feat: Update ingestion config validation for ctf and alignment entities

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.2
+  rev: v0.6.8
   hooks:
   - id: ruff
     # These files are codegen'd and non-compliant :'(
@@ -15,7 +15,7 @@ repos:
     - --config
     - ingestion_tools/pyproject.toml
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.2
+  rev: v0.6.8
   hooks:
   - id: ruff
     args:

ingestion_tools/dataset_configs/10000.yaml

@@ -1,3 +1,10 @@
+alignments:
+- metadata:
+    format: IMOD
+  sources:
+  - source_multi_glob:
+      list_globs:
+      - metadata/{run_name}.xf
 annotations:
 - metadata:
     annotation_method: Cumulative template-matching trained 2D CNN predictions + visual
@@ -55,6 +62,7 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_fas.csv
+      is_portal_standard: false
       is_visualization_default: false
 - metadata:
     annotation_method: Cumulative template-matching trained 2D CNN predictions + visual
@@ -77,6 +85,7 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_cyto_ribosomes.csv
+      is_portal_standard: false
       is_visualization_default: true
 - metadata:
     annotation_method: spectrum equalization filter + 2D CNN prediction + manual correction
@@ -97,6 +106,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 1
 - metadata:
@@ -118,6 +128,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 2
 - metadata:
@@ -139,6 +150,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 3
 - metadata:
@@ -160,6 +172,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 4
 - metadata:
@@ -181,6 +194,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 5
 - metadata:
@@ -202,6 +216,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 6
 - metadata:
@@ -223,6 +238,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 7
 - metadata:
@@ -244,6 +260,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 8
 - metadata:
@@ -265,6 +282,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 9
 - metadata:
@@ -286,6 +304,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 10
 - metadata:
@@ -307,6 +326,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 11
 - metadata:
@@ -328,6 +348,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 13
 - metadata:
@@ -349,6 +370,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_membranes.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 1
 collection_metadata:
@@ -458,7 +480,6 @@ rawtilts:
   - source_multi_glob:
       list_globs:
       - metadata/{run_name}.rawtlt
-      - metadata/{run_name}.xf
 runs:
 - sources:
   - source_glob:
@@ -519,7 +540,9 @@ tomograms:
       - 1
     authors: *id004
     ctf_corrected: false
+    dates: *id002
     fiducial_alignment_status: NON_FIDUCIAL
+    is_visualization_default: true
     offset:
       x: 0
       y: 0

ingestion_tools/dataset_configs/10001.yaml

@@ -1,3 +1,10 @@
+alignments:
+- metadata:
+    format: IMOD
+  sources:
+  - source_multi_glob:
+      list_globs:
+      - metadata/{run_name}.xf
 annotations:
 - metadata:
     annotation_method: Cumulative template-matching trained 2D CNN predictions + visual
@@ -55,6 +62,7 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_fas.csv
+      is_portal_standard: false
       is_visualization_default: false
 - metadata:
     annotation_method: Cumulative template-matching trained 2D CNN predictions + visual
@@ -77,6 +85,7 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_cyto_ribosomes.csv
+      is_portal_standard: false
       is_visualization_default: true
 - metadata:
     annotation_method: manual segmentation
@@ -96,6 +105,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 1
 - metadata:
@@ -116,6 +126,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 2
 - metadata:
@@ -136,6 +147,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 3
 - metadata:
@@ -156,6 +168,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 4
 - metadata:
@@ -176,6 +189,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 5
 - metadata:
@@ -196,6 +210,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 6
 - metadata:
@@ -216,6 +231,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 7
 - metadata:
@@ -236,6 +252,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 8
 - metadata:
@@ -256,6 +273,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 9
 - metadata:
@@ -276,6 +294,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: true
       mask_label: 10
 - metadata:
@@ -296,6 +315,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 11
 - metadata:
@@ -316,6 +336,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 13
 - metadata:
@@ -337,6 +358,7 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_membranes.mrc
+      is_portal_standard: false
       is_visualization_default: false
       mask_label: 1
 collection_metadata:
@@ -428,7 +450,6 @@ rawtilts:
   - source_multi_glob:
       list_globs:
       - metadata/{run_name}.rawtlt
-      - metadata/{run_name}.xf
 runs:
 - sources:
   - source_glob:
@@ -489,7 +510,12 @@ tomograms:
       - 1
     authors: *id004
     ctf_corrected: false
+    dates:
+      deposition_date: '2023-04-01'
+      last_modified_date: '2023-06-01'
+      release_date: '2023-06-01'
     fiducial_alignment_status: NON_FIDUCIAL
+    is_visualization_default: true
     offset:
       x: 0
       y: 0