MVP stage, test, review

.github/workflows/stage_call.yaml

@@ -31,9 +31,10 @@ env:
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
 
 jobs:
-  validate:
+  stage:
     runs-on: ubuntu-latest
     outputs:
+      version: ${{ steps.stage.outputs.version }}
       dynamic_test_cases: ${{ steps.stage.outputs.dynamic_test_cases }}
       has_dynamic_test_cases: ${{ steps.stage.outputs.has_dynamic_test_cases }}
     steps:
@@ -47,21 +48,9 @@ jobs:
         run: |
           python scripts/stage.py "${{ inputs.resource_id }}" "${{ inputs.package_url }}"
 
-      - name: Validate format
-        id: validate
-        run: |
-          python scripts/update_status.py "${{ inputs.resource_path }}" "Starting validation" "2"
-          python scripts/validate_format.py "${{ inputs.resource_path }}"
-      - run: |
-          python scripts/update_status.py "${{ inputs.resource_path }}" "Starting additional tests" "3"
-        if: steps.validate.outputs.has_dynamic_test_cases == 'yes'
-      - run: |
-          python scripts/update_status.py "${{ inputs.resource_path }}" "Validation done" "3"
-        if: steps.validate.outputs.has_dynamic_test_cases == 'no'
-
   test:
-    needs: validate
-    if: needs.validate.outputs.has_dynamic_test_cases == 'yes'
+    needs: stage
+    if: needs.stage.outputs.has_dynamic_test_cases == 'yes'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -88,11 +77,11 @@ jobs:
         run: pip install typer bioimageio.spec minio loguru
       - name: dynamic validation
         shell: bash -l {0}
-        run: python scripts/test_dynamically.py "https://${{env.S3_HOST}}/${{env.S3_BUCKET}}/${{env.S3_FOLDER}}/${{inputs.resource_path}}/files/rdf.yaml" ${{ matrix.weight_format }} --create-env-outcome ${{ steps.create_env.outcome }} --${{ contains(inputs.deploy_to, 'gh-pages') && 'no-ignore' || 'ignore' }}-rdf-source-field-in-validation
+        run: python scripts/test_dynamically.py "${{inputs.resource_id}}" "${{needs.stage.outputs.version}}" "${{ matrix.weight_format }}" "${{ steps.create_env.outcome }}"
         timeout-minutes: 60
 
   conclude:
-    needs: test
+    needs: [stage, test]
     if: always() # run even if test job fails
     runs-on: ubuntu-latest
     steps:
@@ -103,4 +92,6 @@ jobs:
           cache: "pip" # caching pip dependencies
       - run: pip install -r scripts/requirements.txt
       - run: |
-          python scripts/update_status.py "${{ inputs.resource_path }}" "Awaiting review" "4"
+          python scripts/conclude.py "${{ inputs.resource_id }}" "${{needs.stage.outputs.version}}"
+
+  # TODO: call emailer

.github/workflows/test.yaml

@@ -15,7 +15,7 @@ jobs:
       S3_FOLDER: ${{vars.S3_TEST_FOLDER}}/ci # using test folder
     secrets: inherit
 
-  test-scripts:
+  build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -24,7 +24,32 @@ jobs:
           python-version: "3.12"
           cache: "pip" # caching pip dependencies
       - run: pip install -r .github/scripts/requirements.txt
-      - run: pip install black pyright pytest
       - run: black .
       - run: pyright -p pyproject.toml
       - run: pytest
+      - name: export documentation
+        if: ${{ github.ref == 'ref/head/main' }}
+        run: pdoc . -o ./docs
+      - uses: actions/upload-pages-artifact@v3
+        if: ${{ github.ref == 'ref/head/main' }}
+        with:
+          path: docs/
+
+  deploy:
+    needs: build
+    if: ${{ github.ref == 'ref/head/main' }}
+    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
+    permissions:
+      pages: write # to deploy to Pages
+      id-token: write # to verify the deployment originates from an appropriate source
+
+    # Deploy to the github-pages environment
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

requirements.txt

@@ -1,7 +1,12 @@
-bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@cfacdc292784e0ecc7f5dbcdce0e4fc2b0ca3cad  # TODO: chenage to released version
+bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@0b707b83b061da7584e554cedbf0c6d725980619  # TODO: chenage to released version
+bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@fcb6bcd674e211c61161105eaf3552cf3cb804ec  # TODO: chenage to released version
+black==24.2.0
 loguru==0.7.2
 minio==7.2.3
 packaging==23.2
+pdoc==14.4.0
+pyright==1.1.351
+pytest==8.0.0
 ruyaml==0.91.0
 spdx-license-list==3.22
 typer==0.9.0

scripts/test_dynamically.py

@@ -3,13 +3,20 @@
 from pathlib import Path
 from typing import Optional
 
+import bioimageio.core
+import bioimageio.spec
 import typer
-from bioimageio.spec import load_description
+from bioimageio.spec.model.v0_5 import WeightsFormat
+from bioimageio.spec.summary import (
+    ErrorEntry,
+    InstalledPackage,
+    ValidationDetail,
+    ValidationSummary,
+)
 from ruyaml import YAML
 from utils.remote_resource import StagedVersion
 from utils.s3_client import Client
 
-yaml = YAML(typ="ssafe")
 try:
     from tqdm import tqdm
 except ImportError:
@@ -18,39 +25,66 @@
     # silence tqdm
     tqdm.__init__ = partialmethod(tqdm.__init__, disable=True)  # type: ignore
 
+yaml = YAML(typ="safe")
 
-def test_summary_from_exception(name: str, exception: Exception):
-    return dict(
+
+def get_summary_detail_from_exception(name: str, exception: Exception):
+    return ValidationDetail(
         name=name,
         status="failed",
-        error=str(exception),
-        traceback=traceback.format_tb(exception.__traceback__),
+        errors=[
+            ErrorEntry(
+                loc=(),
+                msg=str(exception),
+                type="exception",
+                traceback=traceback.format_tb(exception.__traceback__),
+            )
+        ],
     )
 
 
 def test_dynamically(
     resource_id: str,
     version: int,
-    weight_format: Optional[str] = typer.Argument(
+    weight_format: Optional[WeightsFormat] = typer.Argument(
         ..., help="weight format to test model with."
     ),
     create_env_outcome: str = "success",
 ):
     staged = StagedVersion(client=Client(), id=resource_id, version=version)
+    staged.set_status(
+        "testing",
+        "Testing" + ("" if weight_format is None else f" {weight_format} weights"),
+    )
     rdf_source = staged.get_rdf_url()
     if weight_format is None:
         # no dynamic tests for non-model resources yet...
         return
 
+    summary = ValidationSummary(
+        name="bioimageio.core.test_description",
+        status="passed",
+        details=[],
+        env=[
+            InstalledPackage(
+                name="bioimageio.spec", version=bioimageio.spec.__version__
+            ),
+            InstalledPackage(
+                name="bioimageio.core", version=bioimageio.core.__version__
+            ),
+        ],
+        source_name=rdf_source,
+    )
+
     if create_env_outcome == "success":
         try:
-            from bioimageio.core import test_resource
+            from bioimageio.core import test_description
         except Exception as e:
-            summaries = [
-                test_summary_from_exception(
-                    "import test_resource from test environment", e
+            summary.add_detail(
+                get_summary_detail_from_exception(
+                    "import test_description from test environment", e
                 )
-            ]
+            )
         else:
             try:
                 rdf = yaml.load(rdf_source)
@@ -61,15 +95,18 @@ def test_dynamically(
                     .get(weight_format, {})
                 )
             except Exception as e:
-                summaries = [test_summary_from_exception("check for test kwargs", e)]
+                summary.add_detail(
+                    get_summary_detail_from_exception("check for test kwargs", e)
+                )
             else:
                 try:
-                    rd = load_description(rdf_source)
-                    summaries = test_resource(
-                        rd, weight_format=weight_format, **test_kwargs
+                    summary = test_description(
+                        rdf_source, weight_format=weight_format, **test_kwargs
                     )
                 except Exception as e:
-                    summaries = [test_summary_from_exception("call 'test_resource'", e)]
+                    summary.add_detail(
+                        get_summary_detail_from_exception("call 'test_resource'", e)
+                    )
 
     else:
         env_path = Path(f"conda_env_{weight_format}.yaml")
@@ -78,11 +115,15 @@ def test_dynamically(
         else:
             error = f"Conda environment yaml file not found: {env_path}"
 
-        summaries = [
-            dict(name="install test environment", status="failed", error=error)
-        ]
+        summary.add_detail(
+            ValidationDetail(
+                name="install test environment",
+                status="failed",
+                errors=[ErrorEntry(loc=(), msg=error, type="env")],
+            )
+        )
 
-    staged.add_log_entry("validation_summaries", summaries)
+    staged.add_log_entry("bioimageio.core", summary.model_dump(mode="json"))
 
 
 if __name__ == "__main__":

scripts/upload_model_to_zenodo.py

@@ -10,12 +10,12 @@
 
 import requests  # type: ignore
 import spdx_license_list  # type: ignore
-
 from loguru import logger  # type: ignore
 from packaging.version import parse as parse_version
 from ruyaml import YAML  # type: ignore
 from s3_client import create_client, version_from_resource_path_or_s3
-from update_status import update_status
+
+from scripts.conclude import update_status
 
 yaml = YAML(typ="safe")
 
@@ -75,7 +75,9 @@ def main():
     params = {"access_token": ACCESS_TOKEN}
 
     client = create_client()
-    resource_path, version = version_from_resource_path_or_s3(args.resource_path, client)
+    resource_path, version = version_from_resource_path_or_s3(
+        args.resource_path, client
+    )
 
     s3_path = f"{resource_path}/{version}/files"
 

scripts/utils/remote_resource.py

@@ -66,12 +66,16 @@ def stage_new_version(self, package_url: str) -> StagedVersion:
             rdf["id"] = ret.id
         elif rdf_id != ret.id:
             raise ValueError(
-                f"Expected package for {ret.id}, but got packaged {rdf_id}"
+                f"Expected package for {ret.id}, but got packaged {rdf_id} ({package_url})"
             )
 
         # overwrite version information
         rdf["version"] = ret.version
 
+        if rdf.get("id_emoji") is None:
+            # TODO: set `id_emoji` according to id
+            raise ValueError(f"RDF in {package_url} is missing `id_emoji`")
+
         for filename in zipobj.namelist():
             file_data = zipobj.open(filename).read()
             path = f"{ret.folder}files/{filename}"
@@ -136,12 +140,20 @@ def set_status(self, name: StatusName, description: str) -> None:
 
     @staticmethod
     def _create_status(name: StatusName, description: str) -> Status:
-        num_steps = 3
+        num_steps = 5
         if name == "unknown":
             step = 1
             num_steps = 1
         elif name == "staging":
             step = 1
+        elif name == "testing":
+            step = 2
+        elif name == "awaiting review":
+            step = 3
+        elif name == "publishing":
+            step = 4
+        elif name == "published":
+            step = 5
         else:
             assert_never(name)
 
@@ -153,7 +165,11 @@ def get_status(self) -> Status:
         details = self._get_details()
         return details["status"]
 
-    def add_log_entry(self, category: LogCategory, content: Any):
+    def add_log_entry(
+        self,
+        category: LogCategory,
+        content: list[Any] | dict[Any, Any] | int | float | str | None | bool,
+    ):
         log = self.get_log()
         entries = log.setdefault(category, [])
         now = datetime.now().isoformat()

scripts/utils/s3_structure.py

@@ -19,7 +19,7 @@ class Message(TypedDict):
     time: str
 
 
-StatusName = Literal["unknown", "staging"]
+StatusName = Literal["unknown", "staging", "testing", "awaiting review"]
 
 
 class Status(TypedDict):

scripts/utils/validate_format.py

@@ -243,6 +243,7 @@ def prepare_dynamic_test_cases(rd: ResourceDescr) -> list[dict[str, str]]:
 
 
 def validate_format(staged: StagedVersion):
+    staged.set_status("testing", "Testing RDF format")
     rdf_source = staged.get_rdf_url()
     rd = load_description(rdf_source, format_version="discover")
     dynamic_test_cases: list[dict[str, str]] = []
@@ -263,5 +264,6 @@ def validate_format(staged: StagedVersion):
         dict(
             has_dynamic_test_cases=bool(dynamic_test_cases),
             dynamic_test_cases={"include": dynamic_test_cases},
+            version=staged.version,
         )
     )