Merge pull request #53 from bioimage-io/add_run_url

Add run url to status

.github/workflows/backup_call.yaml

@@ -26,6 +26,7 @@ env:
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
   ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}}
   ZENODO_TEST_API_ACCESS_TOKEN: ${{secrets.ZENODO_TEST_API_ACCESS_TOKEN}}
+  RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
 jobs:
   run:

.github/workflows/build_backoffice.yaml

@@ -14,6 +14,7 @@ env:
   ZENODO_URL: ${{vars.ZENODO_TEST_URL}} # testing!
   ZENODO_TEST_URL: ${{vars.ZENODO_TEST_URL}}
   REVIEWERS: ${{vars.REVIEWERS}}
+  RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
   S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}}
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}

.github/workflows/generate_collection_json_call.yaml

@@ -21,6 +21,7 @@ env:
   S3_FOLDER: ${{inputs.S3_FOLDER}}
   S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}}
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
+  RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
 jobs:
   run:

.github/workflows/publish_call.yaml

@@ -40,6 +40,7 @@ env:
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
   ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}}
   REVIEWERS: ${{inputs.REVIEWERS}}
+  RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
 
 jobs:

.github/workflows/request_changes_call.yaml

@@ -48,6 +48,7 @@ env:
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
   ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}}
   REVIEWERS: ${{vars.REVIEWERS}}
+  RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
 
 jobs:

.github/workflows/stage_call.yaml

@@ -34,6 +34,7 @@ env:
   S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}}
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
   MAIL_PASSWORD: ${{secrets.MAIL_PASSWORD}}
+  RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
 jobs:
   run:

.github/workflows/test_call.yaml

@@ -35,6 +35,7 @@ env:
   S3_FOLDER: ${{ inputs.S3_FOLDER }}
   S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}}
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
+  RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
 jobs:
   validate_format:

README.md

@@ -1,3 +1,36 @@
 # collection
 
-Maintains the resources displayed on bioimage.io (Successor to collection-bioimage-io)
+This repository is used to manage the resources displayed on [bioimage.io](http://bioimage.io).
+
+Most users will not directly dispatch the workflows defined in this reporitory, but should instead login on [bioimage.io](http://bioimage.io) and use the front-end to interact with the bioimage.io collection.
+
+We currently do not have a workflow for direct upload, but publicly available resource packages may be staged with a stage workflow dispatch.
+
+## Lifecycle of staged resources
+
+1. unpacking: Uploaded resource packages are unpacked and their individuell files uploaded to our public S3 storage.
+2. testing: Staged resources are automatically tested: Is their metadata valid? Can test outputs be reproduced from test inputs? Are linked URLs available?
+3. awaiting reviewe: After the tests have concluded the bioimageio reviewers are notified.
+4. The reviewer will result in
+    a) changes requested: Please upload an updated version.
+    b) accepted: The resource will be published!
+5. The final status of a staged resource is one of
+    a) superseded: This staged resource version has not been published and also will not be published in the future.
+    b) A copy of this staged version has been published as a new resource version. (note: the resource/publish version is not neccessarily identical with the stage number)
+
+Additionally an 'error' status may be shown if an exception occured.
+This also may be the case for invalid inputs.
+
+```mermaid
+graph TD;
+    unpacking[1: unpacking]-->unpacked[unpacked]
+    unpacked-->testing[2: testing]
+    testing-->ar[3: awaiting review]
+    ar--->cr[4a: changes requestd]
+    cr-->superseded
+    ar--->accepted[4b: accepted]
+    accepted-->published
+    unpacking-->error
+    testing-->error
+
+```

bioimageio_collection_backoffice/_settings.py

@@ -19,10 +19,15 @@ class Settings(BaseSettings, extra="ignore"):
         "https://raw.githubusercontent.com/bioimage-io/collection/main/id_parts.json"
     )
     """describes a file holding all parts to create bioimage.io resource ids"""
+
     reviewers: str = (
         "https://raw.githubusercontent.com/bioimage-io/collection/main/reviewers.json"
     )
     """determines who is a bioimage.io reviewer"""
+
+    run_url: Optional[str] = None
+    """url to logs of the current CI run"""
+
     s3_bucket: str = "public-datasets"
     s3_folder: str = "sandbox.bioimage.io"
     s3_host: str = "uk1s3.embassy.ebi.ac.uk"

bioimageio_collection_backoffice/db_structure/versions.py

@@ -6,6 +6,7 @@
 import pydantic
 from typing_extensions import Annotated
 
+from .._settings import settings
 from .common import Node
 
 PublishNumber = NewType("PublishNumber", int)
@@ -17,10 +18,12 @@
 
 class _StatusBase(Node, frozen=True):
     timestamp: datetime
+    run_url: Optional[str]
 
 
 class _StatusBaseWithDefaults(Node, frozen=True):
     timestamp: datetime = datetime.now()
+    run_url: Optional[str] = settings.run_url
 
 
 class _StagedStatusBase(_StatusBase, frozen=True):

bioimageio_collection_backoffice/generate_collection_json.py

@@ -105,4 +105,5 @@ def maybe_swap_with_thumbnail(
     entry["rdf_source"] = entry["entry_source"]
     entry["version_number"] = p.number
     entry["versions"] = list(p.concept.versions.published)
+    entry["staged_versions"] = [f"staged/{s}" for s in p.concept.versions.staged]
     return entry

bioimageio_collection_backoffice/remote_resource.py

@@ -29,6 +29,7 @@
 from ruyaml import YAML
 from typing_extensions import LiteralString, assert_never
 
+from ._settings import settings
 from ._thumbnails import create_thumbnails
 from .db_structure.chat import Chat, ChatWithDefaults, MessageWithDefaults
 from .db_structure.log import Log, LogWithDefaults
@@ -287,6 +288,7 @@ def report_error(self, error: Exception):
 
         error_status = ErrorStatus(
             timestamp=datetime.now(),
+            run_url=settings.run_url,
             message=str(error),
             traceback=traceback.format_tb(error.__traceback__),
             during=current_status,
@@ -476,9 +478,12 @@ def publish(self, reviewer: str) -> PublishedVersion:
         logger.debug("Publishing {} as version nr {}", self.folder, next_publish_nr)
 
         # load rdf
-        staged_rdf_path = f"{self.folder}files/rdf.yaml"
+        staged_rdf_path = f"{self.folder}files/bioimageio.yaml"
         rdf_data = self.client.load_file(staged_rdf_path)
-        rdf = yaml.load(rdf_data)
+        if rdf_data is None:
+            raise ValueError(f"Failed to load staged RDF from {staged_rdf_path}")
+
+        rdf = yaml.load(io.BytesIO(rdf_data))
 
         sem_ver = rdf.get("version")
         if sem_ver is not None: