From e38ab98a7a972195ed2f89a7393514a0150dfe2d Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 24 Apr 2024 11:29:55 +0200 Subject: [PATCH 1/6] add run_url to status --- bioimageio_collection_backoffice/_settings.py | 5 +++++ bioimageio_collection_backoffice/db_structure/versions.py | 3 +++ bioimageio_collection_backoffice/remote_resource.py | 2 ++ 3 files changed, 10 insertions(+) diff --git a/bioimageio_collection_backoffice/_settings.py b/bioimageio_collection_backoffice/_settings.py index b9d46603..e75affad 100644 --- a/bioimageio_collection_backoffice/_settings.py +++ b/bioimageio_collection_backoffice/_settings.py @@ -19,10 +19,15 @@ class Settings(BaseSettings, extra="ignore"): "https://raw.githubusercontent.com/bioimage-io/collection/main/id_parts.json" ) """describes a file holding all parts to create bioimage.io resource ids""" + reviewers: str = ( "https://raw.githubusercontent.com/bioimage-io/collection/main/reviewers.json" ) """determines who is a bioimage.io reviewer""" + + run_url: Optional[str] = None + """url to logs of the current CI run""" + s3_bucket: str = "public-datasets" s3_folder: str = "sandbox.bioimage.io" s3_host: str = "uk1s3.embassy.ebi.ac.uk" diff --git a/bioimageio_collection_backoffice/db_structure/versions.py b/bioimageio_collection_backoffice/db_structure/versions.py index a2b3281d..b5f962dd 100644 --- a/bioimageio_collection_backoffice/db_structure/versions.py +++ b/bioimageio_collection_backoffice/db_structure/versions.py @@ -6,6 +6,7 @@ import pydantic from typing_extensions import Annotated +from .._settings import settings from .common import Node PublishNumber = NewType("PublishNumber", int) @@ -17,10 +18,12 @@ class _StatusBase(Node, frozen=True): timestamp: datetime + run_url: Optional[str] class _StatusBaseWithDefaults(Node, frozen=True): timestamp: datetime = datetime.now() + run_url: Optional[str] = settings.run_url class _StagedStatusBase(_StatusBase, frozen=True): diff --git a/bioimageio_collection_backoffice/remote_resource.py b/bioimageio_collection_backoffice/remote_resource.py index 19528ec1..f25a8903 100644 --- a/bioimageio_collection_backoffice/remote_resource.py +++ b/bioimageio_collection_backoffice/remote_resource.py @@ -29,6 +29,7 @@ from ruyaml import YAML from typing_extensions import LiteralString, assert_never +from ._settings import settings from ._thumbnails import create_thumbnails from .db_structure.chat import Chat, ChatWithDefaults, MessageWithDefaults from .db_structure.log import Log, LogWithDefaults @@ -287,6 +288,7 @@ def report_error(self, error: Exception): error_status = ErrorStatus( timestamp=datetime.now(), + run_url=settings.run_url, message=str(error), traceback=traceback.format_tb(error.__traceback__), during=current_status, From e5713da9ccc12ffe6b5b9638ebe6f10727e0f552 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 24 Apr 2024 17:01:53 +0200 Subject: [PATCH 2/6] add staged_versions --- bioimageio_collection_backoffice/generate_collection_json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bioimageio_collection_backoffice/generate_collection_json.py b/bioimageio_collection_backoffice/generate_collection_json.py index fc5662b3..712dba2b 100644 --- a/bioimageio_collection_backoffice/generate_collection_json.py +++ b/bioimageio_collection_backoffice/generate_collection_json.py @@ -105,4 +105,5 @@ def maybe_swap_with_thumbnail( entry["rdf_source"] = entry["entry_source"] entry["version_number"] = p.number entry["versions"] = list(p.concept.versions.published) + entry["staged_versions"] = [f"staged/{s}" for s in p.concept.versions.staged] return entry From 08327ea222f5535855e83f8b8076b88499aefe57 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 24 Apr 2024 17:08:39 +0200 Subject: [PATCH 3/6] set RUN_URL env var in call wfs --- .github/workflows/backup_call.yaml | 1 + .github/workflows/build_backoffice.yaml | 1 + .github/workflows/generate_collection_json_call.yaml | 1 + .github/workflows/publish_call.yaml | 1 + .github/workflows/request_changes_call.yaml | 1 + .github/workflows/stage_call.yaml | 1 + .github/workflows/test_call.yaml | 1 + 7 files changed, 7 insertions(+) diff --git a/.github/workflows/backup_call.yaml b/.github/workflows/backup_call.yaml index 10190dd5..a64b29ee 100644 --- a/.github/workflows/backup_call.yaml +++ b/.github/workflows/backup_call.yaml @@ -26,6 +26,7 @@ env: S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}} ZENODO_TEST_API_ACCESS_TOKEN: ${{secrets.ZENODO_TEST_API_ACCESS_TOKEN}} + RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} jobs: run: diff --git a/.github/workflows/build_backoffice.yaml b/.github/workflows/build_backoffice.yaml index 699ad720..88ef43ca 100644 --- a/.github/workflows/build_backoffice.yaml +++ b/.github/workflows/build_backoffice.yaml @@ -14,6 +14,7 @@ env: ZENODO_URL: ${{vars.ZENODO_TEST_URL}} # testing! ZENODO_TEST_URL: ${{vars.ZENODO_TEST_URL}} REVIEWERS: ${{vars.REVIEWERS}} + RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} diff --git a/.github/workflows/generate_collection_json_call.yaml b/.github/workflows/generate_collection_json_call.yaml index cb2fd639..cefb4eab 100644 --- a/.github/workflows/generate_collection_json_call.yaml +++ b/.github/workflows/generate_collection_json_call.yaml @@ -21,6 +21,7 @@ env: S3_FOLDER: ${{inputs.S3_FOLDER}} S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} + RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} jobs: run: diff --git a/.github/workflows/publish_call.yaml b/.github/workflows/publish_call.yaml index c80aa385..160de802 100644 --- a/.github/workflows/publish_call.yaml +++ b/.github/workflows/publish_call.yaml @@ -40,6 +40,7 @@ env: S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}} REVIEWERS: ${{inputs.REVIEWERS}} + RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} jobs: diff --git a/.github/workflows/request_changes_call.yaml b/.github/workflows/request_changes_call.yaml index 1c59596b..39c558de 100644 --- a/.github/workflows/request_changes_call.yaml +++ b/.github/workflows/request_changes_call.yaml @@ -48,6 +48,7 @@ env: S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}} REVIEWERS: ${{vars.REVIEWERS}} + RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} jobs: diff --git a/.github/workflows/stage_call.yaml b/.github/workflows/stage_call.yaml index eaf017d3..212850e1 100644 --- a/.github/workflows/stage_call.yaml +++ b/.github/workflows/stage_call.yaml @@ -34,6 +34,7 @@ env: S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} MAIL_PASSWORD: ${{secrets.MAIL_PASSWORD}} + RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} jobs: run: diff --git a/.github/workflows/test_call.yaml b/.github/workflows/test_call.yaml index 592eec6a..861eef7c 100644 --- a/.github/workflows/test_call.yaml +++ b/.github/workflows/test_call.yaml @@ -35,6 +35,7 @@ env: S3_FOLDER: ${{ inputs.S3_FOLDER }} S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} + RUN_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} jobs: validate_format: From a820851054877c6b8cee6a67365dc16159efd68c Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 24 Apr 2024 17:55:20 +0200 Subject: [PATCH 4/6] update README.md --- README.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f0478bb1..4510ff11 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,31 @@ # collection -Maintains the resources displayed on bioimage.io (Successor to collection-bioimage-io) +This repository is used to manage the resources displayed on [bioimage.io](http://bioimage.io). + +Most users will not directly dispatch the workflows defined in this reporitory, but should instead login on [bioimage.io](http://bioimage.io) and use the front-end to interact with the bioimage.io collection. + +We currently do not have a workflow for direct upload, but publicly available resource packages may be staged with a stage workflow dispatch. + +## Lifecycle of staged resources + +1. unpacking: Uploaded resource packages are unpacked and their individuell files uploaded to our public S3 storage. +2. testing: Staged resources are automatically tested: Is their metadata valid? Can test outputs be reproduced from test inputs? Are linked URLs available? +3. awaiting reviewe: After the tests have concluded the bioimageio reviewers are notified. +4. The reviewer will result in + a) changes requested: Please upload an updated version. + b) accepted: The resource will be published! +5. The final status of a staged resource is one of + a) superseded: This staged resource version has not been published and also will not be published in the future. + b) A copy of this staged version has been published as a new resource version. (note: the resource/publish version is not neccessarily identical with the stage number) + +```mermaid +graph TD; + unpacking[1: unpacking]-->unpacked[unpacked] + unpacked-->testing[2: testing] + testing-->ar[3: awaiting review] + ar--->cr[4a: changes requestd] + cr-->superseded + ar--->accepted[4b: accepted] + accepted-->published + +``` From a19bee997af668c99ea7ff80e2c37eaf38c2c455 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Apr 2024 10:03:50 +0200 Subject: [PATCH 5/6] mention error status --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4510ff11..89bfaf47 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ We currently do not have a workflow for direct upload, but publicly available re a) superseded: This staged resource version has not been published and also will not be published in the future. b) A copy of this staged version has been published as a new resource version. (note: the resource/publish version is not neccessarily identical with the stage number) +Additionally an 'error' status may be shown if an exception occured. +This also may be the case for invalid inputs. + ```mermaid graph TD; unpacking[1: unpacking]-->unpacked[unpacked] @@ -27,5 +30,7 @@ graph TD; cr-->superseded ar--->accepted[4b: accepted] accepted-->published + unpacking-->error + testing-->error ``` From b9ffc3cb0f194be335e067c83ae3628959a65a92 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Apr 2024 10:09:06 +0200 Subject: [PATCH 6/6] raise error if staged rdf is missing --- bioimageio_collection_backoffice/remote_resource.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bioimageio_collection_backoffice/remote_resource.py b/bioimageio_collection_backoffice/remote_resource.py index f25a8903..c3f186bc 100644 --- a/bioimageio_collection_backoffice/remote_resource.py +++ b/bioimageio_collection_backoffice/remote_resource.py @@ -478,9 +478,12 @@ def publish(self, reviewer: str) -> PublishedVersion: logger.debug("Publishing {} as version nr {}", self.folder, next_publish_nr) # load rdf - staged_rdf_path = f"{self.folder}files/rdf.yaml" + staged_rdf_path = f"{self.folder}files/bioimageio.yaml" rdf_data = self.client.load_file(staged_rdf_path) - rdf = yaml.load(rdf_data) + if rdf_data is None: + raise ValueError(f"Failed to load staged RDF from {staged_rdf_path}") + + rdf = yaml.load(io.BytesIO(rdf_data)) sem_ver = rdf.get("version") if sem_ver is not None: