From 7dd8ab39afc2a8cbc8347f2234bb0047d7952fa4 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 21 Feb 2024 14:00:02 +0100 Subject: [PATCH] MVP stage, test, review --- .github/workflows/stage_call.yaml | 27 ++++------ .github/workflows/test.yaml | 29 ++++++++++- requirements.txt | 7 ++- scripts/test_dynamically.py | 83 +++++++++++++++++++++++-------- scripts/update_status.py | 37 -------------- scripts/upload_model_to_zenodo.py | 8 +-- scripts/utils/remote_resource.py | 22 ++++++-- scripts/utils/s3_structure.py | 2 +- scripts/utils/validate_format.py | 2 + 9 files changed, 131 insertions(+), 86 deletions(-) delete mode 100644 scripts/update_status.py diff --git a/.github/workflows/stage_call.yaml b/.github/workflows/stage_call.yaml index eba84138..9adb295a 100644 --- a/.github/workflows/stage_call.yaml +++ b/.github/workflows/stage_call.yaml @@ -31,9 +31,10 @@ env: S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} jobs: - validate: + stage: runs-on: ubuntu-latest outputs: + version: ${{ steps.stage.outputs.version }} dynamic_test_cases: ${{ steps.stage.outputs.dynamic_test_cases }} has_dynamic_test_cases: ${{ steps.stage.outputs.has_dynamic_test_cases }} steps: @@ -47,21 +48,9 @@ jobs: run: | python scripts/stage.py "${{ inputs.resource_id }}" "${{ inputs.package_url }}" - - name: Validate format - id: validate - run: | - python scripts/update_status.py "${{ inputs.resource_path }}" "Starting validation" "2" - python scripts/validate_format.py "${{ inputs.resource_path }}" - - run: | - python scripts/update_status.py "${{ inputs.resource_path }}" "Starting additional tests" "3" - if: steps.validate.outputs.has_dynamic_test_cases == 'yes' - - run: | - python scripts/update_status.py "${{ inputs.resource_path }}" "Validation done" "3" - if: steps.validate.outputs.has_dynamic_test_cases == 'no' - test: - needs: validate - if: needs.validate.outputs.has_dynamic_test_cases == 'yes' + needs: stage + if: needs.stage.outputs.has_dynamic_test_cases == 'yes' runs-on: ubuntu-latest strategy: fail-fast: false @@ -88,11 +77,11 @@ jobs: run: pip install typer bioimageio.spec minio loguru - name: dynamic validation shell: bash -l {0} - run: python scripts/test_dynamically.py "https://${{env.S3_HOST}}/${{env.S3_BUCKET}}/${{env.S3_FOLDER}}/${{inputs.resource_path}}/files/rdf.yaml" ${{ matrix.weight_format }} --create-env-outcome ${{ steps.create_env.outcome }} --${{ contains(inputs.deploy_to, 'gh-pages') && 'no-ignore' || 'ignore' }}-rdf-source-field-in-validation + run: python scripts/test_dynamically.py "${{inputs.resource_id}}" "${{needs.stage.outputs.version}}" "${{ matrix.weight_format }}" "${{ steps.create_env.outcome }}" timeout-minutes: 60 conclude: - needs: test + needs: [stage, test] if: always() # run even if test job fails runs-on: ubuntu-latest steps: @@ -103,4 +92,6 @@ jobs: cache: "pip" # caching pip dependencies - run: pip install -r scripts/requirements.txt - run: | - python scripts/update_status.py "${{ inputs.resource_path }}" "Awaiting review" "4" + python scripts/conclude.py "${{ inputs.resource_id }}" "${{needs.stage.outputs.version}}" + + # TODO: call emailer diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 92ae9ffd..9b5f2d8a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,7 +15,7 @@ jobs: S3_FOLDER: ${{vars.S3_TEST_FOLDER}}/ci # using test folder secrets: inherit - test-scripts: + build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -24,7 +24,32 @@ jobs: python-version: "3.12" cache: "pip" # caching pip dependencies - run: pip install -r .github/scripts/requirements.txt - - run: pip install black pyright pytest - run: black . - run: pyright -p pyproject.toml - run: pytest + - name: export documentation + if: ${{ github.ref == 'ref/head/main' }} + run: pdoc . -o ./docs + - uses: actions/upload-pages-artifact@v3 + if: ${{ github.ref == 'ref/head/main' }} + with: + path: docs/ + + deploy: + needs: build + if: ${{ github.ref == 'ref/head/main' }} + # Grant GITHUB_TOKEN the permissions required to make a Pages deployment + permissions: + pages: write # to deploy to Pages + id-token: write # to verify the deployment originates from an appropriate source + + # Deploy to the github-pages environment + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/requirements.txt b/requirements.txt index 24350983..fae6f6fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,12 @@ -bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@cfacdc292784e0ecc7f5dbcdce0e4fc2b0ca3cad # TODO: chenage to released version +bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@0b707b83b061da7584e554cedbf0c6d725980619 # TODO: chenage to released version +bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@fcb6bcd674e211c61161105eaf3552cf3cb804ec # TODO: chenage to released version +black==24.2.0 loguru==0.7.2 minio==7.2.3 packaging==23.2 +pdoc==14.4.0 +pyright==1.1.351 +pytest==8.0.0 ruyaml==0.91.0 spdx-license-list==3.22 typer==0.9.0 diff --git a/scripts/test_dynamically.py b/scripts/test_dynamically.py index f8c7541e..37523a95 100644 --- a/scripts/test_dynamically.py +++ b/scripts/test_dynamically.py @@ -3,13 +3,20 @@ from pathlib import Path from typing import Optional +import bioimageio.core +import bioimageio.spec import typer -from bioimageio.spec import load_description +from bioimageio.spec.model.v0_5 import WeightsFormat +from bioimageio.spec.summary import ( + ErrorEntry, + InstalledPackage, + ValidationDetail, + ValidationSummary, +) from ruyaml import YAML from utils.remote_resource import StagedVersion from utils.s3_client import Client -yaml = YAML(typ="ssafe") try: from tqdm import tqdm except ImportError: @@ -18,39 +25,66 @@ # silence tqdm tqdm.__init__ = partialmethod(tqdm.__init__, disable=True) # type: ignore +yaml = YAML(typ="safe") -def test_summary_from_exception(name: str, exception: Exception): - return dict( + +def get_summary_detail_from_exception(name: str, exception: Exception): + return ValidationDetail( name=name, status="failed", - error=str(exception), - traceback=traceback.format_tb(exception.__traceback__), + errors=[ + ErrorEntry( + loc=(), + msg=str(exception), + type="exception", + traceback=traceback.format_tb(exception.__traceback__), + ) + ], ) def test_dynamically( resource_id: str, version: int, - weight_format: Optional[str] = typer.Argument( + weight_format: Optional[WeightsFormat] = typer.Argument( ..., help="weight format to test model with." ), create_env_outcome: str = "success", ): staged = StagedVersion(client=Client(), id=resource_id, version=version) + staged.set_status( + "testing", + "Testing" + ("" if weight_format is None else f" {weight_format} weights"), + ) rdf_source = staged.get_rdf_url() if weight_format is None: # no dynamic tests for non-model resources yet... return + summary = ValidationSummary( + name="bioimageio.core.test_description", + status="passed", + details=[], + env=[ + InstalledPackage( + name="bioimageio.spec", version=bioimageio.spec.__version__ + ), + InstalledPackage( + name="bioimageio.core", version=bioimageio.core.__version__ + ), + ], + source_name=rdf_source, + ) + if create_env_outcome == "success": try: - from bioimageio.core import test_resource + from bioimageio.core import test_description except Exception as e: - summaries = [ - test_summary_from_exception( - "import test_resource from test environment", e + summary.add_detail( + get_summary_detail_from_exception( + "import test_description from test environment", e ) - ] + ) else: try: rdf = yaml.load(rdf_source) @@ -61,15 +95,18 @@ def test_dynamically( .get(weight_format, {}) ) except Exception as e: - summaries = [test_summary_from_exception("check for test kwargs", e)] + summary.add_detail( + get_summary_detail_from_exception("check for test kwargs", e) + ) else: try: - rd = load_description(rdf_source) - summaries = test_resource( - rd, weight_format=weight_format, **test_kwargs + summary = test_description( + rdf_source, weight_format=weight_format, **test_kwargs ) except Exception as e: - summaries = [test_summary_from_exception("call 'test_resource'", e)] + summary.add_detail( + get_summary_detail_from_exception("call 'test_resource'", e) + ) else: env_path = Path(f"conda_env_{weight_format}.yaml") @@ -78,11 +115,15 @@ def test_dynamically( else: error = f"Conda environment yaml file not found: {env_path}" - summaries = [ - dict(name="install test environment", status="failed", error=error) - ] + summary.add_detail( + ValidationDetail( + name="install test environment", + status="failed", + errors=[ErrorEntry(loc=(), msg=error, type="env")], + ) + ) - staged.add_log_entry("validation_summaries", summaries) + staged.add_log_entry("bioimageio.core", summary.model_dump(mode="json")) if __name__ == "__main__": diff --git a/scripts/update_status.py b/scripts/update_status.py deleted file mode 100644 index 18ffbb14..00000000 --- a/scripts/update_status.py +++ /dev/null @@ -1,37 +0,0 @@ -import datetime -from typing import Optional - -from loguru import logger -from typer import Argument, run -from typing_extensions import Annotated -from utils.s3_client import Client - - -def update_status( - resource_id: str, - version: int, - message: Annotated[str, Argument(help="status message")], - step: Annotated[ - Optional[int], Argument(help="optional step in multi-step process") - ] = None, - num_steps: Annotated[int, Argument(help="total steps of multi-step process")] = 6, -): - assert step is None or step <= num_steps, (step, num_steps) - timenow = datetime.datetime.now().isoformat() - - resource_path, version = version_from_resource_path_or_s3(resource_path, client) - status = client.get_status(resource_path, version) - - if "messages" not in status: - status["messages"] = [] - if step is not None: - status["step"] = step - if num_steps is not None: - status["num_steps"] = num_steps - status["last_message"] = message - status["messages"].append({"timestamp": timenow, "text": message}) - client.put_status(resource_path, version, status) - - -if __name__ == "__main__": - run(update_status) diff --git a/scripts/upload_model_to_zenodo.py b/scripts/upload_model_to_zenodo.py index ce1b24d8..fd89b449 100644 --- a/scripts/upload_model_to_zenodo.py +++ b/scripts/upload_model_to_zenodo.py @@ -10,12 +10,12 @@ import requests # type: ignore import spdx_license_list # type: ignore - from loguru import logger # type: ignore from packaging.version import parse as parse_version from ruyaml import YAML # type: ignore from s3_client import create_client, version_from_resource_path_or_s3 -from update_status import update_status + +from scripts.conclude import update_status yaml = YAML(typ="safe") @@ -75,7 +75,9 @@ def main(): params = {"access_token": ACCESS_TOKEN} client = create_client() - resource_path, version = version_from_resource_path_or_s3(args.resource_path, client) + resource_path, version = version_from_resource_path_or_s3( + args.resource_path, client + ) s3_path = f"{resource_path}/{version}/files" diff --git a/scripts/utils/remote_resource.py b/scripts/utils/remote_resource.py index b40240b8..132c6f70 100644 --- a/scripts/utils/remote_resource.py +++ b/scripts/utils/remote_resource.py @@ -66,12 +66,16 @@ def stage_new_version(self, package_url: str) -> StagedVersion: rdf["id"] = ret.id elif rdf_id != ret.id: raise ValueError( - f"Expected package for {ret.id}, but got packaged {rdf_id}" + f"Expected package for {ret.id}, but got packaged {rdf_id} ({package_url})" ) # overwrite version information rdf["version"] = ret.version + if rdf.get("id_emoji") is None: + # TODO: set `id_emoji` according to id + raise ValueError(f"RDF in {package_url} is missing `id_emoji`") + for filename in zipobj.namelist(): file_data = zipobj.open(filename).read() path = f"{ret.folder}files/{filename}" @@ -136,12 +140,20 @@ def set_status(self, name: StatusName, description: str) -> None: @staticmethod def _create_status(name: StatusName, description: str) -> Status: - num_steps = 3 + num_steps = 5 if name == "unknown": step = 1 num_steps = 1 elif name == "staging": step = 1 + elif name == "testing": + step = 2 + elif name == "awaiting review": + step = 3 + elif name == "publishing": + step = 4 + elif name == "published": + step = 5 else: assert_never(name) @@ -153,7 +165,11 @@ def get_status(self) -> Status: details = self._get_details() return details["status"] - def add_log_entry(self, category: LogCategory, content: Any): + def add_log_entry( + self, + category: LogCategory, + content: list[Any] | dict[Any, Any] | int | float | str | None | bool, + ): log = self.get_log() entries = log.setdefault(category, []) now = datetime.now().isoformat() diff --git a/scripts/utils/s3_structure.py b/scripts/utils/s3_structure.py index 427e2a46..c15bbfd3 100644 --- a/scripts/utils/s3_structure.py +++ b/scripts/utils/s3_structure.py @@ -19,7 +19,7 @@ class Message(TypedDict): time: str -StatusName = Literal["unknown", "staging"] +StatusName = Literal["unknown", "staging", "testing", "awaiting review"] class Status(TypedDict): diff --git a/scripts/utils/validate_format.py b/scripts/utils/validate_format.py index 5c0698a0..2d1d603e 100644 --- a/scripts/utils/validate_format.py +++ b/scripts/utils/validate_format.py @@ -243,6 +243,7 @@ def prepare_dynamic_test_cases(rd: ResourceDescr) -> list[dict[str, str]]: def validate_format(staged: StagedVersion): + staged.set_status("testing", "Testing RDF format") rdf_source = staged.get_rdf_url() rd = load_description(rdf_source, format_version="discover") dynamic_test_cases: list[dict[str, str]] = [] @@ -263,5 +264,6 @@ def validate_format(staged: StagedVersion): dict( has_dynamic_test_cases=bool(dynamic_test_cases), dynamic_test_cases={"include": dynamic_test_cases}, + version=staged.version, ) )