From 37a128bd35a11c2d0a042a37be546fdf91dee1b0 Mon Sep 17 00:00:00 2001 From: dhanshreea Date: Sun, 29 Sep 2024 20:47:44 +0530 Subject: [PATCH] Cleaned up workflows using reusable actions --- .github/scripts/resolve_dockerfile.py | 16 +- .github/workflows/post-model-upload.yml | 172 ++++++++++++++++++ ...-legacy-bentoml.yml => upload-bentoml.yml} | 80 ++++---- .github/workflows/upload-ersilia-pack.yml | 78 ++++---- .../workflows/upload-model-to-dockerhub.yml | 163 +++-------------- .../workflows/upload-multistage-condapack.yml | 140 -------------- 6 files changed, 280 insertions(+), 369 deletions(-) create mode 100644 .github/workflows/post-model-upload.yml rename .github/workflows/{upload-legacy-bentoml.yml => upload-bentoml.yml} (68%) delete mode 100644 .github/workflows/upload-multistage-condapack.yml diff --git a/.github/scripts/resolve_dockerfile.py b/.github/scripts/resolve_dockerfile.py index ca09330..4b06921 100644 --- a/.github/scripts/resolve_dockerfile.py +++ b/.github/scripts/resolve_dockerfile.py @@ -1,3 +1,4 @@ +import sys import os import requests @@ -11,7 +12,7 @@ '3.11': 'py311', '3.12': 'py312' } - +model_id = sys.argv[1] def resolve_parser(): if os.path.exists(os.path.abspath(os.path.join(REPO_PATH, "Dockerfile"))): return DockerfileInstallParser(file_dir=REPO_PATH) @@ -24,17 +25,18 @@ def resolve_python_version(parser): return parser._get_python_version() def read_dockerfile(parser): - if isinstance(parser, DockerfileInstallParser): + commands = parser._get_commands() + has_conda = parser._has_conda(commands) + if has_conda: file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.conda" - elif isinstance(parser, YAMLInstallParser): - file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.pip" else: - raise ValueError("Invalid parser type") + file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.pip" response = requests.get(file_url) return response.text -def write_version(file_content, python_version): +def write_version_and_model_id(file_content, python_version): python_version = PY_VERSION_MAP[python_version] + file_content = file_content.replace("eos_identifier", model_id) lines = file_content.split("\n") lines[0] = lines[0].replace("VERSION", python_version) with open(os.path.join(REPO_PATH, "../", "Dockerfile"), "w") as f: @@ -44,4 +46,4 @@ def write_version(file_content, python_version): parser = resolve_parser() python_version = resolve_python_version(parser) dockerfile = read_dockerfile(parser) - write_version(dockerfile, python_version) \ No newline at end of file + write_version_and_model_id(dockerfile, python_version) \ No newline at end of file diff --git a/.github/workflows/post-model-upload.yml b/.github/workflows/post-model-upload.yml new file mode 100644 index 0000000..91235d9 --- /dev/null +++ b/.github/workflows/post-model-upload.yml @@ -0,0 +1,172 @@ +name: Post Model Upload actions + +on: + workflow_run: + workflows: ["Upload model to DockerHub"] + types: + - completed + +jobs: + post-model-upload: + if: ${{ github.repository != 'ersilia-os/eos-template' }} + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download arch.txt + uses: actions/download-artifact@v4 + with: + name: arch + path: . + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check metadata file + id: checkMetadata + continue-on-error: true + run: | + if [[ ! -f metadata.yml ]]; then + echo "metadata.yml file not found" + exit 1 + fi + + - name: Update Metadata YAML file with DockerHub info + id: updateMetadataYAML + if: steps.checkMetadata.outcome == 'success' + run: | + python3 -c " + import yaml + with open('metadata.yml', 'r') as f: + data = yaml.safe_load(f) + print(data) + with open('arch.txt', 'r') as f: + arch = f.read().rstrip() + arch = arch.split(',') + data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) + data['Docker Architecture'] = arch + with open('metadata.yml', 'w') as f: + yaml.dump(data, default_flow_style=False, sort_keys=False) + " + rm arch.txt + + - name: Update Metadata JSON file with DockerHub info + id: updateMetadataJSON + if: steps.checkMetadata.outcome == 'failure' + run: | + python3 -c " + import json + with open('metadata.json', 'r') as f: + data = json.load(f) + print(data) + with open('arch.txt', 'r') as f: + arch = f.read().rstrip() + arch = arch.split(',') + data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) + data['Docker Architecture'] = arch + with open('metadata.json', 'w') as f: + json.dump(data, f, indent=4) + " + rm arch.txt + + + - name: Commit and push changes done to the Metadata file + uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 + with: + author_name: "ersilia-bot" + author_email: "ersilia-bot@users.noreply.github.com" + message: "updating metadata [skip ci]" + repository: "ersilia-os/${{ github.event.repository.name }}" + github_token: ${{ secrets.GITHUB_TOKEN }} + amend: true + force: true + + # Setup conda + - name: Setup conda + id: setupConda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: "3.10.10" + + # Install ersilia + - name: Install dependencies in Conda environment + id: installDependenciesInConda + run: | + conda install gh -c conda-forge + python -m pip install git+https://github.com/ersilia-os/ersilia.git + + - name: Update metadata to AirTable + id: update-metadata-to-airtable + env: + USER_NAME: ${{ github.repository_owner }} + BRANCH: "main" + REPO_NAME: ${{ github.event.repository.name }} + AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} + run: | + echo "Updating metadata to AirTable looking at owner: $USER_NAME" + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py + python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY + rm update_metadata_to_airtable.py + + - name: Update README file + id: update-readme-file + env: + MODEL_ID: ${{ github.event.repository.name }} + run: | + echo "Updating README file with AirTable metadata for model: $MODEL_ID" + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py + python3 update_readme_from_airtable.py $MODEL_ID . + rm update_readme_from_airtable.py + less README.md + + - name: Commit and push changes done to the README file + uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 + with: + author_name: "ersilia-bot" + author_email: "ersilia-bot@users.noreply.github.com" + message: "updating readme [skip ci]" + repository: "ersilia-os/${{ github.event.repository.name }}" + github_token: ${{ secrets.GITHUB_TOKEN }} + amend: true + force: true + + - name: Docker Hub Description + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ersiliaos/${{ github.event.repository.name }} + short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}" + + # Create an issue within the repository to track that this model is ready for testing + + - name: Shuffle assignees + id: shuffle + run: | + export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') + echo "$assignees" >> $GITHUB_STEP_SUMMARY + echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT + echo "shuffled_assignee=$assignees" >> $GITHUB_ENV + + - name: Check for existing issue + id: check_existing_test_issue + run: | + gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} + issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') + echo "::set-output name=issue_number::$issue_number" + + - name: Create a Test issue + uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 + id: create_test_issue + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + title: ${{ vars.TEST_ISSUE_TITLE }} + assignees: | + ${{ steps.shuffle.outputs.shuffled_assignee }} + body: | + This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works! + labels: | + test + if: steps.check_existing_test_issue.outputs.issue_number == '' diff --git a/.github/workflows/upload-legacy-bentoml.yml b/.github/workflows/upload-bentoml.yml similarity index 68% rename from .github/workflows/upload-legacy-bentoml.yml rename to .github/workflows/upload-bentoml.yml index 576c0a7..9ad4f9e 100644 --- a/.github/workflows/upload-legacy-bentoml.yml +++ b/.github/workflows/upload-bentoml.yml @@ -1,13 +1,28 @@ -name: Upload Legacy Bentoml Dockerized Model +name: Upload BentoML Dockerized Model with or without Multi-stage Conda Pack on: workflow_call: + inputs: + version: + required: true + type: string jobs: - build-legacy-bentoml-image: - if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }} + build-bentoml-image: + if: ${{ github.repository != 'ersilia-os/eos-template' }} runs-on: ubuntu-latest steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check if we can use this workflow + run: | + if [[ -f install.yml ]]; then + echo "This workflow is not supported for this repository" + exit 1 + fi + # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -24,29 +39,22 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout persist credentials - uses: actions/checkout@master - with: - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - lfs: 'true' - # This might stop working in the future, so we need to keep an eye on it - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: true + # # This might stop working in the future, so we need to keep an eye on it + # - name: Free Disk Space (Ubuntu) + # uses: jlumbroso/free-disk-space@main + # with: + # # this might remove tools that are actually needed, + # # if set to "true" but frees about 6 GB + # tool-cache: true - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - swap-storage: true + # # all of these default to true, but feel free to set to + # # "false" if necessary for your workflow + # android: true + # dotnet: true + # haskell: true + # large-packages: true + # swap-storage: true - name: Setup conda id: setupConda @@ -67,11 +75,11 @@ jobs: id: generateDockerfile env: REPO_NAME: ${{ github.event.repository.name }} + VERSION: ${{ inputs.version}} run: | - rm Dockerfile # Remove the V2 Dockerfile wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py python -m pip install requests - python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME "legacy-bentoml" + python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME $VERSION rm place_a_dockerfile_in_current_eos_repo.py # We cannot tag it as anything other than latest because @@ -87,6 +95,8 @@ jobs: # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this - name: Test built image id: testBuiltImage + env: + PULL_IMAGE: n run: | ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub ersilia -v serve ${{ github.event.repository.name }} @@ -99,8 +109,6 @@ jobs: exit 1 fi rm output.csv - env: - PULL_IMAGE: n - name: Build and push id: buildMultiple @@ -125,17 +133,7 @@ jobs: fi - name: Upload arch.txt - uses: actions/upload-artifact@v2 - with: - name: arch - path: arch.txt - - - name: Build only for Linux/amd64 - id: buildSingle - if: steps.buildCheck.outputs.failed == 'true' - uses: docker/build-push-action@v6.7.0 + uses: actions/upload-artifact@v4 with: - context: . - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ersiliaos/${{ github.event.repository.name }}:latest \ No newline at end of file + name: arch + path: arch.txt diff --git a/.github/workflows/upload-ersilia-pack.yml b/.github/workflows/upload-ersilia-pack.yml index 7954f8d..30fad4c 100644 --- a/.github/workflows/upload-ersilia-pack.yml +++ b/.github/workflows/upload-ersilia-pack.yml @@ -5,9 +5,20 @@ on: jobs: build-ersilia-pack-image: - if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.repository != 'ersilia-os/eos-template' }} runs-on: ubuntu-latest steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check if we can use this workflow + run: | + if [[ -f install.yml ]]; then + echo "This workflow is not supported for this repository" + exit 1 + fi + # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -24,29 +35,22 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + + # # This might stop working in the future, so we need to keep an eye on it + # - name: Free Disk Space (Ubuntu) + # uses: jlumbroso/free-disk-space@main + # with: + # # this might remove tools that are actually needed, + # # if set to "true" but frees about 6 GB + # tool-cache: true - - name: Checkout persist credentials - uses: actions/checkout@master - with: - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - lfs: 'true' - - # This might stop working in the future, so we need to keep an eye on it - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: true - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - swap-storage: true + # # all of these default to true, but feel free to set to + # # "false" if necessary for your workflow + # android: true + # dotnet: true + # haskell: true + # large-packages: true + # swap-storage: true # Install ersilia-pack, requests, and ersilia to test the built image with ersilia CLI - name: Setup Python for Ersilia Pack @@ -56,23 +60,27 @@ jobs: python-version: '3.10' - name: Install ersilia-pack and generate the right Dockerfile + env: + REPO_NAME: ${{ github.event.repository.name }} run: | python -m pip install git+https://github.com/ersilia-os/ersilia-pack.git python -m pip install requests python -m pip install git+https://github.com/ersilia-os/ersilia.git - python .github/scripts/resolve_dockerfile.py + python .github/scripts/resolve_dockerfile.py $REPO_NAME - name: Build only AMD64 Image for Testing id: buildForTestErsiliaPack continue-on-error: true # Allow this to fail; uses: docker/build-push-action@v6.7.0 with: - context: . + context: ../ # We need to go back to the root directory to find the Dockerfile and copy the model repository load: true tags: ersiliaos/${{ github.event.repository.name }}:latest - name: Test built image id: testBuiltImageErsiliaPack + env: + PULL_IMAGE: n if: steps.buildForTestErsiliaPack.outcome == 'success' continue-on-error: true # Allow this to fail run: | @@ -87,8 +95,6 @@ jobs: exit 1 fi rm output.csv - env: - PULL_IMAGE: n - name: Build and push id: buildMultiple @@ -96,7 +102,7 @@ jobs: uses: docker/build-push-action@v6.7.0 timeout-minutes: 45 with: - context: . + context: ../ platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ersiliaos/${{ github.event.repository.name }}:latest @@ -113,17 +119,7 @@ jobs: fi - name: Upload arch.txt - uses: actions/upload-artifact@v2 - with: - name: arch - path: arch.txt - - - name: Build only for Linux/amd64 - id: buildSingle - if: steps.buildCheck.outputs.failed == 'true' - uses: docker/build-push-action@v6.7.0 + uses: actions/upload-artifact@v4 with: - context: . - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ersiliaos/${{ github.event.repository.name }}:latest \ No newline at end of file + name: arch + path: arch.txt diff --git a/.github/workflows/upload-model-to-dockerhub.yml b/.github/workflows/upload-model-to-dockerhub.yml index 45869a1..82654a9 100644 --- a/.github/workflows/upload-model-to-dockerhub.yml +++ b/.github/workflows/upload-model-to-dockerhub.yml @@ -1,145 +1,28 @@ name: Upload model to DockerHub on: workflow_dispatch: - workflow_run: - workflows: ["Upload model to S3"] - types: - - completed + # workflow_run: + # workflows: ["Upload model to S3"] + # types: + # - completed jobs: - build-and-push-ersilia-pack: - uses: ./.github/workflows/upload-ersilia-pack.yml - - build-and-push-multistage-condapack: - if: hashFiles('install.yml') == '' - uses: ./.github/workflows/upload-multistage-condapack.yml - - build-and-push-legacy-bentoml: - if: hashFiles('install.yml') == '' - uses: ./.github/workflows/upload-legacy-bentoml.yml - - post-model-upload: - runs-on: ubuntu-latest - steps: - - - name: Check if repo follows the Ersilia Pack eos-template - id: isErsiliaPackTemplate - run: | - if [ -f install.yml ]; then - echo "ERSILIA_PACK=true" >> $GITHUB_ENV - else - echo "ERSILIA_PACK=false" >> $GITHUB_ENV - fi - - - - name: Download arch.txt - uses: actions/download-artifact@v3 - with: - name: arch - path: ./arch.txt - - - name: Update Metadata YAML file with DockerHub info - id: updateMetadata - run: | - mv Dockerfile_legacy Dockerfile - python3 -c " - import yaml - with open('metadata.yml', 'r') as f: - data = yaml.safe_load(f) - print(data) - with open('arch.txt', 'r') as f: - arch = f.read().rstrip() - arch = arch.split(',') - data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) - data['Docker Architecture'] = arch - with open('metadata.yml', 'w') as f: - yaml.dump(data, default_flow_style=False, sort_keys=False) - " - rm arch.txt - - - name: Commit and push changes done to the Metadata YAML file - uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 - with: - author_name: "ersilia-bot" - author_email: "ersilia-bot@users.noreply.github.com" - message: "updating metadata [skip ci]" - repository: "ersilia-os/${{ github.event.repository.name }}" - github_token: ${{ secrets.GITHUB_TOKEN }} - amend: true - force: true - - - name: Update metadata to AirTable - id: update-metadata-to-airtable - env: - USER_NAME: ${{ github.repository_owner }} - BRANCH: "main" - REPO_NAME: ${{ github.event.repository.name }} - AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} - run: | - source activate - echo "Updating metadata to AirTable looking at owner: $USER_NAME" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py - python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY - rm update_metadata_to_airtable.py - - - name: Update README file - id: update-readme-file - env: - MODEL_ID: ${{ github.event.repository.name }} - run: | - source activate - echo "Updating README file with AirTable metadata for model: $MODEL_ID" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py - python3 update_readme_from_airtable.py $MODEL_ID . - rm update_readme_from_airtable.py - less README.md - - - name: Commit and push changes done to the README file - uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 - with: - author_name: "ersilia-bot" - author_email: "ersilia-bot@users.noreply.github.com" - message: "updating readme [skip ci]" - repository: "ersilia-os/${{ github.event.repository.name }}" - github_token: ${{ secrets.GITHUB_TOKEN }} - amend: true - force: true - - - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: ersiliaos/${{ github.event.repository.name }} - short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}" - - # Create an issue within the repository to track that this model is ready for testing - - - name: Shuffle assignees - id: shuffle - run: | - export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') - echo "$assignees" >> $GITHUB_STEP_SUMMARY - echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT - echo "shuffled_assignee=$assignees" >> $GITHUB_ENV - - - name: Check for existing issue - id: check_existing_test_issue - run: | - gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} - issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') - echo "::set-output name=issue_number::$issue_number" - - - name: Create a Test issue - uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 - id: create_test_issue - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - title: ${{ vars.TEST_ISSUE_TITLE }} - assignees: | - ${{ steps.shuffle.outputs.shuffled_assignee }} - body: | - This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works! - labels: | - test - if: steps.check_existing_test_issue.outputs.issue_number == '' + upload-ersilia-pack: + uses: ./.github/workflows/upload-ersilia-pack.yml + secrets: inherit + + upload-multi-stage-condapack: + needs: [upload-ersilia-pack] + if: ${{ failure() }} + uses: ./.github/workflows/upload-bentoml.yml + secrets: inherit + with: + version: multistage-condapack + + upload-legacy-bentoml: + needs: [upload-multi-stage-condapack] + if: ${{ failure() }} + uses: ./.github/workflows/upload-bentoml.yml + secrets: inherit + with: + version: legacy-bentoml \ No newline at end of file diff --git a/.github/workflows/upload-multistage-condapack.yml b/.github/workflows/upload-multistage-condapack.yml deleted file mode 100644 index ef42942..0000000 --- a/.github/workflows/upload-multistage-condapack.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: Upload Multi-stage Conda Pack Dockerized Model - -on: - workflow_call: - -jobs: - build-multistage-condapack-image: - if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - steps: - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - # log in to dockerhub - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout persist credentials - uses: actions/checkout@master - with: - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - lfs: 'true' - - # This might stop working in the future, so we need to keep an eye on it - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: true - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - swap-storage: true - - # Install ersilia in the next few steps if the repo follows the old eos-template - - name: Setup conda - id: setupConda - uses: conda-incubator/setup-miniconda@v3 - with: - auto-update-conda: true - python-version: "3.10.10" - - - name: Install dependencies in Conda environment - id: installDependenciesInConda - run: | - conda install git-lfs -c conda-forge - git-lfs install - conda install gh -c conda-forge - python -m pip install git+https://github.com/ersilia-os/ersilia.git - - - name: Generate the Dockerfile - id: generateDockerfile - env: - REPO_NAME: ${{ github.event.repository.name }} - run: | - wget -O place_a_dockerfile_in_current_eos_repo.py https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py - python -m pip install requests - python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME - rm place_a_dockerfile_in_current_eos_repo.py - - - name: Build only AMD64 Image for Testing - id: buildForTest - continue-on-error: true # Allow this to fail if conda-pack doesn't work for this model - uses: docker/build-push-action@v6.7.0 - with: - context: . - load: true - tags: ersiliaos/${{ github.event.repository.name }}:latest - - - name: Test built image - id: testBuiltImage - continue-on-error: true # Allow this to fail - run: | - ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub - ersilia -v serve ${{ github.event.repository.name }} - ersilia example -n 1 -f input.csv --predefined - ersilia -v run -i "input.csv" -o "output.csv" - ersilia close - output=$(python .github/scripts/verify_model_outcome.py output.csv) - if echo "$output" | grep -q "All outcomes are null"; then - echo "Error in model outcome, aborting build" - exit 1 - fi - rm output.csv - env: - PULL_IMAGE: n - - - name: Build and push - id: buildMultiple - continue-on-error: true - uses: docker/build-push-action@v6.7.0 - timeout-minutes: 45 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ersiliaos/${{ github.event.repository.name }}:latest - - - name: Set build failure output - id: buildCheck - run: | - if [[ "${{ steps.buildMultiple.outcome }}" == "failure" ]]; then - echo "::set-output name=failed::true" - echo "AMD64" > arch.txt - else - echo "::set-output name=failed::false" - echo "AMD64,ARM64" > arch.txt - fi - - - name: Upload arch.txt - uses: actions/upload-artifact@v3 - with: - name: arch - path: arch.txt - - - name: Build only for Linux/amd64 - id: buildSingle - if: steps.buildCheck.outputs.failed == 'true' - uses: docker/build-push-action@v6.7.0 - with: - context: . - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ersiliaos/${{ github.event.repository.name }}:latest \ No newline at end of file