Cleaned up workflows using reusable actions

ersilia-os · Sep 29, 2024 · 37a128b · 37a128b
1 parent d096c30
commit 37a128b
Show file tree

Hide file tree

Showing 6 changed files with 280 additions and 369 deletions.
diff --git a/.github/scripts/resolve_dockerfile.py b/.github/scripts/resolve_dockerfile.py
@@ -1,3 +1,4 @@
+import sys
 import os
 import requests
 
@@ -11,7 +12,7 @@
     '3.11': 'py311',
     '3.12': 'py312'
 }
-
+model_id = sys.argv[1]
 def resolve_parser():
     if os.path.exists(os.path.abspath(os.path.join(REPO_PATH, "Dockerfile"))):
         return DockerfileInstallParser(file_dir=REPO_PATH)
@@ -24,17 +25,18 @@ def resolve_python_version(parser):
     return parser._get_python_version()
 
 def read_dockerfile(parser):
-    if isinstance(parser, DockerfileInstallParser):
+    commands = parser._get_commands()
+    has_conda = parser._has_conda(commands)
+    if has_conda:
         file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.conda"
-    elif isinstance(parser, YAMLInstallParser):
-        file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.pip"
     else:
-        raise ValueError("Invalid parser type")
+        file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.pip"
     response = requests.get(file_url)
     return response.text
 
-def write_version(file_content, python_version):
+def write_version_and_model_id(file_content, python_version):
     python_version = PY_VERSION_MAP[python_version]
+    file_content = file_content.replace("eos_identifier", model_id)
     lines = file_content.split("\n")
     lines[0] = lines[0].replace("VERSION", python_version)
     with open(os.path.join(REPO_PATH, "../", "Dockerfile"), "w") as f:
@@ -44,4 +46,4 @@ def write_version(file_content, python_version):
     parser = resolve_parser()
     python_version = resolve_python_version(parser)
     dockerfile = read_dockerfile(parser)
-    write_version(dockerfile, python_version)
+    write_version_and_model_id(dockerfile, python_version)
diff --git a/.github/workflows/post-model-upload.yml b/.github/workflows/post-model-upload.yml
@@ -0,0 +1,172 @@
+name: Post Model Upload actions
+
+on:
+  workflow_run:
+    workflows: ["Upload model to DockerHub"]
+    types:
+      - completed
+
+jobs:
+  post-model-upload:
+    if: ${{ github.repository != 'ersilia-os/eos-template' }}
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download arch.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: arch
+          path: .
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check metadata file
+        id: checkMetadata
+        continue-on-error: true
+        run: |
+          if [[ ! -f metadata.yml ]]; then
+            echo "metadata.yml file not found"
+            exit 1
+          fi
+
+      - name: Update Metadata YAML file with DockerHub info
+        id: updateMetadataYAML
+        if: steps.checkMetadata.outcome == 'success'
+        run: |
+          python3 -c "
+          import yaml
+          with open('metadata.yml', 'r') as f:
+            data = yaml.safe_load(f)
+          print(data)
+          with open('arch.txt', 'r') as f:
+            arch = f.read().rstrip()
+            arch = arch.split(',')
+          data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier'])
+          data['Docker Architecture'] = arch
+          with open('metadata.yml', 'w') as f:
+            yaml.dump(data, default_flow_style=False, sort_keys=False)
+          "
+          rm arch.txt
+
+      - name: Update Metadata JSON file with DockerHub info
+        id: updateMetadataJSON
+        if: steps.checkMetadata.outcome == 'failure'
+        run: |
+          python3 -c "
+          import json
+          with open('metadata.json', 'r') as f:
+            data = json.load(f)
+          print(data)
+          with open('arch.txt', 'r') as f:
+            arch = f.read().rstrip()
+            arch = arch.split(',')
+          data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier'])
+          data['Docker Architecture'] = arch
+          with open('metadata.json', 'w') as f:
+            json.dump(data, f, indent=4)
+          "
+          rm arch.txt
+      
+
+      - name: Commit and push changes done to the Metadata file
+        uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected]
+        with:
+          author_name: "ersilia-bot"
+          author_email: "[email protected]"
+          message: "updating metadata [skip ci]"
+          repository: "ersilia-os/${{ github.event.repository.name }}"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          amend: true
+          force: true
+
+      # Setup conda
+      - name: Setup conda
+        id: setupConda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          python-version: "3.10.10"
+
+      # Install ersilia
+      - name: Install dependencies in Conda environment
+        id: installDependenciesInConda
+        run: |
+          conda install gh -c conda-forge
+          python -m pip install git+https://github.com/ersilia-os/ersilia.git
+
+      - name: Update metadata to AirTable
+        id: update-metadata-to-airtable
+        env:
+          USER_NAME: ${{ github.repository_owner }}
+          BRANCH: "main"
+          REPO_NAME: ${{ github.event.repository.name }}
+          AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
+        run: |
+          echo "Updating metadata to AirTable looking at owner: $USER_NAME"
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py
+          python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY
+          rm update_metadata_to_airtable.py
+  
+      - name: Update README file
+        id: update-readme-file
+        env:
+          MODEL_ID: ${{ github.event.repository.name }}
+        run: |
+          echo "Updating README file with AirTable metadata for model: $MODEL_ID"
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py
+          python3 update_readme_from_airtable.py $MODEL_ID .
+          rm update_readme_from_airtable.py
+          less README.md
+
+      - name: Commit and push changes done to the README file
+        uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected]
+        with:
+          author_name: "ersilia-bot"
+          author_email: "[email protected]"
+          message: "updating readme [skip ci]"
+          repository: "ersilia-os/${{ github.event.repository.name }}"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          amend: true
+          force: true
+
+      - name: Docker Hub Description
+        uses: peter-evans/dockerhub-description@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          repository: ersiliaos/${{ github.event.repository.name }}
+          short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}"
+
+      # Create an issue within the repository to track that this model is ready for testing
+
+      - name: Shuffle assignees
+        id: shuffle
+        run: |
+          export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}')
+          echo "$assignees" >> $GITHUB_STEP_SUMMARY
+          echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT
+          echo "shuffled_assignee=$assignees" >> $GITHUB_ENV
+
+      - name: Check for existing issue
+        id: check_existing_test_issue
+        run: |
+          gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }}
+          issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number')
+          echo "::set-output name=issue_number::$issue_number"
+  
+      - name: Create a Test issue
+        uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49
+        id: create_test_issue 
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          title: ${{ vars.TEST_ISSUE_TITLE }}
+          assignees: |
+            ${{ steps.shuffle.outputs.shuffled_assignee }}
+          body: |
+            This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works!
+          labels: |
+            test
+        if: steps.check_existing_test_issue.outputs.issue_number == ''
diff --git a/.github/workflows/upload-legacy-bentoml.yml → .github/workflows/upload-bentoml.yml b/.github/workflows/upload-legacy-bentoml.yml → .github/workflows/upload-bentoml.yml
@@ -1,13 +1,28 @@
-name: Upload Legacy Bentoml Dockerized Model
+name: Upload BentoML Dockerized Model with or without Multi-stage Conda Pack
 
 on:
   workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
 
 jobs:
-  build-legacy-bentoml-image:
-    if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }}
+  build-bentoml-image:
+    if: ${{ github.repository != 'ersilia-os/eos-template' }}
     runs-on: ubuntu-latest
     steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check if we can use this workflow
+        run: |
+          if [[ -f install.yml ]]; then
+            echo "This workflow is not supported for this repository"
+            exit 1
+          fi
+      
       # https://github.com/docker/setup-qemu-action
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -24,29 +39,22 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-      - name: Checkout persist credentials
-        uses: actions/checkout@master
-        with:
-          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
-          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
-          lfs: 'true'
 
-      # This might stop working in the future, so we need to keep an eye on it
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: true
+      # # This might stop working in the future, so we need to keep an eye on it
+      # - name: Free Disk Space (Ubuntu)
+      #   uses: jlumbroso/free-disk-space@main
+      #   with:
+      #     # this might remove tools that are actually needed,
+      #     # if set to "true" but frees about 6 GB
+      #     tool-cache: true
 
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: true
-          swap-storage: true
+      #     # all of these default to true, but feel free to set to
+      #     # "false" if necessary for your workflow
+      #     android: true
+      #     dotnet: true
+      #     haskell: true
+      #     large-packages: true
+      #     swap-storage: true
 
       - name: Setup conda
         id: setupConda
@@ -67,11 +75,11 @@ jobs:
         id: generateDockerfile
         env:
           REPO_NAME: ${{ github.event.repository.name }}
+          VERSION: ${{ inputs.version}}
         run: |
-          rm Dockerfile # Remove the V2 Dockerfile
           wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py
           python -m pip install requests
-          python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME "legacy-bentoml"
+          python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME $VERSION
           rm place_a_dockerfile_in_current_eos_repo.py
 
       # We cannot tag it as anything other than latest because 
@@ -87,6 +95,8 @@ jobs:
       # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this
       - name: Test built image
         id: testBuiltImage
+        env:
+          PULL_IMAGE: n
         run: |
           ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub
           ersilia -v serve ${{ github.event.repository.name }}
@@ -99,8 +109,6 @@ jobs:
             exit 1
           fi
           rm output.csv
-        env:
-          PULL_IMAGE: n
 
       - name: Build and push
         id: buildMultiple
@@ -125,17 +133,7 @@ jobs:
           fi
       
       - name: Upload arch.txt
-      uses: actions/upload-artifact@v2
-      with:
-        name: arch
-        path: arch.txt
-
-      - name: Build only for Linux/amd64
-        id: buildSingle
-        if: steps.buildCheck.outputs.failed == 'true'
-        uses: docker/[email protected]
+        uses: actions/upload-artifact@v4
         with:
-          context: .
-          platforms: linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ersiliaos/${{ github.event.repository.name }}:latest
+          name: arch
+          path: arch.txt