-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Modify upload model to dockerhub workflow to support backward compati…
…bility (#68) * Separate different docker building strategies into reusable workflows; clean up main pipeline * Cleaned up workflows using reusable actions
- Loading branch information
1 parent
a400370
commit cce1bb6
Showing
5 changed files
with
476 additions
and
252 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,179 @@ | ||
| name: Post Model Upload actions | ||
|
|
||
| on: | ||
| workflow_run: | ||
| workflows: ["Upload model to DockerHub"] | ||
| types: | ||
| - completed | ||
|
|
||
| jobs: | ||
| post-model-upload: | ||
| if: ${{ github.repository != 'ersilia-os/eos-template' }} | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
|
|
||
| - name: Checkout repository | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - name: Download arch.txt | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: arch | ||
| path: . | ||
| run-id: ${{ github.event.workflow_run.id }} | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Check metadata file | ||
| id: checkMetadata | ||
| continue-on-error: true | ||
| run: | | ||
| if [[ ! -f metadata.yml ]]; then | ||
| echo "metadata.yml file not found" | ||
| exit 1 | ||
| fi | ||
| - name: Update Metadata YAML file with DockerHub info | ||
| id: updateMetadataYAML | ||
| if: steps.checkMetadata.outcome == 'success' | ||
| run: | | ||
| python3 -c " | ||
| import yaml | ||
| with open('metadata.yml', 'r') as f: | ||
| data = yaml.safe_load(f) | ||
| print(data) | ||
| with open('arch.txt', 'r') as f: | ||
| arch = f.read().rstrip() | ||
| arch = arch.split(',') | ||
| data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) | ||
| data['Docker Architecture'] = arch | ||
| with open('metadata.yml', 'w') as f: | ||
| yaml.dump(data, default_flow_style=False, sort_keys=False) | ||
| " | ||
| rm arch.txt | ||
| - name: Update Metadata JSON file with DockerHub info | ||
| id: updateMetadataJSON | ||
| if: steps.checkMetadata.outcome == 'failure' | ||
| run: | | ||
| python3 -c " | ||
| import json | ||
| with open('metadata.json', 'r') as f: | ||
| data = json.load(f) | ||
| print(data) | ||
| with open('arch.txt', 'r') as f: | ||
| arch = f.read().rstrip() | ||
| arch = arch.split(',') | ||
| data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) | ||
| data['Docker Architecture'] = arch | ||
| with open('metadata.json', 'w') as f: | ||
| json.dump(data, f, indent=4) | ||
| " | ||
| rm arch.txt | ||
|
|
||
| - name: Commit and push changes done to the Metadata file | ||
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected] | ||
| with: | ||
| author_name: "ersilia-bot" | ||
| author_email: "[email protected]" | ||
| message: "updating metadata [skip ci]" | ||
| repository: "ersilia-os/${{ github.event.repository.name }}" | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| amend: true | ||
| force: true | ||
|
|
||
| # Setup conda | ||
| - name: Setup conda | ||
| id: setupConda | ||
| uses: conda-incubator/setup-miniconda@v3 | ||
| with: | ||
| auto-update-conda: true | ||
| python-version: "3.10.10" | ||
|
|
||
| # Install ersilia | ||
| - name: Install dependencies in Conda environment | ||
| id: installDependenciesInConda | ||
| run: | | ||
| conda install gh -c conda-forge | ||
| python -m pip install git+https://github.com/ersilia-os/ersilia.git | ||
| - name: Update metadata to AirTable | ||
| id: update-metadata-to-airtable | ||
| env: | ||
| USER_NAME: ${{ github.repository_owner }} | ||
| BRANCH: "main" | ||
| REPO_NAME: ${{ github.event.repository.name }} | ||
| AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} | ||
| run: | | ||
| echo "Updating metadata to AirTable looking at owner: $USER_NAME" | ||
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py | ||
| python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY | ||
| rm update_metadata_to_airtable.py | ||
| - name: Update README file | ||
| id: update-readme-file | ||
| env: | ||
| MODEL_ID: ${{ github.event.repository.name }} | ||
| run: | | ||
| echo "Updating README file with AirTable metadata for model: $MODEL_ID" | ||
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py | ||
| python3 update_readme_from_airtable.py $MODEL_ID . | ||
| rm update_readme_from_airtable.py | ||
| less README.md | ||
| - name: Commit and push changes done to the README file | ||
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected] | ||
| with: | ||
| author_name: "ersilia-bot" | ||
| author_email: "[email protected]" | ||
| message: "updating readme [skip ci]" | ||
| repository: "ersilia-os/${{ github.event.repository.name }}" | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| amend: true | ||
| force: true | ||
|
|
||
| - name: Docker Hub Description | ||
| uses: peter-evans/dockerhub-description@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | ||
| repository: ersiliaos/${{ github.event.repository.name }} | ||
| short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}" | ||
|
|
||
| # Create an issue within the repository to track that this model is ready for testing | ||
|
|
||
| - name: Shuffle assignees | ||
| id: shuffle | ||
| run: | | ||
| export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') | ||
| echo "$assignees" >> $GITHUB_STEP_SUMMARY | ||
| echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT | ||
| echo "shuffled_assignee=$assignees" >> $GITHUB_ENV | ||
| - name: Check for existing issue | ||
| id: check_existing_test_issue | ||
| run: | | ||
| gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} | ||
| issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') | ||
| echo "::set-output name=issue_number::$issue_number" | ||
| - name: Create a Test issue | ||
| uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 | ||
| id: create_test_issue | ||
| with: | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| title: ${{ vars.TEST_ISSUE_TITLE }} | ||
| assignees: | | ||
| ${{ steps.shuffle.outputs.shuffled_assignee }} | ||
| body: | | ||
| This model is a new incorporation to the Ersilia Model Hub or it has been modified. If you are assigned to this issue, please try it out and ensure everything works! | ||
| To test a model, first clone it in your local system (ideally, from dockerhub) using the CLI commands: | ||
| ``` | ||
| ersilia -v fetch eosxxxx --from_dockerhub | ||
| ersilia serve eosxxxx | ||
| ersilia test | ||
| ``` | ||
| The test command will automatically check that the model can handle null outputs and whether it produces consistent results. Please copy here the result of the test command. If it passes, simply close the issue as completed. If it fails, please detail at which step and whether you have taken any steps to solve it. Please tag the original model contributor and one of Ersilia's maintainers for support. | ||
| labels: | | ||
| test | ||
| if: steps.check_existing_test_issue.outputs.issue_number == '' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| name: Upload BentoML Dockerized Model with or without Multi-stage Conda Pack | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| version: | ||
| required: true | ||
| type: string | ||
|
|
||
| jobs: | ||
| build-bentoml-image: | ||
| if: ${{ github.repository != 'ersilia-os/eos-template' }} | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
|
|
||
| - name: Checkout repository | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - name: Check if we can use this workflow | ||
| run: | | ||
| if [[ -f install.yml ]]; then | ||
| echo "This workflow is not supported for this repository" | ||
| exit 1 | ||
| fi | ||
| # https://github.com/docker/setup-qemu-action | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v3 | ||
|
|
||
| # https://github.com/docker/setup-buildx-action | ||
| - name: Set up Docker Buildx | ||
| id: buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
|
|
||
| # log in to dockerhub | ||
| - name: Login to Docker Hub | ||
| if: github.event_name != 'pull_request' | ||
| uses: docker/login-action@v1 | ||
| with: | ||
| username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | ||
|
|
||
| # # This might stop working in the future, so we need to keep an eye on it | ||
| # - name: Free Disk Space (Ubuntu) | ||
| # uses: jlumbroso/free-disk-space@main | ||
| # with: | ||
| # # this might remove tools that are actually needed, | ||
| # # if set to "true" but frees about 6 GB | ||
| # tool-cache: true | ||
|
|
||
| # # all of these default to true, but feel free to set to | ||
| # # "false" if necessary for your workflow | ||
| # android: true | ||
| # dotnet: true | ||
| # haskell: true | ||
| # large-packages: true | ||
| # swap-storage: true | ||
|
|
||
| - name: Setup conda | ||
| id: setupConda | ||
| uses: conda-incubator/setup-miniconda@v3 | ||
| with: | ||
| auto-update-conda: true | ||
| python-version: "3.10.10" | ||
|
|
||
| - name: Install dependencies in Conda environment | ||
| id: installDependenciesInConda | ||
| run: | | ||
| conda install git-lfs -c conda-forge | ||
| git-lfs install | ||
| conda install gh -c conda-forge | ||
| python -m pip install git+https://github.com/ersilia-os/ersilia.git | ||
| - name: Generate the Dockerfile | ||
| id: generateDockerfile | ||
| env: | ||
| REPO_NAME: ${{ github.event.repository.name }} | ||
| VERSION: ${{ inputs.version}} | ||
| run: | | ||
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py | ||
| python -m pip install requests | ||
| python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME $VERSION | ||
| rm place_a_dockerfile_in_current_eos_repo.py | ||
| # We cannot tag it as anything other than latest because | ||
| # ersilia cli only looks for the 'latest' tag | ||
| - name: Build only AMD64 Image for Testing | ||
| id: buildForTest | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: . | ||
| load: true | ||
| tags: ersiliaos/${{ github.event.repository.name }}:latest | ||
|
|
||
| # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this | ||
| - name: Test built image | ||
| id: testBuiltImage | ||
| env: | ||
| PULL_IMAGE: n | ||
| run: | | ||
| ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub | ||
| ersilia -v serve ${{ github.event.repository.name }} | ||
| ersilia example -n 1 -f input.csv --predefined | ||
| ersilia -v run -i "input.csv" -o "output.csv" | ||
| ersilia close | ||
| output=$(python .github/scripts/verify_model_outcome.py output.csv) | ||
| if echo "$output" | grep -q "All outcomes are null"; then | ||
| echo "Error in model outcome, aborting build" | ||
| exit 1 | ||
| fi | ||
| rm output.csv | ||
| - name: Build and push | ||
| id: buildMultiple | ||
| continue-on-error: true | ||
| uses: docker/[email protected] | ||
| timeout-minutes: 45 | ||
| with: | ||
| context: . | ||
| platforms: linux/amd64,linux/arm64 | ||
| push: ${{ github.event_name != 'pull_request' }} | ||
| tags: ersiliaos/${{ github.event.repository.name }}:latest | ||
|
|
||
| - name: Set build failure output | ||
| id: buildCheck | ||
| run: | | ||
| if [[ "${{ steps.buildMultiple.outcome }}" == "failure" ]]; then | ||
| echo "::set-output name=failed::true" | ||
| echo "AMD64" > arch.txt | ||
| else | ||
| echo "::set-output name=failed::false" | ||
| echo "AMD64,ARM64" > arch.txt | ||
| fi | ||
| - name: Upload arch.txt | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: arch | ||
| path: arch.txt |
Oops, something went wrong.