generated from ersilia-os/eos-template
-
Notifications
You must be signed in to change notification settings - Fork 3
156 lines (140 loc) · 5.92 KB
/
upload-model-to-s3.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: Upload model to S3
on:
workflow_dispatch:
workflow_run:
workflows: ["Model test on push"]
types:
- completed
jobs:
upload_model_to_s3:
if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest
steps:
- name: Checkout persist credentials
uses: actions/checkout@master
with:
persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
lfs: 'true'
- name: Add conda to system path
run: echo $CONDA/bin >> $GITHUB_PATH
- name: Source conda
run: source $CONDA/etc/profile.d/conda.sh
- name: Set Python to 3.10.10
run:
conda install -y python=3.10.10
- name: Install dependencies
run: |
source activate
conda init
conda install git-lfs -c conda-forge
git-lfs install
conda install gh -c conda-forge
- name: Install ersilia
run: |
source activate
python --version
echo "After conda init"
conda init
python -m pip install git+https://github.com/ersilia-os/ersilia.git
- name: Check metadata file
id: checkMetadata
continue-on-error: true
run: |
if [[ ! -f metadata.yml ]]; then
echo "metadata.yml file not found"
exit 1
fi
- name: Update Metadata JSON file with DockerHub info
if: steps.checkMetadata.outcome == 'failure'
id: UpdateMetadataJSON
run: |
python3 -c "
import json
with open('metadata.json', 'r') as f:
data = json.load(f)
data['S3'] = 'https://ersilia-models-zipped.s3.eu-central-1.amazonaws.com/{0}.zip'.format(data['Identifier'])
with open('metadata.json', 'w') as f:
json.dump(data, f, indent=4)
"
- name: Update Metadata YAML file with DockerHub info
if: steps.checkMetadata.outcome == 'success'
id: UpdateMetadataYAML
run: |
python3 -c "
import yaml
with open('metadata.yml', 'r') as f:
data = yaml.safe_load(f)
data['S3'] = 'https://ersilia-models-zipped.s3.eu-central-1.amazonaws.com/{0}.zip'.format(data['Identifier'])
with open('metadata.yml', 'w') as f:
yaml.dump(data, f, default_flow_style=False, sort_keys=False)
"
- name: Commit and push changes done to the Metadata JSON file
uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected]
with:
author_name: "ersilia-bot"
author_email: "[email protected]"
message: "updating metadata [skip ci]"
repository: "ersilia-os/${{ github.event.repository.name }}"
github_token: ${{ secrets.GITHUB_TOKEN }}
amend: true
force: true
- name: Update metadata to AirTable
id: update-metadata-to-airtable
env:
USER_NAME: ${{ github.repository_owner }}
BRANCH: "main"
REPO_NAME: ${{ github.event.repository.name }}
AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
run: |
source activate
pip install requests pyairtable
echo "Updating metadata to AirTable looking at owner: $USER_NAME"
wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY
rm airtableops.py
- name: sync metadata to S3 JSON
id: sync-metadata-to-s3
env:
AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
source activate
wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py
pip install boto3 requests pyairtable
python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
rm convert_airtable_to_json.py
- name: Update README file
id: update-readme-file
env:
MODEL_ID: ${{ github.event.repository.name }}
run: |
source activate
echo "Updating README file with AirTable metadata for model: $MODEL_ID"
wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
python3 airtableops.py readme-update --repo $MODEL_ID --path .
rm airtableops.py
less README.md
- name: Commit and push changes done to the README file
uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected]
with:
author_name: "ersilia-bot"
author_email: "[email protected]"
message: "updating readme [skip ci]"
repository: "ersilia-os/${{ github.event.repository.name }}"
github_token: ${{ secrets.GITHUB_TOKEN }}
amend: true
force: true
- name: Upload model to S3
id: upload-model-to-s3
env:
REPO_NAME: ${{ github.event.repository.name }}
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
source activate
echo "Uploading model to S3 bucket"
wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/upload_model_to_s3.py
python3 upload_model_to_s3.py $REPO_NAME $AWS_ACCESS_KEY $AWS_SECRET_ACCESS_KEY .
rm upload_model_to_s3.py