Feat/ci cd (#5)

* lint * add linting in ci * add unit tests * fix version pytest * add docker image build * makefile * deploy * add cd * fix workflows * wip: cd * change permission * cm hash * permission * update new way to auth gcp from github action * wip * fix prometheus config * fix cicd * wip * wip * fix dockerhub repo name * chagnge triggering to push to main only * documentation
anhtu293 · Nov 20, 2024 · 5a4967a · 5a4967a
1 parent 9315ff1
commit 5a4967a
Show file tree

Hide file tree

Showing 619 changed files with 1,359 additions and 49,980 deletions.
diff --git a/.github/workflows/deployment.yaml b/.github/workflows/deployment.yaml
@@ -0,0 +1,88 @@
+name: Deploy Model Serving Cluster GKE
+on:
+  push:
+    branches:
+      - "master"
+    paths-ignore:
+      - "*.md"
+
+permissions:
+  contents: "read"
+  id-token: "write"
+jobs:
+  Build-Docker-Image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and Push Lens Docker Image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./lens
+          file: ./lens/Dockerfile
+          tags: anhtu293/lens-app:${{ github.sha }}
+          push: true
+      - name: Build and Push OCR Docker Image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./ocr-app
+          file: ./ocr-app/Dockerfile
+          tags: anhtu293/ocr-app:${{ github.sha }}
+          push: true
+      - name: Build and Push Translation Docker Image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./trans-app
+          file: ./trans-app/Dockerfile
+          tags: anhtu293/translation-app:${{ github.sha }}
+          push: true
+
+  Deploy-Model-Serving-Cluster:
+    runs-on: ubuntu-latest
+    needs: [Build-Docker-Image]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - id: "auth"
+        name: "Authenticate to Google Cloud"
+        uses: "google-github-actions/auth@f112390a2df9932162083945e46d439060d66ec2" # google-github-actions/auth@v2
+        with:
+          credentials_json: "${{ secrets.GCP_CREDENTIALS }}"
+
+      - name: "Set up Cloud SDK"
+        uses: "google-github-actions/setup-gcloud@v2"
+
+      - name: "Use gcloud CLI"
+        run: "gcloud info"
+
+      # Get the GKE credentials so we can deploy to the cluster
+      - name: "Set up GKE credentials"
+        uses: "google-github-actions/get-gke-credentials@6051de21ad50fbb1767bc93c11357a49082ad116" # google-github-actions/get-gke-credentials@v2
+        with:
+          cluster_name: "${{ vars.GKE_CLUSTER }}"
+          location: "${{ vars.GKE_ZONE }}"
+
+      - name: Install Helm
+        run: |
+          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+
+      - name: Deploy Lens App
+        run: |
+          echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+          curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+          sudo apt update
+          sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin kubectl
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud container clusters get-credentials ${{ vars.GKE_CLUSTER }} \
+            --zone ${{ vars.GKE_ZONE }} \
+            --project ${{ secrets.GKE_PROJECT }}
+          cd deployments/model-serving
+          helm upgrade --install app ./lens --set lens_app.image.tag=${{ github.sha }} --set ocr_app.image.tag=${{ github.sha }} --set translation_app.image.tag=${{ github.sha }} --namespace model-serving
diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml
@@ -0,0 +1,44 @@
+name: lens-translation Unit Tests
+on:
+  push:
+jobs:
+  Lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python 3.11.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.9"
+      - name: Install flake8
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8==7.1.1
+          pip install flake8_black==0.3.6
+      - name: Lint with flake8
+        run: |
+          flake8 --max-line-length=119 --ignore=E722,I201,I100,BLK100,W503 --per-file-ignores=__init__.py:F401 lens/app/*.py
+          flake8 --max-line-length=119 --ignore=E722,I201,I100,BLK100,W503 --per-file-ignores=__init__.py:F401 trans-app/app/*.py
+          flake8 --max-line-length=119 --ignore=E722,I201,I100,BLK100,W503 --per-file-ignores=__init__.py:F401 ocr-app/app/*.py
+
+  Unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python 3.10.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10.12"
+      - name: Install libraries
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest==8.3.3
+          pip install pytest-asyncio==0.24.0
+          pip install numpy==1.23.1
+          pip install Pillow==9.5.0
+      - name: Run unit tests
+        run: |
+          export PYTHONPATH=$PYTHONPATH:$(pwd)
+          pytest unittests/test_draw_fct.py