From 0cd5dc6e9873d4cfaa8eb60c6d3f5a3fadcddcaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=9B=A7=E5=9B=A7?= Date: Wed, 4 Sep 2024 16:12:10 -0400 Subject: [PATCH] Deploy extensions to Dockerhub; add draft purge script (#4192) (cherry picked from commit 40dadb8576f45026fb97bea1c2bc2d5676d2ce40) --- .../workflows/build-and-deploy-extension.yml | 7 ++- .github/workflows/deploy-extension.yml | 46 +++++++++++++++++++ .github/workflows/purge-extension.yml | 11 +++++ scripts/extension/Dockerfile | 5 ++ scripts/extension/PRODUCTION_RELEASES | 5 ++ scripts/extension/compose.yaml | 10 ++++ scripts/extension/nginx.conf | 32 +++++++++++++ scripts/extension/purge-beta.py | 33 +++++++++++++ 8 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy-extension.yml create mode 100644 .github/workflows/purge-extension.yml create mode 100644 scripts/extension/Dockerfile create mode 100644 scripts/extension/PRODUCTION_RELEASES create mode 100644 scripts/extension/compose.yaml create mode 100644 scripts/extension/nginx.conf create mode 100644 scripts/extension/purge-beta.py diff --git a/.github/workflows/build-and-deploy-extension.yml b/.github/workflows/build-and-deploy-extension.yml index 196208c7226..45621611e56 100644 --- a/.github/workflows/build-and-deploy-extension.yml +++ b/.github/workflows/build-and-deploy-extension.yml @@ -35,7 +35,7 @@ jobs: echo "RELEASE_VERSION=v$(cat CMakeLists.txt | grep "DKUZU_EXTENSION_VERSION" | cut -d '"' -f 2 )" >> $GITHUB_ENV - name: Delete checked out repository - run: cd .. && rm -rf kuzu + run: rm -rf ./* - uses: actions/download-artifact@v4 with: @@ -111,3 +111,8 @@ jobs: path: | releases dataset + + deploy-extensions: + needs: package-built-extensions + uses: ./.github/workflows/deploy-extension.yml + secrets: inherit diff --git a/.github/workflows/deploy-extension.yml b/.github/workflows/deploy-extension.yml new file mode 100644 index 00000000000..e1d302d4b0a --- /dev/null +++ b/.github/workflows/deploy-extension.yml @@ -0,0 +1,46 @@ +name: Deploy Extensions + +on: + workflow_call: + +jobs: + deploy-extensions: + runs-on: ubuntu-latest + steps: + - name: Free disk space on Ubuntu runner + uses: kfir4444/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - uses: actions/checkout@v4 + + - name: Restructure directories + run: | + mv scripts/extension ../extension-scripts + rm -rf * + mv ../extension-scripts/* . + + - uses: actions/download-artifact@v4 + with: + name: kuzu-extensions + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: kuzudb/extension-repo:latest diff --git a/.github/workflows/purge-extension.yml b/.github/workflows/purge-extension.yml new file mode 100644 index 00000000000..8b207e70c24 --- /dev/null +++ b/.github/workflows/purge-extension.yml @@ -0,0 +1,11 @@ +name: Purge Extensions + +on: + workflow_dispatch: + +jobs: + get-dockerhub-extensions: + uses: ./.github/workflows/get-extensions-from-dockerhub.yml + + # purge-extensions: + # needs: get-dockerhub-extensions \ No newline at end of file diff --git a/scripts/extension/Dockerfile b/scripts/extension/Dockerfile new file mode 100644 index 00000000000..9b95aabf98c --- /dev/null +++ b/scripts/extension/Dockerfile @@ -0,0 +1,5 @@ +FROM nginx:stable +COPY --chown=nginx:nginx ./releases /usr/share/nginx/html +COPY --chown=nginx:nginx ./dataset /usr/share/nginx/html/dataset +COPY nginx.conf /etc/nginx/nginx.conf +RUN rm /usr/share/nginx/html/index.html /usr/share/nginx/html/50x.html diff --git a/scripts/extension/PRODUCTION_RELEASES b/scripts/extension/PRODUCTION_RELEASES new file mode 100644 index 00000000000..9f7c8f9e447 --- /dev/null +++ b/scripts/extension/PRODUCTION_RELEASES @@ -0,0 +1,5 @@ +v0.1.0 +v0.3.0 +v0.3.1 +v0.5.0 +v0.5.1.3 diff --git a/scripts/extension/compose.yaml b/scripts/extension/compose.yaml new file mode 100644 index 00000000000..e4f049fb02d --- /dev/null +++ b/scripts/extension/compose.yaml @@ -0,0 +1,10 @@ +version: '3' +services: + file-server: + image: kuzudb/extension-repo + pull_policy: always + ports: + - "8081:80" + restart: always + labels: + - com.centurylinklabs.watchtower.enable=true diff --git a/scripts/extension/nginx.conf b/scripts/extension/nginx.conf new file mode 100644 index 00000000000..8ddf616324a --- /dev/null +++ b/scripts/extension/nginx.conf @@ -0,0 +1,32 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + +http { + expires 15552000s; + add_header Cache-Control "public, no-transform"; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + autoindex on; + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + gzip on; + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/scripts/extension/purge-beta.py b/scripts/extension/purge-beta.py new file mode 100644 index 00000000000..a536c381437 --- /dev/null +++ b/scripts/extension/purge-beta.py @@ -0,0 +1,33 @@ +import pathlib +import os +from packaging.version import Version + + +RELEASES_PATH = pathlib.Path(__file__).parent.parent.joinpath('releases').resolve() + +production_releases = open(RELEASES_PATH.joinpath('PRODUCTION_RELEASES')).read().splitlines() + +releases_to_purge = [r for r in os.listdir(RELEASES_PATH) if r.startswith('v')] + +releases_to_purge = [r for r in releases_to_purge if r not in production_releases] +releases_to_purge = [r[1:] for r in releases_to_purge] +releases_to_purge.sort(key=Version) + +releases_to_purge.pop() + +releases_to_purge = ['v' + r for r in releases_to_purge] + +if len(releases_to_purge) == 0: + print('No releases to purge.') + exit(1) + +print('Releases to purge:') +for r in releases_to_purge: + print(' ' + r) + +for r in releases_to_purge: + path_to_purge = RELEASES_PATH.joinpath(r) + print('Deleting ' + str(path_to_purge)) + os.system('rm -rf ' + str(path_to_purge)) + +print('Done.')