refactor: delta, delta-polars #75
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dagster-delta-[polars] CI/CD | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - master | |
| tags: | |
| - "*" | |
| pull_request: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| lint: | |
| name: Linting checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Install pypa/build | |
| run: pip install uv && make .venv && VIRTUAL_ENV=./.venv | |
| - name: CI-check | |
| run: source .venv/bin/activate && make ci-check | |
| tests: | |
| name: Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Install pypa/build | |
| run: pip install uv && make .venv && VIRTUAL_ENV=./.venv | |
| - name: Execute tests | |
| run: source .venv/bin/activate && pytest . | |
| build: | |
| name: Build wheels | |
| runs-on: ubuntu-latest | |
| needs: | |
| - lint | |
| - tests | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Install pypa/build | |
| run: pip install uv && make .venv && VIRTUAL_ENV=./.venv | |
| - name: Build a binary wheel | |
| run: source .venv/bin/activate && python3 -m build -w "libraries/dagster-delta" && python3 -m build -w "libraries/dagster-delta-polars" && python3 -m build -w "libraries/dagster-unity-catalog-polars" | |
| - name: Store the distribution packages | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: python-package-distributions | |
| path: libraries/**/dist/ | |
| publish-dagster-delta: | |
| name: >- | |
| Publish Python 🐍 distribution 📦 to PyPI | |
| if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes | |
| needs: | |
| - build | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/dagster-delta | |
| steps: | |
| - name: Download all the dists | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-package-distributions | |
| path: my_dists/ | |
| - name: list files | |
| run: ls my_dists/dagster-delta/dist/ | |
| - name: Publish distribution 📦 to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: my_dists/dagster-delta/dist/ | |
| password: ${{ secrets.PYPI_API_TOKEN_DD }} | |
| verbose: true | |
| publish-dagster-delta-polars: | |
| name: >- | |
| Publish Python 🐍 distribution 📦 to PyPI | |
| if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes | |
| needs: | |
| - build | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/dagster-delta-polars | |
| steps: | |
| - name: Download all the dists | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-package-distributions | |
| path: my_dists/ | |
| - name: list files | |
| run: ls my_dists/dagster-delta-polars/dist/ | |
| - name: Publish distribution 📦 to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: my_dists/dagster-delta-polars/dist/ | |
| password: ${{ secrets.PYPI_API_TOKEN_DDP }} | |
| verbose: true | |
| publish-dagster-unity-catalog-polars: | |
| name: >- | |
| Publish Python 🐍 distribution 📦 to PyPI | |
| if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes | |
| needs: | |
| - build | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/dagster-unity-catalog-polars | |
| steps: | |
| - name: Download all the dists | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-package-distributions | |
| path: my_dists/ | |
| - name: list files | |
| run: ls my_dists/dagster-unity-catalog-polars/dist/ | |
| - name: Publish distribution 📦 to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: my_dists/dagster-unity-catalog-polars/dist/ | |
| password: ${{ secrets.PYPI_API_TOKEN_DDUC }} | |
| verbose: true | |