512 sarus

.gitattributes

@@ -0,0 +1 @@
+karabo/_version.py export-subst

.github/workflows/build-docker-image.yml

@@ -0,0 +1,121 @@
+name: Build Docker Image
+
+on:
+  workflow_call:
+    inputs:
+      verstag:
+        type: string
+        required: true
+      latest:
+        type: boolean
+        required: false
+        default: false
+
+  workflow_dispatch:
+    inputs:
+      gitrev:
+        type: string
+        required: false
+        default: ""
+        description: "gitrev: commit-hash (full) | branch | tag if not current commit."
+      verstag:
+        type: string
+        required: true
+        description: "version: PEP440 version-tag of Karabo. DON't trigger build if you don't know what PEP440 is!"
+      latest:
+        type: boolean
+        required: false
+        default: false
+        description: "tag image as 'latest'?"
+      test:
+        type: boolean
+        required: false
+        default: false
+        description: "create env from environment.yaml instead of conda-wheel?"
+
+env:
+  REGISTRY: ghcr.io
+  IMG_NAME: karabo-pipeline
+
+jobs:
+  build-test-and-push-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup metadata img-name & img-tag
+        shell: bash -l {0}
+        run: |
+          if [[ ${{ github.event_name }} == "workflow_call" ]]; then
+            echo "gitrev=$GITHUB_SHA" >> "$GITHUB_ENV"
+            echo "build=user" >> "$GITHUB_ENV"
+          elif [[ ${{ github.event_name }} == "workflow_dispatch" ]]; then
+            if [[ "${{ inputs.gitrev }}" != "" ]]; then
+              echo "gitrev=${{ inputs.gitrev }}" >> "$GITHUB_ENV"
+            else
+              echo "gitrev=$GITHUB_SHA" >> "$GITHUB_ENV"
+            fi
+            if [[ "${{ inputs.test }}" == 'true' ]]; then
+              echo "build=test" >> "$GITHUB_ENV"
+            else
+              echo "build=user" >> "$GITHUB_ENV"
+            fi
+          else
+            echo "Invalid github-event!"
+            exit 1
+          fi
+          echo "latest=${{ inputs.latest }}" >> "$GITHUB_ENV"
+          echo "version=${{ inputs.verstag }}" >> "$GITHUB_ENV"
+          REPO_OWNER=${{ github.repository_owner }}
+          echo "IMG_ADDR=${{ env.REGISTRY }}/${REPO_OWNER@L}/${{ env.IMG_NAME }}" >> "$GITHUB_ENV"
+          DEV_STR="dev"
+          if [[ "${{ inputs.verstag }}" == *"$DEV_STR"* ]] && [[ "${{ inputs.latest }}" == 'true' ]]; then
+            echo "Invalid configuration of workflow-inputs!"
+            exit 1
+          fi
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}
+          tags: |
+            type=raw, enable=${{ env.latest }}, value=latest
+            type=raw, value=${{ env.version }}
+
+      - name: Docker build
+        shell: bash -l {0}
+        run: |
+          docker build \
+            --build-arg GIT_REV=${{ env.gitrev }} \
+            --build-arg BUILD=${{ env.build }} \
+            --build-arg KARABO_VERSION=${{ env.version }} \
+            -f Dockerfile \
+            -t ${{ env.IMG_ADDR }}:${{ env.version }} \
+            .
+          if [[ ${{ env.latest }} == 'true' ]]; then
+            docker tag ${{ env.IMG_ADDR }}:${{ env.version }} ${{ env.IMG_ADDR }}:latest
+          fi
+
+      - name: Test image
+        run: |  # karabo-sitepackage-location used for mpirun instead of --pyargs because --only-mpi is a custom-flag of karabo which lives in the site-packages
+          docker run --rm ${{ env.IMG_ADDR }}:${{ env.version }} bash -c \
+            'export IS_GITHUB_RUNNER=true RUN_GPU_TESTS=false RUN_NOTEBOOK_TESTS=false; pytest --pyargs karabo.test; SITE_PKGS=$(pip show karabo-pipeline | grep Location | sed "s/.*\(\/opt\/conda.*\).*/\1/"); mpirun -n 2 pytest --only-mpi $SITE_PKGS/karabo/test'
+      - name: Docker push
+        shell: bash -l {0}
+        run: |
+          docker push --all-tags ${{ env.IMG_ADDR }}

.github/workflows/build-docs.yml

@@ -20,7 +20,7 @@ jobs:
         run: |
           conda env create -n karabo_dev_env -f environment.yaml
           conda activate karabo_dev_env
-          pip install -r requirements.txt
+          pip install ".[dev]"
       - name: Build Docs
         shell: bash -l {0}
         run: |

.github/workflows/conda-build.yml

@@ -4,37 +4,127 @@ on:
   release:
     types: [published]
   workflow_dispatch:
+    inputs:
+      buildNumber:
+        type: string
+        required: true
+        description: "build-nr: anaconda.org build-nr (DON'T trigger build if you don't know how to set it)"
+      version:
+        type: string
+        required: true
+        description: "version: PEP440 package-version (DON'T trigger build if you don't know what PEP440 is)"
+      buildDocker:
+        type: boolean
+        required: false
+        default: false
+        description: "build docker-img if conda-build & tests succeeded?"
+      latestDocker:
+        type: boolean
+        required: false
+        default: false
+        description: "tag docker-img as latest (if `buildDocker` is enabled)"
+  workflow_call:
+    inputs:
+      buildNumber:
+        type: string
+        required: true
+      version:
+        type: string
+        required: true
+      buildDocker:
+        type: boolean
+        required: false
+        default: false
+      latestDocker:
+        type: boolean
+        required: false
+        default: false
 
 
 jobs:
   conda-build:
     runs-on: ubuntu-latest
     container: ghcr.io/i4ds/mambabuild-docker:latest
-
+    outputs:
+      karabo-version: ${{ steps.bcs.outputs.karabo_version }}
+      build-docker: ${{ steps.bcs.outputs.build_docker == 'true' }}
+      latest-docker: ${{ steps.bcs.outputs.latest_docker == 'true' }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get Previous tag
         uses: actions-ecosystem/action-get-latest-tag@v1
         id: get-latest-tag
-      - name: Install conda build
+      - name: Build Conda
+        id: bcs
         shell: bash -l {0}
-        id: channels
         run: |
-          KARABO_TAG=${{ steps.get-latest-tag.outputs.tag }}
+          DEV_STR="dev"
+          if [[ ${{ github.event_name }} == 'release' ]]
+          then
+            echo "Conda build from release"
+            build="0"
+            KARABO_TAG=${{ steps.get-latest-tag.outputs.tag }}
+            if [[ ${KARABO_TAG:0:1} == "v" ]]; then
+              KARABO_VERSION="${KARABO_TAG:1}"
+            else
+              echo "invalid karabo-tag: has no leading v"
+              exit 1
+            fi
+            BUILD_DOCKER=true
+            LATEST_DOCKER=true
+          elif [[ ${{ github.event_name }} == 'workflow_dispatch' ]] || [[ ${{ github.event_name }} == 'workflow_call' ]]; then
+            echo "Conda build from dispatch | call"
+            build=${{ inputs.buildNumber }}
+            BUILD_DOCKER=${{ inputs.buildDocker }}
+            LATEST_DOCKER=${{ inputs.latestDocker }}
+            KARABO_VERSION=${{ inputs.version }}
+            conda install -c conda-forge python versioneer
+            VERSIONEER_VERSION=$(/opt/conda/bin/python -c 'import versioneer; print(versioneer.get_version())')
+            if [[ "$VERSIONEER_VERSION" == *"+"* ]]; then
+              if [[ "$KARABO_VERSION" != *"$DEV_STR"* ]]; then
+                echo "Dirty commit-version needs to be a PEP440 conform dev-version"
+                exit 1
+              fi
+            else
+              if [[ "$KARABO_VERSION" != "$VERSIONEER_VERSION" ]]; then
+                echo "Provided version $KARABO_VERSION doesn't match the actual version $VERSIONEER_VERSION"
+                exit 1
+              fi
+            fi
+          else
+            echo "Not specified github-event occured"
+            exit 1
+          fi
+          if [[ "$KARABO_VERSION" == *"$DEV_STR"* ]]; then
+            if [[ $LATEST_DOCKER == 'true' ]]; then
+              echo "Dev build docker image can not be tagged as latest!"
+              exit 1
+            fi
+            build="$(($build + 1000))"
+          fi
+
+          export KARABO_VERSION=$KARABO_VERSION build=$build
+          echo "karabo_version=$KARABO_VERSION" >> $GITHUB_OUTPUT
+          echo "build_docker=$BUILD_DOCKER" >> $GITHUB_OUTPUT
+          echo "latest_docker=$LATEST_DOCKER" >> $GITHUB_OUTPUT
+
           conda config --append channels i4ds
           conda config --append channels nvidia/label/cuda-11.7.0
           conda config --append channels conda-forge
-      - name: Build Conda
-        shell: bash -l {0}
-        run: |
+
           cd conda
-          KARABO_TAG=${{ steps.get-latest-tag.outputs.tag }}
-          export KARABO_VERSION="${KARABO_TAG:1}"
-          echo "__version__ = \"${KARABO_VERSION}\"" > karabo/version.py
           conda mambabuild .
       - name: Publish to Conda
         shell: bash -l {0}
         run: |
           conda activate base
           anaconda -t ${{ secrets.ANACONDA_SECRET }} upload /opt/conda/conda-bld/linux-64/karabo-pipeline-*.tar.bz2 --force
+
+  test-build:
+    needs: conda-build
+    uses: ./.github/workflows/test-user-package.yml
+    with:
+      version: ${{ needs.conda-build.outputs.karabo-version }}
+      buildDocker: ${{ needs.conda-build.outputs.build-docker == 'true' }}
+      latestDocker: ${{ needs.conda-build.outputs.latest-docker == 'true' }}