Updating to latest changes

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "maven" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    open-pull-requests-limit: 20
+    schedule:
+      interval: "weekly"
+    ignore:
+      - dependency-name: "com.google.cloud:google-cloud-bigquery"
+      - dependency-name: "com.google.cloud:google-cloud-resourcemanager"
+      - dependency-name: "com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1"
+      - dependency-name: "io.grpc:grpc-api"
+      - dependency-name: "software.amazon.msk:aws-msk-iam-auth"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/auto-approve.yml

@@ -0,0 +1,39 @@
+name: Dependabot reviewer
+
+on: pull_request_target
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  review-dependabot-pr:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: dependabot-metadata
+        uses: dependabot/fetch-metadata@v2
+      - name: Enable auto-merge for Dependabot PRs
+        run: gh pr merge --auto --merge "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Approve patch and minor updates
+        if: ${{ (steps.dependabot-metadata.outputs.update-type == 'version-update:semver-patch' || steps.dependabot-metadata.outputs.update-type == 'version-update:semver-minor') &&
+            !contains(steps.dependabot-metadata.outputs.new-version, 'preview') &&
+            !contains(steps.dependabot-metadata.outputs.new-version, 'alpha') &&
+            !contains(steps.dependabot-metadata.outputs.new-version, 'beta') &&
+            !contains(steps.dependabot-metadata.outputs.new-version, 'rc') }}
+        run: gh pr review $PR_URL --approve -b "I'm **approving** this pull request because **it includes a patch or minor update**"
+        env:
+            PR_URL: ${{github.event.pull_request.html_url}}
+            GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Comment on major updates of non-development dependencies
+        if: ${{steps.dependabot-metadata.outputs.update-type == 'version-update:semver-major'}}
+        run: |
+          gh pr comment $PR_URL --body "I'm **not approving** this PR because **it includes a major update of a dependency**"
+          gh pr edit $PR_URL --add-label "requires-manual-qa"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/codeql-analysis.yml

@@ -2,50 +2,40 @@ name: "Code scanning - action"
 
 on:
   push:
+    branches: [master]
   pull_request:
+    branches: [master]
   schedule:
     - cron: '0 12 * * 6'
 
 jobs:
-  CodeQL-Build:
-
+  # This is from: https://github.com/fkirc/skip-duplicate-actions#example-1-skip-entire-jobs
+  pre_job:
     runs-on: ubuntu-latest
-
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      # Override language selection by uncommenting this and choosing your languages
-      # with:
-      #   languages: go, javascript, csharp, python, cpp, java
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      - id: skip_check
+        uses: fkirc/[email protected]
+        with:
+          cancel_others: 'true'
+          concurrent_skipping: 'same_content_newer'
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+  CodeQL-Build:
+    needs: pre_job
+    if: needs.pre_job.outputs.should_skip != 'true'
+    runs-on: aws-athena-query-federation_ubuntu-latest_16-core
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+      # Specify custom java build since the autobuild doesn't take advantage of multiple cores.
+      # For more info on specifying custom build steps, see:
+      # https://docs.github.com/en/[email protected]/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language
+      - name: Build java
+        run: mvn clean package -T 1C -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/cut_release.yml

@@ -0,0 +1,139 @@
+name: Cut release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  cut_release:
+    name: Cut a release
+    runs-on: aws-athena-query-federation_ubuntu-latest_16-core
+    steps:
+      - name: Setup dependencies
+        run: |
+          sudo apt-get install -y libxml2-utils python3
+          pip3 install lxml beautifulsoup4
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '11'
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # See: https://stackoverflow.com/a/57969570
+      - name: Setup previous release version environment variable
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: echo "PREVIOUS_RELEASE_VERSION=$(gh release list --exclude-drafts --exclude-pre-releases -L 1 | sed 's/.*\s\+Latest\s\+v\(.*\)\s\+.*/\1/g')" >> $GITHUB_ENV
+
+      - name: Setup previous release version tag environment variable
+        run: echo "PREVIOUS_RELEASE_TAG=v$PREVIOUS_RELEASE_VERSION" >> $GITHUB_ENV
+
+      - name: Setup new release version environment variable
+        run: |
+          NEW_RELEASE_VERSION=$((echo $PREVIOUS_RELEASE_VERSION | grep "$(date +'%Y.%-U')" || date +'%Y.%-U.0') | python3 -c 'version = input().split("."); print(f"{version[0]}.{version[1]}.{int(version[2]) + 1}")')
+          echo "NEW_RELEASE_VERSION=$NEW_RELEASE_VERSION" >> $GITHUB_ENV
+
+      - name: Create branch locally on the runner
+        run: git checkout -b release_${NEW_RELEASE_VERSION} origin/master
+
+      - name: Bump the versions in the branch
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python3 tools/bump_versions/bump_sdk_version.py
+          python3 tools/bump_versions/bump_connectors_version.py
+
+      # See: https://joht.github.io/johtizen/build/2022/01/20/github-actions-push-into-repository.html
+      - name: Setup workflow git committer user
+        run: |
+          git config --global user.name "athena_federation_cut_release_workflow"
+          git config --global user.email "[email protected]"
+
+      - name: Create the release bump commit
+        run: |
+          echo "Cut release $NEW_RELEASE_VERSION" > /tmp/RELEASE_MESSAGE
+          echo >> /tmp/RELEASE_MESSAGE
+          git fetch --tag origin
+          git log --format='  - %s' $PREVIOUS_RELEASE_TAG..HEAD >> /tmp/RELEASE_MESSAGE
+          git commit -a -F /tmp/RELEASE_MESSAGE
+
+      - name: Build Java 11 jars
+        run: |
+          mvn clean package -T 1C -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress
+
+          # Copy over the sdk jars
+          mkdir -p /tmp/java11_sdk_jars/
+          cp ./athena-federation-sdk/target/aws-athena-federation-sdk-${NEW_RELEASE_VERSION}.jar /tmp/java11_sdk_jars/
+          cp ./athena-federation-sdk/target/aws-athena-federation-sdk-${NEW_RELEASE_VERSION}-withdep.jar /tmp/java11_sdk_jars/
+
+          # Copy over the connector jars
+          mkdir -p /tmp/java11_connector_jars/
+          find athena-*/target -name "*.jar" -type f | grep -v "test" | grep -v "arrow" | grep -v "/original" | grep -v "example" | grep -v "\-sdk-" | grep -v "\-dsv2/" | grep -v "athena-jdbc" | xargs -I{} cp {} /tmp/java11_connector_jars/
+
+          # Copy over the connector zips
+          mkdir -p /tmp/java11_connector_zips/
+          find athena-*/target/ -name "*.zip" -type f | grep -v "test" | grep -v "arrow" | grep -v "/original" | grep -v "example" | grep -v "\-sdk-" | grep -v "\-dsv2/" | xargs -I{} cp {} /tmp/java11_connector_zips/
+
+      - name: Push the release branch
+        run: |
+          git push origin release_${NEW_RELEASE_VERSION}:release_${NEW_RELEASE_VERSION}
+
+      - name: Sync up with origin again now that the new branch has been pushed
+        run: |
+          git fetch origin
+          git status
+          git pull --rebase origin release_${NEW_RELEASE_VERSION}
+
+      - name: Create the release on github
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          cat <<EOF > /tmp/RELEASE_NOTES
+          This version includes improvements and bugfixes in some connectors. See [here](https://github.com/awslabs/aws-athena-query-federation/compare/v$PREVIOUS_RELEASE_VERSION..v$NEW_RELEASE_VERSION) for changes since the last release.
+
+          Binary distribution of the SDK can be found here. For pre-built versions of the connector and UDF suite please:
+
+          1. Navigate to Serverless Application Repository and search for "athena-federation".  Also check the checkbox labeled "Show apps that create custom IAM roles or resource policies".
+
+          2. Look for entries published by the "Amazon Athena Federation" author.
+
+          You can assert the validity of the binary distribution by comparing against the below cksums.
+
+          Athena Federation SDK jars
+          |CheckSum|File|
+          |----------|----|
+          $(cd /tmp/java11_sdk_jars && ls | xargs -I{} cksum {} | sed 's/\([0-9]\) \([a-zA-Z]\)/\1|\2/g' | sed 's/^/|/g' | sed 's/$/|/g')
+
+          Athena Federation Connector Lambda jars
+          |CheckSum|File|
+          |----------|----|
+          $(cd /tmp/java11_connector_jars && ls | xargs -I{} cksum {} | sed 's/\([0-9]\) \([a-zA-Z]\)/\1|\2/g' | sed 's/^/|/g' | sed 's/$/|/g')
+
+          Athena Federation Connector Lambda zips
+          |CheckSum|File|
+          |----------|----|
+          $(cd /tmp/java11_connector_zips && ls | xargs -I{} cksum {} | sed 's/\([0-9]\) \([a-zA-Z]\)/\1|\2/g' | sed 's/^/|/g' | sed 's/$/|/g')
+
+          ## What's Changed
+          $(git log --format='- %s' v$PREVIOUS_RELEASE_VERSION..HEAD~1)
+
+          **Full Changelog**: https://github.com/awslabs/aws-athena-query-federation/compare/v$PREVIOUS_RELEASE_VERSION..v$NEW_RELEASE_VERSION
+          EOF
+          # ------------------------------------------------------------------------------------------
+
+          # Create the draft github release:
+          gh release create "v$NEW_RELEASE_VERSION" -d --latest \
+            --target release_${NEW_RELEASE_VERSION} \
+            -t "Release v$NEW_RELEASE_VERSION of Athena Query Federation" \
+            --notes-file /tmp/RELEASE_NOTES
+          # Upload the jar attachments
+          gh release upload "v$NEW_RELEASE_VERSION" \
+            $(find /tmp/java11_sdk_jars -type f) \
+            $(find /tmp/java11_connector_jars -type f) \
+            $(find /tmp/java11_connector_zips -type f) \
+            --clobber

.github/workflows/javadoc_sync.yaml

@@ -5,27 +5,30 @@ on:
       - master
 jobs:
   build-and-deploy:
-    runs-on: self-hosted
+    runs-on: aws-athena-query-federation_ubuntu-latest_16-core
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
 
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
         with:
-          java-version: 1.8
+          distribution: 'corretto'
+          java-version: '11'
 
       - name: Build Javadoc with Maven
-        run: mvn -B javadoc:aggregate --file pom.xml
+        run: mvn -B javadoc:aggregate --file pom.xml -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress
 
       - name: Switch to gh-pages branch
-        run: git checkout gh-pages
+        run: |
+          git fetch
+          git checkout gh-pages
 
       - name: Remove old files
         run: rm -rf com
 
       - name: Copy new files
-        run: cp -r target/site/apidocs/* .
+        run: cp -r target/reports/apidocs/* .
 
       - name: Commit files
         run: |
@@ -40,4 +43,4 @@ jobs:
         uses: ad-m/github-push-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          branch: gh-pages
+          branch: gh-pages

.github/workflows/maven_push.yml

@@ -3,15 +3,46 @@ name: Java CI Push
 on: [push, pull_request]
 
 jobs:
-  build:
-
-    runs-on: self-hosted
+  # This is from: https://github.com/fkirc/skip-duplicate-actions#example-1-skip-entire-jobs
+  pre_job:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/[email protected]
+        with:
+          cancel_others: 'true'
+          concurrent_skipping: 'same_content_newer'
+          do_not_skip: '["workflow_dispatch", "schedule", "merge_group", "pull_request"]'
 
+  build:
+    needs: pre_job
+    if: needs.pre_job.outputs.should_skip != 'true'
+    runs-on: aws-athena-query-federation_ubuntu-latest_16-core
     steps:
-    - uses: actions/checkout@v1
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
-      with:
-        java-version: 1.8
-    - name: Build with Maven
-      run: mvn -B package --file pom.xml
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '11'
+      # Target java 8 to ensure that the source is compatible with java 8
+      - name: Build with Maven
+        env:
+          AWS_DEFAULT_REGION: us-east-1
+          AWS_REGION: us-east-1
+        run: mvn -B clean package -T 1C --file pom.xml -Dmaven.compiler.release=8 -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress
+      # Identify if any files were modified as a result of running maven build.
+      - name: Identify any Maven Build changes
+        run: >
+          ! (git status | grep "modified: " )
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}