From 4c22b7fe66080891af60a5abd7c56114b4984b01 Mon Sep 17 00:00:00 2001
From: Peter Zhu <zhujiaxi@amazon.com>
Date: Tue, 31 Oct 2023 16:10:11 -0400
Subject: [PATCH] Add more

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
---
 .github/workflows/add-untriaged.yml           |  19 +++
 .github/workflows/auto-release.yml            |  29 ++++
 .github/workflows/backport.yml                |  40 ++++++
 .github/workflows/bwc-test-workflow.yml       |  33 +++++
 .../workflows/create-documentation-issue.yml  |  42 ++++++
 .github/workflows/delete_backport_branch.yml  |  15 ++
 .../docker-security-test-workflow.yml         |  98 +++++++++++++
 .../draft-release-notes-workflow.yml          |  20 +++
 .github/workflows/links.yml                   |  21 +++
 .github/workflows/maven-publish.yml           |  35 +++++
 .github/workflows/security-test-workflow.yml  |  47 +++++++
 .github/workflows/test-and-build-workflow.yml | 132 ++++++++++++++++++
 12 files changed, 531 insertions(+)
 create mode 100644 .github/workflows/add-untriaged.yml
 create mode 100644 .github/workflows/auto-release.yml
 create mode 100644 .github/workflows/backport.yml
 create mode 100644 .github/workflows/bwc-test-workflow.yml
 create mode 100644 .github/workflows/create-documentation-issue.yml
 create mode 100644 .github/workflows/delete_backport_branch.yml
 create mode 100644 .github/workflows/docker-security-test-workflow.yml
 create mode 100644 .github/workflows/draft-release-notes-workflow.yml
 create mode 100644 .github/workflows/links.yml
 create mode 100644 .github/workflows/maven-publish.yml
 create mode 100644 .github/workflows/security-test-workflow.yml
 create mode 100644 .github/workflows/test-and-build-workflow.yml

diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml
new file mode 100644
index 000000000..9dcc7020d
--- /dev/null
+++ b/.github/workflows/add-untriaged.yml
@@ -0,0 +1,19 @@
+name: Apply 'untriaged' label during issue lifecycle
+
+on:
+  issues:
+    types: [opened, reopened, transferred]
+
+jobs:
+  apply-label:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['untriaged']
+            })
diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml
new file mode 100644
index 000000000..dea080139
--- /dev/null
+++ b/.github/workflows/auto-release.yml
@@ -0,0 +1,29 @@
+name: Releases
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v1.5.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+      - uses: actions/checkout@v2
+      - uses: ncipollo/release-action@v1
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          bodyFile: release-notes/opensearch-index-management.release-notes-${{steps.tag.outputs.tag}}.md
\ No newline at end of file
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
new file mode 100644
index 000000000..06681a263
--- /dev/null
+++ b/.github/workflows/backport.yml
@@ -0,0 +1,40 @@
+
+name: Backport
+on:
+  pull_request_target:
+    types:
+      - closed
+      - labeled
+
+jobs:
+  backport:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.pull_request.merged
+      && (
+        github.event.action == 'closed'
+        || (
+          github.event.action == 'labeled'
+          && contains(github.event.label.name, 'backport')
+        )
+      )
+    permissions:
+      contents: write
+      pull-requests: write
+    name: Backport
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v1.5.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Backport
+        uses: VachaShah/backport@v2.2.0
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>
+          labels_template: "<%= JSON.stringify([...labels, 'autocut']) %>"
+          failure_labels: "failed backport"
diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml
new file mode 100644
index 000000000..51f6c04a8
--- /dev/null
+++ b/.github/workflows/bwc-test-workflow.yml
@@ -0,0 +1,33 @@
+name: Backward compatibility test workflow
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - "*"
+
+jobs:
+  test:
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 17
+      # index-management
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - name: Run IM Backwards Compatibility Tests
+        run: |
+          echo "Running backwards compatibility tests..."
+          ./gradlew bwcTestSuite
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: build/testclusters/indexmanagementBwcCluster*/logs/*
diff --git a/.github/workflows/create-documentation-issue.yml b/.github/workflows/create-documentation-issue.yml
new file mode 100644
index 000000000..b20b78117
--- /dev/null
+++ b/.github/workflows/create-documentation-issue.yml
@@ -0,0 +1,42 @@
+name: Create Documentation Issue
+on:
+  pull_request:
+    types:
+      - closed
+      - labeled
+env:
+  PR_NUMBER: ${{ github.event.number }}
+
+jobs:
+  create-issue:
+    if: ${{ github.event.label.name == 'needs-documentation' }}
+    runs-on: ubuntu-latest
+    name: Create Documentation Issue
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v1.5.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Edit the issue template
+        run: |
+          echo "https://github.com/opensearch-project/index-management/pull/${{ env.PR_NUMBER }}." >> ./.github/ISSUE_TEMPLATE/documentation.md
+
+      - name: Create Issue From File
+        id: create-issue
+        uses: peter-evans/create-issue-from-file@v4
+        with:
+          title: Add documentation related to new feature
+          content-filepath: ./.github/ISSUE_TEMPLATE/documentation.md
+          labels: documentation
+          repository: opensearch-project/documentation-website
+          token: ${{ steps.github_app_token.outputs.token }}
+
+      - name: Print Issue
+        run: echo Created related documentation issue ${{ steps.create-issue.outputs.issue-number }}
diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml
new file mode 100644
index 000000000..f24f022b0
--- /dev/null
+++ b/.github/workflows/delete_backport_branch.yml
@@ -0,0 +1,15 @@
+name: Delete merged branch of the backport PRs
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  delete-branch:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.pull_request.head.ref,'backport/')
+    steps:
+      - name: Delete merged branch
+        uses: SvanBoxel/delete-merged-branch@main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/docker-security-test-workflow.yml b/.github/workflows/docker-security-test-workflow.yml
new file mode 100644
index 000000000..a3177cd1b
--- /dev/null
+++ b/.github/workflows/docker-security-test-workflow.yml
@@ -0,0 +1,98 @@
+name: Docker Security Test Workflow
+on:
+  pull_request:
+    branches:
+      - "**"
+  push:
+    branches:
+      - "**"
+
+jobs:
+  docker-test:
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set Up JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 17
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - name: Build Index Management
+        run: ./gradlew assemble
+      - name: Pull and Run Docker
+        run: |
+          plugin=`basename $(ls build/distributions/*.zip)`
+          list_of_files=`ls`
+          list_of_all_files=`ls build/distributions/`
+          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
+          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
+          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
+          candidate_version=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
+          if qualifier
+          then
+            docker_version=$version-$qualifier
+          else
+            docker_version=$version
+          fi
+
+          [[ -z $candidate_version ]] && candidate_version=$qualifier && qualifier=""
+
+          echo plugin version plugin_version qualifier candidate_version docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($candidate_version) ($docker_version)"
+          echo $ls $list_of_all_files
+
+          if docker pull opensearchstaging/opensearch:$docker_version
+          then
+            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-index-management ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-index-management; fi" >> Dockerfile
+            echo "ADD build/distributions/$plugin /tmp/" >> Dockerfile
+            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
+            echo "RUN echo 'path.repo: ["/usr/share/opensearch/data/repo"]' >> /usr/share/opensearch/config/opensearch.yml" >> Dockerfile
+
+            docker build -t opensearch-index-management:test .
+            echo "imagePresent=true" >> $GITHUB_ENV
+          else
+            echo "imagePresent=false" >> $GITHUB_ENV
+          fi
+      - name: Run Docker Image
+        if: env.imagePresent == 'true'
+        run: |
+          cd ..
+          container_id=`docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-index-management:test`
+          sleep 120
+          echo `docker logs $container_id`
+      - name: Run Index Management Test for security enabled test cases
+        if: env.imagePresent == 'true'
+        run: |
+          container_id=`docker ps -q`
+          plugins=`docker exec $container_id /usr/share/opensearch/bin/opensearch-plugin list`
+          echo "plugins: $plugins"
+          security=`echo $plugins | grep opensearch-security | wc -l`
+          echo $security
+          if [ $security -gt 0 ]
+          then
+            echo "Security plugin is available"
+            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin --tests '*SecurityBehaviorIT'
+          else
+            echo "Security plugin is NOT available skipping this run as tests without security have already been run"
+          fi
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: build/testclusters/integTest-*/logs/*
+      - name: Collect docker logs on failure
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: './logs'
+      - name: Tar logs
+        run: tar cvzf ./logs.tgz ./logs
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs.tgz
+          path: ./logs.tgz
diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml
new file mode 100644
index 000000000..6b3d89c33
--- /dev/null
+++ b/.github/workflows/draft-release-notes-workflow.yml
@@ -0,0 +1,20 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  update_release_draft:
+    name: Update draft release notes
+    runs-on: ubuntu-latest
+    steps:
+      - name: Update draft release notes
+        uses: release-drafter/release-drafter@v5
+        with:
+          config-name: draft-release-notes-config.yml
+          name: Version (set here)
+          tag: (None)
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
new file mode 100644
index 000000000..af6c25f3a
--- /dev/null
+++ b/.github/workflows/links.yml
@@ -0,0 +1,21 @@
+name: Link Checker
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: lychee Link Checker
+        id: lychee
+        uses: lycheeverse/lychee-action@master
+        with:
+          args: --accept=200,403,429  **/*.html **/*.md **/*.txt **/*.json
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Fail if there were link errors
+        run: exit ${{ steps.lychee.outputs.exit_code }}
\ No newline at end of file
diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml
new file mode 100644
index 000000000..8b7b1164b
--- /dev/null
+++ b/.github/workflows/maven-publish.yml
@@ -0,0 +1,35 @@
+name: Publish snapshots to maven
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
+
+jobs:
+  build-and-publish-snapshots:
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 11
+      - uses: actions/checkout@v3
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
+          aws-region: us-east-1
+      - name: publish snapshots to maven
+        run: |
+          export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
+          export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
+          echo "::add-mask::$SONATYPE_USERNAME"
+          echo "::add-mask::$SONATYPE_PASSWORD"
+          ./gradlew publishPluginZipPublicationToSnapshotsRepository
\ No newline at end of file
diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
new file mode 100644
index 000000000..a4c5aa16f
--- /dev/null
+++ b/.github/workflows/security-test-workflow.yml
@@ -0,0 +1,47 @@
+name: Security test workflow
+# This workflow is triggered on pull requests to main
+on:
+  pull_request:
+    branches:
+      - "**"
+  push:
+    branches:
+      - "**"
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  security-test:
+    needs: Get-CI-Image-Tag
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 17
+      # index-management
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - name: Run integration tests
+        run: | 
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: build/testclusters/integTest-*/logs/*
diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml
new file mode 100644
index 000000000..eb486d243
--- /dev/null
+++ b/.github/workflows/test-and-build-workflow.yml
@@ -0,0 +1,132 @@
+name: Test and Build Workflow
+on:
+  pull_request:
+    branches:
+      - "**"
+  push:
+    branches:
+      - "**"
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  test-and-build-linux:
+    needs: Get-CI-Image-Tag
+    env:
+      TEST_FILTER: ${{ matrix.test_filter }}
+    strategy:
+      # This setting says that all jobs should finish, even if one fails
+      fail-fast: false
+      # This starts three jobs, setting these environment variables uniquely for the different jobs
+      matrix:
+        java: [11]
+        #java: [11, 17]
+        feature: [ism, non-ism]
+        include:
+          - feature: ism
+            test_filter: -PincludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+          - feature: non-ism
+            test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v2
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: ${{ matrix.java }}
+      # build index management
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      # This is a hack, but this step creates a link to the X: mounted drive, which makes the path
+      # short enough to work on Windows
+      - name: Build with Gradle
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew build ${{ env.TEST_FILTER }}"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: logs
+          path: build/testclusters/integTest-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p index-management-artifacts
+          cp ./build/distributions/*.zip index-management-artifacts
+      - name: Uploads coverage
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: index-management-plugin-ubuntu-latest
+          path: index-management-artifacts
+
+#  test-and-build-windows-macos:
+#    env:
+#      BUILD_ARGS: ${{ matrix.os_build_args }}
+#      WORKING_DIR: ${{ matrix.working_directory }}.
+#      TEST_FILTER: ${{ matrix.test_filter }}
+#    strategy:
+#      # This setting says that all jobs should finish, even if one fails
+#      fail-fast: false
+#      # This starts three jobs, setting these environment variables uniquely for the different jobs
+#      matrix:
+#        java: [11, 17]
+#        os: [windows-latest, macos-latest]
+#        feature: [ism, non-ism]
+#        include:
+#          - os: windows-latest
+#            os_build_args: -x integTest -x jacocoTestReport
+#            working_directory: X:\
+#            os_java_options: -Xmx4096M
+#          - os: macos-latest
+#            os_build_args: -x integTest -x jacocoTestReport
+#          - feature: ism
+#            test_filter: -PincludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+#          - feature: non-ism
+#            test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+#    runs-on: ${{ matrix.os }}
+#    steps:
+#      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+#      - name: Set Up JDK ${{ matrix.java }}
+#        uses: actions/setup-java@v2
+#        with:
+#          distribution: temurin # Temurin is a distribution of adoptium
+#          java-version: ${{ matrix.java }}
+#      # build index management
+#      - name: Checkout Branch
+#        uses: actions/checkout@v2
+#      # This is a hack, but this step creates a link to the X: mounted drive, which makes the path
+#      # short enough to work on Windows
+#      - name: Shorten Path
+#        if: ${{ matrix.os == 'windows-latest' }}
+#        run: subst 'X:' .
+#      - name: Build with Gradle
+#        working-directory: ${{ env.WORKING_DIR }}
+#        run: ./gradlew build ${{ env.BUILD_ARGS }} ${{ env.TEST_FILTER }}
+#        env:
+#          _JAVA_OPTIONS: ${{ matrix.os_java_options }}
+#      - name: Create Artifact Path
+#        run: |
+#          mkdir -p index-management-artifacts
+#          cp ./build/distributions/*.zip index-management-artifacts
+#      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+#      - name: Upload Artifacts
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: index-management-plugin-${{ matrix.os }}
+#          path: index-management-artifacts