From 2e33f72976fb788c79f226fa5cc52a9f3afeae19 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 15:24:34 -0500 Subject: [PATCH 01/12] Update codecov to v5 with new token Signed-off-by: Peter Zhu --- .github/workflows/groovy-tests.yml | 3 ++- .github/workflows/python-tests.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/groovy-tests.yml b/.github/workflows/groovy-tests.yml index 9371638f92..abb88826f2 100644 --- a/.github/workflows/groovy-tests.yml +++ b/.github/workflows/groovy-tests.yml @@ -16,6 +16,7 @@ jobs: run: | ./gradlew test --info - name: Upload Coverage Report - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v5 with: files: ./build/reports/jacoco/test/jacocoTestReport.xml + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 618dafb0d0..29c4bde960 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -77,6 +77,7 @@ jobs: run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml - name: Upload Coverage Report - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v5 with: files: ./coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} From b4be1c5d1d4d132eebd0fe93b2e536e6df187677 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 15:45:42 -0500 Subject: [PATCH 02/12] More updates Signed-off-by: Peter Zhu --- .github/workflows/groovy-tests.yml | 5 ----- .github/workflows/python-tests.yml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/groovy-tests.yml b/.github/workflows/groovy-tests.yml index abb88826f2..e77f602650 100644 --- a/.github/workflows/groovy-tests.yml +++ b/.github/workflows/groovy-tests.yml @@ -15,8 +15,3 @@ jobs: - name: Run Tests run: | ./gradlew test --info - - name: Upload Coverage Report - uses: codecov/codecov-action@v5 - with: - files: ./build/reports/jacoco/test/jacocoTestReport.xml - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 29c4bde960..dd6d5bc2ef 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -77,7 +77,7 @@ jobs: run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml - name: Upload Coverage Report - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v3 with: files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} From 0042d0d2c92c3c1b4bfa958568cec6a42954f9fa Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 15:56:21 -0500 Subject: [PATCH 03/12] More updates Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index dd6d5bc2ef..540b0dfd2c 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -79,5 +79,5 @@ jobs: - name: Upload Coverage Report uses: codecov/codecov-action@v3 with: - files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml From dc4c0a7849599a66fbf7ba14d9a96e119ec32343 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 16:09:29 -0500 Subject: [PATCH 04/12] Separate codecov upload to its own steps Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 540b0dfd2c..01545795b1 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -44,10 +44,11 @@ jobs: python-tests-linux: needs: Get-CI-Image-Tag + name: python-tests-linux-${{ matrix.os }} strategy: matrix: os: - - ubuntu-24.04-arm # arm64-preview + # - ubuntu-24.04-arm # arm64-preview - ubuntu-24.04 # x64 runs-on: ${{ matrix.os }} container: @@ -76,8 +77,26 @@ jobs: - name: Run Tests with Coverage run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml + - uses: actions/upload-artifact@v4 + with: + name: coverage-report-${{ matrix.os }} + path: ./coverage.xml + + python-tests-codecov: + needs: python-tests-linux-${{ matrix.os }} + strategy: + matrix: + os: + # - ubuntu-24.04-arm # arm64-preview + - ubuntu-24.04 # x64 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/download-artifact@v4 + with: + name: my-artifact + path: ./coverage.xml - name: Upload Coverage Report - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml From 91527523e2bb48b0d2543a3e018ce3a2f86d057f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 16:12:00 -0500 Subject: [PATCH 05/12] Separate codecov upload to its own steps Signed-off-by: Peter Zhu --- .github/workflows/add-untriaged.yml | 20 -- .github/workflows/automatic-merges.yml | 69 ------- .github/workflows/dockerfile-lint.yml | 27 --- .github/workflows/get-ci-image-tag.yml | 82 -------- .github/workflows/github-merit-badger.yml | 21 -- .github/workflows/groovy-tests.yml | 17 -- .github/workflows/license-header-checker.yml | 12 -- .github/workflows/link-checker.yml | 20 -- .github/workflows/manifests.yml | 63 ------ .../os-increment-plugin-versions.yml | 156 -------------- .github/workflows/os-release-issues.yml | 123 ----------- .../osd-increment-plugin-versions.yml | 195 ------------------ .github/workflows/osd-release-issues.yml | 116 ----------- .github/workflows/publish-release.yml | 19 -- .github/workflows/publish-wiki.yml | 25 --- .github/workflows/python-tests.yml | 8 +- .github/workflows/releases.yml | 48 ----- .github/workflows/yaml-lint.yml | 23 --- 18 files changed, 4 insertions(+), 1040 deletions(-) delete mode 100644 .github/workflows/add-untriaged.yml delete mode 100644 .github/workflows/automatic-merges.yml delete mode 100644 .github/workflows/dockerfile-lint.yml delete mode 100644 .github/workflows/get-ci-image-tag.yml delete mode 100644 .github/workflows/github-merit-badger.yml delete mode 100644 .github/workflows/groovy-tests.yml delete mode 100644 .github/workflows/license-header-checker.yml delete mode 100644 .github/workflows/link-checker.yml delete mode 100644 .github/workflows/manifests.yml delete mode 100644 .github/workflows/os-increment-plugin-versions.yml delete mode 100644 .github/workflows/os-release-issues.yml delete mode 100644 .github/workflows/osd-increment-plugin-versions.yml delete mode 100644 .github/workflows/osd-release-issues.yml delete mode 100644 .github/workflows/publish-release.yml delete mode 100644 .github/workflows/publish-wiki.yml delete mode 100644 .github/workflows/releases.yml delete mode 100644 .github/workflows/yaml-lint.yml diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml deleted file mode 100644 index 88e31ef9d3..0000000000 --- a/.github/workflows/add-untriaged.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Apply 'untriaged' label during issue lifecycle - -on: - issues: - types: [opened, reopened, transferred] - -jobs: - apply-label: - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['untriaged'] - }) diff --git a/.github/workflows/automatic-merges.yml b/.github/workflows/automatic-merges.yml deleted file mode 100644 index 0772885e3b..0000000000 --- a/.github/workflows/automatic-merges.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- -name: automatic-merges - -on: - workflow_run: - # If new workflows triggered on pull request are added, they should be added to this list - workflows: - - Link Checker - - yaml-lint - - dockerfile-lint - - groovy-tests - - manifests - - python-tests - types: completed - -jobs: - automatic-merge-manifests: - if: github.repository == 'opensearch-project/opensearch-build' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - id: find-triggering-pr - uses: peternied/find-triggering-pr@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: peternied/discerning-merger@v3 - if: steps.find-triggering-pr.outputs.pr-number != null - with: - token: ${{ secrets.GITHUB_TOKEN }} - pull-request-number: ${{ steps.find-triggering-pr.outputs.pr-number }} - allowed-authors: | - dependabot - opensearch-trigger-bot - opensearch-ci-bot - peterzhuamazon - gaiksaya - rishabh6788 - zelinh - prudhvigodithi - Divyaasm - dblock - tianleh - peternied - allowed-files: | - manifests/*/*.yml - - automatic-merge-new-manifests: - if: github.repository == 'opensearch-project/opensearch-build' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - id: find-triggering-pr - uses: peternied/find-triggering-pr@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: peternied/discerning-merger@v3 - if: steps.find-triggering-pr.outputs.pr-number != null - with: - token: ${{ secrets.GITHUB_TOKEN }} - pull-request-number: ${{ steps.find-triggering-pr.outputs.pr-number }} - allowed-authors: | - opensearch-ci-bot - allowed-files: | - manifests/*/*.yml - jenkins/check-for-build.jenkinsfile diff --git a/.github/workflows/dockerfile-lint.yml b/.github/workflows/dockerfile-lint.yml deleted file mode 100644 index a363432719..0000000000 --- a/.github/workflows/dockerfile-lint.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: dockerfile-lint - -on: - pull_request: - paths: - - 'docker/ci/dockerfiles/current/**' - - 'docker/release/dockerfiles/**' - -jobs: - dockerfile-lint-test_ci: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: jbergstroem/hadolint-gh-action@v1 - with: - dockerfile: 'docker/ci/dockerfiles/current/*.dockerfile' - annotate: false - - dockerfile-lint-test_release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: jbergstroem/hadolint-gh-action@v1 - with: - dockerfile: 'docker/release/dockerfiles/*.dockerfile' - annotate: false diff --git a/.github/workflows/get-ci-image-tag.yml b/.github/workflows/get-ci-image-tag.yml deleted file mode 100644 index b1e7e41c15..0000000000 --- a/.github/workflows/get-ci-image-tag.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: Get CI Image Tag -on: - workflow_call: - inputs: - product: - required: false - type: string - default: 'opensearch' - platform: - required: false - type: string - build_ref: - required: false - type: string - default: 'main' - version: - required: false - type: string - type: - required: false - type: string - default: 'build' - outputs: - ci-image-version-linux: - description: The ci image version for linux build - value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-version-linux }} - ci-image-start-options: - description: The ci image start options to set when starting the container - value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-options }} - ci-image-start-command: - description: The ci image start commands to run after initialization - value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-command }} - -jobs: - Get-CI-Image-Tag: - runs-on: ubuntu-latest - outputs: - output-ci-image-version-linux: ${{ steps.step-ci-image-setups.outputs.ci-image-version-linux }} - output-ci-image-start-command: ${{ steps.step-ci-image-setups.outputs.ci-image-start-command }} - output-ci-image-start-options: ${{ steps.step-ci-image-setups.outputs.ci-image-start-options }} - steps: - - name: Install crane - uses: iarekylew00t/crane-installer@v1 - with: - crane-release: v0.15.2 - - name: Checkout opensearch-build repository - uses: actions/checkout@v4 - with: - repository: 'opensearch-project/opensearch-build' - ref: ${{ inputs.build_ref }} - path: 'opensearch-build' - - name: Get ci image version from opensearch-build repository scripts - id: step-ci-image-setups - run: | - PRODUCT=${{ inputs.product }} - PLATFORM=${{ inputs.platform }} - TYPE=${{ inputs.type }} - if [[ -z "$PLATFORM" ]]; then - if [[ "$PRODUCT" = "opensearch" ]]; then - PLATFORM="al2" # Temp measure before al2 deprecation on opensearch for k-NN - else - PLATFORM="almalinux8" - fi - fi - - if [[ "$PLATFORM" = "al2" ]]; then - CI_IMAGE_CMD="cp -a /node_al2/* /node && /node/bin/node -v" - echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT - CI_IMAGE_OPTIONS="--user root -v /node:/node:rw,rshared -v /node:/__e/node20:ro,rshared" - echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT - else - CI_IMAGE_CMD="echo pass" - echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT - CI_IMAGE_OPTIONS="--user root" - echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT - fi - crane version - echo $PRODUCT $PLATFORM - CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p $PLATFORM -u $PRODUCT -t $TYPE | head -1` - echo $CI_IMAGE_VERSION - echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/github-merit-badger.yml b/.github/workflows/github-merit-badger.yml deleted file mode 100644 index 00f0199f15..0000000000 --- a/.github/workflows/github-merit-badger.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: github-merit-badger -on: - pull_request_target: - types: - - opened - -jobs: - call-action: - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - uses: aws-github-ops/github-merit-badger@v0.0.98 - id: merit-badger - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - badges: '[first-time-contributor,repeat-contributor,valued-contributor,seasoned-contributor,all-star-contributor,distinguished-contributor]' - thresholds: '[0,3,6,13,25,50]' - badge-type: 'achievement' - ignore-usernames: '[opensearch-ci-bot]' diff --git a/.github/workflows/groovy-tests.yml b/.github/workflows/groovy-tests.yml deleted file mode 100644 index e77f602650..0000000000 --- a/.github/workflows/groovy-tests.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: groovy-tests - -on: [push, pull_request] - -jobs: - groovy-tests: - runs-on: ubuntu-latest - steps: - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: 11 - - uses: actions/checkout@v3 - - name: Run Tests - run: | - ./gradlew test --info diff --git a/.github/workflows/license-header-checker.yml b/.github/workflows/license-header-checker.yml deleted file mode 100644 index d026f0b47b..0000000000 --- a/.github/workflows/license-header-checker.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -name: License Header Checker - -on: [push, pull_request] - -jobs: - license-header-checker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Add License Header - run: npx @kt3k/license-checker diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml deleted file mode 100644 index 52cdf7f139..0000000000 --- a/.github/workflows/link-checker.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Link Checker -on: [push, pull_request] - -permissions: - contents: read -jobs: - linkchecker: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: lychee Link Checker - id: lychee - uses: lycheeverse/lychee-action@v1 - with: - args: --verbose --max-retries 5 --retry-wait-time 10 --accept=200,403,429 --exclude-path tests/ --exclude-path release-notes/ ./ - fail: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/manifests.yml b/.github/workflows/manifests.yml deleted file mode 100644 index 5850e33b85..0000000000 --- a/.github/workflows/manifests.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -name: manifests-ci-check - -on: - pull_request: - types: [opened, synchronize] - paths: - - 'manifests/**/*.yml' - - '!manifests/templates/**/' - - 'legacy-manifests/**/*.yml' - -jobs: - list-changed-manifests: - if: ${{ github.repository == 'opensearch-project/opensearch-build' }} - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Get changed manifest files - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - id: list-changed-manifests - with: - files: manifests/**/opensearch*.yml - json: true - quotepath: false - dir_names: false - - - name: Set unique changed manifests as matrix - id: set-matrix - run: echo "matrix={\"manifest\":${{ steps.list-changed-manifests.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT" - - manifest-checks: - needs: [list-changed-manifests] - runs-on: ubuntu-latest - env: - PYTHON_VERSION: 3.9 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.list-changed-manifests.outputs.matrix) }} - steps: - - uses: actions/checkout@v3 - - name: Check JDK Version - run: | - java_version=`cat ${{ matrix.manifest }} | yq -r .ci.image.args | grep -Eo '[0-9]+' || echo ''` - echo $java_version - echo "JAVA_VERSION=$java_version" >> "$GITHUB_ENV" - - name: Set Up JDK ${{ env.JAVA_VERSION }} - if: ${{ env.JAVA_VERSION }} - uses: actions/setup-java@v1 - with: - java-version: ${{ env.JAVA_VERSION }} - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install Pipenv and Dependencies - run: | - python -m pip install --upgrade pipenv wheel - - name: OpenSearch Manifests - run: |- - ./ci.sh ${{ matrix.manifest }} --snapshot diff --git a/.github/workflows/os-increment-plugin-versions.yml b/.github/workflows/os-increment-plugin-versions.yml deleted file mode 100644 index 4b1f4a9adf..0000000000 --- a/.github/workflows/os-increment-plugin-versions.yml +++ /dev/null @@ -1,156 +0,0 @@ ---- -name: Increment OpenSearch Plugins Version - -on: - schedule: - - cron: 0 0 * * * - workflow_dispatch: - inputs: - logLevel: - description: Log level - required: true - default: warning - type: choice - options: - - info - - warning - - debug - -jobs: - plugin-version-increment-sync: - if: github.repository == 'opensearch-project/opensearch-build' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - entry: - # Adding the core repo OpenSearch for the label creation automation - - {repo: OpenSearch} - - {repo: alerting} - - {repo: anomaly-detection} - - {repo: asynchronous-search} - - {repo: common-utils} - - {repo: cross-cluster-replication} - - {repo: geospatial} - - {repo: index-management} - - {repo: job-scheduler} - - {repo: k-NN} - - {repo: neural-search} - - {repo: ml-commons} - - {repo: notifications, path: notifications} - - {repo: observability} - - {repo: performance-analyzer} - - {repo: performance-analyzer-rca} - - {repo: reporting} - - {repo: security} - - {repo: security-analytics} - - {repo: sql} - - {repo: custom-codecs} - - {repo: flow-framework} - - {repo: skills} - - {repo: query-insights} - - {repo: opensearch-system-templates} - - {repo: opensearch-remote-metadata-sdk} - - {repo: opensearch-learning-to-rank-base} - branch: - - 2.x - - main - - '2.18' - - '2.19' - exclude: - - {entry: {repo: geospatial}, branch: '1.3'} - - {entry: {repo: neural-search}, branch: '1.3'} - - {entry: {repo: security-analytics}, branch: '1.3'} - - {entry: {repo: notifications, path: notifications}, branch: '1.3'} - - {entry: {repo: custom-codecs}, branch: '1.3'} - steps: - - name: Check out OpenSearch repo - uses: actions/checkout@v3 - with: - repository: opensearch-project/OpenSearch - ref: ${{ matrix.branch }} - - name: Fetch OpenSearch version - run: | - OPENSEARCH_VERSION=$(cat buildSrc/version.properties | grep opensearch | cut -d= -f2 | grep -oE '[0-9.]+') - echo "OPENSEARCH_VERSION_NUMBER=$OPENSEARCH_VERSION" >> $GITHUB_ENV - OPENSEARCH_VERSION=$OPENSEARCH_VERSION-SNAPSHOT - echo "OPENSEARCH_VERSION=$OPENSEARCH_VERSION" >> $GITHUB_ENV - - name: Check out plugin repo - if: ${{ matrix.entry.repo != 'OpenSearch' }} - uses: actions/checkout@v3 - with: - repository: opensearch-project/${{ matrix.entry.repo }} - ref: ${{ matrix.branch }} - - name: Increment Version in ${{ matrix.entry.repo }} - if: ${{ matrix.entry.repo != 'OpenSearch' }} - run: | - echo "OpenSearch Core repo version on branch ${{ matrix.branch }} is ${{ env.OPENSEARCH_VERSION_NUMBER }}" - if [ ${{ matrix.entry.path }} ]; then - echo "The gradle path is ${{ matrix.entry.path }}" - cd ${{ matrix.entry.path }} - fi - ./gradlew updateVersion -DnewVersion=${{ env.OPENSEARCH_VERSION }} - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.6.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Check and Create label - id: check_create_label - uses: actions/github-script@v6 - with: - github-token: ${{ steps.github_app_token.outputs.token }} - result-encoding: string - script: | - const labelName = "v${{ env.OPENSEARCH_VERSION_NUMBER }}"; - let labelFound = false; - try { - const label = await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: "${{ matrix.entry.repo }}", - name: labelName - }); - labelFound = true; - } catch (error) { - if (error.status === 404) { - const randomColor = Math.floor(Math.random() * 16777215).toString(16); - const newLabel = { - owner: context.repo.owner, - repo: "${{ matrix.entry.repo }}", - name: labelName, - color: randomColor, - description: "Issues targeting release " + labelName - }; - await github.rest.issues.createLabel(newLabel); - labelFound = true; - } else { - throw error; - } - } - console.log(labelFound); - return labelFound - - name: Create Pull Request - if: ${{ matrix.entry.repo != 'OpenSearch' }} - id: cpr - uses: peter-evans/create-pull-request@v4 - with: - token: ${{ steps.github_app_token.outputs.token }} - committer: opensearch-ci-bot - author: opensearch-ci-bot - commit-message: | - Increment version to ${{ env.OPENSEARCH_VERSION }} - - Signed-off-by: opensearch-ci-bot - delete-branch: true - branch: create-pull-request/${{ env.OPENSEARCH_VERSION }} - title: '[AUTO] Increment version to ${{ env.OPENSEARCH_VERSION }}' - labels: | - v${{ env.OPENSEARCH_VERSION_NUMBER }} - body: | - - Incremented version to **${{ env.OPENSEARCH_VERSION }}**. - - name: Check outputs - run: |- - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/os-release-issues.yml b/.github/workflows/os-release-issues.yml deleted file mode 100644 index c5b883bb21..0000000000 --- a/.github/workflows/os-release-issues.yml +++ /dev/null @@ -1,123 +0,0 @@ ---- -name: release-issue-os - -on: - workflow_dispatch: - inputs: - logLevel: - description: Log level - required: true - default: warning - type: choice - options: - - info - - warning - - debug - schedule: - - cron: 0 1 * * * - -jobs: - list-manifest-versions: - if: github.repository == 'opensearch-project/opensearch-build' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - with: - repository: opensearch-project/opensearch-build - ref: main - - id: set-matrix - # produces a list of major versions, e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] - run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" - component-release-issue: - needs: list-manifest-versions - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - entry: - - {repo: OpenSearch} - - {repo: alerting} - - {repo: anomaly-detection} - - {repo: asynchronous-search} - - {repo: common-utils} - - {repo: cross-cluster-replication} - - {repo: geospatial} - - {repo: index-management} - - {repo: job-scheduler} - - {repo: k-NN} - - {repo: neural-search} - - {repo: ml-commons} - - {repo: notifications} - - {repo: observability} - - {repo: performance-analyzer} - - {repo: performance-analyzer-rca} - - {repo: reporting} - - {repo: security} - - {repo: security-analytics} - - {repo: sql} - - {repo: custom-codecs} - - {repo: flow-framework} - - {repo: skills} - - {repo: query-insights} - - {repo: opensearch-system-templates} - release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.6.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Checkout - uses: actions/checkout@v3 - - name: Check if build repo release issue exists - id: check_if_build_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/opensearch-build - token: ${{ steps.github_app_token.outputs.token }} - issue-state: 'open' - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Check out plugin repo - uses: actions/checkout@v3 - with: - path: plugin-repo - repository: opensearch-project/${{ matrix.entry.repo }} - - name: Check if plugin repo release issue exists - if: steps.check_if_build_repo_issue_exists.outputs.issues != '[]' - id: check_if_plugin_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/${{ matrix.entry.repo }} - token: ${{ steps.github_app_token.outputs.token }} - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Replace Placeholders - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' - run: | - # Read the file contents and replace the placeholders - file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" - RELEASE_VERSION="${{ matrix.release_version }}" - RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2) - BUILD_REPO_ISSUE_OUTPUT=$(cat < "$file_path.tmp" && mv "$file_path.tmp" "$file_path" - - name: Create component release issue from file - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' - uses: peter-evans/create-issue-from-file@v4 - with: - title: '[RELEASE] Release version ${{ matrix.release_version }}' - content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md - labels: | - v${{ matrix.release_version }} - token: ${{ steps.github_app_token.outputs.token }} - repository: opensearch-project/${{ matrix.entry.repo }} diff --git a/.github/workflows/osd-increment-plugin-versions.yml b/.github/workflows/osd-increment-plugin-versions.yml deleted file mode 100644 index c581f060e0..0000000000 --- a/.github/workflows/osd-increment-plugin-versions.yml +++ /dev/null @@ -1,195 +0,0 @@ ---- -name: Increment OpenSearch Dashboards Plugins Version - -on: - schedule: - - cron: 0 0 * * * - workflow_dispatch: - inputs: - logLevel: - description: Log level - required: true - default: warning - type: choice - options: - - info - - warning - - debug -jobs: - plugin-version-increment-sync: - if: github.repository == 'opensearch-project/opensearch-build' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - entry: - # Adding the core repo OpenSearch-Dashboards for the label creation automation - - {repo: OpenSearch-Dashboards} - - {repo: dashboards-observability} - - {repo: dashboards-reporting} - - {repo: dashboards-visualizations} - - {repo: dashboards-query-workbench} - - {repo: dashboards-assistant} - - {repo: dashboards-maps} - - {repo: dashboards-flow-framework} - - {repo: anomaly-detection-dashboards-plugin} - - {repo: ml-commons-dashboards} - - {repo: index-management-dashboards-plugin} - - {repo: dashboards-notifications} - - {repo: alerting-dashboards-plugin} - - {repo: security-analytics-dashboards-plugin} - - {repo: security-dashboards-plugin} - - {repo: dashboards-search-relevance} - - {repo: opensearch-dashboards-functional-test} - - {repo: query-insights-dashboards} - branch: - - 2.x - - main - - '2.18' - - '2.19' - steps: - - name: Check out OpenSearch Dashboards repo - uses: actions/checkout@v3 - with: - repository: opensearch-project/OpenSearch-Dashboards - ref: ${{ matrix.branch }} - path: OpenSearch-Dashboards - - name: Check out plugin repo - if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' }} - uses: actions/checkout@v3 - with: - repository: opensearch-project/${{ matrix.entry.repo }} - ref: ${{ matrix.branch }} - path: OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }} - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: './OpenSearch-Dashboards/.nvmrc' - registry-url: 'https://registry.npmjs.org' - - name: Install Yarn and Setup Dashboard Version - shell: bash - run: | - YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn") - echo "Installing yarn@$YARN_VERSION" - npm i -g yarn@$YARN_VERSION - DASHBOARD_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').version") - echo "DASHBOARD_VERSION=$DASHBOARD_VERSION" >> $GITHUB_ENV - - run: node -v - - run: yarn -v - - name: Bootstrap and Version Increment - if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' }} - run: | - cd OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }} - if [ ${{ matrix.entry.path }} ]; then - yarn osd bootstrap --single-version=loose - cp -R ${{ matrix.entry.path }} ../ - cd ../${{ matrix.entry.path }} - node ../../scripts/plugin_helpers version --sync legacy - OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version") - echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV - cd ../ - cp -R ${{ matrix.entry.path }} ${{ matrix.entry.repo }}/ - cd ${{ matrix.entry.repo }}/ - # tmp `elif` solution for opensearch-dashboards-functional-test (ref: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1801#issuecomment-1545947935) - elif [ ${{ matrix.entry.repo }} == "opensearch-dashboards-functional-test" ]; then - jq --arg DASHBOARD_VERSION "${{ env.DASHBOARD_VERSION }}" '.version = $DASHBOARD_VERSION' package.json > package-tmp.json - mv package-tmp.json package.json - OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version") - echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV - else - yarn osd bootstrap --single-version=loose - node ../../scripts/plugin_helpers version --sync legacy - OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version") - echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV - fi - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.6.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Check if label exists - id: check_label - uses: actions/github-script@v6 - with: - github-token: ${{ steps.github_app_token.outputs.token }} - result-encoding: string - script: | - const labelName = "v${{ env.DASHBOARD_VERSION }}"; - let labelFound = false; - try { - const label = await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: "${{ matrix.entry.repo }}", - name: labelName - }); - labelFound = true; - } catch (error) { - if (error.status === 404) { - const randomColor = Math.floor(Math.random() * 16777215).toString(16); - const newLabel = { - owner: context.repo.owner, - repo: "${{ matrix.entry.repo }}", - name: labelName, - color: randomColor, - description: "Issues targeting release " + labelName - }; - await github.rest.issues.createLabel(newLabel); - labelFound = true; - } else { - throw error; - } - } - console.log(labelFound); - return labelFound - - name: Create Pull Request for plugins - if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' && matrix.entry.repo - != 'opensearch-dashboards-functional-test' }} - id: cpr - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ steps.github_app_token.outputs.token }} - committer: opensearch-ci-bot - author: opensearch-ci-bot - commit-message: | - Increment version to ${{ env.OSD_PLUGIN_VERSION }} - - Signed-off-by: opensearch-ci-bot - delete-branch: true - branch: create-pull-request/${{ env.OSD_PLUGIN_VERSION }} - title: '[AUTO] Increment version to ${{ env.OSD_PLUGIN_VERSION }}' - labels: | - v${{ env.DASHBOARD_VERSION }} - body: | - - Incremented version to **${{ env.OSD_PLUGIN_VERSION }}**. - path: 'OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }}' - add-paths: | - opensearch_dashboards.json - package.json - - name: Create Pull Request for opensearch-dashboards-functional-test - if: ${{ matrix.entry.repo == 'opensearch-dashboards-functional-test' }} - id: cprft - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ steps.github_app_token.outputs.token }} - committer: opensearch-ci-bot - author: opensearch-ci-bot - commit-message: | - Increment version to ${{ env.OSD_PLUGIN_VERSION }} - - Signed-off-by: opensearch-ci-bot - delete-branch: true - branch: create-pull-request/${{ env.OSD_PLUGIN_VERSION }} - title: '[AUTO] Increment version to ${{ env.OSD_PLUGIN_VERSION }}' - labels: | - v${{ env.DASHBOARD_VERSION }} - body: | - - Incremented version to **${{ env.OSD_PLUGIN_VERSION }}**. - path: 'OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }}' - add-paths: | - package.json - - name: Check outputs - run: |- - echo "Pull Request Number - ${{ steps.cprft.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cprft.outputs.pull-request-url }}" diff --git a/.github/workflows/osd-release-issues.yml b/.github/workflows/osd-release-issues.yml deleted file mode 100644 index 9a2dcbc9d2..0000000000 --- a/.github/workflows/osd-release-issues.yml +++ /dev/null @@ -1,116 +0,0 @@ ---- -name: release-issue-osd - -on: - workflow_dispatch: - inputs: - logLevel: - description: Log level - required: true - default: warning - type: choice - options: - - info - - warning - - debug - schedule: - - cron: 0 1 * * * - -jobs: - list-manifest-versions: - if: github.repository == 'opensearch-project/opensearch-build' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - with: - repository: opensearch-project/opensearch-build - ref: main - - id: set-matrix - # produces a list of major versions, e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] - run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" - component-release-issue: - needs: list-manifest-versions - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - entry: - - {repo: OpenSearch-Dashboards} - - {repo: dashboards-observability} - - {repo: dashboards-reporting} - - {repo: dashboards-visualizations} - - {repo: dashboards-query-workbench} - - {repo: dashboards-assistant} - - {repo: dashboards-maps} - - {repo: dashboards-flow-framework} - - {repo: anomaly-detection-dashboards-plugin} - - {repo: ml-commons-dashboards} - - {repo: index-management-dashboards-plugin} - - {repo: dashboards-notifications} - - {repo: alerting-dashboards-plugin} - - {repo: security-analytics-dashboards-plugin} - - {repo: security-dashboards-plugin} - - {repo: dashboards-search-relevance} - - {repo: opensearch-dashboards-functional-test} - - {repo: query-insights-dashboards} - release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.6.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Checkout - uses: actions/checkout@v3 - - name: Check if build repo release issue exists - id: check_if_build_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/opensearch-build - token: ${{ steps.github_app_token.outputs.token }} - issue-state: 'open' - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Check out plugin repo - uses: actions/checkout@v3 - with: - path: plugin-repo - repository: opensearch-project/${{ matrix.entry.repo }} - - name: Check if plugin repo release issue exists - if: steps.check_if_build_repo_issue_exists.outputs.issues != '[]' - id: check_if_plugin_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/${{ matrix.entry.repo }} - token: ${{ steps.github_app_token.outputs.token }} - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Replace Placeholders - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' - run: | - # Read the file contents and replace the placeholders - file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" - RELEASE_VERSION="${{ matrix.release_version }}" - RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2) - BUILD_REPO_ISSUE_OUTPUT=$(cat < "$file_path.tmp" && mv "$file_path.tmp" "$file_path" - - name: Create component release issue from file - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' - uses: peter-evans/create-issue-from-file@v4 - with: - title: '[RELEASE] Release version ${{ matrix.release_version }}' - content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md - labels: | - v${{ matrix.release_version }} - token: ${{ steps.github_app_token.outputs.token }} - repository: opensearch-project/${{ matrix.entry.repo }} diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml deleted file mode 100644 index 44fce89530..0000000000 --- a/.github/workflows/publish-release.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Publish a release - -on: - push: - tags: - - '*' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Release - uses: softprops/action-gh-release@v2 - with: - generate_release_notes: true diff --git a/.github/workflows/publish-wiki.yml b/.github/workflows/publish-wiki.yml deleted file mode 100644 index a25e59dca6..0000000000 --- a/.github/workflows/publish-wiki.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: Publish wiki - -on: - push: - branches: [main] - paths: - - docs/** - - .github/workflows/publish-wiki.yml - -concurrency: - group: publish-wiki - cancel-in-progress: true - -permissions: - contents: write - -jobs: - publish-wiki: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: Andrew-Chen-Wang/github-wiki-action@v4 - with: - path: docs/ diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 01545795b1..e1600b92fd 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -44,7 +44,6 @@ jobs: python-tests-linux: needs: Get-CI-Image-Tag - name: python-tests-linux-${{ matrix.os }} strategy: matrix: os: @@ -78,12 +77,13 @@ jobs: run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml - uses: actions/upload-artifact@v4 + if: ${{ matrix.os }} = "ubuntu-24.04" with: - name: coverage-report-${{ matrix.os }} + name: coverage-report path: ./coverage.xml python-tests-codecov: - needs: python-tests-linux-${{ matrix.os }} + needs: python-tests-linux strategy: matrix: os: @@ -93,7 +93,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: my-artifact + name: coverage-report path: ./coverage.xml - name: Upload Coverage Report uses: codecov/codecov-action@v5 diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml deleted file mode 100644 index 8fa22d2cd1..0000000000 --- a/.github/workflows/releases.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: releases - -on: - workflow_dispatch: - schedule: - - cron: 0 0 * * * - -jobs: - list-manifest-versions: - if: github.repository == 'opensearch-project/opensearch-build' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - id: set-matrix - # produces a list of versions, e.g. ["1.0.0","1.0.0","1.0.1","1.1.0","1.2.0","2.0.0"] - run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" - check: - needs: list-manifest-versions - runs-on: ubuntu-latest - strategy: - matrix: - release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} - steps: - - uses: actions/checkout@v3 - - name: Create release issue - uses: dblock/create-a-github-issue@v3.0.0 - id: release-issue - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ matrix.release_version }} - with: - search_existing: all - update_existing: false - filename: .github/ISSUE_TEMPLATE/release_template.md - - name: Create retrospective issue - uses: dblock/create-a-github-issue@v3.0.0 - if: ${{ steps.release-issue.outputs.status == 'created' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ matrix.release_version }} - RELEASE_ISSUE_URL: ${{ steps.release-issue.outputs.url }} - with: - search_existing: all - update_existing: false - filename: .github/ISSUE_TEMPLATE/release_retrospective_template.md diff --git a/.github/workflows/yaml-lint.yml b/.github/workflows/yaml-lint.yml deleted file mode 100644 index 34b77e7926..0000000000 --- a/.github/workflows/yaml-lint.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: yaml-lint - -on: [push, pull_request] - -jobs: - yaml-lint: - runs-on: ubuntu-latest - env: - PYTHON_VERSION: 3.9.12 - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install Pipenv and Dependencies - run: | - python -m pip install --upgrade pipenv wheel - pipenv install --deploy --dev - - name: Lint YAML files - run: |- - pipenv run yamllint . From 5f756218cd09e87af2c55ce128eb9cf945d3659b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 16:13:19 -0500 Subject: [PATCH 06/12] Separate codecov upload to its own steps Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index e1600b92fd..52b69da2cb 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: Get-CI-Image-Tag: - uses: ./.github/workflows/get-ci-image-tag.yml + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main with: product: opensearch From c4198487ece808ba132fe7b6344a248929d0bd2c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 16:21:31 -0500 Subject: [PATCH 07/12] Restore yamls Signed-off-by: Peter Zhu --- .github/workflows/add-untriaged.yml | 20 ++ .github/workflows/automatic-merges.yml | 69 +++++++ .github/workflows/dockerfile-lint.yml | 27 +++ .github/workflows/get-ci-image-tag.yml | 82 ++++++++ .github/workflows/github-merit-badger.yml | 21 ++ .github/workflows/groovy-tests.yml | 17 ++ .github/workflows/license-header-checker.yml | 12 ++ .github/workflows/link-checker.yml | 20 ++ .github/workflows/manifests.yml | 63 ++++++ .../os-increment-plugin-versions.yml | 156 ++++++++++++++ .github/workflows/os-release-issues.yml | 123 +++++++++++ .../osd-increment-plugin-versions.yml | 195 ++++++++++++++++++ .github/workflows/osd-release-issues.yml | 116 +++++++++++ .github/workflows/publish-release.yml | 19 ++ .github/workflows/publish-wiki.yml | 25 +++ .github/workflows/releases.yml | 48 +++++ .github/workflows/yaml-lint.yml | 23 +++ 17 files changed, 1036 insertions(+) create mode 100644 .github/workflows/add-untriaged.yml create mode 100644 .github/workflows/automatic-merges.yml create mode 100644 .github/workflows/dockerfile-lint.yml create mode 100644 .github/workflows/get-ci-image-tag.yml create mode 100644 .github/workflows/github-merit-badger.yml create mode 100644 .github/workflows/groovy-tests.yml create mode 100644 .github/workflows/license-header-checker.yml create mode 100644 .github/workflows/link-checker.yml create mode 100644 .github/workflows/manifests.yml create mode 100644 .github/workflows/os-increment-plugin-versions.yml create mode 100644 .github/workflows/os-release-issues.yml create mode 100644 .github/workflows/osd-increment-plugin-versions.yml create mode 100644 .github/workflows/osd-release-issues.yml create mode 100644 .github/workflows/publish-release.yml create mode 100644 .github/workflows/publish-wiki.yml create mode 100644 .github/workflows/releases.yml create mode 100644 .github/workflows/yaml-lint.yml diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml new file mode 100644 index 0000000000..88e31ef9d3 --- /dev/null +++ b/.github/workflows/add-untriaged.yml @@ -0,0 +1,20 @@ +--- +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/automatic-merges.yml b/.github/workflows/automatic-merges.yml new file mode 100644 index 0000000000..0772885e3b --- /dev/null +++ b/.github/workflows/automatic-merges.yml @@ -0,0 +1,69 @@ +--- +name: automatic-merges + +on: + workflow_run: + # If new workflows triggered on pull request are added, they should be added to this list + workflows: + - Link Checker + - yaml-lint + - dockerfile-lint + - groovy-tests + - manifests + - python-tests + types: completed + +jobs: + automatic-merge-manifests: + if: github.repository == 'opensearch-project/opensearch-build' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - id: find-triggering-pr + uses: peternied/find-triggering-pr@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: peternied/discerning-merger@v3 + if: steps.find-triggering-pr.outputs.pr-number != null + with: + token: ${{ secrets.GITHUB_TOKEN }} + pull-request-number: ${{ steps.find-triggering-pr.outputs.pr-number }} + allowed-authors: | + dependabot + opensearch-trigger-bot + opensearch-ci-bot + peterzhuamazon + gaiksaya + rishabh6788 + zelinh + prudhvigodithi + Divyaasm + dblock + tianleh + peternied + allowed-files: | + manifests/*/*.yml + + automatic-merge-new-manifests: + if: github.repository == 'opensearch-project/opensearch-build' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - id: find-triggering-pr + uses: peternied/find-triggering-pr@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: peternied/discerning-merger@v3 + if: steps.find-triggering-pr.outputs.pr-number != null + with: + token: ${{ secrets.GITHUB_TOKEN }} + pull-request-number: ${{ steps.find-triggering-pr.outputs.pr-number }} + allowed-authors: | + opensearch-ci-bot + allowed-files: | + manifests/*/*.yml + jenkins/check-for-build.jenkinsfile diff --git a/.github/workflows/dockerfile-lint.yml b/.github/workflows/dockerfile-lint.yml new file mode 100644 index 0000000000..a363432719 --- /dev/null +++ b/.github/workflows/dockerfile-lint.yml @@ -0,0 +1,27 @@ +--- +name: dockerfile-lint + +on: + pull_request: + paths: + - 'docker/ci/dockerfiles/current/**' + - 'docker/release/dockerfiles/**' + +jobs: + dockerfile-lint-test_ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: jbergstroem/hadolint-gh-action@v1 + with: + dockerfile: 'docker/ci/dockerfiles/current/*.dockerfile' + annotate: false + + dockerfile-lint-test_release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: jbergstroem/hadolint-gh-action@v1 + with: + dockerfile: 'docker/release/dockerfiles/*.dockerfile' + annotate: false diff --git a/.github/workflows/get-ci-image-tag.yml b/.github/workflows/get-ci-image-tag.yml new file mode 100644 index 0000000000..b1e7e41c15 --- /dev/null +++ b/.github/workflows/get-ci-image-tag.yml @@ -0,0 +1,82 @@ +--- +name: Get CI Image Tag +on: + workflow_call: + inputs: + product: + required: false + type: string + default: 'opensearch' + platform: + required: false + type: string + build_ref: + required: false + type: string + default: 'main' + version: + required: false + type: string + type: + required: false + type: string + default: 'build' + outputs: + ci-image-version-linux: + description: The ci image version for linux build + value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-version-linux }} + ci-image-start-options: + description: The ci image start options to set when starting the container + value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-options }} + ci-image-start-command: + description: The ci image start commands to run after initialization + value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-command }} + +jobs: + Get-CI-Image-Tag: + runs-on: ubuntu-latest + outputs: + output-ci-image-version-linux: ${{ steps.step-ci-image-setups.outputs.ci-image-version-linux }} + output-ci-image-start-command: ${{ steps.step-ci-image-setups.outputs.ci-image-start-command }} + output-ci-image-start-options: ${{ steps.step-ci-image-setups.outputs.ci-image-start-options }} + steps: + - name: Install crane + uses: iarekylew00t/crane-installer@v1 + with: + crane-release: v0.15.2 + - name: Checkout opensearch-build repository + uses: actions/checkout@v4 + with: + repository: 'opensearch-project/opensearch-build' + ref: ${{ inputs.build_ref }} + path: 'opensearch-build' + - name: Get ci image version from opensearch-build repository scripts + id: step-ci-image-setups + run: | + PRODUCT=${{ inputs.product }} + PLATFORM=${{ inputs.platform }} + TYPE=${{ inputs.type }} + if [[ -z "$PLATFORM" ]]; then + if [[ "$PRODUCT" = "opensearch" ]]; then + PLATFORM="al2" # Temp measure before al2 deprecation on opensearch for k-NN + else + PLATFORM="almalinux8" + fi + fi + + if [[ "$PLATFORM" = "al2" ]]; then + CI_IMAGE_CMD="cp -a /node_al2/* /node && /node/bin/node -v" + echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT + CI_IMAGE_OPTIONS="--user root -v /node:/node:rw,rshared -v /node:/__e/node20:ro,rshared" + echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT + else + CI_IMAGE_CMD="echo pass" + echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT + CI_IMAGE_OPTIONS="--user root" + echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT + fi + crane version + echo $PRODUCT $PLATFORM + CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p $PLATFORM -u $PRODUCT -t $TYPE | head -1` + echo $CI_IMAGE_VERSION + echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/github-merit-badger.yml b/.github/workflows/github-merit-badger.yml new file mode 100644 index 0000000000..00f0199f15 --- /dev/null +++ b/.github/workflows/github-merit-badger.yml @@ -0,0 +1,21 @@ +--- +name: github-merit-badger +on: + pull_request_target: + types: + - opened + +jobs: + call-action: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - uses: aws-github-ops/github-merit-badger@v0.0.98 + id: merit-badger + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + badges: '[first-time-contributor,repeat-contributor,valued-contributor,seasoned-contributor,all-star-contributor,distinguished-contributor]' + thresholds: '[0,3,6,13,25,50]' + badge-type: 'achievement' + ignore-usernames: '[opensearch-ci-bot]' diff --git a/.github/workflows/groovy-tests.yml b/.github/workflows/groovy-tests.yml new file mode 100644 index 0000000000..e77f602650 --- /dev/null +++ b/.github/workflows/groovy-tests.yml @@ -0,0 +1,17 @@ +--- +name: groovy-tests + +on: [push, pull_request] + +jobs: + groovy-tests: + runs-on: ubuntu-latest + steps: + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: 11 + - uses: actions/checkout@v3 + - name: Run Tests + run: | + ./gradlew test --info diff --git a/.github/workflows/license-header-checker.yml b/.github/workflows/license-header-checker.yml new file mode 100644 index 0000000000..d026f0b47b --- /dev/null +++ b/.github/workflows/license-header-checker.yml @@ -0,0 +1,12 @@ +--- +name: License Header Checker + +on: [push, pull_request] + +jobs: + license-header-checker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Add License Header + run: npx @kt3k/license-checker diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml new file mode 100644 index 0000000000..52cdf7f139 --- /dev/null +++ b/.github/workflows/link-checker.yml @@ -0,0 +1,20 @@ +--- +name: Link Checker +on: [push, pull_request] + +permissions: + contents: read +jobs: + linkchecker: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: lychee Link Checker + id: lychee + uses: lycheeverse/lychee-action@v1 + with: + args: --verbose --max-retries 5 --retry-wait-time 10 --accept=200,403,429 --exclude-path tests/ --exclude-path release-notes/ ./ + fail: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/manifests.yml b/.github/workflows/manifests.yml new file mode 100644 index 0000000000..5850e33b85 --- /dev/null +++ b/.github/workflows/manifests.yml @@ -0,0 +1,63 @@ +--- +name: manifests-ci-check + +on: + pull_request: + types: [opened, synchronize] + paths: + - 'manifests/**/*.yml' + - '!manifests/templates/**/' + - 'legacy-manifests/**/*.yml' + +jobs: + list-changed-manifests: + if: ${{ github.repository == 'opensearch-project/opensearch-build' }} + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Get changed manifest files + uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 + id: list-changed-manifests + with: + files: manifests/**/opensearch*.yml + json: true + quotepath: false + dir_names: false + + - name: Set unique changed manifests as matrix + id: set-matrix + run: echo "matrix={\"manifest\":${{ steps.list-changed-manifests.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT" + + manifest-checks: + needs: [list-changed-manifests] + runs-on: ubuntu-latest + env: + PYTHON_VERSION: 3.9 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.list-changed-manifests.outputs.matrix) }} + steps: + - uses: actions/checkout@v3 + - name: Check JDK Version + run: | + java_version=`cat ${{ matrix.manifest }} | yq -r .ci.image.args | grep -Eo '[0-9]+' || echo ''` + echo $java_version + echo "JAVA_VERSION=$java_version" >> "$GITHUB_ENV" + - name: Set Up JDK ${{ env.JAVA_VERSION }} + if: ${{ env.JAVA_VERSION }} + uses: actions/setup-java@v1 + with: + java-version: ${{ env.JAVA_VERSION }} + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install Pipenv and Dependencies + run: | + python -m pip install --upgrade pipenv wheel + - name: OpenSearch Manifests + run: |- + ./ci.sh ${{ matrix.manifest }} --snapshot diff --git a/.github/workflows/os-increment-plugin-versions.yml b/.github/workflows/os-increment-plugin-versions.yml new file mode 100644 index 0000000000..4b1f4a9adf --- /dev/null +++ b/.github/workflows/os-increment-plugin-versions.yml @@ -0,0 +1,156 @@ +--- +name: Increment OpenSearch Plugins Version + +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: + inputs: + logLevel: + description: Log level + required: true + default: warning + type: choice + options: + - info + - warning + - debug + +jobs: + plugin-version-increment-sync: + if: github.repository == 'opensearch-project/opensearch-build' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + entry: + # Adding the core repo OpenSearch for the label creation automation + - {repo: OpenSearch} + - {repo: alerting} + - {repo: anomaly-detection} + - {repo: asynchronous-search} + - {repo: common-utils} + - {repo: cross-cluster-replication} + - {repo: geospatial} + - {repo: index-management} + - {repo: job-scheduler} + - {repo: k-NN} + - {repo: neural-search} + - {repo: ml-commons} + - {repo: notifications, path: notifications} + - {repo: observability} + - {repo: performance-analyzer} + - {repo: performance-analyzer-rca} + - {repo: reporting} + - {repo: security} + - {repo: security-analytics} + - {repo: sql} + - {repo: custom-codecs} + - {repo: flow-framework} + - {repo: skills} + - {repo: query-insights} + - {repo: opensearch-system-templates} + - {repo: opensearch-remote-metadata-sdk} + - {repo: opensearch-learning-to-rank-base} + branch: + - 2.x + - main + - '2.18' + - '2.19' + exclude: + - {entry: {repo: geospatial}, branch: '1.3'} + - {entry: {repo: neural-search}, branch: '1.3'} + - {entry: {repo: security-analytics}, branch: '1.3'} + - {entry: {repo: notifications, path: notifications}, branch: '1.3'} + - {entry: {repo: custom-codecs}, branch: '1.3'} + steps: + - name: Check out OpenSearch repo + uses: actions/checkout@v3 + with: + repository: opensearch-project/OpenSearch + ref: ${{ matrix.branch }} + - name: Fetch OpenSearch version + run: | + OPENSEARCH_VERSION=$(cat buildSrc/version.properties | grep opensearch | cut -d= -f2 | grep -oE '[0-9.]+') + echo "OPENSEARCH_VERSION_NUMBER=$OPENSEARCH_VERSION" >> $GITHUB_ENV + OPENSEARCH_VERSION=$OPENSEARCH_VERSION-SNAPSHOT + echo "OPENSEARCH_VERSION=$OPENSEARCH_VERSION" >> $GITHUB_ENV + - name: Check out plugin repo + if: ${{ matrix.entry.repo != 'OpenSearch' }} + uses: actions/checkout@v3 + with: + repository: opensearch-project/${{ matrix.entry.repo }} + ref: ${{ matrix.branch }} + - name: Increment Version in ${{ matrix.entry.repo }} + if: ${{ matrix.entry.repo != 'OpenSearch' }} + run: | + echo "OpenSearch Core repo version on branch ${{ matrix.branch }} is ${{ env.OPENSEARCH_VERSION_NUMBER }}" + if [ ${{ matrix.entry.path }} ]; then + echo "The gradle path is ${{ matrix.entry.path }}" + cd ${{ matrix.entry.path }} + fi + ./gradlew updateVersion -DnewVersion=${{ env.OPENSEARCH_VERSION }} + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.6.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Check and Create label + id: check_create_label + uses: actions/github-script@v6 + with: + github-token: ${{ steps.github_app_token.outputs.token }} + result-encoding: string + script: | + const labelName = "v${{ env.OPENSEARCH_VERSION_NUMBER }}"; + let labelFound = false; + try { + const label = await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: "${{ matrix.entry.repo }}", + name: labelName + }); + labelFound = true; + } catch (error) { + if (error.status === 404) { + const randomColor = Math.floor(Math.random() * 16777215).toString(16); + const newLabel = { + owner: context.repo.owner, + repo: "${{ matrix.entry.repo }}", + name: labelName, + color: randomColor, + description: "Issues targeting release " + labelName + }; + await github.rest.issues.createLabel(newLabel); + labelFound = true; + } else { + throw error; + } + } + console.log(labelFound); + return labelFound + - name: Create Pull Request + if: ${{ matrix.entry.repo != 'OpenSearch' }} + id: cpr + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ steps.github_app_token.outputs.token }} + committer: opensearch-ci-bot + author: opensearch-ci-bot + commit-message: | + Increment version to ${{ env.OPENSEARCH_VERSION }} + + Signed-off-by: opensearch-ci-bot + delete-branch: true + branch: create-pull-request/${{ env.OPENSEARCH_VERSION }} + title: '[AUTO] Increment version to ${{ env.OPENSEARCH_VERSION }}' + labels: | + v${{ env.OPENSEARCH_VERSION_NUMBER }} + body: | + - Incremented version to **${{ env.OPENSEARCH_VERSION }}**. + - name: Check outputs + run: |- + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/os-release-issues.yml b/.github/workflows/os-release-issues.yml new file mode 100644 index 0000000000..c5b883bb21 --- /dev/null +++ b/.github/workflows/os-release-issues.yml @@ -0,0 +1,123 @@ +--- +name: release-issue-os + +on: + workflow_dispatch: + inputs: + logLevel: + description: Log level + required: true + default: warning + type: choice + options: + - info + - warning + - debug + schedule: + - cron: 0 1 * * * + +jobs: + list-manifest-versions: + if: github.repository == 'opensearch-project/opensearch-build' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + with: + repository: opensearch-project/opensearch-build + ref: main + - id: set-matrix + # produces a list of major versions, e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] + run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" + component-release-issue: + needs: list-manifest-versions + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + entry: + - {repo: OpenSearch} + - {repo: alerting} + - {repo: anomaly-detection} + - {repo: asynchronous-search} + - {repo: common-utils} + - {repo: cross-cluster-replication} + - {repo: geospatial} + - {repo: index-management} + - {repo: job-scheduler} + - {repo: k-NN} + - {repo: neural-search} + - {repo: ml-commons} + - {repo: notifications} + - {repo: observability} + - {repo: performance-analyzer} + - {repo: performance-analyzer-rca} + - {repo: reporting} + - {repo: security} + - {repo: security-analytics} + - {repo: sql} + - {repo: custom-codecs} + - {repo: flow-framework} + - {repo: skills} + - {repo: query-insights} + - {repo: opensearch-system-templates} + release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.6.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Checkout + uses: actions/checkout@v3 + - name: Check if build repo release issue exists + id: check_if_build_repo_issue_exists + uses: actions-cool/issues-helper@v3 + with: + actions: 'find-issues' + repo: opensearch-project/opensearch-build + token: ${{ steps.github_app_token.outputs.token }} + issue-state: 'open' + title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' + - name: Check out plugin repo + uses: actions/checkout@v3 + with: + path: plugin-repo + repository: opensearch-project/${{ matrix.entry.repo }} + - name: Check if plugin repo release issue exists + if: steps.check_if_build_repo_issue_exists.outputs.issues != '[]' + id: check_if_plugin_repo_issue_exists + uses: actions-cool/issues-helper@v3 + with: + actions: 'find-issues' + repo: opensearch-project/${{ matrix.entry.repo }} + token: ${{ steps.github_app_token.outputs.token }} + title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' + - name: Replace Placeholders + if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + run: | + # Read the file contents and replace the placeholders + file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" + RELEASE_VERSION="${{ matrix.release_version }}" + RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2) + BUILD_REPO_ISSUE_OUTPUT=$(cat < "$file_path.tmp" && mv "$file_path.tmp" "$file_path" + - name: Create component release issue from file + if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + uses: peter-evans/create-issue-from-file@v4 + with: + title: '[RELEASE] Release version ${{ matrix.release_version }}' + content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md + labels: | + v${{ matrix.release_version }} + token: ${{ steps.github_app_token.outputs.token }} + repository: opensearch-project/${{ matrix.entry.repo }} diff --git a/.github/workflows/osd-increment-plugin-versions.yml b/.github/workflows/osd-increment-plugin-versions.yml new file mode 100644 index 0000000000..c581f060e0 --- /dev/null +++ b/.github/workflows/osd-increment-plugin-versions.yml @@ -0,0 +1,195 @@ +--- +name: Increment OpenSearch Dashboards Plugins Version + +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: + inputs: + logLevel: + description: Log level + required: true + default: warning + type: choice + options: + - info + - warning + - debug +jobs: + plugin-version-increment-sync: + if: github.repository == 'opensearch-project/opensearch-build' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + entry: + # Adding the core repo OpenSearch-Dashboards for the label creation automation + - {repo: OpenSearch-Dashboards} + - {repo: dashboards-observability} + - {repo: dashboards-reporting} + - {repo: dashboards-visualizations} + - {repo: dashboards-query-workbench} + - {repo: dashboards-assistant} + - {repo: dashboards-maps} + - {repo: dashboards-flow-framework} + - {repo: anomaly-detection-dashboards-plugin} + - {repo: ml-commons-dashboards} + - {repo: index-management-dashboards-plugin} + - {repo: dashboards-notifications} + - {repo: alerting-dashboards-plugin} + - {repo: security-analytics-dashboards-plugin} + - {repo: security-dashboards-plugin} + - {repo: dashboards-search-relevance} + - {repo: opensearch-dashboards-functional-test} + - {repo: query-insights-dashboards} + branch: + - 2.x + - main + - '2.18' + - '2.19' + steps: + - name: Check out OpenSearch Dashboards repo + uses: actions/checkout@v3 + with: + repository: opensearch-project/OpenSearch-Dashboards + ref: ${{ matrix.branch }} + path: OpenSearch-Dashboards + - name: Check out plugin repo + if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' }} + uses: actions/checkout@v3 + with: + repository: opensearch-project/${{ matrix.entry.repo }} + ref: ${{ matrix.branch }} + path: OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }} + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version-file: './OpenSearch-Dashboards/.nvmrc' + registry-url: 'https://registry.npmjs.org' + - name: Install Yarn and Setup Dashboard Version + shell: bash + run: | + YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn") + echo "Installing yarn@$YARN_VERSION" + npm i -g yarn@$YARN_VERSION + DASHBOARD_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').version") + echo "DASHBOARD_VERSION=$DASHBOARD_VERSION" >> $GITHUB_ENV + - run: node -v + - run: yarn -v + - name: Bootstrap and Version Increment + if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' }} + run: | + cd OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }} + if [ ${{ matrix.entry.path }} ]; then + yarn osd bootstrap --single-version=loose + cp -R ${{ matrix.entry.path }} ../ + cd ../${{ matrix.entry.path }} + node ../../scripts/plugin_helpers version --sync legacy + OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version") + echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV + cd ../ + cp -R ${{ matrix.entry.path }} ${{ matrix.entry.repo }}/ + cd ${{ matrix.entry.repo }}/ + # tmp `elif` solution for opensearch-dashboards-functional-test (ref: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1801#issuecomment-1545947935) + elif [ ${{ matrix.entry.repo }} == "opensearch-dashboards-functional-test" ]; then + jq --arg DASHBOARD_VERSION "${{ env.DASHBOARD_VERSION }}" '.version = $DASHBOARD_VERSION' package.json > package-tmp.json + mv package-tmp.json package.json + OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version") + echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV + else + yarn osd bootstrap --single-version=loose + node ../../scripts/plugin_helpers version --sync legacy + OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version") + echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV + fi + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.6.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Check if label exists + id: check_label + uses: actions/github-script@v6 + with: + github-token: ${{ steps.github_app_token.outputs.token }} + result-encoding: string + script: | + const labelName = "v${{ env.DASHBOARD_VERSION }}"; + let labelFound = false; + try { + const label = await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: "${{ matrix.entry.repo }}", + name: labelName + }); + labelFound = true; + } catch (error) { + if (error.status === 404) { + const randomColor = Math.floor(Math.random() * 16777215).toString(16); + const newLabel = { + owner: context.repo.owner, + repo: "${{ matrix.entry.repo }}", + name: labelName, + color: randomColor, + description: "Issues targeting release " + labelName + }; + await github.rest.issues.createLabel(newLabel); + labelFound = true; + } else { + throw error; + } + } + console.log(labelFound); + return labelFound + - name: Create Pull Request for plugins + if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' && matrix.entry.repo + != 'opensearch-dashboards-functional-test' }} + id: cpr + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ steps.github_app_token.outputs.token }} + committer: opensearch-ci-bot + author: opensearch-ci-bot + commit-message: | + Increment version to ${{ env.OSD_PLUGIN_VERSION }} + + Signed-off-by: opensearch-ci-bot + delete-branch: true + branch: create-pull-request/${{ env.OSD_PLUGIN_VERSION }} + title: '[AUTO] Increment version to ${{ env.OSD_PLUGIN_VERSION }}' + labels: | + v${{ env.DASHBOARD_VERSION }} + body: | + - Incremented version to **${{ env.OSD_PLUGIN_VERSION }}**. + path: 'OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }}' + add-paths: | + opensearch_dashboards.json + package.json + - name: Create Pull Request for opensearch-dashboards-functional-test + if: ${{ matrix.entry.repo == 'opensearch-dashboards-functional-test' }} + id: cprft + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ steps.github_app_token.outputs.token }} + committer: opensearch-ci-bot + author: opensearch-ci-bot + commit-message: | + Increment version to ${{ env.OSD_PLUGIN_VERSION }} + + Signed-off-by: opensearch-ci-bot + delete-branch: true + branch: create-pull-request/${{ env.OSD_PLUGIN_VERSION }} + title: '[AUTO] Increment version to ${{ env.OSD_PLUGIN_VERSION }}' + labels: | + v${{ env.DASHBOARD_VERSION }} + body: | + - Incremented version to **${{ env.OSD_PLUGIN_VERSION }}**. + path: 'OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }}' + add-paths: | + package.json + - name: Check outputs + run: |- + echo "Pull Request Number - ${{ steps.cprft.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cprft.outputs.pull-request-url }}" diff --git a/.github/workflows/osd-release-issues.yml b/.github/workflows/osd-release-issues.yml new file mode 100644 index 0000000000..9a2dcbc9d2 --- /dev/null +++ b/.github/workflows/osd-release-issues.yml @@ -0,0 +1,116 @@ +--- +name: release-issue-osd + +on: + workflow_dispatch: + inputs: + logLevel: + description: Log level + required: true + default: warning + type: choice + options: + - info + - warning + - debug + schedule: + - cron: 0 1 * * * + +jobs: + list-manifest-versions: + if: github.repository == 'opensearch-project/opensearch-build' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + with: + repository: opensearch-project/opensearch-build + ref: main + - id: set-matrix + # produces a list of major versions, e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] + run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" + component-release-issue: + needs: list-manifest-versions + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + entry: + - {repo: OpenSearch-Dashboards} + - {repo: dashboards-observability} + - {repo: dashboards-reporting} + - {repo: dashboards-visualizations} + - {repo: dashboards-query-workbench} + - {repo: dashboards-assistant} + - {repo: dashboards-maps} + - {repo: dashboards-flow-framework} + - {repo: anomaly-detection-dashboards-plugin} + - {repo: ml-commons-dashboards} + - {repo: index-management-dashboards-plugin} + - {repo: dashboards-notifications} + - {repo: alerting-dashboards-plugin} + - {repo: security-analytics-dashboards-plugin} + - {repo: security-dashboards-plugin} + - {repo: dashboards-search-relevance} + - {repo: opensearch-dashboards-functional-test} + - {repo: query-insights-dashboards} + release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.6.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Checkout + uses: actions/checkout@v3 + - name: Check if build repo release issue exists + id: check_if_build_repo_issue_exists + uses: actions-cool/issues-helper@v3 + with: + actions: 'find-issues' + repo: opensearch-project/opensearch-build + token: ${{ steps.github_app_token.outputs.token }} + issue-state: 'open' + title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' + - name: Check out plugin repo + uses: actions/checkout@v3 + with: + path: plugin-repo + repository: opensearch-project/${{ matrix.entry.repo }} + - name: Check if plugin repo release issue exists + if: steps.check_if_build_repo_issue_exists.outputs.issues != '[]' + id: check_if_plugin_repo_issue_exists + uses: actions-cool/issues-helper@v3 + with: + actions: 'find-issues' + repo: opensearch-project/${{ matrix.entry.repo }} + token: ${{ steps.github_app_token.outputs.token }} + title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' + - name: Replace Placeholders + if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + run: | + # Read the file contents and replace the placeholders + file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" + RELEASE_VERSION="${{ matrix.release_version }}" + RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2) + BUILD_REPO_ISSUE_OUTPUT=$(cat < "$file_path.tmp" && mv "$file_path.tmp" "$file_path" + - name: Create component release issue from file + if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + uses: peter-evans/create-issue-from-file@v4 + with: + title: '[RELEASE] Release version ${{ matrix.release_version }}' + content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md + labels: | + v${{ matrix.release_version }} + token: ${{ steps.github_app_token.outputs.token }} + repository: opensearch-project/${{ matrix.entry.repo }} diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 0000000000..44fce89530 --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,19 @@ +--- +name: Publish a release + +on: + push: + tags: + - '*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true diff --git a/.github/workflows/publish-wiki.yml b/.github/workflows/publish-wiki.yml new file mode 100644 index 0000000000..a25e59dca6 --- /dev/null +++ b/.github/workflows/publish-wiki.yml @@ -0,0 +1,25 @@ +--- +name: Publish wiki + +on: + push: + branches: [main] + paths: + - docs/** + - .github/workflows/publish-wiki.yml + +concurrency: + group: publish-wiki + cancel-in-progress: true + +permissions: + contents: write + +jobs: + publish-wiki: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: Andrew-Chen-Wang/github-wiki-action@v4 + with: + path: docs/ diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml new file mode 100644 index 0000000000..8fa22d2cd1 --- /dev/null +++ b/.github/workflows/releases.yml @@ -0,0 +1,48 @@ +--- +name: releases + +on: + workflow_dispatch: + schedule: + - cron: 0 0 * * * + +jobs: + list-manifest-versions: + if: github.repository == 'opensearch-project/opensearch-build' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - id: set-matrix + # produces a list of versions, e.g. ["1.0.0","1.0.0","1.0.1","1.1.0","1.2.0","2.0.0"] + run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" + check: + needs: list-manifest-versions + runs-on: ubuntu-latest + strategy: + matrix: + release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} + steps: + - uses: actions/checkout@v3 + - name: Create release issue + uses: dblock/create-a-github-issue@v3.0.0 + id: release-issue + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ matrix.release_version }} + with: + search_existing: all + update_existing: false + filename: .github/ISSUE_TEMPLATE/release_template.md + - name: Create retrospective issue + uses: dblock/create-a-github-issue@v3.0.0 + if: ${{ steps.release-issue.outputs.status == 'created' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ matrix.release_version }} + RELEASE_ISSUE_URL: ${{ steps.release-issue.outputs.url }} + with: + search_existing: all + update_existing: false + filename: .github/ISSUE_TEMPLATE/release_retrospective_template.md diff --git a/.github/workflows/yaml-lint.yml b/.github/workflows/yaml-lint.yml new file mode 100644 index 0000000000..34b77e7926 --- /dev/null +++ b/.github/workflows/yaml-lint.yml @@ -0,0 +1,23 @@ +--- +name: yaml-lint + +on: [push, pull_request] + +jobs: + yaml-lint: + runs-on: ubuntu-latest + env: + PYTHON_VERSION: 3.9.12 + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install Pipenv and Dependencies + run: | + python -m pip install --upgrade pipenv wheel + pipenv install --deploy --dev + - name: Lint YAML files + run: |- + pipenv run yamllint . From 6519f2de4e9c2b59ca22d880f0592b729e846b90 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jan 2025 16:40:56 -0500 Subject: [PATCH 08/12] Separate codecov upload to its own steps Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 52b69da2cb..fe595f2d0b 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -47,7 +47,7 @@ jobs: strategy: matrix: os: - # - ubuntu-24.04-arm # arm64-preview + - ubuntu-24.04-arm # arm64-preview - ubuntu-24.04 # x64 runs-on: ${{ matrix.os }} container: @@ -77,9 +77,8 @@ jobs: run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml - uses: actions/upload-artifact@v4 - if: ${{ matrix.os }} = "ubuntu-24.04" with: - name: coverage-report + name: coverage-report-${{ matrix.os }} path: ./coverage.xml python-tests-codecov: @@ -87,13 +86,13 @@ jobs: strategy: matrix: os: - # - ubuntu-24.04-arm # arm64-preview + - ubuntu-24.04-arm # arm64-preview - ubuntu-24.04 # x64 runs-on: ${{ matrix.os }} steps: - uses: actions/download-artifact@v4 with: - name: coverage-report + name: coverage-report-${{ matrix.os }} path: ./coverage.xml - name: Upload Coverage Report uses: codecov/codecov-action@v5 From 84909f09eb6de2337e297815482263c6e1093389 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 31 Jan 2025 15:07:39 -0500 Subject: [PATCH 09/12] Only upload and push to codecov once Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index fe595f2d0b..ab84fc18d4 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -77,6 +77,7 @@ jobs: run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml - uses: actions/upload-artifact@v4 + if: ${{ matrix.os }} == "ubuntu-24.04" with: name: coverage-report-${{ matrix.os }} path: ./coverage.xml @@ -86,7 +87,6 @@ jobs: strategy: matrix: os: - - ubuntu-24.04-arm # arm64-preview - ubuntu-24.04 # x64 runs-on: ${{ matrix.os }} steps: From e066337b5ad94ce5528130ba9698c42d35dd1d0d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 31 Jan 2025 15:34:46 -0500 Subject: [PATCH 10/12] Update upload section Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index ab84fc18d4..b35feb1790 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -79,20 +79,16 @@ jobs: - uses: actions/upload-artifact@v4 if: ${{ matrix.os }} == "ubuntu-24.04" with: - name: coverage-report-${{ matrix.os }} + name: coverage-report path: ./coverage.xml python-tests-codecov: needs: python-tests-linux - strategy: - matrix: - os: - - ubuntu-24.04 # x64 - runs-on: ${{ matrix.os }} + runs-on: ubuntu-24.04 steps: - uses: actions/download-artifact@v4 with: - name: coverage-report-${{ matrix.os }} + name: coverage-report path: ./coverage.xml - name: Upload Coverage Report uses: codecov/codecov-action@v5 From d301de3ad308ec6d978df88f21e95ff50c137ebb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 31 Jan 2025 15:53:01 -0500 Subject: [PATCH 11/12] Update upload logics Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index b35feb1790..e45283090f 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -77,9 +77,8 @@ jobs: run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml - uses: actions/upload-artifact@v4 - if: ${{ matrix.os }} == "ubuntu-24.04" with: - name: coverage-report + name: coverage-report-${{ matrix.os }} path: ./coverage.xml python-tests-codecov: @@ -88,7 +87,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: coverage-report + name: coverage-report-ubuntu-24.04 path: ./coverage.xml - name: Upload Coverage Report uses: codecov/codecov-action@v5 From 0a396939ad077d19909421d475d7e69cc59ab562 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 31 Jan 2025 16:21:13 -0500 Subject: [PATCH 12/12] Update upload section Signed-off-by: Peter Zhu --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index e45283090f..9bc7208448 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -88,7 +88,7 @@ jobs: - uses: actions/download-artifact@v4 with: name: coverage-report-ubuntu-24.04 - path: ./coverage.xml + path: ./ - name: Upload Coverage Report uses: codecov/codecov-action@v5 with: