diff --git a/.github/ISSUE_TEMPLATE/component_release_template.md b/.github/ISSUE_TEMPLATE/component_release_template.md index 7b9e3b1b8e..9c04608f31 100644 --- a/.github/ISSUE_TEMPLATE/component_release_template.md +++ b/.github/ISSUE_TEMPLATE/component_release_template.md @@ -30,7 +30,7 @@ If including changes in this release, increment the version on `{{RELEASE_VERSIO ### Preparation - [ ] Assign this issue to a release owner. -- [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/1). +- [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/206). - [ ] All the tasks in this issue have been reviewed by the release owner. - [ ] Create, update, triage and label all features and issues targeted for this release with `v{{RELEASE_VERSION}}`. - [ ] Finalize the code and create the the release branch `{{RELEASE_BRANCH}}` from the `{{RELEASE_VERSION_X}}` branch. diff --git a/.github/ISSUE_TEMPLATE/release_template.md b/.github/ISSUE_TEMPLATE/release_template.md index f37703cf0f..fe526a1f61 100644 --- a/.github/ISSUE_TEMPLATE/release_template.md +++ b/.github/ISSUE_TEMPLATE/release_template.md @@ -81,11 +81,13 @@ Release blog is ready | :red_circle: | | - [ ] [Core Release Branch](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#core). - [ ] [Components Release Branch](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#components). - [ ] [Generate Release Candidate](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#release-candidate). -- [ ] [Integ Test TAR](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integ-test-tar). -- [ ] [Integ Test RPM](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integ-test-rpm). -- [ ] [Docker Build and Scan](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#docker-build-and-scan). +- [ ] [Docker Scan](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#docker-scan). +- [ ] [Integ Test](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integration-test). + - [ ] TAR + - [ ] RPM + - [ ] DEB + - [ ] Windows - [ ] [Backwards Compatibility Tests](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#backwards-compatibility-tests). -- [ ] [Windows Integration Test](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#windows-integration-test). - [ ] [Broadcast and Communication](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#broadcast-and-communication). - [ ] [Release Candidate Lock](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#release-candidate-lock). - [ ] [Stop Release Candidate periodic auto builds](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#stop-periodic-auto-builds) diff --git a/.github/workflows/os-increment-plugin-versions.yml b/.github/workflows/os-increment-plugin-versions.yml index 33933d3bd3..4b1f4a9adf 100644 --- a/.github/workflows/os-increment-plugin-versions.yml +++ b/.github/workflows/os-increment-plugin-versions.yml @@ -50,12 +50,11 @@ jobs: - {repo: skills} - {repo: query-insights} - {repo: opensearch-system-templates} + - {repo: opensearch-remote-metadata-sdk} + - {repo: opensearch-learning-to-rank-base} branch: - - 1.x - - '1.3' - 2.x - main - - '2.17' - '2.18' - '2.19' exclude: diff --git a/.github/workflows/osd-increment-plugin-versions.yml b/.github/workflows/osd-increment-plugin-versions.yml index c48fc17ffe..c581f060e0 100644 --- a/.github/workflows/osd-increment-plugin-versions.yml +++ b/.github/workflows/osd-increment-plugin-versions.yml @@ -41,12 +41,10 @@ jobs: - {repo: security-dashboards-plugin} - {repo: dashboards-search-relevance} - {repo: opensearch-dashboards-functional-test} + - {repo: query-insights-dashboards} branch: - - 1.x - - '1.3' - 2.x - main - - '2.17' - '2.18' - '2.19' steps: diff --git a/assests/distribution_build_os.png b/assests/distribution_build_os.png index 23fcb9f793..547a9fe900 100644 Binary files a/assests/distribution_build_os.png and b/assests/distribution_build_os.png differ diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index 4d1f2136f4..ae45e39048 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -67,7 +67,7 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ SHELL ["/bin/bash", "-lc"] CMD ["/bin/bash", "-l"] -# Install ruby / rpm / fpm related dependencies +# Install ruby / rpm / fpm / openssl / gcc / binutils related dependencies RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo && yum clean all ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin @@ -100,19 +100,47 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 # Upgrade gcc -RUN yum install -y gcc10* && \ - mv -v /usr/bin/gcc /usr/bin/gcc7-gcc && \ - mv -v /usr/bin/g++ /usr/bin/gcc7-g++ && \ - mv -v /usr/bin/gfortran /usr/bin/gcc7-gfortran && \ - update-alternatives --install /usr/bin/gcc gcc $(which gcc10-gcc) 1 && \ - update-alternatives --install /usr/bin/g++ g++ $(which gcc10-g++) 1 && \ - update-alternatives --install /usr/bin/gfortran gfortran $(which gcc10-gfortran) 1 +# Only x64 requires gcc 12+ for k-NN avx512_spr fp16 feature +# https://github.com/opensearch-project/opensearch-build/issues/5226 +# Due to cross-compilation being too slow on arm64, it will stay on gcc 10 for the time being +RUN if [ `uname -m` = "x86_64" ]; then \ + curl -SL https://ci.opensearch.org/ci/dbc/tools/gcc/gcc-12.4.0.tar.gz -o gcc12.tgz && \ + tar -xzf gcc12.tgz && cd gcc-12.4.0 && \ + sed -i 's@base_url=.*@base_url=https://ci.opensearch.org/ci/dbc/tools/gcc/@g' ./contrib/download_prerequisites && \ + ./contrib/download_prerequisites && \ + mkdir build && cd build && \ + ../configure --enable-languages=all --prefix=/usr --disable-multilib --disable-bootstrap && \ + make && make install && gcc --version && g++ --version && gfortran --version && \ + cd ../../ && rm -rf gcc12.tgz gcc-12.4.0; \ + else \ + yum install -y gcc10* && \ + mv -v /usr/bin/gcc /usr/bin/gcc7-gcc && \ + mv -v /usr/bin/g++ /usr/bin/gcc7-g++ && \ + mv -v /usr/bin/gfortran /usr/bin/gcc7-gfortran && \ + update-alternatives --install /usr/bin/gcc gcc $(which gcc10-gcc) 1 && \ + update-alternatives --install /usr/bin/g++ g++ $(which gcc10-g++) 1 && \ + update-alternatives --install /usr/bin/gfortran gfortran $(which gcc10-gfortran) 1; \ + fi + +# Upgrade binutils +# This is only required if gcc upgrade to 12 or above +RUN if [ `uname -m` = "x86_64" ]; then \ + yum install -y texinfo && \ + curl -SLO https://ci.opensearch.org/ci/dbc/tools/gcc/binutils-2.42.90.tar.xz && \ + tar -xf binutils-2.42.90.tar.xz && cd binutils-2.42.90 && \ + mkdir build && cd build && \ + ../configure --prefix=/usr && \ + make && make install && ld --version && \ + cd ../../ && rm -rf binutils-2.42.90.tar.xz binutils-2.42.90 && \ + yum remove -y texinfo; \ + fi + ENV FC=gfortran ENV CXX=g++ # Add k-NN Library dependencies -RUN yum repolist && yum install lapack -y -RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git && \ +RUN yum repolist && yum install lapack -y && yum clean all && rm -rf /var/cache/yum/* +RUN git clone -b v0.3.27 --single-branch https://github.com/OpenMathLib/OpenBLAS.git && \ cd OpenBLAS && \ if [ "$(uname -m)" = "x86_64" ]; then \ echo "Machine is x86_64. Adding DYNAMIC_ARCH=1 to openblas make command."; \ @@ -120,7 +148,8 @@ RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git else \ make USE_OPENMP=1 FC=gfortran; \ fi && \ - make PREFIX=/usr/local install + make PREFIX=/usr/local install && \ + cd ../ && rm -rf OpenBLAS ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" RUN pip3 install cmake==3.26.4 @@ -131,19 +160,22 @@ RUN pip3 install cmake==3.26.4 # GitHub enforce nodejs 20 official build in runner 2.317.0 of their actions and CentOS7/AL2 would fail due to having older glibc versions # Until https://github.com/actions/runner/pull/3128 is merged or AL2 is deprecated (2025/06) this is a quick fix with unofficial builds support glibc 2.17 # With changes done similar to this PR (https://github.com/opensearch-project/job-scheduler/pull/702) alongside the image here -# Only linux x64 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged +# Only linux x64 glibc217 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged for pre-compiled arm64 binaries +# The linux arm64 glibc226 tarball here is directly compiled from the source code on AL2 host for the time being RUN if [ `uname -m` = "x86_64" ]; then \ - curl -SL https://unofficial-builds.nodejs.org/download/release/v20.10.0/node-v20.10.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \ - mkdir /node_al2; \ - tar -xf /node20.tar.xz --strip-components 1 -C /node_al2; \ - rm -v /node20.tar.xz; \ - fi + curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \ + else \ + curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-arm64-glibc-226.tar.xz -o /node20.tar.xz; \ + fi; \ + mkdir /node_al2 && \ + tar -xf /node20.tar.xz --strip-components 1 -C /node_al2 && \ + rm -v /node20.tar.xz # Change User USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME # Install fpm for opensearch dashboards core -RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2 +RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0 && gem install fpm -v 1.14.2 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH RUN fpm -v diff --git a/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile b/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile index a8ca5ed47d..c0eb3bed31 100644 --- a/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile +++ b/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile @@ -58,7 +58,7 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ SHELL ["/bin/bash", "-lc"] # Install ruby / rpm / fpm related dependencies -RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && gem install rchardet -v 1.8.0 && dnf install -y rpm-build createrepo && dnf clean all ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin ENV RVM_HOME=/usr/local/rvm/bin diff --git a/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile b/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile index 4648f784f9..62d9aa915c 100644 --- a/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile +++ b/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile @@ -102,6 +102,6 @@ USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME # Install fpm for opensearch dashboards core -RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2 +RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0&& gem install fpm -v 1.14.2 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH RUN fpm -v diff --git a/docs/Releasing-the-Distribution.md b/docs/Releasing-the-Distribution.md index 3766e84ce1..74dcd4419f 100644 --- a/docs/Releasing-the-Distribution.md +++ b/docs/Releasing-the-Distribution.md @@ -133,21 +133,23 @@ This section is not required for a patch release. The component release issues are auto created by the workflows part of the build repo [OpenSearch components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/os-release-issues.yml), [OpenSearch Dashboards components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/osd-release-issues.yml). These workflows create the release issues based on the template [component_release_template.md](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md) and links back the global release issue part of the build. Sample [component release issue](https://github.com/opensearch-project/job-scheduler/issues/642) created for 2.16.0 release. The release manager must verify that each component release issue has an assigned owner, as this is one of the entry criteria for the release. -#### Issue Creation Process Overview - -Inside the template [component_release_template.md](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md), replace the fields `RELEASE_VERSION`, `RELEASE_BRANCH_X`, `RELEASE_BRANCH` and `RELEASE_ISSUE` to desired release values before creating the release issues across the component/plugin repos. Once the fields are replaced, use the `meta` and `gh` cli to create the issues. Find the list of components/plugins from the [opensearch-plugins](https://github.com/opensearch-project/opensearch-plugins) repo (for [OpenSearch](https://github.com/opensearch-project/opensearch-plugins/tree/main/plugins), for [OpenSearch Dashboards](https://github.com/opensearch-project/opensearch-plugins/tree/main/dashboards-plugins)) and use the `meta` cli to create the release issues. For more details check the [create-an-issue-in-all-plugin-repos](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#create-an-issue-in-all-plugin-repos) section. +
Manual release issue creation process +

+Inside the template [component_release_template.md](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md), replace the fields `RELEASE_VERSION`, `RELEASE_BRANCH_X`, `RELEASE_BRANCH` and `RELEASE_ISSUE` to desired release values before creating the release issues across the component/plugin repos. Once the fields are replaced, use the `meta` and `gh` cli to create the issues. Find the list of components/plugins from the [opensearch-plugins](https://github.com/opensearch-project/opensearch-plugins) repo (for [OpenSearch](https://github.com/opensearch-project/opensearch-plugins/tree/main/plugins), for [OpenSearch Dashboards](https://github.com/opensearch-project/opensearch-plugins/tree/main/dashboards-plugins)) and use the `meta` cli to create the release issues. For more details check the [create-an-issue-in-all-plugin-repos](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#create-an-issue-in-all-plugin-repos) section. ``` meta exec "gh issue create --label v2.8.0 --title 'Release version 2.8.0' --body-file /tmp/opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" ``` +

+
### Release Campaigns -If exists any release specific issues/campaigns, link it back to the release issue. Sample linked [issues/campaigns](https://github.com/opensearch-project/opensearch-build/issues/3434#issuecomment-1552138916) +If any release specific issues/campaigns exists, link it back to the release issue. Sample linked [issues/campaigns](https://github.com/opensearch-project/opensearch-build/issues/3434#issuecomment-1552138916) ## Release Branch Readiness -The `Release Branch Readiness date` is determined as the release date minus 4 days. +The `Release Branch Readiness date` is determined as the release date minus 6 days. However, after receiving some feedback from the maintainers as to avoid multiple backports of the Pull Requests, it is recommended to create release branch (if one does not exist) on the day of first Release Candidate generation. The release branch creation (if one does not exist) is [automated](#other-release-related-workflows). ### Release Branch @@ -155,11 +157,11 @@ This release branch creation is not applicable for patch release. #### Core -This step requires both OpenSearch and OpenSearch Dashboards to create a release branch that will be used for the release. +This step requires both OpenSearch and OpenSearch Dashboards to create a release branch that will be used for the release. If not created already, [release-branch-creation workflow](https://build.ci.opensearch.org/view/Release/job/release-branch-creation/) will create one based of `major.x` branch. #### Components -This step requires that every team participating in a release has their release branch created for the corresponding release by the date listed on this step. The [Distribution Build](#distribution-build) workflow will also start using the release branch to create release candidate instead of `.x` branches. +This step requires that every team participating in a release has their release branch created for the corresponding release by the date listed on this step. If not created already, [release-branch-creation workflow](https://build.ci.opensearch.org/view/Release/job/release-branch-creation/) will create one based of `major.x` branch. The [Distribution Build](#distribution-build) workflow will also start using the release branch to create release candidate instead of `major.x` branches. ### Version Increment @@ -171,7 +173,7 @@ Versions are incremented as soon as development starts on a given version to avo #### Core Version Increment -To ensure the version incrementation process is handled correctly, it is important to follow to increment the version of the release branch. Currently, the version incrementation is being done manually by an individual from the core repositories. Sample OpenSearch Version Increment [PR](https://github.com/opensearch-project/OpenSearch/pull/7864/) for the release branch. +To ensure the version incrementation process is handled correctly, it is important to increment the version of the release branch. Currently, the version incrementation is being done manually by an individual from the core repositories. Sample OpenSearch Version Increment [PR](https://github.com/opensearch-project/OpenSearch/pull/7864/). #### Components Version Increment @@ -183,7 +185,7 @@ The objective is to merge these pull requests in order to synchronize all the co ## Code Complete and Feature Freeze -Coordinate with the Core and component teams to ensure that the code for this particular release version is fully prepared and that the corresponding branch has been included in the release version [Input Manifest](#input-manifest). Update Jenkins workflows that execute daily snapshot builds for both OpenSearch and OpenSearch Dashboards (Ref [Increase the build frequency](#increase-the-build-frequency)). Submit pull requests to incorporate each component into the respective version level [Input Manifest](#input-manifest) along with the necessary checks. Sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3501/files). The `Feature Freeze date` and the `Code Complete date` is determined as the release date minus 4 days. +Coordinate with the Core and component teams to ensure that the code for this particular release version is fully prepared and that the corresponding branch has been included in the release version [Input Manifest](#input-manifest). Update Jenkins workflows that execute daily snapshot builds for both OpenSearch and OpenSearch Dashboards (Ref [Increase the build frequency](#increase-the-build-frequency)). For new components on-boarding to the distribution, please submit pull requests to incorporate the component into the [Input Manifest](#input-manifest) along with the necessary checks. Sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3501/files). The `Feature Freeze date` and the `Code Complete date` is determined as the release date minus 14 days. ## Release Candidate Creation and Testing @@ -197,27 +199,50 @@ Ensure the proper inputs are used to initiate the distribution. For instance, he ![Alt Text](https://github.com/opensearch-project/opensearch-build/blob/main/assests/distribution_build_os.png) -**COMPONENT_NAME**: To trigger a specific component, this includes standalone OpenSearch or specific plugin.
+
Parameters info +

+ +**COMPONENT_NAME**: \ To trigger a specific component, this includes standalone OpenSearch or specific plugin.
+ +**INPUT_MANIFEST**: \ The release input manifest that drives the workflow.
+ +**TEST_MANIFEST**: \ The release test input manifest that is used for the integ tests.
-**INPUT_MANIFEST**: The release input manifest that drives the workflow.
+**INTEG_TEST_JOB_NAME**: \ The integ test job name. Default already added to this input `integ-test` for OpenSearch and `integ-test-opensearch-dashboards` for OpenSearch Dashboards.
-**TEST_MANIFEST**: The release test input manifest that is used for the integ tests.
+**BWC_TEST_JOB_NAME**: \ The BWC test job name. Default already added to this input `bwc-test` for OpenSearch and `bwc-test-opensearch-dashboards` for OpenSearch Dashboards.
-**INTEG_TEST_JOB_NAME**: The integ test job name. Default already added to this input `integ-test` for OpenSearch and `integ-test-opensearch-dashboards` for OpenSearch Dashboards.
+**BUILD_PLATFORM**: \ Platforms to build. Example linux, windows.
-**BUILD_PLATFORM**: The input used to build for a specific platform, followed by its own distributions within the `platform`.
+**BUILD_DISTRIBUTION**: \ Distribution to build. Choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions separated by space. In order to build Docker distribution (option specified later), tar needs to be built.
-**BUILD_DISTRIBUTION**: Input to build selected distribution related artifacts, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar).
+**TEST_PLATFORM**: \ Platforms to tests. Choices include 'linux', 'windows'. Can combine multiple platforms with space in between.
+ +**TEST_DISTRIBUTION**: \ Distributions to tests. Choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between.
+ +**RC_NUMBER**: \ The RC build count. Default is 0 which means its not a Release Candidate.
**BUILD_DOCKER**: Input with a dropdown that has 3 options `build_docker`, `build_docker_with_build_number_tag`, `do_not_build_docker`, the release manager has to take a call with right inputs.
-**UPDATE_LATEST_URL**: To update the `/latest` CFN URL, Visit [latest-distribution-url](https://github.com/opensearch-project/opensearch-build#latest-distribution-url) for more details.
+**UPDATE_LATEST_URL**: To update the `/latest` CFN URL, Visit [latest-distribution-url](https://github.com/opensearch-project/opensearch-build/wiki/Building-an-OpenSearch-and-OpenSearch-Dashboards-Distribution#latest-distributions-buildsl) for more details.
+ +**UPDATE_GITHUB_ISSUE**: To create/close/update a github issue for all component or not.
+ +**CONTINUE_ON_ERROR**: Continues to build the distribution even if one of the non-core or non-essential plugins fails to build.
+ +**INCREMENTAL**: Builds the distribution incrementally when a previous build is provided.
+ +**PREVIOUS_BUILD_ID**: \ Previous Ditribution Build ID to download the artifacts from.
+

+
+ +In case of failure, go to the workflow run on [Jenkins](https://build.ci.opensearch.org/view/Build/). For easy debugging and visibility, click on `Open Blue Ocean` from the left panel. This should show exactly which stage failed indicated by :x . For stages marked with as `Unstable` (:warning), it indicates that some components failed to build but a distribution was built with the successfully built components. Click on any unstable stage and then click on `./build.sh ...` to see more logs. ##### Order of Execution Following is the order of execution of the distribution build to address the components dependencies. -Note: The execution order specified is necessary only for versions up to `1.3.x`. For `2.x` and above Maven dependencies are published through each component repository using the GH workflow. For more details check the [META issue](https://github.com/opensearch-project/opensearch-build/issues/3185). +Note: The execution order specified is necessary only for versions up to `1.3.x`. For `2.x` and above the snapshots Maven dependencies are published through each component repository using the GH workflow. For more details check the [META issue](https://github.com/opensearch-project/opensearch-build/issues/3185). ###### OpenSearch ``` @@ -237,9 +262,15 @@ All components (which are ready after completion of version increment) ### Release Candidate -Now once all the version increment PRs are completed and all the components are part of the input manifest, now it's time to generate the RC. Use the [Distribution Build](#distribution-build) to generate the release candidate. Use the following section as the reference to generate the RC, validate it and broadcast it for a given release. The process of `Release Candidate Generation and Testing` should commence at least 6 days prior to the release date. +Once all the version increment PRs are merged and all the components are part of the input manifest, now it is time to generate the Release Candidate (RC). Use the [Distribution Build](#distribution-build) to generate the release candidate. Use the following section as the reference to generate the RC, validate it and broadcast it for a given release. The process of `Release Candidate Generation and Testing` should commence at least 6 days prior to the release date. + +#### What sets Release Candidate different than regular builds? +Well, in terms of build, nothing. However, while triggering the build please be careful to take care of the following [workflow-triggers](#workflow-trigger) along with the other parameters. -#### Sample Build details +**RC_NUMBER**: Starts with `1` and keeps on incrementing as we build more RCs through out the release process. +**BUILD_DOCKER**: Select `build_docker_with_build_number_tag`. This ensures that [check-for-build.jenkinsfile](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/check-for-build.jenkinsfile) won't re-build periodically and override the docker image, the RC docker is created with build number. Example: [opensearchstaging/opensearch:2.18.0.10454](https://hub.docker.com/layers/opensearchstaging/opensearch/2.18.0.10454/images/sha256-7755492728968e6f7afbd20014f3e13d0d765a8690e4bc80cb2403460999b0cc) +**INCREMENTAL**: Recommend to disable it for first RC and let it build from scratch. Moving forward, it can be enabled for future builds. +**PREVIOUS_BUILD_ID**: Build number of previous Release Candidate. Applicable only when **INCREMENTAL** parameter is enabled. Following is the generated build number after triggering the [Distribution Build](#distribution-build) workflow. The distribution build number denotes the RC, now with the example below the finalized RC’s are `OS: 7848`, `OSD: 6126`. @@ -247,29 +278,45 @@ Following is the generated build number after triggering the [Distribution Build |----------|----------| | [build_7848](https://build.ci.opensearch.org/job/distribution-build-opensearch/7848/console) | [build_6126](https://build.ci.opensearch.org/job/distribution-build-opensearch-dashboards/6126/) | -#### Docker Build and Scan +
What if? +

+ +- What if I forgot to choose `build_docker_with_build_number_tag` while building RC? + If the docker image was build regulary overriding the default `major.minor.patch` tag, you can simply copy the image to `major.minor.patch.build_id` tag using [docker-copy](https://build.ci.opensearch.org/job/docker-copy/) workflow. + | Docker Freeze | copy | + |----------|----------| + | OpenSearch | [docker-copy](https://build.ci.opensearch.org/job/docker-copy/466/console) | + | OpenSearch Dashboards | [docker-copy](https://build.ci.opensearch.org/job/docker-copy/467/console) | -Following are the details for the docker image build and scan. The docker images are built using the TAR artifact generated as part of the [Distribution Build](#distribution-build) (From the above example `OS: 7848`, `OSD: 6126`). The [Distribution Build](#distribution-build) workflow with input `BUILD_DOCKER` (Ref [Workflow Trigger](#workflow-trigger)) triggers the [docker-build] workflow as downstream. +

+
+ +#### Docker Scan + +Following are the details for the docker image build and scan. The docker images are built using the TAR artifact generated as part of the [Distribution Build](#distribution-build) (From the above example `OS: 7848`, `OSD: 6126`). The [Distribution Build](#distribution-build) workflow with input `BUILD_DOCKER` or `build_docker_with_build_number_tag` (Ref [Workflow Trigger](#workflow-trigger)) triggers the [docker-build](https://build.ci.opensearch.org/job/docker-build/) workflow as downstream which also triggers [docker-scan](https://build.ci.opensearch.org/job/docker-scan/). Docker scan is responsible for parsing the docker image for possible vulnerabilities. It uses [trivy](https://github.com/aquasecurity/trivy) to scan the docker images. | Docker | build | scan | |----------|----------|----------| | OpenSearch | [Build](https://build.ci.opensearch.org/job/docker-build/3371/) | [Scan](https://build.ci.opensearch.org/job/docker-scan/1558/artifact/scan_docker_image.txt) | | OpenSearch Dashboards | [Build](https://build.ci.opensearch.org/job/docker-build/3370/) | [Scan](https://build.ci.opensearch.org/job/docker-scan/1557/artifact/scan_docker_image.txt) | +#### Integration Tests -##### Docker RC Freeze +For more details on running integration tests, refer [wiki](https://github.com/opensearch-project/opensearch-build/wiki/Testing-the-Distribution#integration-tests). As one of the exit criteria, it is important to have all the integration tests passing for all the platforms. See [build-workflow](#build-workflows) for details on integration test workflow for OpenSearch and OpenSearch Dashboards. As a part of build-workflow if the parameters (TEST_MANIFEST, INTEG_TEST_JOB_NAME, TEST_PLATFORM, TEST_DISTRIBUTION) are provided during the release candidate build, integration tests will be triggered for all distributions. -This to ensure that [check-for-build.jenkinsfile](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/check-for-build.jenkinsfile) won't re-build periodically and override the docker, the RC docker is created with build number. This step can be skipped if the input `BUILD_DOCKER: build_docker_with_build_number_tag` (Ref [Workflow Trigger](#workflow-trigger) used in the [Distribution Build](#distribution-build)). +Failing integration tests create GitHub issues with all the details included in the respective repository. [Sample issue](https://github.com/opensearch-project/security-analytics/issues/1451) -| Docker Freeze | copy | -|----------|----------| -| OpenSearch | [docker-copy](https://build.ci.opensearch.org/job/docker-copy/466/console) | -| OpenSearch Dashboards | [docker-copy](https://build.ci.opensearch.org/job/docker-copy/467/console) | +
What if? +

-#### Benchmark Tests +- What if I missed one of the integration test parameter while triggering RC? + Integration test workflows can be triggered and run independently irrespective of build workflows. Feel free to trigger the respective workflow. The results of the integration tests for per component can be viewed in pipeline view as well as on the [metrics portal](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/21aad140-49f6-11ef-bbdd-39a9b324a5aa?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-90d,to:now))&_a=(description:'OpenSearch%20Release%20Build%20and%20Integration%20Test%20Results',filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,useMargins:!t),query:(language:kuery,query:''),timeRestore:!t,title:'OpenSearch%20Release%20Build%20and%20Integration%20Test%20Results',viewMode:view)). -For running the benchmark tests, use the `benchmark-test` job part of the [Build Workflows](#build-workflows). For more details in running the benchmark tests refer [Benchmarking Tests](https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#benchmarking-tests) section part of the [test workflow](https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow). This job offers multiple options to test the performance of a specific version cluster using various metrics. The benchmark performance results can be accessed via the [OpenSearch Performance Benchmarks dashboard](https://opensearch.org/benchmarks). Sample benchmark tests for [2.9.0 release](https://github.com/opensearch-project/opensearch-build/issues/3616#issuecomment-1642764515). +- What is I want to re-run the integration test? Is there a limit to how many times the workflow can be triggered? + There is no limit as such. The tests for the given component(s) can be run any number of times. However, please be mindful of the resources consumption used to run these tests. For flaky integration test, please create an issue in the respective component repository for it to be addressed. +

+
#### Backwards Compatibility Tests @@ -282,29 +329,24 @@ On board the components/plugins to the test [Test Manifest](#test-manifest) with - with-security ``` -#### Windows Integration Test - -Currently, the windows integration tests for a release is manual. The manually tested windows zip is being evaluated and approved by the plugin teams for sign off. In order to test the windows distribution, two instances need to be created: one with security features enabled and another without security. Afterward, API calls should be tested by launching the OpenSearch and Dashboard processes through direct execution of the `.bat` file. - #### Broadcast and Communication -Broadcast the release candidate in OpenSearch public slack workspace and the release GitHub issue using format [sample broadcast message](https://github.com/opensearch-project/opensearch-build/issues/3434#issuecomment-1571201919) to gather votes. +Broadcast the release candidate in OpenSearch public slack workspace (releases channel) and the GitHub release issue using format [sample broadcast message](https://github.com/opensearch-project/opensearch-build/issues/3434#issuecomment-1571201919) to gather votes. As a release manager, it is essential to ensure the successful completion of all the above mentioned jobs. In the event of failures during integration tests or scans, the release manager should collaborate with the component teams and initiate a re-run to ensure that all jobs are executed successfully. Post all the job related failures in the `Release issue`, Sample [post](https://github.com/opensearch-project/opensearch-build/issues/3331#issuecomment-1550461519). -Note: Sometimes the integ-test jobs are flaky and might not pass due to several reasons with the component code, in that case coordinate with the respective component team and get a manual sign off. Sample [manual sign off](https://github.com/opensearch-project/opensearch-build/issues/3331#issuecomment-1552191673). -All the failed logs are in s3 accessed through the cloudfront. Sample [link](https://github.com/opensearch-project/opensearch-build/issues/3331#issuecomment-1552148546). - - #### Release Candidate Lock -Stop builds for this version of OpenSearch and/or OpenSearch Dashboards in order to avoid accidental commits going in unknowingly. Restart only if necessary, else manually run the build workflow and declare new release candidate. +Stop builds for this version of OpenSearch and/or OpenSearch Dashboards in order to avoid accidental commits going in unknowingly. This is done by locking the commits in the [input manifest](#input-manifest). Use [release-manifest-commit-lock workflow](https://build.ci.opensearch.org/job/release-manifest-commit-lock/) with appropriate action. #### Stop Periodic Auto Builds Once the RC is finalized, in order to exclude the release from running periodically, at this point it is necessary for the release manager to lock the input manifest and update the `check-for-build.jenkins` to remove it from the scheduled execution, sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3523/files). +#### Benchmark Tests + +For running the benchmark tests, use the `benchmark-test` job part of the [Build Workflows](#build-workflows). For more details in running the benchmark tests refer [Benchmarking Tests](https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#benchmarking-tests) section part of the [test workflow](https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow). This job offers multiple options to test the performance of a specific version cluster using various metrics. The benchmark performance results can be accessed via the [OpenSearch Performance Benchmarks dashboard](https://opensearch.org/benchmarks). Sample benchmark tests for [2.9.0 release](https://github.com/opensearch-project/opensearch-build/issues/3616#issuecomment-1642764515). ## Release diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index 9acac6a468..2af130c662 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -23,12 +23,11 @@ pipeline { } triggers { parameterizedCron ''' - H 1 * * * %INPUT_MANIFEST=1.3.21/opensearch-1.3.21.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=1.3.21/opensearch-1.3.21-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar H 1 * * * %INPUT_MANIFEST=2.19.0/opensearch-dashboards-2.19.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.19.0/opensearch-dashboards-2.19.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar - H 1 * * * %INPUT_MANIFEST=2.18.1/opensearch-2.18.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.18.1/opensearch-2.18.1-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar H 1 * * * %INPUT_MANIFEST=2.19.0/opensearch-2.19.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.19.0/opensearch-2.19.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar - H 4 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 4 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.18.1/opensearch-2.18.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 4 * * * %INPUT_MANIFEST=3.0.0-alpha1/opensearch-3.0.0-alpha1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 4 * * * %INPUT_MANIFEST=3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip ''' } parameters { diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 9d79e594ff..fa82860d87 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -29,61 +29,61 @@ pipeline { parameters { string( name: 'COMPONENT_NAME', - description: 'If this field contains one or more component names (e.g. OpenSearch-Dashboards reportsDashboards ...), will build with "--component ...", else build everything in the INPUT_MANIFEST.', + description: ' If this field contains one or more component names (e.g. OpenSearch-Dashboards reportsDashboards ...), will build with "--component ...", else build everything in the INPUT_MANIFEST.', trim: true ) string( name: 'INPUT_MANIFEST', - description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0.yml.', + description: ' Input manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0.yml.', trim: true ) string( name: 'TEST_MANIFEST', - description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.', + description: ' Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.', trim: true ) string( name: 'INTEG_TEST_JOB_NAME', - description: 'Name of integration test job that will be triggered, e.g. Playground/integ-test-opensearch-dashboards. A non-null empty value here will skip integration tests.', + description: ' Name of integration test job that will be triggered, e.g. Playground/integ-test-opensearch-dashboards. A non-null empty value here will skip integration tests.', defaultValue: 'integ-test-opensearch-dashboards', trim: true ) string( name: 'BWC_TEST_JOB_NAME', - description: 'Name of backwards compatibility test job that will be triggered, e.g. Playground/bwc-test-opensearch-dashboards. A non-null empty value here will skip BWC tests.', + description: ' Name of backwards compatibility test job that will be triggered, e.g. Playground/bwc-test-opensearch-dashboards. A non-null empty value here will skip BWC tests.', defaultValue: 'bwc-test-opensearch-dashboards', trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'BUILD_PLATFORM', - description: "Build selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)", + description: " Build selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)", defaultValue: 'linux windows', trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new distribution(s) name: 'BUILD_DISTRIBUTION', - description: "Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", + description: " Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", defaultValue: 'tar rpm deb zip', trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'TEST_PLATFORM', - description: "Test selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)", + description: " Test selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)", trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new distribution(s) name: 'TEST_DISTRIBUTION', - description: "Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", + description: " Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", trim: true ) string( name: 'RC_NUMBER', - description: 'The RC build count. Default is 0 which means its not an RC build.', + description: ' The RC build count. Default is 0 which means its not a Release Candidate.', defaultValue: '0' ) choice( name: 'BUILD_DOCKER', - description: 'Build docker image or not with options.', + description: ' Build docker image or not with options.', choices: ['build_docker', 'build_docker_with_build_number_tag', 'do_not_build_docker'], ) booleanParam( diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index f82ffd995a..03b6aeeba6 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -28,61 +28,61 @@ pipeline { parameters { string( name: 'COMPONENT_NAME', - description: 'If this field contains one or more component names (e.g. OpenSearch common-utils ...), will build with "--component ...", else build everything in the INPUT_MANIFEST.', + description: ' If this field contains one or more component names (e.g. OpenSearch common-utils ...), will build with "--component ...", else build everything in the INPUT_MANIFEST.', trim: true ) string( name: 'INPUT_MANIFEST', - description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.', + description: ' Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.', trim: true ) string( name: 'TEST_MANIFEST', - description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', + description: ' Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', trim: true ) string( name: 'INTEG_TEST_JOB_NAME', - description: 'Name of integration test job that will be triggered, e.g. Playground/integ-test. A non-null empty value here will skip integration tests.', + description: ' Name of integration test job that will be triggered, e.g. Playground/integ-test. A non-null empty value here will skip integration tests.', defaultValue: 'integ-test', trim: true ) string( name: 'BWC_TEST_JOB_NAME', - description: 'Name of backwards compatibility test job that will be triggered, e.g. Playground/bwc-test. A non-null empty value here will skip BWC tests.', + description: ' Name of backwards compatibility test job that will be triggered, e.g. Playground/bwc-test. A non-null empty value here will skip BWC tests.', defaultValue: 'bwc-test', trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'BUILD_PLATFORM', - description: "Build selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)", + description: " Build selected platform, choices include 'linux', 'windows'. Can combine multiple platforms separated by space (docker is only available on linux)", defaultValue: 'linux windows', trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new distribution(s) name: 'BUILD_DISTRIBUTION', - description: "Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", + description: " Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions separated by space (docker is only available on tar)", defaultValue: 'tar rpm deb zip', trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'TEST_PLATFORM', - description: "Test selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)", + description: " Test selected platform, choices include 'linux', 'windows'. Can combine multiple platforms separated by space", trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new distribution(s) name: 'TEST_DISTRIBUTION', - description: "Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", + description: " Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions separated by space", trim: true ) string( name: 'RC_NUMBER', - description: 'The RC build count. Default is 0 which means its not an RC build.', + description: ' The RC build count. Default is 0 which means its not a Release Candidate.', defaultValue: '0' ) choice( name: 'BUILD_DOCKER', - description: 'Build docker image or not with options.', + description: ' Build docker image or not with options.', choices: ['build_docker', 'build_docker_with_build_number_tag', 'do_not_build_docker'], ) booleanParam( diff --git a/manifests/2.19.0/opensearch-2.19.0-test.yml b/manifests/2.19.0/opensearch-2.19.0-test.yml index 6083a0e9fc..81e9ee8e47 100644 --- a/manifests/2.19.0/opensearch-2.19.0-test.yml +++ b/manifests/2.19.0/opensearch-2.19.0-test.yml @@ -82,6 +82,11 @@ components: test-configs: - with-security - without-security + - name: opensearch-learning-to-rank-base + integ-test: + test-configs: + - with-security + - without-security - name: neural-search integ-test: test-configs: diff --git a/manifests/2.19.0/opensearch-dashboards-2.19.0-test.yml b/manifests/2.19.0/opensearch-dashboards-2.19.0-test.yml index d1a4216bd2..f5b041c241 100644 --- a/manifests/2.19.0/opensearch-dashboards-2.19.0-test.yml +++ b/manifests/2.19.0/opensearch-dashboards-2.19.0-test.yml @@ -3,7 +3,7 @@ schema-version: '1.0' name: OpenSearch Dashboards ci: image: - name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4 + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 components: - name: OpenSearch-Dashboards integ-test: diff --git a/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml new file mode 100644 index 0000000000..6c263bc206 --- /dev/null +++ b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml @@ -0,0 +1,159 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-23 +components: + - name: opensearch + smoke-test: + test-spec: opensearch.yml + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: + - 10.0.0.0/8 + - 127.0.0.1 + bwc-test: + test-configs: + - with-security + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: flow-framework + integ-test: + test-configs: + - with-security + - without-security + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 2 + - cluster_name: follower + data_nodes: 2 + test-configs: + - with-security + - without-security + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: + - /tmp + bwc-test: + test-configs: + - with-security + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + - name: opensearch-learning-to-rank-base + integ-test: + test-configs: + - with-security + - without-security + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + - name: security + integ-test: + test-configs: + - with-security + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5 + bwc-test: + test-configs: + - with-security + - name: custom-codecs + integ-test: + test-configs: + - with-security + - without-security + - name: skills + integ-test: + test-configs: + - with-security + - without-security + - name: query-insights + integ-test: + test-configs: + - with-security + - without-security + - name: opensearch-system-templates + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.application_templates.enabled: true + cluster.application_templates.enabled: true diff --git a/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml new file mode 100644 index 0000000000..8caac8b456 --- /dev/null +++ b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml @@ -0,0 +1,213 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 3.0.0 + qualifier: alpha1 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-23 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: main + platforms: + - linux + - windows + - name: opensearch-learning-to-rank-base + repository: https://github.com/opensearch-project/opensearch-learning-to-rank-base.git + ref: main + platforms: + - linux + - windows + - name: opensearch-remote-metadata-sdk + repository: https://github.com/opensearch-project/opensearch-remote-metadata-sdk.git + ref: main + platforms: + - linux + - windows + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: main + platforms: + - linux + - windows + - name: security + repository: https://github.com/opensearch-project/security.git + ref: main + platforms: + - linux + - windows + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: main + platforms: + - linux + - windows + - name: geospatial + repository: https://github.com/opensearch-project/geospatial.git + ref: main + platforms: + - linux + - windows + depends_on: + - job-scheduler + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: neural-search + repository: https://github.com/opensearch-project/neural-search.git + ref: main + platforms: + - linux + - windows + depends_on: + - ml-commons + - k-NN + - name: notifications-core + repository: https://github.com/opensearch-project/notifications.git + ref: main + working_directory: notifications + platforms: + - linux + - windows + depends_on: + - common-utils + - name: notifications + repository: https://github.com/opensearch-project/notifications.git + ref: main + working_directory: notifications + platforms: + - linux + - windows + depends_on: + - common-utils + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: opensearch-reports + repository: https://github.com/opensearch-project/reporting.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: main + platforms: + - linux + - windows + depends_on: + - ml-commons + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: security-analytics + repository: https://github.com/opensearch-project/security-analytics.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - alerting + - job-scheduler + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: performance-analyzer + repository: https://github.com/opensearch-project/performance-analyzer.git + ref: main + platforms: + - linux + - name: custom-codecs + repository: https://github.com/opensearch-project/custom-codecs.git + ref: main + platforms: + - linux + - windows + - name: flow-framework + repository: https://github.com/opensearch-project/flow-framework.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - opensearch-remote-metadata-sdk + - name: skills + repository: https://github.com/opensearch-project/skills.git + ref: main + platforms: + - linux + - windows + depends_on: + - job-scheduler + - anomaly-detection + - sql + - ml-commons + - name: query-insights + repository: https://github.com/opensearch-project/query-insights.git + ref: main + platforms: + - linux + - windows + - name: opensearch-system-templates + repository: https://github.com/opensearch-project/opensearch-system-templates.git + ref: main + platforms: + - linux + - windows diff --git a/manifests/3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1-test.yml b/manifests/3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1-test.yml new file mode 100644 index 0000000000..f5b041c241 --- /dev/null +++ b/manifests/3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1-test.yml @@ -0,0 +1,95 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: alertingDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: anomalyDetectionDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: ganttChartDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: observabilityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: queryWorkbenchDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: reportsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: notificationsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: customImportMapDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: searchRelevanceDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityAnalyticsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: mlCommonsDashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + ml_commons_dashboards.enabled: true + - name: assistantDashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + assistant.chat.enabled: true diff --git a/manifests/3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1.yml b/manifests/3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1.yml new file mode 100644 index 0000000000..27ac7780c5 --- /dev/null +++ b/manifests/3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1.yml @@ -0,0 +1,64 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch Dashboards + version: 3.0.0 + qualifier: alpha1 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-build-v1 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: main + - name: observabilityDashboards + repository: https://github.com/opensearch-project/dashboards-observability.git + ref: main + - name: reportsDashboards + repository: https://github.com/opensearch-project/dashboards-reporting.git + ref: main + - name: ganttChartDashboards + repository: https://github.com/opensearch-project/dashboards-visualizations.git + ref: main + - name: queryWorkbenchDashboards + repository: https://github.com/opensearch-project/dashboards-query-workbench.git + ref: main + - name: customImportMapDashboards + repository: https://github.com/opensearch-project/dashboards-maps.git + ref: main + - name: anomalyDetectionDashboards + repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin.git + ref: main + - name: mlCommonsDashboards + repository: https://github.com/opensearch-project/ml-commons-dashboards.git + ref: main + - name: indexManagementDashboards + repository: https://github.com/opensearch-project/index-management-dashboards-plugin.git + ref: main + - name: notificationsDashboards + repository: https://github.com/opensearch-project/dashboards-notifications.git + ref: main + - name: alertingDashboards + repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git + ref: main + - name: securityAnalyticsDashboards + repository: https://github.com/opensearch-project/security-analytics-dashboards-plugin.git + ref: main + - name: securityDashboards + repository: https://github.com/opensearch-project/security-dashboards-plugin.git + ref: main + - name: searchRelevanceDashboards + repository: https://github.com/opensearch-project/dashboards-search-relevance.git + ref: main + - name: assistantDashboards + repository: https://github.com/opensearch-project/dashboards-assistant.git + ref: main + - name: flowFrameworkDashboards + repository: https://github.com/opensearch-project/dashboards-flow-framework.git + ref: main + - name: queryInsightsDashboards + repository: https://github.com/opensearch-project/query-insights-dashboards.git + ref: main diff --git a/manifests/3.0.0/PLEASE_UPDATE_3.0.0-alpha1_MANIFESTS_THANKS.txt b/manifests/3.0.0/PLEASE_UPDATE_3.0.0-alpha1_MANIFESTS_THANKS.txt new file mode 100644 index 0000000000..8caa7b6b34 --- /dev/null +++ b/manifests/3.0.0/PLEASE_UPDATE_3.0.0-alpha1_MANIFESTS_THANKS.txt @@ -0,0 +1 @@ +PLEASE_UPDATE_3.0.0-alpha1_MANIFESTS_THANKS diff --git a/manifests/3.0.0/opensearch-3.0.0.yml b/manifests/3.0.0/opensearch-3.0.0.yml index cc0934b43d..19818093df 100644 --- a/manifests/3.0.0/opensearch-3.0.0.yml +++ b/manifests/3.0.0/opensearch-3.0.0.yml @@ -10,7 +10,7 @@ ci: components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: main + ref: '3.0.0-before-alpha1' checks: - gradle:publish - gradle:properties:version diff --git a/manifests/3.0.0/opensearch-dashboards-3.0.0.yml b/manifests/3.0.0/opensearch-dashboards-3.0.0.yml index 1e7391b6ea..bab530e5ce 100644 --- a/manifests/3.0.0/opensearch-dashboards-3.0.0.yml +++ b/manifests/3.0.0/opensearch-dashboards-3.0.0.yml @@ -58,7 +58,7 @@ components: - name: queryInsightsDashboards repository: https://github.com/opensearch-project/query-insights-dashboards.git ref: main - # Commenting SA until https://github.com/opensearch-project/security-analytics-dashboards-plugin/issues/1185 is resolved - # - name: securityAnalyticsDashboards - # repository: https://github.com/opensearch-project/security-analytics-dashboards-plugin.git - # ref: main +# Commenting SA until https://github.com/opensearch-project/security-analytics-dashboards-plugin/issues/1185 is resolved +# - name: securityAnalyticsDashboards +# repository: https://github.com/opensearch-project/security-analytics-dashboards-plugin.git +# ref: main