Merge remote-tracking branch 'upstream/main' into update-arm64-runner

opensearch-project · Jan 24, 2025 · 8faaf88 · 8faaf88
2 parents 7009d99 + 30d39eb
commit 8faaf88
Show file tree

Hide file tree

Showing 21 changed files with 711 additions and 102 deletions.
diff --git a/.github/ISSUE_TEMPLATE/component_release_template.md b/.github/ISSUE_TEMPLATE/component_release_template.md
@@ -30,7 +30,7 @@ If including changes in this release, increment the version on `{{RELEASE_VERSIO
 ### Preparation
 
 - [ ] Assign this issue to a release owner.
-- [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/1).
+- [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/206).
 - [ ] All the tasks in this issue have been reviewed by the release owner.
 - [ ] Create, update, triage and label all features and issues targeted for this release with `v{{RELEASE_VERSION}}`.
 - [ ] Finalize the code and create the the release branch `{{RELEASE_BRANCH}}` from the `{{RELEASE_VERSION_X}}` branch.

diff --git a/.github/ISSUE_TEMPLATE/release_template.md b/.github/ISSUE_TEMPLATE/release_template.md
@@ -81,11 +81,13 @@ Release blog is ready | :red_circle: |   |
 - [ ] [Core Release Branch](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#core).
 - [ ] [Components Release Branch](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#components).
 - [ ] [Generate Release Candidate](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#release-candidate).
-- [ ] [Integ Test TAR](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integ-test-tar).
-- [ ] [Integ Test RPM](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integ-test-rpm).
-- [ ] [Docker Build and Scan](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#docker-build-and-scan).
+- [ ] [Docker Scan](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#docker-scan).
+- [ ] [Integ Test](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integration-test).
+   - [ ] TAR
+   - [ ] RPM
+   - [ ] DEB
+   - [ ] Windows
 - [ ] [Backwards Compatibility Tests](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#backwards-compatibility-tests).
-- [ ] [Windows Integration Test](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#windows-integration-test).
 - [ ] [Broadcast and Communication](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#broadcast-and-communication).
 - [ ] [Release Candidate Lock](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#release-candidate-lock).
 - [ ] [Stop Release Candidate periodic auto builds](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#stop-periodic-auto-builds)

diff --git a/.github/workflows/os-increment-plugin-versions.yml b/.github/workflows/os-increment-plugin-versions.yml
@@ -50,12 +50,11 @@ jobs:
           - {repo: skills}
           - {repo: query-insights}
           - {repo: opensearch-system-templates}
+          - {repo: opensearch-remote-metadata-sdk}
+          - {repo: opensearch-learning-to-rank-base}
         branch:
-          - 1.x
-          - '1.3'
           - 2.x
           - main
-          - '2.17'
           - '2.18'
           - '2.19'
         exclude:

diff --git a/.github/workflows/osd-increment-plugin-versions.yml b/.github/workflows/osd-increment-plugin-versions.yml
@@ -41,12 +41,10 @@ jobs:
           - {repo: security-dashboards-plugin}
           - {repo: dashboards-search-relevance}
           - {repo: opensearch-dashboards-functional-test}
+          - {repo: query-insights-dashboards}
         branch:
-          - 1.x
-          - '1.3'
           - 2.x
           - main
-          - '2.17'
           - '2.18'
           - '2.19'
     steps:

diff --git a/assests/distribution_build_os.png b/assests/distribution_build_os.png
diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile
@@ -67,7 +67,7 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \
 SHELL ["/bin/bash", "-lc"]
 CMD ["/bin/bash", "-l"]
 
-# Install ruby / rpm / fpm related dependencies
+# Install ruby / rpm / fpm / openssl / gcc / binutils related dependencies
 RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo && yum clean all
 
 ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
@@ -100,27 +100,56 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
     pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17
 
 # Upgrade gcc
-RUN yum install -y gcc10* && \
-    mv -v /usr/bin/gcc /usr/bin/gcc7-gcc && \
-    mv -v /usr/bin/g++ /usr/bin/gcc7-g++ && \
-    mv -v /usr/bin/gfortran /usr/bin/gcc7-gfortran && \
-    update-alternatives --install /usr/bin/gcc gcc $(which gcc10-gcc) 1 && \
-    update-alternatives --install /usr/bin/g++ g++ $(which gcc10-g++) 1 && \
-    update-alternatives --install /usr/bin/gfortran gfortran $(which gcc10-gfortran) 1
+# Only x64 requires gcc 12+ for k-NN avx512_spr fp16 feature
+# https://github.com/opensearch-project/opensearch-build/issues/5226
+# Due to cross-compilation being too slow on arm64, it will stay on gcc 10 for the time being
+RUN if [ `uname -m` = "x86_64" ]; then \
+        curl -SL https://ci.opensearch.org/ci/dbc/tools/gcc/gcc-12.4.0.tar.gz -o gcc12.tgz && \
+        tar -xzf gcc12.tgz && cd gcc-12.4.0 && \
+        sed -i 's@base_url=.*@base_url=https://ci.opensearch.org/ci/dbc/tools/gcc/@g' ./contrib/download_prerequisites && \
+        ./contrib/download_prerequisites && \
+        mkdir build && cd build && \
+        ../configure --enable-languages=all --prefix=/usr --disable-multilib --disable-bootstrap && \
+        make && make install && gcc --version && g++ --version && gfortran --version && \
+        cd  ../../ && rm -rf gcc12.tgz gcc-12.4.0; \
+    else \
+        yum install -y gcc10* && \
+        mv -v /usr/bin/gcc /usr/bin/gcc7-gcc && \
+        mv -v /usr/bin/g++ /usr/bin/gcc7-g++ && \
+        mv -v /usr/bin/gfortran /usr/bin/gcc7-gfortran && \
+        update-alternatives --install /usr/bin/gcc gcc $(which gcc10-gcc) 1 && \
+        update-alternatives --install /usr/bin/g++ g++ $(which gcc10-g++) 1 && \
+        update-alternatives --install /usr/bin/gfortran gfortran $(which gcc10-gfortran) 1; \
+    fi
+
+# Upgrade binutils
+# This is only required if gcc upgrade to 12 or above
+RUN if [ `uname -m` = "x86_64" ]; then \
+        yum install -y texinfo && \
+        curl -SLO https://ci.opensearch.org/ci/dbc/tools/gcc/binutils-2.42.90.tar.xz && \
+        tar -xf binutils-2.42.90.tar.xz && cd binutils-2.42.90 && \
+        mkdir build && cd build && \
+        ../configure --prefix=/usr && \
+        make && make install && ld --version && \
+        cd ../../ && rm -rf binutils-2.42.90.tar.xz binutils-2.42.90 && \
+        yum remove -y texinfo; \
+    fi
+
 ENV FC=gfortran
 ENV CXX=g++
 
 # Add k-NN Library dependencies
-RUN yum repolist && yum install lapack -y
-RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git && \
+RUN yum repolist && yum install lapack -y && yum clean all && rm -rf /var/cache/yum/*
+RUN git clone -b v0.3.27 --single-branch https://github.com/OpenMathLib/OpenBLAS.git && \
     cd OpenBLAS && \
     if [ "$(uname -m)" = "x86_64" ]; then \
         echo "Machine is x86_64. Adding DYNAMIC_ARCH=1 to openblas make command."; \
         make USE_OPENMP=1 FC=gfortran DYNAMIC_ARCH=1; \
     else \
         make USE_OPENMP=1 FC=gfortran; \
     fi && \
-    make PREFIX=/usr/local install
+    make PREFIX=/usr/local install && \
+    cd ../ && rm -rf OpenBLAS
 ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
 RUN pip3 install cmake==3.26.4
 
@@ -131,19 +160,22 @@ RUN pip3 install cmake==3.26.4
 # GitHub enforce nodejs 20 official build in runner 2.317.0 of their actions and CentOS7/AL2 would fail due to having older glibc versions
 # Until https://github.com/actions/runner/pull/3128 is merged or AL2 is deprecated (2025/06) this is a quick fix with unofficial builds support glibc 2.17
 # With changes done similar to this PR (https://github.com/opensearch-project/job-scheduler/pull/702) alongside the image here
-# Only linux x64 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged
+# Only linux x64 glibc217 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged for pre-compiled arm64 binaries
+# The linux arm64 glibc226 tarball here is directly compiled from the source code on AL2 host for the time being
 RUN if [ `uname -m` = "x86_64" ]; then \
-        curl -SL https://unofficial-builds.nodejs.org/download/release/v20.10.0/node-v20.10.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \
-        mkdir /node_al2; \
-        tar -xf /node20.tar.xz --strip-components 1 -C /node_al2; \
-        rm -v /node20.tar.xz; \
-    fi
+        curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \
+    else \
+        curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-arm64-glibc-226.tar.xz -o /node20.tar.xz; \
+    fi; \
+    mkdir /node_al2 && \
+    tar -xf /node20.tar.xz --strip-components 1 -C /node_al2 && \
+    rm -v /node20.tar.xz
 
 # Change User
 USER $CONTAINER_USER
 WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
-RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2
+RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0 && gem install fpm -v 1.14.2
 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 RUN fpm -v
diff --git a/...i/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile b/...i/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile
@@ -58,7 +58,7 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \
 SHELL ["/bin/bash", "-lc"]
 
 # Install ruby / rpm / fpm related dependencies
-RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all
+RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && gem install rchardet -v 1.8.0 && dnf install -y rpm-build createrepo && dnf clean all
 
 ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
 ENV RVM_HOME=/usr/local/rvm/bin

diff --git a/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile b/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile
@@ -102,6 +102,6 @@ USER $CONTAINER_USER
 WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
-RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2
+RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0&& gem install fpm -v 1.14.2
 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 RUN fpm -v