From 724bd3604ac5715126e80943ece64840cefac47d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 7 Dec 2023 15:04:12 -0800 Subject: [PATCH] Initial commit of BWC Test Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 147 ++++++++++++++++++ build.gradle | 20 +++ gradle.properties | 3 + qa/build.gradle | 114 ++++++++++++++ qa/restart-upgrade/build.gradle | 115 ++++++++++++++ .../AbstractRestartUpgradeRestTestCase.java | 62 ++++++++ .../neuralsearch/bwc/TextSearch.java | 9 ++ qa/rolling-upgrade/build.gradle | 108 +++++++++++++ .../bwc/AbstractRollingUpgradeTestCase.java | 90 +++++++++++ repositories.gradle | 11 ++ settings.gradle | 4 + .../opensearch/neuralsearch/TestUtils.java | 1 + 12 files changed, 684 insertions(+) create mode 100644 .github/workflows/backwards_compatibility_tests_workflow.yml create mode 100644 qa/build.gradle create mode 100644 qa/restart-upgrade/build.gradle create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java create mode 100644 qa/rolling-upgrade/build.gradle create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java create mode 100644 repositories.gradle diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml new file mode 100644 index 000000000..2c69dc441 --- /dev/null +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -0,0 +1,147 @@ +name: Backwards Compatibility Tests NeuralSearch +on: + push: + branches: + - "*" + - "feature/**" + pull_request: + branches: + - "*" + - "feature/**" + +jobs: + Restart-Upgrade-BWCTests-NeuralSearch: + strategy: + matrix: + java: [ 11, 17 ] + os: [ubuntu-latest] + bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0-SNAPSHOT"] + opensearch_version : [ "3.0.0-SNAPSHOT" ] + exclude: + - os: windows-latest + bwc_version: "2.0.1" + - os: windows-latest + bwc_version: "2.1.0" + - os: windows-latest + bwc_version: "2.2.1" + - os: windows-latest + bwc_version: "2.3.0" + + name: NeuralSearch Restart-Upgrade BWC Tests + runs-on: ${{ matrix.os }} + env: + BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }} + + steps: + - name: Checkout NeuralSearch + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - if: startsWith(matrix.os,'ubuntu') + name: Install dependencies on ubuntu + run: | + sudo apt-get install libopenblas-dev gfortran -y + + - if: startsWith(matrix.os,'windows') + name: Install MinGW Using Scoop on Windows + run: | + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + scoop bucket add main + scoop install mingw + + - if: startsWith(matrix.os,'windows') + name: Add MinGW to PATH on Windows + run: | + echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + refreshenv + + - if: startsWith(matrix.os,'windows') + name: Download OpenBLAS on Windows + run: | + curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip + mkdir OpenBLAS + Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ + mkdir ./src/main/resources/windowsDependencies + cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ + rm .\OpenBLAS-0.3.21-x64.zip + rm -r .\OpenBLAS\ + + - if: startsWith(matrix.os,'windows') + name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows + run: | + echo "Running restart-upgrade backwards compatibility tests ..." + ./gradlew :qa:restart-upgrade:testRestartUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' + + - if: startsWith(matrix.os,'ubuntu') + name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu + run: | + echo "Running restart-upgrade backwards compatibility tests ..." + ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE + + + Rolling-Upgrade-BWCTests-NeuralSearch: + strategy: + matrix: + java: [ 11, 17 ] + os: [ubuntu-latest] + bwc_version: [ "2.11.0-SNAPSHOT" ] + opensearch_version: [ "3.0.0-SNAPSHOT" ] + + name: NeuralSearch Rolling-Upgrade BWC Tests + runs-on: ${{ matrix.os }} + env: + BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }} + + steps: + - name: Checkout NeuralSearch + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - if: startsWith(matrix.os,'ubuntu') + name: Install dependencies on ubuntu + run: | + sudo apt-get install libopenblas-dev gfortran -y + + - if: startsWith(matrix.os,'windows') + name: Install MinGW Using Scoop on Windows + run: | + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + scoop bucket add main + scoop install mingw + + - if: startsWith(matrix.os,'windows') + name: Add MinGW to PATH on Windows + run: | + echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + refreshenv + + - if: startsWith(matrix.os,'windows') + name: Download OpenBLAS on Windows + run: | + curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip + mkdir OpenBLAS + Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ + mkdir ./src/main/resources/windowsDependencies + cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ + rm .\OpenBLAS-0.3.21-x64.zip + rm -r .\OpenBLAS\ + + - if: startsWith(matrix.os,'windows') + name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows + run: | + echo "Running rolling-upgrade backwards compatibility tests ..." + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' + + - if: startsWith(matrix.os,'ubuntu') + name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu + run: | + echo "Running rolling-upgrade backwards compatibility tests ..." + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE \ No newline at end of file diff --git a/build.gradle b/build.gradle index f2b545eca..9d9115327 100644 --- a/build.gradle +++ b/build.gradle @@ -124,6 +124,26 @@ allprojects { version = "${opensearch_build}" targetCompatibility = JavaVersion.VERSION_11 sourceCompatibility = JavaVersion.VERSION_11 + + apply from: rootProject.file('repositories.gradle').absoluteFile + plugins.withId('java') { + sourceCompatibility = targetCompatibility = "11" + } + + afterEvaluate { + project.dependencyLicenses.enabled = false + project.thirdPartyAudit.enabled = false + project.loggerUsageCheck.enabled = false + project.forbiddenApis.ignoreFailures = true + project.forbiddenPatterns { + setEnabled(false) + } + project.testingConventions.enabled = false + project.validateNebulaPom.enabled = false + project.licenseFile = rootProject.file('LICENSE.txt') + project.noticeFile = rootProject.file('NOTICE.txt') + project.forbiddenApis.ignoreFailures = true + } } repositories { diff --git a/gradle.properties b/gradle.properties index 5e5cd9ced..5d73e7e51 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # +version=2.0.0 +systemProp.bwc.version=2.10.0 + # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ diff --git a/qa/build.gradle b/qa/build.gradle new file mode 100644 index 000000000..c8342970d --- /dev/null +++ b/qa/build.gradle @@ -0,0 +1,114 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.apache.tools.ant.taskdefs.condition.Os +import java.nio.file.Files + +apply plugin: 'opensearch.testclusters' +apply plugin: 'opensearch.build' +apply plugin: 'opensearch.rest-test' +apply plugin: 'io.freefair.lombok' + +// Disable a few tasks that come with build +build.enabled = false +integTest.enabled = false +test.enabled = false +assemble.enabled = false +dependenciesInfo.enabled = false + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +dependencies { + api "org.opensearch:opensearch:${opensearch_version}" + api group: 'commons-lang', name: 'commons-lang', version: '2.6' + api "org.apache.logging.log4j:log4j-api:${versions.log4j}" + api "org.apache.logging.log4j:log4j-core:${versions.log4j}" + api "junit:junit:${versions.junit}" + testImplementation "org.opensearch.test:framework:${opensearch_version}" + testImplementation(testFixtures(rootProject)) +} + +ext { + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE.txt') +} + +def tmp_dir = project.file('build/private/artifact_tmp').absoluteFile +tmp_dir.mkdirs() +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version + +String os_platform = "linux" +String artifact_type = "tar" +String file_ext = "tar.gz" + +if (Os.isFamily(Os.FAMILY_WINDOWS)) { + os_platform = "windows" + artifact_type = "zip" + file_ext = "zip" +} + +// Task to pull neural search plugin from archive +task pullBwcPlugin { + doFirst { + File[] tempFiles = tmp_dir.listFiles() + for (File child : tempFiles) { + if (child.exists() && child.toString().contains("opensearch-")) { + Files.delete(child.toPath()); + } + } + } + + doLast { + ext{ + if (isSnapshot) { + srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" + } else { + srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}" + } + } + ant.get( + src: srcUrl, + dest: tmp_dir.absolutePath, + httpusecaches: false + ) + copy { + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } else { + from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } + into tmp_dir.absolutePath + } + copy { + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) + into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + } + delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") + } +} + +// Task to zip plugin from archive +task zipBwcPlugin(type: Zip) { + dependsOn "pullBwcPlugin" + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) + destinationDirectory = tmp_dir + archiveFileName = "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}.zip" + doLast { + delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + } +} + +task bwcTestSuite { + dependsOn ":qa:restart-upgrade:testRestartUpgrade" + dependsOn ":qa:rolling-upgrade:testRollingUpgrade" +} \ No newline at end of file diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle new file mode 100644 index 000000000..e3f8e4e93 --- /dev/null +++ b/qa/restart-upgrade/build.gradle @@ -0,0 +1,115 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.apache.tools.ant.taskdefs.condition.Os + +apply from : "$rootDir/qa/build.gradle" + +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version +String baseName = "neuralSearchBwcCluster-restart" + +// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +testClusters { + "${baseName}" { + testDistribution = "ARCHIVE" + versions = [neural_search_bwc_version, opensearch_version] + numberOfNodes = 3 + plugin(project.tasks.zipBwcPlugin.archiveFile) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' +// environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// if (Os.isFamily(Os.FAMILY_WINDOWS)) { +// // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies +// environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } else { +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } + + } +} + +// Task to run BWC tests against the old cluster +task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "zipBwcPlugin" + useCluster testClusters."${baseName}" + systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' + systemProperty 'tests.is_old_cluster', 'true' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + + // Skip test if version is 1.0 or 1.1 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) { + filter { +// excludeTestsMatching "org.opensearch.knn.bwc.ModelIT" +// excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping" +// excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping" + } + } + + // Skip test if version is 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) { + filter { + // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade" + } + } + + // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) { + filter { + // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// All nodes are upgraded to latest version and run the tests +task testRestartUpgrade(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstOldCluster" + //dependsOn rootProject.tasks.buildJniLib + dependsOn rootProject.tasks.assemble + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.is_old_cluster', 'false' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + + // Skip test if version is 1.0 or 1.1 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) { + filter { +// excludeTestsMatching "org.opensearch.knn.bwc.ModelIT" +// excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping" +// excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping" + } + } + + // Skip test if version is 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) { + filter { + //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade" + } + } + + // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) { + filter { + //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java new file mode 100644 index 000000000..aa2fbbe0b --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +import java.util.Locale; +import java.util.Optional; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; +import static org.opensearch.neuralsearch.TestUtils.CLIENT_TIMEOUT_VALUE; +import static org.opensearch.neuralsearch.TestUtils.RESTART_UPGRADE_OLD_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.BWC_VERSION; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearchIT { + protected static String testIndex; + + @Before + protected void setIndex() { + // Creating index name by concatenating "knn-bwc-" prefix with test method name + // for all the tests in this sub-project + testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + } + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) + .build(); + } + + protected static final boolean isRunningAgainstOldCluster() { + return Boolean.parseBoolean(System.getProperty(RESTART_UPGRADE_OLD_CLUSTER)); + } + + protected final Optional getBWCVersion() { + return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); + } +} \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java new file mode 100644 index 000000000..c3232815d --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +public class TextSearch { +} diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle new file mode 100644 index 000000000..863bfca9a --- /dev/null +++ b/qa/rolling-upgrade/build.gradle @@ -0,0 +1,108 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.apache.tools.ant.taskdefs.condition.Os + +apply from : "$rootDir/qa/build.gradle" + +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version +String baseName = "neuralSearchBwcCluster-rolling" + +// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +testClusters { + "${baseName}" { + testDistribution = "ARCHIVE" + versions = [neural_search_bwc_version, opensearch_version] + numberOfNodes = 3 + plugin(project.tasks.zipBwcPlugin.archiveFile) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' +// environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// if (Os.isFamily(Os.FAMILY_WINDOWS)) { +// // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies +// environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } else { +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } + } +} + + +// Task to run BWC tests against the old cluster +task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "zipBwcPlugin" + useCluster testClusters."${baseName}" + systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + systemProperty 'tests.skip_delete_model_index', 'true' + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + + +// Part of rolling upgrade. Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version +// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}" + //dependsOn rootProject.tasks.buildJniLib + dependsOn rootProject.tasks.assemble + dependsOn "testAgainstOldCluster" + doFirst { + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' + systemProperty 'tests.rest.first_round', 'true' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// Part of rolling upgrade. Upgrades the second node to new OpenSearch version with upgraded plugin version after the +// first node is upgraded. This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstOneThirdUpgradedCluster" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' + systemProperty 'tests.rest.first_round', 'false' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// Part of rolling upgrade. Upgrades the third node to new OpenSearch version with upgraded plugin version after the +// second node is upgraded. This results in a fully upgraded cluster. +task testRollingUpgrade(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstTwoThirdsUpgradedCluster" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + mustRunAfter "testAgainstOneThirdUpgradedCluster" + systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} \ No newline at end of file diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java new file mode 100644 index 000000000..089ade9f3 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +import java.util.Locale; +import java.util.Optional; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import static org.opensearch.neuralsearch.TestUtils.CLIENT_TIMEOUT_VALUE; +import static org.opensearch.neuralsearch.TestUtils.OLD_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.MIXED_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.UPGRADED_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.BWC_VERSION; +import static org.opensearch.neuralsearch.TestUtils.ROLLING_UPGRADE_FIRST_ROUND; +import static org.opensearch.neuralsearch.TestUtils.BWCSUITE_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; + +public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT { + protected String testIndex; + + @Before + protected void setIndex() { + // Creating index name by concatenating "knn-bwc-" prefix with test method name + // for all the tests in this sub-project + testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + } + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) + .build(); + } + + protected enum ClusterType { + OLD, + MIXED, + UPGRADED; + + public static ClusterType instance(String value) { + switch (value) { + case OLD_CLUSTER: + return OLD; + case MIXED_CLUSTER: + return MIXED; + case UPGRADED_CLUSTER: + return UPGRADED; + default: + throw new IllegalArgumentException("unknown cluster type: " + value); + } + } + } + + protected final ClusterType getClusterType() { + return ClusterType.instance(System.getProperty(BWCSUITE_CLUSTER)); + } + + protected final boolean isFirstMixedRound() { + return Boolean.parseBoolean(System.getProperty(ROLLING_UPGRADE_FIRST_ROUND, "false")); + } + + protected final Optional getBWCVersion() { + return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); + } + +} \ No newline at end of file diff --git a/repositories.gradle b/repositories.gradle new file mode 100644 index 000000000..0ce55eed3 --- /dev/null +++ b/repositories.gradle @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 57fb25f3c..5bfca1add 100644 --- a/settings.gradle +++ b/settings.gradle @@ -9,3 +9,7 @@ */ rootProject.name = 'neural-search' + +include ":qa" +include ":qa:rolling-upgrade" +include ":qa:restart-upgrade" \ No newline at end of file diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index 1d6c12eb4..b07744730 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -48,6 +48,7 @@ public class TestUtils { public static final String NODES_BWC_CLUSTER = "3"; public static final int TEST_DIMENSION = 768; public static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; + public static final String CLIENT_TIMEOUT_VALUE = "90s"; /** * Convert an xContentBuilder to a map