Skip to content

Commit

Permalink
Text Search bug fix
Browse files Browse the repository at this point in the history 
Signed-off-by: Varun Jain <[email protected]>
  • Loading branch information
@vibrantvarun
vibrantvarun committed Dec 20, 2023
1 parent 8c0f22d commit 26eb401
Show file tree
Hide file tree
Showing 5 changed files with 149 additions and 58 deletions.
43 changes: 0 additions & 43 deletions qa/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,29 +81,7 @@ task deletetempDirectories {
}

task pullMlCommonsBwcPlugin {
dependsOn "deletetempDirectories"

doLast {
ext{
if (isSnapshot) {
srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"
} else {
srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}"
}
}
ant.get(
src: srcUrl,
dest: tmp_dir.absolutePath,
httpusecaches: false
)
copy {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
} else {
from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
}
into tmp_dir.absolutePath
}
copy {
from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml"))
into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")
Expand Down Expand Up @@ -144,29 +122,8 @@ task pullKnnBwcPlugin {

// Task to pull neural search plugin from archive
task pullBwcPlugin {
dependsOn "deletetempDirectories"

doLast {
ext{
if (isSnapshot) {
srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"
} else {
srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}"
}
}
ant.get(
src: srcUrl,
dest: tmp_dir.absolutePath,
httpusecaches: false
)
copy {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
} else {
from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
}
into tmp_dir.absolutePath
}
copy {
from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search"))
into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search")
Expand Down
44 changes: 41 additions & 3 deletions qa/restart-upgrade/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.apache.tools.ant.taskdefs.condition.Os
import java.util.concurrent.Callable

apply from : "$rootDir/qa/build.gradle"

Expand All @@ -15,6 +16,44 @@ String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_v
String baseName = "neuralSearchBwcCluster-restart"
String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version

List<Provider<RegularFile>> plugins = [provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile()
}
}
}
}), provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile()
}
}
}
}), rootProject.tasks.bundlePlugin.archiveFile]

// task unzip {
// configurations.zipArchive.asFileTree.each {
// println(it.toPath())
// }
// }

// def tmp_dir = project.file('build/private/artifact_tmp').absoluteFile


// task unzip(type: Copy) {
// configurations.zipArchive.asFileTree.each {
// from(zipTree(it))
// }
// into tmp_dir
// }

// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion
testClusters {
"${baseName}" {
Expand Down Expand Up @@ -57,13 +96,12 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
// All nodes are upgraded to latest version and run the tests
task testRestartUpgrade(type: StandaloneRestIntegTestTask) {
dependsOn "testAgainstOldCluster"
//dependsOn rootProject.tasks.buildJniLib
dependsOn rootProject.tasks.assemble
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".environment("LD_LIBRARY_PATH", "$rootDir/jni/release")
testClusters."${baseName}".systemProperty("java.library.path", "$rootDir/jni/release")
testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion([rootProject.tasks.bundlePlugin.archiveFile])
testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster'
systemProperty 'tests.skip_delete_model_index', 'true'
Expand All @@ -73,4 +111,4 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) {
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
systemProperty 'tests.security.manager', 'false'
}
}
6 changes: 4 additions & 2 deletions qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
import com.carrotsearch.randomizedtesting.RandomizedTest;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.opensearch.client.Request;
import org.opensearch.client.Response;

import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER;

public class TextSearch extends AbstractRestartUpgradeRestTestCase{
Expand All @@ -29,10 +33,8 @@ public void testIndex() throws Exception{
Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())),
PIPELINE_NAME
);
logger.info("=================================================================================Document Added");
addDocument(testIndex, DOC_ID,TEST_FIELD,TEXT);
}else {
System.out.println("===========================================================================================Cluster Upgraded");
validateTestIndex();
}
}
Expand Down
110 changes: 100 additions & 10 deletions src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,16 @@

import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE;
import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL;
import static org.opensearch.knn.common.KNNConstants.MODELS;
import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME;
import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX;
import static org.opensearch.neuralsearch.TestUtils.OPENDISTRO_SECURITY;
import static org.opensearch.neuralsearch.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX;
import static org.opensearch.neuralsearch.TestUtils.SECURITY_AUDITLOG_PREFIX;
import static org.opensearch.neuralsearch.TestUtils.SKIP_DELETE_MODEL_INDEX;

import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import java.util.stream.Collectors;

import org.apache.hc.client5.http.auth.AuthScope;
Expand All @@ -24,22 +28,26 @@
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
import org.apache.hc.core5.ssl.SSLContextBuilder;
import org.apache.hc.core5.util.Timeout;
import org.junit.After;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Request;
import org.opensearch.client.Response;
import org.opensearch.client.RestClient;
import org.opensearch.client.RestClientBuilder;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.*;
import org.opensearch.knn.plugin.KNNPlugin;
import org.opensearch.search.SearchHit;
import org.opensearch.test.rest.OpenSearchRestTestCase;

/**
Expand All @@ -58,6 +66,12 @@ public abstract class OpenSearchSecureRestTestCase extends OpenSearchRestTestCas
private static final String INTERNAL_INDICES_PREFIX = ".";
private static String protocol;

private final Set<String> IMMUTABLE_INDEX_PREFIXES = Set.of(
NEURAL_SEARCH_BWC_PREFIX,
SECURITY_AUDITLOG_PREFIX,
OPENSEARCH_SYSTEM_INDEX_PREFIX
);

@Override
protected String getProtocol() {
if (protocol == null) {
Expand Down Expand Up @@ -148,7 +162,7 @@ protected boolean preserveIndicesUponCompletion() {
}

@After
public void deleteExternalIndices() throws IOException {
public void deleteExternalIndices() throws IOException, ParseException {
final Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json" + "&expand_wildcards=all"));
final MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType());
try (
Expand All @@ -174,8 +188,84 @@ public void deleteExternalIndices() throws IOException {
.collect(Collectors.toList());

for (final String indexName : externalIndices) {
adminClient().performRequest(new Request("DELETE", "/" + indexName));
if (isIndexCleanupRequired(indexName)) {
wipeIndexContent(indexName);
continue;
}
if (!skipDeleteIndex(indexName)) {
adminClient().performRequest(new Request("DELETE", "/" + indexName));
}
}
}
}

private boolean isIndexCleanupRequired(final String index) {
return MODEL_INDEX_NAME.equals(index) && !getSkipDeleteModelIndexFlag();
}

private void wipeIndexContent(String indexName) throws IOException, ParseException {
deleteModels(getModelIds());
deleteAllDocs(indexName);
}

private List<String> getModelIds() throws IOException, ParseException {
final String restURIGetModels = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search");
final Response response = adminClient().performRequest(new Request("GET", restURIGetModels));

assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode()));

final String responseBody = EntityUtils.toString(response.getEntity());
assertNotNull(responseBody);

final XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
final SearchResponse searchResponse = SearchResponse.fromXContent(parser);

return Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList());
}

private void deleteModels(final List<String> modelIds) throws IOException {
for (final String testModelID : modelIds) {
final String restURIGetModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID);
final Response getModelResponse = adminClient().performRequest(new Request("GET", restURIGetModel));
if (RestStatus.OK != RestStatus.fromCode(getModelResponse.getStatusLine().getStatusCode())) {
continue;
}
final String restURIDeleteModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID);
adminClient().performRequest(new Request("DELETE", restURIDeleteModel));
}
}

private void deleteAllDocs(final String indexName) throws IOException {
final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query");
final Request request = new Request("POST", restURIDeleteByQuery);
final XContentBuilder matchAllDocsQuery = XContentFactory.jsonBuilder()
.startObject()
.startObject("query")
.startObject("match_all")
.endObject()
.endObject()
.endObject();

request.setJsonEntity(matchAllDocsQuery.toString());
adminClient().performRequest(request);
}

private boolean getSkipDeleteModelIndexFlag() {
return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false"));
}

private boolean skipDeleteModelIndex(String indexName) {
return (MODEL_INDEX_NAME.equals(indexName) && getSkipDeleteModelIndexFlag());
}

private boolean skipDeleteIndex(String indexName) {
if (indexName != null
&& !OPENDISTRO_SECURITY.equals(indexName)
&& IMMUTABLE_INDEX_PREFIXES.stream().noneMatch(indexName::startsWith)
&& !skipDeleteModelIndex(indexName)) {
return false;
}

return true;
}
}
4 changes: 4 additions & 0 deletions src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ public class TestUtils {
public static final int TEST_DIMENSION = 768;
public static final SpaceType TEST_SPACE_TYPE = SpaceType.L2;
public static final String CLIENT_TIMEOUT_VALUE = "90s";
public static final String OPENDISTRO_SECURITY = ".opendistro_security";
public static final String SKIP_DELETE_MODEL_INDEX = "tests.skip_delete_model_index";
public static final String SECURITY_AUDITLOG_PREFIX = "security-auditlog";
public static final String OPENSEARCH_SYSTEM_INDEX_PREFIX = ".opensearch";

/**
* Convert an xContentBuilder to a map
Expand Down

0 comments on commit 26eb401

Please sign in to comment.