most of the XContent related updates have been applied; code compiles…

…, but there are integ tests that need to be fixed Signed-off-by: Mark Cohen <[email protected]>
opensearch-project · Dec 28, 2023 · ff90402 · ff90402
1 parent 099aebf
commit ff90402
Show file tree

Hide file tree

Showing 60 changed files with 222 additions and 193 deletions.
diff --git a/build.gradle b/build.gradle
@@ -112,6 +112,7 @@ allprojects {
 test {
   include '**/*Tests.class'
   finalizedBy jacocoTestReport
+  systemProperty 'tests.security.manager', 'false'
 }
 
 task integTest(type: RestIntegTestTask) {
@@ -131,6 +132,7 @@ integTest {
   if (System.getProperty("test.debug") != null) {
     jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
   }
+  systemProperty 'tests.security.manager', 'false'
 }
 
 testClusters.integTest {

diff --git a/src/javaRestTest/java/com/o19s/es/ltr/NodeSettingsIT.java b/src/javaRestTest/java/com/o19s/es/ltr/NodeSettingsIT.java
@@ -24,8 +24,8 @@
 import com.o19s.es.ltr.ranker.LtrRanker;
 import org.apache.lucene.util.Accountable;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
 
 import java.io.IOException;
 

diff --git a/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java b/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java
@@ -107,7 +107,7 @@ public <E extends StorableElement> E getElement(Class<E> clazz, String type, Str
     }
 
     public <E extends StorableElement> E getElement(Class<E> clazz, String type, String name, String store) throws IOException {
-        return new IndexFeatureStore(store, this::client, parserFactory()).getAndParse(name, clazz, type);
+        return new IndexFeatureStore(store, this::client, parserFactory(), client().threadPool()).getAndParse(name, clazz, type);
     }
 
     protected LtrRankerParserFactory parserFactory() {
@@ -118,7 +118,7 @@ public FeatureStoreResponse addElement(StorableElement element,
                                            @Nullable FeatureValidation validation,
                                            String store) throws ExecutionException, InterruptedException {
         FeatureStoreRequestBuilder builder =
-            new FeatureStoreRequestBuilder(client(), FeatureStoreAction.INSTANCE);
+                new FeatureStoreRequestBuilder(client(), FeatureStoreAction.INSTANCE);
         builder.request().setStorableElement(element);
         builder.request().setAction(FeatureStoreAction.FeatureStoreRequest.Action.CREATE);
         builder.request().setStore(store);
@@ -167,11 +167,11 @@ public <FactoryType> FactoryType compile(String scriptName, String scriptSource,
                                 + "]");
                     }
                     // we use the script "source" as the script identifier
-                    ScoreScript.Factory factory = (p, lookup) ->
+                    ScoreScript.Factory factory = (p, lookup, searcher) ->
                             new ScoreScript.LeafFactory() {
                                 @Override
                                 public ScoreScript newInstance(LeafReaderContext ctx) throws IOException {
-                                    return new ScoreScript(p, lookup, ctx) {
+                                    return new ScoreScript(p, lookup, searcher, ctx) {
                                         @Override
                                         public double execute(ExplanationHolder explainationHolder) {
                                             // For testing purposes just look for the "terms" key and see if stats were injected
@@ -239,7 +239,7 @@ public <FactoryType> FactoryType compile(String scriptName, String scriptSource,
                     }
                     // we use the script "source" as the script identifier
                     if (FEATURE_EXTRACTOR.equals(scriptSource)) {
-                        ScoreScript.Factory factory = (p, lookup) ->
+                        ScoreScript.Factory factory = (p, lookup, searcher) ->
                                 new ScoreScript.LeafFactory() {
                                     final Map<String, Float> featureSupplier;
                                     final String dependentFeature;
@@ -267,7 +267,7 @@ public <FactoryType> FactoryType compile(String scriptName, String scriptSource,
 
                                     @Override
                                     public ScoreScript newInstance(LeafReaderContext ctx) throws IOException {
-                                        return new ScoreScript(p, lookup, ctx) {
+                                        return new ScoreScript(p, lookup, searcher, ctx) {
                                             @Override
                                             public double execute(ExplanationHolder explainationHolder ) {
                                                 return extraMultiplier == 0.0d ?
@@ -286,7 +286,7 @@ public boolean needs_score() {
                         return context.factoryClazz.cast(factory);
                     }
                     else if (scriptSource.equals(FEATURE_EXTRACTOR + "_extra_logging")) {
-                        ScoreScript.Factory factory = (p, lookup) ->
+                        ScoreScript.Factory factory = (p, lookup, searcher) ->
                                 new ScoreScript.LeafFactory() {
                                     {
                                         if (!p.containsKey(FEATURE_VECTOR)) {
@@ -299,7 +299,7 @@ else if (scriptSource.equals(FEATURE_EXTRACTOR + "_extra_logging")) {
 
                                     @Override
                                     public ScoreScript newInstance(LeafReaderContext ctx) throws IOException {
-                                        return new ScoreScript(p, lookup, ctx) {
+                                        return new ScoreScript(p, lookup, searcher, ctx) {
 
                                             @Override
                                             public double execute(ExplanationHolder explanation) {
@@ -332,4 +332,4 @@ public Set<ScriptContext<?>> getSupportedContexts() {
             };
         }
     }
-}
+}
diff --git a/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java b/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java
@@ -17,10 +17,10 @@
 
 import org.apache.lucene.search.Query;
 import org.opensearch.core.ParseField;
-import org.opensearch.common.ParsingException;
-import org.opensearch.common.io.stream.NamedWriteable;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.core.common.ParsingException;
+import org.opensearch.core.common.io.stream.NamedWriteable;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ObjectParser;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;

diff --git a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java
@@ -69,15 +69,15 @@
 import org.apache.lucene.analysis.miscellaneous.LengthFilter;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
 import org.opensearch.action.ActionRequest;
-import org.opensearch.action.ActionResponse;
+import org.opensearch.core.action.ActionResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.core.ParseField;
-import org.opensearch.common.io.stream.NamedWriteableRegistry;
-import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry.Entry;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Setting;
@@ -87,7 +87,7 @@
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
-import org.opensearch.index.Index;
+import org.opensearch.core.index.Index;
 import org.opensearch.index.analysis.PreConfiguredTokenFilter;
 import org.opensearch.index.analysis.PreConfiguredTokenizer;
 import org.opensearch.plugins.ActionPlugin;
@@ -272,7 +272,7 @@ private LTRStats getStats(Client client, ClusterService clusterService, IndexNam
 
     protected FeatureStoreLoader getFeatureStoreLoader() {
         return (storeName, clientSupplier) ->
-            new CachedFeatureStore(new IndexFeatureStore(storeName, clientSupplier, parserFactory), caches);
+            new CachedFeatureStore(new IndexFeatureStore(storeName, clientSupplier, parserFactory, null), caches);
     }
 
     // A simplified version of some token filters needed by the feature stores.

diff --git a/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java b/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java
@@ -22,16 +22,16 @@
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.ActionRequestValidationException;
-import org.opensearch.action.ActionResponse;
+import org.opensearch.core.action.ActionResponse;
 import org.opensearch.action.ActionType;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.client.OpenSearchClient;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
-import org.opensearch.common.io.stream.Writeable.Reader;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable.Reader;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.rest.RestStatus;
+import org.opensearch.core.rest.RestStatus;
 
 import java.io.IOException;
 import java.util.List;

diff --git a/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java b/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java
@@ -27,9 +27,9 @@
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
-import org.opensearch.common.io.stream.Writeable;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
 

diff --git a/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java b/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java
@@ -27,13 +27,13 @@
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
-import org.opensearch.common.io.stream.Writeable.Reader;
+import org.opensearch.core.common.io.stream.Writeable.Reader;
 
 import static org.opensearch.action.ValidateActions.addValidationError;
 

diff --git a/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java b/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java
@@ -22,15 +22,15 @@
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.ActionRequestValidationException;
-import org.opensearch.action.ActionResponse;
+import org.opensearch.core.action.ActionResponse;
 import org.opensearch.action.ActionType;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.client.OpenSearchClient;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.rest.RestStatus;
+import org.opensearch.core.rest.RestStatus;
 
 import java.io.IOException;
 

diff --git a/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java b/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java
@@ -23,16 +23,16 @@
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.ActionRequestValidationException;
-import org.opensearch.action.ActionResponse;
+import org.opensearch.core.action.ActionResponse;
 import org.opensearch.action.ActionType;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.client.OpenSearchClient;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
-import org.opensearch.common.io.stream.Writeable.Reader;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable.Reader;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.rest.RestStatus;
+import org.opensearch.core.rest.RestStatus;
 
 import java.io.IOException;
 import java.util.Objects;

diff --git a/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java b/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java
@@ -26,8 +26,8 @@
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;

diff --git a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java
@@ -20,14 +20,14 @@
 import com.o19s.es.ltr.feature.store.index.IndexFeatureStore;
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.ActionRequestValidationException;
-import org.opensearch.action.ActionResponse;
+import org.opensearch.core.action.ActionResponse;
 import org.opensearch.action.ActionType;
 import org.opensearch.action.support.master.MasterNodeReadRequest;
 import org.opensearch.client.OpenSearchClient;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
-import org.opensearch.common.io.stream.Writeable;
-import org.opensearch.common.io.stream.Writeable.Reader;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.io.stream.Writeable.Reader;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;

diff --git a/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java b/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java
@@ -24,7 +24,7 @@
 import com.o19s.es.ltr.feature.store.StoredFeature;
 import com.o19s.es.ltr.feature.store.StoredFeatureSet;
 import com.o19s.es.ltr.feature.store.index.IndexFeatureStore;
-import org.opensearch.action.ActionListener;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.get.GetRequest;
 import org.opensearch.action.get.GetResponse;
 import org.opensearch.action.get.TransportGetAction;
@@ -52,7 +52,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.opensearch.action.ActionListener.wrap;
+import static org.opensearch.core.action.ActionListener.wrap;
 
 public class TransportAddFeatureToSetAction extends HandledTransportAction<AddFeaturesToSetRequest, AddFeaturesToSetResponse> {
     private final ClusterService clusterService;

diff --git a/src/main/java/com/o19s/es/ltr/action/TransportCacheStatsAction.java b/src/main/java/com/o19s/es/ltr/action/TransportCacheStatsAction.java
@@ -27,7 +27,7 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
-import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;

diff --git a/src/main/java/com/o19s/es/ltr/action/TransportClearCachesAction.java b/src/main/java/com/o19s/es/ltr/action/TransportClearCachesAction.java
@@ -27,8 +27,8 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;

diff --git a/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java b/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java
@@ -23,7 +23,7 @@
 import com.o19s.es.ltr.feature.store.StoredFeatureSet;
 import com.o19s.es.ltr.feature.store.StoredLtrModel;
 import com.o19s.es.ltr.feature.store.index.IndexFeatureStore;
-import org.opensearch.action.ActionListener;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.get.GetRequest;
 import org.opensearch.action.get.GetResponse;
 import org.opensearch.action.get.TransportGetAction;

diff --git a/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java b/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java
@@ -29,7 +29,7 @@
 import com.o19s.es.ltr.ranker.parser.LtrRankerParserFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.action.ActionListener;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.index.IndexAction;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.search.SearchAction;
@@ -47,7 +47,7 @@
 import java.io.IOException;
 import java.util.Optional;
 
-import static org.opensearch.action.ActionListener.wrap;
+import static org.opensearch.core.action.ActionListener.wrap;
 
 public class TransportFeatureStoreAction extends HandledTransportAction<FeatureStoreRequest, FeatureStoreResponse> {
     private final LtrRankerParserFactory factory;

diff --git a/src/main/java/com/o19s/es/ltr/action/TransportLTRStatsAction.java b/src/main/java/com/o19s/es/ltr/action/TransportLTRStatsAction.java
@@ -26,7 +26,7 @@
 import org.opensearch.action.support.nodes.TransportNodesAction;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
-import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 

diff --git a/src/main/java/com/o19s/es/ltr/action/TransportListStoresAction.java b/src/main/java/com/o19s/es/ltr/action/TransportListStoresAction.java
@@ -19,7 +19,7 @@
 import com.o19s.es.ltr.action.ListStoresAction.ListStoresActionRequest;
 import com.o19s.es.ltr.action.ListStoresAction.ListStoresActionResponse;
 import com.o19s.es.ltr.feature.store.index.IndexFeatureStore;
-import org.opensearch.action.ActionListener;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
 import org.opensearch.action.search.MultiSearchRequestBuilder;
 import org.opensearch.action.search.MultiSearchResponse;
@@ -34,7 +34,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.inject.Inject;
-import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.aggregations.AggregationBuilders;
@@ -54,7 +54,7 @@
 
 import static com.o19s.es.ltr.feature.store.index.IndexFeatureStore.STORE_VERSION_PROP;
 import static java.util.stream.Collectors.toMap;
-import static org.opensearch.action.ActionListener.wrap;
+import static org.opensearch.core.action.ActionListener.wrap;
 import static org.opensearch.common.collect.Tuple.tuple;
 
 public class TransportListStoresAction extends TransportMasterNodeReadAction<ListStoresActionRequest, ListStoresActionResponse> {

diff --git a/src/main/java/com/o19s/es/ltr/feature/FeatureValidation.java b/src/main/java/com/o19s/es/ltr/feature/FeatureValidation.java
@@ -17,9 +17,9 @@
 package com.o19s.es.ltr.feature;
 
 import org.opensearch.core.ParseField;
-import org.opensearch.common.io.stream.StreamInput;
-import org.opensearch.common.io.stream.StreamOutput;
-import org.opensearch.common.io.stream.Writeable;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.ObjectParser;
 import org.opensearch.core.xcontent.ToXContentObject;