Merge pull request #30613 from vespa-engine/balder/cleanup-backend-co…

…nstruction

Balder/cleanup backend construction

container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java

@@ -12,7 +12,6 @@
 import com.yahoo.prelude.fastsearch.ClusterParams;
 import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
 import com.yahoo.prelude.fastsearch.IndexedBackend;
-import com.yahoo.prelude.fastsearch.SummaryParameters;
 import com.yahoo.prelude.fastsearch.VespaBackend;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
@@ -92,23 +91,19 @@ public ClusterSearcher(ComponentId id,
         maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT);
         maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT);
 
-        SummaryParameters docSumParams = new SummaryParameters(qrsConfig
-                .com().yahoo().prelude().fastsearch().IndexedBackend().docsum()
-                .defaultclass());
-
-        String uniqueServerId = UUID.randomUUID().toString();
         VespaBackend streaming = null, indexed = null;
+        ClusterParams clusterParams = makeClusterParams(searchClusterIndex, qrsConfig
+                .com().yahoo().prelude().fastsearch().IndexedBackend().docsum()
+                .defaultclass(), documentDbConfig, schemaInfo);
         for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) {
             if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX) {
                 if (indexed == null) {
-                    indexed = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId,
-                                             docSumParams, documentDbConfig, schemaInfo, dispatchers);
+                    indexed = searchDispatch(clusterParams, searchClusterName, dispatchers);
                 }
                 schema2Searcher.put(docDb.name(), indexed);
             } else if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING) {
                 if (streaming == null) {
-                    streaming = streamingCluster(uniqueServerId, searchClusterIndex,
-                            searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access);
+                    streaming = streamingCluster(clusterParams, searchClusterConfig, access);
                     vipStatus.addToRotation(streaming.getName());
                 }
                 schema2Searcher.put(docDb.name(), streaming);
@@ -126,42 +121,30 @@ private static QrSearchersConfig.Searchcluster getSearchClusterConfigFromCluster
                 config.searchcluster().stream().map(QrSearchersConfig.Searchcluster::name).toList());
     }
 
-    private static ClusterParams makeClusterParams(int searchclusterIndex) {
-        return new ClusterParams("sc" + searchclusterIndex + ".num" + 0);
+    private static ClusterParams makeClusterParams(int searchclusterIndex, String defaultSummary,
+                                                   DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo)
+    {
+        return new ClusterParams("sc" + searchclusterIndex + ".num" + 0, UUID.randomUUID().toString(),
+                                 defaultSummary, documentDbConfig, schemaInfo);
     }
 
-    private static IndexedBackend searchDispatch(int searchclusterIndex,
+    private static IndexedBackend searchDispatch(ClusterParams clusterParams,
                                                  String searchClusterName,
-                                                 String serverId,
-                                                 SummaryParameters docSumParams,
-                                                 DocumentdbInfoConfig documentdbInfoConfig,
-                                                 SchemaInfo schemaInfo,
-                                                 ComponentRegistry<Dispatcher> dispatchers) {
-        ClusterParams clusterParams = makeClusterParams(searchclusterIndex);
+                                                 ComponentRegistry<Dispatcher> dispatchers)
+    {
         ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName);
         Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId);
         if (dispatcher == null)
-            throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId +
-                                               " is configured");
-        return new IndexedBackend(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo);
+            throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + " is configured");
+        return new IndexedBackend(clusterParams, dispatcher);
     }
 
-    private static StreamingBackend streamingCluster(String serverId,
-                                                     int searchclusterIndex,
+    private static StreamingBackend streamingCluster(ClusterParams clusterParams,
                                                      QrSearchersConfig.Searchcluster searchClusterConfig,
-                                                     SummaryParameters docSumParams,
-                                                     DocumentdbInfoConfig documentdbInfoConfig,
-                                                     SchemaInfo schemaInfo,
-                                                     VespaDocumentAccess access) {
-        if (searchClusterConfig.searchdef().size() != 1)
-            throw new IllegalArgumentException("Streaming search clusters can only contain a single schema but got " +
-                                               searchClusterConfig.searchdef());
-        ClusterParams clusterParams = makeClusterParams(searchclusterIndex);
-        StreamingBackend searcher = new StreamingBackend(access);
-        searcher.setSearchClusterName(searchClusterConfig.rankprofiles_configid());
-        searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec());
-        searcher.init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo);
-        return searcher;
+                                                     VespaDocumentAccess access)
+    {
+        return new StreamingBackend(clusterParams, searchClusterConfig.rankprofiles_configid(),
+                                    access, searchClusterConfig.storagecluster().routespec());
     }
 
     /** Do not use, for internal testing purposes only. **/

container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java

@@ -1,21 +1,40 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.fastsearch;
 
+import com.yahoo.search.schema.SchemaInfo;
+
 /**
  * Helper class for carrying around cluster-related
- * config parameters to the FastSearcher class.
+ * config parameters to the VespaBackend class.
  *
  * @author arnej27959
  */
 public class ClusterParams {
 
-    public final String searcherName;
+    private final String searcherName;
+    private final String serverId;
+    private final String defaultSummary;
+    private final DocumentdbInfoConfig documentdbInfoConfig;
+    private final SchemaInfo schemaInfo;
 
-    /**
-     * Make up full ClusterParams
-     */
     public ClusterParams(String name) {
+        this(name, "server.0", null, null, null);
+    }
+    public ClusterParams(String name, String serverId, String defaultSummary,
+                         DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) {
         this.searcherName = name;
+        this.serverId = serverId;
+        if (defaultSummary != null && defaultSummary.isEmpty())
+            this.defaultSummary = null;
+        else
+            this.defaultSummary = defaultSummary;
+        this.documentdbInfoConfig = documentdbInfoConfig;
+        this.schemaInfo = schemaInfo;
     }
 
+    public String getServerId() { return serverId; }
+    public String getSearcherName() { return searcherName; }
+    public String getDefaultSummary() { return defaultSummary; }
+    public DocumentdbInfoConfig getDocumentdbInfoConfig() { return documentdbInfoConfig; }
+    public SchemaInfo getSchemaInfo() { return schemaInfo; }
 }

container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java

@@ -13,7 +13,6 @@
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.result.Hit;
 import com.yahoo.search.result.HitGroup;
-import com.yahoo.search.schema.SchemaInfo;
 
 import java.io.IOException;
 import java.util.Optional;
@@ -38,23 +37,15 @@ public class IndexedBackend extends VespaBackend {
     /**
      * Creates a Fastsearcher.
      *
-     * @param serverId the resource pool used to create direct connections to the local search nodes when
-     *                        bypassing the dispatch node
      * @param dispatcher the dispatcher used (when enabled) to send summary requests over the rpc protocol.
      *                   Eventually we will move everything to this protocol and never use dispatch nodes.
      *                   At that point we won't need a cluster searcher above this to select and pass the right
      *                   backend.
-     * @param docSumParams  document summary parameters
      * @param clusterParams the cluster number, and other cluster backend parameters
-     * @param documentdbInfoConfig document database parameters
      */
-    public IndexedBackend(String serverId,
-                          Dispatcher dispatcher,
-                          SummaryParameters docSumParams,
-                          ClusterParams clusterParams,
-                          DocumentdbInfoConfig documentdbInfoConfig,
-                          SchemaInfo schemaInfo) {
-        init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo);
+    public IndexedBackend(ClusterParams clusterParams, Dispatcher dispatcher)
+    {
+        super(clusterParams);
         this.dispatcher = dispatcher;
     }
 

container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java

@@ -16,15 +16,14 @@
 import com.yahoo.search.grouping.vespa.GroupingExecutor;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.result.Hit;
-import com.yahoo.search.schema.SchemaInfo;
 import com.yahoo.searchlib.aggregation.Grouping;
 
 import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 /**
  * Superclass for backend searchers.
@@ -36,29 +35,44 @@ public abstract class VespaBackend {
     /** for vespa-internal use only; consider renaming the summary class */
     public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch";
 
-    private String serverId;
+    private final String serverId;
 
     /** The set of all document databases available in the backend handled by this searcher */
-    private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>();
-    private DocumentDatabase defaultDocumentDb = null;
+    private final Map<String, DocumentDatabase> documentDbs;
+    private final DocumentDatabase defaultDocumentDb;
 
     /** Default docsum class. null means "unset" and is the default value */
-    private String defaultDocsumClass = null;
+    private final String defaultDocsumClass;
 
     /** Returns an iterator which returns all hits below this result **/
     private static Iterator<Hit> hitIterator(Result result) {
         return result.hits().unorderedDeepIterator();
     }
 
     /** The name of this source */
-    private String name;
+    private final String name;
+
+    protected VespaBackend(ClusterParams clusterParams) {
+        this.serverId = clusterParams.getServerId();
+        this.name = clusterParams.getSearcherName();
+        this.defaultDocsumClass = clusterParams.getDefaultSummary();
+
+        Validator.ensureNotNull("Name of Vespa backend integration", name);
+
+        List<DocumentDatabase> dbs = new ArrayList<>();
+        if (clusterParams.getDocumentdbInfoConfig() != null) {
+            for (DocumentdbInfoConfig.Documentdb docDb : clusterParams.getDocumentdbInfoConfig().documentdb()) {
+                DocumentDatabase db = new DocumentDatabase(clusterParams.getSchemaInfo().schemas().get(docDb.name()));
+                dbs.add(db);
+            }
+        }
+        this.defaultDocumentDb = dbs.isEmpty() ? null : dbs.get(0);
+        this.documentDbs = dbs.stream().collect(Collectors.toMap(db -> db.schema().name(), db -> db));
+    }
 
     public final String getName() { return name; }
     protected final String getDefaultDocsumClass() { return defaultDocsumClass; }
 
-    /** Sets default document summary class. Default is null */
-    private void setDefaultDocsumClass(String docsumClass) { defaultDocsumClass = docsumClass; }
-
     /**
      * Searches a search cluster
      * This is an endpoint - searchers will never propagate the search to any nested searcher.
@@ -127,25 +141,6 @@ private void resolveDocumentDatabase(Query query) {
         }
     }
 
-    public final void init(String serverId, SummaryParameters docSumParams, ClusterParams clusterParams,
-                           DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) {
-        this.serverId = serverId;
-        this.name = clusterParams.searcherName;
-
-        Validator.ensureNotNull("Name of Vespa backend integration", getName());
-
-        setDefaultDocsumClass(docSumParams.defaultClass);
-
-        if (documentdbInfoConfig != null) {
-            for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) {
-                DocumentDatabase db = new DocumentDatabase(schemaInfo.schemas().get(docDb.name()));
-                if (documentDbs.isEmpty())
-                    defaultDocumentDb = db;
-                documentDbs.put(docDb.name(), db);
-            }
-        }
-    }
-
     protected void transformQuery(Query query) { }
 
     public Result search(String schema, Query query) {

container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java

@@ -11,6 +11,7 @@
 import com.yahoo.messagebus.routing.Route;
 import com.yahoo.prelude.Ping;
 import com.yahoo.prelude.Pong;
+import com.yahoo.prelude.fastsearch.ClusterParams;
 import com.yahoo.prelude.fastsearch.FastHit;
 import com.yahoo.prelude.fastsearch.GroupingListHit;
 import com.yahoo.prelude.fastsearch.TimeoutException;
@@ -46,48 +47,43 @@
  */
 public class StreamingBackend extends VespaBackend {
 
+    private static final Logger log = Logger.getLogger(StreamingBackend.class.getName());
     private static final CompoundName streamingUserid = CompoundName.from("streaming.userid");
     private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname");
     private static final CompoundName streamingSelection = CompoundName.from("streaming.selection");
 
     static final String STREAMING_STATISTICS = "streaming.statistics";
     private final VisitorFactory visitorFactory;
     private final TracingOptions tracingOptions;
-    private static final Logger log = Logger.getLogger(StreamingBackend.class.getName());
 
-    private Route route;
+    private final Route route;
 
     /** The configId used to access the searchcluster. */
-    private String searchClusterName = null;
+    private final String searchClusterName;
 
     /** The route to the storage cluster. */
-    private String storageClusterRouteSpec = null;
+    private final String storageClusterRouteSpec;
 
-    StreamingBackend(VisitorFactory visitorFactory) {
-        this.visitorFactory = visitorFactory;
-        tracingOptions = TracingOptions.DEFAULT;
+    StreamingBackend(ClusterParams clusterParams, String searchClusterName, VisitorFactory visitorFactory, String storageClusterRouteSpec) {
+        this(clusterParams, searchClusterName, visitorFactory, storageClusterRouteSpec, TracingOptions.DEFAULT);
     }
 
-    StreamingBackend(VisitorFactory visitorFactory, TracingOptions tracingOptions) {
+    StreamingBackend(ClusterParams clusterParams, String searchClusterName, VisitorFactory visitorFactory, String storageClusterRouteSpec, TracingOptions tracingOptions) {
+        super(clusterParams);
         this.visitorFactory = visitorFactory;
         this.tracingOptions = tracingOptions;
+        this.searchClusterName = searchClusterName;
+        this.storageClusterRouteSpec = storageClusterRouteSpec;
+        this.route = Route.parse(storageClusterRouteSpec);
     }
 
-    public StreamingBackend(VespaDocumentAccess access) {
-        this(new VespaVisitorFactory(access));
+    public StreamingBackend(ClusterParams clusterParams, String searchClusterName, VespaDocumentAccess access, String storageClusterRouteSpec) {
+        this(clusterParams, searchClusterName, new VespaVisitorFactory(access), storageClusterRouteSpec);
     }
 
     private String getSearchClusterName() { return searchClusterName; }
-    private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; }
-    public final void setSearchClusterName(String clusterName) { this.searchClusterName = clusterName; }
 
-    public final void setStorageClusterRouteSpec(String storageClusterRouteSpec) {
-        this.storageClusterRouteSpec = storageClusterRouteSpec;
-    }
-
-    @Override
-    protected void doPartialFill(Result result, String summaryClass) {
-    }
+    @Override protected void doPartialFill(Result result, String summaryClass) { }
 
     private double durationInMillisFromNanoTime(long startTimeNanos) {
         return (tracingOptions.getClock().nanoTimeNow() - startTimeNanos) / (double)TimeUnit.MILLISECONDS.toNanos(1);
@@ -165,11 +161,7 @@ public Result doSearch2(String schema, Query query) {
     }
 
     private void initializeMissingQueryFields(Query query) {
-        lazyTrace(query, 7, "Routing to storage cluster ", getStorageClusterRouteSpec());
-
-        if (route == null) {
-            route = Route.parse(getStorageClusterRouteSpec());
-        }
+        lazyTrace(query, 7, "Routing to storage cluster ", storageClusterRouteSpec);
         lazyTrace(query, 8, "Route is ", route);
 
         lazyTrace(query, 7, "doSearch2(): query docsum class=",

container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java

@@ -7,6 +7,7 @@
 import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.container.handler.ClustersStatus;
 import com.yahoo.container.handler.VipStatus;
+import com.yahoo.prelude.fastsearch.ClusterParams;
 import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
 import com.yahoo.prelude.fastsearch.FastHit;
 import com.yahoo.prelude.fastsearch.VespaBackend;
@@ -213,6 +214,7 @@ private void init() {
         }
 
         MyMockBackend(boolean expectAttributePrefetch) {
+            super(new ClusterParams("container.0"));
             this.expectAttributePrefetch = expectAttributePrefetch;
             init();
         }