Merge main

.asf.yaml

@@ -27,6 +27,7 @@ github:
     branch_9_3: {}
     branch_9_4: {}
     branch_9_5: {}
+    branch_9_6: {}
     branch_9x: {}
 
   protected_tags:

.github/renovate.json

@@ -30,12 +30,12 @@
     {
       "description": "Test-dependencies are checked less often than the shipped deps",
       "matchDepTypes": ["test"],
-      "extends": ["schedule:monthly"]
+      "schedule": ["before 9am on the first day of the month"]
     },
     {
       "description": "Noisy, frequently updated dependencies checked less often",
       "matchPackagePrefixes": ["software.amazon.awssdk", "com.google.cloud"],
-      "extends": ["schedule:monthly"]
+      "schedule": ["before 9am on the first day of the month"]
     },
     {
       "description": "Workaround for https://github.com/renovatebot/renovate/issues/19226",
@@ -44,12 +44,38 @@
     },
     {
       "description": "Lucene dependencies should be skipped",
-      "matchPackageNames": ["org.apache.lucene"],
+      "matchPackagePrefixes": ["org.apache.lucene"],
       "enabled": false
+    },
+    {
+      "description": "Skip major jetty upgrades - must be done manually",
+      "matchPackagePrefixes": ["org.eclipse.jetty"],
+      "matchUpdateTypes": ["major"],
+      "enabled": false
+    },
+    {
+      "description": "Changelog for commons-io",
+      "matchSourceUrls": ["https://gitbox.apache.org/repos/asf?p=commons-io.git"],
+      "customChangelogUrl": "https://commons.apache.org/proper/commons-io/changes-report.html"
+    },
+    {
+      "description": "Changelog for zookeeper",
+      "matchSourceUrls": ["https://gitbox.apache.org/repos/asf/zookeeper.git"],
+      "customChangelogUrl": "https://zookeeper.apache.org/releases.html"
+    },
+    {
+      "description": "Changelog for commons-compress",
+      "matchSourceUrls": ["https://gitbox.apache.org/repos/asf?p=commons-compress.git"],
+      "customChangelogUrl": "https://commons.apache.org/proper/commons-compress/changes-report.html"
+    },
+    {
+      "description": "Changelog for commons-configuration",
+      "matchSourceUrls": ["https://gitbox.apache.org/repos/asf?p=commons-configuration.git"],
+      "customChangelogUrl": "https://commons.apache.org/proper/commons-configuration/changes-report.html"
     }
   ],
   "schedule": [
-    "* * * * *"
+    "* * * * 0"
   ],
   "prConcurrentLimit": 50,
   "prHourlyLimit": 10,

.github/workflows/stale.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   stale:
-
+    if: github.repository == 'apache/solr'
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write

dev-tools/README.txt

@@ -5,7 +5,6 @@ as to the usefulness of the tools.
 
 Description of dev-tools/ contents:
 
-./size-estimator-lucene-solr.xls -- Spreadsheet for estimating memory and disk usage in Lucene/Solr
 ./missing-doclet -- JavaDoc validation doclet subproject
 ./doap/       -- Lucene and Solr project descriptors in DOAP RDF format.
 ./scripts/    -- Odds and ends for building releases, etc.

dev-tools/doap/solr.rdf

@@ -68,6 +68,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+         <Version>
+           <name>solr-9.6.0</name>
+           <created>2024-04-27</created>
+           <revision>9.6.0</revision>
+         </Version>
+    </release>
     <release>
          <Version>
            <name>solr-9.5.0</name>

dev-tools/scripts/cloud.sh

@@ -59,6 +59,9 @@
 #           that not using the embedded zookeeper is key to being able
 #           switch between testing setups and to test vs alternate versions
 #           of zookeeper if desired.
+# 
+#           An option is:
+#           docker run --name my-zookeeper -p 2181:2181 -d zookeeper
 #
 # SETUP: 1. Place this script in a directory intended to hold all your
 #           testing installations of solr.
@@ -79,7 +82,7 @@
 #
 #   ./cloud.sh stop
 #
-# Compile and push new code to a running cluster (incl bounce the cluster)
+# Compile and push new code to a running cluster (including bounce the cluster)
 #
 #   ./cloud.sh restart -r
 #
@@ -310,13 +313,9 @@ start(){
   findSolr
 
   echo "SOLR=$SOLR"
-  SOLR_ROOT=$("${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd getfile "/solr_${SAFE_DEST}" /dev/stdout);
-  if [[ -z ${SOLR_ROOT} ]]; then
-    # Need a fresh root in zookeeper...
-    "${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd makepath "/solr_${SAFE_DEST}";
-    "${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd put "/solr_${SAFE_DEST}" "created by cloud.sh"; # so we can test for existence next time
-  fi
-
+  # Create the root if it doesn't already exist
+  ${SOLR}/bin/solr zk mkroot "/solr_${SAFE_DEST}" -z localhost:${ZK_PORT}
+
   ACTUAL_NUM_NODES=$(ls -1 -d ${CLUSTER_WD}/n* | wc -l )
   if [[ "$NUM_NODES" -eq 0 ]]; then
     NUM_NODES=${ACTUAL_NUM_NODES}

dev-tools/scripts/releaseWizard.py

@@ -110,6 +110,7 @@ def expand_jinja(text, vars=None):
         'get_next_version': state.get_next_version(),
         'current_git_rev': state.get_current_git_rev(),
         'keys_downloaded': keys_downloaded(),
+        'docker_version_to_remove': state.get_docker_version_to_remove(),
         'editor': get_editor(),
         'rename_cmd': 'ren' if is_windows() else 'mv',
         'vote_close_72h': vote_close_72h_date().strftime("%Y-%m-%d %H:00 UTC"),
@@ -602,6 +603,12 @@ def get_next_version(self):
             return "%s.%s.%s" % (self.release_version_major, self.release_version_minor, self.release_version_bugfix + 1)
         return None
 
+    def get_docker_version_to_remove(self):
+        if self.release_type == 'minor' and self.release_version_minor >= 2:
+            return "%s.%s" % (self.release_version_major, self.release_version_minor - 2)
+        else:
+            return None
+
     def get_refguide_release(self):
         return "%s_%s" % (self.release_version_major, self.release_version_minor)
 

dev-tools/scripts/releaseWizard.yaml

@@ -1078,7 +1078,12 @@ groups:
           comment: Add the new slim distribution Dockerfile to the repo
           logfile: solr_docker_add_dockerfile_slim.log
         - !Command
-          cmd: 'git add {{ docker_version }}/Dockerfile {{ docker_version }}-slim/Dockerfile && git commit -m "Apache Solr release {{ release_version }}"'
+          cmd: "{% if docker_version_to_remove %}rm -rf {{ docker_version_to_remove }} {{ docker_version_to_remove }}-slim{% else %}echo Nothing to delete{% endif %}"
+          cwd: solr-docker
+          comment: "{% if docker_version_to_remove %}Mark docker image version {{ docker_version_to_remove }} as not supported{% else %}N/A - No docker versions to delete for this release{% endif %}"
+          logfile: solr_remove_old_docker_images.log
+        - !Command
+          cmd: 'git add . && git commit -m "Apache Solr release {{ release_version }}"'
           cwd: solr-docker
           comment: Commit the new Dockerfile
           logfile: solr_docker_commit.log

solr/CHANGES.txt

@@ -59,6 +59,13 @@ Deprecation Removals
 * SOLR-12089: Remove deprecated `breakSugestionTieBreaker` parameter in favor of `breakSuggestionTieBreaker`
   in WordBreakSolrSpellChecker (Andrey Bozhko via Eric Pugh)
 
+* SOLR-14763: Remove deprecated asynchronous request methods from `Http2SolrClient`, `HttpJdkSolrClient` and `LBHttp2SolrClient`
+  in favor of the new CompletableFuture based methods.  Remove the related deprecated interfaces `AsyncListener` and ``Cancellable`
+  (James Dyer)
+
+* SOLR-14115: Remove deprecated zkcli script in favour of equivalent bin/solr sub commmands. (Eric Pugh)
+
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -88,6 +95,46 @@ Other Changes
 
 * SOLR-17205: De-couple SolrJ required Java version from server Java version (janhoy)
 
+==================  9.7.0 ==================
+New Features
+---------------------
+* SOLR-13350: Multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble)
+
+* SOLR-17192: Put an UpdateRequestProcessor-enforced soft-limit on the number of fields allowed in a core.  The `NumFieldLimitingUpdateRequestProcessorFactory`
+  limit may be adjusted by raising the factory's `maxFields` setting, toggled in and out of "warning-only" mode using the `warnOnly` setting, or disabled entirely
+  by removing it solrconfig.xml.  The limit is set at 1000 fields in the "_default" configset, but left in warning-only mode. (David Smiley, Eric Pugh,
+  Jason Gerlowski)
+
+Improvements
+---------------------
+* SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh)
+
+* SOLR-14115: Add linkconfig, cluster, and updateacls as commands to SolrCLI.  Allow bin/solr commands to have parity with zkcli.sh commands. (Eric Pugh)
+
+Optimizations
+---------------------
+* SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather
+  cluster metrics.  AttributeFetcherImpl was improved accordingly.  (Yohann Callea)
+
+Bug Fixes
+---------------------
+* SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh)
+
+* SOLR-17261: Remove unintended timeout of 60 seconds for core loading. (Houston Putman)
+
+Dependency Upgrades
+---------------------
+(No changes)
+
+Other Changes
+---------------------
+* SOLR-17248: Refactor ZK related SolrCli tools to separate SolrZkClient and CloudSolrClient instantiation/usage (Lamine Idjeraoui via Eric Pugh)
+
+* SOLR-16505: Use Jetty HTTP2 for index replication and other "recovery" operations
+  (Sanjay Dutt, David Smiley)
+
+* SOLR-16503: Introduce new default Http2SolrClient in UpdateShardHandler and deprecated old default HttpClient (Sanjay Dutt, David Smiley)
+
 ==================  9.6.0 ==================
 New Features
 ---------------------
@@ -97,6 +144,8 @@ New Features
 
 * SOLR-16403: A new cluster singleton plugin to automatically remove inactive shards. (Paul McArthur, David Smiley)
 
+* SOLR-16466: Admin UI - Make it optional to sort list of commandline args (Shawn Heisey, Vincenzo D'Amore via Christine Poerschke)
+
 Improvements
 ---------------------
 * SOLR-17119: When registering or updating a ConfigurablePlugin through the `/cluster/plugin` API,
@@ -126,6 +175,13 @@ Improvements
 
 * SOLR-17164: Add 2 arg variant of vectorSimilarity() function (Sanjay Dutt, hossman)
 
+* SOLR-17211: New SolrJ JDK client supports Async (James Dyer)
+
+* SOLR-17204: REPLACENODE now supports the source node not being live (Vincent Primault)
+
+* SOLR-14763: Add a CompletableFuture based asynchronous API to Http2SolrClient, HttpJdkSolrClient and LBHttp2SolrClient.
+  The previous asynchronous API is deprecated. (Rishi Sankar, James Dyer)
+
 Optimizations
 ---------------------
 * SOLR-17144: Close searcherExecutor thread per core after 1 minute (Pierre Salagnac, Christine Poerschke)
@@ -151,6 +207,9 @@ Bug Fixes
 * SOLR-17198: AffinityPlacementFactory can fail if Shard leadership changes occur while it is collecting metrics.
   (Paul McArthur)
 
+* SOLR-17018: Add QueryLimits support to Learning To Rank rescoring.
+    (Alessandro Benedetti)
+
 * SOLR-14892: Queries with shards.info and shards.tolerant can yield multiple null keys in place of shard names
   (Mathieu Marie, David Smiley)
 
@@ -164,15 +223,30 @@ Bug Fixes
 
 * SOLR-17218: Fix indexFetcher logging to include MDC details (hossman)
 
+* SOLR-17206: Eliminate the possibility of a -1 status code for SolrCloud update requests that are distributed.
+  (Paul McArthur)
+
+* SOLR-17213: Fix spurious warnings about solr url format in Solr CLI when users aren't providing a deprecated solr url.  (Eric Pugh)
+
+* SOLR-17176: Fix log history V2 API serialization (Michael Gibney)
+
+* SOLR-11535: Fix race condition in singleton-per-collection StateWatcher creation (Michael Gibney)
+
+* SOLR-16866: Reorder nested directory deletions to avoid logging spurious NoSuchFileException (Michael Gibney)
+
+* SOLR-17194: Fix CloudHttp2SolrClient doesn't preserve the passed httpClient to Http2ClusterStateProvider when using the internalClientBuilder (Eric Pugh, Lamine Idjeraoui, Houston Putman, Anshum Gupta)
+
 Dependency Upgrades
 ---------------------
-
 * SOLR-17157: Upgrade Lucene to 9.10.0 (hossman, Christine Poerschke)
 
 * SOLR-17214: Update forbiddenapis to 3.7 (hossman)
 
+* GITHUB#2408: Update commons-cli to 1.7 (Eric Pugh)
+
 Other Changes
 ---------------------
+
 * SOLR-17126: Cut over System.getProperty to EnvUtils.getProperty (janhoy)
 
 * SOLR-17066: GenericSolrRequest now has a `setRequiresCollection` setter that allows it to specify whether
@@ -189,7 +263,10 @@ Other Changes
 
 * SOLR-17217: Simplify verbose MatcherAssert.assertThat usage in tests (hossman)
 
-* SOLR-16503: Introduce new default Http2SolrClient in UpdateShardHandler and deprecated old default HttpClient
+* SOLR-17232: PropertiesOutputStream is renamed IndexOutputOutputStream and overrides write(byte[], int, int).
+  (Bruno Roustant)
+
+* SOLR-17222: QueryResponseWriterUtil.NonFlushingStream should support bulk write methods (Michael Gibney)
 
 ==================  9.5.0 ==================
 New Features

solr/benchmark/src/java/org/apache/solr/bench/BaseBenchState.java

@@ -41,9 +41,29 @@
 @State(Scope.Benchmark)
 public class BaseBenchState {
 
-  private static final long RANDOM_SEED = System.nanoTime();
+  public static final long RANDOM_SEED;
 
-  private static final SplittableRandom random = new SplittableRandom(getInitRandomSeed());
+  static {
+    Long seed = Long.getLong("solr.bench.seed");
+
+    if (seed == null) {
+      String prop = System.getProperty("tests.seed"); // RandomizedTesting framework
+      if (prop != null) {
+        // if there is a test failure we remain reproducible based on the test seed:
+        prop = prop.split(":")[0]; // main seed
+        seed = Long.parseUnsignedLong(prop, 16);
+      } else {
+        seed = System.nanoTime();
+      }
+    }
+
+    log("");
+    log("benchmark random seed: " + seed);
+
+    RANDOM_SEED = seed;
+  }
+
+  private static final SplittableRandom random = new SplittableRandom(RANDOM_SEED);
 
   /**
    * Gets random seed.
@@ -134,17 +154,4 @@ public static void dumpHeap(BenchmarkParams benchmarkParams) throws IOException
       }
     }
   }
-
-  private static Long getInitRandomSeed() {
-    Long seed = Long.getLong("solr.bench.seed");
-
-    if (seed == null) {
-      seed = RANDOM_SEED;
-    }
-
-    log("");
-    log("benchmark random seed: " + seed);
-
-    return seed;
-  }
 }