Merge branch 'main' into SOLR-17023

.github/PULL_REQUEST_TEMPLATE.md

@@ -35,7 +35,7 @@ Please describe the tests you've developed or run to confirm this patch implemen
 
 Please review the following and check all that apply:
 
-- [ ] I have reviewed the guidelines for [How to Contribute](https://wiki.apache.org/solr/HowToContribute) and my code conforms to the standards described there to the best of my ability.
+- [ ] I have reviewed the guidelines for [How to Contribute](https://github.com/apache/solr/blob/main/CONTRIBUTING.md) and my code conforms to the standards described there to the best of my ability.
 - [ ] I have created a Jira issue and added the issue ID to my pull request title.
 - [ ] I have given Solr maintainers [access](https://help.github.com/en/articles/allowing-changes-to-a-pull-request-branch-created-from-a-fork) to contribute to my PR branch. (optional but recommended)
 - [ ] I have developed this patch against the `main` branch.

.github/renovate.json

@@ -49,9 +49,9 @@
     }
   ],
   "schedule": [
-    "* * * * 0"
+    "* * * * *"
   ],
-  "prConcurrentLimit": 30,
+  "prConcurrentLimit": 50,
   "prHourlyLimit": 10,
   "stabilityDays": 5
 }

.github/workflows/tests-via-crave.yml

@@ -9,13 +9,21 @@ jobs:
   test:
     name: Run Solr Tests using Crave.io resources
 
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
 
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: Get the Crave binary
-      run: curl -s https://raw.githubusercontent.com/accupara/crave/master/get_crave.sh | bash -s --
+    - name: Destroy previous clone
+      run: crave clone destroy -y /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER} || echo "Clone did not exist"
+      continue-on-error: true
+    - name: Crave clone sources
+      run: crave clone create --projectID 39 /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}
+    - name: Checkout the correct branch
+      run: |
+        git -C /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}/solr fetch origin ${GITHUB_REF}:${GITHUB_REF}
+        git -C /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}/solr checkout ${GITHUB_REF}
     - name: Initialize, build, test
-      run: ./crave run --clean
+      run: |
+        cd /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}/solr
+        crave run --clean
+    - name: Delete Clone
+      run: crave clone destroy -y /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}

dev-tools/scripts/cloud.sh

@@ -280,10 +280,11 @@ copyTarball() {
       curl -o "$RC_FILE" "$SMOKE_RC_URL"
       pushd
     else
-      if [[ ! -f $(ls "$VCS_WORK"/solr/packaging/build/distributions/solr-*.tgz) ]]; then
+      TARBALL=$(find "$VCS_WORK" -regex '.*/solr-.*\.tgz' | grep -v slim)
+      if [[ ! -f "$TARBALL" ]]; then
         echo "No solr tarball found try again with -r"; popd; exit 10;
       fi
-      cp "$VCS_WORK"/solr/packaging/build/distributions/solr-*.tgz ${CLUSTER_WD}
+      cp "$TARBALL" "${CLUSTER_WD}"
     fi
     pushd # back into cluster wd to unpack
     tar xzvf solr-*.tgz

dev-tools/scripts/smokeTestRelease.py

@@ -220,6 +220,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
   ents = getDirEntries(urlString)
   artifact = None
   changesURL = None
+  openApiURL = None
   mavenURL = None
   dockerURL = None
   artifactURL = None
@@ -243,6 +244,10 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
       if text not in ('changes/', 'changes-%s/' % version):
         raise RuntimeError('solr: found %s vs expected changes-%s/' % (text, version))
       changesURL = subURL
+    elif text.startswith('openApi'):
+      if text not in ('openApi/', 'openApi-%s/' % version):
+        raise RuntimeError('solr: found %s vs expected openApi-%s/' % (text, version))
+      openApiURL = subURL
     elif artifact is None:
       artifact = text
       artifactURL = subURL
@@ -296,6 +301,12 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
     raise RuntimeError('solr is missing changes-%s' % version)
   testChanges(version, changesURL)
 
+  if openApiURL is None:
+    stopGpgAgent(gpgHomeDir, logfile)
+    raise RuntimeError('solr is missing OpenAPI specification' % version)
+  testOpenApi(version, openApiURL)
+
+
   for artifact, urlString in artifacts: # pylint: disable=redefined-argument-from-local
     print('  download %s...' % artifact)
     scriptutil.download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
@@ -349,6 +360,17 @@ def testChanges(version, changesURLString):
   s = load(changesURL)
   checkChangesContent(s, version, changesURL, True)
 
+def testOpenApi(version, openApiDirUrl):
+  print('  check OpenAPI specification...')
+  specFound = False
+  expectedSpecFileName = 'solr-openapi-' + version + '.json'
+  for text, subURL in getDirEntries(openApiDirUrl):
+    if text == expectedSpecFileName:
+      specFound = True
+
+  if not specFound:
+    raise RuntimeError('Did not see %s in %s' % expectedSpecFileName, openApiDirUrl)
+
 
 def testChangesText(dir, version):
   "Checks all CHANGES.txt under this dir."

gradle/testing/randomization/policies/solr-tests.policy

@@ -162,6 +162,11 @@ grant {
   permission javax.security.auth.AuthPermission "createLoginContext.Server";
   permission javax.security.auth.AuthPermission "createLoginContext.Client";
 
+  // Needed by BouncyCastle in jwt-auth tests
+  permission java.security.SecurityPermission "putProviderProperty.BC";
+  permission java.security.SecurityPermission "removeProviderProperty.BC";
+  permission java.security.SecurityPermission "getProperty.org.bouncycastle.x509.allow_non-der_tbscert";
+
   // may only be necessary with Java 7?
   permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
   permission javax.security.auth.PrivateCredentialPermission "sun.security.jgss.krb5.Krb5Util$KeysFromKeyTab * \"*\"", "read";

solr/CHANGES.txt

@@ -8,6 +8,7 @@ https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-not
 
 New Features
 ---------------------
+* SOLR-14496: Solr CLI commands now can interact with a Solr secured using Basic Authentication.  (Eric Pugh)
 
 * SOLR-16943: Extend Solr client tracing coverage to both Jetty Client and Apache HttpClient (Alex Deparvu, David Smiley)
 
@@ -20,6 +21,8 @@ Improvements
 
 * SOLR-16536: Replace OpenTracing instrumentation with OpenTelemetry (Alex Deparvu, janhoy)
 
+* SOLR-17077: When a shard rejoins leader election, leave previous election only once to save unneeded calls to Zookeeper. (Pierre Salagnac)
+
 Optimizations
 ---------------------
 (No changes)
@@ -76,22 +79,25 @@ Other Changes
 ==================  9.5.0 ==================
 New Features
 ---------------------
-* SOLR-14496: Solr CLI commands now can interact with a Solr secured using Basic Authentication.  (Eric Pugh)
-
 * SOLR-17006: Collection creation & adding replicas: User-defined properties are persisted to state.json and
   applied to new replicas, available for use as property substitution in configuration files.  (Vincent Primault)
 
 * SOLR-16974: Circuit Breakers can now be configured globally (janhoy, Christine Poerschke)
 
-* SOLR-17079: Allow to declare replica placement plugins in solr.xml  (Vincent Primault)
+* SOLR-16743: When using TLS, Solr can now auto-reload the keystore and truststore without the need to restart the process.
+  This is enabled by default when running with TLS and can be disabled or configured in solr.in.sh (Houston Putman, Tomás Fernández Löbbe)
 
 Improvements
 ---------------------
+* SOLR-17053: Distributed search with shards.tolerant: if all shards fail, fail the request (Aparna Suresh via David Smiley)
+
 * SOLR-16924: RESTORECORE now sets the UpdateLog to ACTIVE state instead of requiring a separate
-  REQUESTAPPLYUPDATES call in Collection restore.  (Julia Lamoine, David Smiley)
+  REQUESTAPPLYUPDATES call in Collection restore.  (Julia Maimone, David Smiley)
 
 * SOLR-16907: Fail when parsing an invalid custom permission definition from security.json (janhoy, Uwe Schindler)
 
+* SOLR-13748: Add support for mm (min should match) parameter to bool query parser (Andrey Bozhko)
+
 * SOLR-17046: SchemaCodecFactory is now the implicit default codec factory.  (hossman)
 
 * SOLR-16397: Swap core v2 endpoints have been updated to be more REST-ful.
@@ -106,9 +112,36 @@ Improvements
 
 * SOLR-17035: Add trace id to jetty thread names to improve debuggability via stack traces (Alex Deparvu)
 
+* SOLR-17079: Allow to declare replica placement plugins in solr.xml  (Vincent Primault)
+
+* SOLR-16959: Make the internal CoreSorter implementation configurable in solr.xml  (Vincent Primault)
+
+* SOLR-17050: Use compact JSON for Learning to Rank (LTR) feature and model storage. (Florin Babes, Christine Poerschke, Alessandro Benedetti)
+
+* SOLR-17094: Close objects contained inside an ObjectCache.  (Vincent Primault)
+
+* SOLR-16577: Ensure core load failures are always logged. (Haythem Khiri, David Smiley)
+
+* SOLR-17063: Do not retain log param references in LogWatcher (Michael Gibney)
+
+* SOLR-17066: SolrClient builders now allow users to specify a "default" collection or core
+  using the `withDefaultDataStore` method  ("Cloud" client builders retain the more specific
+  `withDefaultCollection`).  Use of the Builder methods is preferable to including the
+  collection in the base URL accepted by certain client implementations. (Jason Gerlowski)
+
+* SOLR-15960: Unified use of system properties and environment variables (janhoy)
+
+* SOLR-16397: The MERGEINDEXES v2 endpoint has been updated to be more REST-ful.
+  MERGEINDEXES is now available at `POST /api/cores/coreName/merge-indices` (Sanjay Dutt via Jason Gerlowski)
+
 Optimizations
 ---------------------
-(No changes)
+* SOLR-17084: LBSolrClient (used by CloudSolrClient) now returns the count of core tracked as not live AKA zombies
+  instead of the full list of cores.  This list is potentially as long as the number of cores.  When there are many
+  cores, this leads to high CPU and memory usage. (Gilles Bellaton, David Smiley)
+
+* SOLR-17036: UpdateLog lazy creates VersionBucket large array, and VersionBucket do not keep the highest version anymore.
+  This optimization reduces the memory usage, specially when the SolrCore is not used for indexing (Bruno Roustant)
 
 Bug Fixes
 ---------------------
@@ -122,6 +155,23 @@ Bug Fixes
 * SOLR-10653: When there's a UUIDField in the schema and atomic update touches another field
   the error occurs when leader updates replica (Mikhail Khludnev)
 
+* SOLR-17093: Collection restore API command now returns "requestid" when executed asynchronously like other APIs
+  (Tomás Fernández Löbbe)
+
+* SOLR-16952: In distributed i.e. multi-shard cloud mode returned dense vector 'fl' fields are now a list of numbers instead of strings. (Qais Qadri, Christine Poerschke)
+
+* SOLR-17090: The v2 "delete alias" API no longer errantly returns a 405 status code (Jason Gerlowski)
+
+* SOLR-17057: JSON Query regression: If "query" is specified with a String (not JSON structure),
+  "defType" should parse it.  Since 9.4 defType was ignored.  (David Smiley)
+
+* SOLR-17060: CoreContainer#create may deadlock with concurrent requests for metrics (Alex Deparvu, David Smiley)
+
+* SOLR-17098: ZK Credentials and ACLs are no longer sent to all ZK Servers when using Streaming Expressions.
+  They will only be used when sent to the default ZK Host. (Houston Putman, Jan Høydahl, David Smiley, Gus Heck, Qing Xu)
+
+* SOLR-16203: Properly initialize schema plugins loaded by SPI name (janhoy, hossman, Uwe Schindler)
+
 Dependency Upgrades
 ---------------------
 * SOLR-17012: Update Apache Hadoop to 3.3.6 and Apache Curator to 5.5.0 (Kevin Risden)
@@ -134,6 +184,16 @@ Other Changes
 
 * SOLR-17072: package CLI tool prints error JSONPath (Mikhail Khludnev)
 
+* SOLR-17078: The `train_and_upload_demo_model.py` script referenced in LTR documentation now uses Python3 (Jason Gerlowski)
+
+* SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expecting
+  cloud.sh has been updated to ignore the -slim.tgz version of the tarball.
+
+* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)
+
+* SOLR-16880: Solr now produces an OpenAPI Specification artifact on releases ("solr-openapi-x.y.z.json") that covers
+  Solr's v2 APIs. (Jason Gerlowski, Houston Putman)
+
 ==================  9.4.0 ==================
 New Features
 ---------------------

solr/api/build.gradle

@@ -26,8 +26,9 @@ description = 'API - Interfaces and classes used to represent Solrs APIs'
 
 ext {
     jsClientDir = "${buildDir}/generated/js"
+    pythonClientDir = "${buildDir}/generated/python"
     openApiSpecDir = "${buildDir}/generated/openapi"
-    openApiSpecFile = "${project.openApiSpecDir}/openapi.json"
+    openApiSpecFile = "${project.openApiSpecDir}/solr-openapi-${version}.json"
 }
 
 configurations {
@@ -39,12 +40,17 @@ configurations {
         canBeConsumed = true
         canBeResolved = false
     }
+    pythonClient {
+        canBeConsumed = true
+        canBeResolved = false
+    }
 }
 
 resolve {
   classpath = sourceSets.main.runtimeClasspath
   resourcePackages = ["org.apache.solr.client.api.util", "org.apache.solr.client.api.endpoint"]
   outputDir = file(project.openApiSpecDir)
+  outputFileName = "solr-openapi-${version}"
   prettyPrint = true
 }
 
@@ -62,7 +68,6 @@ dependencies {
 }
 
 // Non-Java client generation tasks below:
-
 task buildJSClient(type: org.openapitools.generator.gradle.plugin.tasks.GenerateTask) {
   generatorName.set("javascript")
   inputSpec.set("$openApiSpecFile")
@@ -72,18 +77,30 @@ task buildJSClient(type: org.openapitools.generator.gradle.plugin.tasks.Generate
   generateModelTests.set(false)
 }
 
+task buildPythonClient(type: org.openapitools.generator.gradle.plugin.tasks.GenerateTask) {
+  generatorName.set("python")
+  inputSpec.set("$openApiSpecFile")
+  outputDir.set("$pythonClientDir")
+  packageName.set("solr")
+  generateApiTests.set(false)
+  generateModelTests.set(false)
+}
+
 tasks.withType(org.openapitools.generator.gradle.plugin.tasks.GenerateTask) {
   dependsOn(resolve)
 }
 
 artifacts {
     // Ensure the OAS is available to other modules who want to generate code (i.e. solrj)
-    openapiSpec resolve.outputDir, {
+    openapiSpec file(openApiSpecFile), {
         builtBy resolve
     }
 
-    // Makes our Javascript client available to the Admin UI build
+    // Makes generated clients available to other build modules
     jsClient file(project.jsClientDir), {
         builtBy buildJSClient
     }
+    pythonClient file(project.pythonClientDir), {
+        builtBy buildPythonClient
+    }
 }

solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceShardUniqueApi.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.api.endpoint;
+
+import io.swagger.v3.oas.annotations.Operation;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import org.apache.solr.client.api.model.BalanceShardUniqueRequestBody;
+import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse;
+
+/**
+ * V2 API definition for insuring that a specified per-shard property is distributed evenly amongst
+ * the physical nodes comprising a collection.
+ *
+ * <p>The new API (POST /v2/collections/collectionName/balance-shard-unique {...} ) is analogous to
+ * the v1 /admin/collections?action=BALANCESHARDUNIQUE command.
+ */
+@Path("/collections/{collectionName}/balance-shard-unique")
+public interface BalanceShardUniqueApi {
+  @POST
+  @Operation(
+      summary =
+          "Ensure a specified per-shard property is distributed evenly amongst physical nodes comprising a collection",
+      tags = {"collections"})
+  SubResponseAccumulatingJerseyResponse balanceShardUnique(
+      @PathParam("collectionName") String collectionName, BalanceShardUniqueRequestBody requestBody)
+      throws Exception;
+}