SOLR-17556: Fix smokeTest to use films as example for slim distribution

Also remove test_utf8 from the smoke test, and the data for the test.

(cherry picked from commit 80dfe11)

dev-tools/scripts/smokeTestRelease.py

@@ -690,7 +690,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
     shutil.copytree(unpackPath, java11UnpackPath)
     os.chdir(java11UnpackPath)
     print('    test solr example w/ Java 11...')
-    testSolrExample(java11UnpackPath, java.java11_home)
+    testSolrExample(java11UnpackPath, java.java11_home, isSlim)
 
     if java.run_java17:
       print('    copying unpacked distribution for Java 17 ...')
@@ -700,7 +700,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
       shutil.copytree(unpackPath, java17UnpackPath)
       os.chdir(java17UnpackPath)
       print('    test solr example w/ Java 17...')
-      testSolrExample(java17UnpackPath, java.java17_home)
+      testSolrExample(java17UnpackPath, java.java17_home, isSlim)
 
     os.chdir(unpackPath)
 
@@ -742,7 +742,7 @@ def is_port_in_use(port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         return s.connect_ex(('localhost', port)) == 0
 
-def testSolrExample(binaryDistPath, javaPath):
+def testSolrExample(binaryDistPath, javaPath, isSlim):
   # test solr using some examples it comes with
   logFile = '%s/solr-example.log' % binaryDistPath
   old_cwd = os.getcwd() # So we can back-track
@@ -754,6 +754,10 @@ def testSolrExample(binaryDistPath, javaPath):
   env['JAVA_HOME'] = javaPath
   env['PATH'] = '%s/bin:%s' % (javaPath, env['PATH'])
 
+  example = "techproducts"
+  if isSlim:
+    example = "films"
+
   # Stop Solr running on port 8983 (in case a previous run didn't shutdown cleanly)
   try:
       if not cygwin:
@@ -763,22 +767,20 @@ def testSolrExample(binaryDistPath, javaPath):
   except:
      print('      Stop failed due to: '+sys.exc_info()[0])
 
-  print('      Running techproducts example on port 8983 from %s' % binaryDistPath)
+  print('      Running %s example on port 8983 from %s' % (example, binaryDistPath))
   try:
     if not cygwin:
-      runExampleStatus = subprocess.call(['bin/solr','-e','techproducts'])
+      runExampleStatus = subprocess.call(['bin/solr','-e',example])
     else:
-      runExampleStatus = subprocess.call('env "PATH=`cygpath -S -w`:$PATH" bin/solr.cmd -e techproducts', shell=True)
+      runExampleStatus = subprocess.call('env "PATH=`cygpath -S -w`:$PATH" bin/solr.cmd -e ' + example, shell=True)
 
     if runExampleStatus != 0:
-      raise RuntimeError('Failed to run the techproducts example, check log for previous errors.')
+      raise RuntimeError('Failed to run the %s example, check log for previous errors.' % example)
 
     os.chdir('example')
-    print('      test utf8...')
-    run('sh ./exampledocs/test_utf8.sh http://localhost:8983/solr/techproducts', 'utf8.log')
     print('      run query...')
-    s = load('http://localhost:8983/solr/techproducts/select/?q=video')
-    if s.find('"numFound":3,') == -1:
+    s = load('http://localhost:8983/solr/%s/select/?q=video' % example)
+    if s.find('"numFound":%d,' % (8 if isSlim else 3)) == -1:
       print('FAILED: response is:\n%s' % s)
       raise RuntimeError('query on solr example instance failed')
     s = load('http://localhost:8983/api/cores')

solr/core/src/java/org/apache/solr/cli/RunExampleTool.java

@@ -48,6 +48,7 @@
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.EnvUtils;
 import org.noggit.CharArr;
 import org.noggit.JSONWriter;
 
@@ -67,6 +68,7 @@ public class RunExampleTool extends ToolBase {
   protected String script;
   protected File serverDir;
   protected File exampleDir;
+  protected File solrHomeDir;
   protected String urlScheme;
 
   /** Default constructor used by the framework when running as a command-line application. */
@@ -123,7 +125,7 @@ public List<Option> getOptions() {
         Option.builder()
             .longOpt("server-dir")
             .hasArg()
-            .argName("DIR")
+            .argName("SERVER_DIR")
             .required(true)
             .desc("Path to the Solr server directory.")
             .build(),
@@ -135,11 +137,19 @@ public List<Option> getOptions() {
         Option.builder()
             .longOpt("example-dir")
             .hasArg()
-            .argName("DIR")
+            .argName("EXAMPLE_DIR")
             .required(false)
             .desc(
                 "Path to the Solr example directory; if not provided, ${serverDir}/../example is expected to exist.")
             .build(),
+        Option.builder()
+            .longOpt("solr-home-dir")
+            .hasArg()
+            .argName("SOLR_HOME_DIR")
+            .required(false)
+            .desc(
+                "Path to the Solr home directory; if not provided, ${serverDir}/solr is expected to exist.")
+            .build(),
         Option.builder()
             .longOpt("url-scheme")
             .hasArg()
@@ -190,6 +200,7 @@ public List<Option> getOptions() {
   @Override
   public void runImpl(CommandLine cli) throws Exception {
     this.urlScheme = cli.getOptionValue("url-scheme", "http");
+    String exampleType = cli.getOptionValue("example");
 
     serverDir = new File(cli.getOptionValue("server-dir"));
     if (!serverDir.isDirectory())
@@ -228,16 +239,36 @@ public void runImpl(CommandLine cli) throws Exception {
               + exampleDir.getAbsolutePath()
               + " is not a directory!");
 
+    if (cli.hasOption("solr-home-dir")) {
+      solrHomeDir = new File(cli.getOptionValue("solr-home-dir"));
+    } else {
+      String solrHomeProp = EnvUtils.getProperty("solr.home");
+      if (solrHomeProp != null && !solrHomeProp.isEmpty()) {
+        solrHomeDir = new File(solrHomeProp);
+      } else if ("cloud".equals(exampleType)) {
+        solrHomeDir = new File(exampleDir, "cloud");
+        if (!solrHomeDir.isDirectory()) solrHomeDir.mkdir();
+      } else {
+        solrHomeDir = new File(serverDir, "solr");
+      }
+    }
+    if (!solrHomeDir.isDirectory())
+      throw new IllegalArgumentException(
+          "Value of --solr-home-dir option is invalid! "
+              + solrHomeDir.getAbsolutePath()
+              + " is not a directory!");
+
     echoIfVerbose(
         "Running with\nserverDir="
             + serverDir.getAbsolutePath()
             + ",\nexampleDir="
             + exampleDir.getAbsolutePath()
+            + ",\nsolrHomeDir="
+            + solrHomeDir.getAbsolutePath()
             + "\nscript="
             + script,
         cli);
 
-    String exampleType = cli.getOptionValue("example");
     if ("cloud".equals(exampleType)) {
       runCloudExample(cli);
     } else if ("techproducts".equals(exampleType)
@@ -262,8 +293,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
     int port =
         Integer.parseInt(
             cli.getOptionValue('p', System.getenv().getOrDefault("SOLR_PORT", "8983")));
-    Map<String, Object> nodeStatus =
-        startSolr(new File(serverDir, "solr"), isCloudMode, cli, port, zkHost, 30);
+    Map<String, Object> nodeStatus = startSolr(solrHomeDir, isCloudMode, cli, port, zkHost, 30);
 
     String solrUrl = SolrCLI.normalizeSolrUrl((String) nodeStatus.get("baseUrl"), false);
 
@@ -365,7 +395,8 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
                 + "        }\n"
                 + "      }");
 
-        echo("Adding name, initial_release_date, and film_vector fields to films schema");
+        echo(
+            "Adding name, genre, directed_by, initial_release_date, and film_vector fields to films schema");
         SolrCLI.postJsonToSolr(
             solrClient,
             "/" + collectionName + "/schema",
@@ -377,6 +408,18 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
                 + "          \"stored\":true\n"
                 + "        },\n"
                 + "        \"add-field\" : {\n"
+                + "          \"name\":\"genre\",\n"
+                + "          \"type\":\"text_general\",\n"
+                + "          \"multiValued\":true,\n"
+                + "          \"stored\":true\n"
+                + "        },\n"
+                + "        \"add-field\" : {\n"
+                + "          \"name\":\"directed_by\",\n"
+                + "          \"type\":\"text_general\",\n"
+                + "          \"multiValued\":true,\n"
+                + "          \"stored\":true\n"
+                + "        },\n"
+                + "        \"add-field\" : {\n"
                 + "          \"name\":\"initial_release_date\",\n"
                 + "          \"type\":\"pdate\",\n"
                 + "          \"stored\":true\n"
@@ -386,6 +429,18 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
                 + "          \"type\":\"knn_vector_10\",\n"
                 + "          \"indexed\":true\n"
                 + "          \"stored\":true\n"
+                + "        },\n"
+                + "        \"add-copy-field\" : {\n"
+                + "          \"source\":\"genre\",\n"
+                + "          \"dest\":\"_text_\"\n"
+                + "        },\n"
+                + "        \"add-copy-field\" : {\n"
+                + "          \"source\":\"name\",\n"
+                + "          \"dest\":\"_text_\"\n"
+                + "        },\n"
+                + "        \"add-copy-field\" : {\n"
+                + "          \"source\":\"directed_by\",\n"
+                + "          \"dest\":\"_text_\"\n"
                 + "        }\n"
                 + "      }");
 
@@ -452,8 +507,6 @@ protected void runCloudExample(CommandLine cli) throws Exception {
       // Override the old default port numbers if user has started the example overriding SOLR_PORT
       cloudPorts = new int[] {defaultPort, defaultPort + 1, defaultPort + 2, defaultPort + 3};
     }
-    File cloudDir = new File(exampleDir, "cloud");
-    if (!cloudDir.isDirectory()) cloudDir.mkdir();
 
     echo("\nWelcome to the SolrCloud example!\n");
 
@@ -499,9 +552,9 @@ protected void runCloudExample(CommandLine cli) throws Exception {
     }
 
     // setup a unique solr.solr.home directory for each node
-    File node1Dir = setupExampleDir(serverDir, cloudDir, "node1");
+    File node1Dir = setupExampleDir(serverDir, solrHomeDir, "node1");
     for (int n = 2; n <= numNodes; n++) {
-      File nodeNDir = new File(cloudDir, "node" + n);
+      File nodeNDir = new File(solrHomeDir, "node" + n);
       if (!nodeNDir.isDirectory()) {
         echo("Cloning " + node1Dir.getAbsolutePath() + " into\n   " + nodeNDir.getAbsolutePath());
         FileUtils.copyDirectory(node1Dir, nodeNDir);
@@ -532,7 +585,12 @@ protected void runCloudExample(CommandLine cli) throws Exception {
       // start the other nodes
       for (int n = 1; n < numNodes; n++)
         startSolr(
-            new File(cloudDir, "node" + (n + 1) + "/solr"), true, cli, cloudPorts[n], zkHost, 30);
+            new File(solrHomeDir, "node" + (n + 1) + "/solr"),
+            true,
+            cli,
+            cloudPorts[n],
+            zkHost,
+            30);
     }
 
     String solrUrl = SolrCLI.normalizeSolrUrl((String) nodeStatus.get("baseUrl"), false);
@@ -896,7 +954,7 @@ protected Map<String, Object> getNodeStatus(String solrUrl, int maxWaitSecs, Com
     return nodeStatus;
   }
 
-  protected File setupExampleDir(File serverDir, File exampleParentDir, String dirName)
+  protected File setupExampleDir(File serverDir, File solrHomeParentDir, String dirName)
       throws IOException {
     File solrXml = new File(serverDir, "solr/solr.xml");
     if (!solrXml.isFile())
@@ -908,7 +966,7 @@ protected File setupExampleDir(File serverDir, File exampleParentDir, String dir
       throw new IllegalArgumentException(
           "Value of --server-dir option is invalid! " + zooCfg.getAbsolutePath() + " not found!");
 
-    File solrHomeDir = new File(exampleParentDir, dirName + "/solr");
+    File solrHomeDir = new File(solrHomeParentDir, dirName + "/solr");
     if (!solrHomeDir.isDirectory()) {
       echo("Creating Solr home directory " + solrHomeDir);
       solrHomeDir.mkdirs();

solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java

@@ -39,7 +39,6 @@
 import org.apache.commons.exec.CommandLine;
 import org.apache.commons.exec.DefaultExecutor;
 import org.apache.commons.exec.ExecuteResultHandler;
-import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
@@ -318,19 +317,16 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  @LuceneTestCase.Nightly
   public void testTechproductsExample() throws Exception {
     testExample("techproducts");
   }
 
   @Test
-  @LuceneTestCase.Nightly
   public void testSchemalessExample() throws Exception {
     testExample("schemaless");
   }
 
   @Test
-  @LuceneTestCase.Nightly
   public void testFilmsExample() throws Exception {
     testExample("films");
   }
@@ -357,7 +353,7 @@ protected void testExample(String exampleName) throws Exception {
           new String[] {
             "-e", exampleName,
             "--server-dir", solrServerDir.getAbsolutePath(),
-            "--example-dir", solrExampleDir.getAbsolutePath(),
+            "--solr-home-dir", solrExampleDir.getAbsolutePath(),
             "-p", String.valueOf(bindPort)
           };
 
@@ -413,13 +409,13 @@ protected void testExample(String exampleName) throws Exception {
             numFound = solrClient.query(query).getResults().getNumFound();
           }
           assertEquals(
-              "expected 32 docs in the "
+              "expected 31 docs in the "
                   + exampleName
                   + " example but found "
                   + numFound
                   + ", output: "
                   + toolOutput,
-              32,
+              31,
               numFound);
         }
       }

solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java

@@ -128,7 +128,7 @@ public void testLoadDocsIntoGettingStartedCollection() throws Exception {
     CommandLine postCli = SolrCLI.processCommandLineArgs(postTool, argsForPost);
     postTool.runTool(postCli);
 
-    int expectedXmlDocCount = 32;
+    int expectedXmlDocCount = 31;
 
     int numFound = 0;
 

solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java

@@ -239,7 +239,7 @@ public void testAddTechproductsProgressively() throws Exception {
     schemaDesignerAPI.query(req, rsp);
     assertNotNull(rsp.getResponseHeader());
     SolrDocumentList results = (SolrDocumentList) rsp.getResponse();
-    assertEquals(48, results.getNumFound());
+    assertEquals(47, results.getNumFound());
 
     // publish schema to a config set that can be used by real collections
     reqParams.clear();

solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java

@@ -323,7 +323,7 @@ public void testMultipleSearchResults() throws Exception {
     assertU(commit());
 
     assertJQ(
-        req("q", "id:myid*", "fl", "*"),
+        req("q", "id:myid*", "fl", "*", "sort", "id asc"),
         "/response/docs==["
             + "{'id':'myid1','test_is_dvo':[101,102,103]},"
             + "{'id':'myid2','test_is_dvo':[201,202]},"