Merge branch 'main' into 2025/01/22/forbid-write-block-removal

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -72,18 +72,19 @@ public void apply(Project project) {
         var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
         var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true");
         var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true");
+        int minJavaVersion = Integer.parseInt(buildParams.getMinimumCompilerVersion().getMajorVersion());
 
         // Ignore version-specific source sets if we are importing into IntelliJ and have not explicitly enabled this.
         // Avoids an IntelliJ bug:
         // https://youtrack.jetbrains.com/issue/IDEA-285640/Compiler-Options-Settings-language-level-is-set-incorrectly-with-JDK-19ea
         if (isIdeaSync == false || ideaSourceSetsEnabled) {
-            List<Integer> mainVersions = findSourceVersions(project);
+            List<Integer> mainVersions = findSourceVersions(project, minJavaVersion);
             List<String> mainSourceSets = new ArrayList<>();
             mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
-            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), 21);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), minJavaVersion);
             List<String> testSourceSets = new ArrayList<>(mainSourceSets);
             testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
-            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), minJavaVersion);
             for (int javaVersion : mainVersions) {
                 String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
                 SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion, true);
@@ -103,6 +104,7 @@ public void apply(Project project) {
     }
 
     private void configureMrjar(Project project) {
+
         var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
         jarTask.configure(task -> { task.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); }); });
 
@@ -222,7 +224,7 @@ private void createTestTask(
         project.getTasks().named("check").configure(checkTask -> checkTask.dependsOn(testTaskProvider));
     }
 
-    private static List<Integer> findSourceVersions(Project project) {
+    private static List<Integer> findSourceVersions(Project project, int minJavaVersion) {
         var srcDir = project.getProjectDir().toPath().resolve("src");
         List<Integer> versions = new ArrayList<>();
         try (var subdirStream = Files.list(srcDir)) {
@@ -231,7 +233,23 @@ private static List<Integer> findSourceVersions(Project project) {
                 String sourcesetName = sourceSetPath.getFileName().toString();
                 Matcher sourcesetMatcher = MRJAR_SOURCESET_PATTERN.matcher(sourcesetName);
                 if (sourcesetMatcher.matches()) {
-                    versions.add(Integer.parseInt(sourcesetMatcher.group(1)));
+                    int version = Integer.parseInt(sourcesetMatcher.group(1));
+                    if (version < minJavaVersion) {
+                        // NOTE: We allow mainNN for the min java version so that incubating modules can be used without warnings.
+                        // It is a workaround for https://bugs.openjdk.org/browse/JDK-8187591. Once min java is 22, we
+                        // can use the SuppressWarnings("preview") in the code using incubating modules and this check
+                        // can change to <=
+                        throw new IllegalArgumentException(
+                            "Found src dir '"
+                                + sourcesetName
+                                + "' for Java "
+                                + version
+                                + " but multi-release jar sourceset should have version "
+                                + minJavaVersion
+                                + " or greater"
+                        );
+                    }
+                    versions.add(version);
                 }
             }
         } catch (IOException e) {

docs/changelog/120392.yaml

@@ -0,0 +1,6 @@
+pr: 120392
+summary: Test/107515 restore template with match only text mapper it fail
+area: Search
+type: bug
+issues:
+ - 107515

docs/changelog/120487.yaml

@@ -0,0 +1,5 @@
+pr: 120487
+summary: Fix cat_component_templates documentation
+area: CAT APIs
+type: bug
+issues: []

docs/changelog/120494.yaml

@@ -0,0 +1,5 @@
+pr: 120494
+summary: Update grammar to rely on `indexPattern` instead of identifier in join target
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/120617.yaml

@@ -0,0 +1,5 @@
+pr: 120617
+summary: Fix queries with document level security on lookup indexes
+area: ES|QL
+type: bug
+issues: [120509]

docs/changelog/120717.yaml

@@ -0,0 +1,6 @@
+pr: 120717
+summary: Fix LTR rescorer throws 'local model reference is null' on multi-shards index when explained is enabled
+area: Ranking
+type: bug
+issues:
+ - 120739

docs/changelog/120748.yaml

@@ -0,0 +1,15 @@
+pr: 120748
+summary: Removing support for types field in watcher search
+area: Watcher
+type: breaking
+issues: []
+breaking:
+  title: Removing support for types field in watcher search
+  area: REST API
+  details: >-
+    Previously, setting the `input.search.request.types` field in the payload when creating a watcher to an empty array
+    was  allowed, although it resulted in a deprecation warning and had no effect (and any value other than an empty
+    array would result in an error). Now, support for this field is entirely removed, and the empty array will also
+    result in an error.
+  impact: Users should stop setting this field (which did not have any effect anyway).
+  notable: false

docs/changelog/120752.yaml

@@ -0,0 +1,6 @@
+pr: 120752
+summary: Refresh source index before reindexing data stream index
+area: Data streams
+type: bug
+issues:
+ - 120314

docs/changelog/120753.yaml

@@ -0,0 +1,5 @@
+pr: 120753
+summary: Optimize `IngestDocMetadata` `isAvailable`
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/120781.yaml

@@ -0,0 +1,5 @@
+pr: 120781
+summary: Add back `keep_alive` to `async_search.submit` rest-api-spec
+area: Search
+type: bug
+issues: []

docs/changelog/120809.yaml

@@ -0,0 +1,6 @@
+pr: 120809
+summary: LTR sometines throw `NullPointerException:` Cannot read field "approximation"
+  because "top" is null
+area: Ranking
+type: bug
+issues: []

docs/reference/indices/index-templates.asciidoc

@@ -40,6 +40,7 @@ template with the highest priority is used.
 following index patterns:
 
 // tag::built-in-index-template-patterns[]
+- `.kibana-reporting*`
 - `logs-*-*`
 - `metrics-*-*`
 - `synthetics-*-*`

docs/reference/indices/put-component-template.asciidoc

@@ -97,6 +97,7 @@ Name of the component template to create.
 {es} includes the following built-in component templates:
 
 // tag::built-in-component-templates[]
+- `kibana-reporting@settings`
 - `logs@mappings`
 - `logs@settings`
 - `metrics@mappings`

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java

@@ -14,21 +14,51 @@
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.net.spi.InetAddressResolver;
+import java.net.spi.InetAddressResolverProvider;
 
 class VersionSpecificNetworkChecks {
-    static void createInetAddressResolverProvider() {}
+    static void createInetAddressResolverProvider() {
+        var x = new InetAddressResolverProvider() {
+            @Override
+            public InetAddressResolver get(Configuration configuration) {
+                return null;
+            }
+
+            @Override
+            public String name() {
+                return "TEST";
+            }
+        };
+    }
 
     static void httpClientSend() throws InterruptedException {
-        HttpClient httpClient = HttpClient.newBuilder().build();
-        try {
-            httpClient.send(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
-        } catch (IOException e) {
-            // Expected, the send action may fail with these parameters (but after it run the entitlement check in the prologue)
+        try (HttpClient httpClient = HttpClient.newBuilder().build()) {
+            // Shutdown the client, so the send action will shortcut before actually executing any network operation
+            // (but after it run our check in the prologue)
+            httpClient.shutdown();
+            try {
+                httpClient.send(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
+            } catch (IOException e) {
+                // Expected, since we shut down the client
+            }
         }
     }
 
     static void httpClientSendAsync() {
-        HttpClient httpClient = HttpClient.newBuilder().build();
-        httpClient.sendAsync(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
+        try (HttpClient httpClient = HttpClient.newBuilder().build()) {
+            // Shutdown the client, so the send action will return before actually executing any network operation
+            // (but after it run our check in the prologue)
+            httpClient.shutdown();
+            var future = httpClient.sendAsync(
+                HttpRequest.newBuilder(URI.create("http://localhost")).build(),
+                HttpResponse.BodyHandlers.discarding()
+            );
+            assert future.isCompletedExceptionally();
+            future.exceptionally(ex -> {
+                assert ex instanceof IOException;
+                return null;
+            });
+        }
     }
 }

libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public abstract class AbstractEntitlementsIT extends ESRestTestCase {
+
+    static final EntitlementsTestRule.PolicyBuilder ALLOWED_TEST_ENTITLEMENTS = (builder, tempDir) -> {
+        builder.value("create_class_loader");
+        builder.value("set_https_connection_properties");
+        builder.value("inbound_network");
+        builder.value("outbound_network");
+        builder.value("load_native_libraries");
+        builder.value(
+            Map.of(
+                "write_system_properties",
+                Map.of("properties", List.of("es.entitlements.checkSetSystemProperty", "es.entitlements.checkClearSystemProperty"))
+            )
+        );
+    };
+
+    private final String actionName;
+    private final boolean expectAllowed;
+
+    AbstractEntitlementsIT(String actionName, boolean expectAllowed) {
+        this.actionName = actionName;
+        this.expectAllowed = expectAllowed;
+    }
+
+    private Response executeCheck() throws IOException {
+        var request = new Request("GET", "/_entitlement_check");
+        request.addParameter("action", actionName);
+        return client().performRequest(request);
+    }
+
+    public void testAction() throws IOException {
+        logger.info("Executing Entitlement test for [{}]", actionName);
+        if (expectAllowed) {
+            Response result = executeCheck();
+            assertThat(result.getStatusLine().getStatusCode(), equalTo(200));
+        } else {
+            var exception = expectThrows(IOException.class, this::executeCheck);
+            assertThat(exception.getMessage(), containsString("not_entitled_exception"));
+        }
+    }
+}