Merge branch 'main' into 2024/05/30/do-not-request-gaps-when-sub-rang…

…e-is-completed

docs/changelog/105709.yaml

@@ -1,6 +1,5 @@
 pr: 105709
-summary: Disable validate when rewrite parameter is sent and the index access control
-  list is non-null
+summary: Apply stricter Document Level Security (DLS) rules for the validate query API with the rewrite parameter.
 area: Security
 type: bug
 issues: []

docs/changelog/105714.yaml

@@ -1,5 +1,5 @@
 pr: 105714
-summary: Cross check livedocs for terms aggs when index access control list is non-null
+summary: Apply stricter Document Level Security (DLS) rules for terms aggregations when min_doc_count is set to 0.
 area: "Aggregations"
 type: bug
 issues: []

docs/changelog/109078.yaml

@@ -0,0 +1,5 @@
+pr: 109078
+summary: Expose API Key cache metrics
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/109174.yaml

@@ -0,0 +1,5 @@
+pr: 109174
+summary: "ESQL: Change \"substring\" function to not return null on empty string"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/109205.yaml

@@ -0,0 +1,6 @@
+pr: 109205
+summary: "ESQL: Fix `IpPrefix` function not handling correctly `ByteRefs`"
+area: ES|QL
+type: bug
+issues:
+ - 109198

docs/reference/migration/migrate_8_14.asciidoc

@@ -16,5 +16,28 @@ coming::[8.14.0]
 [[breaking-changes-8.14]]
 === Breaking changes
 
-There are no breaking changes in {es} 8.14.
+The following changes in {es} 8.14 might affect your applications
+and prevent them from operating normally.
+Before upgrading to 8.14, review these changes and take the described steps
+to mitigate the impact.
 
+[discrete]
+[[breaking-changes-8.14-0]]
+
+[discrete]
+[[breaking_814_dls_changes]]
+==== Stricter Document Level Security (DLS)
+
+[[stricter_dls_814]]
+.Document Level Security (DLS) applies stricter checks for the validate query API and for terms aggregations when min_doc_count is set to 0.
+
+[%collapsible]
+====
+*Details* +
+When Document Level Security (DLS) is applied to terms aggregations and min_doc_count is set to 0, stricter security rules apply.
+
+When Document Level Security (DLS) is applied to the validate query API with the rewrite parameter, stricter security rules apply.
+
+*Impact* +
+If needed, test workflows with DLS enabled to ensure that the stricter security rules do not impact your application.
+====

muted-tests.yml

@@ -46,12 +46,6 @@ tests:
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
-- class: "org.elasticsearch.upgrades.SearchStatesIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109190"
-  method: "testBWCSearchStates"
-- class: "org.elasticsearch.xpack.esql.CsvTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/109198"
-  method: "test {ip.IpPrefixLengthFromColumn}"
 - class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109200"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"

server/src/main/java/org/elasticsearch/cluster/routing/package-info.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * <p>The cluster routing package provides routing information and manages shard
+ * allocations in a cluster. The routing part contains two different views of
+ * shards in {@link org.elasticsearch.cluster.routing.RoutingTable} and
+ * {@link org.elasticsearch.cluster.routing.RoutingNodes}. RoutingTable provides
+ * a view from index to shard to node. RoutingNodes provides view from node to
+ * shard. Shard allocation is a process of assigning and moving shards between
+ * nodes. For more details about allocation see
+ * {@link org.elasticsearch.cluster.routing.allocation}.</p>
+ *
+ * <b>Routing Table</b>
+ *
+ * <p>The routing table provide a view of global cluster state from an index
+ * perspective. It's a mapping from indices to shards and shards to nodes, where
+ * the relationship between shard and node may not exist if a shard
+ * allocation is not possible.
+ *
+ * <p>{@link org.elasticsearch.cluster.routing.RoutingTable} is the access
+ * point to the routing information. RoutingTable is a part of the
+ * {@link org.elasticsearch.cluster.ClusterState}. It maps index names to
+ * {@link org.elasticsearch.cluster.routing.IndexRoutingTable}.
+ * {@link org.elasticsearch.cluster.routing.IndexRoutingTable}, in turn,
+ * holds a list of shards in that index,
+ * {@link org.elasticsearch.cluster.routing.IndexShardRoutingTable}. Each
+ * shard has one or more instances: a primary and replicas. An
+ * IndexShardRoutingTable contains information about all shard instances for
+ * a specific shard, {@link org.elasticsearch.cluster.routing.ShardRouting}.
+ * The ShardRouting is the state of a shard instance in a cluster. There are
+ * several states of ShardRouting: unassigned, initializing, relocating,
+ * started.</p>
+ *
+ * An example of routing table:
+ *
+ * <pre>
+ * {@code
+ * └── ClusterState
+ *     └── RoutingTable
+ *         ├── index1-IndexRoutingTable
+ *         │   ├── shard1-IndexShardRoutingTable
+ *         │   │   ├── primary-ShardRouting
+ *         │   │   │   └── STARTED-node1
+ *         │   │   ├── replica1-ShardRouting
+ *         │   │   │   └── INITIALIZING-node2
+ *         │   │   └── replica2-ShardRouting
+ *         │   │       └── UNASSIGNED
+ *         │   └── shard2-IndexShardRoutingTable ...
+ *         └── index2-IndexRoutingTable ...
+ * }
+ * </pre>
+ *
+ *
+ * <b>Routing Nodes</b>
+ *
+ * <p>{@link org.elasticsearch.cluster.routing.RoutingNode} provides a view
+ * from node to shard routing. There is a RoutingNode for every
+ * {@link org.elasticsearch.cluster.node.DiscoveryNode}. It contains
+ * information about  all shards and their state on this node.
+ * {@link org.elasticsearch.cluster.routing.RoutingNodes} (plural) provide
+ * aggregated view from all cluster nodes. It supports finding all shards by
+ * specific node or finding all nodes for specific shard.</p>
+ *
+ * <b>Reroute</b>
+ *
+ * <p>Reroute is a process of routing update in the cluster. Routing update
+ * may start allocation process, which assign or move shards around. When
+ * cluster has an update that impacts routing (for example: node join
+ * cluster, index created, snapshot restored, ...) the code that handles
+ * cluster update should trigger reroute. There are 2 ways to trigger reroute -
+ * batched with
+ * {@link org.elasticsearch.cluster.routing.BatchedRerouteService} and
+ * unbatched
+ * {@link org.elasticsearch.cluster.routing.allocation.AllocationService}.
+ * Batched reroute can combine multiple requests into one (used when starting
+ * initializing shards). Unbatched reroute allows to mix other cluster state
+ * changes that might be required to create or delete index.</p>
+ */
+package org.elasticsearch.cluster.routing;

server/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
@@ -1843,11 +1844,18 @@ public void diff(Settings.Builder builder, Settings source, Settings defaultSett
     }
 
     static void logSettingUpdate(Setting<?> setting, Settings current, Settings previous, Logger logger) {
-        if (logger.isInfoEnabled()) {
+        Level level = setting.hasIndexScope() ? Level.DEBUG : Level.INFO;
+        if (logger.isEnabled(level)) {
             if (setting.isFiltered()) {
-                logger.info("updating [{}]", setting.key);
+                logger.log(level, "updating [{}]", setting.key);
             } else {
-                logger.info("updating [{}] from [{}] to [{}]", setting.key, setting.getLogString(previous), setting.getLogString(current));
+                logger.log(
+                    level,
+                    "updating [{}] from [{}] to [{}]",
+                    setting.key,
+                    setting.getLogString(previous),
+                    setting.getLogString(current)
+                );
             }
         }
     }

server/src/test/java/org/elasticsearch/common/settings/SettingTests.java

@@ -1423,7 +1423,7 @@ public void testNonSecureSettingInKeystore() {
     }
 
     @TestLogging(
-        value = "org.elasticsearch.common.settings.IndexScopedSettings:INFO",
+        value = "org.elasticsearch.common.settings.IndexScopedSettings:DEBUG",
         reason = "to ensure we log INFO-level messages from IndexScopedSettings"
     )
     public void testLogSettingUpdate() throws Exception {
@@ -1438,7 +1438,7 @@ public void testLogSettingUpdate() throws Exception {
                 new MockLog.SeenEventExpectation(
                     "message",
                     "org.elasticsearch.common.settings.IndexScopedSettings",
-                    Level.INFO,
+                    Level.DEBUG,
                     "updating [index.refresh_interval] from [20s] to [10s]"
                 ) {
                     @Override

server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java

@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configurator;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.rest.RestRequest;
@@ -99,6 +100,31 @@ public void testFilteredSettingIsNotLogged() throws Exception {
         );
     }
 
+    public void testIndexScopeSettingUpdateLoggedAsDebug() throws Exception {
+        Settings oldSettings = Settings.builder().put("key", "old").build();
+        Settings newSettings = Settings.builder().put("key", "new").build();
+
+        // With INFO log level nothing gets logged.
+        Setting<String> filteredSetting = Setting.simpleString("key", Property.IndexScope);
+        assertExpectedLogMessages((testLogger) -> Setting.logSettingUpdate(filteredSetting, newSettings, oldSettings, testLogger));
+
+        try {
+            // With DEBUG log level something gets logged
+            Configurator.setLevel("org.elasticsearch.test", Level.DEBUG);
+            assertExpectedLogMessages(
+                (logger) -> Setting.logSettingUpdate(filteredSetting, newSettings, oldSettings, logger),
+                new MockLog.SeenEventExpectation(
+                    "regular logging",
+                    "org.elasticsearch.test",
+                    Level.DEBUG,
+                    "updating [key] from [old] to [new]"
+                )
+            );
+        } finally {
+            Configurator.setLevel("org.elasticsearch.test", Level.INFO);
+        }
+    }
+
     public void testRegularSettingUpdateIsFullyLogged() throws Exception {
         Settings oldSettings = Settings.builder().put("key", "old").build();
         Settings newSettings = Settings.builder().put("key", "new").build();

server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java

@@ -39,11 +39,11 @@ public void append(LogEvent event) {
             String message = event.getMessage().getFormattedMessage();
             if (event.getLevel() == Level.TRACE && event.getLoggerName().endsWith("lucene.iw")) {
             }
-            if (event.getLevel() == Level.INFO
+            if (event.getLevel() == Level.DEBUG
                 && message.contains("updating [index.merge.scheduler.max_thread_count] from [10000] to [1]")) {
                 sawUpdateMaxThreadCount = true;
             }
-            if (event.getLevel() == Level.INFO
+            if (event.getLevel() == Level.DEBUG
                 && message.contains("updating [index.merge.scheduler.auto_throttle] from [true] to [false]")) {
                 sawUpdateAutoThrottle = true;
             }

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java

@@ -10,7 +10,6 @@
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.core.Tuple;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -25,11 +24,13 @@
  */
 class ProgressListenableActionFuture extends PlainActionFuture<Long> {
 
+    private record PositionAndListener(long position, ActionListener<Long> listener) {}
+
     protected final long start;
     protected final long end;
 
     // modified under 'this' mutex
-    private volatile List<Tuple<Long, ActionListener<Long>>> listeners;
+    private volatile List<PositionAndListener> listeners;
     protected volatile long progress;
     private volatile boolean completed;
 
@@ -55,7 +56,7 @@ private boolean invariant() {
             assert completed == false || listeners == null;
             assert start <= progress : start + " <= " + progress;
             assert progress <= end : progress + " <= " + end;
-            assert listeners == null || listeners.stream().allMatch(listener -> progress < listener.v1());
+            assert listeners == null || listeners.stream().allMatch(listener -> progress < listener.position());
         }
         return true;
     }
@@ -78,17 +79,20 @@ public void onProgress(final long progressValue) {
             assert false : end + " < " + progressValue;
             throw new IllegalArgumentException("Cannot update progress with a value greater than [end=" + end + ']');
         }
+        if (progressValue == end) {
+            return; // reached the end of the range, listeners will be completed by {@link #onResponse(Long)}
+        }
 
         List<ActionListener<Long>> listenersToExecute = null;
         synchronized (this) {
             assert this.progress < progressValue : this.progress + " < " + progressValue;
             this.progress = progressValue;
 
-            final List<Tuple<Long, ActionListener<Long>>> listenersCopy = this.listeners;
+            final List<PositionAndListener> listenersCopy = this.listeners;
             if (listenersCopy != null) {
-                List<Tuple<Long, ActionListener<Long>>> listenersToKeep = null;
-                for (Tuple<Long, ActionListener<Long>> listener : listenersCopy) {
-                    if (progressValue < listener.v1()) {
+                List<PositionAndListener> listenersToKeep = null;
+                for (PositionAndListener listener : listenersCopy) {
+                    if (progressValue < listener.position()) {
                         if (listenersToKeep == null) {
                             listenersToKeep = new ArrayList<>();
                         }
@@ -97,7 +101,7 @@ public void onProgress(final long progressValue) {
                         if (listenersToExecute == null) {
                             listenersToExecute = new ArrayList<>();
                         }
-                        listenersToExecute.add(listener.v2());
+                        listenersToExecute.add(listener.listener());
                     }
                 }
                 this.listeners = listenersToKeep;
@@ -134,15 +138,16 @@ private void ensureNotCompleted() {
     @Override
     protected void done(boolean success) {
         super.done(success);
-        final List<Tuple<Long, ActionListener<Long>>> listenersToExecute;
+        final List<PositionAndListener> listenersToExecute;
+        assert invariant();
         synchronized (this) {
             assert completed == false;
             completed = true;
             listenersToExecute = this.listeners;
             listeners = null;
         }
         if (listenersToExecute != null) {
-            listenersToExecute.stream().map(Tuple::v2).forEach(listener -> executeListener(listener, this::actionResult));
+            listenersToExecute.forEach(listener -> executeListener(listener.listener(), this::actionResult));
         }
         assert invariant();
     }
@@ -162,11 +167,11 @@ public void addListener(ActionListener<Long> listener, long value) {
             if (completed || value <= progressValue) {
                 executeImmediate = true;
             } else {
-                List<Tuple<Long, ActionListener<Long>>> listenersCopy = this.listeners;
+                List<PositionAndListener> listenersCopy = this.listeners;
                 if (listenersCopy == null) {
                     listenersCopy = new ArrayList<>();
                 }
-                listenersCopy.add(Tuple.tuple(value, listener));
+                listenersCopy.add(new PositionAndListener(value, listener));
                 this.listeners = listenersCopy;
             }
         }

x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java

@@ -203,21 +203,19 @@ public void testCallsListenerWhenWholeRangeIsAvailable() {
                 final SparseFileTracker.Gap gap = gaps.get(gapIndex);
                 assertThat(gap.start(), greaterThanOrEqualTo(start));
                 assertThat(gap.end(), lessThanOrEqualTo(end));
-                // listener is notified when the last gap is completed
-                final AtomicBoolean shouldNotifyListener = new AtomicBoolean();
                 for (long i = gap.start(); i < gap.end(); i++) {
                     assertThat(fileContents[toIntBytes(i)], equalTo(UNAVAILABLE));
                     fileContents[toIntBytes(i)] = AVAILABLE;
-                    // listener is notified when the progress reached the last byte of the last gap
-                    if ((gapIndex == gaps.size() - 1) && (i == gap.end() - 1L)) {
-                        assertTrue(shouldNotifyListener.compareAndSet(false, true));
-                        expectNotification.set(true);
-                    }
                     gap.onProgress(i + 1L);
-                    assertThat(wasNotified.get(), equalTo(shouldNotifyListener.get()));
+                    assertThat(wasNotified.get(), equalTo(false));
+                }
+                // listener is notified when the last gap is completed
+                if (gapIndex == gaps.size() - 1) {
+                    expectNotification.set(true);
                 }
-                assertThat(wasNotified.get(), equalTo(shouldNotifyListener.get()));
+                assertThat(wasNotified.get(), equalTo(false));
                 gap.onCompletion();
+                assertThat(wasNotified.get(), equalTo(expectNotification.get()));
             }
             assertTrue(wasNotified.get());
         }