From 1a4ceb47115fa1d241c54af703f05f688891032b Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 28 Jan 2025 08:23:44 -0500 Subject: [PATCH 01/10] [8.x] Esql - Support date nanos in date extract function (#120727) (#120908) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Esql - Support date nanos in date extract function (#120727) Resolves https://github.com/elastic/elasticsearch/issues/110000 Add support for running the date extract function on nanosecond dates. * Fix switch error * ESQL: Fix DateExtract with nanos tests --------- Co-authored-by: Iván Cea Fontenla Co-authored-by: Iván Cea Fontenla --- docs/changelog/120727.yaml | 6 + .../kibana/definition/date_extract.json | 36 ++++ .../functions/types/date_extract.asciidoc | 2 + .../src/main/resources/date_nanos.csv-spec | 22 +++ .../DateExtractConstantMillisEvaluator.java | 137 ++++++++++++++ .../DateExtractConstantNanosEvaluator.java | 137 ++++++++++++++ .../date/DateExtractMillisEvaluator.java | 169 ++++++++++++++++++ .../date/DateExtractNanosEvaluator.java | 169 ++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 +- .../function/scalar/date/DateExtract.java | 66 +++++-- .../esql/type/EsqlDataTypeConverter.java | 25 ++- .../scalar/date/DateExtractErrorTests.java | 2 +- .../scalar/date/DateExtractTests.java | 108 ++++++----- 13 files changed, 826 insertions(+), 58 deletions(-) create mode 100644 docs/changelog/120727.yaml create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantMillisEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractMillisEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractNanosEvaluator.java diff --git a/docs/changelog/120727.yaml b/docs/changelog/120727.yaml new file mode 100644 index 0000000000000..4d0241e6baad1 --- /dev/null +++ b/docs/changelog/120727.yaml @@ -0,0 +1,6 @@ +pr: 120727 +summary: Esql - Support date nanos in date extract function +area: ES|QL +type: enhancement +issues: + - 110000 diff --git a/docs/reference/esql/functions/kibana/definition/date_extract.json b/docs/reference/esql/functions/kibana/definition/date_extract.json index c6dc6583f324d..0ababf80d9137 100644 --- a/docs/reference/esql/functions/kibana/definition/date_extract.json +++ b/docs/reference/esql/functions/kibana/definition/date_extract.json @@ -22,6 +22,24 @@ "variadic" : false, "returnType" : "long" }, + { + "params" : [ + { + "name" : "datePart", + "type" : "keyword", + "optional" : false, + "description" : "Part of the date to extract. Can be: `aligned_day_of_week_in_month`, `aligned_day_of_week_in_year`, `aligned_week_of_month`, `aligned_week_of_year`, `ampm_of_day`, `clock_hour_of_ampm`, `clock_hour_of_day`, `day_of_month`, `day_of_week`, `day_of_year`, `epoch_day`, `era`, `hour_of_ampm`, `hour_of_day`, `instant_seconds`, `micro_of_day`, `micro_of_second`, `milli_of_day`, `milli_of_second`, `minute_of_day`, `minute_of_hour`, `month_of_year`, `nano_of_day`, `nano_of_second`, `offset_seconds`, `proleptic_month`, `second_of_day`, `second_of_minute`, `year`, or `year_of_era`. Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values. If `null`, the function returns `null`." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { @@ -39,6 +57,24 @@ ], "variadic" : false, "returnType" : "long" + }, + { + "params" : [ + { + "name" : "datePart", + "type" : "text", + "optional" : false, + "description" : "Part of the date to extract. Can be: `aligned_day_of_week_in_month`, `aligned_day_of_week_in_year`, `aligned_week_of_month`, `aligned_week_of_year`, `ampm_of_day`, `clock_hour_of_ampm`, `clock_hour_of_day`, `day_of_month`, `day_of_week`, `day_of_year`, `epoch_day`, `era`, `hour_of_ampm`, `hour_of_day`, `instant_seconds`, `micro_of_day`, `micro_of_second`, `milli_of_day`, `milli_of_second`, `minute_of_day`, `minute_of_hour`, `month_of_year`, `nano_of_day`, `nano_of_second`, `offset_seconds`, `proleptic_month`, `second_of_day`, `second_of_minute`, `year`, or `year_of_era`. Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values. If `null`, the function returns `null`." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "long" } ], "examples" : [ diff --git a/docs/reference/esql/functions/types/date_extract.asciidoc b/docs/reference/esql/functions/types/date_extract.asciidoc index ec9bf70c221cc..207e09b00f786 100644 --- a/docs/reference/esql/functions/types/date_extract.asciidoc +++ b/docs/reference/esql/functions/types/date_extract.asciidoc @@ -6,5 +6,7 @@ |=== datePart | date | result keyword | date | long +keyword | date_nanos | long text | date | long +text | date_nanos | long |=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 2e660e501a43e..e48f1e9d0b9ae 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -500,6 +500,28 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +Date nanos date extract +required_capability: date_nanos_date_extract + +FROM date_nanos +| EVAL nn = MV_MAX(nanos) +| EVAL year = DATE_EXTRACT("year", nn), ns = DATE_EXTRACT("nano_of_second", nn) +| KEEP nn, year, ns +| SORT nn DESC; + +nn:date_nanos | year:long | ns:long +2023-10-23T13:55:01.543123456Z | 2023 | 543123456 +2023-10-23T13:53:55.832987654Z | 2023 | 832987654 +2023-10-23T13:52:55.015787878Z | 2023 | 015787878 +2023-10-23T13:51:54.732102837Z | 2023 | 732102837 +2023-10-23T13:33:34.937193000Z | 2023 | 937193000 +2023-10-23T12:27:28.948000000Z | 2023 | 948000000 +2023-10-23T12:15:03.360103847Z | 2023 | 360103847 +2023-10-23T12:15:03.360103847Z | 2023 | 360103847 +2023-03-23T12:15:03.360103847Z | 2023 | 360103847 +2023-03-23T12:15:03.360103847Z | 2023 | 360103847 +; + date nanos to long, index version required_capability: to_date_nanos diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantMillisEvaluator.java new file mode 100644 index 0000000000000..11da518a01ce1 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantMillisEvaluator.java @@ -0,0 +1,137 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import java.time.temporal.ChronoField; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractConstantMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractConstantMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + ChronoField chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock); + } + return eval(page.getPositionCount(), valueVector).asBlock(); + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendLong(DateExtract.processMillis(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), this.chronoField, this.zone)); + } + return result.build(); + } + } + + public LongVector eval(int positionCount, LongVector valueVector) { + try(LongVector.FixedBuilder result = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendLong(p, DateExtract.processMillis(valueVector.getLong(p), this.chronoField, this.zone)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractConstantMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + ChronoField chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractConstantMillisEvaluator get(DriverContext context) { + return new DateExtractConstantMillisEvaluator(source, value.get(context), chronoField, zone, context); + } + + @Override + public String toString() { + return "DateExtractConstantMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantNanosEvaluator.java new file mode 100644 index 0000000000000..bbd0a59c87ceb --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantNanosEvaluator.java @@ -0,0 +1,137 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import java.time.temporal.ChronoField; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractConstantNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractConstantNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + ChronoField chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock); + } + return eval(page.getPositionCount(), valueVector).asBlock(); + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendLong(DateExtract.processNanos(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), this.chronoField, this.zone)); + } + return result.build(); + } + } + + public LongVector eval(int positionCount, LongVector valueVector) { + try(LongVector.FixedBuilder result = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendLong(p, DateExtract.processNanos(valueVector.getLong(p), this.chronoField, this.zone)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractConstantNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + ChronoField chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractConstantNanosEvaluator get(DriverContext context) { + return new DateExtractConstantNanosEvaluator(source, value.get(context), chronoField, zone, context); + } + + @Override + public String toString() { + return "DateExtractConstantNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractMillisEvaluator.java new file mode 100644 index 0000000000000..edc0b2cb0f0ce --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractMillisEvaluator.java @@ -0,0 +1,169 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final EvalOperator.ExpressionEvaluator chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + EvalOperator.ExpressionEvaluator chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + try (BytesRefBlock chronoFieldBlock = (BytesRefBlock) chronoField.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + BytesRefVector chronoFieldVector = chronoFieldBlock.asVector(); + if (chronoFieldVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + return eval(page.getPositionCount(), valueVector, chronoFieldVector); + } + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock, BytesRefBlock chronoFieldBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (chronoFieldBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (chronoFieldBlock.getValueCount(p) != 1) { + if (chronoFieldBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendLong(DateExtract.processMillis(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), chronoFieldBlock.getBytesRef(chronoFieldBlock.getFirstValueIndex(p), chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public LongBlock eval(int positionCount, LongVector valueVector, + BytesRefVector chronoFieldVector) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendLong(DateExtract.processMillis(valueVector.getLong(p), chronoFieldVector.getBytesRef(p, chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value, chronoField); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final EvalOperator.ExpressionEvaluator.Factory chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + EvalOperator.ExpressionEvaluator.Factory chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractMillisEvaluator get(DriverContext context) { + return new DateExtractMillisEvaluator(source, value.get(context), chronoField.get(context), zone, context); + } + + @Override + public String toString() { + return "DateExtractMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractNanosEvaluator.java new file mode 100644 index 0000000000000..97a04f0d06a74 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractNanosEvaluator.java @@ -0,0 +1,169 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final EvalOperator.ExpressionEvaluator chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + EvalOperator.ExpressionEvaluator chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + try (BytesRefBlock chronoFieldBlock = (BytesRefBlock) chronoField.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + BytesRefVector chronoFieldVector = chronoFieldBlock.asVector(); + if (chronoFieldVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + return eval(page.getPositionCount(), valueVector, chronoFieldVector); + } + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock, BytesRefBlock chronoFieldBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (chronoFieldBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (chronoFieldBlock.getValueCount(p) != 1) { + if (chronoFieldBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendLong(DateExtract.processNanos(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), chronoFieldBlock.getBytesRef(chronoFieldBlock.getFirstValueIndex(p), chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public LongBlock eval(int positionCount, LongVector valueVector, + BytesRefVector chronoFieldVector) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendLong(DateExtract.processNanos(valueVector.getLong(p), chronoFieldVector.getBytesRef(p, chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value, chronoField); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final EvalOperator.ExpressionEvaluator.Factory chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + EvalOperator.ExpressionEvaluator.Factory chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractNanosEvaluator get(DriverContext context) { + return new DateExtractNanosEvaluator(source, value.get(context), chronoField.get(context), zone, context); + } + + @Override + public String toString() { + return "DateExtractNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 213cf011f03f9..0c35ccf477996 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -358,7 +358,10 @@ public enum Cap { * Support Least and Greatest functions on Date Nanos type */ LEAST_GREATEST_FOR_DATENANOS(), - + /** + * support date extract function for date nanos + */ + DATE_NANOS_DATE_EXTRACT(), /** * Support add and subtract on date nanos */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index 7fc5d82441802..7d8648a672ff8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -33,10 +33,10 @@ import java.time.temporal.ChronoField; import java.util.List; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isDate; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.EsqlConverter.STRING_TO_CHRONO_FIELD; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.chronoToLong; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.chronoToLongNanos; public class DateExtract extends EsqlConfigurationFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -72,7 +72,11 @@ public DateExtract( Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values.\n If `null`, the function returns `null`.""") Expression chronoFieldExp, - @Param(name = "date", type = "date", description = "Date expression. If `null`, the function returns `null`.") Expression field, + @Param( + name = "date", + type = { "date", "date_nanos" }, + description = "Date expression. If `null`, the function returns `null`." + ) Expression field, Configuration configuration ) { super(source, List.of(chronoFieldExp, field), configuration); @@ -109,17 +113,42 @@ public String getWriteableName() { @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - var fieldEvaluator = toEvaluator.apply(children().get(1)); + boolean isNanos = switch (field().dataType()) { + case DATETIME -> false; + case DATE_NANOS -> true; + default -> throw new UnsupportedOperationException( + "Unsupported field type [" + + field().dataType().name() + + "]. " + + "If you're seeing this, there's a bug in DateExtract.resolveType" + ); + }; + + ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(children().get(1)); + + // Constant chrono field if (children().get(0).foldable()) { ChronoField chrono = chronoField(toEvaluator.foldCtx()); if (chrono == null) { BytesRef field = (BytesRef) children().get(0).fold(toEvaluator.foldCtx()); throw new InvalidArgumentException("invalid date field for [{}]: {}", sourceText(), field.utf8ToString()); } - return new DateExtractConstantEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); + + if (isNanos) { + return new DateExtractConstantNanosEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); + } else { + return new DateExtractConstantMillisEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); + } } + var chronoEvaluator = toEvaluator.apply(children().get(0)); - return new DateExtractEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); + + if (isNanos) { + return new DateExtractNanosEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); + } else { + return new DateExtractMillisEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); + } + } private ChronoField chronoField(FoldContext ctx) { @@ -138,16 +167,26 @@ private ChronoField chronoField(FoldContext ctx) { return chronoField; } - @Evaluator(warnExceptions = { IllegalArgumentException.class }) - static long process(long value, BytesRef chronoField, @Fixed ZoneId zone) { + @Evaluator(extraName = "Millis", warnExceptions = { IllegalArgumentException.class }) + static long processMillis(long value, BytesRef chronoField, @Fixed ZoneId zone) { return chronoToLong(value, chronoField, zone); } - @Evaluator(extraName = "Constant") - static long process(long value, @Fixed ChronoField chronoField, @Fixed ZoneId zone) { + @Evaluator(extraName = "ConstantMillis") + static long processMillis(long value, @Fixed ChronoField chronoField, @Fixed ZoneId zone) { return chronoToLong(value, chronoField, zone); } + @Evaluator(extraName = "Nanos", warnExceptions = { IllegalArgumentException.class }) + static long processNanos(long value, BytesRef chronoField, @Fixed ZoneId zone) { + return chronoToLongNanos(value, chronoField, zone); + } + + @Evaluator(extraName = "ConstantNanos") + static long processNanos(long value, @Fixed ChronoField chronoField, @Fixed ZoneId zone) { + return chronoToLongNanos(value, chronoField, zone); + } + @Override public Expression replaceChildren(List newChildren) { return new DateExtract(source(), newChildren.get(0), newChildren.get(1), configuration()); @@ -168,8 +207,15 @@ protected TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } + String operationName = sourceText(); return isStringAndExact(children().get(0), sourceText(), TypeResolutions.ParamOrdinal.FIRST).and( - isDate(children().get(1), sourceText(), TypeResolutions.ParamOrdinal.SECOND) + TypeResolutions.isType( + children().get(1), + DataType::isDate, + operationName, + TypeResolutions.ParamOrdinal.SECOND, + "datetime or date_nanos" + ) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 23c922cb813f7..b5509883aedc4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -471,13 +471,36 @@ private static ChronoField stringToChrono(Object field) { public static long chronoToLong(long dateTime, BytesRef chronoField, ZoneId zone) { ChronoField chrono = ChronoField.valueOf(chronoField.utf8ToString().toUpperCase(Locale.ROOT)); - return Instant.ofEpochMilli(dateTime).atZone(zone).getLong(chrono); + return chronoToLong(dateTime, chrono, zone); } public static long chronoToLong(long dateTime, ChronoField chronoField, ZoneId zone) { return Instant.ofEpochMilli(dateTime).atZone(zone).getLong(chronoField); } + /** + * Extract the given {@link ChronoField} value from a date specified as a long number of nanoseconds since epoch + * @param dateNanos - long nanoseconds since epoch + * @param chronoField - The field to extract + * @param zone - Timezone for the given date + * @return - long representing the given ChronoField value + */ + public static long chronoToLongNanos(long dateNanos, BytesRef chronoField, ZoneId zone) { + ChronoField chrono = ChronoField.valueOf(chronoField.utf8ToString().toUpperCase(Locale.ROOT)); + return chronoToLongNanos(dateNanos, chrono, zone); + } + + /** + * Extract the given {@link ChronoField} value from a date specified as a long number of nanoseconds since epoch + * @param dateNanos - long nanoseconds since epoch + * @param chronoField - The field to extract + * @param zone - Timezone for the given date + * @return - long representing the given ChronoField value + */ + public static long chronoToLongNanos(long dateNanos, ChronoField chronoField, ZoneId zone) { + return DateUtils.toInstant(dateNanos).atZone(zone).getLong(chronoField); + } + /** * The following conversions are between String and other data types. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java index d5b9a06c8738e..feee1dd06f30f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java @@ -35,7 +35,7 @@ protected Expression build(Source source, List args) { protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 0 -> "string"; - case 1 -> "datetime"; + case 1 -> "datetime or date_nanos"; default -> ""; })); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index cd27ce511b317..01a84d7885ed3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -26,6 +26,7 @@ import java.time.Instant; import java.time.ZonedDateTime; import java.time.temporal.ChronoField; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -40,53 +41,70 @@ public DateExtractTests(@Name("TestCase") Supplier te @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( - true, - List.of( - new TestCaseSupplier( - List.of(DataType.KEYWORD, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataType.KEYWORD, "chrono"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date") - ), - "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", - DataType.LONG, - equalTo(2023L) - ) - ), - new TestCaseSupplier( - List.of(DataType.TEXT, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataType.TEXT, "chrono"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date") - ), - "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", - DataType.LONG, - equalTo(2023L) - ) - ), - new TestCaseSupplier( - List.of(DataType.KEYWORD, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("not a unit"), DataType.KEYWORD, "chrono"), - new TestCaseSupplier.TypedData(0L, DataType.DATETIME, "date") + var suppliers = new ArrayList(); - ), - "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", - DataType.LONG, - is(nullValue()) - ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") - .withWarning( - "Line -1:-1: java.lang.IllegalArgumentException: " - + "No enum constant java.time.temporal.ChronoField.NOT A UNIT" + for (var stringType : DataType.stringTypes()) { + suppliers.addAll( + List.of( + new TestCaseSupplier( + List.of(stringType, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("YeAr"), stringType, "chrono"), + new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date") + ), + "DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + equalTo(2023L) + ) + ), + new TestCaseSupplier( + List.of(stringType, DataType.DATE_NANOS), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("YeAr"), stringType, "chrono"), + new TestCaseSupplier.TypedData(1687944333000000000L, DataType.DATE_NANOS, "date") + ), + "DateExtractNanosEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + equalTo(2023L) ) - .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") + ), + new TestCaseSupplier( + List.of(stringType, DataType.DATE_NANOS), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("nano_of_second"), stringType, "chrono"), + new TestCaseSupplier.TypedData(1687944333000123456L, DataType.DATE_NANOS, "date") + ), + "DateExtractNanosEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + equalTo(123456L) + ) + ), + new TestCaseSupplier( + List.of(stringType, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("not a unit"), stringType, "chrono"), + new TestCaseSupplier.TypedData(0L, DataType.DATETIME, "date") + + ), + "DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + is(nullValue()) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning( + "Line -1:-1: java.lang.IllegalArgumentException: " + + "No enum constant java.time.temporal.ChronoField.NOT A UNIT" + ) + .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") + ) ) - ) - ); + ); + } + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testAllChronoFields() { @@ -102,7 +120,7 @@ public void testAllChronoFields() { assertThat(instance.fold(FoldContext.small()), is(date.getLong(value))); assertThat( - DateExtract.process(epochMilli, new BytesRef(value.name()), EsqlTestUtils.TEST_CFG.zoneId()), + DateExtract.processMillis(epochMilli, new BytesRef(value.name()), EsqlTestUtils.TEST_CFG.zoneId()), is(date.getLong(value)) ); } From f146691fc3d4db06881c328ee0b6469c0c1b74eb Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jan 2025 14:49:32 +0100 Subject: [PATCH 02/10] Fix NPE on disabled API key auth cache (#120483) (#121008) Currently, when `xpack.security.authc.api_key.cache.ttl` is set to `0d` API key creation (and invalidation) fail with NPEs. This PR adds null checks to fix this. --- docs/changelog/120483.yaml | 5 +++ .../xpack/security/authc/ApiKeyService.java | 16 ++++--- .../security/authc/ApiKeyServiceTests.java | 44 +++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/120483.yaml diff --git a/docs/changelog/120483.yaml b/docs/changelog/120483.yaml new file mode 100644 index 0000000000000..20da3b9ab4e8d --- /dev/null +++ b/docs/changelog/120483.yaml @@ -0,0 +1,5 @@ +pr: 120483 +summary: Fix NPE on disabled API auth key cache +area: Authentication +type: bug +issues: [] diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 3c63ec71e74ef..1e80c01b8e71b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -266,7 +266,9 @@ public void invalidate(Collection keys) { if (apiKeyDocCache != null) { apiKeyDocCache.invalidate(keys); } - keys.forEach(apiKeyAuthCache::invalidate); + if (apiKeyAuthCache != null) { + keys.forEach(apiKeyAuthCache::invalidate); + } } @Override @@ -274,7 +276,9 @@ public void invalidateAll() { if (apiKeyDocCache != null) { apiKeyDocCache.invalidateAll(); } - apiKeyAuthCache.invalidateAll(); + if (apiKeyAuthCache != null) { + apiKeyAuthCache.invalidateAll(); + } } }); cacheInvalidatorRegistry.registerCacheInvalidator("api_key_doc", new CacheInvalidatorRegistry.CacheInvalidator() { @@ -589,9 +593,11 @@ private void createApiKeyAndIndexIt( + "])"; assert indexResponse.getResult() == DocWriteResponse.Result.CREATED : "Index response was [" + indexResponse.getResult() + "]"; - final ListenableFuture listenableFuture = new ListenableFuture<>(); - listenableFuture.onResponse(new CachedApiKeyHashResult(true, apiKey)); - apiKeyAuthCache.put(request.getId(), listenableFuture); + if (apiKeyAuthCache != null) { + final ListenableFuture listenableFuture = new ListenableFuture<>(); + listenableFuture.onResponse(new CachedApiKeyHashResult(true, apiKey)); + apiKeyAuthCache.put(request.getId(), listenableFuture); + } listener.onResponse(new CreateApiKeyResponse(request.getName(), request.getId(), apiKey, expiration)); }, listener::onFailure)) ) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 996291c52c71f..185669a6a203b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -2367,6 +2367,50 @@ public void testWillInvalidateAuthCacheWhenDocNotFound() { assertNull(service.getApiKeyAuthCache().get(docId)); } + public void testCanCreateApiKeyWithAuthCacheDisabled() { + final ApiKeyService service = createApiKeyService( + Settings.builder() + .put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true) + .put("xpack.security.authc.api_key.cache.ttl", "0s") + .build() + ); + final Authentication authentication = AuthenticationTestHelper.builder() + .user(new User(randomAlphaOfLengthBetween(8, 16), "superuser")) + .realmRef(new RealmRef(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8))) + .build(false); + final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLengthBetween(3, 8), null, null); + when(client.prepareIndex(anyString())).thenReturn(new IndexRequestBuilder(client)); + when(client.prepareBulk()).thenReturn(new BulkRequestBuilder(client)); + when(client.threadPool()).thenReturn(threadPool); + doAnswer(inv -> { + final Object[] args = inv.getArguments(); + @SuppressWarnings("unchecked") + final ActionListener listener = (ActionListener) args[2]; + final IndexResponse indexResponse = new IndexResponse( + new ShardId(INTERNAL_SECURITY_MAIN_INDEX_7, randomAlphaOfLength(22), randomIntBetween(0, 1)), + createApiKeyRequest.getId(), + randomLongBetween(1, 99), + randomLongBetween(1, 99), + randomIntBetween(1, 99), + true + ); + listener.onResponse( + new BulkResponse( + new BulkItemResponse[] { BulkItemResponse.success(randomInt(), DocWriteRequest.OpType.INDEX, indexResponse) }, + randomLongBetween(0, 100) + ) + ); + return null; + }).when(client).execute(eq(TransportBulkAction.TYPE), any(BulkRequest.class), any()); + + assertThat(service.getFromCache(createApiKeyRequest.getId()), is(nullValue())); + final PlainActionFuture listener = new PlainActionFuture<>(); + service.createApiKey(authentication, createApiKeyRequest, Set.of(), listener); + final CreateApiKeyResponse createApiKeyResponse = listener.actionGet(); + assertThat(createApiKeyResponse.getId(), equalTo(createApiKeyRequest.getId())); + assertThat(service.getFromCache(createApiKeyResponse.getId()), is(nullValue())); + } + public void testGetCreatorRealm() { final User user = AuthenticationTests.randomUser(); From c705aeab3c9f71e2f6618b4b397986f575416000 Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:01:18 +0100 Subject: [PATCH 03/10] Update README.asciidoc (#96455) (#121029) Co-authored-by: ARPIT SHARMA <93235104+ARPIT2128@users.noreply.github.com> --- docs/README.asciidoc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/README.asciidoc b/docs/README.asciidoc index 9b7e280e532f5..05f449795c7fd 100644 --- a/docs/README.asciidoc +++ b/docs/README.asciidoc @@ -157,16 +157,15 @@ used for its modifiers: * `// TESTRESPONSE[skip:reason]`: Skip the assertions specified by this response. * `// TESTSETUP`: Marks this snippet as the "setup" for all other snippets in - this file. This is a somewhat natural way of structuring documentation. You - say "this is the data we use to explain this feature" then you add the - snippet that you mark `// TESTSETUP` and then every snippet will turn into - a test that runs the setup snippet first. See the "painless" docs for a file - that puts this to good use. This is fairly similar to `// TEST[setup:name]` - but rather than the setup defined in `docs/build.gradle` the setup is defined - right in the documentation file. In general, we should prefer `// TESTSETUP` - over `// TEST[setup:name]` because it makes it more clear what steps have to - be taken before the examples will work. Tip: `// TESTSETUP` can only be used - on the first snippet of a document. + this file. In order to enhance clarity and simplify understanding for readers, + a straightforward approach involves marking the first snippet in the documentation file with the + `// TESTSETUP` marker. By doing so, it clearly indicates that this particular snippet serves as the setup + or preparation step for all subsequent snippets in the file. + This helps in explaining the necessary steps that need to be executed before running the examples. + Unlike the alternative convention `// TEST[setup:name]`, which relies on a setup defined in a separate file, + this convention brings the setup directly into the documentation file, making it more self-contained and reducing ambiguity. + By adopting this convention, users can easily identify and follow the correct sequence + of steps to ensure that the examples provided in the documentation work as intended. * `// TEARDOWN`: Ends and cleans up a test series started with `// TESTSETUP` or `// TEST[setup:name]`. You can use `// TEARDOWN` to set up multiple tests in the same file. From 6e3592ec0a69af4cfc6ce0b3f1a1d199ee149b3f Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:10:08 +0100 Subject: [PATCH 04/10] Update match-phrase-query.asciidoc (#118828) (#121034) (cherry picked from commit 8e9cccba6a9aa2eef2e109d250409bcb679881bc) Co-authored-by: Damien RENIER <153135842+damien-renier-elastic@users.noreply.github.com> --- .../query-dsl/match-phrase-query.asciidoc | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/reference/query-dsl/match-phrase-query.asciidoc b/docs/reference/query-dsl/match-phrase-query.asciidoc index f6b0fa19001f6..88046bc009e7d 100644 --- a/docs/reference/query-dsl/match-phrase-query.asciidoc +++ b/docs/reference/query-dsl/match-phrase-query.asciidoc @@ -19,9 +19,45 @@ GET /_search } -------------------------------------------------- +[[match-phrase-field-params]] +==== Parameters for `` +`query`:: ++ +-- +(Required) Text, number, boolean value or date you wish to find in the provided +``. +-- + +`analyzer`:: +(Optional, string) <> used to convert the text in the `query` +value into tokens. Defaults to the <> mapped for the ``. If no analyzer is mapped, the index's +default analyzer is used. + +`slop`:: +(Optional, integer) Maximum number of positions allowed between matching tokens. +Defaults to `0`. Transposed terms have a slop of `2`. + +`zero_terms_query`:: ++ +-- +(Optional, string) Indicates whether no documents are returned if the `analyzer` +removes all tokens, such as when using a `stop` filter. Valid values are: + + `none` (Default):: +No documents are returned if the `analyzer` removes all tokens. + + `all`:: +Returns all documents, similar to a <> +query. +-- + A phrase query matches terms up to a configurable `slop` (which defaults to 0) in any order. Transposed terms have a slop of 2. +[[query-dsl-match-query-phrase-analyzer]] +===== Analyzer in the match phrase query + The `analyzer` can be set to control which analyzer will perform the analysis process on the text. It defaults to the field explicit mapping definition, or the default search analyzer, for example: @@ -40,5 +76,3 @@ GET /_search } } -------------------------------------------------- - -This query also accepts `zero_terms_query`, as explained in <>. From 37fa39d9f602ec27cb4a1467515edb3318afced6 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Tue, 28 Jan 2025 15:15:08 +0100 Subject: [PATCH 05/10] [8.x] Added query param ?include_source_on_error for ingest requests (#120725) (#121010) A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control if to include the document source in the error response in case of parsing errors. The default value is `true`. Relates to ES-9186. --- docs/changelog/120725.yaml | 7 ++ .../XContentParserConfigurationImpl.java | 41 ++++++++-- .../provider/json/JsonXContentImpl.java | 15 +++- .../xcontent/XContentParserConfiguration.java | 7 ++ .../xcontent/XContentParserTests.java | 21 +++++ .../resources/rest-api-spec/api/bulk.json | 4 + .../resources/rest-api-spec/api/create.json | 4 + .../resources/rest-api-spec/api/index.json | 4 + .../resources/rest-api-spec/api/update.json | 4 + .../action/document/RestBulkActionIT.java | 76 +++++++++++++++++++ .../action/document/RestIndexActionIT.java | 53 +++++++++++++ .../action/document/RestUpdateActionIT.java | 53 +++++++++++++ .../org/elasticsearch/TransportVersions.java | 1 + .../action/bulk/BulkRequest.java | 18 ++++- .../action/bulk/BulkRequestParser.java | 9 ++- .../action/bulk/TransportShardBulkAction.java | 1 + .../bulk/TransportSimulateBulkAction.java | 1 + .../action/index/IndexRequest.java | 18 +++++ .../index/mapper/DocumentParser.java | 6 +- .../index/mapper/SourceToParse.java | 23 +++++- .../org/elasticsearch/rest/RestRequest.java | 17 ++++- .../org/elasticsearch/rest/RestUtils.java | 17 +++++ .../rest/action/document/RestBulkAction.java | 31 ++++---- .../rest/action/document/RestIndexAction.java | 2 + .../action/document/RestUpdateAction.java | 3 +- .../action/bulk/BulkRequestParserTests.java | 22 +++--- .../index/mapper/DynamicTemplatesTests.java | 12 +-- .../index/mapper/MapperServiceTestCase.java | 10 +-- .../action/MonitoringBulkRequest.java | 2 +- .../test/CoreTestTranslater.java | 2 +- 30 files changed, 418 insertions(+), 66 deletions(-) create mode 100644 docs/changelog/120725.yaml create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestBulkActionIT.java create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestIndexActionIT.java create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestUpdateActionIT.java diff --git a/docs/changelog/120725.yaml b/docs/changelog/120725.yaml new file mode 100644 index 0000000000000..71d256a559a7d --- /dev/null +++ b/docs/changelog/120725.yaml @@ -0,0 +1,7 @@ +pr: 120725 +summary: |- + A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control + if to include the document source in the error response in case of parsing errors. The default value is `true`. +area: Infra/REST API +type: enhancement +issues: [] diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java index 70adc59b9c6a9..e04c640ad7461 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java @@ -31,7 +31,8 @@ public class XContentParserConfigurationImpl implements XContentParserConfigurat RestApiVersion.current(), null, null, - false + false, + true ); final NamedXContentRegistry registry; @@ -40,6 +41,7 @@ public class XContentParserConfigurationImpl implements XContentParserConfigurat final FilterPath[] includes; final FilterPath[] excludes; final boolean filtersMatchFieldNamesWithDots; + final boolean includeSourceOnError; private XContentParserConfigurationImpl( NamedXContentRegistry registry, @@ -47,7 +49,8 @@ private XContentParserConfigurationImpl( RestApiVersion restApiVersion, FilterPath[] includes, FilterPath[] excludes, - boolean filtersMatchFieldNamesWithDots + boolean filtersMatchFieldNamesWithDots, + boolean includeSourceOnError ) { this.registry = registry; this.deprecationHandler = deprecationHandler; @@ -55,6 +58,28 @@ private XContentParserConfigurationImpl( this.includes = includes; this.excludes = excludes; this.filtersMatchFieldNamesWithDots = filtersMatchFieldNamesWithDots; + this.includeSourceOnError = includeSourceOnError; + } + + @Override + public boolean includeSourceOnError() { + return includeSourceOnError; + } + + @Override + public XContentParserConfiguration withIncludeSourceOnError(boolean includeSourceOnError) { + if (includeSourceOnError == this.includeSourceOnError) { + return this; + } + return new XContentParserConfigurationImpl( + registry, + deprecationHandler, + restApiVersion, + includes, + excludes, + filtersMatchFieldNamesWithDots, + includeSourceOnError + ); } @Override @@ -65,7 +90,8 @@ public XContentParserConfigurationImpl withRegistry(NamedXContentRegistry regist restApiVersion, includes, excludes, - filtersMatchFieldNamesWithDots + filtersMatchFieldNamesWithDots, + includeSourceOnError ); } @@ -80,7 +106,8 @@ public XContentParserConfiguration withDeprecationHandler(DeprecationHandler dep restApiVersion, includes, excludes, - filtersMatchFieldNamesWithDots + filtersMatchFieldNamesWithDots, + includeSourceOnError ); } @@ -95,7 +122,8 @@ public XContentParserConfiguration withRestApiVersion(RestApiVersion restApiVers restApiVersion, includes, excludes, - filtersMatchFieldNamesWithDots + filtersMatchFieldNamesWithDots, + includeSourceOnError ); } @@ -143,7 +171,8 @@ public XContentParserConfiguration withFiltering( restApiVersion, includePaths, excludePaths, - filtersMatchFieldNamesWithDots + filtersMatchFieldNamesWithDots, + includeSourceOnError ); } diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java index c842e3bbc50f4..7f52467caf49b 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java @@ -87,23 +87,30 @@ public XContentGenerator createGenerator(OutputStream os, Set includes, return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes); } + private XContentParser createParser(XContentParserConfiguration config, JsonParser parser) { + if (config.includeSourceOnError() == false) { + parser.disable(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION); // enabled by default, disable if requested + } + return new JsonXContentParser(config, parser); + } + @Override public XContentParser createParser(XContentParserConfiguration config, String content) throws IOException { - return new JsonXContentParser(config, jsonFactory.createParser(content)); + return createParser(config, jsonFactory.createParser(content)); } @Override public XContentParser createParser(XContentParserConfiguration config, InputStream is) throws IOException { - return new JsonXContentParser(config, jsonFactory.createParser(is)); + return createParser(config, jsonFactory.createParser(is)); } @Override public XContentParser createParser(XContentParserConfiguration config, byte[] data, int offset, int length) throws IOException { - return new JsonXContentParser(config, jsonFactory.createParser(data, offset, length)); + return createParser(config, jsonFactory.createParser(data, offset, length)); } @Override public XContentParser createParser(XContentParserConfiguration config, Reader reader) throws IOException { - return new JsonXContentParser(config, jsonFactory.createParser(reader)); + return createParser(config, jsonFactory.createParser(reader)); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java index 59e5cd5d6485c..73ebdfce222ad 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java @@ -26,6 +26,13 @@ public interface XContentParserConfiguration { */ XContentParserConfiguration EMPTY = XContentProvider.provider().empty(); + /** + * Disable to not include the source in case of parsing errors (defaults to true). + */ + XContentParserConfiguration withIncludeSourceOnError(boolean includeSourceOnError); + + boolean includeSourceOnError(); + /** * Replace the registry backing {@link XContentParser#namedObject}. */ diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java index 5aff60b1a4c75..5aa3b1e140074 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -33,6 +34,7 @@ import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage; @@ -655,6 +657,25 @@ public void testCreateRootSubParser() throws IOException { } + public void testJsonIncludeSourceOnParserError() throws IOException { + var xContent = XContentFactory.xContent(XContentType.JSON); + var source = "{\"field\": invalid}"; // causes parse exception + var sourceEnabled = XContentParserConfiguration.EMPTY; + var sourceDisabled = XContentParserConfiguration.EMPTY.withIncludeSourceOnError(false); + + var parseException = expectThrows(XContentParseException.class, () -> createParser(xContent, sourceEnabled, source).map()); + assertThat(parseException.getMessage(), containsString(source)); + + parseException = expectThrows(XContentParseException.class, () -> createParser(xContent, sourceDisabled, source).map()); + assertThat(parseException.getMessage(), not(containsString(source))); + } + + private XContentParser createParser(XContent xContent, XContentParserConfiguration config, String content) throws IOException { + return randomBoolean() + ? xContent.createParser(config, content) + : xContent.createParser(config, content.getBytes(StandardCharsets.UTF_8)); + } + /** * Generates a random object {"first_field": "foo", "marked_field": {...random...}, "last_field": "bar} * diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json index 9ced5d3e8c454..2b6e99bfa36ca 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json @@ -87,6 +87,10 @@ "list_executed_pipelines": { "type": "boolean", "description": "Sets list_executed_pipelines for all incoming documents. Defaults to unset (false)" + }, + "include_source_on_error": { + "type": "boolean", + "description": "True or false if to include the document source in the error message in case of parsing errors. Defaults to true." } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json index 8ed4c04917d3a..65cb0da4753cc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json @@ -69,6 +69,10 @@ "pipeline":{ "type":"string", "description":"The pipeline id to preprocess incoming documents with" + }, + "include_source_on_error": { + "type": "boolean", + "description": "True or false if to include the document source in the error message in case of parsing errors. Defaults to true." } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json index 102ca4e012e85..79ecbd794024a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json @@ -105,6 +105,10 @@ "require_data_stream": { "type": "boolean", "description": "When true, requires the destination to be a data stream (existing or to-be-created). Default is false" + }, + "include_source_on_error": { + "type": "boolean", + "description": "True or false if to include the document source in the error message in case of parsing errors. Defaults to true." } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json index e588777e990ec..9e47e80547e88 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json @@ -83,6 +83,10 @@ "require_alias": { "type": "boolean", "description": "When true, requires destination is an alias. Default is false" + }, + "include_source_on_error": { + "type": "boolean", + "description": "True or false if to include the document source in the error message in case of parsing errors. Defaults to true." } }, "body":{ diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestBulkActionIT.java new file mode 100644 index 0000000000000..d0b5ec4562903 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestBulkActionIT.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.rest.action.document; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.test.ESIntegTestCase; + +import java.io.InputStreamReader; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + +public class RestBulkActionIT extends ESIntegTestCase { + @Override + protected boolean addMockHttpTransport() { + return false; + } + + public void testBulkIndexWithSourceOnErrorDisabled() throws Exception { + var source = "{\"field\": \"index\",}"; + var sourceEscaped = "{\\\"field\\\": \\\"index\\\",}"; + + var request = new Request("PUT", "/test_index/_bulk"); + request.setJsonEntity(Strings.format("{\"index\":{\"_id\":\"1\"}}\n%s\n", source)); + + Response response = getRestClient().performRequest(request); + String responseContent = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), UTF_8)); + assertThat(responseContent, containsString(sourceEscaped)); + + request.addParameter(RestUtils.INCLUDE_SOURCE_ON_ERROR_PARAMETER, "false"); + + response = getRestClient().performRequest(request); + responseContent = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), UTF_8)); + assertThat( + responseContent, + both(not(containsString(sourceEscaped))).and( + containsString("REDACTED (`StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION` disabled)") + ) + ); + } + + public void testBulkUpdateWithSourceOnErrorDisabled() throws Exception { + var source = "{\"field\": \"index\",}"; + var sourceEscaped = "{\\\"field\\\": \\\"index\\\",}"; + + var request = new Request("PUT", "/test_index/_bulk"); + request.addParameter(RestUtils.INCLUDE_SOURCE_ON_ERROR_PARAMETER, "false"); + request.setJsonEntity(Strings.format("{\"update\":{\"_id\":\"1\"}}\n{\"doc\":%s}}\n", source)); + + // note: this behavior is not consistent with bulk index actions + // In case of updates by doc, the source is eagerly parsed and will fail the entire request if it cannot be parsed + var exception = assertThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + String response = Streams.copyToString(new InputStreamReader(exception.getResponse().getEntity().getContent(), UTF_8)); + + assertThat( + response, + both(not(containsString(sourceEscaped))).and( + containsString("REDACTED (`StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION` disabled)") + ) + ); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestIndexActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestIndexActionIT.java new file mode 100644 index 0000000000000..1a27e704ad497 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestIndexActionIT.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.rest.action.document; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.test.ESIntegTestCase; + +import java.io.InputStreamReader; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + +public class RestIndexActionIT extends ESIntegTestCase { + @Override + protected boolean addMockHttpTransport() { + return false; + } + + public void testIndexWithSourceOnErrorDisabled() throws Exception { + var source = "{\"field\": \"value}"; + var sourceEscaped = "{\\\"field\\\": \\\"value}"; + + var request = new Request("POST", "/test_index/_doc/1"); + request.setJsonEntity(source); + + var exception = assertThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + String response = Streams.copyToString(new InputStreamReader(exception.getResponse().getEntity().getContent(), UTF_8)); + assertThat(response, containsString(sourceEscaped)); + + // disable source on error + request.addParameter(RestUtils.INCLUDE_SOURCE_ON_ERROR_PARAMETER, "false"); + exception = assertThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + response = Streams.copyToString(new InputStreamReader(exception.getResponse().getEntity().getContent(), UTF_8)); + assertThat( + response, + both(not(containsString(sourceEscaped))).and( + containsString("REDACTED (`StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION` disabled)") + ) + ); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestUpdateActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestUpdateActionIT.java new file mode 100644 index 0000000000000..f25a2b8855c06 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/document/RestUpdateActionIT.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.rest.action.document; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.test.ESIntegTestCase; + +import java.io.InputStreamReader; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + +public class RestUpdateActionIT extends ESIntegTestCase { + @Override + protected boolean addMockHttpTransport() { + return false; + } + + public void testUpdateByDocWithSourceOnErrorDisabled() throws Exception { + var updateRequest = "{\"doc\":{\"field\": \"value}}"; + var sourceEscaped = "{\\\"field\\\": \\\"value}"; + + var request = new Request("POST", "/test_index/_update/1"); + request.setJsonEntity(updateRequest); + + var exception = assertThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + String response = Streams.copyToString(new InputStreamReader(exception.getResponse().getEntity().getContent(), UTF_8)); + assertThat(response, containsString(sourceEscaped)); + + // disable source on error + request.addParameter(RestUtils.INCLUDE_SOURCE_ON_ERROR_PARAMETER, "false"); + exception = assertThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + response = Streams.copyToString(new InputStreamReader(exception.getResponse().getEntity().getContent(), UTF_8)); + assertThat( + response, + both(not(containsString(sourceEscaped))).and( + containsString("REDACTED (`StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION` disabled)") + ) + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 95e5fc57c51bf..20d6e967b6a62 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -172,6 +172,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_RESPONSE_PARTIAL = def(8_832_00_0); public static final TransportVersion RANK_DOC_OPTIONAL_METADATA_FOR_EXPLAIN = def(8_833_00_0); public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_ADD_REPLICATE_FOR = def(8_834_00_0); + public static final TransportVersion INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR = def(8_835_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 91caebc420ffb..cd4602ead42b5 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -84,6 +84,7 @@ public class BulkRequest extends ActionRequest private String globalIndex; private Boolean globalRequireAlias; private Boolean globalRequireDatsStream; + private boolean includeSourceOnError = true; private long sizeInBytes = 0; @@ -103,6 +104,9 @@ public BulkRequest(StreamInput in) throws IOException { } else { incrementalState = BulkRequest.IncrementalState.EMPTY; } + if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { + includeSourceOnError = in.readBoolean(); + } // else default value is true } public BulkRequest(@Nullable String globalIndex) { @@ -278,7 +282,7 @@ public BulkRequest add( String pipeline = valueOrDefault(defaultPipeline, globalPipeline); Boolean requireAlias = valueOrDefault(defaultRequireAlias, globalRequireAlias); Boolean requireDataStream = valueOrDefault(defaultRequireDataStream, globalRequireDatsStream); - new BulkRequestParser(true, restApiVersion).parse( + new BulkRequestParser(true, includeSourceOnError, restApiVersion).parse( data, defaultIndex, routing, @@ -341,6 +345,11 @@ public void incrementalState(IncrementalState incrementalState) { this.incrementalState = incrementalState; } + public final BulkRequest includeSourceOnError(boolean includeSourceOnError) { + this.includeSourceOnError = includeSourceOnError; + return this; + } + /** * Note for internal callers (NOT high level rest client), * the global parameter setting is ignored when used with: @@ -399,6 +408,10 @@ public Boolean requireDataStream() { return globalRequireDatsStream; } + public boolean includeSourceOnError() { + return includeSourceOnError; + } + /** * Note for internal callers (NOT high level rest client), * the global parameter setting is ignored when used with: @@ -457,6 +470,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { incrementalState.writeTo(out); } + if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { + out.writeBoolean(includeSourceOnError); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java index 3a61c548ebecb..7c6a0bda190f8 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java @@ -78,12 +78,14 @@ public final class BulkRequestParser { * Create a new parser. * * @param deprecateOrErrorOnType whether to allow _type information in the index line; used by BulkMonitoring + * @param includeSourceOnError if to include the source in parser error messages * @param restApiVersion */ - public BulkRequestParser(boolean deprecateOrErrorOnType, RestApiVersion restApiVersion) { + public BulkRequestParser(boolean deprecateOrErrorOnType, boolean includeSourceOnError, RestApiVersion restApiVersion) { this.deprecateOrErrorOnType = deprecateOrErrorOnType; this.config = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE) - .withRestApiVersion(restApiVersion); + .withRestApiVersion(restApiVersion) + .withIncludeSourceOnError(includeSourceOnError); } private static int findNextMarker(byte marker, int from, BytesReference data, boolean lastData) { @@ -485,7 +487,8 @@ private boolean parseActionLine(BytesReference data, int from, int to) throws IO .setDynamicTemplates(dynamicTemplates) .setRequireAlias(requireAlias) .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(currentListExecutedPipelines); + .setListExecutedPipelines(currentListExecutedPipelines) + .setIncludeSourceOnError(config.includeSourceOnError()); if ("create".equals(action)) { indexRequest = indexRequest.create(true); } else if (opType != null) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 89cee714a9ff2..33c73898c0394 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -376,6 +376,7 @@ static boolean executeBulkItemRequest( request.getContentType(), request.routing(), request.getDynamicTemplates(), + request.getIncludeSourceOnError(), meteringParserDecorator ); result = primary.applyIndexOperationOnPrimary( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 081d3ed031644..a3eab93b94d6e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -196,6 +196,7 @@ private Tuple, Exception> validateMappings( request.getContentType(), request.routing(), request.getDynamicTemplates(), + request.getIncludeSourceOnError(), XContentMeteringParserDecorator.NOOP ); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index bcb8a7fb78bf3..e774384f87343 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -116,6 +116,8 @@ public class IndexRequest extends ReplicatedWriteRequest implement private boolean requireDataStream; + private boolean includeSourceOnError = true; + /** * Transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire. */ @@ -210,6 +212,10 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio in.readBoolean(); // obsolete originatesFromUpdateByDoc } } + + if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { + includeSourceOnError = in.readBoolean(); + } // else default value is true } public IndexRequest() { @@ -806,6 +812,9 @@ private void writeBody(StreamOutput out) throws IOException { out.writeBoolean(false); // obsolete originatesFromUpdateByDoc } } + if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { + out.writeBoolean(includeSourceOnError); + } } @Override @@ -874,6 +883,15 @@ public IndexRequest setRequireDataStream(boolean requireDataStream) { return this; } + public boolean getIncludeSourceOnError() { + return includeSourceOnError; + } + + public IndexRequest setIncludeSourceOnError(boolean includeSourceOnError) { + this.includeSourceOnError = includeSourceOnError; + return this; + } + @Override public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { if (DataStream.isFailureStoreFeatureFlagEnabled() && writeToFailureStore) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index e2d0b7efa6be9..ceb137e6d099f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -85,7 +85,11 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL XContentMeteringParserDecorator meteringParserDecorator = source.getDocumentSizeObserver(); try ( XContentParser parser = meteringParserDecorator.decorate( - XContentHelper.createParser(parserConfiguration, source.source(), xContentType) + XContentHelper.createParser( + parserConfiguration.withIncludeSourceOnError(source.getIncludeSourceOnError()), + source.source(), + xContentType + ) ) ) { context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index a8cb03c223833..0bb50d8051099 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -29,6 +29,9 @@ public class SourceToParse { private final XContentType xContentType; private final Map dynamicTemplates; + + private final boolean includeSourceOnError; + private final XContentMeteringParserDecorator meteringParserDecorator; public SourceToParse( @@ -37,6 +40,7 @@ public SourceToParse( XContentType xContentType, @Nullable String routing, Map dynamicTemplates, + boolean includeSourceOnError, XContentMeteringParserDecorator meteringParserDecorator ) { this.id = id; @@ -46,15 +50,26 @@ public SourceToParse( this.xContentType = Objects.requireNonNull(xContentType); this.routing = routing; this.dynamicTemplates = Objects.requireNonNull(dynamicTemplates); + this.includeSourceOnError = includeSourceOnError; this.meteringParserDecorator = meteringParserDecorator; } public SourceToParse(String id, BytesReference source, XContentType xContentType) { - this(id, source, xContentType, null, Map.of(), XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, null, Map.of(), true, XContentMeteringParserDecorator.NOOP); } public SourceToParse(String id, BytesReference source, XContentType xContentType, String routing) { - this(id, source, xContentType, routing, Map.of(), XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, routing, Map.of(), true, XContentMeteringParserDecorator.NOOP); + } + + public SourceToParse( + String id, + BytesReference source, + XContentType xContentType, + String routing, + Map dynamicTemplates + ) { + this(id, source, xContentType, routing, dynamicTemplates, true, XContentMeteringParserDecorator.NOOP); } public BytesReference source() { @@ -94,4 +109,8 @@ public XContentType getXContentType() { public XContentMeteringParserDecorator getDocumentSizeObserver() { return meteringParserDecorator; } + + public boolean getIncludeSourceOnError() { + return includeSourceOnError; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index a04bdcb32f2b4..fb8a8b44d8ec3 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -543,9 +543,24 @@ public XContentParserConfiguration contentParserConfig() { * {@link #contentOrSourceParamParser()} for requests that support specifying the request body in the {@code source} param. */ public final XContentParser contentParser() throws IOException { + return contentParser(parserConfig); + } + + private XContentParser contentParser(XContentParserConfiguration parserConfig) throws IOException { BytesReference content = requiredContent(); // will throw exception if body or content type missing return XContentHelper.createParserNotCompressed(parserConfig, content, xContentType.get()); + } + /** + * If there is any content then call {@code applyParser} with the parser modified by {@code includeSourceOnError}, otherwise do nothing. + */ + public final void applyContentParser(boolean includeSourceOnError, CheckedConsumer applyParser) + throws IOException { + if (hasContent()) { + try (XContentParser parser = contentParser(parserConfig.withIncludeSourceOnError(includeSourceOnError))) { + applyParser.accept(parser); + } + } } /** @@ -553,7 +568,7 @@ public final XContentParser contentParser() throws IOException { */ public final void applyContentParser(CheckedConsumer applyParser) throws IOException { if (hasContent()) { - try (XContentParser parser = contentParser()) { + try (XContentParser parser = contentParser(parserConfig)) { applyParser.accept(parser); } } diff --git a/server/src/main/java/org/elasticsearch/rest/RestUtils.java b/server/src/main/java/org/elasticsearch/rest/RestUtils.java index df51b57f0859c..36cfe32b02d73 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestUtils.java +++ b/server/src/main/java/org/elasticsearch/rest/RestUtils.java @@ -278,6 +278,12 @@ public static Optional extractTraceId(String traceparent) { */ public static final String REST_TIMEOUT_PARAM = "timeout"; + /** + * The name of the common {@code ?include_source_on_error} query parameter. + * By default, the document source is included in the error response in case of parsing errors. This parameter allows to disable this. + */ + public static final String INCLUDE_SOURCE_ON_ERROR_PARAMETER = "include_source_on_error"; + /** * Extract the {@code ?master_timeout} parameter from the request, imposing the common default of {@code 30s} in case the parameter is * missing. @@ -315,4 +321,15 @@ public static TimeValue getTimeout(RestRequest restRequest) { assert restRequest != null; return restRequest.paramAsTime(REST_TIMEOUT_PARAM, null); } + + /** + * Extract the {@code ?include_source_on_error} parameter from the request, returning {@code true} in case the parameter is missing. + * + * @param restRequest The request from which to extract the {@code ?include_source_on_error} parameter + * @return the value of the {@code ?include_source_on_error} parameter from the request, with a default of {@code true} if the request + */ + public static boolean getIncludeSourceOnError(RestRequest restRequest) { + assert restRequest != null; + return restRequest.paramAsBoolean(INCLUDE_SOURCE_ON_ERROR_PARAMETER, true); + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index e5f3b91575d41..6d7e93f914c7d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; @@ -114,6 +115,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC boolean defaultRequireDataStream = request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, false); bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); + bulkRequest.includeSourceOnError(RestUtils.getIncludeSourceOnError(request)); ReleasableBytesReference content = request.requiredContent(); try { @@ -175,20 +177,21 @@ static class ChunkHandler implements BaseRestHandler.RequestBodyChunkConsumer { ChunkHandler(boolean allowExplicitIndex, RestRequest request, Supplier handlerSupplier) { this.request = request; this.handlerSupplier = handlerSupplier; - this.parser = new BulkRequestParser(true, request.getRestApiVersion()).incrementalParser( - request.param("index"), - request.param("routing"), - FetchSourceContext.parseFromRestRequest(request), - request.param("pipeline"), - request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, false), - request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, false), - request.paramAsBoolean("list_executed_pipelines", false), - allowExplicitIndex, - request.getXContentType(), - (indexRequest, type) -> items.add(indexRequest), - items::add, - items::add - ); + this.parser = new BulkRequestParser(true, RestUtils.getIncludeSourceOnError(request), request.getRestApiVersion()) + .incrementalParser( + request.param("index"), + request.param("routing"), + FetchSourceContext.parseFromRestRequest(request), + request.param("pipeline"), + request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, false), + request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, false), + request.paramAsBoolean("list_executed_pipelines", false), + allowExplicitIndex, + request.getXContentType(), + (indexRequest, type) -> items.add(indexRequest), + items::add, + items::add + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 6d1d3776545e8..8a593f6742927 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestActions; @@ -149,6 +150,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC indexRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", indexRequest.ifPrimaryTerm())); indexRequest.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, indexRequest.isRequireAlias())); indexRequest.setRequireDataStream(request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, indexRequest.isRequireDataStream())); + indexRequest.setIncludeSourceOnError(RestUtils.getIncludeSourceOnError(request)); String sOpType = request.param("op_type"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java index ee5f030b0a45a..33398eb4fe65e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestActions; @@ -85,7 +86,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC updateRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", updateRequest.ifPrimaryTerm())); updateRequest.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, updateRequest.isRequireAlias())); - request.applyContentParser(parser -> { + request.applyContentParser(RestUtils.getIncludeSourceOnError(request), parser -> { updateRequest.fromXContent(parser); IndexRequest upsertRequest = updateRequest.upsertRequest(); if (upsertRequest != null) { diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java index 7be1b2574a120..7c3cf472057d3 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java @@ -32,7 +32,7 @@ public void testParserCannotBeReusedAfterFailure() { {} """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); BulkRequestParser.IncrementalParser incrementalParser = parser.incrementalParser( null, null, @@ -66,7 +66,7 @@ public void testIncrementalParsing() throws IOException { ArrayList> updateRequests = new ArrayList<>(); ArrayList> deleteRequests = new ArrayList<>(); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); BulkRequestParser.IncrementalParser incrementalParser = parser.incrementalParser( null, null, @@ -112,7 +112,7 @@ public void testIndexRequest() throws IOException { { "index":{ "_id": "bar" } } {} """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); final AtomicBoolean parsed = new AtomicBoolean(); parser.parse(request, "foo", null, null, null, null, null, null, false, XContentType.JSON, (indexRequest, type) -> { assertFalse(parsed.get()); @@ -148,7 +148,7 @@ public void testDeleteRequest() throws IOException { BytesArray request = new BytesArray(""" { "delete":{ "_id": "bar" } } """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); final AtomicBoolean parsed = new AtomicBoolean(); parser.parse( request, @@ -178,7 +178,7 @@ public void testUpdateRequest() throws IOException { { "update":{ "_id": "bar" } } {} """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); final AtomicBoolean parsed = new AtomicBoolean(); parser.parse(request, "foo", null, null, null, null, null, null, false, XContentType.JSON, (req, type) -> fail(), updateRequest -> { assertFalse(parsed.get()); @@ -214,7 +214,7 @@ public void testBarfOnLackOfTrailingNewline() throws IOException { BytesArray request = new BytesArray(""" { "index":{ "_id": "bar" } } {}"""); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> parser.parse( @@ -262,7 +262,7 @@ public void testFailOnExplicitIndex() { { "index":{ "_index": "foo", "_id": "bar" } } {} """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, @@ -290,7 +290,7 @@ public void testTypesStillParsedForBulkMonitoring() throws IOException { { "index":{ "_type": "quux", "_id": "bar" } } {} """); - BulkRequestParser parser = new BulkRequestParser(false, RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(false, true, RestApiVersion.current()); final AtomicBoolean parsed = new AtomicBoolean(); parser.parse(request, "foo", null, null, null, null, null, null, false, XContentType.JSON, (indexRequest, type) -> { assertFalse(parsed.get()); @@ -309,7 +309,7 @@ public void testParseDeduplicatesParameterStrings() throws IOException { { "index":{ "_index": "bar", "pipeline": "foo", "routing": "blub" } } {} """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); final List indexRequests = new ArrayList<>(); parser.parse( request, @@ -339,7 +339,7 @@ public void testFailOnInvalidAction() { { "invalidaction":{ } } {} """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), randomFrom(RestApiVersion.values())); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, randomFrom(RestApiVersion.values())); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, @@ -370,7 +370,7 @@ public void testListExecutedPipelines() throws IOException { { "index":{ "_id": "bar" } } {} """); - BulkRequestParser parser = new BulkRequestParser(randomBoolean(), RestApiVersion.current()); + BulkRequestParser parser = new BulkRequestParser(randomBoolean(), true, RestApiVersion.current()); parser.parse(request, "foo", null, null, null, null, null, null, false, XContentType.JSON, (indexRequest, type) -> { assertFalse(indexRequest.getListExecutedPipelines()); }, req -> fail(), req -> fail()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index bc7b3faf0d417..68b234af2c25f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -733,16 +732,7 @@ public void testTemplateWithoutMatchPredicates() throws Exception { {"foo": "41.12,-71.34", "bar": "41.12,-71.34"} """; ParsedDocument doc = mapperService.documentMapper() - .parse( - new SourceToParse( - "1", - new BytesArray(json), - XContentType.JSON, - null, - Map.of("foo", "geo_point"), - XContentMeteringParserDecorator.NOOP - ) - ); + .parse(new SourceToParse("1", new BytesArray(json), XContentType.JSON, null, Map.of("foo", "geo_point"))); assertThat(doc.rootDoc().getFields("foo"), hasSize(1)); assertThat(doc.rootDoc().getFields("bar"), hasSize(1)); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 8c44b49f36357..459480d1d7316 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -61,7 +61,6 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.TelemetryPlugin; -import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; @@ -389,14 +388,7 @@ protected static SourceToParse source( XContentBuilder builder = JsonXContent.contentBuilder().startObject(); build.accept(builder); builder.endObject(); - return new SourceToParse( - id, - BytesReference.bytes(builder), - XContentType.JSON, - routing, - dynamicTemplates, - XContentMeteringParserDecorator.NOOP - ); + return new SourceToParse(id, BytesReference.bytes(builder), XContentType.JSON, routing, dynamicTemplates); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequest.java index 638e57207fbeb..36bc2db95932d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequest.java @@ -83,7 +83,7 @@ public MonitoringBulkRequest add( ) throws IOException { // MonitoringBulkRequest accepts a body request that has the same format as the BulkRequest - new BulkRequestParser(false, RestApiVersion.current()).parse( + new BulkRequestParser(false, true, RestApiVersion.current()).parse( content, null, null, diff --git a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index d34303ea803d6..51c9e35c95a3d 100644 --- a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -366,7 +366,7 @@ private boolean handleBulk(ApiCallSection bulk) { bos.write(JsonXContent.jsonXContent.bulkSeparator()); } List indexRequests = new ArrayList<>(); - new BulkRequestParser(false, RestApiVersion.current()).parse( + new BulkRequestParser(false, true, RestApiVersion.current()).parse( bos.bytes(), defaultIndex, defaultRouting, From b1085300e08ace2f81321e1b25c43be549aa6a80 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Tue, 28 Jan 2025 16:34:12 +0200 Subject: [PATCH 06/10] Lazy initialization for `SyntheticSourceSupport.loader()` (#120896) (#120915) * Lazy initialization for `SyntheticSourceSupport.loader()` * [CI] Auto commit changes from spotless * add missing --------- Co-authored-by: elasticsearchmachine --- .../extras/MatchOnlyTextFieldMapper.java | 14 ++++---- .../mapper/extras/ScaledFloatFieldMapper.java | 14 ++++---- .../AnnotatedTextFieldMapper.java | 8 ++--- .../index/mapper/BinaryFieldMapper.java | 6 ++-- .../index/mapper/BooleanFieldMapper.java | 14 ++++---- .../index/mapper/DateFieldMapper.java | 14 ++++---- .../index/mapper/DocCountFieldMapper.java | 2 +- .../index/mapper/DocumentParser.java | 2 +- .../index/mapper/FieldMapper.java | 18 +++++++++- .../index/mapper/GeoPointFieldMapper.java | 20 +++++------ .../mapper/IgnoredSourceFieldMapper.java | 2 +- .../index/mapper/IpFieldMapper.java | 35 ++++++++++--------- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/MetadataFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 2 +- .../index/mapper/RangeFieldMapper.java | 6 ++-- .../index/mapper/TextFieldMapper.java | 8 ++--- .../flattened/FlattenedFieldMapper.java | 14 ++++---- .../vectors/DenseVectorFieldMapper.java | 10 +++--- .../vectors/SparseVectorFieldMapper.java | 2 +- .../index/mapper/DocumentParserTests.java | 6 ++-- .../mapper/HistogramFieldMapper.java | 14 ++++---- .../AggregateDoubleMetricFieldMapper.java | 14 ++++---- .../mapper/ConstantKeywordFieldMapper.java | 8 ++--- .../CountedKeywordFieldMapper.java | 5 +-- .../unsignedlong/UnsignedLongFieldMapper.java | 14 ++++---- .../VersionStringFieldMapper.java | 26 +++++++------- .../mapper/RankVectorsFieldMapper.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 28 +++++++-------- 29 files changed, 159 insertions(+), 153 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 5904169308fab..b103a23772a47 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -463,13 +463,13 @@ public MatchOnlyTextFieldType fieldType() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var loader = new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), fieldType().name(), leafName()) { - @Override - protected void write(XContentBuilder b, Object value) throws IOException { - b.value((String) value); + return new SyntheticSourceSupport.Native( + () -> new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), fieldType().name(), leafName()) { + @Override + protected void write(XContentBuilder b, Object value) throws IOException { + b.value((String) value); + } } - }; - - return new SyntheticSourceSupport.Native(loader); + ); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index b845545133e19..a91ca66faa405 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -707,14 +707,14 @@ public int docValueCount() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(decodeForSyntheticSource(value, scalingFactor)); + return new SyntheticSourceSupport.Native( + () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(decodeForSyntheticSource(value, scalingFactor)); + } } - }; - - return new SyntheticSourceSupport.Native(loader); + ); } return super.syntheticSourceSupport(); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index f58fd138905c3..d04828b067f0c 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -563,19 +563,17 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (fieldType.stored()) { - var loader = new StringStoredFieldFieldLoader(fullPath(), leafName()) { + return new SyntheticSourceSupport.Native(() -> new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); } - }; - - return new SyntheticSourceSupport.Native(loader); + }); } var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(fullPath(), leafName())); + return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName())); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 87c123d71aae5..2093909876567 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -195,7 +195,7 @@ protected String contentType() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - var loader = new BinaryDocValuesSyntheticFieldLoader(fullPath()) { + return new SyntheticSourceSupport.Native(() -> new BinaryDocValuesSyntheticFieldLoader(fullPath()) { @Override protected void writeValue(XContentBuilder b, BytesRef value) throws IOException { var in = new ByteArrayStreamInput(); @@ -221,9 +221,7 @@ protected void writeValue(XContentBuilder b, BytesRef value) throws IOException b.endArray(); } } - }; - - return new SyntheticSourceSupport.Native(loader); + }); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 1a28bf2a10acb..fafd041f887d4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -552,14 +552,14 @@ protected String contentType() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(value == 1); + return new SyntheticSourceSupport.Native( + () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(value == 1); + } } - }; - - return new SyntheticSourceSupport.Native(loader); + ); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 012f08a1db01d..823170fcfc1b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -1012,14 +1012,14 @@ public Long getNullValue() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(fieldType().format(value, fieldType().dateTimeFormatter())); + return new SyntheticSourceSupport.Native( + () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(fieldType().format(value, fieldType().dateTimeFormatter())); + } } - }; - - return new SyntheticSourceSupport.Native(loader); + ); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java index d57564ca5d696..52828c5e430ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java @@ -128,7 +128,7 @@ public static IndexableField field(int count) { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native(new SyntheticFieldLoader()); + return new SyntheticSourceSupport.Native(SyntheticFieldLoader::new); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index ceb137e6d099f..99f4be4dfb6d4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -1027,7 +1027,7 @@ protected String contentType() { protected SyntheticSourceSupport syntheticSourceSupport() { // Opt out of fallback synthetic source implementation // since there is custom logic in #parseCreateField(). - return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); + return new SyntheticSourceSupport.Native(() -> SourceLoader.SyntheticFieldLoader.NOTHING); } }; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 32681871b346e..7cf012fd298ff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -560,11 +560,27 @@ public SourceLoader.SyntheticFieldLoader loader() { SyntheticSourceSupport FALLBACK = new Fallback(); - record Native(SourceLoader.SyntheticFieldLoader loader) implements SyntheticSourceSupport { + final class Native implements SyntheticSourceSupport { + Supplier loaderSupplier; + private SourceLoader.SyntheticFieldLoader loader; + + @SuppressWarnings("checkstyle:RedundantModifier") + public Native(Supplier loaderSupplier) { + this.loaderSupplier = loaderSupplier; + } + @Override public SyntheticSourceMode mode() { return SyntheticSourceMode.NATIVE; } + + @Override + public SourceLoader.SyntheticFieldLoader loader() { + if (loader == null) { + loader = loaderSupplier.get(); + } + return loader; + } } SyntheticSourceMode mode(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index c99400c122f42..0d91c7525a9e2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -622,17 +622,17 @@ protected void onMalformedValue(DocumentParserContext context, XContentBuilder m @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (fieldType().hasDocValues()) { - var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { - final GeoPoint point = new GeoPoint(); - - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - point.reset(GeoEncodingUtils.decodeLatitude((int) (value >>> 32)), GeoEncodingUtils.decodeLongitude((int) value)); - point.toXContent(b, ToXContent.EMPTY_PARAMS); + return new SyntheticSourceSupport.Native( + () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { + final GeoPoint point = new GeoPoint(); + + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + point.reset(GeoEncodingUtils.decodeLatitude((int) (value >>> 32)), GeoEncodingUtils.decodeLongitude((int) value)); + point.toXContent(b, ToXContent.EMPTY_PARAMS); + } } - }; - - return new SyntheticSourceSupport.Native(loader); + ); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 9e08298fecdfd..d0f8a0facb06c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -231,7 +231,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { // not being available. // We would like to have an option to lose some values in synthetic source // but have search not fail. - return new SyntheticSourceSupport.Native(new SourceLoader.SyntheticFieldLoader() { + return new SyntheticSourceSupport.Native(() -> new SourceLoader.SyntheticFieldLoader() { @Override public Stream> storedFieldLoaders() { if (indexSettings.getSkipIgnoredSourceRead()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 2f64955b48627..7e2380a357ec5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -610,26 +610,27 @@ public void doValidate(MappingLookup lookup) { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - var layers = new ArrayList(); - layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { - @Override - protected BytesRef convert(BytesRef value) { - byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); - return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes))); - } + return new SyntheticSourceSupport.Native(() -> { + var layers = new ArrayList(); + layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { + @Override + protected BytesRef convert(BytesRef value) { + byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); + return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes))); + } - @Override - protected BytesRef preserve(BytesRef value) { - // No need to copy because convert has made a deep copy - return value; + @Override + protected BytesRef preserve(BytesRef value) { + // No need to copy because convert has made a deep copy + return value; + } + }); + + if (ignoreMalformed) { + layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); } + return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); }); - - if (ignoreMalformed) { - layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); - } - - return new SyntheticSourceSupport.Native(new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers)); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index ea461855738fa..8104ed806410a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -1052,7 +1052,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { } if (fieldType.stored() || hasDocValues) { - return new SyntheticSourceSupport.Native(syntheticFieldLoader(fullPath(), leafName())); + return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName())); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 9a2c9517dfd05..a4a9f3a9069d5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -218,6 +218,6 @@ public void postParse(DocumentParserContext context) throws IOException { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); + return new SyntheticSourceSupport.Native(() -> SourceLoader.SyntheticFieldLoader.NOTHING); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 963646495f8d1..6b4e87a70ab9e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -2039,7 +2039,7 @@ public void doValidate(MappingLookup lookup) { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - return new SyntheticSourceSupport.Native(type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value())); + return new SyntheticSourceSupport.Native(() -> type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value())); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 36f61311ddfc7..e70f902657cd1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -467,7 +467,7 @@ private static Range parseIpRangeFromCidr(final XContentParser parser) throws IO @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - var loader = new BinaryDocValuesSyntheticFieldLoader(fullPath()) { + return new SyntheticSourceSupport.Native(() -> new BinaryDocValuesSyntheticFieldLoader(fullPath()) { @Override protected void writeValue(XContentBuilder b, BytesRef value) throws IOException { List ranges = type.decodeRanges(value); @@ -487,9 +487,7 @@ protected void writeValue(XContentBuilder b, BytesRef value) throws IOException b.endArray(); } } - }; - - return new SyntheticSourceSupport.Native(loader); + }); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index d9dcfb6aeddfa..5242b33666b21 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1481,19 +1481,17 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (store) { - var loader = new StringStoredFieldFieldLoader(fullPath(), leafName()) { + return new SyntheticSourceSupport.Native(() -> new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); } - }; - - return new SyntheticSourceSupport.Native(loader); + }); } var kwd = SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(fullPath(), leafName())); + return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName())); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 0aca39f7573f7..d13daa1f416ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -914,14 +914,14 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (fieldType().hasDocValues()) { - var loader = new FlattenedSortedSetDocValuesSyntheticFieldLoader( - fullPath(), - fullPath() + KEYED_FIELD_SUFFIX, - ignoreAbove() < Integer.MAX_VALUE ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, - leafName() + return new SyntheticSourceSupport.Native( + () -> new FlattenedSortedSetDocValuesSyntheticFieldLoader( + fullPath(), + fullPath() + KEYED_FIELD_SUFFIX, + ignoreAbove() < Integer.MAX_VALUE ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, + leafName() + ) ); - - return new SyntheticSourceSupport.Native(loader); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 70e458456cde0..6d01ea21a6478 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2375,11 +2375,11 @@ public String toString() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var loader = fieldType().indexed - ? new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity) - : new DocValuesSyntheticFieldLoader(indexCreatedVersion); - - return new SyntheticSourceSupport.Native(loader); + return new SyntheticSourceSupport.Native( + () -> fieldType().indexed + ? new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity) + : new DocValuesSyntheticFieldLoader(indexCreatedVersion) + ); } private class IndexedSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java index 266fee7b3fc70..ca2312c8c708a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java @@ -164,7 +164,7 @@ private SparseVectorFieldMapper(String simpleName, MappedFieldType mappedFieldTy @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (fieldType().isStored()) { - return new SyntheticSourceSupport.Native(new SparseVectorSyntheticFieldLoader(fullPath(), leafName())); + return new SyntheticSourceSupport.Native(() -> new SparseVectorSyntheticFieldLoader(fullPath(), leafName())); } return super.syntheticSourceSupport(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 5b32350cbf4b8..c394d1a82e28e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -3303,15 +3303,13 @@ protected String contentType() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var loader = new StringStoredFieldFieldLoader(fullPath(), leafName()) { + return new SyntheticSourceSupport.Native(() -> new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { BytesRef ref = (BytesRef) value; b.utf8Value(ref.bytes, ref.offset, ref.length); } - }; - - return new SyntheticSourceSupport.Native(loader); + }); } private static final TypeParser PARSER = new FixedTypeParser(c -> new MockMetadataMapper()); diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index d597d7b59f240..fcf2d572f2d14 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -497,14 +497,14 @@ public long count() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var loader = new CompositeSyntheticFieldLoader( - leafName(), - fullPath(), - new HistogramSyntheticFieldLoader(), - new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()) + return new SyntheticSourceSupport.Native( + () -> new CompositeSyntheticFieldLoader( + leafName(), + fullPath(), + new HistogramSyntheticFieldLoader(), + new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()) + ) ); - - return new SyntheticSourceSupport.Native(loader); } private class HistogramSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer { diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 6944f91042311..df4a0aed01bc2 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -709,14 +709,14 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var loader = new CompositeSyntheticFieldLoader( - leafName(), - fullPath(), - new AggregateMetricSyntheticFieldLoader(fullPath(), metrics), - new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()) + return new SyntheticSourceSupport.Native( + () -> new CompositeSyntheticFieldLoader( + leafName(), + fullPath(), + new AggregateMetricSyntheticFieldLoader(fullPath(), metrics), + new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()) + ) ); - - return new SyntheticSourceSupport.Native(loader); } public static class AggregateMetricSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index f17ff728afd0c..4c04458af503d 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -355,16 +355,14 @@ protected SyntheticSourceSupport syntheticSourceSupport() { String const_value = fieldType().value(); if (const_value == null) { - return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); + return new SyntheticSourceSupport.Native(() -> SourceLoader.SyntheticFieldLoader.NOTHING); } - var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), false) { + return new SyntheticSourceSupport.Native(() -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), false) { @Override protected void writeValue(XContentBuilder b, long ignored) throws IOException { b.value(const_value); } - }; - - return new SyntheticSourceSupport.Native(loader); + }); } } diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java index da03577d6a19e..0e09fb44f9e5d 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java @@ -507,8 +507,9 @@ protected SyntheticSourceSupport syntheticSourceSupport() { return super.syntheticSourceSupport(); } - var loader = new CountedKeywordFieldSyntheticSourceLoader(fullPath(), countFieldMapper.fullPath(), leafName()); - return new SyntheticSourceSupport.Native(loader); + return new SyntheticSourceSupport.Native( + () -> new CountedKeywordFieldSyntheticSourceLoader(fullPath(), countFieldMapper.fullPath(), leafName()) + ); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 807cb16a363b1..7af69a1a2b453 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -754,14 +754,14 @@ public void doValidate(MappingLookup lookup) { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value)); + return new SyntheticSourceSupport.Native( + () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value)); + } } - }; - - return new SyntheticSourceSupport.Native(loader); + ); } return super.syntheticSourceSupport(); diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index b49b4500ce7b7..37b0300651a9a 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -435,19 +435,19 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var loader = new CompositeSyntheticFieldLoader(leafName(), fullPath(), new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { - @Override - protected BytesRef convert(BytesRef value) { - return VersionEncoder.decodeVersion(value); - } - - @Override - protected BytesRef preserve(BytesRef value) { - // Convert copies the underlying bytes - return value; - } - }); + return new SyntheticSourceSupport.Native( + () -> new CompositeSyntheticFieldLoader(leafName(), fullPath(), new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { + @Override + protected BytesRef convert(BytesRef value) { + return VersionEncoder.decodeVersion(value); + } - return new SyntheticSourceSupport.Native(loader); + @Override + protected BytesRef preserve(BytesRef value) { + // Convert copies the underlying bytes + return value; + } + }) + ); } } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index 873d67e76b04a..a595eedaf4b8d 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -375,7 +375,7 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native(new DocValuesSyntheticFieldLoader()); + return new SyntheticSourceSupport.Native(DocValuesSyntheticFieldLoader::new); } private class DocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 7784e7ffdda12..bbf02561426af 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -992,20 +992,20 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var layers = new ArrayList(); - layers.add(new WildcardSyntheticFieldLoader()); - if (ignoreAbove != Integer.MAX_VALUE) { - layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { - @Override - protected void writeValue(Object value, XContentBuilder b) throws IOException { - BytesRef r = (BytesRef) value; - b.utf8Value(r.bytes, r.offset, r.length); - } - }); - } - - var loader = new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); - return new SyntheticSourceSupport.Native(loader); + return new SyntheticSourceSupport.Native(() -> { + var layers = new ArrayList(); + layers.add(new WildcardSyntheticFieldLoader()); + if (ignoreAbove != Integer.MAX_VALUE) { + layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { + @Override + protected void writeValue(Object value, XContentBuilder b) throws IOException { + BytesRef r = (BytesRef) value; + b.utf8Value(r.bytes, r.offset, r.length); + } + }); + } + return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); + }); } private class WildcardSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer { From c3e15ac7d333287c8f666dffc2002325153002d0 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 28 Jan 2025 15:40:33 +0100 Subject: [PATCH 07/10] Breakup release tests in PR (#120692) (#120728) * Breakup release tests in PR * fix test release task dependencies (cherry picked from commit 1484f783d8d7131c82a2377647c2421f57f28e39) # Conflicts: # build.gradle --- .../pipelines/pull-request/release-tests.yml | 27 +++++++--- .buildkite/scripts/release-tests.sh | 2 +- build.gradle | 51 ++++++++++++++++--- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/.buildkite/pipelines/pull-request/release-tests.yml b/.buildkite/pipelines/pull-request/release-tests.yml index 7d7a5c77d3320..101f13e569514 100644 --- a/.buildkite/pipelines/pull-request/release-tests.yml +++ b/.buildkite/pipelines/pull-request/release-tests.yml @@ -1,11 +1,22 @@ config: allow-labels: test-release steps: - - label: release-tests - command: .buildkite/scripts/release-tests.sh - timeout_in_minutes: 300 - agents: - provider: gcp - image: family/elasticsearch-ubuntu-2004 - diskSizeGb: 350 - machineType: custom-32-98304 + - group: release-tests + steps: + - label: "{{matrix.CHECK_TASK}} / release-tests" + key: "packaging-tests-unix" + command: .buildkite/scripts/release-tests.sh {{matrix.CHECK_TASK}} + timeout_in_minutes: 120 + matrix: + setup: + CHECK_TASK: + - checkPart1 + - checkPart2 + - checkPart3 + - checkPart4 + - checkPart5 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + diskSizeGb: 350 + machineType: custom-32-98304 diff --git a/.buildkite/scripts/release-tests.sh b/.buildkite/scripts/release-tests.sh index e4185c642f244..700704bca0129 100755 --- a/.buildkite/scripts/release-tests.sh +++ b/.buildkite/scripts/release-tests.sh @@ -20,4 +20,4 @@ curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/m curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT.zip .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \ - -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef assemble functionalTests + -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef ${@:-functionalTests} diff --git a/build.gradle b/build.gradle index 3c0b456a14d4b..9d22d196a3ca4 100644 --- a/build.gradle +++ b/build.gradle @@ -296,22 +296,57 @@ allprojects { } } + ext.withReleaseBuild = { Closure config -> + if(buildParams.snapshotBuild == false) { + config.call() + } + } + plugins.withId('lifecycle-base') { if (project.path.startsWith(":x-pack:")) { if (project.path.contains("security") || project.path.contains(":ml")) { - tasks.register('checkPart4') { dependsOn 'check' } - } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) { - tasks.register('checkPart3') { dependsOn 'check' } + tasks.register('checkPart4') { + dependsOn 'check' + withReleaseBuild { + dependsOn 'assemble' + } + } + } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) { + tasks.register('checkPart3') { + dependsOn 'check' + withReleaseBuild { + dependsOn 'assemble' + } + } } else if (project.path.contains("multi-node")) { - tasks.register('checkPart5') { dependsOn 'check' } + tasks.register('checkPart5') { + dependsOn 'check' + withReleaseBuild { + dependsOn 'assemble' + } + } } else { - tasks.register('checkPart2') { dependsOn 'check' } + tasks.register('checkPart2') { + dependsOn 'check' + withReleaseBuild { + dependsOn 'assemble' + } + } } } else { - tasks.register('checkPart1') { dependsOn 'check' } + tasks.register('checkPart1') { + dependsOn 'check' + withReleaseBuild { + dependsOn 'assemble' + } + } + } + tasks.register('functionalTests') { + dependsOn 'check' + withReleaseBuild { + dependsOn 'assemble' + } } - - tasks.register('functionalTests') { dependsOn 'check' } } /* From ec0dcf23a16adc730c30f37a502705645bb9199e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 29 Jan 2025 02:04:23 +1100 Subject: [PATCH 08/10] Mute org.elasticsearch.xpack.security.authc.service.ServiceAccountIT testAuthenticateShouldNotFallThroughInCaseOfFailure #120902 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 7be53f57c78f0..a228738276195 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -476,3 +476,6 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.RestEsqlIT method: testOutOfRangeComparisons {ASYNC} issue: https://github.com/elastic/elasticsearch/issues/121007 +- class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT + method: testAuthenticateShouldNotFallThroughInCaseOfFailure + issue: https://github.com/elastic/elasticsearch/issues/120902 From e807b443e5241c34f18522f66766d503c74793cc Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 28 Jan 2025 16:09:01 +0100 Subject: [PATCH 09/10] Fix incorrect use of "updateable" flag in synonyms documentation (#120866) (#121046) Co-authored-by: Amine GANI Co-authored-by: Amine GANI --- .../search-with-synonyms.asciidoc | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/reference/search/search-your-data/search-with-synonyms.asciidoc b/docs/reference/search/search-your-data/search-with-synonyms.asciidoc index 61d3a1d8f925b..5a1897febb1fa 100644 --- a/docs/reference/search/search-your-data/search-with-synonyms.asciidoc +++ b/docs/reference/search/search-your-data/search-with-synonyms.asciidoc @@ -145,15 +145,35 @@ The following example adds `my_analyzer` as a search analyzer to the `title` fie [source,JSON] ---- +{ "mappings": { "properties": { "title": { "type": "text", - "search_analyzer": "my_analyzer", - "updateable": true + "search_analyzer": "my_analyzer" + } + } + }, + "settings": { + "analysis": { + "analyzer": { + "my_analyzer": { + "tokenizer": "whitespace", + "filter": [ + "synonyms_filter" + ] + } + }, + "filter": { + "synonyms_filter": { + "type": "synonym", + "synonyms_path": "analysis/synonym-set.txt", + "updateable": true + } } } } +} ---- From ff4f9d7dde9f6d9f31d0eccc475be3b8d6e4882d Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Tue, 28 Jan 2025 10:15:35 -0500 Subject: [PATCH 10/10] Re-enable geoip FullClusterRestartIT (#120966) (#120976) --- modules/ingest-geoip/qa/full-cluster-restart/build.gradle | 7 +------ .../elasticsearch/ingest/geoip/FullClusterRestartIT.java | 5 ++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle index c6f4e1667957d..e91ffd7e8de6c 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle +++ b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle @@ -21,12 +21,7 @@ dependencies { javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest")) } -assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() == 8 : - "If we are targeting a branch other than 8, we should enable migration tests" - -// once we are ready to test migrations from 8.x to 9.x, we can set the compatible version to 8.0.0 -// see https://github.com/elastic/elasticsearch/pull/93666 -buildParams.bwcVersions.withWireCompatible(v -> v.before("7.0.0")) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(v -> v.before("8.0.0")) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java index cf66a978c6b5f..13462f465e6bc 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java +++ b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java @@ -42,9 +42,8 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas .distribution(DistributionType.DEFAULT) .version(getOldClusterTestVersion()) .nodes(2) - .setting("indices.memory.shard_inactive_time", "60m") - .setting("xpack.security.enabled", "false") .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture) + .setting("xpack.security.enabled", "false") .feature(FeatureFlag.TIME_SERIES_MODE) .build(); @@ -93,7 +92,7 @@ public void testGeoIpSystemFeaturesMigration() throws Exception { Request migrateSystemFeatures = new Request("POST", "/_migration/system_features"); assertOK(client().performRequest(migrateSystemFeatures)); - assertBusy(() -> testCatIndices(".geoip_databases-reindexed-for-8", "my-index-00001")); + assertBusy(() -> testCatIndices(".geoip_databases-reindexed-for-9", "my-index-00001")); assertBusy(() -> testIndexGeoDoc()); Request disableDownloader = new Request("PUT", "/_cluster/settings");