diff --git a/CHANGELOG.md b/CHANGELOG.md index 15fda1333343b..f57f9f262659c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support object fields in star-tree index([#16728](https://github.com/opensearch-project/OpenSearch/pull/16728/)) - Support searching from doc_value using termQueryCaseInsensitive/termQuery in flat_object/keyword field([#16974](https://github.com/opensearch-project/OpenSearch/pull/16974/)) - Added a new `time` field to replace the deprecated `getTime` field in `GetStats`. ([#17009](https://github.com/opensearch-project/OpenSearch/pull/17009)) +- Improve flat_object field parsing performance by reducing two passes to a single pass ([#16297](https://github.com/opensearch-project/OpenSearch/pull/16297)) - Improve performance of the bitmap filtering([#16936](https://github.com/opensearch-project/OpenSearch/pull/16936/)) ### Dependencies diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java deleted file mode 100644 index 21270b4241b15..0000000000000 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.common.xcontent; - -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.common.ParsingException; -import org.opensearch.core.common.Strings; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.xcontent.AbstractXContentParser; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentLocation; -import org.opensearch.core.xcontent.XContentParser; - -import java.io.IOException; -import java.math.BigInteger; -import java.nio.CharBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashSet; -import java.util.LinkedList; - -/** - * JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser - * returns XContentParser with one parent field and subfields - * fieldName, fieldName._value, fieldName._valueAndPath - * @opensearch.internal - */ -public class JsonToStringXContentParser extends AbstractXContentParser { - public static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - public static final String VALUE_SUFFIX = "._value"; - public static final String DOT_SYMBOL = "."; - public static final String EQUAL_SYMBOL = "="; - private final String fieldTypeName; - private final XContentParser parser; - - private final ArrayList valueList = new ArrayList<>(); - private final ArrayList valueAndPathList = new ArrayList<>(); - private final ArrayList keyList = new ArrayList<>(); - - private final XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent); - - private final NamedXContentRegistry xContentRegistry; - - private final DeprecationHandler deprecationHandler; - - public JsonToStringXContentParser( - NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, - XContentParser parser, - String fieldTypeName - ) throws IOException { - super(xContentRegistry, deprecationHandler); - this.deprecationHandler = deprecationHandler; - this.xContentRegistry = xContentRegistry; - this.parser = parser; - this.fieldTypeName = fieldTypeName; - } - - public XContentParser parseObject() throws IOException { - assert currentToken() == Token.START_OBJECT; - parser.nextToken(); // Skip the outer START_OBJECT. Need to return on END_OBJECT. - - builder.startObject(); - LinkedList path = new LinkedList<>(Collections.singleton(fieldTypeName)); - while (currentToken() != Token.END_OBJECT) { - parseToken(path); - } - // deduplication the fieldName,valueList,valueAndPathList - builder.field(this.fieldTypeName, new HashSet<>(keyList)); - builder.field(this.fieldTypeName + VALUE_SUFFIX, new HashSet<>(valueList)); - builder.field(this.fieldTypeName + VALUE_AND_PATH_SUFFIX, new HashSet<>(valueAndPathList)); - builder.endObject(); - String jString = XContentHelper.convertToJson(BytesReference.bytes(builder), false, MediaTypeRegistry.JSON); - return JsonXContent.jsonXContent.createParser(this.xContentRegistry, this.deprecationHandler, String.valueOf(jString)); - } - - /** - * @return true if the child object contains no_null value, false otherwise - */ - private boolean parseToken(Deque path) throws IOException { - boolean isChildrenValueValid = false; - boolean visitFieldName = false; - if (this.parser.currentToken() == Token.FIELD_NAME) { - final String currentFieldName = this.parser.currentName(); - path.addLast(currentFieldName); // Pushing onto the stack *must* be matched by pop - visitFieldName = true; - String parts = currentFieldName; - while (parts.contains(".")) { // Extract the intermediate keys maybe present in fieldName - int dotPos = parts.indexOf('.'); - String part = parts.substring(0, dotPos); - this.keyList.add(part); - parts = parts.substring(dotPos + 1); - } - this.keyList.add(parts); // parts has no dot, so either it's the original fieldName or it's the last part - this.parser.nextToken(); // advance to the value of fieldName - isChildrenValueValid = parseToken(path); // parse the value for fieldName (which will be an array, an object, - // or a primitive value) - path.removeLast(); // Here is where we pop fieldName from the stack (since we're done with the value of fieldName) - // Note that whichever other branch we just passed through has already ended with nextToken(), so we - // don't need to call it. - } else if (this.parser.currentToken() == Token.START_ARRAY) { - parser.nextToken(); - while (this.parser.currentToken() != Token.END_ARRAY) { - isChildrenValueValid |= parseToken(path); - } - this.parser.nextToken(); - } else if (this.parser.currentToken() == Token.START_OBJECT) { - parser.nextToken(); - while (this.parser.currentToken() != Token.END_OBJECT) { - isChildrenValueValid |= parseToken(path); - } - this.parser.nextToken(); - } else { - String parsedValue = parseValue(); - if (parsedValue != null) { - this.valueList.add(parsedValue); - this.valueAndPathList.add(Strings.collectionToDelimitedString(path, ".") + EQUAL_SYMBOL + parsedValue); - isChildrenValueValid = true; - } - this.parser.nextToken(); - } - - if (visitFieldName && isChildrenValueValid == false) { - removeKeyOfNullValue(); - } - return isChildrenValueValid; - } - - public void removeKeyOfNullValue() { - // it means that the value of the sub child (or the last brother) is invalid, - // we should delete the key from keyList. - assert keyList.size() > 0; - this.keyList.remove(keyList.size() - 1); - } - - private String parseValue() throws IOException { - switch (this.parser.currentToken()) { - case VALUE_BOOLEAN: - case VALUE_NUMBER: - case VALUE_STRING: - case VALUE_NULL: - return this.parser.textOrNull(); - // Handle other token types as needed - default: - throw new ParsingException(parser.getTokenLocation(), "Unexpected value token type [" + parser.currentToken() + "]"); - } - } - - @Override - public MediaType contentType() { - return MediaTypeRegistry.JSON; - } - - @Override - public Token nextToken() throws IOException { - return this.parser.nextToken(); - } - - @Override - public void skipChildren() throws IOException { - this.parser.skipChildren(); - } - - @Override - public Token currentToken() { - return this.parser.currentToken(); - } - - @Override - public String currentName() throws IOException { - return this.parser.currentName(); - } - - @Override - public String text() throws IOException { - return this.parser.text(); - } - - @Override - public CharBuffer charBuffer() throws IOException { - return this.parser.charBuffer(); - } - - @Override - public Object objectText() throws IOException { - return this.parser.objectText(); - } - - @Override - public Object objectBytes() throws IOException { - return this.parser.objectBytes(); - } - - @Override - public boolean hasTextCharacters() { - return this.parser.hasTextCharacters(); - } - - @Override - public char[] textCharacters() throws IOException { - return this.parser.textCharacters(); - } - - @Override - public int textLength() throws IOException { - return this.parser.textLength(); - } - - @Override - public int textOffset() throws IOException { - return this.parser.textOffset(); - } - - @Override - public Number numberValue() throws IOException { - return this.parser.numberValue(); - } - - @Override - public NumberType numberType() throws IOException { - return this.parser.numberType(); - } - - @Override - public byte[] binaryValue() throws IOException { - return this.parser.binaryValue(); - } - - @Override - public XContentLocation getTokenLocation() { - return this.parser.getTokenLocation(); - } - - @Override - protected boolean doBooleanValue() throws IOException { - return this.parser.booleanValue(); - } - - @Override - protected short doShortValue() throws IOException { - return this.parser.shortValue(); - } - - @Override - protected int doIntValue() throws IOException { - return this.parser.intValue(); - } - - @Override - protected long doLongValue() throws IOException { - return this.parser.longValue(); - } - - @Override - protected float doFloatValue() throws IOException { - return this.parser.floatValue(); - } - - @Override - protected double doDoubleValue() throws IOException { - return this.parser.doubleValue(); - } - - @Override - protected BigInteger doBigIntegerValue() throws IOException { - return this.parser.bigIntegerValue(); - } - - @Override - public boolean isClosed() { - return this.parser.isClosed(); - } - - @Override - public void close() throws IOException { - this.parser.close(); - } -} diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index 4fe821ff74d34..424172e844989 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -8,28 +8,29 @@ package org.opensearch.index.mapper; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; -import org.opensearch.Version; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; +import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; -import org.opensearch.common.collect.Iterators; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.unit.Fuzziness; -import org.opensearch.common.xcontent.JsonToStringXContentParser; import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.fielddata.IndexFieldData; @@ -44,18 +45,20 @@ import java.io.UncheckedIOException; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Supplier; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.DOT_SYMBOL; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.EQUAL_SYMBOL; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; import static org.opensearch.index.mapper.FlatObjectFieldMapper.FlatObjectFieldType.getKeywordFieldType; +import static org.opensearch.index.mapper.KeywordFieldMapper.normalizeValue; +import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; +import static org.apache.lucene.search.MultiTermQuery.DOC_VALUES_REWRITE; /** * A field mapper for flat_objects. @@ -67,6 +70,11 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper { public static final String CONTENT_TYPE = "flat_object"; public static final Object DOC_VALUE_NO_MATCH = new Object(); + static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; + static final String VALUE_SUFFIX = "._value"; + static final String DOT_SYMBOL = "."; + static final String EQUAL_SYMBOL = "="; + /** * In flat_object field mapper, field type is similar to keyword field type * Cannot be tokenized, can OmitNorms, and can setIndexOption. @@ -81,28 +89,11 @@ public static class Defaults { FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); FIELD_TYPE.freeze(); } - } @Override public MappedFieldType keyedFieldType(String key) { - return new FlatObjectFieldType( - this.name() + DOT_SYMBOL + key, - this.name(), - (KeywordFieldType) valueFieldMapper.fieldType(), - (KeywordFieldType) valueAndPathFieldMapper.fieldType() - ); - } - - /** - * FlatObjectFieldType is the parent field type. - */ - public static class FlatObjectField extends Field { - - public FlatObjectField(String field, BytesRef term, FieldType ft) { - super(field, term, ft); - } - + return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key, this.name(), valueFieldType, valueAndPathFieldType); } /** @@ -116,24 +107,6 @@ public Builder(String name) { builder = this; } - /** - * ValueFieldMapper is the subfield type for values in the Json. - * use a {@link KeywordFieldMapper.KeywordField} - */ - private ValueFieldMapper buildValueFieldMapper(FieldType fieldType, KeywordFieldType valueFieldType) { - FieldType vft = new FieldType(fieldType); - return new ValueFieldMapper(vft, valueFieldType); - } - - /** - * ValueAndPathFieldMapper is the subfield type for path=value format in the Json. - * also use a {@link KeywordFieldMapper.KeywordField} - */ - private ValueAndPathFieldMapper buildValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType valueAndPathFieldType) { - FieldType vft = new FieldType(fieldType); - return new ValueAndPathFieldMapper(vft, valueAndPathFieldType); - } - @Override public FlatObjectFieldMapper build(BuilderContext context) { boolean isSearchable = true; @@ -147,15 +120,7 @@ public FlatObjectFieldMapper build(BuilderContext context) { ); FlatObjectFieldType fft = new FlatObjectFieldType(buildFullName(context), null, valueFieldType, valueAndPathFieldType); - return new FlatObjectFieldMapper( - name, - Defaults.FIELD_TYPE, - fft, - buildValueFieldMapper(Defaults.FIELD_TYPE, valueFieldType), - buildValueAndPathFieldMapper(Defaults.FIELD_TYPE, valueAndPathFieldType), - CopyTo.empty(), - this - ); + return new FlatObjectFieldMapper(name, Defaults.FIELD_TYPE, fft); } } @@ -173,8 +138,7 @@ public TypeParser(BiFunction builderFunction) { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = builderFunction.apply(name, parserContext); - return builder; + return builderFunction.apply(name, parserContext); } } @@ -186,40 +150,22 @@ public static final class FlatObjectFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; - - private final String mappedFieldTypeName; - + private final String rootFieldName; private final KeywordFieldType valueFieldType; - private final KeywordFieldType valueAndPathFieldType; - public FlatObjectFieldType( - String name, - String mappedFieldTypeName, - boolean isSearchable, - boolean hasDocValues, - NamedAnalyzer analyzer, - Map meta - ) { - super( + public FlatObjectFieldType(String name, String rootFieldName, boolean isSearchable, boolean hasDocValues) { + this( name, - isSearchable, - false, - hasDocValues, - analyzer == null ? TextSearchInfo.SIMPLE_MATCH_ONLY : new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), - meta + rootFieldName, + getKeywordFieldType(rootFieldName == null ? name : rootFieldName, VALUE_SUFFIX, isSearchable, hasDocValues), + getKeywordFieldType(rootFieldName == null ? name : rootFieldName, VALUE_AND_PATH_SUFFIX, isSearchable, hasDocValues) ); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); - this.ignoreAbove = Integer.MAX_VALUE; - this.nullValue = null; - this.mappedFieldTypeName = mappedFieldTypeName; - this.valueFieldType = getKeywordFieldType(name, VALUE_SUFFIX, isSearchable, hasDocValues); - this.valueAndPathFieldType = getKeywordFieldType(name, VALUE_AND_PATH_SUFFIX, isSearchable, hasDocValues); } public FlatObjectFieldType( String name, - String mappedFieldTypeName, + String rootFieldName, KeywordFieldType valueFieldType, KeywordFieldType valueAndPathFieldType ) { @@ -231,19 +177,20 @@ public FlatObjectFieldType( new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); + assert rootFieldName == null || (name.length() > rootFieldName.length() && name.startsWith(rootFieldName)); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; - this.mappedFieldTypeName = mappedFieldTypeName; + this.rootFieldName = rootFieldName; this.valueFieldType = valueFieldType; this.valueAndPathFieldType = valueAndPathFieldType; } - static KeywordFieldType getKeywordFieldType(String fullName, String valueType, boolean isSearchable, boolean hasDocValue) { - return new KeywordFieldType(fullName + valueType, isSearchable, hasDocValue, Collections.emptyMap()) { + static KeywordFieldType getKeywordFieldType(String rootField, String suffix, boolean isSearchable, boolean hasDocValue) { + return new KeywordFieldType(rootField + suffix, isSearchable, hasDocValue, Collections.emptyMap()) { @Override protected String rewriteForDocValue(Object value) { assert value instanceof String; - return fullName + DOT_SYMBOL + value; + return getDVPrefix(rootField) + value; } }; } @@ -317,8 +264,8 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { "Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones" ); } - if (mappedFieldTypeName != null) { - return new FlatObjectDocValueFormat(mappedFieldTypeName + DOT_SYMBOL + name() + EQUAL_SYMBOL); + if (rootFieldName != null) { + return new FlatObjectDocValueFormat(getDVPrefix(rootFieldName) + getPathPrefix(name())); } else { throw new IllegalArgumentException( "Field [" + name() + "] of type [" + typeName() + "] does not support doc_value in root field" @@ -355,32 +302,37 @@ protected BytesRef indexedValueForSearch(Object value) { if (value == null) { return null; } - value = inputToString(value); + if (value instanceof BytesRef) { + value = ((BytesRef) value).utf8ToString(); + } return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } private KeywordFieldType valueFieldType() { - return (mappedFieldTypeName == null) ? valueFieldType : valueAndPathFieldType; + return (rootFieldName == null) ? valueFieldType : valueAndPathFieldType; } @Override public Query termQueryCaseInsensitive(Object value, QueryShardContext context) { - return valueFieldType().termQueryCaseInsensitive(rewriteValue(inputToString(value)), context); + failIfNotIndexedAndNoDocValues(); + return valueFieldType().termQueryCaseInsensitive(rewriteSearchValue(value), context); } /** * redirect queries with rewrite value to rewriteSearchValue and directSubFieldName */ @Override - public Query termQuery(Object value, @Nullable QueryShardContext context) { - return valueFieldType().termQuery(rewriteValue(inputToString(value)), context); + public Query termQuery(Object value, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); + return valueFieldType().termQuery(rewriteSearchValue(value), context); } @Override public Query termsQuery(List values, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); List parsedValues = new ArrayList<>(values.size()); for (Object value : values) { - parsedValues.add(rewriteValue(inputToString(value))); + parsedValues.add(rewriteSearchValue(value)); } return valueFieldType().termsQuery(parsedValues, context); @@ -392,12 +344,8 @@ public Query termsQuery(List values, QueryShardContext context) { * else, direct to flatObjectFieldName._value subfield. * @return directedSubFieldName */ - public String directSubfield() { - if (mappedFieldTypeName == null) { - return new StringBuilder().append(this.name()).append(VALUE_SUFFIX).toString(); - } else { - return new StringBuilder().append(this.mappedFieldTypeName).append(VALUE_AND_PATH_SUFFIX).toString(); - } + public String getSearchField() { + return isSubField() ? rootFieldName + VALUE_AND_PATH_SUFFIX : name() + VALUE_SUFFIX; } /** @@ -406,73 +354,21 @@ public String directSubfield() { * then rewrite the searchValueString as the format "dotpath=value", * @return rewriteSearchValue */ - public String rewriteValue(String searchValueString) { - if (!hasMappedFieldTyeNameInQueryFieldName(name())) { - return searchValueString; - } else { - String rewriteSearchValue = new StringBuilder().append(name()).append(EQUAL_SYMBOL).append(searchValueString).toString(); - return rewriteSearchValue; + public String rewriteSearchValue(Object value) { + if (value instanceof BytesRef) { + value = ((BytesRef) value).utf8ToString(); } - + return isSubField() ? getPathPrefix(name()) + value : value.toString(); } - boolean hasMappedFieldTyeNameInQueryFieldName(String input) { - String prefix = this.mappedFieldTypeName; - if (prefix == null) { - return false; - } - if (!input.startsWith(prefix)) { - return false; - } - String rest = input.substring(prefix.length()); - - if (rest.isEmpty()) { - return false; - } else { - return true; - } - } - - private String inputToString(Object inputValue) { - if (inputValue == null) { - return null; - } - if (inputValue instanceof Integer) { - String inputToString = Integer.toString((Integer) inputValue); - return inputToString; - } else if (inputValue instanceof Float) { - String inputToString = Float.toString((Float) inputValue); - return inputToString; - } else if (inputValue instanceof Boolean) { - String inputToString = Boolean.toString((Boolean) inputValue); - return inputToString; - } else if (inputValue instanceof Short) { - String inputToString = Short.toString((Short) inputValue); - return inputToString; - } else if (inputValue instanceof Long) { - String inputToString = Long.toString((Long) inputValue); - return inputToString; - } else if (inputValue instanceof Double) { - String inputToString = Double.toString((Double) inputValue); - return inputToString; - } else if (inputValue instanceof BytesRef) { - String inputToString = (((BytesRef) inputValue).utf8ToString()); - return inputToString; - } else if (inputValue instanceof String) { - String inputToString = (String) inputValue; - return inputToString; - } else if (inputValue instanceof Version) { - String inputToString = inputValue.toString(); - return inputToString; - } else { - // default to cast toString - return inputValue.toString(); - } + boolean isSubField() { + return rootFieldName != null; } @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { - return valueFieldType().prefixQuery(rewriteValue(value), method, caseInsensitive, context); + failIfNotIndexedAndNoDocValues(); + return valueFieldType().prefixQuery(rewriteSearchValue(value), method, caseInsensitive, context); } @Override @@ -484,7 +380,8 @@ public Query regexpQuery( @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context ) { - return valueFieldType().regexpQuery(rewriteValue(value), syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + failIfNotIndexedAndNoDocValues(); + return valueFieldType().regexpQuery(rewriteSearchValue(value), syntaxFlags, matchFlags, maxDeterminizedStates, method, context); } @Override @@ -497,8 +394,9 @@ public Query fuzzyQuery( @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context ) { + failIfNotIndexedAndNoDocValues(); return valueFieldType().fuzzyQuery( - rewriteValue(inputToString(value)), + rewriteSearchValue(value), fuzziness, prefixLength, maxExpansions, @@ -510,13 +408,71 @@ public Query fuzzyQuery( @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - return valueFieldType().rangeQuery( - rewriteValue(inputToString(lowerTerm)), - rewriteValue(inputToString(upperTerm)), - includeLower, - includeUpper, - context - ); + if (context.allowExpensiveQueries() == false) { + throw new OpenSearchException( + "[range] queries on [text] or [keyword] fields cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false." + ); + } + failIfNotIndexedAndNoDocValues(); + + if ((lowerTerm != null && upperTerm != null)) { + return valueFieldType().rangeQuery( + rewriteSearchValue(lowerTerm), + rewriteSearchValue(upperTerm), + includeLower, + includeUpper, + context + ); + } + + // when either the upper term or lower term is null, + // we can't delegate to valueFieldType() and need to process the prefix ourselves + Query indexQuery = null; + Query dvQuery = null; + if (isSearchable()) { + if (isSubField() == false) { + indexQuery = new TermRangeQuery( + getSearchField(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper + ); + } else { + Automaton a1 = PrefixQuery.toAutomaton(indexedValueForSearch(getPathPrefix(name()))); + BytesRef lowerTermBytes = lowerTerm == null ? null : indexedValueForSearch(rewriteSearchValue(lowerTerm)); + BytesRef upperTermBytes = upperTerm == null ? null : indexedValueForSearch(rewriteSearchValue(upperTerm)); + Automaton a2 = TermRangeQuery.toAutomaton(lowerTermBytes, upperTermBytes, includeLower, includeUpper); + Automaton termAutomaton = Operations.intersection(a1, a2); + indexQuery = new AutomatonQuery( + new Term(getSearchField()), + termAutomaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + true + ); + } + } + if (hasDocValues()) { + String dvPrefix = isSubField() ? getDVPrefix(rootFieldName) : getDVPrefix(name()); + String prefix = dvPrefix + (isSubField() ? getPathPrefix(name()) : ""); + Automaton a1 = PrefixQuery.toAutomaton(indexedValueForSearch(prefix)); + BytesRef lowerDvBytes = lowerTerm == null ? null : indexedValueForSearch(dvPrefix + rewriteSearchValue(lowerTerm)); + BytesRef upperDvBytes = upperTerm == null ? null : indexedValueForSearch(dvPrefix + rewriteSearchValue(upperTerm)); + Automaton a2 = TermRangeQuery.toAutomaton(lowerDvBytes, upperDvBytes, includeLower, includeUpper); + Automaton dvAutomaton = Operations.intersection(a1, a2); + dvQuery = new AutomatonQuery( + new Term(getSearchField()), + dvAutomaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + true, + DOC_VALUES_REWRITE + ); + } + + assert indexQuery != null || dvQuery != null; + return indexQuery == null ? dvQuery : (dvQuery == null ? indexQuery : new IndexOrDocValuesQuery(indexQuery, dvQuery)); } /** @@ -527,8 +483,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower public Query existsQuery(QueryShardContext context) { String searchKey; String searchField; - if (hasMappedFieldTyeNameInQueryFieldName(name())) { - searchKey = this.mappedFieldTypeName; + if (isSubField()) { + searchKey = this.rootFieldName; searchField = name(); } else { if (hasDocValues()) { @@ -548,7 +504,8 @@ public Query wildcardQuery( boolean caseInsensitve, QueryShardContext context ) { - return valueFieldType().wildcardQuery(rewriteValue(value), method, caseInsensitve, context); + failIfNotIndexedAndNoDocValues(); + return valueFieldType().wildcardQuery(rewriteSearchValue(value), method, caseInsensitve, context); } /** @@ -572,7 +529,7 @@ public void writeTo(StreamOutput out) {} @Override public Object format(BytesRef value) { - String parsedValue = inputToString(value); + String parsedValue = value.utf8ToString(); if (parsedValue.startsWith(prefix) == false) { return DOC_VALUE_NO_MATCH; } @@ -581,29 +538,19 @@ public Object format(BytesRef value) { @Override public BytesRef parseBytesRef(String value) { - return new BytesRef((String) valueFieldType.rewriteForDocValue(rewriteValue(value))); + return new BytesRef((String) valueFieldType.rewriteForDocValue(rewriteSearchValue(value))); } } } - private final ValueFieldMapper valueFieldMapper; - private final ValueAndPathFieldMapper valueAndPathFieldMapper; - - FlatObjectFieldMapper( - String simpleName, - FieldType fieldType, - FlatObjectFieldType mappedFieldType, - ValueFieldMapper valueFieldMapper, - ValueAndPathFieldMapper valueAndPathFieldMapper, - CopyTo copyTo, - Builder builder - ) { - super(simpleName, fieldType, mappedFieldType, copyTo); + private final KeywordFieldType valueFieldType; + private final KeywordFieldType valueAndPathFieldType; + + FlatObjectFieldMapper(String simpleName, FieldType fieldType, FlatObjectFieldType mappedFieldType) { + super(simpleName, fieldType, mappedFieldType, CopyTo.empty()); assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; - this.fieldType = fieldType; - this.valueFieldMapper = valueFieldMapper; - this.valueAndPathFieldMapper = valueAndPathFieldMapper; - this.mappedFieldType = mappedFieldType; + valueFieldType = mappedFieldType.valueFieldType; + valueAndPathFieldType = mappedFieldType.valueAndPathFieldType; } @Override @@ -623,224 +570,123 @@ public FlatObjectFieldType fieldType() { @Override protected void parseCreateField(ParseContext context) throws IOException { - String fieldName = null; - - if (context.externalValueSet()) { - String value = context.externalValue().toString(); - parseValueAddFields(context, value, fieldType().name()); - } else { - XContentParser ctxParser = context.parser(); - if (ctxParser.currentToken() != XContentParser.Token.VALUE_NULL) { - if (ctxParser.currentToken() != XContentParser.Token.START_OBJECT) { - throw new ParsingException( - ctxParser.getTokenLocation(), - "[" + this.name() + "] unexpected token [" + ctxParser.currentToken() + "] in flat_object field value" - ); - } + XContentParser ctxParser = context.parser(); + if (fieldType().isSearchable() == false && fieldType().isStored() == false && fieldType().hasDocValues() == false) { + ctxParser.skipChildren(); + return; + } - JsonToStringXContentParser jsonToStringParser = new JsonToStringXContentParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.IGNORE_DEPRECATIONS, - ctxParser, - fieldType().name() + if (ctxParser.currentToken() != XContentParser.Token.VALUE_NULL) { + if (ctxParser.currentToken() != XContentParser.Token.START_OBJECT) { + throw new ParsingException( + ctxParser.getTokenLocation(), + "[" + this.name() + "] unexpected token [" + ctxParser.currentToken() + "] in flat_object field value" ); - /* - JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser - It reads the JSON object and parsed to a list of string - */ - XContentParser parser = jsonToStringParser.parseObject(); - - XContentParser.Token currentToken; - while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - switch (currentToken) { - case FIELD_NAME: - fieldName = parser.currentName(); - break; - case VALUE_STRING: - String value = parser.textOrNull(); - parseValueAddFields(context, value, fieldName); - break; - } - - } } + parseObject(ctxParser, context); } } - @Override - public Iterator iterator() { - List subIterators = new ArrayList<>(); - if (valueFieldMapper != null) { - subIterators.add(valueFieldMapper); - } - if (valueAndPathFieldMapper != null) { - subIterators.add(valueAndPathFieldMapper); - } - if (subIterators.size() == 0) { - return super.iterator(); - } - @SuppressWarnings("unchecked") - Iterator concat = Iterators.concat(super.iterator(), subIterators.iterator()); - return concat; - } - - /** - * parseValueAddFields method will store data to Lucene. - * the JsonToStringXContentParser returns XContentParser with 3 string fields - * fieldName, fieldName._value, fieldName._valueAndPath. - * parseValueAddFields recognized string by the stringfield name, - * fieldName will be store through the parent FlatObjectFieldMapper,which contains all the keys - * fieldName._value will be store through the valueFieldMapper, which contains the values of the Json Object - * fieldName._valueAndPath will be store through the valueAndPathFieldMapper, which contains the "path=values" format - */ - private void parseValueAddFields(ParseContext context, String value, String fieldName) throws IOException { + private void parseObject(XContentParser parser, ParseContext context) throws IOException { + assert parser.currentToken() == XContentParser.Token.START_OBJECT; + parser.nextToken(); // Skip the outer START_OBJECT. Need to return on END_OBJECT. - assert valueFieldMapper != null; - assert valueAndPathFieldMapper != null; - NamedAnalyzer normalizer = fieldType().normalizer(); - if (normalizer != null) { - value = normalizeValue(normalizer, name(), value); + LinkedList path = new LinkedList<>(Collections.singleton(fieldType().name())); + HashSet pathParts = new HashSet<>(); + while (parser.currentToken() != XContentParser.Token.END_OBJECT) { + parseToken(parser, context, path, pathParts); } - String[] valueTypeList = fieldName.split("\\._"); - String valueType = "._" + valueTypeList[valueTypeList.length - 1]; - - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - // convert to utf8 only once before feeding postings/dv/stored fields - final BytesRef binaryValue = new BytesRef(fieldType().name() + DOT_SYMBOL + value); - - if (fieldType().hasDocValues() == false) { - createFieldNamesField(context); - } - if (fieldName.equals(fieldType().name())) { - Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); - } else if (valueType.equals(VALUE_SUFFIX)) { - valueFieldMapper.addField(context, value); - } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - valueAndPathFieldMapper.addField(context, value); - } - - if (fieldType().hasDocValues()) { - if (fieldName.equals(fieldType().name())) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } else if (valueType.equals(VALUE_SUFFIX)) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); - } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); - } - } - } + createPathFields(context, pathParts); } - private static String normalizeValue(NamedAnalyzer normalizer, String field, String value) throws IOException { - try (TokenStream ts = normalizer.tokenStream(field, value)) { - final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - ts.reset(); - if (ts.incrementToken() == false) { - throw new IllegalStateException(errorMessage(normalizer, value)); + private void createPathFields(ParseContext context, HashSet pathParts) { + for (String part : pathParts) { + final BytesRef value = new BytesRef(name() + DOT_SYMBOL + part); + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + context.doc().add(new Field(name(), value, fieldType)); } - final String newValue = termAtt.toString(); - if (ts.incrementToken()) { - throw new IllegalStateException(errorMessage(normalizer, value)); + if (fieldType().hasDocValues()) { + context.doc().add(new SortedSetDocValuesField(name(), value)); + } else { + createFieldNamesField(context); } - ts.end(); - return newValue; } } - private static String errorMessage(NamedAnalyzer normalizer, String value) { - return "The normalization token stream is " - + "expected to produce exactly 1 token, but got 0 for analyzer " - + normalizer - + " and input \"" - + value - + "\""; + private static String getDVPrefix(String rootFieldName) { + return rootFieldName + DOT_SYMBOL; } - @Override - protected String contentType() { - return CONTENT_TYPE; + private static String getPathPrefix(String path) { + return path + EQUAL_SYMBOL; } - private static final class ValueAndPathFieldMapper extends FieldMapper { - - protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { - super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); - } - - void addField(ParseContext context, String value) { - final BytesRef binaryValue = new BytesRef(value); - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - Field field = new KeywordFieldMapper.KeywordField(fieldType().name(), binaryValue, fieldType); - - context.doc().add(field); - - if (fieldType().hasDocValues() == false) { - createFieldNamesField(context); - } + private void parseToken(XContentParser parser, ParseContext context, Deque path, HashSet pathParts) throws IOException { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + final String currentFieldName = parser.currentName(); + path.addLast(currentFieldName); // Pushing onto the stack *must* be matched by pop + parser.nextToken(); // advance to the value of fieldName + parseToken(parser, context, path, pathParts); // parse the value for fieldName (which will be an array, an object, + // or a primitive value) + path.removeLast(); // Here is where we pop fieldName from the stack (since we're done with the value of fieldName) + // Note that whichever other branch we just passed through has already ended with nextToken(), so we + // don't need to call it. + } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + parser.nextToken(); + while (parser.currentToken() != XContentParser.Token.END_ARRAY) { + parseToken(parser, context, path, pathParts); } - } - - @Override - protected void parseCreateField(ParseContext context) { - throw new UnsupportedOperationException(); - } - - @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - - } - - @Override - protected String contentType() { - return "valueAndPath"; - } - - @Override - public String toString() { - return fieldType().toString(); - } - - } - - private static final class ValueFieldMapper extends FieldMapper { - - protected ValueFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { - super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); - } - - void addField(ParseContext context, String value) { - final BytesRef binaryValue = new BytesRef(value); - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - Field field = new KeywordFieldMapper.KeywordField(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); - - if (fieldType().hasDocValues() == false) { - createFieldNamesField(context); - } + parser.nextToken(); + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + while (parser.currentToken() != XContentParser.Token.END_OBJECT) { + parseToken(parser, context, path, pathParts); + } + parser.nextToken(); + } else { + String value = parseValue(parser); + if (value == null || value.length() > fieldType().ignoreAbove) { + parser.nextToken(); + return; + } + NamedAnalyzer normalizer = fieldType().normalizer(); + if (normalizer != null) { + value = normalizeValue(normalizer, name(), value); + } + final String leafPath = Strings.collectionToDelimitedString(path, "."); + final String valueAndPath = getPathPrefix(leafPath) + value; + if (fieldType().isSearchable() || fieldType().isStored()) { + context.doc().add(new Field(valueFieldType.name(), new BytesRef(value), fieldType)); + context.doc().add(new Field(valueAndPathFieldType.name(), new BytesRef(valueAndPath), fieldType)); } - } - - @Override - protected void parseCreateField(ParseContext context) { - throw new UnsupportedOperationException(); - } - - @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - } + if (fieldType().hasDocValues()) { + context.doc().add(new SortedSetDocValuesField(valueFieldType.name(), new BytesRef(getDVPrefix(name()) + value))); + context.doc() + .add(new SortedSetDocValuesField(valueAndPathFieldType.name(), new BytesRef(getDVPrefix(name()) + valueAndPath))); + } - @Override - protected String contentType() { - return "value"; + pathParts.addAll(Arrays.asList(leafPath.substring(name().length() + 1).split("\\."))); + parser.nextToken(); } + } - @Override - public String toString() { - return fieldType().toString(); + private static String parseValue(XContentParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_BOOLEAN: + case VALUE_NUMBER: + case VALUE_STRING: + case VALUE_NULL: + return parser.textOrNull(); + // Handle other token types as needed + default: + throw new ParsingException(parser.getTokenLocation(), "Unexpected value token type [" + parser.currentToken() + "]"); } } + @Override + protected String contentType() { + return CONTENT_TYPE; + } } diff --git a/server/src/test/java/org/opensearch/common/xcontent/JsonToStringXContentParserTests.java b/server/src/test/java/org/opensearch/common/xcontent/JsonToStringXContentParserTests.java deleted file mode 100644 index 3c292181b4d8f..0000000000000 --- a/server/src/test/java/org/opensearch/common/xcontent/JsonToStringXContentParserTests.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.common.xcontent; - -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; - -public class JsonToStringXContentParserTests extends OpenSearchTestCase { - - private String flattenJsonString(String fieldName, String in) throws IOException { - try ( - XContentParser parser = JsonXContent.jsonXContent.createParser( - xContentRegistry(), - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - in - ) - ) { - JsonToStringXContentParser jsonToStringXContentParser = new JsonToStringXContentParser( - xContentRegistry(), - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - parser, - fieldName - ); - // Point to the first token (should be START_OBJECT) - jsonToStringXContentParser.nextToken(); - - XContentParser transformedParser = jsonToStringXContentParser.parseObject(); - assertSame(XContentParser.Token.END_OBJECT, jsonToStringXContentParser.currentToken()); - try (XContentBuilder jsonBuilder = XContentFactory.jsonBuilder()) { - jsonBuilder.copyCurrentStructure(transformedParser); - return jsonBuilder.toString(); - } - } - } - - public void testNestedObjects() throws IOException { - String jsonExample = "{" + "\"first\" : \"1\"," + "\"second\" : {" + " \"inner\": \"2.0\"" + "}," + "\"third\": \"three\"" + "}"; - - assertEquals( - "{" - + "\"flat\":[\"third\",\"inner\",\"first\",\"second\"]," - + "\"flat._value\":[\"1\",\"2.0\",\"three\"]," - + "\"flat._valueAndPath\":[\"flat.second.inner=2.0\",\"flat.first=1\",\"flat.third=three\"]" - + "}", - flattenJsonString("flat", jsonExample) - ); - } - - public void testChildHasDots() throws IOException { - // This should be exactly the same as testNestedObjects. We're just using the "flat" notation for the inner - // object. - String jsonExample = "{" + "\"first\" : \"1\"," + "\"second.inner\" : \"2.0\"," + "\"third\": \"three\"" + "}"; - - assertEquals( - "{" - + "\"flat\":[\"third\",\"inner\",\"first\",\"second\"]," - + "\"flat._value\":[\"1\",\"2.0\",\"three\"]," - + "\"flat._valueAndPath\":[\"flat.second.inner=2.0\",\"flat.first=1\",\"flat.third=three\"]" - + "}", - flattenJsonString("flat", jsonExample) - ); - } - - public void testNestChildObjectWithDots() throws IOException { - String jsonExample = "{" - + "\"first\" : \"1\"," - + "\"second.inner\" : {" - + " \"really_inner\" : \"2.0\"" - + "}," - + "\"third\": \"three\"" - + "}"; - - assertEquals( - "{" - + "\"flat\":[\"really_inner\",\"third\",\"inner\",\"first\",\"second\"]," - + "\"flat._value\":[\"1\",\"2.0\",\"three\"]," - + "\"flat._valueAndPath\":[\"flat.first=1\",\"flat.second.inner.really_inner=2.0\",\"flat.third=three\"]" - + "}", - flattenJsonString("flat", jsonExample) - ); - } - - public void testNestChildObjectWithDotsAndFieldWithDots() throws IOException { - String jsonExample = "{" - + "\"first\" : \"1\"," - + "\"second.inner\" : {" - + " \"totally.absolutely.inner\" : \"2.0\"" - + "}," - + "\"third\": \"three\"" - + "}"; - - assertEquals( - "{" - + "\"flat\":[\"third\",\"absolutely\",\"totally\",\"inner\",\"first\",\"second\"]," - + "\"flat._value\":[\"1\",\"2.0\",\"three\"]," - + "\"flat._valueAndPath\":[\"flat.first=1\",\"flat.second.inner.totally.absolutely.inner=2.0\",\"flat.third=three\"]" - + "}", - flattenJsonString("flat", jsonExample) - ); - } - - public void testArrayOfObjects() throws IOException { - String jsonExample = "{" - + "\"field\": {" - + " \"detail\": {" - + " \"foooooooooooo\": [" - + " {\"name\":\"baz\"}," - + " {\"name\":\"baz\"}" - + " ]" - + " }" - + "}}"; - - assertEquals( - "{" - + "\"flat\":[\"field\",\"name\",\"detail\",\"foooooooooooo\"]," - + "\"flat._value\":[\"baz\"]," - + "\"flat._valueAndPath\":[" - + "\"flat.field.detail.foooooooooooo.name=baz\"" - + "]}", - flattenJsonString("flat", jsonExample) - ); - } - - public void testArraysOfObjectsAndValues() throws IOException { - String jsonExample = "{" - + "\"field\": {" - + " \"detail\": {" - + " \"foooooooooooo\": [" - + " {\"name\":\"baz\"}," - + " {\"name\":\"baz\"}" - + " ]" - + " }," - + " \"numbers\" : [" - + " 1," - + " 2," - + " 3" - + " ]" - + "}}"; - - assertEquals( - "{" - + "\"flat\":[\"field\",\"name\",\"numbers\",\"detail\",\"foooooooooooo\"]," - + "\"flat._value\":[\"1\",\"2\",\"3\",\"baz\"]," - + "\"flat._valueAndPath\":[" - + "\"flat.field.detail.foooooooooooo.name=baz\"," - + "\"flat.field.numbers=1\"," - + "\"flat.field.numbers=3\"," - + "\"flat.field.numbers=2\"" - + "]}", - flattenJsonString("flat", jsonExample) - ); - } -} diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java index 7e6aa00c87290..118f58cf5e855 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java @@ -16,18 +16,22 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.TriFunction; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; +import org.hamcrest.MatcherAssert; import java.io.IOException; +import java.util.List; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; import static org.opensearch.index.mapper.FlatObjectFieldMapper.CONTENT_TYPE; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.VALUE_SUFFIX; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.core.IsEqual.equalTo; @@ -57,7 +61,7 @@ protected void assertExistsQuery(MapperService mapperService) throws IOException protected void assertExistsQuery(FlatObjectFieldMapper.FlatObjectFieldType fieldType, Query query, ParseContext.Document fields) { - if (fieldType.hasDocValues() && fieldType.hasMappedFieldTyeNameInQueryFieldName(fieldType.name()) == false) { + if (fieldType.hasDocValues() && fieldType.isSubField() == false) { assertThat(query, instanceOf(FieldExistsQuery.class)); FieldExistsQuery fieldExistsQuery = (FieldExistsQuery) query; assertEquals(fieldType.name(), fieldExistsQuery.getField()); @@ -133,12 +137,12 @@ public void testDefaults() throws Exception { // Test internal substring fields as well IndexableField[] fieldValues = doc.rootDoc().getFields("field" + VALUE_SUFFIX); assertEquals(2, fieldValues.length); - assertTrue(fieldValues[0] instanceof KeywordFieldMapper.KeywordField); + assertEquals(IndexOptions.DOCS, fieldValues[0].fieldType().indexOptions()); assertEquals(new BytesRef("bar"), fieldValues[0].binaryValue()); IndexableField[] fieldValueAndPaths = doc.rootDoc().getFields("field" + VALUE_AND_PATH_SUFFIX); assertEquals(2, fieldValues.length); - assertTrue(fieldValueAndPaths[0] instanceof KeywordFieldMapper.KeywordField); + assertEquals(IndexOptions.DOCS, fieldValues[0].fieldType().indexOptions()); assertEquals(new BytesRef("field.foo=bar"), fieldValueAndPaths[0].binaryValue()); } @@ -223,37 +227,39 @@ public void testNullValue() throws IOException { assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field" + VALUE_SUFFIX)); assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field" + VALUE_AND_PATH_SUFFIX)); + TriFunction makeFieldsAssertion = (v1, v2, fs) -> { + MatcherAssert.assertThat( + List.of(new BytesRef(v1), new BytesRef(v2)), + containsInAnyOrder(fs[0].binaryValue(), fs[2].binaryValue()) + ); + return null; + }; + // test9: {"field":{"name": [null,3],"age":4}} json = "{\"field\":{\"name\": [null,3],\"age\":4}}"; doc = mapper.parse(source(json)); fields = doc.rootDoc().getFields("field"); assertEquals(4, fields.length); - assertEquals(new BytesRef("field.name"), fields[0].binaryValue()); - assertEquals(new BytesRef("field.age"), fields[2].binaryValue()); + makeFieldsAssertion.apply("field.name", "field.age", fields); fieldValues = doc.rootDoc().getFields("field" + VALUE_SUFFIX); assertEquals(4, fieldValues.length); - assertEquals(new BytesRef("3"), fieldValues[0].binaryValue()); - assertEquals(new BytesRef("4"), fieldValues[2].binaryValue()); + makeFieldsAssertion.apply("3", "4", fieldValues); fieldValueAndPaths = doc.rootDoc().getFields("field" + VALUE_AND_PATH_SUFFIX); assertEquals(4, fieldValueAndPaths.length); - assertEquals(new BytesRef("field.age=4"), fieldValueAndPaths[0].binaryValue()); - assertEquals(new BytesRef("field.name=3"), fieldValueAndPaths[2].binaryValue()); + makeFieldsAssertion.apply("field.name=3", "field.age=4", fieldValueAndPaths); // test10: {"field":{"age": 4,"name": [null,"3"]}} json = "{\"field\":{\"age\": 4,\"name\": [null,\"3\"]}}"; doc = mapper.parse(source(json)); fields = doc.rootDoc().getFields("field"); assertEquals(4, fields.length); - assertEquals(new BytesRef("field.name"), fields[0].binaryValue()); - assertEquals(new BytesRef("field.age"), fields[2].binaryValue()); + makeFieldsAssertion.apply("field.name", "field.age", fields); fieldValues = doc.rootDoc().getFields("field" + VALUE_SUFFIX); assertEquals(4, fieldValues.length); - assertEquals(new BytesRef("3"), fieldValues[0].binaryValue()); - assertEquals(new BytesRef("4"), fieldValues[2].binaryValue()); + makeFieldsAssertion.apply("3", "4", fieldValues); fieldValueAndPaths = doc.rootDoc().getFields("field" + VALUE_AND_PATH_SUFFIX); assertEquals(4, fieldValueAndPaths.length); - assertEquals(new BytesRef("field.age=4"), fieldValueAndPaths[0].binaryValue()); - assertEquals(new BytesRef("field.name=3"), fieldValueAndPaths[2].binaryValue()); + makeFieldsAssertion.apply("field.name=3", "field.age=4", fieldValueAndPaths); // test11: {"field":{"age":"4","labels": [null]}} json = "{\"field\":{\"age\":\"4\",\"labels\": [null]}}"; @@ -318,12 +324,10 @@ public void testNullValue() throws IOException { doc = mapper.parse(source(json)); fields = doc.rootDoc().getFields("field"); assertEquals(4, fields.length); - assertEquals(new BytesRef("field.d"), fields[0].binaryValue()); - assertEquals(new BytesRef("field.name"), fields[2].binaryValue()); + makeFieldsAssertion.apply("field.d", "field.name", fields); fieldValues = doc.rootDoc().getFields("field" + VALUE_SUFFIX); assertEquals(4, fieldValues.length); - assertEquals(new BytesRef("dsds"), fieldValues[0].binaryValue()); - assertEquals(new BytesRef("age1"), fieldValues[2].binaryValue()); + makeFieldsAssertion.apply("dsds", "age1", fieldValues); fieldValueAndPaths = doc.rootDoc().getFields("field" + VALUE_AND_PATH_SUFFIX); assertEquals(4, fieldValueAndPaths.length); assertEquals(new BytesRef("field.name.name=age1"), fieldValueAndPaths[0].binaryValue()); @@ -375,29 +379,6 @@ public void testInfiniteLoopWithNullValue() throws IOException { assertEquals(new BytesRef("field.age=3"), fieldValueAndPaths[0].binaryValue()); } - // test deduplicationValue of keyList, valueList, valueAndPathList - public void testDeduplicationValue() throws IOException { - DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - - // test: {"field":{"age": 3,"labels": [null,"3"], "abc":{"abc":{"labels":"n"}}}} - String json = "{\"field\":{\"age\": 3,\"labels\": [null,\"3\"], \"abc\":{\"abc\":{\"labels\":\"n\"}}}}"; - ParsedDocument doc = mapper.parse(source(json)); - IndexableField[] fields = doc.rootDoc().getFields("field"); - assertEquals(6, fields.length); - assertEquals(new BytesRef("field.abc"), fields[0].binaryValue()); - assertEquals(new BytesRef("field.age"), fields[2].binaryValue()); - assertEquals(new BytesRef("field.labels"), fields[4].binaryValue()); - IndexableField[] fieldValues = doc.rootDoc().getFields("field" + VALUE_SUFFIX); - assertEquals(4, fieldValues.length); - assertEquals(new BytesRef("3"), fieldValues[0].binaryValue()); - assertEquals(new BytesRef("n"), fieldValues[2].binaryValue()); - IndexableField[] fieldValueAndPaths = doc.rootDoc().getFields("field" + VALUE_AND_PATH_SUFFIX); - assertEquals(6, fieldValueAndPaths.length); - assertEquals(new BytesRef("field.abc.abc.labels=n"), fieldValueAndPaths[0].binaryValue()); - assertEquals(new BytesRef("field.age=3"), fieldValueAndPaths[2].binaryValue()); - assertEquals(new BytesRef("field.labels=3"), fieldValueAndPaths[4].binaryValue()); - } - public void testFetchDocValues() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "flat_object"))); { diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java index 4160108342534..5b158b204b289 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; @@ -24,22 +25,23 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.unit.Fuzziness; import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.mapper.FlatObjectFieldMapper.FlatObjectFieldType; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.VALUE_SUFFIX; import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; import static org.apache.lucene.search.MultiTermQuery.DOC_VALUES_REWRITE; @@ -56,9 +58,7 @@ private static MappedFieldType getFlatParentFieldType( fieldName, mappedFieldTypeName, isSearchable, - hasDocValues, - null, - Collections.emptyMap() + hasDocValues ); FieldType fieldtype = new FieldType(FlatObjectFieldMapper.Defaults.FIELD_TYPE); FieldType vft = new FieldType(fieldtype); @@ -89,29 +89,29 @@ public void testFetchSourceValue() throws IOException { } - public void testDirectSubfield() { + public void testGetSearchField() { { FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = (FlatObjectFieldMapper.FlatObjectFieldType) (getFlatParentFieldType("field", null, true, true)); // when searching for "foo" in "field", the directSubfield is field._value field - String searchFieldName = (flatParentFieldType).directSubfield(); + String searchFieldName = (flatParentFieldType).getSearchField(); assertEquals("field._value", searchFieldName); MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( - "bar", + flatParentFieldType.name() + ".bar", flatParentFieldType.name(), flatParentFieldType.getValueFieldType(), flatParentFieldType.getValueAndPathFieldType() ); // when searching for "foo" in "field.bar", the directSubfield is field._valueAndPath field - String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); + String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).getSearchField(); assertEquals("field._valueAndPath", searchFieldNameDocPath); } { NamedAnalyzer analyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, null); - MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, true, true, analyzer, Collections.emptyMap()); - assertEquals("field._value", ((FlatObjectFieldMapper.FlatObjectFieldType) ft).directSubfield()); + MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, true, true); + assertEquals("field._value", ((FlatObjectFieldMapper.FlatObjectFieldType) ft).getSearchField()); } } @@ -124,7 +124,7 @@ public void testRewriteValue() { ); // when searching for "foo" in "field", the rewrite value is "foo" - String searchValues = (flatParentFieldType).rewriteValue("foo"); + String searchValues = (flatParentFieldType).rewriteSearchValue("foo"); assertEquals("foo", searchValues); MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( @@ -135,8 +135,8 @@ public void testRewriteValue() { ); // when searching for "foo" in "field.bar", the rewrite value is "field.bar=foo" - String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); - String searchValuesDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).rewriteValue("foo"); + String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).getSearchField(); + String searchValuesDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).rewriteSearchValue("foo"); assertEquals("field.bar=foo", searchValuesDocPath); } @@ -176,12 +176,12 @@ public void testTermQueryCaseInsensitive() { // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, false ); - Query expected = new TermQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("fOo"))); + Query expected = new TermQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.field1=fOo"))); assertEquals(expected, ft.termQuery("fOo", MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -205,14 +205,14 @@ public void testTermQueryCaseInsensitive() { // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, true ); Query expected = AutomatonQueries.createAutomatonQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "field.fOo"), - AutomatonQueries.toCaseInsensitiveString("field.fOo", Integer.MAX_VALUE), + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field.field1=fOo"), + AutomatonQueries.toCaseInsensitiveString("field.field.field1=fOo", Integer.MAX_VALUE), MultiTermQuery.DOC_VALUES_REWRITE ); @@ -232,7 +232,7 @@ public void testTermQueryCaseInsensitive() { () -> ft.termQueryCaseInsensitive("foo", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -240,7 +240,7 @@ public void testTermQueryCaseInsensitive() { // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, false @@ -250,27 +250,17 @@ public void testTermQueryCaseInsensitive() { () -> ft.termQuery("foo", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } - MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType( - "field", - null, - false, - false, - null, - Collections.emptyMap() - ); + MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, false, false); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> unsearchable.termQuery("bar", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values enabled.", - e.getMessage() - ); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testTermQuery() { @@ -281,12 +271,12 @@ public void testTermQuery() { (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType("field", null, true, true); // when searching for "foo" in "field", the term query is directed to search "foo" in field._value field - String searchFieldName = (flatParentFieldType).directSubfield(); - String searchValues = (flatParentFieldType).rewriteValue("foo"); + String searchFieldName = flatParentFieldType.getSearchField(); + String searchValues = flatParentFieldType.rewriteSearchValue("foo"); assertEquals("foo", searchValues); assertEquals(new TermQuery(new Term(searchFieldName, searchValues)), flatParentFieldType.termQuery(searchValues, null)); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + FlatObjectFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( "field.bar", flatParentFieldType.name(), flatParentFieldType.getValueFieldType(), @@ -294,14 +284,13 @@ public void testTermQuery() { ); // when searching for "foo" in "field.bar", the term query is directed to search in field._valueAndPath field - String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); - String searchValuesDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).rewriteValue("foo"); + String searchFieldNameDocPath = dynamicMappedFieldType.getSearchField(); + String searchValuesDocPath = dynamicMappedFieldType.rewriteSearchValue("foo"); assertEquals("field.bar=foo", searchValuesDocPath); assertEquals( new TermQuery(new Term(searchFieldNameDocPath, searchValuesDocPath)), dynamicMappedFieldType.termQuery("foo", null) ); - } // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null @@ -319,12 +308,12 @@ public void testTermQuery() { // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, false ); - Query expected = new TermQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo"))); + Query expected = new TermQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.field1=foo"))); assertEquals(expected, ft.termQuery("foo", MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -351,15 +340,15 @@ public void testTermQuery() { // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, true ); Query expected = SortedSetDocValuesField.newSlowRangeQuery( "field" + VALUE_AND_PATH_SUFFIX, - new BytesRef("field.foo"), - new BytesRef("field.foo"), + new BytesRef("field.field.field1=foo"), + new BytesRef("field.field.field1=foo"), true, true ); @@ -379,7 +368,7 @@ public void testTermQuery() { () -> ft.termQuery("foo", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -387,7 +376,7 @@ public void testTermQuery() { // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, false @@ -397,27 +386,17 @@ public void testTermQuery() { () -> ft.termQuery("foo", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } - MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType( - "field", - null, - false, - false, - null, - Collections.emptyMap() - ); + MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, false, false); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> unsearchable.termQuery("bar", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values enabled.", - e.getMessage() - ); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testExistsQuery() { @@ -443,14 +422,7 @@ public void testExistsQuery() { } { - FlatObjectFieldMapper.FlatObjectFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType( - "field", - null, - true, - false, - null, - Collections.emptyMap() - ); + FlatObjectFieldMapper.FlatObjectFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, true, false); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } } @@ -482,18 +454,18 @@ public void testTermsQuery() { // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, true ); List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); + indexTerms.add(new BytesRef("field.field1=foo")); + indexTerms.add(new BytesRef("field.field1=bar")); List docValueterms = new ArrayList<>(); - docValueterms.add(new BytesRef("field.foo")); - docValueterms.add(new BytesRef("field.bar")); + docValueterms.add(new BytesRef("field.field.field1=foo")); + docValueterms.add(new BytesRef("field.field.field1=bar")); Query expected = new IndexOrDocValuesQuery( new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms), new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms) @@ -521,14 +493,14 @@ public void testTermsQuery() { // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, false ); List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); + indexTerms.add(new BytesRef("field.field1=foo")); + indexTerms.add(new BytesRef("field.field1=bar")); Query expected = new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms); assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null)); @@ -557,19 +529,16 @@ public void testTermsQuery() { // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, true ); - List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); - List docValueterms = new ArrayList<>(); - docValueterms.add(new BytesRef("field.foo")); - docValueterms.add(new BytesRef("field.bar")); - Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms); + List docValueTerms = new ArrayList<>(); + docValueTerms.add(new BytesRef("field.field.field1=foo")); + docValueTerms.add(new BytesRef("field.field.field1=bar")); + Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueTerms); assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -587,7 +556,7 @@ public void testTermsQuery() { () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -595,7 +564,7 @@ public void testTermsQuery() { // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, false @@ -605,7 +574,7 @@ public void testTermsQuery() { () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -630,14 +599,14 @@ public void testPrefixQuery() { // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, true ); Query expected = new IndexOrDocValuesQuery( - new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE), - new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE) + new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field1=foo"), CONSTANT_SCORE_REWRITE), + new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field.field1=foo"), DOC_VALUES_REWRITE) ); assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -657,12 +626,12 @@ public void testPrefixQuery() { // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, false ); - Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE); + Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field1=foo"), CONSTANT_SCORE_REWRITE); assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -681,12 +650,12 @@ public void testPrefixQuery() { // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, true ); - Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE); + Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field.field1=foo"), DOC_VALUES_REWRITE); assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -703,7 +672,7 @@ public void testPrefixQuery() { () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -711,7 +680,7 @@ public void testPrefixQuery() { // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, false @@ -721,7 +690,7 @@ public void testPrefixQuery() { () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -760,14 +729,14 @@ public void testRegexpQuery() { // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, true ); Query expected = new IndexOrDocValuesQuery( new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.field1=foo")), 0, 0, RegexpQuery.DEFAULT_PROVIDER, @@ -775,7 +744,7 @@ public void testRegexpQuery() { CONSTANT_SCORE_REWRITE ), new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.field.field1=foo")), 0, 0, RegexpQuery.DEFAULT_PROVIDER, @@ -808,13 +777,13 @@ public void testRegexpQuery() { // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, false ); Query expected = new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.field1=foo")), 0, 0, RegexpQuery.DEFAULT_PROVIDER, @@ -846,13 +815,13 @@ public void testRegexpQuery() { // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, true ); Query expected = new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.field.field1=foo")), 0, 0, RegexpQuery.DEFAULT_PROVIDER, @@ -875,7 +844,7 @@ public void testRegexpQuery() { () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -883,7 +852,7 @@ public void testRegexpQuery() { // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, false @@ -893,7 +862,7 @@ public void testRegexpQuery() { () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -918,14 +887,21 @@ public void testFuzzyQuery() { // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, true ); Query expected = new IndexOrDocValuesQuery( - new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true), - new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) + new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field1=foo"), 2, 1, 50, true), + new FuzzyQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field.field1=foo"), + 2, + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE + ) ); assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -945,12 +921,12 @@ public void testFuzzyQuery() { // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, false ); - Query expected = new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true); + Query expected = new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field1=foo"), 2, 1, 50, true); assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } @@ -976,13 +952,13 @@ public void testFuzzyQuery() { // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, true ); Query expected = new FuzzyQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field.field1=foo"), 2, 1, 50, @@ -1005,7 +981,7 @@ public void testFuzzyQuery() { () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -1013,7 +989,7 @@ public void testFuzzyQuery() { // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, false @@ -1023,153 +999,236 @@ public void testFuzzyQuery() { () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } } public void testRangeQuery() { - // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), - new TermRangeQuery( - "field" + VALUE_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), - new TermRangeQuery( - "field" + VALUE_AND_PATH_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - false - ); - Query expected = new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - false - ); - Query expected = new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - true - ); - Query expected = new TermRangeQuery( - "field" + VALUE_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - true - ); - Query expected = new TermRangeQuery( - "field" + VALUE_AND_PATH_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - - // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); + for (boolean searchable : new boolean[] { true, false }) { + for (boolean hasDocValue : new boolean[] { true, false }) { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + searchable, + hasDocValue + ); + + if (searchable == false && hasDocValue == false) { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + continue; + } + + Query indexQuery = new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); + Query dvQuery = new TermRangeQuery( + "field" + VALUE_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ); + Query expected = searchable == false + ? dvQuery + : (hasDocValue ? new IndexOrDocValuesQuery(indexQuery, dvQuery) : indexQuery); + assertEquals( + expected, + ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + } + } + } + + { + for (boolean searchable : new boolean[] { true, false }) { + for (boolean hasDocValue : new boolean[] { true, false }) { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field.field1", + "field", + searchable, + hasDocValue + ); + + if (searchable == false && hasDocValue == false) { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + + "enabled.", + e.getMessage() + ); + continue; + } + + Query indexQuery = new TermRangeQuery( + "field" + VALUE_AND_PATH_SUFFIX, + new BytesRef("field.field1=2"), + new BytesRef("field.field1=10"), + true, + true + ); + Query dvQuery = new TermRangeQuery( + "field" + VALUE_AND_PATH_SUFFIX, + new BytesRef("field.field.field1=2"), + new BytesRef("field.field.field1=10"), + true, + true, + DOC_VALUES_REWRITE + ); + Query expected = searchable == false + ? dvQuery + : (hasDocValue ? new IndexOrDocValuesQuery(indexQuery, dvQuery) : indexQuery); + assertEquals( + expected, + ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + } + } + } + + { + for (boolean searchable : new boolean[] { true, false }) { + for (boolean hasDocValue : new boolean[] { true, false }) { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + searchable, + hasDocValue + ); + + if (searchable == false && hasDocValue == false) { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + continue; + } + boolean nullLowerTerm = randomBoolean(); + boolean nullUpperTerm = nullLowerTerm == false || randomBoolean(); + + Automaton a1 = PrefixQuery.toAutomaton(new BytesRef("field.")); + Automaton a2 = TermRangeQuery.toAutomaton( + nullLowerTerm ? null : new BytesRef("field.2"), + nullUpperTerm ? null : new BytesRef("field.10"), + true, + true + ); + Automaton dvAutomaton = Operations.intersection(a1, a2); + Query indexQuery = new TermRangeQuery( + "field" + VALUE_SUFFIX, + nullLowerTerm ? null : new BytesRef("2"), + nullUpperTerm ? null : new BytesRef("10"), + true, + true + ); + Query dvQuery = new AutomatonQuery( + new Term("field" + VALUE_SUFFIX), + dvAutomaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + true, + DOC_VALUES_REWRITE + ); + Query expected = searchable == false + ? dvQuery + : (hasDocValue ? new IndexOrDocValuesQuery(indexQuery, dvQuery) : indexQuery); + assertEquals( + expected, + ft.rangeQuery( + nullLowerTerm ? null : new BytesRef("2"), + nullUpperTerm ? null : new BytesRef("10"), + true, + true, + MOCK_QSC_ENABLE_INDEX_DOC_VALUES + ) + ); + } + } + } + + { + for (boolean searchable : new boolean[] { true, false }) { + for (boolean hasDocValue : new boolean[] { true, false }) { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field.field1", + "field", + searchable, + hasDocValue + ); + if (searchable == false && hasDocValue == false) { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + + "enabled.", + e.getMessage() + ); + continue; + } + boolean nullLowerTerm = true;// randomBoolean(); + boolean nullUpperTerm = true;// nullLowerTerm == false || randomBoolean(); + + Automaton a1 = PrefixQuery.toAutomaton(new BytesRef("field.field1=")); + Automaton a2 = TermRangeQuery.toAutomaton( + nullLowerTerm ? null : new BytesRef("field.field1=2"), + nullUpperTerm ? null : new BytesRef("field.field1=10"), + true, + true + ); + Automaton termAutomaton = Operations.intersection(a1, a2); + + Automaton dvA1 = PrefixQuery.toAutomaton(new BytesRef("field.field.field1=")); + Automaton dvA2 = TermRangeQuery.toAutomaton( + nullLowerTerm ? null : new BytesRef("field.field.field1=2"), + nullUpperTerm ? null : new BytesRef("field.field.field1=10"), + true, + true + ); + Automaton dvAutomaton = Operations.intersection(dvA1, dvA2); + Query indexQuery = new AutomatonQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX), + termAutomaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + true + ); + Query dvQuery = new AutomatonQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX), + dvAutomaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + true, + DOC_VALUES_REWRITE + ); + Query expected = searchable == false + ? dvQuery + : (hasDocValue ? new IndexOrDocValuesQuery(indexQuery, dvQuery) : indexQuery); + assertEquals( + expected, + ft.rangeQuery( + nullLowerTerm ? null : new BytesRef("2"), + nullUpperTerm ? null : new BytesRef("10"), + true, + true, + MOCK_QSC_ENABLE_INDEX_DOC_VALUES + ) + ); + } + } } } @@ -1200,19 +1259,19 @@ public void testWildcardQuery() { // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, true ); Query expected = new IndexOrDocValuesQuery( new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field1=foo*"), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.CONSTANT_SCORE_REWRITE ), new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field.field1=foo*"), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE ) @@ -1239,13 +1298,13 @@ public void testWildcardQuery() { // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", true, false ); Query expected = new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field1=foo*"), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.CONSTANT_SCORE_REWRITE ); @@ -1271,13 +1330,13 @@ public void testWildcardQuery() { // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, true ); Query expected = new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.field.field1=foo*"), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE ); @@ -1297,7 +1356,7 @@ public void testWildcardQuery() { () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); } @@ -1305,7 +1364,7 @@ public void testWildcardQuery() { // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null { FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", + "field.field1", "field", false, false @@ -1315,7 +1374,7 @@ public void testWildcardQuery() { () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + "Cannot search on field [field.field1] since it is both not indexed, and does not have doc_values " + "enabled.", e.getMessage() ); }