diff --git a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java index bd3a5a9779..1d53504566 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java @@ -105,4 +105,8 @@ public static String format(final String format, Object... args) { private static boolean isQuoted(String text, String mark) { return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark); } + + public static String removeParenthesis(String qualifier) { + return qualifier.replaceAll("\\[\\d+\\]", ""); + } } diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 601e3e00cc..488f7f4f33 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -17,14 +17,17 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import java.util.stream.Collectors; import lombok.Getter; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.Between; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; @@ -49,7 +52,7 @@ import org.opensearch.sql.ast.expression.When; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.expression.Xor; -import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -57,6 +60,7 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.HighlightExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.NamedExpression; @@ -368,8 +372,28 @@ public Expression visitAllFields(AllFields node, AnalysisContext context) { public Expression visitQualifiedName(QualifiedName node, AnalysisContext context) { QualifierAnalyzer qualifierAnalyzer = new QualifierAnalyzer(context); - // check for reserved words in the identifier - for (String part : node.getParts()) { + Expression reserved = checkForReservedIdentifier(node.getParts(), context, qualifierAnalyzer, node); + if (reserved != null) { + return reserved; + } + + return visitIdentifier(qualifierAnalyzer.unqualified(node), context); + } + + @Override + public Expression visitArrayQualifiedName(ArrayQualifiedName node, AnalysisContext context) { + QualifierAnalyzer qualifierAnalyzer = new QualifierAnalyzer(context); + + Expression reserved = checkForReservedIdentifier(node.getParts(), context, qualifierAnalyzer, node); + if (reserved != null) { + return reserved; + } + + return visitArrayIdentifier(qualifierAnalyzer.unqualified(node), context, node.getPartsAndIndexes()); + } + + private Expression checkForReservedIdentifier(List parts, AnalysisContext context, QualifierAnalyzer qualifierAnalyzer, QualifiedName node) { + for (String part : parts) { for (TypeEnvironment typeEnv = context.peek(); typeEnv != null; typeEnv = typeEnv.getParent()) { @@ -384,7 +408,7 @@ public Expression visitQualifiedName(QualifiedName node, AnalysisContext context } } } - return visitIdentifier(qualifierAnalyzer.unqualified(node), context); + return null; } @Override @@ -422,9 +446,27 @@ private Expression visitIdentifier(String ident, AnalysisContext context) { } TypeEnvironment typeEnv = context.peek(); + var type = typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident)); ReferenceExpression ref = DSL.ref(ident, - typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); + if (type.equals(ExprCoreType.ARRAY)) { + return new ArrayReferenceExpression(ref); + } return ref; } + + private Expression visitArrayIdentifier(String ident, AnalysisContext context, + List> partsAndIndexes) { + // ParseExpression will always override ReferenceExpression when ident conflicts + for (NamedExpression expr : context.getNamedParseExpressions()) { + if (expr.getNameOrAlias().equals(ident) && expr.getDelegated() instanceof ParseExpression) { + return expr.getDelegated(); + } + } + + TypeEnvironment typeEnv = context.peek(); + return new ArrayReferenceExpression(ident, + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, StringUtils.removeParenthesis(ident))), partsAndIndexes); + } } diff --git a/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java index d1e31d0079..d75603be13 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java @@ -11,8 +11,10 @@ import lombok.RequiredArgsConstructor; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; /** @@ -38,6 +40,10 @@ public String unqualified(QualifiedName fullName) { return isQualifierIndexOrAlias(fullName) ? fullName.rest().toString() : fullName.toString(); } + public String unqualified(ArrayQualifiedName fullName) { + return isQualifierIndexOrAlias(fullName) ? fullName.rest().toString() : fullName.toString(); + } + private boolean isQualifierIndexOrAlias(QualifiedName fullName) { Optional qualifier = fullName.first(); if (qualifier.isPresent()) { @@ -50,10 +56,22 @@ private boolean isQualifierIndexOrAlias(QualifiedName fullName) { return false; } + private boolean isQualifierIndexOrAlias(ArrayQualifiedName fullName) { + Optional qualifier = fullName.first(); + if (qualifier.isPresent()) { + if (isFieldName(qualifier.get())) { + return false; + } + resolveQualifierSymbol(fullName, qualifier.get()); + return true; + } + return false; + } + private boolean isFieldName(String qualifier) { try { // Resolve the qualifier in Namespace.FIELD_NAME - context.peek().resolve(new Symbol(Namespace.FIELD_NAME, qualifier)); + context.peek().resolve(new Symbol(Namespace.FIELD_NAME, StringUtils.removeParenthesis(qualifier))); return true; } catch (SemanticCheckException e2) { return false; diff --git a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java index 734f37378b..7798aaf198 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java @@ -23,9 +23,11 @@ import org.opensearch.sql.ast.expression.NestedAllTupleFields; import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; @@ -105,8 +107,17 @@ public List visitAllFields(AllFields node, AnalysisContext context) { TypeEnvironment environment = context.peek(); Map lookupAllFields = environment.lookupAllFields(Namespace.FIELD_NAME); - return lookupAllFields.entrySet().stream().map(entry -> DSL.named(entry.getKey(), - new ReferenceExpression(entry.getKey(), entry.getValue()))).collect(Collectors.toList()); + return lookupAllFields.entrySet().stream().map(entry -> + { + ReferenceExpression ref = new ReferenceExpression(entry.getKey(), entry.getValue()); + if (entry.getValue().equals(ExprCoreType.ARRAY)) { + return DSL.named(entry.getKey(), + new ArrayReferenceExpression(ref)); + } else { + return DSL.named(entry.getKey(), ref); + } + } + ).collect(Collectors.toList()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index f02bc07ccc..adb666669e 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -11,6 +11,7 @@ import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.AttributeList; import org.opensearch.sql.ast.expression.Between; import org.opensearch.sql.ast.expression.Case; @@ -194,6 +195,9 @@ public T visitField(Field node, C context) { public T visitQualifiedName(QualifiedName node, C context) { return visitChildren(node, context); } + public T visitArrayQualifiedName(ArrayQualifiedName node, C context) { + return visitChildren(node, context); + } public T visitRename(Rename node, C context) { return visitChildren(node, context); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java new file mode 100644 index 0000000000..910cfdc5be --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ast.expression; + + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +import java.util.List; +import java.util.OptionalInt; +import java.util.stream.Collectors; + +@Getter +@EqualsAndHashCode(callSuper = false) +public class ArrayQualifiedName extends QualifiedName { + + private final List> partsAndIndexes; + + public ArrayQualifiedName(List> parts) { + super(parts.stream().map(p -> p.getLeft()).collect(Collectors.toList())); + this.partsAndIndexes = parts; + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitArrayQualifiedName(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java new file mode 100644 index 0000000000..6674a266ae --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression; + +import static org.opensearch.sql.utils.ExpressionUtils.PATH_SEP; + +import java.util.Arrays; +import java.util.List; +import java.util.OptionalInt; +import java.util.stream.Collectors; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprMissingValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.env.Environment; + +@EqualsAndHashCode +public class ArrayReferenceExpression extends ReferenceExpression { + @Getter + private final List> partsAndIndexes; + @Getter + private final ExprType type; + public ArrayReferenceExpression(String ref, ExprType type, List> partsAndIndexes) { + super(StringUtils.removeParenthesis(ref), type); + this.partsAndIndexes = partsAndIndexes; + this.type = type; + } + + public ArrayReferenceExpression(ReferenceExpression ref) { + super(StringUtils.removeParenthesis(ref.toString()), ref.type()); + this.partsAndIndexes = Arrays.stream(ref.toString().split("\\.")).map(e -> Pair.of(e, OptionalInt.empty())).collect( + Collectors.toList()); + this.type = ref.type(); + } + + @Override + public ExprValue valueOf(Environment env) { + return env.resolve(this); + } + + @Override + public ExprType type() { + return type; + } + + @Override + public T accept(ExpressionNodeVisitor visitor, C context) { + return visitor.visitArrayReference(this, context); + } + + public ExprValue resolve(ExprTupleValue value) { + return resolve(value, partsAndIndexes); + } + + private ExprValue resolve(ExprValue value, List> paths) { + List pathsWithoutParenthesis = + paths.stream().map(p -> StringUtils.removeParenthesis(p.getLeft())).collect(Collectors.toList()); + ExprValue wholePathValue = value.keyValue(String.join(PATH_SEP, pathsWithoutParenthesis)); + + if (!paths.get(0).getRight().isEmpty()) { + if (value.keyValue(pathsWithoutParenthesis.get(0)) instanceof ExprCollectionValue) { // TODO check array size + wholePathValue = value + .keyValue(pathsWithoutParenthesis.get(0)) + .collectionValue() + .get(paths.get(0).getRight().getAsInt()); + if (paths.size() != 1) { + return resolve(wholePathValue, paths.subList(1, paths.size())); + } + } else { + return ExprValueUtils.missingValue(); + } + } else if (wholePathValue.isMissing()) { + return resolve(value.keyValue(pathsWithoutParenthesis.get(0)), paths.subList(1, paths.size())); + } + + if (!wholePathValue.isMissing() || paths.size() == 1) { + return wholePathValue; + } else { + return resolve(value.keyValue(pathsWithoutParenthesis.get(0)), paths.subList(1, paths.size())); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java index e3d4e38674..1999cef44f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java @@ -64,6 +64,10 @@ public T visitReference(ReferenceExpression node, C context) { return visitNode(node, context); } + public T visitArrayReference(ArrayReferenceExpression node, C context) { + return visitNode(node, context); + } + public T visitParse(ParseExpression node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java index 804c38a6f7..0557ac727c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java @@ -51,7 +51,7 @@ public ExprValue valueOf(Environment valueEnv) { if (this.type == ExprCoreType.ARRAY) { refName += "." + StringUtils.unquoteText(getHighlightField().toString()); } - ExprValue value = valueEnv.resolve(DSL.ref(refName, ExprCoreType.STRING)); + ExprValue value = valueEnv.resolve(new ArrayReferenceExpression(DSL.ref(refName, ExprCoreType.STRING))); // In the event of multiple returned highlights and wildcard being // used in conjunction with other highlight calls, we need to ensure diff --git a/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java index 3c5b2af23c..4bac2fb14d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java @@ -106,6 +106,9 @@ private ExprValue resolve(ExprValue value, List paths) { if (value.type().equals(ExprCoreType.ARRAY)) { wholePathValue = value.collectionValue().get(0).keyValue(paths.get(0)); } + if (wholePathValue.type().equals(ExprCoreType.ARRAY)) { + return getFirstValueOfCollection(wholePathValue); + } if (!wholePathValue.isMissing() || paths.size() == 1) { return wholePathValue; @@ -113,4 +116,12 @@ private ExprValue resolve(ExprValue value, List paths) { return resolve(value.keyValue(paths.get(0)), paths.subList(1, paths.size())); } } + + private ExprValue getFirstValueOfCollection(ExprValue value) { + ExprValue collectionVal = value; + while(collectionVal.type().equals(ExprCoreType.ARRAY)) { + collectionVal = collectionVal.collectionValue().get(0); + } + return collectionVal; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index c5fcb010f5..7aa7f7df4b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -17,7 +17,9 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; @UtilityClass @@ -106,7 +108,7 @@ public Pair resolve( new FunctionExpression(BuiltinFunctionName.NESTED.getName(), arguments) { @Override public ExprValue valueOf(Environment valueEnv) { - return valueEnv.resolve(getArguments().get(0)); + return valueEnv.resolve(new ArrayReferenceExpression((ReferenceExpression) getArguments().get(0))); } @Override diff --git a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java index 51a0348116..00890fe2d8 100644 --- a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java +++ b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java @@ -10,6 +10,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; @@ -30,7 +31,9 @@ public ExprValue resolve(ReferenceExpression ref) { */ @Override public ExprValue resolve(Expression var) { - if (var instanceof ReferenceExpression) { + if (var instanceof ArrayReferenceExpression) { + return resolve(((ArrayReferenceExpression) var)); + } else if (var instanceof ReferenceExpression) { return resolve(((ReferenceExpression) var)); } else { throw new ExpressionEvaluationException(String.format("can resolve expression: %s", var)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 7216c03d08..5c2c599c5f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -38,6 +38,7 @@ import static com.google.common.base.Strings.isNullOrEmpty; import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient; import static org.opensearch.sql.legacy.TestUtils.getAccountIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getArraysIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping; @@ -681,7 +682,11 @@ public enum Index { NESTED_WITH_NULLS(TestsConstants.TEST_INDEX_NESTED_WITH_NULLS, "multi_nested", getNestedTypeIndexMapping(), - "src/test/resources/nested_with_nulls.json"); + "src/test/resources/nested_with_nulls.json"), + ARRAYS(TestsConstants.TEST_INDEX_ARRAYS, + "arrays", + getArraysIndexMapping(), + "src/test/resources/arrays.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 30cee86e15..49641c0c29 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -223,6 +223,11 @@ public static String getDateIndexMapping() { return getMappingFile(mappingFile); } + public static String getArraysIndexMapping() { + String mappingFile = "arrays_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getDateTimeIndexMapping() { String mappingFile = "date_time_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 338be25a0c..ea9101b3f0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -48,6 +48,7 @@ public class TestsConstants { public final static String TEST_INDEX_ORDER = TEST_INDEX + "_order"; public final static String TEST_INDEX_WEBLOG = TEST_INDEX + "_weblog"; public final static String TEST_INDEX_DATE = TEST_INDEX + "_date"; + public final static String TEST_INDEX_ARRAYS = TEST_INDEX + "_arrays"; public final static String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime"; public final static String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested"; public final static String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ArraysIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ArraysIT.java new file mode 100644 index 0000000000..65e5af0cc8 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ArraysIT.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ARRAYS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import java.util.List; + +import com.google.common.collect.ImmutableMap; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class ArraysIT extends SQLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.ARRAYS); + } + + @Test + public void object_array_index_1_test() { + String query = "SELECT objectArray[0] FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(new JSONObject(ImmutableMap.of("innerObject", List.of(1, 2))))); + } + + @Test + public void object_array_index_1_inner_object_test() { + String query = "SELECT objectArray[0].innerObject FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(new JSONArray(List.of(1, 2)))); + } + + @Test + public void object_array_index_1_inner_object_index_1_test() { + String query = "SELECT objectArray[0].innerObject[0] FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(1)); + } + + @Test + public void multi_object_array_index_1_test() { + String query = "SELECT multiObjectArray[0] FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(new JSONObject(ImmutableMap.of("id", 1, "name", "blah")))); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index d3230188b7..e09e9a4f53 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -254,7 +254,7 @@ public void nested_function_and_field_with_order_by_clause() { rows("a", 4), rows("b", 2), rows("c", 3), - rows("zz", new JSONArray(List.of(3, 4)))); + rows("zz", 3)); } // Nested function in GROUP BY clause is not yet implemented for JDBC format. This test ensures @@ -535,7 +535,7 @@ public void nested_function_all_subfields_and_non_nested_field() { rows("g", 1, "c", 3), rows("h", 4, "c", 4), rows("i", 5, "a", 4), - rows("zz", 6, "zz", new JSONArray(List.of(3, 4)))); + rows("zz", 6, "zz", 3)); } @Test diff --git a/integ-test/src/test/resources/arrays.json b/integ-test/src/test/resources/arrays.json new file mode 100644 index 0000000000..ce70bf08e9 --- /dev/null +++ b/integ-test/src/test/resources/arrays.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"objectArray": [{"innerObject": [1, 2]}, {"innerObject": 3}], "multiObjectArray":[{"id": 1, "name": "blah"}, {"id": 2,"name": "hoo"}]} diff --git a/integ-test/src/test/resources/indexDefinitions/arrays_mapping.json b/integ-test/src/test/resources/indexDefinitions/arrays_mapping.json new file mode 100644 index 0000000000..2e905dd8b7 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/arrays_mapping.json @@ -0,0 +1,23 @@ +{ + "mappings": { + "properties": { + "objectArray": { + "properties": { + "innerObject": { + "type": "keyword" + } + } + }, + "multiObjectArray": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "type": "keyword" + } + } + } + } + } +} \ No newline at end of file diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 22a43d3444..55c69eebe9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -190,7 +190,8 @@ private ExprValue parse( ExprType type = fieldType.get(); if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) - || content.isArray()) { + || content.objectValue() instanceof ArrayNode + ) { return parseArray(content, field, type, supportArrays); } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) || type == STRUCT) { @@ -330,12 +331,6 @@ private ExprValue parseArray( // ARRAY is mapped to nested but can take the json structure of an Object. if (content.objectValue() instanceof ObjectNode) { result.add(parseStruct(content, prefix, supportArrays)); - // non-object type arrays are only supported when parsing inner_hits of OS response. - } else if ( - !(type instanceof OpenSearchDataType - && ((OpenSearchDataType) type).getExprType().equals(ARRAY)) - && !supportArrays) { - return parseInnerArrayValue(content.array().next(), prefix, type, supportArrays); } else { content.array().forEachRemaining(v -> { result.add(parseInnerArrayValue(v, prefix, type, supportArrays)); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java index 590272a9f1..bd3023b340 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java @@ -19,6 +19,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.OpenSearchFunctions; @@ -156,6 +157,10 @@ public static List findReferenceExpression(NamedExpression public Object visitReference(ReferenceExpression node, Object context) { return results.add(node); } + @Override + public Object visitArrayReference(ArrayReferenceExpression node, Object context) { + return results.add(node); + } }, null); return results; } diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index e68edbbc58..9fc72b19f5 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -303,6 +303,7 @@ expressions expressionAtom : constant #constantExpressionAtom | columnName #fullColumnNameExpressionAtom +// | arrayColumnName #arrayColumnNameExpressionAtom | functionCall #functionCallExpressionAtom | LR_BRACKET expression RR_BRACKET #nestedExpressionAtom | left=expressionAtom @@ -814,6 +815,12 @@ columnName : qualifiedName ; +//arrayColumnName +// : arrayQualifiedName +// | qualifiedName LT_SQR_PRTHS COLON_SYMB RT_SQR_PRTHS (arrayColumnName | columnName) * +// | qualifiedName LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS (arrayColumnName | columnName) * +// ; + allTupleFields : path=qualifiedName DOT STAR ; @@ -822,12 +829,21 @@ alias : ident ; +//arrayQualifiedName +// : (ident | indexedIdentifier) (DOT (ident | indexedIdentifier))* +//// (( LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS ) | (DOT ident ( LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS )* ))+ +// ; + +//indexedIdentifier +// : ident LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS +// ; + qualifiedName : ident (DOT ident)* ; ident - : DOT? ID + : DOT? ID (LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS)? | BACKTICK_QUOTE_ID | keywordsCanBeId | scalarFunctionName diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 7279553106..d1f0c9ee1c 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -69,11 +69,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; + +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.OptionalInt; import java.util.stream.Collectors; import org.antlr.v4.runtime.RuleContext; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -82,6 +85,7 @@ import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.DataType; @@ -103,7 +107,6 @@ import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchQueryContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AndExpressionContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnNameContext; @@ -129,6 +132,27 @@ public UnresolvedExpression visitColumnName(ColumnNameContext ctx) { return visit(ctx.qualifiedName()); } +// @Override +// public UnresolvedExpression visitArrayColumnName(ArrayColumnNameContext ctx) { +//// return new QualifiedName( +//// identifiers.stream() +//// .map(RuleContext::getText) +//// .map(StringUtils::unquoteIdentifier) +//// .collect(Collectors.toList())); +// +// var blah = ctx.arrayQualifiedName().indexedIdentifier(); +// var hmm = ctx.arrayQualifiedName().ident(); +// +// +// UnresolvedExpression qualifiedName = visit(ctx.arrayQualifiedName()); +//// if (ctx.arrayQualifiedName().decimalLiteral() == null) { +// return new ArrayQualifiedName(qualifiedName.toString()); +//// } else { +// return new ArrayQualifiedName( +// qualifiedName.toString(), Integer.parseInt(ctx.arrayQualifiedName().decimalLiteral().toString())); +//// } +// } + @Override public UnresolvedExpression visitIdent(IdentContext ctx) { return visitIdentifiers(Collections.singletonList(ctx)); @@ -532,6 +556,32 @@ public UnresolvedExpression visitExtractFunctionCall(ExtractFunctionCallContext private QualifiedName visitIdentifiers(List identifiers) { + + List> parts = new ArrayList<>(); + boolean supportsArrays = false; + for(var blah : identifiers) { + if (blah.decimalLiteral() != null) { + parts.add( + Pair.of( + StringUtils.unquoteIdentifier(blah.getText()), + OptionalInt.of(Integer.parseInt(blah.decimalLiteral().getText())) + ) + ); + supportsArrays = true; + } else { + parts.add( + Pair.of( + StringUtils.unquoteIdentifier(blah.getText()), + OptionalInt.empty() + ) + ); + } + } + + if (supportsArrays) { + return new ArrayQualifiedName(parts); + } + return new QualifiedName( identifiers.stream() .map(RuleContext::getText)