diff --git a/src/java/org/apache/cassandra/cql3/Operator.java b/src/java/org/apache/cassandra/cql3/Operator.java index 41b7985ffc4d..cd34eb1ff17f 100644 --- a/src/java/org/apache/cassandra/cql3/Operator.java +++ b/src/java/org/apache/cassandra/cql3/Operator.java @@ -41,10 +41,14 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - if (analyzer != null) - return ANALYZER_MATCHES.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + if (indexAnalyzer != null) + return ANALYZER_MATCHES.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); return type.compareForCQL(leftOperand, rightOperand) == 0; } @@ -58,7 +62,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return type.compareForCQL(leftOperand, rightOperand) < 0; } @@ -72,7 +80,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return type.compareForCQL(leftOperand, rightOperand) <= 0; } @@ -86,7 +98,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return type.compareForCQL(leftOperand, rightOperand) >= 0; } @@ -100,7 +116,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return type.compareForCQL(leftOperand, rightOperand) > 0; } @@ -113,7 +133,11 @@ public String toString() return "IN"; } - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { List inValues = ListType.getInstance(type, false).getSerializer().deserialize(rightOperand); return inValues.contains(type.getSerializer().deserialize(leftOperand)); @@ -128,7 +152,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { CollectionType collectionType = (CollectionType) type; return collectionType.contains(leftOperand, rightOperand); @@ -143,7 +171,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { MapType mapType = (MapType) type; return mapType.containsKey(leftOperand, rightOperand); @@ -159,7 +191,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return type.compareForCQL(leftOperand, rightOperand) != 0; @@ -174,7 +210,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { throw new UnsupportedOperationException(); } @@ -188,7 +228,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return ByteBufferUtil.startsWith(leftOperand, rightOperand); } @@ -202,7 +246,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return ByteBufferUtil.endsWith(leftOperand, rightOperand); } @@ -216,7 +264,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return ByteBufferUtil.contains(leftOperand, rightOperand); } @@ -229,7 +281,11 @@ public String toString() return "LIKE ''"; } - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return ByteBufferUtil.contains(leftOperand, rightOperand); } @@ -243,7 +299,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { throw new UnsupportedOperationException(); } @@ -257,7 +317,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { return true; } @@ -270,9 +334,13 @@ public String toString() return "NOT IN"; } - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !IN.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !IN.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, NOT_CONTAINS(17) @@ -284,9 +352,13 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !CONTAINS.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !CONTAINS.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, @@ -299,9 +371,13 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !CONTAINS_KEY.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !CONTAINS_KEY.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, NOT_LIKE_PREFIX(19) @@ -313,9 +389,13 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !LIKE_PREFIX.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !LIKE_PREFIX.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, NOT_LIKE_SUFFIX(20) @@ -327,9 +407,13 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !LIKE_SUFFIX.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !LIKE_SUFFIX.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, NOT_LIKE_CONTAINS(21) @@ -341,9 +425,13 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !LIKE_CONTAINS.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !LIKE_CONTAINS.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, NOT_LIKE_MATCHES(22) @@ -354,9 +442,13 @@ public String toString() return "NOT LIKE ''"; } - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !LIKE_MATCHES.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !LIKE_MATCHES.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, NOT_LIKE(23) @@ -368,9 +460,13 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - return !LIKE.isSatisfiedBy(type, leftOperand, rightOperand, analyzer); + return !LIKE.isSatisfiedBy(type, leftOperand, rightOperand, indexAnalyzer, queryAnalyzer); } }, @@ -386,12 +482,16 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - assert analyzer != null : ": operation can only be computed by an indexed column with a configured analyzer"; + assert indexAnalyzer != null && queryAnalyzer != null : ": operation can only be computed by an indexed column with a configured analyzer"; - List leftTokens = analyzer.analyze(leftOperand); - List rightTokens = analyzer.analyze(rightOperand); + List leftTokens = indexAnalyzer.analyze(leftOperand); + List rightTokens = queryAnalyzer.analyze(rightOperand); Iterator it = rightTokens.iterator(); @@ -431,7 +531,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { throw new UnsupportedOperationException(); } @@ -446,7 +550,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { throw new UnsupportedOperationException(); } @@ -460,7 +568,11 @@ public String toString() } @Override - public boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer) + public boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { throw new UnsupportedOperationException(); } @@ -519,10 +631,16 @@ public static Operator readFrom(DataInput input) throws IOException * @param type the type of the values to compare. * @param leftOperand the left operand of the comparison. * @param rightOperand the right operand of the comparison. - * @param analyzer an index-provided function to transform the compared values before comparison, or {@code null} if - * the values don't need to be transformed. + * @param indexAnalyzer an index-provided function to transform the left-side compared value before comparison, + * or {@code null} if the values don't need to be transformed. + * @param queryAnalyzer an index-provided function to transform the right-side compared value before comparison, + * or {@code null} if the values don't need to be transformed. */ - public abstract boolean isSatisfiedBy(AbstractType type, ByteBuffer leftOperand, ByteBuffer rightOperand, @Nullable Index.Analyzer analyzer); + public abstract boolean isSatisfiedBy(AbstractType type, + ByteBuffer leftOperand, + ByteBuffer rightOperand, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer); public int serializedSize() { diff --git a/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java index aa6ac45621f9..2533a758927b 100644 --- a/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java +++ b/src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java @@ -257,7 +257,7 @@ else if (otherValue == null) // the condition value is not null, so only NEQ can return true return operator == Operator.NEQ; } - return operator.isSatisfiedBy(type, otherValue, value, null); // We don't use any analyzers in LWT, see CNDB-11658 + return operator.isSatisfiedBy(type, otherValue, value, null, null); // We don't use any analyzers in LWT, see CNDB-11658 } } diff --git a/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java b/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java index 41a249da4427..2f8d3c1cb8e0 100644 --- a/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java +++ b/src/java/org/apache/cassandra/cql3/conditions/ColumnConditions.java @@ -93,7 +93,7 @@ public Set getAnalyzedColumns(IndexRegistry indexRegistry) for (ColumnCondition condition : this) { - if (indexRegistry.getAnalyzerFor(condition.column, condition.operator).isPresent()) + if (indexRegistry.getIndexAnalyzerFor(condition.column, condition.operator).isPresent()) { analyzedColumns.add(condition.column); } diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java index af5076fd757d..dad3bb212d8a 100644 --- a/src/java/org/apache/cassandra/db/filter/RowFilter.java +++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java @@ -261,7 +261,7 @@ public boolean partitionKeyRestrictionsAreSatisfiedBy(DecoratedKey key, Abstract ByteBuffer value = keyValidator instanceof CompositeType ? ((CompositeType) keyValidator).split(key.getKey())[e.column.position()] : key.getKey(); - if (!e.operator().isSatisfiedBy(e.column.type, value, e.value, e.analyzer())) + if (!e.operator().isSatisfiedBy(e.column.type, value, e.value, e.indexAnalyzer(), e.queryAnalyzer())) return false; } return true; @@ -278,7 +278,7 @@ public boolean clusteringKeyRestrictionsAreSatisfiedBy(Clustering clustering) if (!e.column.isClusteringColumn()) continue; - if (!e.operator().isSatisfiedBy(e.column.type, clustering.bufferAt(e.column.position()), e.value, e.analyzer())) + if (!e.operator().isSatisfiedBy(e.column.type, clustering.bufferAt(e.column.position()), e.value, e.indexAnalyzer(), e.queryAnalyzer())) return false; } return true; @@ -465,7 +465,7 @@ else if (builder.current.children.size() == 1 && builder.current.expressions.isE public SimpleExpression add(ColumnMetadata def, Operator op, ByteBuffer value) { assert op != Operator.ANN : "ANN expressions should be added with the addANNExpression method"; - SimpleExpression expression = new SimpleExpression(def, op, value, analyzer(def, op), null); + SimpleExpression expression = new SimpleExpression(def, op, value, indexAnalyzer(def, op), queryAnalyzer(def, op), null); add(expression); return expression; } @@ -479,18 +479,24 @@ public SimpleExpression add(ColumnMetadata def, Operator op, ByteBuffer value) */ public void addANNExpression(ColumnMetadata def, ByteBuffer value, ANNOptions annOptions) { - add(new SimpleExpression(def, Operator.ANN, value, null, annOptions)); + add(new SimpleExpression(def, Operator.ANN, value, null, null, annOptions)); } public void addMapComparison(ColumnMetadata def, ByteBuffer key, Operator op, ByteBuffer value) { - add(new MapComparisonExpression(def, key, op, value, analyzer(def, op))); + add(new MapComparisonExpression(def, key, op, value, indexAnalyzer(def, op), queryAnalyzer(def, op))); } @Nullable - private Index.Analyzer analyzer(ColumnMetadata def, Operator op) + private Index.Analyzer indexAnalyzer(ColumnMetadata def, Operator op) { - return indexRegistry == null ? null : indexRegistry.getAnalyzerFor(def, op).orElse(null); + return indexRegistry == null ? null : indexRegistry.getIndexAnalyzerFor(def, op).orElse(null); + } + + @Nullable + private Index.Analyzer queryAnalyzer(ColumnMetadata def, Operator op) + { + return indexRegistry == null ? null : indexRegistry.getQueryAnalyzerFor(def, op).orElse(null); } public void addGeoDistanceExpression(ColumnMetadata def, ByteBuffer point, Operator op, ByteBuffer distance) @@ -870,7 +876,13 @@ public Operator operator() } @Nullable - public Index.Analyzer analyzer() + public Index.Analyzer indexAnalyzer() + { + return null; + } + + @Nullable + public Index.Analyzer queryAnalyzer() { return null; } @@ -1046,7 +1058,9 @@ public Expression deserialize(DataInputPlus in, int version, TableMetadata metad ByteBuffer name = ByteBufferUtil.readWithShortLength(in); Operator operator = Operator.readFrom(in); ColumnMetadata column = metadata.getColumn(name); - Index.Analyzer analyzer = IndexRegistry.obtain(metadata).getAnalyzerFor(column, operator).orElse(null); + IndexRegistry indexRegistry = IndexRegistry.obtain(metadata); + Index.Analyzer indexAnalyzer = indexRegistry.getIndexAnalyzerFor(column, operator).orElse(null); + Index.Analyzer queryAnalyzer = indexRegistry.getQueryAnalyzerFor(column, operator).orElse(null); // Compact storage tables, when used with thrift, used to allow falling through this withouot throwing an // exception. However, since thrift was removed in 4.0, this behaviour was not restored in CASSANDRA-16217 @@ -1058,11 +1072,11 @@ public Expression deserialize(DataInputPlus in, int version, TableMetadata metad case SIMPLE: ByteBuffer value = ByteBufferUtil.readWithShortLength(in); ANNOptions annOptions = operator == Operator.ANN ? ANNOptions.serializer.deserialize(in, version) : null; - return new SimpleExpression(column, operator, value, analyzer, annOptions); + return new SimpleExpression(column, operator, value, indexAnalyzer, queryAnalyzer, annOptions); case MAP_COMPARISON: ByteBuffer key = ByteBufferUtil.readWithShortLength(in); ByteBuffer val = ByteBufferUtil.readWithShortLength(in); - return new MapComparisonExpression(column, key, operator, val, analyzer); + return new MapComparisonExpression(column, key, operator, val, indexAnalyzer, queryAnalyzer); case VECTOR_RADIUS: Operator boundaryOperator = Operator.readFrom(in); ByteBuffer distance = ByteBufferUtil.readWithShortLength(in); @@ -1119,26 +1133,40 @@ public long serializedSize(Expression expression, int version) public abstract static class AnalyzableExpression extends Expression { @Nullable - protected final Index.Analyzer analyzer; + protected final Index.Analyzer indexAnalyzer; + + @Nullable + protected final Index.Analyzer queryAnalyzer; - public AnalyzableExpression(ColumnMetadata column, Operator operator, ByteBuffer value, @Nullable Index.Analyzer analyzer) + public AnalyzableExpression(ColumnMetadata column, + Operator operator, + ByteBuffer value, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { super(column, operator, value); - this.analyzer = analyzer; + this.indexAnalyzer = indexAnalyzer; + this.queryAnalyzer = queryAnalyzer; + } + + @Nullable + public final Index.Analyzer indexAnalyzer() + { + return indexAnalyzer; } @Nullable - public final Index.Analyzer analyzer() + public final Index.Analyzer queryAnalyzer() { - return analyzer; + return queryAnalyzer; } @Override public int numFilteredValues() { - return analyzer == null + return indexAnalyzer == null ? super.numFilteredValues() - : analyzer().analyze(value).size(); + : indexAnalyzer().analyze(value).size(); } } @@ -1153,10 +1181,11 @@ public static class SimpleExpression extends AnalyzableExpression public SimpleExpression(ColumnMetadata column, Operator operator, ByteBuffer value, - @Nullable Index.Analyzer analyzer, + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer, @Nullable ANNOptions annOptions) { - super(column, operator, value, analyzer); + super(column, operator, value, indexAnalyzer, queryAnalyzer); this.annOptions = annOptions; } @@ -1194,13 +1223,13 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, return false; ByteBuffer counterValue = LongType.instance.decompose(CounterContext.instance().total(foundValue, ByteBufferAccessor.instance)); - return operator.isSatisfiedBy(LongType.instance, counterValue, value, analyzer); + return operator.isSatisfiedBy(LongType.instance, counterValue, value, indexAnalyzer, queryAnalyzer); } else { // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left. ByteBuffer foundValue = getValue(metadata, partitionKey, row); - return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value, analyzer); + return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value, indexAnalyzer, queryAnalyzer); } } case NEQ: @@ -1214,7 +1243,7 @@ public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, assert !column.isComplex() : "Only CONTAINS and CONTAINS_KEY are supported for 'complex' types"; ByteBuffer foundValue = getValue(metadata, partitionKey, row); // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left. - return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value, analyzer); + return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value, indexAnalyzer, queryAnalyzer); } case CONTAINS: return contains(metadata, partitionKey, row); @@ -1345,9 +1374,10 @@ public MapComparisonExpression(ColumnMetadata column, ByteBuffer key, Operator operator, ByteBuffer value, - @Nullable Index.Analyzer analyzer) + @Nullable Index.Analyzer indexAnalyzer, + @Nullable Index.Analyzer queryAnalyzer) { - super(column, operator, value, analyzer); + super(column, operator, value, indexAnalyzer, queryAnalyzer); assert column.type instanceof MapType && (operator == Operator.EQ || operator == Operator.NEQ || operator.isSlice()); this.key = key; } diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java index 168e60c985f2..beb8280e674a 100644 --- a/src/java/org/apache/cassandra/index/Index.java +++ b/src/java/org/apache/cassandra/index/Index.java @@ -431,12 +431,23 @@ default RowFilter.CustomExpression customExpressionFor(TableMetadata metadata, B } /** - * Returns the {@link Analyzer} for this index, if any. If the index doesn't transform the column values, this - * method will return an empty optional. + * Returns the write-time {@link Analyzer} for this index, if any. If the index doesn't transform the column values, + * this method will return an empty optional. * - * @return the transforming column value analyzer for the index, if any + * @return the write-time transforming column value analyzer for the index, if any */ - default Optional getAnalyzer() + default Optional getIndexAnalyzer() + { + return Optional.empty(); + } + + /** + * Returns the query-time {@link Analyzer} for this index, if any. If the index doesn't transform the column values, + * this method will return an empty optional. + * + * @return the query-time transforming column value analyzer for the index, if any + */ + default Optional getQueryAnalyzer() { return Optional.empty(); } diff --git a/src/java/org/apache/cassandra/index/IndexRegistry.java b/src/java/org/apache/cassandra/index/IndexRegistry.java index cc6b0d103ea9..639d9aec350b 100644 --- a/src/java/org/apache/cassandra/index/IndexRegistry.java +++ b/src/java/org/apache/cassandra/index/IndexRegistry.java @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -314,13 +315,25 @@ default void registerIndex(Index index) Index getIndex(IndexMetadata indexMetadata); Collection listIndexes(); - default Optional getAnalyzerFor(ColumnMetadata column, Operator operator) + default Optional getIndexAnalyzerFor(ColumnMetadata column, Operator operator) + { + return getAnalyzerFor(column, operator, Index::getIndexAnalyzer); + } + + default Optional getQueryAnalyzerFor(ColumnMetadata column, Operator operator) + { + return getAnalyzerFor(column, operator, Index::getQueryAnalyzer); + } + + default Optional getAnalyzerFor(ColumnMetadata column, + Operator operator, + Function> analyzerGetter) { for (Index index : listIndexes()) { if (index.supportsExpression(column, operator)) { - Optional analyzer = index.getAnalyzer(); + Optional analyzer = analyzerGetter.apply(index); if (analyzer.isPresent()) return analyzer; } diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java index dccf0e1bd4b7..28cb42ab8dc8 100644 --- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java +++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java @@ -660,26 +660,36 @@ public AbstractType customExpressionValueType() } @Override - public Optional getAnalyzer() + public Optional getIndexAnalyzer() { - if (!indexContext.isAnalyzed()) - return Optional.empty(); + return indexContext.isAnalyzed() + ? Optional.of(value -> analyze(indexContext.getAnalyzerFactory(), value)) + : Optional.empty(); + } - return Optional.of(value -> { - List tokens = new ArrayList<>(); - AbstractAnalyzer analyzer = indexContext.getQueryAnalyzerFactory().create(); - try - { - analyzer.reset(value); - while (analyzer.hasNext()) - tokens.add(analyzer.next()); - } - finally - { - analyzer.end(); - } - return tokens; - }); + @Override + public Optional getQueryAnalyzer() + { + return indexContext.isAnalyzed() + ? Optional.of(value -> analyze(indexContext.getQueryAnalyzerFactory(), value)) + : Optional.empty(); + } + + private static List analyze(AbstractAnalyzer.AnalyzerFactory factory, ByteBuffer value) + { + List tokens = new ArrayList<>(); + AbstractAnalyzer analyzer = factory.create(); + try + { + analyzer.reset(value.duplicate()); + while (analyzer.hasNext()) + tokens.add(analyzer.next()); + } + finally + { + analyzer.end(); + } + return tokens; } @Override diff --git a/src/java/org/apache/cassandra/index/sai/plan/Operation.java b/src/java/org/apache/cassandra/index/sai/plan/Operation.java index 1277b5d6a83f..7722ada87463 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/Operation.java +++ b/src/java/org/apache/cassandra/index/sai/plan/Operation.java @@ -311,7 +311,8 @@ static Node buildExpression(QueryController controller, RowFilter.Expression exp ByteBufferAccessor.instance, offset, ProtocolVersion.V3), - expression.analyzer(), + expression.indexAnalyzer(), + expression.queryAnalyzer(), expression.annOptions()))); offset += TypeSizes.INT_SIZE + ByteBufferAccessor.instance.getInt(expression.getIndexValue(), offset); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java index df7895b77e43..57d44ef209b8 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java +++ b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java @@ -32,6 +32,7 @@ import org.junit.Assert; import org.junit.BeforeClass; +import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; @@ -180,4 +181,38 @@ void assertCannotStartDueToConfigurationException(Cluster cluster) Assert.assertEquals(ConfigurationException.class.getName(), tr.getClass().getName()); } } + + /** + * Runs the given function before and after a flush of sstables. This is useful for checking that behavior is + * the same whether data is in memtables or sstables. + * + * @param cluster the tested cluster + * @param keyspace the keyspace to flush + * @param runnable the test to run + */ + public static void beforeAndAfterFlush(Cluster cluster, String keyspace, CQLTester.CheckedFunction runnable) throws Throwable + { + try + { + runnable.apply(); + } + catch (Throwable t) + { + throw new AssertionError("Test failed before flush:\n" + t, t); + } + + for (int i = 1; i <= cluster.size(); i++) + { + cluster.get(i).flush(keyspace); + + try + { + runnable.apply(); + } + catch (Throwable t) + { + throw new AssertionError("Test failed after flushing node " + i + ":\n" + t, t); + } + } + } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/AnalyzerDistributedTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/AnalyzerDistributedTest.java new file mode 100644 index 000000000000..3b2643ab93bf --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/sai/AnalyzerDistributedTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.sai; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.index.sai.SAITester; + +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; + +public class AnalyzerDistributedTest extends TestBaseImpl +{ + @Rule + public SAITester.FailureWatcher failureRule = new SAITester.FailureWatcher(); + + private static final String CREATE_KEYSPACE = "CREATE KEYSPACE IF NOT EXISTS %%s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}"; + private static final int NUM_REPLICAS = 3; + private static final int RF = 2; + + private static final AtomicInteger seq = new AtomicInteger(); + private static String table; + + private static Cluster cluster; + + @BeforeClass + public static void setupCluster() throws Exception + { + cluster = Cluster.build(NUM_REPLICAS) + .withConfig(config -> config.with(GOSSIP).with(NETWORK)) + .start(); + + cluster.schemaChange(withKeyspace(String.format(CREATE_KEYSPACE, RF))); + } + + @AfterClass + public static void closeCluster() + { + if (cluster != null) + cluster.close(); + } + + @Before + public void before() + { + table = "table_" + seq.getAndIncrement(); + } + + @After + public void after() + { + cluster.schemaChange(formatQuery("DROP TABLE IF EXISTS %s")); + } + + @Test + public void testAnalyzerSearch() + { + cluster.schemaChange(formatQuery("CREATE TABLE %s (pk int PRIMARY KEY, not_analyzed int, val text)")); + cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer': 'standard'}")); + SAIUtil.waitForIndexQueryable(cluster, KEYSPACE); + + var iterations = 15000; + for (int i = 0; i < iterations; i++) + { + var x = i % 100; + if (i % 100 == 0) + { + execute(String.format( + "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')", + KEYSPACE + '.' + table, i, x, "this will be tokenized")); + } + else if (i % 2 == 0) + { + execute(String.format( + "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')", + KEYSPACE + '.' + table, i, x, "this is different")); + } + else + { + execute(String.format( + "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')", + KEYSPACE + '.' + table, i, x, "basic test")); + } + } + // We match the first inserted statement here, and that one is just written 1/100 times + var result = execute("SELECT * FROM %s WHERE val : 'tokenized'"); + assertThat(result).hasNumberOfRows(iterations / 100); + // We match the first and second inserted statements here, and those account for 1/2 the inserts + result = execute("SELECT * FROM %s WHERE val : 'this'"); + assertThat(result).hasNumberOfRows(iterations / 2); + // We match the last write here, and that accounts for the other 1/2 of the inserts + result = execute("SELECT * FROM %s WHERE val : 'test'"); + assertThat(result).hasNumberOfRows(iterations / 2); + } + + /** + * See CNDB-12739 for more details. + */ + @Test + public void testIndexAndQueryAnalyzerSearch() + { + cluster.schemaChange(formatQuery("CREATE TABLE %s (c1 int PRIMARY KEY , c2 text)")); + cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(c2) USING 'StorageAttachedIndex' WITH OPTIONS = {" + + "'index_analyzer': '{" + + " \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," + + " \"filters\" : [ { \"name\" : \"lowercase\", \"args\": {} }, " + + " { \"name\" : \"edgengram\", \"args\": { \"minGramSize\":\"1\", \"maxGramSize\":\"30\" } }]," + + " \"charFilters\" : []}', " + + "'query_analyzer': '{" + + " \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," + + " \"filters\" : [ {\"name\" : \"lowercase\",\"args\": {}} ]}'}")); + SAIUtil.waitForIndexQueryable(cluster, KEYSPACE); + + execute("INSERT INTO %s(c1,c2) VALUES (1, 'astra quick fox')"); + execute("INSERT INTO %s(c1,c2) VALUES (2, 'astra quick foxes')"); + execute("INSERT INTO %s(c1,c2) VALUES (3, 'astra1')"); + execute("INSERT INTO %s(c1,c2) VALUES (4, 'astra4 -1@a#')"); + + Object[][] result = execute("SELECT * FROM %s WHERE c2 :'ast' "); + assertThat(result).hasNumberOfRows(4); + } + + @Test + public void testEdgeNgramFilterWithOR() throws Throwable + { + cluster.schemaChange(formatQuery("CREATE TABLE %s (id text PRIMARY KEY, val text)")); + cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" + + "'index_analyzer': '{\n" + + "\t\"tokenizer\":{\"name\":\"standard\", \"args\":{}}," + + "\t\"filters\":[{\"name\":\"lowercase\", \"args\":{}}, " + + "{\"name\":\"edgengram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" + + "\t\"charFilters\":[]" + + "}'};")); + SAIUtil.waitForIndexQueryable(cluster, KEYSPACE); + + execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')"); + execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')"); + execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other word')"); + execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')"); + execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')"); + + beforeAndAfterFlush(cluster, KEYSPACE, () -> { + + // match (:) + assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL0133AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val : 'WFS2684AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val : ''").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val : 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val : '' OR val : 'WFS2684AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val : '' AND val : 'WFS2684AU'").length); + + // equals (=) + assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL0133AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val = 'WFS2684AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val = ''").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val = 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val = '' OR val = 'WFS2684AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val = '' AND val = 'WFS2684AU'").length); + + // mixed match (:) and equals (=) + assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val : 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val = '' OR val : 'WFS2684AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val = '' AND val : 'WFS2684AU'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val = 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val : '' OR val = 'WFS2684AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val : '' AND val = 'WFS2684AU'").length); + }); + } + + @Test + public void testNgramFilterWithOR() throws Throwable + { + cluster.schemaChange(formatQuery("CREATE TABLE %s (id text PRIMARY KEY, val text)")); + cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {" + + "'index_analyzer': '{\n" + + "\t\"tokenizer\":{\"name\":\"standard\", \"args\":{}}," + + "\t\"filters\":[{\"name\":\"lowercase\", \"args\":{}}, " + + "{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" + + "\t\"charFilters\":[]" + + "}'};")); + SAIUtil.waitForIndexQueryable(cluster, KEYSPACE); + + execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')"); + execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')"); + execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other words')"); + execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')"); + execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')"); + + beforeAndAfterFlush(cluster, KEYSPACE, () -> { + + // match (:) + assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL0133AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val : 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val : '268'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val : 'WFS2684AU'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val : '133' OR val : 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL' AND val : 'AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val : 'XYZ' AND val : 'AU'").length); + + // equals (=) + assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL0133AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val = 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val = '268'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val = 'WFS2684AU'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val = '133' OR val = 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL' AND val = 'AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val = 'XYZ' AND val = 'AU'").length); + + // mixed match (:) and equals (=) + assertEquals(2, execute("SELECT val FROM %s WHERE val : 'MAL0133AU' OR val = 'WFS2684AU'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val : '133' OR val = 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val : 'MAL' AND val = 'AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val : 'XYZ' AND val = 'AU'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val = 'MAL0133AU' OR val : 'WFS2684AU'").length); + assertEquals(2, execute("SELECT val FROM %s WHERE val = '133' OR val : 'WFS2684AU'").length); + assertEquals(1, execute("SELECT val FROM %s WHERE val = 'MAL' AND val : 'AU'").length); + assertEquals(0, execute("SELECT val FROM %s WHERE val = 'XYZ' AND val : 'AU'").length); + }); + } + + private static Object[][] execute(String query) + { + return cluster.coordinator(1).execute(formatQuery(query), ConsistencyLevel.QUORUM); + } + + private static String formatQuery(String query) + { + return String.format(query, KEYSPACE + '.' + table); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/AnalzyerDistributedTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/AnalzyerDistributedTest.java deleted file mode 100644 index 2375f5dbff7a..000000000000 --- a/test/distributed/org/apache/cassandra/distributed/test/sai/AnalzyerDistributedTest.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.distributed.test.sai; - -import java.util.concurrent.atomic.AtomicInteger; - -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; - -import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.distributed.api.ConsistencyLevel; -import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.index.sai.SAITester; - -import static org.apache.cassandra.cql3.CQLTester.getRandom; -import static org.apache.cassandra.distributed.api.Feature.GOSSIP; -import static org.apache.cassandra.distributed.api.Feature.NETWORK; -import static org.assertj.core.api.Assertions.assertThat; - -public class AnalzyerDistributedTest extends TestBaseImpl -{ - @Rule - public SAITester.FailureWatcher failureRule = new SAITester.FailureWatcher(); - - private static final String CREATE_KEYSPACE = "CREATE KEYSPACE %%s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}"; - private static final String CREATE_TABLE = "CREATE TABLE %%s (pk int PRIMARY KEY, not_analyzed int, val text)"; - private static final String CREATE_INDEX = "CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer': 'standard'}"; - private static final int NUM_REPLICAS = 3; - private static final int RF = 2; - - private static final AtomicInteger seq = new AtomicInteger(); - private static String table; - - private static Cluster cluster; - - private static int dimensionCount; - - @BeforeClass - public static void setupCluster() throws Exception - { - cluster = Cluster.build(NUM_REPLICAS) - .withConfig(config -> config.with(GOSSIP).with(NETWORK)) - .start(); - - cluster.schemaChange(withKeyspace(String.format(CREATE_KEYSPACE, RF))); - } - - @AfterClass - public static void closeCluster() - { - if (cluster != null) - cluster.close(); - } - - @Before - public void before() - { - table = "table_" + seq.getAndIncrement(); - dimensionCount = getRandom().nextIntBetween(100, 2048); - } - - @After - public void after() - { - cluster.schemaChange(formatQuery("DROP TABLE IF EXISTS %s")); - } - - @Test - public void testAnalyzerSearch() - { - cluster.schemaChange(formatQuery(String.format(CREATE_TABLE, dimensionCount))); - cluster.schemaChange(formatQuery(String.format(CREATE_INDEX, "val"))); - SAIUtil.waitForIndexQueryable(cluster, KEYSPACE); - - var iterations = 15000; - for (int i = 0; i < iterations; i++) - { - var x = i % 100; - if (i % 100 == 0) - { - execute(String.format( - "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')", - KEYSPACE + '.' + table, i, x, "this will be tokenized")); - } - else if (i % 2 == 0) - { - execute(String.format( - "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')", - KEYSPACE + '.' + table, i, x, "this is different")); - } - else - { - execute(String.format( - "INSERT INTO %s (pk, not_analyzed, val) VALUES (%s, %s, '%s')", - KEYSPACE + '.' + table, i, x, "basic test")); - } - } - // We match the first inserted statement here, and that one is just written 1/100 times - var result = execute("SELECT * FROM %s WHERE val : 'tokenized'"); - assertThat(result).hasNumberOfRows(iterations / 100); - // We match the first and second inserted statements here, and those account for 1/2 the inserts - result = execute("SELECT * FROM %s WHERE val : 'this'"); - assertThat(result).hasNumberOfRows(iterations / 2); - // We match the last write here, and that accounts for the other 1/2 of the inserts - result = execute("SELECT * FROM %s WHERE val : 'test'"); - assertThat(result).hasNumberOfRows(iterations / 2); - } - - private static Object[][] execute(String query) - { - return execute(query, ConsistencyLevel.QUORUM); - } - - private static Object[][] execute(String query, ConsistencyLevel consistencyLevel) - { - return cluster.coordinator(1).execute(formatQuery(query), consistencyLevel); - } - - private static String formatQuery(String query) - { - return String.format(query, KEYSPACE + '.' + table); - } -} diff --git a/test/unit/org/apache/cassandra/cql3/OperatorTest.java b/test/unit/org/apache/cassandra/cql3/OperatorTest.java index 779203bf80dd..182c3d9b6880 100644 --- a/test/unit/org/apache/cassandra/cql3/OperatorTest.java +++ b/test/unit/org/apache/cassandra/cql3/OperatorTest.java @@ -71,14 +71,14 @@ private static void testAnalyzer(AbstractType type, { // test that EQ and ANALYZER_MATCHES are satisfied by the same value with an analyzer for (Operator operator : Arrays.asList(Operator.EQ, Operator.ANALYZER_MATCHES)) - Assertions.assertThat(operator.isSatisfiedBy(type, left, right, analyzer)).isEqualTo(shouldBeSatisfied); + Assertions.assertThat(operator.isSatisfiedBy(type, left, right, analyzer, analyzer)).isEqualTo(shouldBeSatisfied); // test that EQ without an analyzer behaves as type-based identity - Assertions.assertThat(Operator.EQ.isSatisfiedBy(type, left, right, null)) + Assertions.assertThat(Operator.EQ.isSatisfiedBy(type, left, right, null, null)) .isEqualTo(type.compareForCQL(left, right) == 0); // test that ANALYZER_MATCHES throws an exception when no analyzer is provided - Assertions.assertThatThrownBy(() -> Operator.ANALYZER_MATCHES.isSatisfiedBy(type, left, right, null)) + Assertions.assertThatThrownBy(() -> Operator.ANALYZER_MATCHES.isSatisfiedBy(type, left, right, null, null)) .isInstanceOf(AssertionError.class) .hasMessageContaining(": operation can only be computed by an indexed column with a configured analyzer"); @@ -91,19 +91,19 @@ private static void testAnalyzer(AbstractType type, boolean supported = false; try { - shouldBeSatisfied = operator.isSatisfiedBy(type, left, right, null); + shouldBeSatisfied = operator.isSatisfiedBy(type, left, right, null, null); supported = true; } catch (Exception e) { - Assertions.assertThatThrownBy(() -> operator.isSatisfiedBy(type, left, right, analyzer)) + Assertions.assertThatThrownBy(() -> operator.isSatisfiedBy(type, left, right, analyzer, analyzer)) .isInstanceOf(e.getClass()) .hasMessage(e.getMessage()); } if (supported) { - Assertions.assertThat(operator.isSatisfiedBy(type, left, right, analyzer)) + Assertions.assertThat(operator.isSatisfiedBy(type, left, right, analyzer, analyzer)) .isEqualTo(shouldBeSatisfied); } } diff --git a/test/unit/org/apache/cassandra/index/sai/cql/LuceneAnalyzerTest.java b/test/unit/org/apache/cassandra/index/sai/cql/LuceneAnalyzerTest.java index 520ff6e17d9b..8ee2e2e511d9 100644 --- a/test/unit/org/apache/cassandra/index/sai/cql/LuceneAnalyzerTest.java +++ b/test/unit/org/apache/cassandra/index/sai/cql/LuceneAnalyzerTest.java @@ -469,7 +469,7 @@ public void testEdgeNgramFilterWithOR() throws Throwable execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')"); execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')"); - execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 Mercer High Growth Managed')"); + execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other words')"); execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')"); execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')"); @@ -515,7 +515,7 @@ public void testNgramFilterWithOR() throws Throwable execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')"); execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')"); - execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 Mercer High Growth Managed')"); + execute("INSERT INTO %s (id, val) VALUES ('3', 'FPWMCR005 some other words')"); execute("INSERT INTO %s (id, val) VALUES ('4', 'WFS7093AU')"); execute("INSERT INTO %s (id, val) VALUES ('5', 'WFS0565AU')");