From 896bc19fa0684158a2948a7483e17bad788df380 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Thu, 20 Feb 2025 13:23:55 +0800 Subject: [PATCH] add geometry and geography types to iceberg-api and iceberg-core --- .../java/org/apache/iceberg/Geography.java | 127 ++++++++ .../main/java/org/apache/iceberg/Schema.java | 4 +- .../expressions/BoundLiteralPredicate.java | 64 ++++ .../apache/iceberg/expressions/Evaluator.java | 48 +++ .../iceberg/expressions/Expression.java | 12 + .../iceberg/expressions/ExpressionUtil.java | 16 + .../expressions/ExpressionVisitors.java | 76 +++++ .../iceberg/expressions/Expressions.java | 70 +++++ .../InclusiveMetricsEvaluator.java | 152 ++++++++++ .../apache/iceberg/expressions/Literal.java | 10 + .../apache/iceberg/expressions/Literals.java | 65 +++- .../expressions/StrictMetricsEvaluator.java | 20 ++ .../iceberg/expressions/UnboundPredicate.java | 8 + .../org/apache/iceberg/types/Conversions.java | 64 ++++ .../java/org/apache/iceberg/types/Type.java | 4 + .../java/org/apache/iceberg/types/Types.java | 139 +++++++++ .../org/apache/iceberg/util/GeometryUtil.java | 221 ++++++++++++++ .../iceberg/expressions/TestEvaluator.java | 191 ++++++++++++ .../expressions/TestExpressionHelpers.java | 31 +- .../TestExpressionSerialization.java | 22 +- .../expressions/TestExpressionUtil.java | 88 +++++- .../TestInclusiveMetricsEvaluator.java | 277 +++++++++++++++++- .../expressions/TestLiteralSerialization.java | 12 + .../TestMiscLiteralConversions.java | 13 +- .../apache/iceberg/types/TestConversions.java | 105 +++++++ .../iceberg/types/TestReadabilityChecks.java | 8 +- .../iceberg/types/TestSerializableTypes.java | 8 +- .../org/apache/iceberg/types/TestTypes.java | 46 +++ .../org/apache/iceberg/util/RandomUtil.java | 13 + .../apache/iceberg/util/TestGeometryUtil.java | 266 +++++++++++++++++ build.gradle | 1 + .../java/org/apache/iceberg/SchemaParser.java | 34 ++- .../org/apache/iceberg/SingleValueParser.java | 26 ++ .../org/apache/iceberg/avro/TypeToSchema.java | 2 + .../iceberg/expressions/ExpressionParser.java | 11 + .../apache/iceberg/TestGeospatialTable.java | 68 +++++ .../org/apache/iceberg/TestSchemaParser.java | 24 +- .../org/apache/iceberg/TestSchemaUpdate.java | 6 +- .../apache/iceberg/TestSingleValueParser.java | 19 ++ .../org/apache/iceberg/TestTableMetadata.java | 37 +++ .../expressions/TestExpressionParser.java | 45 ++- gradle/libs.versions.toml | 2 + 42 files changed, 2437 insertions(+), 18 deletions(-) create mode 100644 api/src/main/java/org/apache/iceberg/Geography.java create mode 100644 api/src/main/java/org/apache/iceberg/util/GeometryUtil.java create mode 100644 api/src/test/java/org/apache/iceberg/util/TestGeometryUtil.java create mode 100644 core/src/test/java/org/apache/iceberg/TestGeospatialTable.java diff --git a/api/src/main/java/org/apache/iceberg/Geography.java b/api/src/main/java/org/apache/iceberg/Geography.java new file mode 100644 index 000000000000..f60183b0e76c --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/Geography.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; +import java.util.Locale; +import java.util.Objects; +import org.locationtech.jts.geom.Geometry; + +/** + * Geospatial features from OGC – Simple feature access. The geometry is on a spherical or + * ellipsoidal surface. An edge-interpolation algorithm is used to evaluate spatial predicates. + */ +public class Geography implements Comparable, Serializable { + + /** The algorithm for interpolating edges. */ + public enum EdgeInterpolationAlgorithm { + /** Edges are interpolated as geodesics on a sphere. */ + SPHERICAL("spherical"), + /** See Vincenty's formulae */ + VINCENTY("vincenty"), + /** + * Thomas, Paul D. Spheroidal geodesics, reference systems, & local geometry. US Naval + * Oceanographic Office, 1970. + */ + THOMAS("thomas"), + /** + * Thomas, Paul D. Mathematical models for navigation systems. US Naval Oceanographic Office, + * 1965. + */ + ANDOYER("andoyer"), + /** + * Karney, Charles + * FF. "Algorithms for geodesics." Journal of Geodesy 87 (2013): 43-55 , and GeographicLib. + */ + KARNEY("karney"); + + private final String value; + + EdgeInterpolationAlgorithm(String value) { + this.value = value; + } + + public String value() { + return value; + } + + public static EdgeInterpolationAlgorithm fromName(String algorithmName) { + try { + return EdgeInterpolationAlgorithm.valueOf(algorithmName.toUpperCase(Locale.ENGLISH)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format("Invalid edge interpolation algorithm name: %s", algorithmName), e); + } + } + } + + private final Geometry geometry; + + public Geography(Geometry geometry) { + this.geometry = geometry; + } + + public Geometry geometry() { + return geometry; + } + + @Override + public String toString() { + return "Geography(" + geometry + ")"; + } + + @Override + public int compareTo(Geography o) { + return geometry.compareTo(o.geometry); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Geography)) { + return false; + } + Geography geography = (Geography) o; + return Objects.equals(geometry, geography.geometry); + } + + @Override + public int hashCode() { + return Objects.hashCode(geometry); + } + + public boolean intersects(Geography other, EdgeInterpolationAlgorithm algorithm) { + if (algorithm != EdgeInterpolationAlgorithm.SPHERICAL) { + throw new UnsupportedOperationException( + "Interpolation algorithm other than spherical is not supported yet"); + } + + // TODO: implement a correct spherical intersection algorithm using S2 + return geometry.intersects(other.geometry); + } + + public boolean covers(Geography other, EdgeInterpolationAlgorithm algorithm) { + if (algorithm != EdgeInterpolationAlgorithm.SPHERICAL) { + throw new UnsupportedOperationException( + "Interpolation algorithm other than spherical is not supported yet"); + } + // TODO: implement a correct spherical covers algorithm using S2 + return geometry.covers(other.geometry); + } +} diff --git a/api/src/main/java/org/apache/iceberg/Schema.java b/api/src/main/java/org/apache/iceberg/Schema.java index 07ed44b65cf7..e497b8e69afc 100644 --- a/api/src/main/java/org/apache/iceberg/Schema.java +++ b/api/src/main/java/org/apache/iceberg/Schema.java @@ -63,7 +63,9 @@ public class Schema implements Serializable { ImmutableMap.of( Type.TypeID.TIMESTAMP_NANO, 3, Type.TypeID.VARIANT, 3, - Type.TypeID.UNKNOWN, 3); + Type.TypeID.UNKNOWN, 3, + Type.TypeID.GEOMETRY, 3, + Type.TypeID.GEOGRAPHY, 3); private final StructType struct; private final int schemaId; diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java index 127d46e6a48f..8f7ddb3529e8 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java +++ b/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java @@ -20,9 +20,12 @@ import java.util.Comparator; import java.util.Set; +import org.apache.iceberg.Geography; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.locationtech.jts.geom.Geometry; public class BoundLiteralPredicate extends BoundPredicate { private static final Set INTEGRAL_TYPES = @@ -88,11 +91,64 @@ public boolean test(T value) { return String.valueOf(value).startsWith((String) literal.value()); case NOT_STARTS_WITH: return !String.valueOf(value).startsWith((String) literal.value()); + case ST_INTERSECTS: + case ST_COVERS: + case ST_DISJOINT: + case ST_NOT_COVERS: + return testSpatial(value); default: throw new IllegalStateException("Invalid operation for BoundLiteralPredicate: " + op()); } } + private boolean testSpatial(T value) { + Type type = term().type(); + Type.TypeID typeId = type.typeId(); + if (typeId == Type.TypeID.GEOMETRY) { + return testGeometry((Geometry) value); + } else if (typeId == Type.TypeID.GEOGRAPHY) { + Types.GeographyType geographyType = (Types.GeographyType) type; + Geography.EdgeInterpolationAlgorithm algorithm = geographyType.algorithm(); + return testGeography((Geography) value, algorithm); + } else { + throw new IllegalStateException("Invalid term type for spatial predicate: " + type); + } + } + + private boolean testGeometry(Geometry value) { + Geometry literalGeometry = (Geometry) literal.value(); + switch (op()) { + case ST_INTERSECTS: + return value.intersects(literalGeometry); + case ST_COVERS: + return value.covers(literalGeometry); + case ST_DISJOINT: + return value.disjoint(literalGeometry); + case ST_NOT_COVERS: + return !value.covers(literalGeometry); + default: + throw new IllegalStateException( + "Invalid spatial operation for BoundLiteralPredicate: " + op()); + } + } + + private boolean testGeography(Geography value, Geography.EdgeInterpolationAlgorithm algorithm) { + Geography literalGeography = (Geography) literal.value(); + switch (op()) { + case ST_INTERSECTS: + return value.intersects(literalGeography, algorithm); + case ST_COVERS: + return value.covers(literalGeography, algorithm); + case ST_DISJOINT: + return !value.intersects(literalGeography, algorithm); + case ST_NOT_COVERS: + return !value.covers(literalGeography, algorithm); + default: + throw new IllegalStateException( + "Invalid spatial operation for BoundLiteralPredicate: " + op()); + } + } + @Override @SuppressWarnings("unchecked") public boolean isEquivalentTo(Expression expr) { @@ -159,6 +215,14 @@ public String toString() { return term() + " startsWith \"" + literal + "\""; case NOT_STARTS_WITH: return term() + " notStartsWith \"" + literal + "\""; + case ST_INTERSECTS: + return term() + " intersects " + literal; + case ST_COVERS: + return term() + " covers " + literal; + case ST_DISJOINT: + return term() + " disjoint " + literal; + case ST_NOT_COVERS: + return term() + " notCovers " + literal; case IN: return term() + " in { " + literal + " }"; case NOT_IN: diff --git a/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java b/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java index 96e148a2d438..856c80463169 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java @@ -21,10 +21,14 @@ import java.io.Serializable; import java.util.Comparator; import java.util.Set; +import org.apache.iceberg.Geography; import org.apache.iceberg.StructLike; import org.apache.iceberg.expressions.ExpressionVisitors.BoundVisitor; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.util.NaNUtil; +import org.locationtech.jts.geom.Geometry; /** * Evaluates an {@link Expression} for data described by a {@link StructType}. @@ -156,5 +160,49 @@ public Boolean startsWith(Bound valueExpr, Literal lit) { public Boolean notStartsWith(Bound valueExpr, Literal lit) { return !startsWith(valueExpr, lit); } + + @Override + public Boolean stIntersects(Bound valueExpr, Literal lit) { + T evalRes = valueExpr.eval(struct); + if (evalRes == null) { + return false; + } + Type type = valueExpr.ref().type(); + if (type.typeId() == Type.TypeID.GEOMETRY) { + return ((Geometry) evalRes).intersects((Geometry) lit.value()); + } else if (type.typeId() == Type.TypeID.GEOGRAPHY) { + Types.GeographyType geographyType = (Types.GeographyType) type; + return ((Geography) evalRes).intersects((Geography) lit.value(), geographyType.algorithm()); + } else { + throw new IllegalArgumentException("Invalid type for spatial predicate: " + type); + } + } + + @Override + public Boolean stCovers(Bound valueExpr, Literal lit) { + T evalRes = valueExpr.eval(struct); + if (evalRes == null) { + return false; + } + Type type = valueExpr.ref().type(); + if (type.typeId() == Type.TypeID.GEOMETRY) { + return ((Geometry) evalRes).covers((Geometry) lit.value()); + } else if (type.typeId() == Type.TypeID.GEOGRAPHY) { + Types.GeographyType geographyType = (Types.GeographyType) type; + return ((Geography) evalRes).covers((Geography) lit.value(), geographyType.algorithm()); + } else { + throw new IllegalArgumentException("Invalid type for spatial predicate: " + type); + } + } + + @Override + public Boolean stDisjoint(Bound valueExpr, Literal lit) { + return !stIntersects(valueExpr, lit); + } + + @Override + public Boolean stNotCovers(Bound valueExpr, Literal lit) { + return !stCovers(valueExpr, lit); + } } } diff --git a/api/src/main/java/org/apache/iceberg/expressions/Expression.java b/api/src/main/java/org/apache/iceberg/expressions/Expression.java index dc88172c590d..ccb605be38e8 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Expression.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Expression.java @@ -44,6 +44,10 @@ enum Operation { OR, STARTS_WITH, NOT_STARTS_WITH, + ST_INTERSECTS, + ST_COVERS, + ST_DISJOINT, + ST_NOT_COVERS, COUNT, COUNT_STAR, MAX, @@ -90,6 +94,14 @@ public Operation negate() { return Operation.NOT_STARTS_WITH; case NOT_STARTS_WITH: return Operation.STARTS_WITH; + case ST_INTERSECTS: + return Operation.ST_DISJOINT; + case ST_COVERS: + return Operation.ST_NOT_COVERS; + case ST_DISJOINT: + return Operation.ST_INTERSECTS; + case ST_NOT_COVERS: + return Operation.ST_COVERS; default: throw new IllegalArgumentException("No negation for operation: " + this); } diff --git a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java index 02f3880dd96a..ef298ae693ea 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java +++ b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java @@ -333,6 +333,10 @@ public Expression predicate(UnboundPredicate pred) { case NOT_EQ: case STARTS_WITH: case NOT_STARTS_WITH: + case ST_INTERSECTS: + case ST_COVERS: + case ST_DISJOINT: + case ST_NOT_COVERS: return new UnboundPredicate<>( pred.op(), pred.term(), (T) sanitize(pred.literal(), now, today)); case IN: @@ -485,6 +489,14 @@ public String predicate(UnboundPredicate pred) { return term + " STARTS WITH " + sanitize(pred.literal(), nowMicros, today); case NOT_STARTS_WITH: return term + " NOT STARTS WITH " + sanitize(pred.literal(), nowMicros, today); + case ST_INTERSECTS: + return term + " ST_INTERSECTS " + sanitize(pred.literal(), nowMicros, today); + case ST_COVERS: + return term + " ST_COVERS " + sanitize(pred.literal(), nowMicros, today); + case ST_DISJOINT: + return term + " ST_DISJOINT " + sanitize(pred.literal(), nowMicros, today); + case ST_NOT_COVERS: + return term + " ST_NOT_COVERS " + sanitize(pred.literal(), nowMicros, today); default: throw new UnsupportedOperationException( "Cannot sanitize unsupported predicate type: " + pred.op()); @@ -568,6 +580,10 @@ private static String sanitize(Literal literal, long now, int today) { return sanitizeNumber(((Literals.FloatLiteral) literal).value(), "float"); } else if (literal instanceof Literals.DoubleLiteral) { return sanitizeNumber(((Literals.DoubleLiteral) literal).value(), "float"); + } else if (literal instanceof Literals.GeometryLiteral) { + return "(geometry)"; + } else if (literal instanceof Literals.GeographyLiteral) { + return "(geography)"; } else { // for uuid, decimal, fixed, variant, and binary, match the string result return sanitizeSimpleString(literal.value().toString()); diff --git a/api/src/main/java/org/apache/iceberg/expressions/ExpressionVisitors.java b/api/src/main/java/org/apache/iceberg/expressions/ExpressionVisitors.java index 79ca6a712887..d304f05d61af 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/ExpressionVisitors.java +++ b/api/src/main/java/org/apache/iceberg/expressions/ExpressionVisitors.java @@ -126,6 +126,26 @@ public R notStartsWith(BoundReference ref, Literal lit) { "notStartsWith expression is not supported by the visitor"); } + public R stIntersects(BoundReference ref, Literal lit) { + throw new UnsupportedOperationException( + "stIntersects expression is not supported by the visitor"); + } + + public R stCovers(BoundReference ref, Literal lit) { + throw new UnsupportedOperationException( + "stContains expression is not supported by the visitor"); + } + + public R stDisjoint(BoundReference ref, Literal lit) { + throw new UnsupportedOperationException( + "stDisjoint expression is not supported by the visitor"); + } + + public R stNotCovers(BoundReference ref, Literal lit) { + throw new UnsupportedOperationException( + "stNotCovers expression is not supported by the visitor"); + } + /** * Handle a non-reference value in this visitor. * @@ -166,6 +186,14 @@ public R predicate(BoundPredicate pred) { return startsWith((BoundReference) pred.term(), literalPred.literal()); case NOT_STARTS_WITH: return notStartsWith((BoundReference) pred.term(), literalPred.literal()); + case ST_INTERSECTS: + return stIntersects((BoundReference) pred.term(), literalPred.literal()); + case ST_COVERS: + return stCovers((BoundReference) pred.term(), literalPred.literal()); + case ST_DISJOINT: + return stDisjoint((BoundReference) pred.term(), literalPred.literal()); + case ST_NOT_COVERS: + return stNotCovers((BoundReference) pred.term(), literalPred.literal()); default: throw new IllegalStateException( "Invalid operation for BoundLiteralPredicate: " + pred.op()); @@ -266,6 +294,22 @@ public R notStartsWith(Bound expr, Literal lit) { throw new UnsupportedOperationException("Unsupported operation."); } + public R stIntersects(Bound expr, Literal lit) { + throw new UnsupportedOperationException("Unsupported operation."); + } + + public R stCovers(Bound expr, Literal lit) { + throw new UnsupportedOperationException("Unsupported operation."); + } + + public R stDisjoint(Bound expr, Literal lit) { + throw new UnsupportedOperationException("Unsupported operation."); + } + + public R stNotCovers(Bound expr, Literal lit) { + throw new UnsupportedOperationException("Unsupported operation."); + } + @Override public R predicate(BoundPredicate pred) { if (pred.isLiteralPredicate()) { @@ -287,6 +331,14 @@ public R predicate(BoundPredicate pred) { return startsWith(pred.term(), literalPred.literal()); case NOT_STARTS_WITH: return notStartsWith(pred.term(), literalPred.literal()); + case ST_INTERSECTS: + return stIntersects(pred.term(), literalPred.literal()); + case ST_COVERS: + return stCovers(pred.term(), literalPred.literal()); + case ST_DISJOINT: + return stDisjoint(pred.term(), literalPred.literal()); + case ST_NOT_COVERS: + return stNotCovers(pred.term(), literalPred.literal()); default: throw new IllegalStateException( "Invalid operation for BoundLiteralPredicate: " + pred.op()); @@ -465,6 +517,14 @@ public R predicate(BoundPredicate pred) { return startsWith(pred.term(), literalPred.literal()); case NOT_STARTS_WITH: return notStartsWith(pred.term(), literalPred.literal()); + case ST_INTERSECTS: + return stIntersects(pred.term(), literalPred.literal()); + case ST_COVERS: + return stCovers(pred.term(), literalPred.literal()); + case ST_DISJOINT: + return stDisjoint(pred.term(), literalPred.literal()); + case ST_NOT_COVERS: + return stNotCovers(pred.term(), literalPred.literal()); default: throw new IllegalStateException( "Invalid operation for BoundLiteralPredicate: " + pred.op()); @@ -555,6 +615,22 @@ public R startsWith(BoundTerm term, Literal lit) { public R notStartsWith(BoundTerm term, Literal lit) { return null; } + + public R stIntersects(BoundTerm term, Literal lit) { + return null; + } + + public R stCovers(BoundTerm term, Literal lit) { + return null; + } + + public R stDisjoint(BoundTerm term, Literal lit) { + return null; + } + + public R stNotCovers(BoundTerm term, Literal lit) { + return null; + } } /** diff --git a/api/src/main/java/org/apache/iceberg/expressions/Expressions.java b/api/src/main/java/org/apache/iceberg/expressions/Expressions.java index 22626866725b..247aa664bfc0 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Expressions.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Expressions.java @@ -19,12 +19,14 @@ package org.apache.iceberg.expressions; import java.util.stream.Stream; +import org.apache.iceberg.Geography; import org.apache.iceberg.expressions.Expression.Operation; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.util.NaNUtil; +import org.locationtech.jts.geom.Geometry; /** Factory methods for creating {@link Expression expressions}. */ public class Expressions { @@ -202,6 +204,74 @@ public static UnboundPredicate notStartsWith(UnboundTerm expr, S return new UnboundPredicate<>(Expression.Operation.NOT_STARTS_WITH, expr, value); } + public static UnboundPredicate stIntersects(String name, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_INTERSECTS, ref(name), value); + } + + public static UnboundPredicate stIntersects( + UnboundTerm expr, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_INTERSECTS, expr, value); + } + + public static UnboundPredicate stCovers(String name, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_COVERS, ref(name), value); + } + + public static UnboundPredicate stCovers(UnboundTerm expr, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_COVERS, expr, value); + } + + public static UnboundPredicate stDisjoint(String name, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_DISJOINT, ref(name), value); + } + + public static UnboundPredicate stDisjoint(UnboundTerm expr, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_DISJOINT, expr, value); + } + + public static UnboundPredicate stNotCovers(String name, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_NOT_COVERS, ref(name), value); + } + + public static UnboundPredicate stNotCovers(UnboundTerm expr, Geometry value) { + return new UnboundPredicate<>(Expression.Operation.ST_NOT_COVERS, expr, value); + } + + public static UnboundPredicate stIntersects(String name, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_INTERSECTS, ref(name), value); + } + + public static UnboundPredicate stIntersects( + UnboundTerm expr, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_INTERSECTS, expr, value); + } + + public static UnboundPredicate stCovers(String name, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_COVERS, ref(name), value); + } + + public static UnboundPredicate stCovers(UnboundTerm expr, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_COVERS, expr, value); + } + + public static UnboundPredicate stDisjoint(String name, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_DISJOINT, ref(name), value); + } + + public static UnboundPredicate stDisjoint( + UnboundTerm expr, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_DISJOINT, expr, value); + } + + public static UnboundPredicate stNotCovers(String name, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_NOT_COVERS, ref(name), value); + } + + public static UnboundPredicate stNotCovers( + UnboundTerm expr, Geography value) { + return new UnboundPredicate<>(Expression.Operation.ST_NOT_COVERS, expr, value); + } + public static UnboundPredicate in(String name, T... values) { return predicate(Operation.IN, name, Lists.newArrayList(values)); } diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java index 172b6a727ddc..a86d3823abb6 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java +++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java @@ -28,13 +28,16 @@ import java.util.stream.Collectors; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.ExpressionVisitors.BoundExpressionVisitor; import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.util.BinaryUtil; +import org.apache.iceberg.util.GeometryUtil; import org.apache.iceberg.util.NaNUtil; +import org.locationtech.jts.geom.Geometry; /** * Evaluates an {@link Expression} on a {@link DataFile} to test whether rows in the file may match. @@ -470,6 +473,155 @@ public Boolean notStartsWith(BoundReference ref, Literal lit) { return ROWS_MIGHT_MATCH; } + @Override + public Boolean stIntersects(BoundReference ref, Literal lit) { + Integer id = ref.fieldId(); + + if (containsNullsOnly(id)) { + return ROWS_CANNOT_MATCH; + } + + if (lowerBounds != null + && upperBounds != null + && lowerBounds.containsKey(id) + && upperBounds.containsKey(id)) { + T lowerBound = Conversions.fromByteBuffer(ref.type(), lowerBounds.get(id)); + T upperBound = Conversions.fromByteBuffer(ref.type(), upperBounds.get(id)); + if (lowerBound != null && upperBound != null) { + switch (ref.type().typeId()) { + case GEOMETRY: + { + Geometry queryWindow = (Geometry) lit.value(); + boolean intersects = + GeometryUtil.boundMayIntersects( + (Geometry) lowerBound, (Geometry) upperBound, queryWindow); + if (!intersects) { + return ROWS_CANNOT_MATCH; + } + break; + } + case GEOGRAPHY: + { + Geography queryWindow = (Geography) lit.value(); + boolean intersects = + GeometryUtil.boundMayIntersects( + (Geography) lowerBound, (Geography) upperBound, queryWindow); + if (!intersects) { + return ROWS_CANNOT_MATCH; + } + break; + } + default: + throw new UnsupportedOperationException( + "Cannot evaluate stIntersects predicate on non-spatial column: " + ref); + } + } + } + + return ROWS_MIGHT_MATCH; + } + + @Override + public Boolean stCovers(BoundReference ref, Literal lit) { + Integer id = ref.fieldId(); + + if (containsNullsOnly(id)) { + return ROWS_CANNOT_MATCH; + } + + if (lowerBounds != null + && upperBounds != null + && lowerBounds.containsKey(id) + && upperBounds.containsKey(id)) { + T lowerBound = Conversions.fromByteBuffer(ref.type(), lowerBounds.get(id)); + T upperBound = Conversions.fromByteBuffer(ref.type(), upperBounds.get(id)); + if (lowerBound != null && upperBound != null) { + switch (ref.type().typeId()) { + case GEOMETRY: + { + Geometry queryWindow = (Geometry) lit.value(); + boolean covers = + GeometryUtil.boundMayCovers( + (Geometry) lowerBound, (Geometry) upperBound, queryWindow); + if (!covers) { + return ROWS_CANNOT_MATCH; + } + break; + } + case GEOGRAPHY: + { + Geography queryWindow = (Geography) lit.value(); + boolean covers = + GeometryUtil.boundMayCovers( + (Geography) lowerBound, (Geography) upperBound, queryWindow); + if (!covers) { + return ROWS_CANNOT_MATCH; + } + break; + } + default: + throw new UnsupportedOperationException( + "Cannot evaluate stCovers predicate on non-spatial column: " + ref); + } + } + } + + return ROWS_MIGHT_MATCH; + } + + @Override + public Boolean stDisjoint(BoundReference ref, Literal lit) { + Integer id = ref.fieldId(); + + if (containsNullsOnly(id)) { + return ROWS_MIGHT_MATCH; + } + + if (lowerBounds != null + && upperBounds != null + && lowerBounds.containsKey(id) + && upperBounds.containsKey(id)) { + T lowerBound = Conversions.fromByteBuffer(ref.type(), lowerBounds.get(id)); + T upperBound = Conversions.fromByteBuffer(ref.type(), upperBounds.get(id)); + if (lowerBound != null && upperBound != null) { + switch (ref.type().typeId()) { + case GEOMETRY: + { + Geometry queryWindow = (Geometry) lit.value(); + boolean covered = + GeometryUtil.boundMustBeCoveredBy( + (Geometry) lowerBound, (Geometry) upperBound, queryWindow); + if (covered) { + return ROWS_CANNOT_MATCH; + } + break; + } + case GEOGRAPHY: + { + Geography queryWindow = (Geography) lit.value(); + boolean covered = + GeometryUtil.boundMustBeCoveredBy( + (Geography) lowerBound, (Geography) upperBound, queryWindow); + if (covered) { + return ROWS_CANNOT_MATCH; + } + break; + } + default: + throw new UnsupportedOperationException( + "Cannot evaluate stDisjoint predicate on non-spatial column: " + ref); + } + } + } + + return ROWS_MIGHT_MATCH; + } + + @Override + public Boolean stNotCovers(BoundReference ref, Literal lit) { + return ROWS_MIGHT_MATCH; + } + private boolean mayContainNull(Integer id) { return nullCounts == null || (nullCounts.containsKey(id) && nullCounts.get(id) != 0); } diff --git a/api/src/main/java/org/apache/iceberg/expressions/Literal.java b/api/src/main/java/org/apache/iceberg/expressions/Literal.java index b5d6f72f74d0..2de5061c0250 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Literal.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Literal.java @@ -23,7 +23,9 @@ import java.nio.ByteBuffer; import java.util.Comparator; import java.util.UUID; +import org.apache.iceberg.Geography; import org.apache.iceberg.types.Type; +import org.locationtech.jts.geom.Geometry; /** * Represents a literal fixed value in an expression predicate @@ -71,6 +73,14 @@ static Literal of(BigDecimal value) { return new Literals.DecimalLiteral(value); } + static Literal of(Geometry value) { + return new Literals.GeometryLiteral(value); + } + + static Literal of(Geography value) { + return new Literals.GeographyLiteral(value); + } + /** Returns the value wrapped by this literal. */ T value(); diff --git a/api/src/main/java/org/apache/iceberg/expressions/Literals.java b/api/src/main/java/org/apache/iceberg/expressions/Literals.java index ee47035b1e72..b54c4768e10f 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Literals.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Literals.java @@ -32,6 +32,7 @@ import java.util.Comparator; import java.util.Objects; import java.util.UUID; +import org.apache.iceberg.Geography; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.io.BaseEncoding; import org.apache.iceberg.types.Comparators; @@ -41,6 +42,7 @@ import org.apache.iceberg.util.ByteBuffers; import org.apache.iceberg.util.DateTimeUtil; import org.apache.iceberg.util.NaNUtil; +import org.locationtech.jts.geom.Geometry; class Literals { private Literals() {} @@ -55,7 +57,7 @@ private Literals() {} * @param Java type of value * @return a Literal for the given value */ - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked", "checkstyle:CyclomaticComplexity"}) static Literal from(T value) { Preconditions.checkNotNull(value, "Cannot create expression literal from null"); Preconditions.checkArgument(!NaNUtil.isNaN(value), "Cannot create expression literal from NaN"); @@ -80,6 +82,10 @@ static Literal from(T value) { return (Literal) new Literals.BinaryLiteral((ByteBuffer) value); } else if (value instanceof BigDecimal) { return (Literal) new Literals.DecimalLiteral((BigDecimal) value); + } else if (value instanceof Geometry) { + return (Literal) new Literals.GeometryLiteral((Geometry) value); + } else if (value instanceof Geography) { + return (Literal) new Literals.GeographyLiteral((Geography) value); } throw new IllegalArgumentException( @@ -687,4 +693,61 @@ public String toString() { return "X'" + BaseEncoding.base16().encode(bytes) + "'"; } } + + static class GeometryLiteral extends BaseLiteral { + @SuppressWarnings("unchecked") + private static final Comparator CMP = + Comparators.nullsFirst().thenComparing(Comparator.naturalOrder()); + + GeometryLiteral(Geometry value) { + super(value); + } + + @Override + @SuppressWarnings("unchecked") + public Literal to(Type type) { + if (type.typeId() == Type.TypeID.GEOMETRY) { + return (Literal) this; + } + return null; + } + + @Override + public Comparator comparator() { + return CMP; + } + + @Override + protected Type.TypeID typeId() { + return Type.TypeID.GEOMETRY; + } + } + + static class GeographyLiteral extends BaseLiteral { + private static final Comparator CMP = + Comparators.nullsFirst().thenComparing(Comparator.naturalOrder()); + + GeographyLiteral(Geography value) { + super(value); + } + + @Override + @SuppressWarnings("unchecked") + public Literal to(Type type) { + if (type.typeId() == Type.TypeID.GEOGRAPHY) { + return (Literal) this; + } + return null; + } + + @Override + public Comparator comparator() { + return CMP; + } + + @Override + protected Type.TypeID typeId() { + return Type.TypeID.GEOGRAPHY; + } + } } diff --git a/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java b/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java index 1a5a884f651a..5f29ac769c4b 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java +++ b/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java @@ -476,6 +476,26 @@ private boolean isNestedColumn(int id) { return struct.field(id) == null; } + @Override + public Boolean stIntersects(BoundReference ref, Literal lit) { + return ROWS_MIGHT_NOT_MATCH; + } + + @Override + public Boolean stCovers(BoundReference ref, Literal lit) { + return ROWS_MIGHT_NOT_MATCH; + } + + @Override + public Boolean stDisjoint(BoundReference ref, Literal lit) { + return ROWS_MIGHT_NOT_MATCH; + } + + @Override + public Boolean stNotCovers(BoundReference ref, Literal lit) { + return ROWS_MIGHT_NOT_MATCH; + } + private boolean canContainNulls(Integer id) { return nullCounts == null || (nullCounts.containsKey(id) && nullCounts.get(id) > 0); } diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java index 4736ca4a8668..42118c6d85b3 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java +++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java @@ -276,6 +276,14 @@ public String toString() { return term() + " startsWith \"" + literal() + "\""; case NOT_STARTS_WITH: return term() + " notStartsWith \"" + literal() + "\""; + case ST_INTERSECTS: + return term() + " stIntersects " + literal(); + case ST_COVERS: + return term() + " stCovers " + literal(); + case ST_DISJOINT: + return term() + " stDisjoint " + literal(); + case ST_NOT_COVERS: + return term() + " stNotCovers " + literal(); case IN: return term() + " in (" + COMMA.join(literals()) + ")"; case NOT_IN: diff --git a/api/src/main/java/org/apache/iceberg/types/Conversions.java b/api/src/main/java/org/apache/iceberg/types/Conversions.java index e18c7b4362e6..4993d04e8eb2 100644 --- a/api/src/main/java/org/apache/iceberg/types/Conversions.java +++ b/api/src/main/java/org/apache/iceberg/types/Conversions.java @@ -29,9 +29,17 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.UUID; +import org.apache.iceberg.Geography; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.util.UUIDUtil; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.CoordinateXYM; +import org.locationtech.jts.geom.CoordinateXYZM; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; public class Conversions { @@ -39,6 +47,8 @@ private Conversions() {} private static final String HIVE_NULL = "__HIVE_DEFAULT_PARTITION__"; + private static final GeometryFactory FACTORY = new GeometryFactory(); + public static Object fromPartitionString(Type type, String asString) { if (asString == null || HIVE_NULL.equals(asString)) { return null; @@ -117,6 +127,10 @@ public static ByteBuffer toByteBuffer(Type.TypeID typeId, Object value) { return (ByteBuffer) value; case DECIMAL: return ByteBuffer.wrap(((BigDecimal) value).unscaledValue().toByteArray()); + case GEOMETRY: + return geometryToByteBuffer((Geometry) value); + case GEOGRAPHY: + return geometryToByteBuffer(((Geography) value).geometry()); default: throw new UnsupportedOperationException("Cannot serialize type: " + typeId); } @@ -177,8 +191,58 @@ private static Object internalFromByteBuffer(Type type, ByteBuffer buffer) { byte[] unscaledBytes = new byte[buffer.remaining()]; tmp.get(unscaledBytes); return new BigDecimal(new BigInteger(unscaledBytes), decimal.scale()); + case GEOMETRY: + case GEOGRAPHY: + Coordinate coordinate = coordinateFromByteBuffer(tmp); + Geometry geometry = FACTORY.createPoint(coordinate); + if (type.typeId() == Type.TypeID.GEOMETRY) { + return geometry; + } else { + return new Geography(geometry); + } default: throw new UnsupportedOperationException("Cannot deserialize type: " + type); } } + + private static ByteBuffer geometryToByteBuffer(Geometry value) { + if (value instanceof Point) { + Coordinate coordinate = value.getCoordinate(); + return coordinateToByteBuffer(coordinate); + } else { + throw new IllegalArgumentException("Only point geometry can be converted to byte buffer"); + } + } + + private static ByteBuffer coordinateToByteBuffer(Coordinate coordinate) { + // The getZ() and getM() for a coordinate will return NaN if the value is not set. + // This is conformant with the Bound Serialization spec. + // See https://iceberg.apache.org/spec/#bound-serialization + return ByteBuffer.allocate(32) + .order(ByteOrder.LITTLE_ENDIAN) + .putDouble(0, coordinate.getX()) + .putDouble(8, coordinate.getY()) + .putDouble(16, coordinate.getZ()) + .putDouble(24, coordinate.getM()); + } + + private static Coordinate coordinateFromByteBuffer(ByteBuffer tmp) { + double coordX = tmp.getDouble(0); + double coordY = tmp.getDouble(8); + double coordZ = tmp.getDouble(16); + double coordM = tmp.getDouble(24); + boolean hasZ = !Double.isNaN(coordZ); + boolean hasM = !Double.isNaN(coordM); + Coordinate coordinate; + if (hasZ && hasM) { + coordinate = new CoordinateXYZM(coordX, coordY, coordZ, coordM); + } else if (hasZ) { + coordinate = new Coordinate(coordX, coordY, coordZ); + } else if (hasM) { + coordinate = new CoordinateXYM(coordX, coordY, coordM); + } else { + coordinate = new CoordinateXY(coordX, coordY); + } + return coordinate; + } } diff --git a/api/src/main/java/org/apache/iceberg/types/Type.java b/api/src/main/java/org/apache/iceberg/types/Type.java index 184a17416eae..d221a2455f7b 100644 --- a/api/src/main/java/org/apache/iceberg/types/Type.java +++ b/api/src/main/java/org/apache/iceberg/types/Type.java @@ -25,8 +25,10 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import org.apache.iceberg.Geography; import org.apache.iceberg.StructLike; import org.apache.iceberg.variants.Variant; +import org.locationtech.jts.geom.Geometry; public interface Type extends Serializable { enum TypeID { @@ -44,6 +46,8 @@ enum TypeID { FIXED(ByteBuffer.class), BINARY(ByteBuffer.class), DECIMAL(BigDecimal.class), + GEOMETRY(Geometry.class), + GEOGRAPHY(Geography.class), STRUCT(StructLike.class), LIST(List.class), MAP(Map.class), diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index a866a31ea005..29fcbf92a5fc 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -26,6 +26,7 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.Literal; @@ -61,6 +62,10 @@ private Types() {} .buildOrThrow(); private static final Pattern FIXED = Pattern.compile("fixed\\[\\s*(\\d+)\\s*\\]"); + private static final Pattern GEOMETRY_PARAMETERS = + Pattern.compile("(?:\\(\\s*([^, ]+)?\\s*\\))?"); + private static final Pattern GEOGRAPHY_PARAMETERS = + Pattern.compile("(?:\\(\\s*([^, ]+)?\\s*(?:,\\s*(\\w+)\\s*)?\\))?"); private static final Pattern DECIMAL = Pattern.compile("decimal\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)"); @@ -70,6 +75,20 @@ public static Type fromTypeName(String typeString) { return TYPES.get(lowerTypeString); } + if (lowerTypeString.startsWith("geometry")) { + Matcher geometry = GEOMETRY_PARAMETERS.matcher(typeString.substring(8)); + if (geometry.matches()) { + return GeometryType.of(geometry.group(1)); + } + } + + if (lowerTypeString.startsWith("geography")) { + Matcher geography = GEOGRAPHY_PARAMETERS.matcher(typeString.substring(9)); + if (geography.matches()) { + return GeographyType.of(geography.group(1), geography.group(2)); + } + } + Matcher fixed = FIXED.matcher(lowerTypeString); if (fixed.matches()) { return FixedType.ofLength(Integer.parseInt(fixed.group(1))); @@ -543,6 +562,126 @@ public int hashCode() { } } + public static class GeometryType extends PrimitiveType { + + public static final String DEFAULT_CRS = "OGC:CRS84"; + + private final String crs; + + private GeometryType(String crs) { + this.crs = crs; + } + + public static GeometryType get() { + return of(DEFAULT_CRS); + } + + public static GeometryType of(String crs) { + return new GeometryType(crs == null ? DEFAULT_CRS : crs); + } + + @Override + public TypeID typeId() { + return TypeID.GEOMETRY; + } + + public String crs() { + return crs; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (!(o instanceof GeometryType)) { + return false; + } + + GeometryType that = (GeometryType) o; + return crs.equals(that.crs); + } + + @Override + public int hashCode() { + return Objects.hash(GeometryType.class, crs); + } + + @Override + public String toString() { + return String.format("geometry(%s)", crs); + } + } + + public static class GeographyType extends PrimitiveType { + + public static final String DEFAULT_CRS = "OGC:CRS84"; + public static final Geography.EdgeInterpolationAlgorithm DEFAULT_ALGORITHM = + Geography.EdgeInterpolationAlgorithm.SPHERICAL; + + private final String crs; + private final Geography.EdgeInterpolationAlgorithm algorithm; + + private GeographyType(String crs, Geography.EdgeInterpolationAlgorithm algorithm) { + this.crs = crs; + this.algorithm = algorithm; + } + + public static GeographyType get() { + return of(DEFAULT_CRS); + } + + public static GeographyType of(String crs) { + return of(crs, DEFAULT_ALGORITHM); + } + + public static GeographyType of(String crs, Geography.EdgeInterpolationAlgorithm algorithm) { + return new GeographyType(crs, algorithm); + } + + public static GeographyType of(String crs, String algorithmName) { + Geography.EdgeInterpolationAlgorithm algorithm = + (algorithmName == null + ? DEFAULT_ALGORITHM + : Geography.EdgeInterpolationAlgorithm.fromName(algorithmName)); + return new GeographyType(crs == null ? DEFAULT_CRS : crs, algorithm); + } + + @Override + public TypeID typeId() { + return TypeID.GEOGRAPHY; + } + + public String crs() { + return crs; + } + + public Geography.EdgeInterpolationAlgorithm algorithm() { + return algorithm; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (!(o instanceof GeographyType)) { + return false; + } + + GeographyType that = (GeographyType) o; + return crs.equals(that.crs) && algorithm.equals(that.algorithm); + } + + @Override + public int hashCode() { + return Objects.hash(GeographyType.class, crs, algorithm); + } + + @Override + public String toString() { + return String.format("geography(%s, %s)", crs, algorithm.value()); + } + } + public static class NestedField implements Serializable { public static NestedField optional(int id, String name, Type type) { return new NestedField(true, id, name, type, null, null, null); diff --git a/api/src/main/java/org/apache/iceberg/util/GeometryUtil.java b/api/src/main/java/org/apache/iceberg/util/GeometryUtil.java new file mode 100644 index 000000000000..e161605d12a8 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/util/GeometryUtil.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.util; + +import org.apache.iceberg.Geography; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; +import org.locationtech.jts.io.WKTReader; +import org.locationtech.jts.io.WKTWriter; + +public class GeometryUtil { + + private GeometryUtil() {} + + private static final GeometryFactory FACTORY = new GeometryFactory(); + + public static byte[] toWKB(Geometry geom) { + WKBWriter wkbWriter = new WKBWriter(getOutputDimension(geom), false); + return wkbWriter.write(geom); + } + + public static String toWKT(Geometry geom) { + WKTWriter wktWriter = new WKTWriter(getOutputDimension(geom)); + return wktWriter.write(geom); + } + + public static Geometry fromWKB(byte[] wkb) { + WKBReader reader = new WKBReader(); + try { + return reader.read(wkb); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse WKB", e); + } + } + + public static Geometry fromWKT(String wkt) { + WKTReader reader = new WKTReader(); + try { + return reader.read(wkt); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse WKT", e); + } + } + + public static int getOutputDimension(Geometry geom) { + int dimension = 2; + Coordinate coordinate = geom.getCoordinate(); + + // We need to set outputDimension = 4 for XYM geometries to make JTS WKTWriter or WKBWriter work + // correctly. + // The WKB/WKT writers will ignore Z ordinate for XYM geometries. + if (!Double.isNaN(coordinate.getZ())) { + dimension = 3; + } + if (!Double.isNaN(coordinate.getM())) { + dimension = 4; + } + return dimension; + } + + /** + * Check if the geometry may intersect with the given bound. The bound represents a rectangle + * crossing the anti-meridian when the x of lower bound is greater than the x of upper bound. + * + * @param lowerBound The lower-left point of the bound + * @param upperBound The upper-right point of the bound + * @param geom The geometry to check + * @return true if the geometry may intersect with the bound; false if the geometry definitely + * does not intersect with the bound + */ + public static boolean boundMayIntersects( + Geometry lowerBound, Geometry upperBound, Geometry geom) { + Preconditions.checkArgument(lowerBound instanceof Point, "Lower bound must be a point"); + Preconditions.checkArgument(upperBound instanceof Point, "Upper bound must be a point"); + + Coordinate lowerCoordinate = lowerBound.getCoordinate(); + Coordinate upperCoordinate = upperBound.getCoordinate(); + if (lowerCoordinate.x <= upperCoordinate.x) { + // Not crossing the anti-meridian + Envelope envelope = new Envelope(lowerBound.getCoordinate(), upperBound.getCoordinate()); + return geom.intersects(FACTORY.toGeometry(envelope)); + } else { + // Crossing the anti-meridian. Use the envelope of geom to evaluate the intersection with + // false positives + Envelope envelope = geom.getEnvelopeInternal(); + if (envelope.getMinY() > upperCoordinate.y || envelope.getMaxY() < lowerCoordinate.y) { + return false; + } + return (envelope.getMinX() <= upperCoordinate.x || envelope.getMaxX() >= lowerCoordinate.x); + } + } + + /** + * Check if the geography may intersect with the given bound. The bound represents a rectangle + * crossing the anti-meridian when the x of lower bound is greater than the x of upper bound. + * + * @param lowerBound The lower-left point of the bound + * @param upperBound The upper-right point of the bound + * @param geog The geography to check + * @return true if the geography may intersect with the bound; false if the geography definitely + * does not intersect with the bound + */ + public static boolean boundMayIntersects( + Geography lowerBound, Geography upperBound, Geography geog) { + // TODO: implement a correct spherical intersection algorithm + return boundMayIntersects(lowerBound.geometry(), upperBound.geometry(), geog.geometry()); + } + + /** + * Check if the bound may cover the geometry. The bound represents a rectangle crossing the + * anti-meridian when the x of lower bound is greater than the x of upper bound. + * + * @param lowerBound The lower-left point of the bound + * @param upperBound The upper-right point of the bound + * @param geom The geometry to check + * @return true if the bound may cover the geometry; false if the bound definitely does not cover + * the geometry + */ + public static boolean boundMayCovers(Geometry lowerBound, Geometry upperBound, Geometry geom) { + Preconditions.checkArgument(lowerBound instanceof Point, "Lower bound must be a point"); + Preconditions.checkArgument(upperBound instanceof Point, "Upper bound must be a point"); + + Coordinate lowerCoordinate = lowerBound.getCoordinate(); + Coordinate upperCoordinate = upperBound.getCoordinate(); + if (lowerCoordinate.x <= upperCoordinate.x) { + // Not crossing the anti-meridian + Envelope envelope = new Envelope(lowerBound.getCoordinate(), upperBound.getCoordinate()); + return FACTORY.toGeometry(envelope).covers(geom); + } else { + // Crossing the anti-meridian. Use the envelope of geom to evaluate the covers with false + // positives + Envelope envelope = geom.getEnvelopeInternal(); + if (envelope.getMinY() < lowerCoordinate.y || envelope.getMaxY() > upperCoordinate.y) { + return false; + } + return (envelope.getMaxX() <= upperCoordinate.x || envelope.getMinX() >= lowerCoordinate.x); + } + } + + /** + * Check if the bound may cover the geography. The bound represents a rectangle crossing the + * anti-meridian when the x of lower bound is greater than the x of upper bound. + * + * @param lowerBound The lower-left point of the bound + * @param upperBound The upper-right point of the bound + * @param geog The geography to check + * @return true if the bound may cover the geography; false if the bound definitely does not cover + * the geography + */ + public static boolean boundMayCovers(Geography lowerBound, Geography upperBound, Geography geog) { + // TODO: implement a correct spherical covers algorithm + return boundMayCovers(lowerBound.geometry(), upperBound.geometry(), geog.geometry()); + } + + /** + * Check if we are sure that the bound must be covered by the geometry. The bound represents a + * rectangle crossing the anti-meridian when the x of lower bound is greater than the x of upper + * bound. + * + * @param lowerBound The lower-left point of the bound + * @param upperBound The upper-right point of the bound + * @param geom The geometry to check + * @return true if the bound is definitely covered by the geometry; false if the bound may or may + * not cover the geometry + */ + public static boolean boundMustBeCoveredBy( + Geometry lowerBound, Geometry upperBound, Geometry geom) { + Preconditions.checkArgument(lowerBound instanceof Point, "Lower bound must be a point"); + Preconditions.checkArgument(upperBound instanceof Point, "Upper bound must be a point"); + + Coordinate lowerCoordinate = lowerBound.getCoordinate(); + Coordinate upperCoordinate = upperBound.getCoordinate(); + if (lowerCoordinate.x <= upperCoordinate.x) { + // Not crossing the anti-meridian + Envelope envelope = new Envelope(lowerBound.getCoordinate(), upperBound.getCoordinate()); + return FACTORY.toGeometry(envelope).coveredBy(geom); + } else { + // Crossing the anti-meridian. This case can be tricky so we always return false to be safe. + return false; + } + } + + /** + * Check if we are sure that the bound must be covered by the geography. The bound represents a + * rectangle crossing the anti-meridian when the x of lower bound is greater than the x of upper + * bound. + * + * @param lowerBound The lower-left point of the bound + * @param upperBound The upper-right point of the bound + * @param geog The geography to check + * @return true if the bound is definitely covered by the geography; false if the bound may or may + * not cover the geography + */ + public static boolean boundMustBeCoveredBy( + Geography lowerBound, Geography upperBound, Geography geog) { + // TODO: implement a correct spherical covered-by algorithm + return boundMustBeCoveredBy(lowerBound.geometry(), upperBound.geometry(), geog.geometry()); + } +} diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestEvaluator.java index 792e651a3d18..48c163c12792 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestEvaluator.java @@ -37,6 +37,10 @@ import static org.apache.iceberg.expressions.Expressions.notStartsWith; import static org.apache.iceberg.expressions.Expressions.or; import static org.apache.iceberg.expressions.Expressions.predicate; +import static org.apache.iceberg.expressions.Expressions.stCovers; +import static org.apache.iceberg.expressions.Expressions.stDisjoint; +import static org.apache.iceberg.expressions.Expressions.stIntersects; +import static org.apache.iceberg.expressions.Expressions.stNotCovers; import static org.apache.iceberg.expressions.Expressions.startsWith; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.apache.iceberg.types.Types.NestedField.required; @@ -47,11 +51,16 @@ import java.util.Collection; import java.util.Collections; import org.apache.avro.util.Utf8; +import org.apache.iceberg.Geography; import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.StructType; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; public class TestEvaluator { private static final StructType STRUCT = @@ -809,4 +818,186 @@ public void testNotInExceptions() { .isInstanceOf(ValidationException.class) .hasMessageContaining("Invalid value for conversion to type int"); } + + @Test + public void testStIntersects() { + StructType struct = StructType.of(required(24, "g", Types.GeometryType.get())); + GeometryFactory factory = new GeometryFactory(); + Geometry queryWindow = factory.toGeometry(new Envelope(0, 1, 0, 1)); + Geography queryWindowGeography = new Geography(queryWindow); + Evaluator evaluator = new Evaluator(struct, stIntersects("g", queryWindow)); + + assertThat(evaluator.eval(TestHelpers.Row.of(factory.createPoint(new Coordinate(0.5, 0.5))))) + .as("Point intersects with the query window") + .isTrue(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.createPoint(new Coordinate(2, 2))))) + .as("Point does not intersect with the query window") + .isFalse(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2))))) + .as("Envelope intersects with the query window") + .isTrue(); + assertThat(evaluator.eval(TestHelpers.Row.of((Geometry) null))) + .as("null does not intersect with the query window") + .isFalse(); + + struct = StructType.of(required(24, "g", Types.GeographyType.get())); + Evaluator geographyEvaluator = new Evaluator(struct, stIntersects("g", queryWindowGeography)); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.createPoint(new Coordinate(0.5, 0.5)))))) + .as("Geography point intersects with the query window") + .isTrue(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.createPoint(new Coordinate(2, 2)))))) + .as("Geography point does not intersect with the query window") + .isFalse(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of( + new Geography(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2)))))) + .as("Envelope intersects with the query window") + .isTrue(); + assertThat(geographyEvaluator.eval(TestHelpers.Row.of((Geography) null))) + .as("null does not intersect with the query window") + .isFalse(); + } + + @Test + public void testStCovers() { + StructType struct = StructType.of(required(24, "g", Types.GeometryType.get())); + GeometryFactory factory = new GeometryFactory(); + Geometry queryWindow = factory.toGeometry(new Envelope(0, 1, 0, 1)); + Geography queryWindowGeography = new Geography(queryWindow); + + // Test geometry type + Evaluator evaluator = new Evaluator(struct, stCovers("g", queryWindow)); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.createPoint(new Coordinate(0.5, 0.5))))) + .as("Point does not cover the query window") + .isFalse(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2))))) + .as("Envelope does not cover the query window") + .isFalse(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.toGeometry(new Envelope(0, 2, 0, 2))))) + .as("Envelope covers the query window") + .isTrue(); + assertThat(evaluator.eval(TestHelpers.Row.of((Geometry) null))) + .as("null does not cover the query window") + .isFalse(); + + // Test geography type + struct = StructType.of(required(24, "g", Types.GeographyType.get())); + Evaluator geographyEvaluator = new Evaluator(struct, stCovers("g", queryWindowGeography)); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.createPoint(new Coordinate(0.5, 0.5)))))) + .as("Geography point does not cover the query window") + .isFalse(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of( + new Geography(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2)))))) + .as("Geography envelope does not cover the query window") + .isFalse(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.toGeometry(new Envelope(0, 2, 0, 2)))))) + .as("Geography envelope covers the query window") + .isTrue(); + assertThat(geographyEvaluator.eval(TestHelpers.Row.of((Geography) null))) + .as("null does not cover the query window") + .isFalse(); + } + + @Test + public void testStDisjoint() { + StructType struct = StructType.of(required(24, "g", Types.GeometryType.get())); + GeometryFactory factory = new GeometryFactory(); + Geometry queryWindow = factory.toGeometry(new Envelope(0, 1, 0, 1)); + Geography queryWindowGeography = new Geography(queryWindow); + + // Test geometry type + Evaluator evaluator = new Evaluator(struct, stDisjoint("g", queryWindow)); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.createPoint(new Coordinate(0.5, 0.5))))) + .as("Point intersects with the query window") + .isFalse(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.createPoint(new Coordinate(2, 2))))) + .as("Point disjoint with the query window") + .isTrue(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2))))) + .as("Envelope intersects with the query window") + .isFalse(); + assertThat(evaluator.eval(TestHelpers.Row.of((Geometry) null))) + .as("null disjoint with the query window") + .isTrue(); + + // Test geography type + struct = StructType.of(required(24, "g", Types.GeographyType.get())); + Evaluator geographyEvaluator = new Evaluator(struct, stDisjoint("g", queryWindowGeography)); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.createPoint(new Coordinate(0.5, 0.5)))))) + .as("Geography point intersects with the query window") + .isFalse(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.createPoint(new Coordinate(2, 2)))))) + .as("Geography point disjoint with the query window") + .isTrue(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of( + new Geography(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2)))))) + .as("Geography envelope intersects with the query window") + .isFalse(); + assertThat(geographyEvaluator.eval(TestHelpers.Row.of((Geography) null))) + .as("null disjoint with the query window") + .isTrue(); + } + + @Test + public void testStNotCovers() { + StructType struct = StructType.of(required(24, "g", Types.GeometryType.get())); + GeometryFactory factory = new GeometryFactory(); + Geometry queryWindow = factory.toGeometry(new Envelope(0, 1, 0, 1)); + Geography queryWindowGeography = new Geography(queryWindow); + + // Test geometry type + Evaluator evaluator = new Evaluator(struct, stNotCovers("g", queryWindow)); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.createPoint(new Coordinate(0.5, 0.5))))) + .as("Point does not cover the query window") + .isTrue(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2))))) + .as("Envelope does not cover the query window") + .isTrue(); + assertThat(evaluator.eval(TestHelpers.Row.of(factory.toGeometry(new Envelope(0, 2, 0, 2))))) + .as("Envelope covers the query window") + .isFalse(); + assertThat(evaluator.eval(TestHelpers.Row.of((Geometry) null))) + .as("null does not cover the query window") + .isTrue(); + + // Test geography type + struct = StructType.of(required(24, "g", Types.GeographyType.get())); + Evaluator geographyEvaluator = new Evaluator(struct, stNotCovers("g", queryWindowGeography)); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.createPoint(new Coordinate(0.5, 0.5)))))) + .as("Geography point does not cover the query window") + .isTrue(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of( + new Geography(factory.toGeometry(new Envelope(0.5, 2, 0.5, 2)))))) + .as("Geography envelope does not cover the query window") + .isTrue(); + assertThat( + geographyEvaluator.eval( + TestHelpers.Row.of(new Geography(factory.toGeometry(new Envelope(0, 2, 0, 2)))))) + .as("Geography envelope covers the query window") + .isFalse(); + assertThat(geographyEvaluator.eval(TestHelpers.Row.of((Geography) null))) + .as("null does not cover the query window") + .isTrue(); + } } diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java index 8bb03c633ab5..4d95a99ece50 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java @@ -41,6 +41,10 @@ import static org.apache.iceberg.expressions.Expressions.predicate; import static org.apache.iceberg.expressions.Expressions.ref; import static org.apache.iceberg.expressions.Expressions.rewriteNot; +import static org.apache.iceberg.expressions.Expressions.stCovers; +import static org.apache.iceberg.expressions.Expressions.stDisjoint; +import static org.apache.iceberg.expressions.Expressions.stIntersects; +import static org.apache.iceberg.expressions.Expressions.stNotCovers; import static org.apache.iceberg.expressions.Expressions.startsWith; import static org.apache.iceberg.expressions.Expressions.truncate; import static org.apache.iceberg.expressions.Expressions.year; @@ -48,11 +52,15 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.concurrent.Callable; +import org.apache.iceberg.Geography; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.types.Types.StructType; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; public class TestExpressionHelpers { private final UnboundPredicate pred = lessThan("x", 7); @@ -96,7 +104,12 @@ public void testRewriteNot() { StructType struct = StructType.of( NestedField.optional(1, "a", Types.IntegerType.get()), - NestedField.optional(2, "s", Types.StringType.get())); + NestedField.optional(2, "s", Types.StringType.get()), + NestedField.optional(3, "g", Types.GeometryType.get()), + NestedField.optional(4, "geog", Types.GeographyType.get())); + GeometryFactory factory = new GeometryFactory(); + Geometry queryWindow = factory.toGeometry(new Envelope(0, 1, 0, 1)); + Geography queryWindowGeography = new Geography(queryWindow); Expression[][] expressions = new Expression[][] { // (rewritten pred, original pred) pairs @@ -126,7 +139,21 @@ public void testRewriteNot() { {or(equal("a", 5), isNull("a")), not(and(notEqual("a", 5), notNull("a")))}, {or(equal("a", 5), notNull("a")), or(equal("a", 5), not(isNull("a")))}, {startsWith("s", "hello"), not(notStartsWith("s", "hello"))}, - {notStartsWith("s", "world"), not(startsWith("s", "world"))} + {notStartsWith("s", "world"), not(startsWith("s", "world"))}, + {stIntersects("g", queryWindow), not(stDisjoint("g", queryWindow))}, + {stCovers("g", queryWindow), not(stNotCovers("g", queryWindow))}, + {stDisjoint("g", queryWindow), not(stIntersects("g", queryWindow))}, + {stNotCovers("g", queryWindow), not(stCovers("g", queryWindow))}, + { + stIntersects("geog", queryWindowGeography), + not(stDisjoint("geog", queryWindowGeography)) + }, + {stCovers("geog", queryWindowGeography), not(stNotCovers("geog", queryWindowGeography))}, + { + stDisjoint("geog", queryWindowGeography), + not(stIntersects("geog", queryWindowGeography)) + }, + {stNotCovers("geog", queryWindowGeography), not(stCovers("geog", queryWindowGeography))} }; for (Expression[] pair : expressions) { diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionSerialization.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionSerialization.java index fc7ddd035bf2..4e808d422d7a 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionSerialization.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionSerialization.java @@ -21,11 +21,15 @@ import static org.assertj.core.api.Assertions.assertThat; import java.util.Collection; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.TestHelpers; import org.apache.iceberg.expressions.Expression.Operation; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; public class TestExpressionSerialization { @Test @@ -33,8 +37,12 @@ public void testExpressions() throws Exception { Schema schema = new Schema( Types.NestedField.optional(34, "a", Types.IntegerType.get()), - Types.NestedField.required(35, "s", Types.StringType.get())); - + Types.NestedField.required(35, "s", Types.StringType.get()), + Types.NestedField.optional(3, "g", Types.GeometryType.get()), + Types.NestedField.optional(4, "geog", Types.GeographyType.get())); + GeometryFactory factory = new GeometryFactory(); + Geometry queryWindow = factory.toGeometry(new Envelope(0, 1, 0, 1)); + Geography queryWindowGeog = new Geography(queryWindow); Expression[] expressions = new Expression[] { Expressions.alwaysFalse(), @@ -61,7 +69,15 @@ public void testExpressions() throws Exception { Expressions.notIn("s", "abc", "xyz").bind(schema.asStruct()), Expressions.isNull("a").bind(schema.asStruct()), Expressions.startsWith("s", "abc").bind(schema.asStruct()), - Expressions.notStartsWith("s", "xyz").bind(schema.asStruct()) + Expressions.notStartsWith("s", "xyz").bind(schema.asStruct()), + Expressions.stIntersects("g", queryWindow).bind(schema.asStruct()), + Expressions.stCovers("g", queryWindow).bind(schema.asStruct()), + Expressions.stDisjoint("g", queryWindow).bind(schema.asStruct()), + Expressions.stNotCovers("g", queryWindow).bind(schema.asStruct()), + Expressions.stIntersects("geog", queryWindowGeog).bind(schema.asStruct()), + Expressions.stCovers("geog", queryWindowGeog).bind(schema.asStruct()), + Expressions.stDisjoint("geog", queryWindowGeog).bind(schema.asStruct()), + Expressions.stNotCovers("geog", queryWindowGeog).bind(schema.asStruct()) }; for (Expression expression : expressions) { diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java index 902c0260d63a..5c56a1fe7718 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java @@ -28,12 +28,16 @@ import java.util.List; import java.util.regex.Pattern; import java.util.stream.IntStream; +import org.apache.iceberg.Geography; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.DateTimeUtil; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; public class TestExpressionUtil { private static final Schema SCHEMA = @@ -47,7 +51,9 @@ public class TestExpressionUtil { Types.NestedField.required(7, "time", Types.DateType.get()), Types.NestedField.optional(8, "data", Types.StringType.get()), Types.NestedField.optional(9, "measurement", Types.DoubleType.get()), - Types.NestedField.optional(10, "test", Types.IntegerType.get())); + Types.NestedField.optional(10, "test", Types.IntegerType.get()), + Types.NestedField.optional(11, "geom", Types.GeometryType.get()), + Types.NestedField.optional(12, "geog", Types.GeographyType.get())); private static final Types.StructType STRUCT = SCHEMA.asStruct(); @@ -800,8 +806,75 @@ public void testSanitizeStringFallback() { } } + @Test + public void testSanitizeSpatialPredicates() { + Pattern geometryFilterPattern = Pattern.compile("^g (\\w+) \\(geometry\\)$"); + Pattern geographyFilterPattern = Pattern.compile("^g (\\w+) \\(geography\\)$"); + GeometryFactory factory = new GeometryFactory(); + Geometry queryWindow = factory.toGeometry(new Envelope(0, 1, 0, 1)); + Geography queryGeography = new Geography(factory.toGeometry(new Envelope(0, 1, 0, 1))); + Geometry emptyGeometry = factory.createPoint(); + Geography emptyGeography = new Geography(factory.createPoint()); + + assertEquals( + ExpressionUtil.sanitize(Expressions.stIntersects("g", queryWindow)), + ExpressionUtil.sanitize(Expressions.stIntersects("g", emptyGeometry))); + String sanitizedFilter = + ExpressionUtil.toSanitizedString(Expressions.stIntersects("g", queryWindow)); + assertThat(geometryFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_INTERSECTS"); + assertEquals( + ExpressionUtil.sanitize(Expressions.stIntersects("g", queryGeography)), + ExpressionUtil.sanitize(Expressions.stIntersects("g", emptyGeography))); + sanitizedFilter = + ExpressionUtil.toSanitizedString(Expressions.stIntersects("g", queryGeography)); + assertThat(geographyFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_INTERSECTS"); + + assertEquals( + ExpressionUtil.sanitize(Expressions.stCovers("g", queryWindow)), + ExpressionUtil.sanitize(Expressions.stCovers("g", emptyGeometry))); + sanitizedFilter = ExpressionUtil.toSanitizedString(Expressions.stCovers("g", queryWindow)); + assertThat(geometryFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_COVERS"); + assertEquals( + ExpressionUtil.sanitize(Expressions.stCovers("g", queryGeography)), + ExpressionUtil.sanitize(Expressions.stCovers("g", emptyGeography))); + sanitizedFilter = ExpressionUtil.toSanitizedString(Expressions.stCovers("g", queryGeography)); + assertThat(geographyFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_COVERS"); + + assertEquals( + ExpressionUtil.sanitize(Expressions.stDisjoint("g", queryWindow)), + ExpressionUtil.sanitize(Expressions.stDisjoint("g", emptyGeometry))); + sanitizedFilter = ExpressionUtil.toSanitizedString(Expressions.stDisjoint("g", queryWindow)); + assertThat(geometryFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_DISJOINT"); + assertEquals( + ExpressionUtil.sanitize(Expressions.stDisjoint("g", queryGeography)), + ExpressionUtil.sanitize(Expressions.stDisjoint("g", emptyGeography))); + sanitizedFilter = ExpressionUtil.toSanitizedString(Expressions.stDisjoint("g", queryGeography)); + assertThat(geographyFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_DISJOINT"); + + assertEquals( + ExpressionUtil.sanitize(Expressions.stNotCovers("g", queryWindow)), + ExpressionUtil.sanitize(Expressions.stNotCovers("g", emptyGeometry))); + sanitizedFilter = ExpressionUtil.toSanitizedString(Expressions.stNotCovers("g", queryWindow)); + assertThat(geometryFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_NOT_COVERS"); + assertEquals( + ExpressionUtil.sanitize(Expressions.stNotCovers("g", queryGeography)), + ExpressionUtil.sanitize(Expressions.stNotCovers("g", emptyGeography))); + sanitizedFilter = + ExpressionUtil.toSanitizedString(Expressions.stNotCovers("g", queryGeography)); + assertThat(geographyFilterPattern.matcher(sanitizedFilter)).matches(); + assertThat(sanitizedFilter).contains("ST_NOT_COVERS"); + } + @Test public void testIdenticalExpressionIsEquivalent() { + GeometryFactory factory = new GeometryFactory(); Expression[] exprs = new Expression[] { Expressions.isNull("data"), @@ -818,10 +891,21 @@ public void testIdenticalExpressionIsEquivalent() { Expressions.notIn("id", 5, 6), Expressions.startsWith("data", "aaa"), Expressions.notStartsWith("data", "aaa"), + Expressions.stIntersects("geom", factory.toGeometry(new Envelope(0, 1, 0, 1))), + Expressions.stCovers("geom", factory.toGeometry(new Envelope(0, 1, 0, 1))), + Expressions.stDisjoint("geom", factory.toGeometry(new Envelope(0, 1, 0, 1))), + Expressions.stNotCovers("geom", factory.toGeometry(new Envelope(0, 1, 0, 1))), + Expressions.stIntersects( + "geog", new Geography(factory.toGeometry(new Envelope(0, 1, 0, 1)))), + Expressions.stCovers("geog", new Geography(factory.toGeometry(new Envelope(0, 1, 0, 1)))), + Expressions.stDisjoint( + "geog", new Geography(factory.toGeometry(new Envelope(0, 1, 0, 1)))), + Expressions.stNotCovers( + "geog", new Geography(factory.toGeometry(new Envelope(0, 1, 0, 1)))), Expressions.alwaysTrue(), Expressions.alwaysFalse(), Expressions.and(Expressions.lessThan("id", 5), Expressions.notNull("data")), - Expressions.or(Expressions.lessThan("id", 5), Expressions.notNull("data")), + Expressions.or(Expressions.lessThan("id", 5), Expressions.notNull("data")) }; for (Expression expr : exprs) { diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 7069d891c38d..bdfc28b85aed 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -34,6 +34,9 @@ import static org.apache.iceberg.expressions.Expressions.notNull; import static org.apache.iceberg.expressions.Expressions.notStartsWith; import static org.apache.iceberg.expressions.Expressions.or; +import static org.apache.iceberg.expressions.Expressions.stCovers; +import static org.apache.iceberg.expressions.Expressions.stDisjoint; +import static org.apache.iceberg.expressions.Expressions.stIntersects; import static org.apache.iceberg.expressions.Expressions.startsWith; import static org.apache.iceberg.types.Conversions.toByteBuffer; import static org.apache.iceberg.types.Types.NestedField.optional; @@ -43,6 +46,7 @@ import java.util.List; import org.apache.iceberg.DataFile; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.TestHelpers.Row; import org.apache.iceberg.TestHelpers.TestDataFile; @@ -54,6 +58,9 @@ import org.apache.iceberg.types.Types.StringType; import org.apache.iceberg.util.UnicodeUtil; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.GeometryFactory; public class TestInclusiveMetricsEvaluator { private static final Schema SCHEMA = @@ -71,7 +78,11 @@ public class TestInclusiveMetricsEvaluator { optional(11, "all_nans_v1_stats", Types.FloatType.get()), optional(12, "nan_and_null_only", Types.DoubleType.get()), optional(13, "no_nan_stats", Types.DoubleType.get()), - optional(14, "some_empty", Types.StringType.get())); + optional(14, "some_empty", Types.StringType.get()), + optional(15, "geom", Types.GeometryType.get()), + optional(16, "all_nulls_geom", Types.GeometryType.get()), + optional(17, "geog", Types.GeographyType.get()), + optional(18, "all_nulls_geog", Types.GeographyType.get())); private static final int INT_MIN_VALUE = 30; private static final int INT_MAX_VALUE = 79; @@ -187,6 +198,46 @@ public class TestInclusiveMetricsEvaluator { // upper bounds ImmutableMap.of(3, toByteBuffer(StringType.get(), "abcdefghi"))); + private static final GeometryFactory FACTORY = new GeometryFactory(); + + private static final DataFile FILE_6 = + new TestDataFile( + "file_6.avro", + Row.of(), + 50, + // any value counts, including nulls + ImmutableMap.builder() + .put(15, 20L) + .put(16, 20L) + .put(17, 20L) + .put(18, 20L) + .buildOrThrow(), + // null value counts + ImmutableMap.builder() + .put(15, 2L) + .put(16, 20L) + .put(17, 2L) + .put(18, 20L) + .buildOrThrow(), + // nan value counts + null, + // lower bounds + ImmutableMap.of( + 15, + toByteBuffer(Types.GeometryType.get(), FACTORY.createPoint(new Coordinate(1, 2))), + 17, + toByteBuffer( + Types.GeographyType.get(), + new Geography(FACTORY.createPoint(new Coordinate(1, 2))))), + // upper bounds + ImmutableMap.of( + 15, + toByteBuffer(Types.GeometryType.get(), FACTORY.createPoint(new Coordinate(10, 20))), + 17, + toByteBuffer( + Types.GeographyType.get(), + new Geography(FACTORY.createPoint(new Coordinate(10, 20)))))); + @Test public void testAllNulls() { boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("all_nulls")).eval(FILE); @@ -863,4 +914,228 @@ public void testIntegerNotIn() { shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nulls", "abc", "def")).eval(FILE); assertThat(shouldRead).as("Should read: notIn on no nulls column").isTrue(); } + + @Test + public void testStIntersects() { + boolean shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stIntersects("geom", FACTORY.createPoint(new Coordinate(1, 2)))) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: query window is within the boundary").isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stIntersects("geom", FACTORY.toGeometry(new Envelope(0, 3, 0, 4)))) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: query window intersects with the boundary").isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stIntersects("geom", FACTORY.toGeometry(new Envelope(0, 0.5, 0, 2)))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should skip: query window does not intersect with the boundary") + .isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator(SCHEMA, stIntersects("geom", FACTORY.createPoint())) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: query window is empty").isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator(SCHEMA, stIntersects("all_nulls_geom", FACTORY.createPoint())) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: geometry are all nulls").isFalse(); + } + + @Test + public void testStCovers() { + boolean shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stCovers("geom", FACTORY.toGeometry(new Envelope(3, 4, 5, 6)))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should read: query window is completely within the boundary") + .isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stCovers("geom", FACTORY.toGeometry(new Envelope(0, 3, 0, 4)))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should skip: query window is not completely within the boundary") + .isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stCovers("geom", FACTORY.toGeometry(new Envelope(0, 0.5, 0, 2)))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should skip: query window does not intersect with the boundary") + .isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator(SCHEMA, stCovers("geom", FACTORY.createPoint())).eval(FILE_6); + assertThat(shouldRead).as("Should skip: query window is empty").isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator(SCHEMA, stCovers("all_nulls_geom", FACTORY.createPoint())) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: geometry are all nulls").isFalse(); + } + + @Test + public void testStDisjoint() { + boolean shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stDisjoint("geom", FACTORY.toGeometry(new Envelope(3, 4, 5, 6)))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should read: query window does not fully cover the boundary") + .isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stDisjoint("geom", FACTORY.createPoint(new Coordinate(1, 1)))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should read: query window does not intersect with the boundary") + .isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stDisjoint("geom", FACTORY.toGeometry(new Envelope(0, 100, 0, 100)))) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: query window fully covers the boundary").isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator(SCHEMA, stDisjoint("geom", FACTORY.createPoint())) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: query window is empty").isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator(SCHEMA, stDisjoint("all_nulls_geom", FACTORY.createPoint())) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: geometry are all nulls").isTrue(); + } + + @Test + public void testStIntersectsGeog() { + boolean shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stIntersects("geog", new Geography(FACTORY.createPoint(new Coordinate(1, 2))))) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: query window is within the boundary").isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stIntersects("geog", new Geography(FACTORY.toGeometry(new Envelope(0, 3, 0, 4))))) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: query window intersects with the boundary").isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stIntersects("geog", new Geography(FACTORY.toGeometry(new Envelope(0, 0.5, 0, 2))))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should skip: query window does not intersect with the boundary") + .isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stIntersects("geog", new Geography(FACTORY.createPoint()))) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: query window is empty").isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stIntersects("all_nulls_geog", new Geography(FACTORY.createPoint()))) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: geography are all nulls").isFalse(); + } + + @Test + public void testStCoversGeog() { + boolean shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stCovers("geog", new Geography(FACTORY.toGeometry(new Envelope(3, 4, 5, 6))))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should read: query window is completely within the boundary") + .isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stCovers("geog", new Geography(FACTORY.toGeometry(new Envelope(0, 3, 0, 4))))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should skip: query window is not completely within the boundary") + .isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stCovers("geog", new Geography(FACTORY.toGeometry(new Envelope(0, 0.5, 0, 2))))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should skip: query window does not intersect with the boundary") + .isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stCovers("geog", new Geography(FACTORY.createPoint()))) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: query window is empty").isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stCovers("all_nulls_geog", new Geography(FACTORY.createPoint()))) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: geography are all nulls").isFalse(); + } + + @Test + public void testStDisjointGeog() { + boolean shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stDisjoint("geog", new Geography(FACTORY.toGeometry(new Envelope(3, 4, 5, 6))))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should read: query window does not fully cover the boundary") + .isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stDisjoint("geog", new Geography(FACTORY.createPoint(new Coordinate(1, 1))))) + .eval(FILE_6); + assertThat(shouldRead) + .as("Should read: query window does not intersect with the boundary") + .isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, + stDisjoint("geog", new Geography(FACTORY.toGeometry(new Envelope(0, 100, 0, 100))))) + .eval(FILE_6); + assertThat(shouldRead).as("Should skip: query window fully covers the boundary").isFalse(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stDisjoint("geog", new Geography(FACTORY.createPoint()))) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: query window is empty").isTrue(); + + shouldRead = + new InclusiveMetricsEvaluator( + SCHEMA, stDisjoint("all_nulls_geog", new Geography(FACTORY.createPoint()))) + .eval(FILE_6); + assertThat(shouldRead).as("Should read: geography are all nulls").isTrue(); + } } diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java b/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java index 24fc458b37b4..18b9ba3abd58 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestLiteralSerialization.java @@ -25,10 +25,16 @@ import org.apache.iceberg.TestHelpers; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.CoordinateXYM; +import org.locationtech.jts.geom.CoordinateXYZM; +import org.locationtech.jts.geom.GeometryFactory; public class TestLiteralSerialization { @Test public void testLiterals() throws Exception { + GeometryFactory factory = new GeometryFactory(); Literal[] literals = new Literal[] { Literal.of(false), @@ -47,6 +53,12 @@ public void testLiterals() throws Exception { Literal.of(new byte[] {1, 2, 3}).to(Types.FixedType.ofLength(3)), Literal.of(new byte[] {3, 4, 5, 6}).to(Types.BinaryType.get()), Literal.of(new BigDecimal("122.50")), + Literal.of(factory.createPoint()), + Literal.of(factory.createPoint(new CoordinateXY(10, 20))), + Literal.of(factory.createPoint(new Coordinate(10, 20))), + Literal.of(factory.createPoint(new Coordinate(10, 20, 30))), + Literal.of(factory.createPoint(new CoordinateXYM(10, 20, 30))), + Literal.of(factory.createPoint(new CoordinateXYZM(10, 20, 30, 40))) }; for (Literal lit : literals) { diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java b/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java index e2611ddb281f..3d2419ffca04 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java @@ -25,11 +25,16 @@ import java.util.Arrays; import java.util.List; import java.util.UUID; +import org.apache.iceberg.Geography; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.GeometryFactory; public class TestMiscLiteralConversions { + private static final GeometryFactory FACTORY = new GeometryFactory(); + @Test public void testIdentityConversions() { List, Type>> pairs = @@ -48,7 +53,13 @@ public void testIdentityConversions() { Pair.of(Literal.of("abc"), Types.StringType.get()), Pair.of(Literal.of(UUID.randomUUID()), Types.UUIDType.get()), Pair.of(Literal.of(new byte[] {0, 1, 2}), Types.FixedType.ofLength(3)), - Pair.of(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BinaryType.get())); + Pair.of(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BinaryType.get()), + Pair.of( + Literal.of(FACTORY.toGeometry(new Envelope(1, 2, 10, 20))), + Types.GeometryType.get()), + Pair.of( + Literal.of(new Geography(FACTORY.toGeometry(new Envelope(1, 2, 10, 20)))), + Types.GeographyType.get())); for (Pair, Type> pair : pairs) { Literal lit = pair.first(); diff --git a/api/src/test/java/org/apache/iceberg/types/TestConversions.java b/api/src/test/java/org/apache/iceberg/types/TestConversions.java index e207cfd8d59a..6f74e426b798 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestConversions.java +++ b/api/src/test/java/org/apache/iceberg/types/TestConversions.java @@ -25,6 +25,7 @@ import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; import java.util.UUID; +import org.apache.iceberg.Geography; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.types.Types.BinaryType; import org.apache.iceberg.types.Types.BooleanType; @@ -41,6 +42,12 @@ import org.apache.iceberg.types.Types.TimestampType; import org.apache.iceberg.types.Types.UUIDType; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.CoordinateXYM; +import org.locationtech.jts.geom.CoordinateXYZM; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; public class TestConversions { @@ -191,6 +198,104 @@ public void testByteBufferConversions() { .isEqualTo(new byte[] {11}); } + @Test + public void testByteBufferConversionsForGeometryType() { + // geometry lower/upper boundaries are stored as 4 8-bytes floating point numbers in little + // endian. + // The 4 components are [x, y, optional z, optional m]. If z and m are not present, NaN is + // filled in. + GeometryFactory factory = new GeometryFactory(); + Geometry pointXY = factory.createPoint(new CoordinateXY(10, 20)); + assertConversion( + pointXY, + Types.GeometryType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, -8, 127, 0, 0, 0, 0, + 0, 0, -8, 127 + }); + pointXY = factory.createPoint(new Coordinate(10, 20)); + assertConversion( + pointXY, + Types.GeometryType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, -8, 127, 0, 0, 0, 0, + 0, 0, -8, 127 + }); + Geometry pointXYZ = factory.createPoint(new Coordinate(10, 20, 30)); + assertConversion( + pointXYZ, + Types.GeometryType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 62, 64, 0, 0, 0, 0, + 0, 0, -8, 127 + }); + Geometry pointXYM = factory.createPoint(new CoordinateXYM(10, 20, 30)); + assertConversion( + pointXYM, + Types.GeometryType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, -8, 127, 0, 0, 0, 0, + 0, 0, 62, 64 + }); + Geometry pointXYZM = factory.createPoint(new CoordinateXYZM(10, 20, 30, 40)); + assertConversion( + pointXYZM, + Types.GeometryType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 62, 64, 0, 0, 0, 0, + 0, 0, 68, 64 + }); + } + + @Test + public void testByteBufferConversionsForGeographyType() { + // geography lower/upper boundaries are stored as 4 8-bytes floating point numbers in little + // endian. This is the same as geometry lower/upper boundaries. + // The 4 components are [x, y, optional z, optional m]. If z and m are not present, NaN is + // filled in. + GeometryFactory factory = new GeometryFactory(); + Geography pointXY = new Geography(factory.createPoint(new CoordinateXY(10, 20))); + assertConversion( + pointXY, + Types.GeographyType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, -8, 127, 0, 0, 0, 0, + 0, 0, -8, 127 + }); + pointXY = new Geography(factory.createPoint(new Coordinate(10, 20))); + assertConversion( + pointXY, + Types.GeographyType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, -8, 127, 0, 0, 0, 0, + 0, 0, -8, 127 + }); + Geography pointXYZ = new Geography(factory.createPoint(new Coordinate(10, 20, 30))); + assertConversion( + pointXYZ, + Types.GeographyType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 62, 64, 0, 0, 0, 0, + 0, 0, -8, 127 + }); + Geography pointXYM = new Geography(factory.createPoint(new CoordinateXYM(10, 20, 30))); + assertConversion( + pointXYM, + Types.GeographyType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, -8, 127, 0, 0, 0, 0, + 0, 0, 62, 64 + }); + Geography pointXYZM = new Geography(factory.createPoint(new CoordinateXYZM(10, 20, 30, 40))); + assertConversion( + pointXYZM, + Types.GeographyType.get(), + new byte[] { + 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 62, 64, 0, 0, 0, 0, + 0, 0, 68, 64 + }); + } + private void assertConversion(T value, Type type, byte[] expectedBinary) { ByteBuffer byteBuffer = Conversions.toByteBuffer(type, value); assertThat(byteBuffer.array()).isEqualTo(expectedBinary); diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index debb9c9dc1d6..e6984b2a20aa 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Stream; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Type.PrimitiveType; import org.junit.jupiter.api.Test; @@ -53,7 +54,12 @@ public class TestReadabilityChecks { Types.BinaryType.get(), Types.DecimalType.of(9, 2), Types.DecimalType.of(11, 2), - Types.DecimalType.of(9, 3) + Types.DecimalType.of(9, 3), + Types.GeometryType.get(), + Types.GeometryType.of("srid:3857"), + Types.GeographyType.get(), + Types.GeographyType.of("srid:4269"), + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY), }; @Test diff --git a/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java b/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java index 790f59587c59..7abc90448244 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java @@ -22,6 +22,7 @@ import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.TestHelpers; import org.junit.jupiter.api.Test; @@ -63,7 +64,12 @@ public void testEqualTypes() throws Exception { Types.DecimalType.of(9, 3), Types.DecimalType.of(11, 0), Types.FixedType.ofLength(4), - Types.FixedType.ofLength(34) + Types.FixedType.ofLength(34), + Types.GeometryType.get(), + Types.GeometryType.of("srid:3857"), + Types.GeographyType.get(), + Types.GeographyType.of("srid:4269"), + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY), }; for (Type type : equalityPrimitives) { diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypes.java b/api/src/test/java/org/apache/iceberg/types/TestTypes.java index f8ee4e2ccbd4..90e02baa90dc 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypes.java @@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import org.apache.iceberg.Geography; import org.junit.jupiter.api.Test; public class TestTypes { @@ -90,5 +91,50 @@ public void testNestedFieldBuilderIdCheck() { assertThatExceptionOfType(NullPointerException.class) .isThrownBy(() -> required("field").ofType(Types.StringType.get()).build()) .withMessage("Id cannot be null"); + + assertThat(Types.fromPrimitiveString("geometry")).isEqualTo(Types.GeometryType.get()); + assertThat(Types.fromPrimitiveString("geometry()")).isEqualTo(Types.GeometryType.get()); + assertThat(Types.fromPrimitiveString("geometry(srid:3857)")) + .isEqualTo(Types.GeometryType.of("srid:3857")); + assertThat(Types.fromPrimitiveString("geometry( srid:3857 )")) + .isEqualTo(Types.GeometryType.of("srid:3857")); + + assertThat(Types.fromPrimitiveString("geography")).isEqualTo(Types.GeographyType.get()); + assertThat(Types.fromPrimitiveString("geography()")).isEqualTo(Types.GeographyType.get()); + assertThat(Types.fromPrimitiveString("geography(srid:4269)")) + .isEqualTo(Types.GeographyType.of("srid:4269")); + assertThat(Types.fromPrimitiveString("geography(srid:4269, spherical)")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.SPHERICAL)); + assertThat(Types.fromPrimitiveString("geography(srid:4269, vincenty)")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.VINCENTY)); + assertThat(Types.fromPrimitiveString("geography(srid:4269, thomas)")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.THOMAS)); + assertThat(Types.fromPrimitiveString("geography(srid:4269, andoyer)")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.ANDOYER)); + assertThat(Types.fromPrimitiveString("geography(srid:4269, karney)")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY)); + + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> Types.fromPrimitiveString("geography(srid:4269, BadAlgorithm)")) + .withMessageContaining("Invalid edge interpolation algorithm name") + .withMessageContaining("BadAlgorithm"); + + // Test geography type with various spacing + assertThat(Types.fromPrimitiveString("geography( srid:4269 )")) + .isEqualTo(Types.GeographyType.of("srid:4269")); + assertThat(Types.fromPrimitiveString("geography( srid:4269 , spherical )")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.SPHERICAL)); + assertThat(Types.fromPrimitiveString("geography(srid:4269,vincenty)")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.VINCENTY)); + assertThat(Types.fromPrimitiveString("geography( srid:4269 , karney )")) + .isEqualTo( + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY)); } } diff --git a/api/src/test/java/org/apache/iceberg/util/RandomUtil.java b/api/src/test/java/org/apache/iceberg/util/RandomUtil.java index 5f3fc370e4c0..ed396ad50c40 100644 --- a/api/src/test/java/org/apache/iceberg/util/RandomUtil.java +++ b/api/src/test/java/org/apache/iceberg/util/RandomUtil.java @@ -20,6 +20,7 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -31,11 +32,16 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; public class RandomUtil { private RandomUtil() {} + private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); + private static boolean negate(int num) { return num % 2 == 1; } @@ -151,6 +157,13 @@ public static Object generatePrimitive(Type.PrimitiveType primitive, Random rand BigDecimal bigDecimal = new BigDecimal(unscaled, type.scale()); return negate(choice) ? bigDecimal.negate() : bigDecimal; + case GEOMETRY: + case GEOGRAPHY: + Point geometry = + GEOMETRY_FACTORY.createPoint(new Coordinate(random.nextDouble(), random.nextDouble())); + byte[] wkb = GeometryUtil.toWKB(geometry); + return ByteBuffer.wrap(wkb); + default: throw new IllegalArgumentException( "Cannot generate random value for unknown type: " + primitive); diff --git a/api/src/test/java/org/apache/iceberg/util/TestGeometryUtil.java b/api/src/test/java/org/apache/iceberg/util/TestGeometryUtil.java new file mode 100644 index 000000000000..6dfb8d9e5023 --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/util/TestGeometryUtil.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.CoordinateXYM; +import org.locationtech.jts.geom.CoordinateXYZM; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; + +public class TestGeometryUtil { + private static final GeometryFactory FACTORY = new GeometryFactory(); + + @Test + public void testToWKB() { + Geometry geometry = FACTORY.createPoint(new Coordinate(1.0, 2.0)); + byte[] wkb = GeometryUtil.toWKB(geometry); + Geometry readGeometry = GeometryUtil.fromWKB(wkb); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isNaN(); + assertThat(coordinate.getM()).isNaN(); + } + + @Test + public void testXYToWKB() { + Geometry geometry = FACTORY.createPoint(new CoordinateXY(1.0, 2.0)); + byte[] wkb = GeometryUtil.toWKB(geometry); + Geometry readGeometry = GeometryUtil.fromWKB(wkb); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isNaN(); + assertThat(coordinate.getM()).isNaN(); + } + + @Test + public void testXYZToWKB() { + Geometry geometry = FACTORY.createPoint(new Coordinate(1.0, 2.0, 3.0)); + byte[] wkb = GeometryUtil.toWKB(geometry); + Geometry readGeometry = GeometryUtil.fromWKB(wkb); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isEqualTo(3.0); + assertThat(coordinate.getM()).isNaN(); + } + + @Test + @Disabled("https://github.com/locationtech/jts/issues/733") + public void testXYMToWKB() { + Geometry geometry = FACTORY.createPoint(new CoordinateXYM(1.0, 2.0, 3.0)); + byte[] wkb = GeometryUtil.toWKB(geometry); + Geometry readGeometry = GeometryUtil.fromWKB(wkb); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isNaN(); + assertThat(coordinate.getM()).isEqualTo(3.0); + } + + @Test + public void testXYZMToWKB() { + Geometry geometry = FACTORY.createPoint(new CoordinateXYZM(1.0, 2.0, 3.0, 4.0)); + byte[] wkb = GeometryUtil.toWKB(geometry); + Geometry readGeometry = GeometryUtil.fromWKB(wkb); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isEqualTo(3.0); + assertThat(coordinate.getM()).isEqualTo(4.0); + } + + @Test + public void testToWKT() { + Geometry geometry = FACTORY.createPoint(new Coordinate(1.0, 2.0)); + String wkt = GeometryUtil.toWKT(geometry); + Geometry readGeometry = GeometryUtil.fromWKT(wkt); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isNaN(); + assertThat(coordinate.getM()).isNaN(); + } + + @Test + public void testXYToWKT() { + Geometry geometry = FACTORY.createPoint(new CoordinateXY(1.0, 2.0)); + String wkt = GeometryUtil.toWKT(geometry); + Geometry readGeometry = GeometryUtil.fromWKT(wkt); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isNaN(); + assertThat(coordinate.getM()).isNaN(); + } + + @Test + public void testXYZToWKT() { + Geometry geometry = FACTORY.createPoint(new Coordinate(1.0, 2.0, 3.0)); + String wkt = GeometryUtil.toWKT(geometry); + Geometry readGeometry = GeometryUtil.fromWKT(wkt); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isEqualTo(3.0); + assertThat(coordinate.getM()).isNaN(); + } + + @Test + public void testXYMToWKT() { + Geometry geometry = FACTORY.createPoint(new CoordinateXYM(1.0, 2.0, 3.0)); + String wkt = GeometryUtil.toWKT(geometry); + Geometry readGeometry = GeometryUtil.fromWKT(wkt); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isNaN(); + assertThat(coordinate.getM()).isEqualTo(3.0); + } + + @Test + public void testXYZMToWKT() { + Geometry geometry = FACTORY.createPoint(new CoordinateXYZM(1.0, 2.0, 3.0, 4.0)); + String wkt = GeometryUtil.toWKT(geometry); + Geometry readGeometry = GeometryUtil.fromWKT(wkt); + assertThat(geometry).isEqualTo(readGeometry); + Coordinate coordinate = readGeometry.getCoordinate(); + assertThat(coordinate.getZ()).isEqualTo(3.0); + assertThat(coordinate.getM()).isEqualTo(4.0); + } + + @Test + public void testBoundMayIntersects() { + GeometryFactory factory = new GeometryFactory(); + + // Test regular case (not crossing anti-meridian) + Point lowerBound = factory.createPoint(new Coordinate(0, 0)); + Point upperBound = factory.createPoint(new Coordinate(10, 10)); + + // Envelope completely inside bound + Geometry geom = factory.toGeometry(new Envelope(2, 8, 2, 8)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isTrue(); + + // Envelope partially overlapping bound + geom = factory.toGeometry(new Envelope(5, 15, 5, 15)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isTrue(); + + // Envelope completely outside bound + geom = factory.toGeometry(new Envelope(15, 20, 15, 20)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isFalse(); + + // Test anti-meridian crossing case + lowerBound = factory.createPoint(new Coordinate(170, 0)); + upperBound = factory.createPoint(new Coordinate(-170, 10)); + + // Envelope in the western part of the bound + geom = factory.toGeometry(new Envelope(172, 178, 2, 8)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isTrue(); + + // Envelope in the eastern part of the bound + geom = factory.toGeometry(new Envelope(-178, -172, 2, 8)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isTrue(); + + // Envelope crossing the anti-meridian within the bound + geom = factory.toGeometry(new Envelope(175, -175, 2, 8)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isTrue(); + + // Envelope outside the bound (latitude) + geom = factory.toGeometry(new Envelope(172, 178, 12, 15)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isFalse(); + + // Envelope outside the bound (longitude) + geom = factory.toGeometry(new Envelope(160, 165, 2, 8)); + assertThat(GeometryUtil.boundMayIntersects(lowerBound, upperBound, geom)).isFalse(); + } + + @Test + public void testBoundMayCovers() { + GeometryFactory factory = new GeometryFactory(); + + // Test regular case (not crossing anti-meridian) + Point lowerBound = factory.createPoint(new Coordinate(0, 0)); + Point upperBound = factory.createPoint(new Coordinate(10, 10)); + + // Envelope completely inside bound + Geometry geom = factory.toGeometry(new Envelope(2, 8, 2, 8)); + assertThat(GeometryUtil.boundMayCovers(lowerBound, upperBound, geom)).isTrue(); + + // Envelope partially inside bound + geom = factory.toGeometry(new Envelope(5, 15, 5, 15)); + assertThat(GeometryUtil.boundMayCovers(lowerBound, upperBound, geom)).isFalse(); + + // Envelope completely outside bound + geom = factory.toGeometry(new Envelope(15, 20, 15, 20)); + assertThat(GeometryUtil.boundMayCovers(lowerBound, upperBound, geom)).isFalse(); + + // Test anti-meridian crossing case + lowerBound = factory.createPoint(new Coordinate(170, 0)); + upperBound = factory.createPoint(new Coordinate(-170, 10)); + + // Envelope in the western part of the bound + geom = factory.toGeometry(new Envelope(172, 178, 2, 8)); + assertThat(GeometryUtil.boundMayCovers(lowerBound, upperBound, geom)).isTrue(); + + // Envelope in the eastern part of the bound + geom = factory.toGeometry(new Envelope(-178, -172, 2, 8)); + assertThat(GeometryUtil.boundMayCovers(lowerBound, upperBound, geom)).isTrue(); + + // Envelope partially outside the bound (latitude) + geom = factory.toGeometry(new Envelope(172, 178, -2, 12)); + assertThat(GeometryUtil.boundMayCovers(lowerBound, upperBound, geom)).isFalse(); + + // Envelope outside the bound (longitude) + geom = factory.toGeometry(new Envelope(160, 165, 2, 8)); + assertThat(GeometryUtil.boundMayCovers(lowerBound, upperBound, geom)).isFalse(); + } + + @Test + public void testBoundMustBeCoveredBy() { + GeometryFactory factory = new GeometryFactory(); + + // Test regular case (not crossing anti-meridian) + Point lowerBound = factory.createPoint(new Coordinate(2, 2)); + Point upperBound = factory.createPoint(new Coordinate(8, 8)); + + // Envelope completely covering the bound + Geometry geom = factory.toGeometry(new Envelope(0, 10, 0, 10)); + assertThat(GeometryUtil.boundMustBeCoveredBy(lowerBound, upperBound, geom)).isTrue(); + + // Envelope partially covering the bound + geom = factory.toGeometry(new Envelope(3, 10, 0, 10)); + assertThat(GeometryUtil.boundMustBeCoveredBy(lowerBound, upperBound, geom)).isFalse(); + + // Envelope not covering the bound + geom = factory.toGeometry(new Envelope(0, 5, 0, 5)); + assertThat(GeometryUtil.boundMustBeCoveredBy(lowerBound, upperBound, geom)).isFalse(); + + // Test anti-meridian crossing case - should always return false + lowerBound = factory.createPoint(new Coordinate(170, 0)); + upperBound = factory.createPoint(new Coordinate(-170, 10)); + + // Large envelope covering the entire region + geom = factory.toGeometry(new Envelope(160, -160, -10, 20)); + assertThat(GeometryUtil.boundMustBeCoveredBy(lowerBound, upperBound, geom)).isFalse(); + + // Envelope exactly matching the bound coordinates + geom = factory.toGeometry(new Envelope(170, -170, 0, 10)); + assertThat(GeometryUtil.boundMustBeCoveredBy(lowerBound, upperBound, geom)).isFalse(); + } +} diff --git a/build.gradle b/build.gradle index 099192abaf84..65751da8e570 100644 --- a/build.gradle +++ b/build.gradle @@ -292,6 +292,7 @@ project(':iceberg-api') { dependencies { implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api libs.jts.core compileOnly libs.errorprone.annotations compileOnly libs.findbugs.jsr305 testImplementation libs.avro.avro diff --git a/core/src/main/java/org/apache/iceberg/SchemaParser.java b/core/src/main/java/org/apache/iceberg/SchemaParser.java index d7c756795711..f49c09fee19e 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaParser.java +++ b/core/src/main/java/org/apache/iceberg/SchemaParser.java @@ -44,6 +44,8 @@ private SchemaParser() {} private static final String STRUCT = "struct"; private static final String LIST = "list"; private static final String MAP = "map"; + private static final String GEOMETRY = "geometry"; + private static final String GEOGRAPHY = "geography"; private static final String FIELDS = "fields"; private static final String ELEMENT = "element"; private static final String KEY = "key"; @@ -141,7 +143,22 @@ static void toJson(Types.MapType map, JsonGenerator generator) throws IOExceptio } static void toJson(Type.PrimitiveType primitive, JsonGenerator generator) throws IOException { - generator.writeString(primitive.toString()); + if (primitive.typeId() == Type.TypeID.GEOMETRY) { + Types.GeometryType geometryType = (Types.GeometryType) primitive; + generator.writeStartObject(); + generator.writeStringField("type", "geometry"); + generator.writeStringField("crs", geometryType.crs()); + generator.writeEndObject(); + } else if (primitive.typeId() == Type.TypeID.GEOGRAPHY) { + Types.GeographyType geographyType = (Types.GeographyType) primitive; + generator.writeStartObject(); + generator.writeStringField("type", "geography"); + generator.writeStringField("crs", geographyType.crs()); + generator.writeStringField("algorithm", geographyType.algorithm().name()); + generator.writeEndObject(); + } else { + generator.writeString(primitive.toString()); + } } static void toJson(Type type, JsonGenerator generator) throws IOException { @@ -192,6 +209,10 @@ private static Type typeFromJson(JsonNode json) { return listFromJson(json); } else if (MAP.equals(type)) { return mapFromJson(json); + } else if (GEOMETRY.equals(type)) { + return geometryFromJson(json); + } else if (GEOGRAPHY.equals(type)) { + return geographyFromJson(json); } } } @@ -277,6 +298,17 @@ private static Types.MapType mapFromJson(JsonNode json) { } } + private static Types.GeometryType geometryFromJson(JsonNode json) { + String crs = JsonUtil.getStringOrNull("crs", json); + return Types.GeometryType.of(crs); + } + + private static Types.GeographyType geographyFromJson(JsonNode json) { + String crs = JsonUtil.getStringOrNull("crs", json); + String algorithm = JsonUtil.getStringOrNull("algorithm", json); + return Types.GeographyType.of(crs, algorithm); + } + public static Schema fromJson(JsonNode json) { Type type = typeFromJson(json); Preconditions.checkArgument( diff --git a/core/src/main/java/org/apache/iceberg/SingleValueParser.java b/core/src/main/java/org/apache/iceberg/SingleValueParser.java index 3de6a0bcc663..1e3998e75a81 100644 --- a/core/src/main/java/org/apache/iceberg/SingleValueParser.java +++ b/core/src/main/java/org/apache/iceberg/SingleValueParser.java @@ -38,7 +38,9 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ByteBuffers; import org.apache.iceberg.util.DateTimeUtil; +import org.apache.iceberg.util.GeometryUtil; import org.apache.iceberg.util.JsonUtil; +import org.locationtech.jts.geom.Geometry; public class SingleValueParser { private SingleValueParser() {} @@ -160,6 +162,20 @@ public static Object fromJson(Type type, JsonNode defaultValue) { byte[] binaryBytes = BaseEncoding.base16().decode(defaultValue.textValue().toUpperCase(Locale.ROOT)); return ByteBuffer.wrap(binaryBytes); + case GEOMETRY: + case GEOGRAPHY: + Preconditions.checkArgument( + defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); + try { + Geometry geom = GeometryUtil.fromWKT(defaultValue.textValue()); + if (type.typeId() == Type.TypeID.GEOGRAPHY) { + return new Geography(geom); + } + return geom; + } catch (Exception e) { + throw new IllegalArgumentException( + String.format("Cannot parse default as a %s value: %s", type, defaultValue), e); + } case LIST: return listFromJson(type, defaultValue); case MAP: @@ -335,6 +351,16 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato generator.writeString(decimalValue.toString()); } break; + case GEOMETRY: + Preconditions.checkArgument( + defaultValue instanceof Geometry, "Invalid default %s value: %s", type, defaultValue); + generator.writeString(GeometryUtil.toWKT((Geometry) defaultValue)); + break; + case GEOGRAPHY: + Preconditions.checkArgument( + defaultValue instanceof Geography, "Invalid default %s value: %s", type, defaultValue); + generator.writeString(GeometryUtil.toWKT(((Geography) defaultValue).geometry())); + break; case LIST: Preconditions.checkArgument( defaultValue instanceof List, "Invalid default %s value: %s", type, defaultValue); diff --git a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java index 05ce4e618662..da686e64cc04 100644 --- a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java +++ b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java @@ -230,6 +230,8 @@ public Schema primitive(Type.PrimitiveType primitive) { primitiveSchema = Schema.createFixed("fixed_" + fixed.length(), null, null, fixed.length()); break; case BINARY: + case GEOMETRY: + case GEOGRAPHY: primitiveSchema = BINARY_SCHEMA; break; case DECIMAL: diff --git a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java index 9bb5b7d05f0b..e197b8a3514c 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java +++ b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java @@ -29,6 +29,7 @@ import java.util.UUID; import java.util.function.Function; import java.util.function.Supplier; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.SingleValueParser; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -37,6 +38,7 @@ import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; +import org.locationtech.jts.geom.Geometry; public class ExpressionParser { @@ -202,6 +204,7 @@ public Void predicate(UnboundPredicate pred) { }); } + @SuppressWarnings("checkstyle:CyclomaticComplexity") private void unboundLiteral(Object object) throws IOException { // this handles each type supported in Literals.from if (object instanceof Integer) { @@ -226,6 +229,10 @@ private void unboundLiteral(Object object) throws IOException { BigDecimal decimal = (BigDecimal) object; SingleValueParser.toJson( Types.DecimalType.of(decimal.precision(), decimal.scale()), decimal, gen); + } else if (object instanceof Geometry) { + SingleValueParser.toJson(Types.GeometryType.get(), object, gen); + } else if (object instanceof Geography) { + SingleValueParser.toJson(Types.GeographyType.get(), object, gen); } } @@ -347,6 +354,10 @@ private static UnboundPredicate predicateFromJson( case NOT_EQ: case STARTS_WITH: case NOT_STARTS_WITH: + case ST_INTERSECTS: + case ST_COVERS: + case ST_DISJOINT: + case ST_NOT_COVERS: // literal predicates Preconditions.checkArgument( node.has(VALUE), "Cannot parse %s predicate: missing value", op); diff --git a/core/src/test/java/org/apache/iceberg/TestGeospatialTable.java b/core/src/test/java/org/apache/iceberg/TestGeospatialTable.java new file mode 100644 index 000000000000..504afc73ad74 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestGeospatialTable.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.Map; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.hadoop.HadoopTableTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +public class TestGeospatialTable extends HadoopTableTestBase { + + @Test + public void testCreateGeospatialTable() throws IOException { + Schema schema = + new Schema( + required(3, "id", Types.IntegerType.get(), "unique ID"), + required(4, "data", Types.StringType.get()), + required(5, "geom", Types.GeometryType.of("srid:3857"), "geometry column"), + required( + 6, + "geog", + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY), + "geography column")); + + TableIdentifier identifier = TableIdentifier.of("a", "geos_t1"); + try (HadoopCatalog catalog = hadoopCatalog()) { + Map properties = ImmutableMap.of(TableProperties.FORMAT_VERSION, "3"); + catalog.createTable(identifier, schema, PartitionSpec.unpartitioned(), properties); + Table table = catalog.loadTable(identifier); + + Types.NestedField geomField = table.schema().findField("geom"); + assertThat(geomField.type().typeId()).isEqualTo(Type.TypeID.GEOMETRY); + Types.GeometryType geomType = (Types.GeometryType) geomField.type(); + assertThat(geomType.crs()).isEqualTo("srid:3857"); + + Types.NestedField geogField = table.schema().findField("geog"); + assertThat(geogField.type().typeId()).isEqualTo(Type.TypeID.GEOGRAPHY); + Types.GeographyType geogType = (Types.GeographyType) geogField.type(); + assertThat(geogType.crs()).isEqualTo("srid:4269"); + assertThat(geogType.algorithm()).isEqualTo(Geography.EdgeInterpolationAlgorithm.KARNEY); + assertThat(catalog.dropTable(identifier)).isTrue(); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaParser.java b/core/src/test/java/org/apache/iceberg/TestSchemaParser.java index acdba85adf55..11e0ec95b49f 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaParser.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaParser.java @@ -34,6 +34,7 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.DateTimeUtil; +import org.apache.iceberg.util.GeometryUtil; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -105,7 +106,11 @@ private static Stream primitiveTypesAndDefaults() { Types.FixedType.ofLength(4), Literal.of(ByteBuffer.wrap(new byte[] {0x0a, 0x0b, 0x0c, 0x0d}))), Arguments.of(Types.BinaryType.get(), Literal.of(ByteBuffer.wrap(new byte[] {0x0a, 0x0b}))), - Arguments.of(Types.DecimalType.of(9, 2), Literal.of(new BigDecimal("12.34")))); + Arguments.of(Types.DecimalType.of(9, 2), Literal.of(new BigDecimal("12.34"))), + Arguments.of(Types.GeometryType.get(), Literal.of(GeometryUtil.fromWKT("POINT (1 2)"))), + Arguments.of( + Types.GeographyType.get(), + Literal.of(new Geography(GeometryUtil.fromWKT("POINT (1 2)"))))); } @ParameterizedTest @@ -137,4 +142,21 @@ public void testVariantType() throws IOException { writeAndValidate(schema); } + + @Test + public void testSpatialType() throws IOException { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "geom0", Types.GeometryType.get()), + Types.NestedField.optional(3, "geom1", Types.GeometryType.of("srid:3857")), + Types.NestedField.optional(4, "geog0", Types.GeographyType.get()), + Types.NestedField.optional(5, "geog1", Types.GeographyType.of("srid:4269")), + Types.NestedField.optional( + 6, + "geog2", + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY))); + + writeAndValidate(schema); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index d1591f80d836..411e6eee1e15 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -364,7 +364,11 @@ public void testUpdateFailure() { Types.FixedType.ofLength(4), Types.DecimalType.of(9, 2), Types.DecimalType.of(9, 3), - Types.DecimalType.of(18, 2)); + Types.DecimalType.of(18, 2), + Types.GeometryType.get(), + Types.GeometryType.of("srid:3857"), + Types.GeographyType.get(), + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY)); for (Type.PrimitiveType fromType : primitives) { for (Type.PrimitiveType toType : primitives) { diff --git a/core/src/test/java/org/apache/iceberg/TestSingleValueParser.java b/core/src/test/java/org/apache/iceberg/TestSingleValueParser.java index cc1578b0e081..3c5b3a9fc7d8 100644 --- a/core/src/test/java/org/apache/iceberg/TestSingleValueParser.java +++ b/core/src/test/java/org/apache/iceberg/TestSingleValueParser.java @@ -53,6 +53,16 @@ public void testValidDefaults() throws IOException { {Types.DecimalType.of(9, 4), "\"123.4500\""}, {Types.DecimalType.of(9, 0), "\"2\""}, {Types.DecimalType.of(9, -20), "\"2E+20\""}, + {Types.GeometryType.get(), "\"POINT (1 2)\""}, + {Types.GeometryType.get(), "\"POINT Z(1 2 3)\""}, + {Types.GeometryType.get(), "\"POINT M(1 2 3)\""}, + {Types.GeometryType.get(), "\"POINT ZM(1 2 3 4)\""}, + {Types.GeometryType.of("srid:3857"), "\"POINT (1 2)\""}, + {Types.GeographyType.get(), "\"POINT ZM(1 2 3 4)\""}, + { + Types.GeographyType.of("srid:4269", Geography.EdgeInterpolationAlgorithm.KARNEY), + "\"POINT ZM(1 2 3 4)\"" + }, {Types.ListType.ofOptional(1, Types.IntegerType.get()), "[1, 2, 3]"}, { Types.MapType.ofOptional(2, 3, Types.IntegerType.get(), Types.StringType.get()), @@ -157,6 +167,15 @@ public void testInvalidTimestamptz() { .hasMessageStartingWith("Cannot parse default as a timestamptz value"); } + @Test + public void testInvalidGeometry() { + Type expectedType = Types.GeometryType.get(); + String defaultJson = "\"POINT (1 2 3 4 5 6)\""; + assertThatThrownBy(() -> defaultValueParseAndUnParseRoundTrip(expectedType, defaultJson)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageMatching("Cannot parse default as a geometry.* value.*"); + } + // serialize to json and deserialize back should return the same result private static String defaultValueParseAndUnParseRoundTrip(Type type, String defaultValue) { Object javaDefaultValue = SingleValueParser.fromJson(type, defaultValue); diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 1cd60fbbd177..c4b07ce6720e 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -1825,6 +1825,43 @@ public void testConstructV3Metadata() { 3); } + @Test + public void testV3GeometryTypeSupport() { + Schema v3SchemaGeom = + new Schema( + Types.NestedField.required(3, "id", Types.LongType.get()), + Types.NestedField.required(4, "geom", Types.GeometryType.get())); + Schema v3SchemaGeog = + new Schema( + Types.NestedField.required(3, "id", Types.LongType.get()), + Types.NestedField.required(4, "geog", Types.GeographyType.get())); + + for (Schema schema : ImmutableList.of(v3SchemaGeom, v3SchemaGeog)) { + for (int unsupportedFormatVersion : ImmutableList.of(1, 2)) { + assertThatThrownBy( + () -> + TableMetadata.newTableMetadata( + schema, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + TEST_LOCATION, + ImmutableMap.of(), + unsupportedFormatVersion)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("not supported until v3"); + } + + // should be allowed in v3 + TableMetadata.newTableMetadata( + schema, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + TEST_LOCATION, + ImmutableMap.of(), + 3); + } + } + @Test public void onlyMetadataLocationIsUpdatedWithoutTimestampAndMetadataLogEntry() { String uuid = "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b"; diff --git a/core/src/test/java/org/apache/iceberg/expressions/TestExpressionParser.java b/core/src/test/java/org/apache/iceberg/expressions/TestExpressionParser.java index 43e2f13b55c9..754151616ebf 100644 --- a/core/src/test/java/org/apache/iceberg/expressions/TestExpressionParser.java +++ b/core/src/test/java/org/apache/iceberg/expressions/TestExpressionParser.java @@ -27,8 +27,10 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.UUID; +import org.apache.iceberg.Geography; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.GeometryUtil; import org.junit.jupiter.api.Test; public class TestExpressionParser { @@ -51,7 +53,9 @@ public class TestExpressionParser { required(114, "dec_9_0", Types.DecimalType.of(9, 0)), required(115, "dec_11_2", Types.DecimalType.of(11, 2)), required(116, "dec_38_10", Types.DecimalType.of(38, 10)), // maximum precision - required(117, "time", Types.TimeType.get())); + required(117, "time", Types.TimeType.get()), + required(118, "geom", Types.GeometryType.get()), + required(119, "geog", Types.GeographyType.get())); private static final Schema SCHEMA = new Schema(SUPPORTED_PRIMITIVES.fields()); @Test @@ -94,7 +98,15 @@ public void testSimpleExpressions() { Expressions.or( Expressions.greaterThan(Expressions.day("ts"), "2022-08-14"), Expressions.equal("date", "2022-08-14")), - Expressions.not(Expressions.in("l", 1, 2, 3, 4)) + Expressions.not(Expressions.in("l", 1, 2, 3, 4)), + Expressions.stIntersects("geom", GeometryUtil.fromWKT("POINT (1 2)")), + Expressions.stCovers("geom", GeometryUtil.fromWKT("POINT (1 2)")), + Expressions.stDisjoint("geom", GeometryUtil.fromWKT("POINT (1 2)")), + Expressions.stNotCovers("geom", GeometryUtil.fromWKT("POINT (1 2)")), + Expressions.stIntersects("geog", new Geography(GeometryUtil.fromWKT("POINT (1 2)"))), + Expressions.stCovers("geog", new Geography(GeometryUtil.fromWKT("POINT (1 2)"))), + Expressions.stDisjoint("geog", new Geography(GeometryUtil.fromWKT("POINT (1 2)"))), + Expressions.stNotCovers("geog", new Geography(GeometryUtil.fromWKT("POINT (1 2)"))) }; for (Expression expr : expressions) { @@ -544,4 +556,33 @@ public void testNegativeScaleDecimalLiteral() { assertThat(ExpressionParser.toJson(ExpressionParser.fromJson(expected), true)) .isEqualTo(expected); } + + @Test + public void testSpatialPredicate() { + String expected = + "{\n" + + " \"type\" : \"st-intersects\",\n" + + " \"term\" : \"column-name\",\n" + + " \"value\" : \"POINT (1 2)\"\n" + + "}"; + + Expression expression = + Expressions.stIntersects("column-name", GeometryUtil.fromWKT("POINT (1 2)")); + assertThat(ExpressionParser.toJson(expression, true)).isEqualTo(expected); + assertThat(ExpressionParser.toJson(ExpressionParser.fromJson(expected), true)) + .isEqualTo(expected); + + expected = + "{\n" + + " \"type\" : \"st-covers\",\n" + + " \"term\" : \"column-name\",\n" + + " \"value\" : \"LINESTRING (1 2, 3 4, 5 6)\"\n" + + "}"; + expression = + Expressions.stCovers( + "column-name", new Geography(GeometryUtil.fromWKT("LINESTRING (1 2, 3 4, 5 6)"))); + assertThat(ExpressionParser.toJson(expression, true)).isEqualTo(expected); + assertThat(ExpressionParser.toJson(ExpressionParser.fromJson(expected), true)) + .isEqualTo(expected); + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5cb71c6cfba3..bf6e84d70abb 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -62,6 +62,7 @@ jakarta-servlet-api = "6.1.0" jaxb-api = "2.3.1" jaxb-runtime = "2.3.9" jetty = "11.0.24" +jts-core = "1.20.0" junit = "5.11.4" junit-platform = "1.11.4" kafka = "3.9.0" @@ -147,6 +148,7 @@ jackson214-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = jackson215-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson215" } jaxb-api = { module = "javax.xml.bind:jaxb-api", version.ref = "jaxb-api" } jaxb-runtime = { module = "org.glassfish.jaxb:jaxb-runtime", version.ref = "jaxb-runtime" } +jts-core = { module = "org.locationtech.jts:jts-core", version.ref = "jts-core" } kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka" } kafka-connect-api = { module = "org.apache.kafka:connect-api", version.ref = "kafka" } kafka-connect-json = { module = "org.apache.kafka:connect-json", version.ref = "kafka" }