diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java b/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java index e00e2241a76d0..75049d52111db 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java @@ -58,6 +58,8 @@ public final class StandardTypes public static final String VARCHAR_ENUM = "VarcharEnum"; public static final String DISTINCT_TYPE = "DistinctType"; public static final String UUID = "uuid"; + public static final String FUNCTION = "function"; + public static final String UNKNOWN = "unknown"; private StandardTypes() {} @@ -70,6 +72,7 @@ private StandardTypes() {} ROW, ARRAY, MAP, + FUNCTION, QDIGEST, TDIGEST, KLL_SKETCH, diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/TypeManager.java b/presto-common/src/main/java/com/facebook/presto/common/type/TypeManager.java index 4eafde00c4d34..b904f20335a41 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/type/TypeManager.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/TypeManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.common.type; import java.util.List; +import java.util.Map; public interface TypeManager { @@ -28,4 +29,13 @@ public interface TypeManager Type getParameterizedType(String baseTypeName, List typeParameters); boolean canCoerce(Type actualType, Type expectedType); + + default Type instantiateParametricType(TypeSignature typeSignature) + { + throw new UnsupportedOperationException(); + } + + List getTypes(); + + Map getParametricTypes(); } diff --git a/presto-common/src/test/java/com/facebook/presto/common/type/TestingTypeManager.java b/presto-common/src/test/java/com/facebook/presto/common/type/TestingTypeManager.java index 5e5042e87be3b..9616b566bfb76 100644 --- a/presto-common/src/test/java/com/facebook/presto/common/type/TestingTypeManager.java +++ b/presto-common/src/test/java/com/facebook/presto/common/type/TestingTypeManager.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.Map; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -54,8 +55,15 @@ public boolean canCoerce(Type actualType, Type expectedType) throw new UnsupportedOperationException(); } - private List getTypes() + @Override + public List getTypes() { return ImmutableList.of(BOOLEAN, INTEGER, BIGINT, DOUBLE, VARCHAR, VARBINARY, TIMESTAMP, DATE, ID, HYPER_LOG_LOG); } + + @Override + public Map getParametricTypes() + { + throw new UnsupportedOperationException(); + } } diff --git a/presto-hudi/src/test/java/com/facebook/presto/hudi/TestingTypeManager.java b/presto-hudi/src/test/java/com/facebook/presto/hudi/TestingTypeManager.java index 516f1805b40d8..7f87c1264355d 100644 --- a/presto-hudi/src/test/java/com/facebook/presto/hudi/TestingTypeManager.java +++ b/presto-hudi/src/test/java/com/facebook/presto/hudi/TestingTypeManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.hudi; +import com.facebook.presto.common.type.ParametricType; import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.common.type.TypeSignature; @@ -21,6 +22,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.Map; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -58,8 +60,15 @@ public boolean canCoerce(Type actualType, Type expectedType) throw new UnsupportedOperationException(); } - private List getTypes() + @Override + public List getTypes() { return ImmutableList.of(BOOLEAN, INTEGER, BIGINT, DOUBLE, VARCHAR, VARBINARY, TIMESTAMP, DATE, HYPER_LOG_LOG); } + + @Override + public Map getParametricTypes() + { + throw new UnsupportedOperationException(); + } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergFileWriter.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergFileWriter.java index 734d47172ded8..298913a4fa496 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergFileWriter.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergFileWriter.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.Page; import com.facebook.presto.common.type.BooleanType; import com.facebook.presto.common.type.DoubleType; +import com.facebook.presto.common.type.ParametricType; import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.common.type.TypeSignature; @@ -50,6 +51,7 @@ import java.io.File; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; @@ -178,9 +180,16 @@ public boolean canCoerce(Type actualType, Type expectedType) throw new UnsupportedOperationException(); } - private List getTypes() + @Override + public List getTypes() { return ImmutableList.of(BooleanType.BOOLEAN, INTEGER, BIGINT, DoubleType.DOUBLE, VARCHAR, VARBINARY, TIMESTAMP, DATE, HYPER_LOG_LOG); } + + @Override + public Map getParametricTypes() + { + throw new UnsupportedOperationException(); + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/jdbc/TypesJdbcTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/jdbc/TypesJdbcTable.java index 5e330237a5faa..e345aff8c0e10 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/jdbc/TypesJdbcTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/jdbc/TypesJdbcTable.java @@ -87,7 +87,7 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect for (Type type : functionAndTypeManager.getTypes()) { addTypeRow(table, type); } - addParametricTypeRows(table, functionAndTypeManager.getParametricTypes()); + addParametricTypeRows(table, functionAndTypeManager.getParametricTypes().values()); return table.build().cursor(); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java index c419822e91405..b5d835c2e4d2b 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java @@ -453,6 +453,7 @@ import static com.facebook.presto.operator.window.AggregateWindowFunction.supplier; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_MISSING; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.UNKNOWN_TYPE; import static com.facebook.presto.spi.function.FunctionImplementationType.JAVA; import static com.facebook.presto.spi.function.FunctionImplementationType.SQL; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; @@ -531,7 +532,7 @@ @ThreadSafe public class BuiltInTypeAndFunctionNamespaceManager - implements FunctionNamespaceManager + implements FunctionNamespaceManager, TypeManager { public static final CatalogSchemaName JAVA_BUILTIN_NAMESPACE = new CatalogSchemaName("presto", "default"); public static final String ID = "builtin"; @@ -1261,14 +1262,15 @@ public ScalarFunctionImplementation getScalarFunctionImplementation(Signature si } } - public Optional getType(TypeSignature typeSignature) + @Override + public Type getType(TypeSignature typeSignature) { Type type = types.get(typeSignature); if (type != null) { - return Optional.of(type); + return type; } try { - return Optional.ofNullable(parametricTypeCache.getUnchecked(new ExactTypeSignature(typeSignature))); + return parametricTypeCache.getUnchecked(new ExactTypeSignature(typeSignature)); } catch (UncheckedExecutionException e) { throwIfUnchecked(e.getCause()); @@ -1276,6 +1278,18 @@ public Optional getType(TypeSignature typeSignature) } } + @Override + public Type getParameterizedType(String baseTypeName, List typeParameters) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean canCoerce(Type actualType, Type expectedType) + { + throw new UnsupportedOperationException(); + } + public List getTypes() { return ImmutableList.copyOf(types.values()); @@ -1295,14 +1309,22 @@ public void addParametricType(ParametricType parametricType) parametricTypes.putIfAbsent(name, parametricType); } - public Collection getParametricTypes() + @Override + public Map getParametricTypes() + { + return parametricTypes; + } + + private Type instantiateParametricType(ExactTypeSignature exactTypeSignature) { - return parametricTypes.values(); + return instantiateParametricType(exactTypeSignature.getTypeSignature(), functionAndTypeManager, parametricTypes); } - private Type instantiateParametricType(ExactTypeSignature exactSignature) + public Type instantiateParametricType( + TypeSignature signature, + FunctionAndTypeManager functionAndTypeManager, + Map parametricTypes) { - TypeSignature signature = exactSignature.getTypeSignature(); List parameters = new ArrayList<>(); for (TypeSignatureParameter parameter : signature.getParameters()) { @@ -1312,7 +1334,7 @@ private Type instantiateParametricType(ExactTypeSignature exactSignature) ParametricType parametricType = parametricTypes.get(signature.getBase().toLowerCase(Locale.ENGLISH)); if (parametricType == null) { - throw new IllegalArgumentException("Unknown type " + signature); + throw new PrestoException(UNKNOWN_TYPE, "Unknown type " + signature); } if (parametricType instanceof MapParametricType) { diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java index 456bdef5dba80..e3e9491534eab 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java @@ -52,6 +52,8 @@ import com.facebook.presto.spi.function.SqlFunctionId; import com.facebook.presto.spi.function.SqlFunctionSupplier; import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.type.TypeManagerContext; +import com.facebook.presto.spi.type.TypeManagerFactory; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionAndTypeResolver; import com.facebook.presto.sql.analyzer.FunctionsConfig; @@ -128,8 +130,10 @@ public class FunctionAndTypeManager private final BuiltInTypeAndFunctionNamespaceManager builtInTypeAndFunctionNamespaceManager; private final FunctionInvokerProvider functionInvokerProvider; private final Map functionNamespaceManagerFactories = new ConcurrentHashMap<>(); + private final Map typeManagerFactories = new ConcurrentHashMap<>(); private final HandleResolver handleResolver; private final Map> functionNamespaceManagers = new ConcurrentHashMap<>(); + private final Map typeManagers = new ConcurrentHashMap<>(); private final FunctionSignatureMatcher functionSignatureMatcher; private final TypeCoercer typeCoercer; private final LoadingCache functionCache; @@ -151,6 +155,7 @@ public FunctionAndTypeManager( this.builtInTypeAndFunctionNamespaceManager = new BuiltInTypeAndFunctionNamespaceManager(blockEncodingSerde, functionsConfig, types, this); this.functionNamespaceManagers.put(JAVA_BUILTIN_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); this.functionInvokerProvider = new FunctionInvokerProvider(this); + this.typeManagers.put(JAVA_BUILTIN_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); this.handleResolver = requireNonNull(handleResolver, "handleResolver is null"); // TODO: Provide a more encapsulated way for TransactionManager to register FunctionNamespaceManager transactionManager.registerFunctionNamespaceManager(JAVA_BUILTIN_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); @@ -234,6 +239,24 @@ public SqlFunctionSupplier getSpecializedFunctionKey(Signature signature) return FunctionAndTypeManager.this.getSpecializedFunctionKey(signature); } + @Override + public Type instantiateParametricType(TypeSignature typeSignature) + { + return FunctionAndTypeManager.this.instantiateParametricType(typeSignature); + } + + @Override + public List getTypes() + { + return FunctionAndTypeManager.this.getTypes(); + } + + @Override + public Map getParametricTypes() + { + return FunctionAndTypeManager.this.getParametricTypes(); + } + @Override public Collection listBuiltInFunctions() { @@ -319,6 +342,15 @@ public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) return functionNamespaceManager.get().getFunctionMetadata(functionHandle); } + @Override + public Type instantiateParametricType(TypeSignature typeSignature) + { + return builtInTypeAndFunctionNamespaceManager.instantiateParametricType( + typeSignature, + this, + getServingTypeManager().getParametricTypes()); + } + @Override public Type getType(TypeSignature signature) { @@ -327,12 +359,12 @@ public Type getType(TypeSignature signature) if (signature.isDistinctType()) { return getDistinctType(signature.getParameters().get(0).getDistinctTypeInfo()); } - Optional type = builtInTypeAndFunctionNamespaceManager.getType(signature.getStandardTypeSignature()); - if (type.isPresent()) { + Type type = getServingTypeManager().getType(signature.getStandardTypeSignature()); + if (type != null) { if (signature.getTypeSignatureBase().hasTypeName()) { - return new TypeWithName(signature.getTypeSignatureBase().getTypeName(), type.get()); + return new TypeWithName(signature.getTypeSignatureBase().getTypeName(), type); } - return type.get(); + return type; } } @@ -364,6 +396,32 @@ public void addFunctionNamespaceFactory(FunctionNamespaceManagerFactory factory) handleResolver.addFunctionNamespace(factory.getName(), factory.getHandleResolver()); } + public void loadTypeManager(String typeManagerName) + { + requireNonNull(typeManagerName, "typeManagerName is null"); + TypeManagerFactory factory = typeManagerFactories.get(typeManagerName); + checkState(factory != null, "No factory for type manager %s", typeManagerName); + TypeManager typeManager = factory.create(new TypeManagerContext(this)); + + if (typeManagers.putIfAbsent(typeManagerName, typeManager) != null) { + throw new IllegalArgumentException(format("Type manager [%s] is already registered", typeManager)); + } + } + + public void loadTypeManagers() + { + for (String typeManagerName : typeManagerFactories.keySet()) { + loadTypeManager(typeManagerName); + } + } + + public void addTypeManagerFactory(TypeManagerFactory factory) + { + if (typeManagerFactories.putIfAbsent(factory.getName(), factory) != null) { + throw new IllegalArgumentException(format("Type manager '%s' is already registered", factory.getName())); + } + } + public void registerBuiltInFunctions(List functions) { builtInTypeAndFunctionNamespaceManager.registerBuiltInFunctions(functions); @@ -509,9 +567,9 @@ public List getTypes() return builtInTypeAndFunctionNamespaceManager.getTypes(); } - public Collection getParametricTypes() + public Map getParametricTypes() { - return ImmutableList.copyOf(builtInTypeAndFunctionNamespaceManager.getParametricTypes()); + return builtInTypeAndFunctionNamespaceManager.getParametricTypes(); } public Optional getCommonSuperType(Type firstType, Type secondType) @@ -835,6 +893,19 @@ public CatalogSchemaName configureDefaultNamespace(String defaultNamespacePrefix return new CatalogSchemaName(catalogSchemaNameString[0], catalogSchemaNameString[1]); } + private TypeManager getServingTypeManager() + { + // If the NativeSidecarPlugin is loaded, typeManagerFactories is non-empty and holds an instance of NativeTypeManagerFactory. + // Check if the nativeTypeManager exists in typeManagers else use BuiltInTypeAndFunctionNamespaceManager. + for (Map.Entry entry : typeManagerFactories.entrySet()) { + if (!typeManagers.containsKey(entry.getKey())) { + throw new PrestoException(GENERIC_USER_ERROR, format("Type manager not loaded for factory: %s", entry.getKey())); + } + return typeManagers.get(entry.getKey()); + } + return builtInTypeAndFunctionNamespaceManager; + } + private static class FunctionResolutionCacheKey { private final QualifiedObjectName functionName; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java index 42fdb77c3bdb5..58204ed851258 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java @@ -21,6 +21,8 @@ import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.common.type.TypeSignatureParameter; import com.facebook.presto.common.type.TypeWithName; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.function.LongVariableConstraint; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.TypeVariableConstraint; @@ -531,8 +533,17 @@ private static List expandVarargFormalTypeSignature(List getWorkerSessionPropertyPr return ImmutableList.of(new NativeSystemSessionPropertyProviderFactory()); } + @Override + public Iterable getTypeManagerFactories() + { + return ImmutableList.of(new NativeTypeManagerFactory()); + } + @Override public Iterable getPlanCheckerProviderFactories() { diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java new file mode 100644 index 0000000000000..5d3a3d7518fb9 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java @@ -0,0 +1,303 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.typemanager; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.type.DistinctTypeInfo; +import com.facebook.presto.common.type.ParametricType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.common.type.TypeSignatureParameter; +import com.facebook.presto.common.type.VarcharType; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.UncheckedExecutionException; +import com.google.inject.Inject; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; + +import static com.facebook.presto.common.type.StandardTypes.ARRAY; +import static com.facebook.presto.common.type.StandardTypes.BIGINT; +import static com.facebook.presto.common.type.StandardTypes.BOOLEAN; +import static com.facebook.presto.common.type.StandardTypes.DATE; +import static com.facebook.presto.common.type.StandardTypes.DECIMAL; +import static com.facebook.presto.common.type.StandardTypes.DOUBLE; +import static com.facebook.presto.common.type.StandardTypes.FUNCTION; +import static com.facebook.presto.common.type.StandardTypes.HYPER_LOG_LOG; +import static com.facebook.presto.common.type.StandardTypes.INTEGER; +import static com.facebook.presto.common.type.StandardTypes.INTERVAL_DAY_TO_SECOND; +import static com.facebook.presto.common.type.StandardTypes.INTERVAL_YEAR_TO_MONTH; +import static com.facebook.presto.common.type.StandardTypes.IPADDRESS; +import static com.facebook.presto.common.type.StandardTypes.IPPREFIX; +import static com.facebook.presto.common.type.StandardTypes.JSON; +import static com.facebook.presto.common.type.StandardTypes.MAP; +import static com.facebook.presto.common.type.StandardTypes.REAL; +import static com.facebook.presto.common.type.StandardTypes.ROW; +import static com.facebook.presto.common.type.StandardTypes.SMALLINT; +import static com.facebook.presto.common.type.StandardTypes.TIMESTAMP; +import static com.facebook.presto.common.type.StandardTypes.TIMESTAMP_WITH_TIME_ZONE; +import static com.facebook.presto.common.type.StandardTypes.TINYINT; +import static com.facebook.presto.common.type.StandardTypes.UNKNOWN; +import static com.facebook.presto.common.type.StandardTypes.UUID; +import static com.facebook.presto.common.type.StandardTypes.VARBINARY; +import static com.facebook.presto.common.type.StandardTypes.VARCHAR; +import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Throwables.throwIfUnchecked; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public class NativeTypeManager + implements TypeManager +{ + private static final Logger log = Logger.get(NativeTypeManager.class); + + private static final Set NATIVE_ENGINE_SUPPORTED_TYPES = + ImmutableSet.of( + BIGINT, + REAL, + VARBINARY, + TIMESTAMP, + TINYINT, + BOOLEAN, + DATE, + INTEGER, + DOUBLE, + SMALLINT, + HYPER_LOG_LOG, + JSON, + TIMESTAMP_WITH_TIME_ZONE, + UUID, + IPADDRESS, + IPPREFIX, + INTERVAL_DAY_TO_SECOND, + INTERVAL_YEAR_TO_MONTH, + VARCHAR, + UNKNOWN); + + private static final Set NATIVE_ENGINE_SUPPORTED_PARAMETRIC_TYPES = + ImmutableSet.of( + ARRAY, + DECIMAL, + MAP, + ROW, + FUNCTION); + + private final TypeManager typeManager; + private final LoadingCache parametricTypeCache; + private final ConcurrentMap types = new ConcurrentHashMap<>(); + private final ConcurrentMap parametricTypes = new ConcurrentHashMap<>(); + + @Inject + public NativeTypeManager(TypeManager typeManager, NativeTypeManagerConfig config) + { + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + parametricTypeCache = CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(config.getTypesCacheExpiration().toMillis(), MILLISECONDS) + .build(CacheLoader.from(this::instantiateParametricType)); + addAllTypes( + filterSupportedTypes(NATIVE_ENGINE_SUPPORTED_TYPES, typeManager.getTypes(), Type::getDisplayName), + filterSupportedTypes( + NATIVE_ENGINE_SUPPORTED_PARAMETRIC_TYPES, + new ArrayList<>(typeManager.getParametricTypes().values()), + ParametricType::getName)); + } + + @Override + public Type getType(TypeSignature typeSignature) + { + // Todo: Fix this hack, native execution does not support parameterized char/varchar type signatures. + if (typeSignature.getBase().equals("char") || typeSignature.getBase().equals("varchar")) { + typeSignature = createUnboundedVarcharType().getTypeSignature(); + } + Type type = types.get(typeSignature); + if (type != null) { + return type; + } + try { + return parametricTypeCache.getUnchecked(new ExactTypeSignature(typeSignature)); + } + catch (UncheckedExecutionException e) { + throwIfUnchecked(e.getCause()); + throw new RuntimeException(e.getCause()); + } + } + + @Override + public Map getParametricTypes() + { + return parametricTypes; + } + + @Override + public List getTypes() + { + return new ArrayList<>(types.values()); + } + + @Override + public Type getParameterizedType(String baseTypeName, List typeParameters) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean canCoerce(Type actualType, Type expectedType) + { + throw new UnsupportedOperationException(); + } + + private void addAllTypes(List typesList, List parametricTypesList) + { + for (Type type : typesList) { + addType(type); + } + // todo: Fix this hack + // Native engine does not support parameterized varchar, and varchar isn't in the lists of types returned from the engine + addType(VarcharType.VARCHAR); + + for (ParametricType type : parametricTypesList) { + addParametricType(type); + } + } + + private Type instantiateParametricType(ExactTypeSignature exactTypeSignature) + { + return typeManager.instantiateParametricType(exactTypeSignature.getTypeSignature()); + } + + private void addType(Type type) + { + requireNonNull(type, "type is null"); + Type existingType = types.putIfAbsent(type.getTypeSignature(), type); + checkState(existingType == null || existingType.equals(type), "Type %s is already registered", type); + } + + private void addParametricType(ParametricType parametricType) + { + String name = parametricType.getName().toLowerCase(Locale.ENGLISH); + checkArgument(!parametricTypes.containsKey(name), "Parametric type already registered: %s", name); + parametricTypes.putIfAbsent(name, parametricType); + } + + private static List filterSupportedTypes( + Set actualTypes, + List types, + Function typeSignatureExtractor) + { + return types.stream() + .filter(type -> actualTypes.contains(typeSignatureExtractor.apply(type))) + .collect(toImmutableList()); + } + + /** + * TypeSignature but has overridden equals(). Here, we compare exact signature of any underlying distinct + * types. Some distinct types may have extra information on their lazily loaded parents, and same parent + * information is compared in equals(). This is needed to cache types in parametricTypesCache. + */ + private static class ExactTypeSignature + { + private final TypeSignature typeSignature; + + public ExactTypeSignature(TypeSignature typeSignature) + { + this.typeSignature = typeSignature; + } + + public TypeSignature getTypeSignature() + { + return typeSignature; + } + + @Override + public int hashCode() + { + return Objects.hash(typeSignature); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ExactTypeSignature other = (ExactTypeSignature) o; + return equals(typeSignature, other.typeSignature); + } + + private static boolean equals(TypeSignature left, TypeSignature right) + { + if (!left.equals(right)) { + return false; + } + + if (left.isDistinctType() && right.isDistinctType()) { + return equals(left.getDistinctTypeInfo(), right.getDistinctTypeInfo()); + } + int index = 0; + for (TypeSignatureParameter leftParameter : left.getParameters()) { + TypeSignatureParameter rightParameter = right.getParameters().get(index++); + if (!leftParameter.getKind().equals(rightParameter.getKind())) { + return false; + } + + switch (leftParameter.getKind()) { + case TYPE: + if (!equals(leftParameter.getTypeSignature(), rightParameter.getTypeSignature())) { + return false; + } + break; + case NAMED_TYPE: + if (!equals(leftParameter.getNamedTypeSignature().getTypeSignature(), rightParameter.getNamedTypeSignature().getTypeSignature())) { + return false; + } + break; + case DISTINCT_TYPE: + if (!equals(leftParameter.getDistinctTypeInfo(), rightParameter.getDistinctTypeInfo())) { + return false; + } + break; + } + } + return true; + } + + private static boolean equals(DistinctTypeInfo left, DistinctTypeInfo right) + { + return Objects.equals(left.getName(), right.getName()) && + Objects.equals(left.getBaseType(), right.getBaseType()) && + Objects.equals(left.isOrderable(), right.isOrderable()) && + Objects.equals(left.getTopMostAncestor(), right.getTopMostAncestor()) && + Objects.equals(left.getOtherAncestors(), right.getOtherAncestors()); + } + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerConfig.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerConfig.java new file mode 100644 index 0000000000000..8f1be9a48fb1b --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerConfig.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.typemanager; + +import com.facebook.airlift.configuration.Config; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import static java.util.concurrent.TimeUnit.MINUTES; + +public class NativeTypeManagerConfig +{ + private Duration typesCacheExpiration = new Duration(5, MINUTES); + + @MinDuration("0ns") + public Duration getTypesCacheExpiration() + { + return typesCacheExpiration; + } + + @Config("types-cache-expiration") + public NativeTypeManagerConfig setTypesCacheExpiration(Duration typesCacheExpiration) + { + this.typesCacheExpiration = typesCacheExpiration; + return this; + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerFactory.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerFactory.java new file mode 100644 index 0000000000000..7a76d8500ef5c --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.typemanager; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.spi.type.TypeManagerContext; +import com.facebook.presto.spi.type.TypeManagerFactory; +import com.google.inject.Injector; + +public class NativeTypeManagerFactory + implements TypeManagerFactory +{ + public static final String NAME = "native"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public TypeManager create(TypeManagerContext context) + { + Bootstrap app = new Bootstrap( + new NativeTypeManagerModule(context.getTypeManager())); + + Injector injector = app + .doNotInitializeLogging() + .noStrictConfig() + .initialize(); + return injector.getInstance(NativeTypeManager.class); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerModule.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerModule.java new file mode 100644 index 0000000000000..1c28e75cadc45 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManagerModule.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.typemanager; + +import com.facebook.presto.common.type.TypeManager; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static java.util.Objects.requireNonNull; + +public class NativeTypeManagerModule + implements Module +{ + private final TypeManager typeManager; + + public NativeTypeManagerModule(TypeManager typeManager) + { + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + public void configure(Binder binder) + { + binder.bind(TypeManager.class).toInstance(typeManager); + binder.bind(NativeTypeManager.class).in(Scopes.SINGLETON); + binder.bind(NativeTypeManagerConfig.class).in(Scopes.SINGLETON); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java index 994d824195600..28a82820d394a 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java @@ -15,6 +15,7 @@ import com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManagerFactory; import com.facebook.presto.sidecar.sessionpropertyproviders.NativeSystemSessionPropertyProviderFactory; +import com.facebook.presto.sidecar.typemanager.NativeTypeManagerFactory; import com.facebook.presto.testing.QueryRunner; import com.google.common.collect.ImmutableMap; @@ -32,5 +33,6 @@ public static void setupNativeSidecarPlugin(QueryRunner queryRunner) ImmutableMap.of( "supported-function-languages", "CPP", "function-implementation-type", "CPP")); + queryRunner.loadTypeManager(NativeTypeManagerFactory.NAME); } } diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSystemSessionPropertyProviderConfig.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/sessionpropertyproviders/TestNativeSystemSessionPropertyProviderConfig.java similarity index 92% rename from presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSystemSessionPropertyProviderConfig.java rename to presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/sessionpropertyproviders/TestNativeSystemSessionPropertyProviderConfig.java index 6784928044478..7cb12c8480699 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSystemSessionPropertyProviderConfig.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/sessionpropertyproviders/TestNativeSystemSessionPropertyProviderConfig.java @@ -11,9 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sidecar; +package com.facebook.presto.sidecar.sessionpropertyproviders; -import com.facebook.presto.sidecar.sessionpropertyproviders.NativeSystemSessionPropertyProviderConfig; import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; import org.testng.annotations.Test; diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/typemanager/TestNativeSidecarPluginTypeManagerConfig.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/typemanager/TestNativeSidecarPluginTypeManagerConfig.java new file mode 100644 index 0000000000000..6fa31633a7b6e --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/typemanager/TestNativeSidecarPluginTypeManagerConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.typemanager; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.MINUTES; + +public class TestNativeSidecarPluginTypeManagerConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(NativeTypeManagerConfig.class) + .setTypesCacheExpiration((new Duration(5, MINUTES)))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("types-cache-expiration", "10m") + .build(); + NativeTypeManagerConfig expected = new NativeTypeManagerConfig() + .setTypesCacheExpiration(new Duration(10, MINUTES)); + assertFullMapping(properties, expected); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java index ca8ea46d58aa6..6b82a6ae2b64e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.plan.PlanCheckerProviderFactory; import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory; import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; +import com.facebook.presto.spi.type.TypeManagerFactory; import static java.util.Collections.emptyList; @@ -46,4 +47,9 @@ default Iterable getExpressionOptimizerFactories() { return emptyList(); } + + default Iterable getTypeManagerFactories() + { + return emptyList(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java index 09aa55cb24111..2c9175b74fff2 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java @@ -73,6 +73,7 @@ public enum StandardErrorCode INVALID_TYPE_DEFINITION(0x0000_002F, USER_ERROR), VIEW_NOT_FOUND(0x0000_0030, USER_ERROR), INVALID_LIMIT_CLAUSE(0x0000_0031, USER_ERROR), + UNKNOWN_TYPE(0x0000_0032, USER_ERROR), GENERIC_INTERNAL_ERROR(0x0001_0000, INTERNAL_ERROR), TOO_MANY_REQUESTS_FAILED(0x0001_0001, INTERNAL_ERROR, true), diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeManagerContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeManagerContext.java new file mode 100644 index 0000000000000..075c3ba2414d6 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeManagerContext.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.type; + +import com.facebook.presto.common.type.TypeManager; + +import static java.util.Objects.requireNonNull; + +public class TypeManagerContext +{ + private final TypeManager typeManager; + + public TypeManagerContext(TypeManager typeManager) + { + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + public TypeManager getTypeManager() + { + return typeManager; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeManagerFactory.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeManagerFactory.java new file mode 100644 index 0000000000000..7f82a527ea864 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeManagerFactory.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.type; + +import com.facebook.presto.common.type.TypeManager; + +public interface TypeManagerFactory +{ + String getName(); + + TypeManager create(TypeManagerContext context); +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index b3f25630cb078..e8006ee02268c 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -994,6 +994,14 @@ public void loadSessionPropertyProvider(String sessionPropertyProviderName) } } + @Override + public void loadTypeManager(String typeManagerName) + { + for (TestingPrestoServer server : servers) { + server.getMetadata().getFunctionAndTypeManager().loadTypeManager(typeManagerName); + } + } + private static void closeUnchecked(AutoCloseable closeable) { try {