From febc2278a7504e63c214ca51ea7ac66a53503cbb Mon Sep 17 00:00:00 2001 From: jakobbraun Date: Thu, 13 Aug 2020 16:38:37 +0200 Subject: [PATCH] #369 Type mapping documentation + refactoring (#367) #369 Type mapping documentation #370 Cleanup of oracle dialect --- doc/dialects/oracle.md | 55 ++++++++++++------- .../oracle/OracleColumnMetadataReader.java | 15 ----- .../oracle/OracleSqlGenerationVisitor.java | 11 ++-- .../OracleColumnMetadataReaderTest.java | 26 +-------- .../dialects/oracle/OracleSqlDialectIT.java | 24 +++++++- .../OracleSqlGenerationVisitorTest.java | 22 +------- 6 files changed, 64 insertions(+), 89 deletions(-) diff --git a/doc/dialects/oracle.md b/doc/dialects/oracle.md index c1fceb53c..a5c9f55b0 100644 --- a/doc/dialects/oracle.md +++ b/doc/dialects/oracle.md @@ -133,31 +133,46 @@ CREATE VIRTUAL SCHEMA ORA_CONNECTION_NAME = 'ORA_CONNECTION'; ``` -## Supported capabilities +## Supported Capabilities The Oracle dialect does not support all capabilities. A complete list can be found in [OracleSqlDialect.getCapabilities()](../../src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlDialect.java). ## Type Mappings and Limitations -Oracle data types are mapped to their equivalents in Exasol. The following exceptions apply: - -- `NUMBER`, `NUMBER with precision > 36` and `LONG` are casted to `VARCHAR` to prevent a loss of precision. - - If you want to return a DECIMAL type for these types you can set the property ORACLE_CAST_NUMBER_TO_DECIMAL_WITH_PRECISION_AND_SCALE: - - `ORACLE_CAST_NUMBER_TO_DECIMAL_WITH_PRECISION_AND_SCALE='36,20'` - - This will cast NUMBER with precision > 36, NUMBER without precision and LONG to DECIMAL(36,20). - Keep in mind that this will yield errors if the data in the Oracle database does not fit into the specified DECIMAL type. - -- `DATE` is casted to `TIMESTAMP`. This data type is only supported for positive year values, i.e., years > 0001. -- `TIMESTAMP WITH [LOCAL] TIME ZONE` is casted to `TIMESTAMP`. -- `INTERVAL` is casted to `VARCHAR`. -- `CLOB`, `BLOB`, and `NCLOB` values are supported up to the maximum size of an Exasol [`VARCHAR`](https://docs.exasol.com/sql_references/data_types/datatypedetails.htm#StringDataType). - - By default an error will be thrown if you try to import a value that is larger. If you want to import from tables which contain bigger values, you need to truncate the values. - -- `RAW` and `LONG RAW` are not supported. +| Orcale Data Type | Supported | Converted Exasol Data Type | Comments | +| -------------------------------------------------------------------------------- | --------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| BINARY\_DOUBLE | ✓ | VARCHAR(2000000) | | +| BINARY\_FLOAT | ✓ | VARCHAR(2000000) | | +| BLOB | × | | | +| BFILE | × | | | +| CHAR \[(size)\] | ✓ | CHAR | | +| CLOB | × | | | +| DATE | ✓ | TIMESTAMP | This data type is only supported for positive year values, i.e., years > 0001 | +| FLOAT \[(p)\] | ✓ | DOUBLE | | +| INTERVAL DAY \[(day\_precision)\] TO SECOND \[(fractional\_seconds\_precision)\] | ✓ | VARCHAR(2000000) | | +| INTERVAL YEAR \[(year\_precision)\] TO MONTH | ✓ | VARCHAR(2000000) | | +| LONG | ✓ | VARCHAR(2000000) | Casted to VARCHAR to prevent a loss of precision. | +| LONG RAW | × | | | +| NCLOB | × | | | +| NCHAR\[(size)\] | ✓ | CHAR | | +| NUMBER \[ (p \[, s\]) \] | ✓ | NUMBER or VARCHAR(2000000) | NUMBER with precision > 36 are casted to VARCHAR to prevent a loss of precision. [*](#Mapping-of-number-types) | +| NVARCHAR2(size) | ✓ | VARCHAR | | +| RAW(size) | × | | | +| ROWID | × | | | +| TIMESTAMP \[(fractional\_seconds\_precision)\] | ✓ | TIMESTAMP | | +| TIMESTAMP \[(fractional\_seconds\_precision)\] WITH TIME ZONE | ✓ | TIMESTAMP | | +| UROWID \[(size)\] | × | | | +| VARCHAR2(size) | ✓ | VARCHAR | | + + + +### Mapping of Number Types: + +`NUMBER`, `NUMBER with precision > 36` and `LONG` are casted to `VARCHAR` to prevent a loss of precision. + +If you want to return a DECIMAL type for these types you can set the property ORACLE_CAST_NUMBER_TO_DECIMAL_WITH_PRECISION_AND_SCALE: +This will cast NUMBER with precision > 36, NUMBER without precision and LONG to DECIMAL(36,20). +Keep in mind that this will yield errors if the data in the Oracle database does not fit into the specified DECIMAL type. ## Testing information diff --git a/src/main/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReader.java b/src/main/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReader.java index 4424078ff..c4c0981ac 100644 --- a/src/main/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReader.java +++ b/src/main/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReader.java @@ -6,7 +6,6 @@ import java.sql.Types; import com.exasol.adapter.AdapterProperties; -import com.exasol.adapter.BinaryColumnHandling; import com.exasol.adapter.dialects.IdentifierConverter; import com.exasol.adapter.jdbc.BaseColumnMetadataReader; import com.exasol.adapter.jdbc.JdbcTypeDescription; @@ -20,7 +19,6 @@ public class OracleColumnMetadataReader extends BaseColumnMetadataReader { private static final int ORACLE_TIMESTAMP_WITH_TIME_ZONE = -102; private static final int ORACLE_BINARY_FLOAT = 100; private static final int ORACLE_BINARY_DOUBLE = 101; - private static final int ORACLE_CLOB = Types.OTHER; private static final int INTERVAL_DAY_TO_SECOND = -104; private static final int INTERVAL_YEAR_TO_MONTH = -103; static final int ORACLE_MAGIC_NUMBER_SCALE = -127; @@ -46,16 +44,11 @@ public DataType mapJdbcType(final JdbcTypeDescription jdbcTypeDescription) { case ORACLE_TIMESTAMP_WITH_TIME_ZONE: case ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE: return DataType.createTimestamp(false); - case Types.NCLOB: - case ORACLE_CLOB: case INTERVAL_YEAR_TO_MONTH: case INTERVAL_DAY_TO_SECOND: case ORACLE_BINARY_FLOAT: case ORACLE_BINARY_DOUBLE: - case Types.ROWID: return DataType.createMaximumSizeVarChar(DataType.ExaCharset.UTF8); - case Types.BLOB: - return mapBlobType(); default: return super.mapJdbcType(jdbcTypeDescription); } @@ -92,12 +85,4 @@ private DataType getOracleNumberTargetType() { return DataType.createMaximumSizeVarChar(DataType.ExaCharset.UTF8); } } - - private DataType mapBlobType() { - if (this.properties.getBinaryColumnHandling() == BinaryColumnHandling.IGNORE) { - return DataType.createUnsupported(); - } else { - return DataType.createMaximumSizeVarChar(DataType.ExaCharset.UTF8); - } - } } \ No newline at end of file diff --git a/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitor.java b/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitor.java index 0fd7b6a81..78878b0f9 100644 --- a/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitor.java +++ b/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitor.java @@ -4,11 +4,12 @@ import static com.exasol.adapter.sql.ScalarFunction.*; import java.util.*; -import java.util.logging.Logger; import com.exasol.adapter.AdapterException; import com.exasol.adapter.dialects.*; -import com.exasol.adapter.metadata.*; +import com.exasol.adapter.metadata.ColumnMetadata; +import com.exasol.adapter.metadata.DataType; +import com.exasol.adapter.metadata.TableMetadata; import com.exasol.adapter.sql.*; /** @@ -18,7 +19,7 @@ public class OracleSqlGenerationVisitor extends SqlGenerationVisitor { private boolean requiresSelectListAliasesForLimit = false; private static final String TIMESTAMP_FORMAT = "'YYYY-MM-DD HH24:MI:SS.FF3'"; private static final List TYPE_NAMES_REQUIRING_CAST = List.of("TIMESTAMP", "INTERVAL", "BINARY_FLOAT", - "BINARY_DOUBLE", "ROWID", "UROWID", "BLOB"); + "BINARY_DOUBLE"); private final Set aggregateFunctionsCast = EnumSet.noneOf(AggregateFunction.class); private final Set scalarFunctionsCast = EnumSet.noneOf(ScalarFunction.class); @@ -273,10 +274,6 @@ private String getProjectionString(final SqlColumn column, final String projecti + ")"; } else if (typeName.equals("NUMBER")) { return getNumberProjectionString(column, projectionString, (OracleSqlDialect) dialect); - } else if (typeName.equals("ROWID") || typeName.equals("UROWID")) { - return "ROWIDTOCHAR(" + projectionString + ")"; - } else if (typeName.equals("BLOB")) { - return "UTL_RAW.CAST_TO_VARCHAR2(UTL_ENCODE.BASE64_DECODE(" + projectionString + "))"; } else { return projectionString; } diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReaderTest.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReaderTest.java index cfba39d87..7e40fe2c7 100644 --- a/src/test/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReaderTest.java +++ b/src/test/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReaderTest.java @@ -1,11 +1,10 @@ package com.exasol.adapter.dialects.oracle; import static com.exasol.adapter.metadata.DataType.createMaximumSizeVarChar; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import java.sql.Types; -import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.BeforeEach; @@ -15,7 +14,6 @@ import com.exasol.adapter.dialects.BaseIdentifierConverter; import com.exasol.adapter.jdbc.JdbcTypeDescription; import com.exasol.adapter.metadata.DataType; -import com.exasol.adapter.metadata.DataType.ExaCharset; class OracleColumnMetadataReaderTest { private OracleColumnMetadataReader columnMetadataReader; @@ -44,21 +42,6 @@ void testMapColumnTypeWithMagicScale() { equalTo(createMaximumSizeVarChar(DataType.ExaCharset.UTF8))); } - @Test - void testMapBlobMappedToUnsupportedTypeByDefault() { - final JdbcTypeDescription typeDescription = new JdbcTypeDescription(Types.BLOB, 0, 0, 0, null); - assertThat(this.columnMetadataReader.mapJdbcType(typeDescription), equalTo(DataType.createUnsupported())); - } - - @Test - void testMapBlobMappedToMaximumSizeVarCharIfBase64EncodingEnabled() { - final Map rawProperties = new HashMap<>(); - rawProperties.put("BINARY_COLUMN_HANDLING", "ENCODE_BASE64"); - final JdbcTypeDescription typeDescription = new JdbcTypeDescription(Types.BLOB, 0, 0, 0, null); - assertThat(createParameterizedColumnMetadataReader(rawProperties).mapJdbcType(typeDescription), - equalTo(createMaximumSizeVarChar(ExaCharset.UTF8))); - } - @Test void testMapNumericColumnTypeWithMaximumDecimalPrecision() { final int precision = DataType.MAX_EXASOL_DECIMAL_PRECISION; @@ -77,13 +60,6 @@ void testMapColumnTypeWithZeroPrecision() { equalTo(DataType.createDecimal(DataType.MAX_EXASOL_DECIMAL_PRECISION, scale))); } - @Test - void testMapRowId() { - final JdbcTypeDescription typeDescription = new JdbcTypeDescription(Types.ROWID, 0, 0, 0, null); - assertThat(this.columnMetadataReader.mapJdbcType(typeDescription), - equalTo(createMaximumSizeVarChar(DataType.ExaCharset.UTF8))); - } - private OracleColumnMetadataReader createParameterizedColumnMetadataReader( final Map rawProperties) { return new OracleColumnMetadataReader(null, new AdapterProperties(rawProperties), diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java index fdac3e3d8..31b2b2cb9 100644 --- a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java @@ -3,8 +3,7 @@ import static com.exasol.adapter.dialects.IntegrationTestConstants.*; import static com.exasol.dbbuilder.dialects.exasol.AdapterScript.Language.JAVA; import static com.exasol.matcher.ResultSetMatcher.matchesResultSet; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertAll; @@ -34,7 +33,10 @@ import com.exasol.bucketfs.BucketAccessException; import com.exasol.containers.ExasolContainer; import com.exasol.containers.ExasolContainerConstants; -import com.exasol.dbbuilder.dialects.exasol.*; +import com.exasol.dbbuilder.dialects.exasol.AdapterScript; +import com.exasol.dbbuilder.dialects.exasol.ConnectionDefinition; +import com.exasol.dbbuilder.dialects.exasol.ExasolObjectFactory; +import com.exasol.dbbuilder.dialects.exasol.ExasolSchema; /** * How to run `OracleSqlDialectIT`: See the documentation statementExasol.execute(query)); + assertThat(exception.getMessage(), startsWith("object " + columnName + " not found")); + } + @ParameterizedTest @CsvSource(value = { // "VIRTUAL_SCHEMA_JDBC | C5 | VARCHAR(2000000) UTF8 | 123456789012345678901234567890123456", // diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitorTest.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitorTest.java index bd29fb544..35d3a7e78 100644 --- a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitorTest.java +++ b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitorTest.java @@ -12,7 +12,9 @@ import java.math.BigDecimal; import java.sql.Connection; import java.sql.SQLException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -147,24 +149,6 @@ void testVisitSqlSelectListSelectStarWithTimestamp() throws AdapterException { "TO_TIMESTAMP(TO_CHAR(\"test_column\", 'YYYY-MM-DD HH24:MI:SS.FF3'), 'YYYY-MM-DD HH24:MI:SS.FF3')")); } - @CsvSource({ "ROWID", "UROWID" }) - @ParameterizedTest - void testVisitSqlSelectListSelectStarNumberCastRowidToChar(final String dataType) throws AdapterException { - final SqlSelectList selectList = createSqlSelectStarListWithOneColumn( - "{\"jdbcDataType\":2, \"typeName\":\"" + dataType + "\"}", - DataType.createVarChar(50, DataType.ExaCharset.UTF8), "test_column"); - assertThat(this.visitor.visit(selectList), equalTo("ROWIDTOCHAR(\"test_column\")")); - } - - @Test - void testVisitSqlSelectListSelectStarNumberCastBlobToChar() throws AdapterException { - final SqlSelectList selectList = createSqlSelectStarListWithOneColumn( - "{\"jdbcDataType\":2, \"typeName\":\"BLOB\"}", DataType.createVarChar(50, DataType.ExaCharset.UTF8), - "test_column"); - assertThat(this.visitor.visit(selectList), - equalTo("UTL_RAW.CAST_TO_VARCHAR2(UTL_ENCODE.BASE64_DECODE(\"test_column\"))")); - } - @Test void testVisitSqlSelectListSelectStarNumberCastToDecimal() throws AdapterException { final SqlSelectList selectList = SqlSelectList.createSelectStarSelectList();