diff --git a/README.md b/README.md index a17c601e7..f8ff3c02e 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ Running the Virtual Schema requires a Java Runtime version 9 or later. | [Java Hamcrest](http://hamcrest.org/JavaHamcrest/) | Checking for conditions in code via matchers | BSD License | | [JUnit](https://junit.org/junit5) | Unit testing framework | Eclipse Public License 1.0 | | [Mockito](http://site.mockito.org/) | Mocking framework | MIT License | +| [Microsoft JDBC Driver for SQL Server][sql-server-jdbc-driver] | JDBC driver for SQL Server database | MIT License | | [MySQL JDBC Driver][mysql-jdbc-driver] | JDBC driver for MySQL database | GNU GPL Version 2.0 | | [Oracle JDBC Driver][oracle-jdbc-driver] | JDBC driver for Oracle database | Oracle Technology Network License| | [PostgreSQL JDBC Driver][postgresql-jdbc-driver] | JDBC driver for PostgreSQL database | BSD-2-Clause License | @@ -166,6 +167,7 @@ Running the Virtual Schema requires a Java Runtime version 9 or later. [mysql-jdbc-driver]: https://dev.mysql.com/downloads/connector/j/ [oracle-jdbc-driver]: https://www.oracle.com/database/technologies/appdev/jdbc.html [postgresql-jdbc-driver]: https://jdbc.postgresql.org/ +[sql-server-jdbc-driver]: https://github.com/microsoft/mssql-jdbc [sonatype-oss-index-maven-plugin]: https://sonatype.github.io/ossindex-maven/maven-plugin/ [test-bd-builder]: https://github.com/exasol/test-db-builder-java [versions-maven-plugin]: https://www.mojohaus.org/versions-maven-plugin/ diff --git a/doc/changes/changes-4.0.3.md b/doc/changes/changes-4.0.3.md index a12a9ffa2..7a0af3b53 100644 --- a/doc/changes/changes-4.0.3.md +++ b/doc/changes/changes-4.0.3.md @@ -1,12 +1,22 @@ -# Exasol Virtual Schemas 4.0.3, released 2020-08-?? +# Exasol Virtual Schemas 4.0.3, released 2020-08-14 + +## Bugs + +* #321: Fixed SQL Server bug. Added a function COUNT_BIG instead of COUNT. +* #364: Fixed Oracle dialect bug with invalid Decimal precision (DECIMAL(0,0)). + +## Refactoring + +* #283: Tested SQL Server with new JDBC driver. Added integration test, updated documentation. +* #370: Cleaned up Oracle dialect. ## Documentation -* #317: Work on reducing redundancy between user_guid.md and docs.exasol.com +* #317: Worked on reducing redundancy between user_guid.md and docs.exasol.com * #354: Added datatypes mapping info to the postgres documentation. * #355: Updated general deployment guide. * #359: Replaced links to products with links to dialect in the README's feature list. -* #364: Fixed Oracle dialect bug with invalid Decimal precision (DECIMAL(0,0)). +* #369: Added supported data types list to Oracle dialect. ## Dependency updates @@ -14,7 +24,7 @@ Click to expand * Added `org.junit.jupiter:junit-jupiter:5.6.2` -* Updated `com.exasol:virtual-schema-common-jdbc` from 5.0.2 to 5.0.3 +* Updated `com.exasol:virtual-schema-common-jdbc` from 5.0.2 to 5.0.4 * Updated `com.exasol:exasol-testcontainers` from 2.0.3 to 2.1.0 * Updated `mysql:mysql-connector-java` from 8.0.20 to 8.0.21 * Updated `org.apache.hbase:hbase-server` from 2.2.5 to 2.3.0 diff --git a/doc/dialects/athena.md b/doc/dialects/athena.md index 8a829cab1..12ae3d8ae 100644 --- a/doc/dialects/athena.md +++ b/doc/dialects/athena.md @@ -49,7 +49,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///AthenaJDBC42-.jar; / ; diff --git a/doc/dialects/aurora.md b/doc/dialects/aurora.md index aef155a23..69b163ce4 100644 --- a/doc/dialects/aurora.md +++ b/doc/dialects/aurora.md @@ -62,7 +62,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///postgresql-.jar; / ``` diff --git a/doc/dialects/bigquery.md b/doc/dialects/bigquery.md index 034a760b5..234231735 100644 --- a/doc/dialects/bigquery.md +++ b/doc/dialects/bigquery.md @@ -33,7 +33,7 @@ List all the JAR files from Magnitude Simba JDBC driver. ```sql CREATE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_BIGQUERY AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///GoogleBigQueryJDBC42.jar; ... ... diff --git a/doc/dialects/db2.md b/doc/dialects/db2.md index 8fea9ca15..9660e8bfd 100644 --- a/doc/dialects/db2.md +++ b/doc/dialects/db2.md @@ -56,7 +56,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///db2jcc4.jar; %jar /buckets///db2jcc_license_cu.jar; / @@ -68,7 +68,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///db2jcc4.jar; %jar /buckets///db2jcc_license_cu.jar; %jar /buckets///db2jcc_license_cisuz.jar; diff --git a/doc/dialects/hive.md b/doc/dialects/hive.md index a23ac60f6..0129ae7d2 100644 --- a/doc/dialects/hive.md +++ b/doc/dialects/hive.md @@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///jars/virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///jars/virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///jars/HiveJDBC41.jar; / ``` @@ -297,7 +297,7 @@ In Virtual Schema adapter: CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %jvmoption -Dsun.security.krb5.disableReferrals=true; %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///jars/virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///jars/virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///jars/HiveJDBC41.jar; / ``` diff --git a/doc/dialects/impala.md b/doc/dialects/impala.md index 4c97b0e5f..97b601a52 100644 --- a/doc/dialects/impala.md +++ b/doc/dialects/impala.md @@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///ImpalaJDBC41.jar; / ; diff --git a/doc/dialects/mysql.md b/doc/dialects/mysql.md index b435b91ab..47a5631d7 100644 --- a/doc/dialects/mysql.md +++ b/doc/dialects/mysql.md @@ -15,7 +15,7 @@ Now register the driver in EXAOperation: 1. Select JDBC driver file 1. Click "Upload" 1. Click "Add" -1. In dialog "Add EXACluster JDBC driver" configure the JDBC driver (see below) +1. In a dialog "Add EXACluster JDBC driver" configure the JDBC driver (see below) You need to specify the following settings when adding the JDBC driver via EXAOperation. @@ -51,7 +51,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_MYSQL AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///mysql-connector-java-.jar; / ; diff --git a/doc/dialects/oracle.md b/doc/dialects/oracle.md index a5c9f55b0..f4b440eb8 100644 --- a/doc/dialects/oracle.md +++ b/doc/dialects/oracle.md @@ -48,7 +48,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///ojdbc.jar; / ; diff --git a/doc/dialects/postgresql.md b/doc/dialects/postgresql.md index 6cd8acce6..eb9871456 100644 --- a/doc/dialects/postgresql.md +++ b/doc/dialects/postgresql.md @@ -25,7 +25,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///postgresql-.jar; / ``` diff --git a/doc/dialects/redshift.md b/doc/dialects/redshift.md index b13553529..a71660ca7 100644 --- a/doc/dialects/redshift.md +++ b/doc/dialects/redshift.md @@ -51,7 +51,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///RedshiftJDBC42-.jar; / ; diff --git a/doc/dialects/saphana.md b/doc/dialects/saphana.md index c4f7b40c2..caa87bea7 100644 --- a/doc/dialects/saphana.md +++ b/doc/dialects/saphana.md @@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///ngdbc-.jar; / ; diff --git a/doc/dialects/sql_server.md b/doc/dialects/sql_server.md index 7e3e86c82..815bb2904 100644 --- a/doc/dialects/sql_server.md +++ b/doc/dialects/sql_server.md @@ -2,9 +2,31 @@ [Microsoft SQL Server](https://www.microsoft.com/en-us/sql-server/sql-server-2017) is a Relational Database Management System (RDBMS) developed by Microsoft. -## Uploading the JDBC Driver to EXAOperation +## Registering the JDBC Driver in EXAOperation + +First download the [SQL Server JDBC driver](https://github.com/microsoft/mssql-jdbc/releases). +We recommend using a `jre8` driver. + +Now register the driver in EXAOperation: + +1. Click "Software" +1. Switch to tab "JDBC Drivers" +1. Click "Browse..." +1. Select JDBC driver file +1. Click "Upload" +1. Click "Add" +1. In a dialog "Add EXACluster JDBC driver" configure the JDBC driver (see below) + +You need to specify the following settings when adding the JDBC driver via EXAOperation. -First download the [jTDS JDBC driver](https://sourceforge.net/projects/jtds/files/). +| Parameter | Value | +|-----------|-----------------------------------------------------| +| Name | `SQLSERVER` | +| Main | `com.microsoft.sqlserver.jdbc.SQLServerDriver` | +| Prefix | `jdbc:sqlserver:` | +| Files | `mssql-jdbc-.jre8.jar` | + +## Uploading the JDBC Driver to EXAOperation 1. [Create a bucket in BucketFS](https://docs.exasol.com/administration/on-premise/bucketfs/create_new_bucket_in_bucketfs_service.htm) 1. Upload the driver to BucketFS @@ -16,16 +38,16 @@ Upload the latest available release of [Virtual Schema JDBC Adapter](https://git Then create a schema to hold the adapter script. ```sql -CREATE SCHEMA ADAPTER; +CREATE SCHEMA SCHEMA_FOR_VS_SCRIPT; ``` The SQL statement below creates the adapter script, defines the Java class that serves as entry point and tells the UDF framework where to find the libraries (JAR files) for Virtual Schema and database driver. ```sql -CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS +CREATE OR REPLACE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_SQLSERVER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; - %jar /buckets///jtds.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; + %jar /buckets///mssql-jdbc-.jre8.jar; / ``` @@ -34,8 +56,8 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS Define the connection to SQL Server as shown below. We recommend using TLS to secure the connection. ```sql -CREATE OR REPLACE CONNECTION SQLSERVER_CONNECTION -TO 'jdbc:jtds:sqlserver://:/' +CREATE OR REPLACE CONNECTION SQLSERVER_JDBC_CONNECTION +TO 'jdbc:sqlserver://:' USER '' IDENTIFIED BY ''; ``` @@ -46,20 +68,63 @@ Below you see how an SQL Server Virtual Schema is created. ```sql CREATE VIRTUAL SCHEMA - USING ADAPTER.JDBC_ADAPTER + USING SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_SQLSERVER WITH SQL_DIALECT = 'SQLSERVER' - CONNECTION_NAME = 'SQLSERVER_CONNECTION' - CATALOG_NAME = '' - SCHEMA_NAME = ''; + CONNECTION_NAME = 'SQLSERVER_JDBC_CONNECTION' + CATALOG_NAME = '' + SCHEMA_NAME = ''; ``` -## Testing inforamtion - -The SQL Server Dialect was tested with the jTDS 1.3.1 JDBC driver and SQL Server 2014. - -## Troubleshooting -- SQL SERVER jTDS JDBC driver contains a [bug](https://sourceforge.net/p/jtds/bugs/679/) in DATE type. -The returned datatype for a SQLServer DATE type is a VARCHAR with a length of 10. If you want to avoid it you can use a newer driver, for example:[mssql-jdbc-7.2.2.jre8.jar](https://www.microsoft.com/en-us/download/details.aspx?id=57782). -Please, be aware that the new driver is not completely tested with Virtual Schemas. The driver's information for this dialect will be updated after we test the driver. +Please, do not forget to specify the `SCHEMA_NAME` property. + +Provide the SQL server's database name using one of the suggested ways: +1. Via the `CATALOG_NAME` property; +1. Via connection string definition: 'jdbc:sqlserver://:/'; + +## Data Types Conversion + +MS SERVER Data Type | Supported | Converted Exasol Data Type| Known limitations +--------------------|-----------|---------------------------|------------------- +BIGINT | ✓ | DECIMAL | +BINARY | × | | +BIT | ✓ | BOOLEAN | +CHAR | ✓ | CHAR | +DATE | ✓ | DATE | +DATETIME | ✓ | TIMESTAMP | +DATETIME2 | ✓ | TIMESTAMP | +DATETIMEOFFSET | ✓ | VARCHAR(34) | +DECIMAL | ✓ | DECIMAL | +FLOAT | ✓ | DOUBLE PRECISION | +GEOMETRY | × | | +GEOGRAPHY | × | | +HIERARCHYID | × | | +IMAGE | × | | +INT | ✓ | DECIMAL | +MONEY | ✓ | DECIMAL | +NCHAR | ✓ | CHAR | +NTEXT | ✓ | VARCHAR(2000000) | +NVARCHAR | ✓ | VARCHAR | +NUMERIC | ✓ | DECIMAL | +SQL_VARIANT | × | | +REAL | ✓ | DOUBLE PRECISION | +ROWVERSION | × | | +SMALLDATETIME | ✓ | TIMESTAMP | +SMALLINT | ✓ | DECIMAL | +SMALLMONEY | ✓ | DECIMAL | +TEXT | ✓ | VARCHAR(2000000) | +TIME | ✓ | VARCHAR(16) | +TINYINT | ✓ | DECIMAL | +UNIQUEIDENTIFIER | ✓ | CHAR(36) | +VARBINARY | × | | +VARCHAR | ✓ | VARCHAR | +XML | × | | + +## Testing information + +In the following matrix you find combinations of JDBC driver and dialect version that we tested. + +| Virtual Schema Version | SQL SERVER Version | Driver Name | Driver Version | +|------------------------|------------------------------|-------------------|----------------| +| Latest | 2019-CU6-ubuntu-16.04 8.0.20 | MS SQL JDBC JRE 8 | 8.4.0 | \ No newline at end of file diff --git a/doc/dialects/sybase.md b/doc/dialects/sybase.md index 6e15e7119..098b8d3c1 100644 --- a/doc/dialects/sybase.md +++ b/doc/dialects/sybase.md @@ -29,7 +29,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///jtds-.jar; / ``` diff --git a/doc/dialects/teradata.md b/doc/dialects/teradata.md index 23d0b65a6..78aa4d347 100644 --- a/doc/dialects/teradata.md +++ b/doc/dialects/teradata.md @@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets///virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets///terajdbc4.jar; %jar /buckets///tdgssconfig.jar; / diff --git a/doc/user-guide/user_guide.md b/doc/user-guide/user_guide.md index 91069852f..d41435619 100644 --- a/doc/user-guide/user_guide.md +++ b/doc/user-guide/user_guide.md @@ -55,7 +55,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.JDBC_ADAPTER_SCRIPT AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets/your-bucket-fs/your-bucket/virtual-schema-dist-5.0.3-bundle-4.0.3.jar; + %jar /buckets/your-bucket-fs/your-bucket/virtual-schema-dist-5.0.4-bundle-4.0.3.jar; %jar /buckets/your-bucket-fs/your-bucket/.jar; / ``` diff --git a/pom.xml b/pom.xml index 5d8562ade..807077b7f 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ UTF-8 11 3.0.0-M4 - 5.0.3 + 5.0.4 1.14.3 target/site/jacoco/jacoco.xml,target/site/jacoco-it/jacoco.xml @@ -215,6 +215,17 @@ + + com.microsoft.sqlserver + mssql-jdbc + 8.4.0.jre11 + + + org.testcontainers + mssqlserver + ${org.testcontainers.version} + test + org.testcontainers mysql @@ -325,6 +336,7 @@ -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} PostgreSQLSqlDialectIT.java + SqlServerSqlDialectIT.java diff --git a/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReader.java b/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReader.java index a51ba3481..6836ed771 100644 --- a/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReader.java +++ b/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReader.java @@ -1,5 +1,7 @@ package com.exasol.adapter.dialects.sqlserver; +import static com.exasol.adapter.metadata.DataType.ExaCharset.UTF8; + import java.sql.Connection; import java.sql.Types; @@ -13,15 +15,9 @@ * This class implements a SQLServer-specific column metadata reader. */ public class SqlServerColumnMetadataReader extends BaseColumnMetadataReader { - static final int SQLSERVER_BLOB_SIZE = 100; - static final int SQLSERVER_HIERARCHYID_SIZE = 4000; - static final int SQLSERVER_MAX_VARCHAR_SIZE = 8000; - static final int SQLSERVER_MAX_CLOB_SIZE = 2000000; - static final int SQLSERVER_TIMESTAMP_TEXT_SIZE = 21; static final String SQLSERVER_DATE_TYPE_NAME = "date"; static final String SQLSERVER_DATETIME2_TYPE_NAME = "datetime2"; - static final String SQLSERVER_GEOMETRY_TYPE_NAME = "geometry"; - static final String SQLSERVER_HIERARCHYID_TYPE_NAME = "hierarchyid"; + private static final int SQL_SERVER_DATETIME_OFFSET = -155; /** * Create a new instance of the {@link SqlServerColumnMetadataReader}. @@ -38,48 +34,12 @@ public SqlServerColumnMetadataReader(final Connection connection, final AdapterP @Override public DataType mapJdbcType(final JdbcTypeDescription jdbcTypeDescription) { switch (jdbcTypeDescription.getJdbcType()) { - case Types.VARCHAR: - return mapVarChar(jdbcTypeDescription); - case Types.TIME: - case Types.TIME_WITH_TIMEZONE: - return DataType.createVarChar(SQLSERVER_TIMESTAMP_TEXT_SIZE, DataType.ExaCharset.UTF8); case Types.NUMERIC: return mapJdbcTypeNumericToDecimalWithFallbackToDouble(jdbcTypeDescription); - case Types.OTHER: - case Types.SQLXML: - return DataType.createVarChar(SqlServerSqlDialect.MAX_SQLSERVER_VARCHAR_SIZE, DataType.ExaCharset.UTF8); - case Types.CLOB: - return DataType.createVarChar(SqlServerSqlDialect.MAX_SQLSERVER_CLOB_SIZE, DataType.ExaCharset.UTF8); - case Types.BLOB: - return mapBlob(jdbcTypeDescription); - case Types.VARBINARY: - case Types.BINARY: - case Types.DISTINCT: - return DataType.createVarChar(SQLSERVER_BLOB_SIZE, DataType.ExaCharset.UTF8); + case SQL_SERVER_DATETIME_OFFSET: + return DataType.createVarChar(jdbcTypeDescription.getPrecisionOrSize(), UTF8); default: return super.mapJdbcType(jdbcTypeDescription); } } - - protected DataType mapVarChar(final JdbcTypeDescription jdbcTypeDescription) { - final String columnTypeName = jdbcTypeDescription.getTypeName(); - if (columnTypeName.equalsIgnoreCase(SQLSERVER_DATE_TYPE_NAME)) { - return DataType.createDate(); - } else if (columnTypeName.equalsIgnoreCase(SQLSERVER_DATETIME2_TYPE_NAME)) { - return DataType.createTimestamp(false); - } else { - return super.mapJdbcType(jdbcTypeDescription); - } - } - - protected DataType mapBlob(final JdbcTypeDescription jdbcTypeDescription) { - switch (jdbcTypeDescription.getTypeName().toLowerCase()) { - case SQLSERVER_HIERARCHYID_TYPE_NAME: - return DataType.createVarChar(SQLSERVER_HIERARCHYID_SIZE, DataType.ExaCharset.UTF8); - case SQLSERVER_GEOMETRY_TYPE_NAME: - return DataType.createVarChar(SQLSERVER_MAX_VARCHAR_SIZE, DataType.ExaCharset.UTF8); - default: - return DataType.createVarChar(SQLSERVER_BLOB_SIZE, DataType.ExaCharset.UTF8); - } - } } \ No newline at end of file diff --git a/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlDialect.java b/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlDialect.java index 01b03a9a3..3966aab93 100644 --- a/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlDialect.java +++ b/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlDialect.java @@ -21,9 +21,6 @@ */ public class SqlServerSqlDialect extends AbstractSqlDialect { static final String NAME = "SQLSERVER"; - static final int MAX_SQLSERVER_VARCHAR_SIZE = 8000; - static final int MAX_SQLSERVER_NVARCHAR_SIZE = 4000; - static final int MAX_SQLSERVER_CLOB_SIZE = 2000000; private static final Capabilities CAPABILITIES = createCapabilityList(); private static Capabilities createCapabilityList() { @@ -94,6 +91,7 @@ public Map getAggregateFunctionAliases() { aggregationAliases.put(AggregateFunction.STDDEV_POP, "STDEVP"); aggregationAliases.put(AggregateFunction.VARIANCE, "VAR"); aggregationAliases.put(AggregateFunction.VAR_POP, "VARP"); + aggregationAliases.put(AggregateFunction.COUNT, "COUNT_BIG"); return aggregationAliases; } diff --git a/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitor.java b/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitor.java index 5539220da..3afe23068 100644 --- a/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitor.java +++ b/src/main/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitor.java @@ -1,11 +1,10 @@ package com.exasol.adapter.dialects.sqlserver; -import static com.exasol.adapter.dialects.sqlserver.SqlServerSqlDialect.MAX_SQLSERVER_NVARCHAR_SIZE; -import static com.exasol.adapter.dialects.sqlserver.SqlServerSqlDialect.MAX_SQLSERVER_VARCHAR_SIZE; - +import java.sql.Types; import java.util.*; import com.exasol.adapter.AdapterException; +import com.exasol.adapter.adapternotes.ColumnAdapterNotesJsonConverter; import com.exasol.adapter.dialects.*; import com.exasol.adapter.metadata.ColumnMetadata; import com.exasol.adapter.metadata.TableMetadata; @@ -15,9 +14,9 @@ * This class generates SQL queries for the {@link SqlServerSqlDialect}. */ public class SqlServerSqlGenerationVisitor extends SqlGenerationVisitor { - private static final List TYPE_NAMES_REQUIRING_CAST = List.of("text", "date", "datetime2", "hierarchyid", - "geometry", "geography", "timestamp", "xml"); - private static final List TYPE_NAME_NOT_SUPPORTED = List.of("varbinary", "binary"); + private static final int SQL_SERVER_DATETIME_OFFSET = -155; + private static final int MAX_SQLSERVER_VARCHAR_SIZE = 8000; + private static final List REQUIRE_CAST = List.of(SQL_SERVER_DATETIME_OFFSET, Types.TIME); /** * Create a new instance of the {@link SqlServerSqlGenerationVisitor}. @@ -29,38 +28,28 @@ public SqlServerSqlGenerationVisitor(final SqlDialect dialect, final SqlGenerati super(dialect, context); } - protected List getListOfTypeNamesRequiringCast() { - return TYPE_NAMES_REQUIRING_CAST; + @Override + public String visit(final SqlColumn column) throws AdapterException { + final String projectionString = super.visit(column); + return getColumnProjectionString(column, projectionString); } - protected List getListOfTypeNamesNotSupported() { - return TYPE_NAME_NOT_SUPPORTED; + private String getColumnProjectionString(final SqlColumn column, final String projectionString) { + return super.isDirectlyInSelectList(column) // + ? buildColumnProjectionString(getJdbcDataType(column), projectionString) // + : projectionString; } - protected String buildColumnProjectionString(final String typeName, final String projectionString) { - final String castTypeNVarchar = "NVARCHAR(" + MAX_SQLSERVER_NVARCHAR_SIZE + ")"; - if (typeName.startsWith("text")) { - return getCastAs(projectionString, castTypeNVarchar); - } else if (typeName.startsWith("date") || typeName.startsWith("datetime2") - || typeName.startsWith("timestamp")) { - return getCastAs(projectionString, "DateTime"); - } else if (typeName.startsWith("hierarchyid")) { - return getCastAs(projectionString, castTypeNVarchar); - } else if (typeName.startsWith("geometry") || typeName.startsWith("geography")) { - return getCastAs(projectionString, "VARCHAR(" + MAX_SQLSERVER_VARCHAR_SIZE + ")"); - } else if (typeName.startsWith("xml")) { - return getCastAs(projectionString, castTypeNVarchar); - } else if (TYPE_NAME_NOT_SUPPORTED.contains(typeName)) { - return "'" + typeName + " NOT SUPPORTED'"; + private String buildColumnProjectionString(final int jdbcDataType, final String projectionString) { + if (jdbcDataType == Types.TIME) { + return "CAST(" + projectionString + " as VARCHAR(16))"; + } else if (jdbcDataType == SQL_SERVER_DATETIME_OFFSET) { + return "CAST(" + projectionString + " as VARCHAR(34))"; } else { return projectionString; } } - private String getCastAs(final String projectionString, final String castType) { - return "CAST(" + projectionString + " as " + castType + " )"; - } - @Override protected String representAsteriskInSelectList(final SqlSelectList selectList) throws AdapterException { final List selectStarList = buildSelectStar(selectList); @@ -70,15 +59,12 @@ protected String representAsteriskInSelectList(final SqlSelectList selectList) t } private List buildSelectStar(final SqlSelectList selectList) throws AdapterException { - - if (SqlGenerationHelper.selectListRequiresCasts(selectList, this.nodeRequiresCast)) { - return buildSelectStarWithNodeCast(selectList); - } else { - return new ArrayList<>(Collections.singletonList("*")); - } + final Optional> selectStartWithCastedColumns = buildSelectStarWithNodeCast(selectList); + return selectStartWithCastedColumns.orElseGet(() -> new ArrayList<>(Collections.singletonList("*"))); } - private List buildSelectStarWithNodeCast(final SqlSelectList selectList) throws AdapterException { + private Optional> buildSelectStarWithNodeCast(final SqlSelectList selectList) throws AdapterException { + boolean requiresCast = false; final SqlStatementSelect select = (SqlStatementSelect) selectList.getParent(); int columnId = 0; final List tableMetadata = new ArrayList<>(); @@ -86,45 +72,32 @@ private List buildSelectStarWithNodeCast(final SqlSelectList selectList) final List selectListElements = new ArrayList<>(tableMetadata.size()); for (final TableMetadata tableMeta : tableMetadata) { for (final ColumnMetadata columnMeta : tableMeta.getColumns()) { + if (requiresCast(new SqlColumn(columnId, columnMeta))) { + requiresCast = true; + } final SqlColumn sqlColumn = new SqlColumn(columnId, columnMeta); - selectListElements.add(buildColumnProjectionString(sqlColumn, super.visit(sqlColumn))); + selectListElements.add(buildColumnProjectionString(getJdbcDataType(sqlColumn), super.visit(sqlColumn))); ++columnId; } } - return selectListElements; + return requiresCast ? Optional.of(selectListElements) : Optional.empty(); } - private String buildColumnProjectionString(final SqlColumn column, final String projectionString) - throws AdapterException { - return buildColumnProjectionString(getTypeNameFromColumn(column), projectionString); + private boolean requiresCast(final SqlColumn column) { + final int typeName = getJdbcDataType(column); + return REQUIRE_CAST.contains(typeName); } - private final java.util.function.Predicate nodeRequiresCast = node -> { + private int getJdbcDataType(final SqlColumn column) { + final ColumnAdapterNotesJsonConverter converter = ColumnAdapterNotesJsonConverter.getInstance(); try { - if (node.getType() == SqlNodeType.COLUMN) { - final SqlColumn column = (SqlColumn) node; - final String typeName = getTypeNameFromColumn(column); - return getListOfTypeNamesRequiringCast().contains(typeName) - || getListOfTypeNamesNotSupported().contains(typeName); - } - return false; + return converter + .convertFromJsonToColumnAdapterNotes(column.getMetadata().getAdapterNotes(), column.getName()) + .getJdbcDataType(); } catch (final AdapterException exception) { - throw new SqlGenerationVisitorException("Exception during deserialization of ColumnAdapterNotes. ", - exception); + throw new SqlGenerationVisitorException( + "Unable to get a JDBC data type for an sql column " + column.getId()); } - }; - - @Override - public String visit(final SqlColumn column) throws AdapterException { - final String projectionString = super.visit(column); - return getColumnProjectionString(column, projectionString); - } - - private String getColumnProjectionString(final SqlColumn column, final String projectionString) - throws AdapterException { - return super.isDirectlyInSelectList(column) // - ? buildColumnProjectionString(getTypeNameFromColumn(column), projectionString) // - : projectionString; } @Override @@ -188,7 +161,7 @@ public String visit(final SqlFunctionScalar function) throws AdapterException { case YEARS_BETWEEN: return getDateTimeBetween(function, argumentsSql); case CURRENT_DATE: - return "CAST( GETDATE() AS DATE)"; + return "CAST(GETDATE() AS DATE)"; case CURRENT_TIMESTAMP: return "GETDATE()"; case SYSDATE: diff --git a/src/test/java/com/exasol/adapter/dialects/AbstractIntegrationTest.java b/src/test/java/com/exasol/adapter/dialects/AbstractIntegrationTest.java index 2c27fcfcf..307ceaa8e 100644 --- a/src/test/java/com/exasol/adapter/dialects/AbstractIntegrationTest.java +++ b/src/test/java/com/exasol/adapter/dialects/AbstractIntegrationTest.java @@ -49,32 +49,44 @@ protected static String getPropertyFromFile(final String resourcesDialectName, f protected static void createTestTablesForJoinTests(final Connection connection, final String schemaName) throws SQLException { - final Statement statement = connection.createStatement(); - statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_1 + "(x INT, y VARCHAR(100))"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (1,'aaa')"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (2,'bbb')"); - statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_2 + "(x INT, y VARCHAR(100))"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (2,'bbb')"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (3,'ccc')"); + try (final Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_1 + "(x INT, y VARCHAR(100))"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (1,'aaa')"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (2,'bbb')"); + statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_2 + "(x INT, y VARCHAR(100))"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (2,'bbb')"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (3,'ccc')"); + } } protected ResultSet getExpectedResultSet(final List expectedColumns, final List expectedRows) throws SQLException { final Connection connection = getExasolConnection(); - final Statement statement = connection.createStatement(); - final String expectedValues = expectedRows.stream().map(row -> "(" + row + ")") - .collect(Collectors.joining(",")); - final String qualifiedExpectedTableName = SCHEMA_EXASOL + ".EXPECTED"; - statement.execute("CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" - + String.join(", ", expectedColumns) + ")"); - statement.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues); - return statement.executeQuery("SELECT * FROM " + qualifiedExpectedTableName); + try (final Statement statement = connection.createStatement()) { + final String expectedValues = expectedRows.stream().map(row -> "(" + row + ")") + .collect(Collectors.joining(",")); + final String qualifiedExpectedTableName = SCHEMA_EXASOL + ".EXPECTED"; + statement.execute("CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" + + String.join(", ", expectedColumns) + ")"); + statement.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues); + return statement.executeQuery("SELECT * FROM " + qualifiedExpectedTableName); + } } protected ResultSet getActualResultSet(final String query) throws SQLException { final Connection connection = getExasolConnection(); - final Statement statement = connection.createStatement(); - return statement.executeQuery(query); + try (final Statement statement = connection.createStatement()) { + return statement.executeQuery(query); + } + } + + protected String getExplainVirtualString(final String query) throws SQLException { + final Connection connection = getExasolConnection(); + try (final Statement statement = connection.createStatement()) { + final ResultSet explainVirtual = statement.executeQuery("EXPLAIN VIRTUAL " + query); + explainVirtual.next(); + return explainVirtual.getString("PUSHDOWN_SQL"); + } } protected abstract Connection getExasolConnection() throws SQLException; diff --git a/src/test/java/com/exasol/adapter/dialects/IntegrationTestConstants.java b/src/test/java/com/exasol/adapter/dialects/IntegrationTestConstants.java index 14ca8e46b..20594c78a 100644 --- a/src/test/java/com/exasol/adapter/dialects/IntegrationTestConstants.java +++ b/src/test/java/com/exasol/adapter/dialects/IntegrationTestConstants.java @@ -3,7 +3,7 @@ import java.nio.file.Path; public final class IntegrationTestConstants { - public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-5.0.3-bundle-4.0.3.jar"; + public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-5.0.4-bundle-4.0.3.jar"; public static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); public static final String SCHEMA_EXASOL = "SCHEMA_EXASOL"; public static final String ADAPTER_SCRIPT_EXASOL = "ADAPTER_SCRIPT_EXASOL"; diff --git a/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReaderTest.java b/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReaderTest.java index 73f163121..d1e11a447 100644 --- a/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReaderTest.java +++ b/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerColumnMetadataReaderTest.java @@ -1,15 +1,12 @@ package com.exasol.adapter.dialects.sqlserver; -import static com.exasol.adapter.dialects.sqlserver.SqlServerColumnMetadataReader.*; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import java.sql.Types; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import com.exasol.adapter.AdapterProperties; import com.exasol.adapter.dialects.BaseIdentifierConverter; @@ -33,30 +30,6 @@ void testMapJdbcTypeVarChar() { assertThat(type, equalTo(DataType.createVarChar(expectedSize, ExaCharset.ASCII))); } - @Test - void testMapJdbcTypeVarCharRepresentingDate() { - final DataType type = mapJdbcTypeWithName(Types.VARCHAR, SQLSERVER_DATE_TYPE_NAME); - assertThat(type, equalTo(DataType.createDate())); - } - - @Test - void testMapJdbcTypeVarCharRepresentingDatetime2() { - final DataType type = mapJdbcTypeWithName(Types.VARCHAR, SQLSERVER_DATETIME2_TYPE_NAME); - assertThat(type, equalTo(DataType.createTimestamp(false))); - } - - @Test - void testMapJdbcTypeTime() { - final DataType jdbcType = mapJdbcType(Types.TIME); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_TIMESTAMP_TEXT_SIZE, DataType.ExaCharset.UTF8))); - } - - @Test - void testMapJdbcTypeTimeWithTimezone() { - final DataType jdbcType = mapJdbcType(Types.TIME_WITH_TIMEZONE); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_TIMESTAMP_TEXT_SIZE, DataType.ExaCharset.UTF8))); - } - @Test void testMapJdbcTypeNumeric() { assertNumericMappedToDecimalWithPrecisionAndScale(DataType.MAX_EXASOL_DECIMAL_PRECISION, 2); @@ -68,44 +41,7 @@ void testMapJdbcTypeNumericExceedingExsolMaxPrecisionToDouble() { } @Test - void testMapJdbcTypeOther() { - final DataType jdbcType = mapJdbcType(Types.OTHER); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_MAX_VARCHAR_SIZE, DataType.ExaCharset.UTF8))); - } - - @Test - void testMapJdbcTypeSqlXml() { - final DataType jdbcType = mapJdbcType(Types.SQLXML); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_MAX_VARCHAR_SIZE, DataType.ExaCharset.UTF8))); - } - - @Test - void testMapJdbcTypeClob() { - final DataType jdbcType = mapJdbcType(Types.CLOB); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_MAX_CLOB_SIZE, DataType.ExaCharset.UTF8))); - } - - @ValueSource(ints = { Types.BLOB, Types.VARBINARY, Types.BINARY, Types.DISTINCT }) - @ParameterizedTest - void testMapJdbcTypeBlob(final int type) { - final DataType jdbcType = mapJdbcType(type); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_BLOB_SIZE, DataType.ExaCharset.UTF8))); - } - - @Test - void testMapJdbcTypeHierarchyId() { - final DataType jdbcType = mapJdbcTypeWithName(Types.BLOB, SQLSERVER_HIERARCHYID_TYPE_NAME); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_HIERARCHYID_SIZE, DataType.ExaCharset.UTF8))); - } - - @Test - void testMapJdbcTypeGeometry() { - final DataType jdbcType = mapJdbcTypeWithName(Types.BLOB, SQLSERVER_GEOMETRY_TYPE_NAME); - assertThat(jdbcType, equalTo(DataType.createVarChar(SQLSERVER_MAX_VARCHAR_SIZE, DataType.ExaCharset.UTF8))); - } - - @Test - void testMapJdbdTypeFallbackToParent() { + void testMapJdbcTypeFallbackToParent() { final DataType jdbcType = mapJdbcType(Types.BOOLEAN); assertThat(jdbcType, equalTo(DataType.createBool())); } diff --git a/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlDialectIT.java new file mode 100644 index 000000000..e41155a25 --- /dev/null +++ b/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlDialectIT.java @@ -0,0 +1,285 @@ +package com.exasol.adapter.dialects.sqlserver; + +import static com.exasol.adapter.dialects.IntegrationTestConstants.*; +import static com.exasol.dbbuilder.dialects.exasol.AdapterScript.Language.JAVA; +import static com.exasol.matcher.ResultSetMatcher.matchesResultSet; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.sql.*; +import java.time.LocalDate; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeoutException; + +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MSSQLServerContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import com.exasol.adapter.dialects.AbstractIntegrationTest; +import com.exasol.bucketfs.BucketAccessException; +import com.exasol.containers.ExasolContainer; +import com.exasol.containers.ExasolContainerConstants; +import com.exasol.dbbuilder.dialects.exasol.*; + +@Tag("integration") +@Testcontainers +class SqlServerSqlDialectIT extends AbstractIntegrationTest { + private static final Logger LOGGER = LoggerFactory.getLogger(SqlServerSqlDialectIT.class); + private static final String MS_SQL_SERVER_CONTAINER_NAME = "mcr.microsoft.com/mssql/server:2019-CU6-ubuntu-16.04"; + private static final String RESOURCES_FOLDER_DIALECT_NAME = "sqlserver"; + private static final String SCHEMA_SQL_SERVER = "SCHEMA_SQL_SERVER"; + private static final String TABLE_SQL_SERVER_NUMERIC_AND_DATE_DATA_TYPES = "TABLE_SQL_SERVER_NUMERIC_AND_DATE"; + private static final String TABLE_SQL_SERVER_STRING_DATA_TYPES = "TABLE_SQL_SERVER_STRING"; + private static final String TABLE_SQL_SERVER_SIMPLE = "TABLE_SQL_SERVER_SIMPLE"; + private static final int MS_SQL_SERVER_PORT = 1433; + private static final String JDBC_CONNECTION_NAME = "JDBC"; + private static final String VIRTUAL_SCHEMA_JDBC = "VIRTUAL_SCHEMA_JDBC"; + @Container + private static final MSSQLServerContainer MS_SQL_SERVER_CONTAINER = new MSSQLServerContainer( + MS_SQL_SERVER_CONTAINER_NAME); + @Container + private static final ExasolContainer> EXASOL_CONTAINER = new ExasolContainer<>( + ExasolContainerConstants.EXASOL_DOCKER_IMAGE_REFERENCE) // + .withLogConsumer(new Slf4jLogConsumer(LOGGER)); + + @Override + protected Connection getExasolConnection() throws SQLException { + return EXASOL_CONTAINER.createConnection(""); + } + + @BeforeAll + static void beforeAll() throws InterruptedException, BucketAccessException, TimeoutException, SQLException { + final String driverName = getPropertyFromFile(RESOURCES_FOLDER_DIALECT_NAME, "driver.name"); + uploadDriverToBucket(driverName, RESOURCES_FOLDER_DIALECT_NAME, EXASOL_CONTAINER.getDefaultBucket()); + uploadVsJarToBucket(EXASOL_CONTAINER.getDefaultBucket()); + createSqlServerSchema(); + createSimpleTable(); + createSqlServerTableNumericAndDateDataTypes(); + createSqlServerTableStringDataTypes(); + final ExasolObjectFactory exasolFactory = new ExasolObjectFactory(EXASOL_CONTAINER.createConnection("")); + final ExasolSchema exasolSchema = exasolFactory.createSchema(SCHEMA_EXASOL); + final AdapterScript adapterScript = createAdapterScript(driverName, exasolSchema); + final String connectionString = "jdbc:sqlserver://" + DOCKER_IP_ADDRESS + ":" + + MS_SQL_SERVER_CONTAINER.getMappedPort(MS_SQL_SERVER_PORT); + final ConnectionDefinition connectionDefinition = exasolFactory.createConnectionDefinition(JDBC_CONNECTION_NAME, + connectionString, MS_SQL_SERVER_CONTAINER.getUsername(), MS_SQL_SERVER_CONTAINER.getPassword()); + exasolFactory.createVirtualSchemaBuilder(VIRTUAL_SCHEMA_JDBC).adapterScript(adapterScript) + .connectionDefinition(connectionDefinition).dialectName("SQLSERVER") + .properties(Map.of("CATALOG_NAME", "master", "SCHEMA_NAME", SCHEMA_SQL_SERVER)).build(); + } + + private static void createSqlServerSchema() throws SQLException { + try (final Statement statement = MS_SQL_SERVER_CONTAINER.createConnection("").createStatement()) { + statement.execute("CREATE SCHEMA " + SCHEMA_SQL_SERVER); + } + } + + private static AdapterScript createAdapterScript(final String driverName, final ExasolSchema schema) { + final String content = "%scriptclass com.exasol.adapter.RequestDispatcher;\n" // + + "%jar /buckets/bfsdefault/default/" + VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION + ";\n" // + + "%jar /buckets/bfsdefault/default/drivers/jdbc/" + driverName + ";\n"; + return schema.createAdapterScript(ADAPTER_SCRIPT_EXASOL, JAVA, content); + } + + @ParameterizedTest + @CsvSource(value = { // + "c1 | DECIMAL(19) | -9223372036854775808 | 9223372036854775807", // + "c2 | DECIMAL(10) | -2147483648 | 2147483647", // + "c3 | DECIMAL(5) | -32768 | 32767", // + "c4 | DECIMAL(3) | 0 | 255", // + "c5 | BOOLEAN | false | true", // + "c6 | DECIMAL(30, 8) | 6 | 999.99999999", // + "c7 | DECIMAL(10, 5) | 7.43 | 6.43", // + "c8 | DECIMAL(19, 4) | -922337203685477.5808 | 922337203685477.5807", // + "c9 | DECIMAL(10, 4) | -214748.3648 | 214748.3647", // + "c10 | DOUBLE PRECISION | -1.79E+308 | 1.79E+308", // + "c11 | DOUBLE PRECISION | -3978.4560546875 | 3978.4560546875" // + }, delimiter = '|') + void testSupportedNumericDataTypes(final String columnName, final String expectedColumnType, + final String expectedValueFirst, final String expectedValueSecond) throws SQLException { + final String query = "SELECT \"" + columnName + "\" FROM " + VIRTUAL_SCHEMA_JDBC + "." + + TABLE_SQL_SERVER_NUMERIC_AND_DATE_DATA_TYPES; + final ResultSet expected = getExpectedResultSet(List.of("col1 " + expectedColumnType), // + List.of(expectedValueFirst, expectedValueSecond)); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @ParameterizedTest + @CsvSource(value = { // + "c12 | VARCHAR(16) | 01:02:03.0000000 | 23:59:59.0000000", // + "c13 | DATE | 0001-01-01 | 9999-12-31", // + "c14 | TIMESTAMP | 1900-01-01 00:00:00 | 2078-12-31 23:59:00", // + "c15 | TIMESTAMP | 1753-01-01 00:00:00.0 | 9999-12-30 23:59:59.000", // + "c16 | TIMESTAMP | 0001-01-01 00:00:00.0 | 9999-12-30 23:59:59.0", // + "c17 | VARCHAR(34) | 0001-01-01 13:00:00.0000000 +12:15 | 9999-12-30 23:59:59.9999999 +12:15" // + }, delimiter = '|') + void testSupportedDateAndTimeDataTypes(final String columnName, final String expectedColumnType, + final String expectedValueFirst, final String expectedValueSecond) throws SQLException { + final String query = "SELECT \"" + columnName + "\" FROM " + VIRTUAL_SCHEMA_JDBC + "." + + TABLE_SQL_SERVER_NUMERIC_AND_DATE_DATA_TYPES; + final ResultSet expected = getExpectedResultSet(List.of("col1 " + expectedColumnType), // + List.of("'" + expectedValueFirst + "'", "'" + expectedValueSecond + "'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @ParameterizedTest + @CsvSource(value = { // + "c18 | CHAR(5) | abcde", // + "c19 | VARCHAR(100) | abc", // + "c20 | VARCHAR(2000000) | def", // + "c21 | CHAR(5) | nabcd", // + "c22 | VARCHAR(100) | nabc", // + "c23 | VARCHAR(2000000) | ndef", // + "c29 | CHAR(36) | B14BC077-F1DF-457C-9F7E-7CB9E0BC1CF3", // + "c31 | VARCHAR(2000000) | " // + }, delimiter = '|') + void testSupportedStringDataTypes(final String columnName, final String expectedColumnType, + final String expectedValue) throws SQLException { + final String query = "SELECT \"" + columnName + "\" FROM " + VIRTUAL_SCHEMA_JDBC + "." + + TABLE_SQL_SERVER_STRING_DATA_TYPES; + final ResultSet expected = getExpectedResultSet(List.of("col1 " + expectedColumnType), // + List.of("'" + expectedValue + "'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @ParameterizedTest + @CsvSource(value = { "c24", "c25", "c26", "c27", "c28", "c30", "c32", "c33" }) + void testUnsupportedDataTypes(final String columnName) { + final String query = "SELECT \"" + columnName + "\" FROM " + VIRTUAL_SCHEMA_JDBC + "." + + TABLE_SQL_SERVER_STRING_DATA_TYPES; + final SQLException exception = assertThrows(SQLException.class, () -> getActualResultSet(query)); + assertThat(exception.getMessage(), containsString("object \"" + columnName + "\" not found")); + } + + @Test + void testSelectStar() throws SQLException { + final String query = "SELECT * FROM " + VIRTUAL_SCHEMA_JDBC + "." + TABLE_SQL_SERVER_SIMPLE; + final ResultSet expected = getExpectedResultSet( + List.of("col1 DECIMAL(19)", "col2 VARCHAR(16)", "col3 VARCHAR(100)"), // + List.of("-9223372036854775808, '00:00:00.0000000', 'first'", // + "0, '01:02:03.0000000', 'second'", // + "9223372036854775807, '23:59:59.0000000', 'third'")); + final String expectedRewrittenQuery = "SELECT [bigint_col], CAST([time_col] as VARCHAR(16)), [varchar_col] FROM"; + assertAll(() -> assertThat(getActualResultSet(query), matchesResultSet(expected)), + () -> assertThat(getExplainVirtualString(query), containsString(expectedRewrittenQuery))); + } + + @Test + void testCount() throws SQLException { + final String query = "SELECT COUNT(*) FROM " + VIRTUAL_SCHEMA_JDBC + "." + TABLE_SQL_SERVER_SIMPLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(19)"), List.of("3")); + final String expectedRewrittenQuery = "SELECT COUNT_BIG(*) FROM"; + assertAll(() -> assertThat(getActualResultSet(query), matchesResultSet(expected)), + () -> assertThat(getExplainVirtualString(query), containsString(expectedRewrittenQuery))); + } + + @Test + void testGetDate() throws SQLException { + final String query = "SELECT CURRENT_DATE FROM " + VIRTUAL_SCHEMA_JDBC + "." + TABLE_SQL_SERVER_SIMPLE + + " LIMIT 1"; + final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), + List.of("'" + LocalDate.now().toString() + "'")); + final String expectedRewrittenQuery = "SELECT TOP 1 CAST(GETDATE() AS DATE) FROM"; + assertAll(() -> assertThat(getActualResultSet(query), matchesResultSet(expected)), + () -> assertThat(getExplainVirtualString(query), containsString(expectedRewrittenQuery))); + } + + private static void createSimpleTable() throws SQLException { + try (final Statement statement = MS_SQL_SERVER_CONTAINER.createConnection("").createStatement()) { + statement.execute("CREATE TABLE " + SCHEMA_SQL_SERVER + "." + TABLE_SQL_SERVER_SIMPLE + " (" // + + "bigint_col bigint, " // + + "time_col time(7), " // + + "varchar_col varchar(100), " // + + ")"); + statement.execute("INSERT INTO " + SCHEMA_SQL_SERVER + "." + TABLE_SQL_SERVER_SIMPLE + " VALUES" // + + "(-9223372036854775808, '00:00:00', 'first'), " // + + "(0, '01:02:03', 'second'), " // + + "(9223372036854775807, '23:59:59', 'third') " // + ); + } + } + + private static void createSqlServerTableNumericAndDateDataTypes() throws SQLException { + try (final Statement statement = MS_SQL_SERVER_CONTAINER.createConnection("").createStatement()) { + statement.execute( + "CREATE TABLE " + SCHEMA_SQL_SERVER + "." + TABLE_SQL_SERVER_NUMERIC_AND_DATE_DATA_TYPES + " (" // + // Exact numerics + + "c1 bigint, " // + + "c2 int, " // + + "c3 smallint, " // + + "c4 tinyint, " // + + "c5 bit, " // + + "c6 decimal(30,8), " // + + "c7 numeric(10,5), " // + + "c8 money, " // + + "c9 smallmoney, " // + // Approximate numerics + + "c10 float(53), " // + + "c11 real, " // + // Date and time + + "c12 time(7), " // + + "c13 date, " // + + "c14 smalldatetime, " // + + "c15 datetime, " // + + "c16 datetime2, " // + + "c17 datetimeoffset " // + + ")"); + statement.execute("INSERT INTO " + SCHEMA_SQL_SERVER + "." + TABLE_SQL_SERVER_NUMERIC_AND_DATE_DATA_TYPES // + + " VALUES(" // + + "-9223372036854775808, -2147483648, -32768, 0, 0, 6, 7.43, -922337203685477.5808, -214748.3648, " // + + "-1.79E+308, -3978.456, " // + + "'01:02:03', '0001-01-01', '1900-01-01 00:00:00', '1753-01-01 00:00:00', '0001-01-01 00:00:00', '0001-01-01 13:00:00.0000000 +12:15' " // + + ")"); + statement.execute("INSERT INTO " + SCHEMA_SQL_SERVER + "." + TABLE_SQL_SERVER_NUMERIC_AND_DATE_DATA_TYPES // + + " VALUES(" // + + "9223372036854775807, 2147483647, 32767, 255, 1, 999.99999999, 6.43, 922337203685477.5807, 214748.3647, " // + + "1.79E+308, 3978.456, " // + + "'23:59:59', '9999-12-31', '2078-12-31 23:59:00', '9999-12-30 23:59:59', '9999-12-30 23:59:59', '9999-12-30 23:59:59.9999999 +12:15' " // + + ")"); + } + } + + private static void createSqlServerTableStringDataTypes() throws SQLException { + try (final Statement statement = MS_SQL_SERVER_CONTAINER.createConnection("").createStatement()) { + statement.execute("CREATE TABLE " + SCHEMA_SQL_SERVER + "." + TABLE_SQL_SERVER_STRING_DATA_TYPES + " (" // + // Character strings + + "c18 char(5), " // + + "c19 varchar(100), " // + + "c20 text, " // + // Unicode character strings + + "c21 nchar(5), " // + + "c22 nvarchar(100), " // + + "c23 ntext, " // + // Binary strings + + "c24 binary(10), " // + + "c25 varbinary(9), " // + + "c26 image, " // + // Other data types + + "c27 rowversion, " // + + "c28 hierarchyid, " // + + "c29 uniqueidentifier, " // + + "c30 sql_variant, " // + + "c31 xml, " // + + "c32 geometry, " // + + "c33 geography " // + + ")"); + statement.execute("INSERT INTO " + SCHEMA_SQL_SERVER + "." + TABLE_SQL_SERVER_STRING_DATA_TYPES // + + "(c18,c19,c20,c21,c22,c23,c24,c25,c26,c28,c29,c30,c31,c32,c33)" // + + " VALUES(" // + + "'abcde', 'abc', 'def', " // + + "'nabcd', 'nabc', 'ndef', " // + + "CAST(123456 AS BINARY(10)), CAST(N'Test' as VARBINARY(9)), CAST(123456 AS BINARY(2)), " // + + "'/1/', CAST('B14BC077-F1DF-457C-9F7E-7CB9E0BC1CF3' AS UNIQUEIDENTIFIER), 'BaseType', '', geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326) " // + + ")"); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitorTest.java b/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitorTest.java index bfb07a52e..ac622e055 100644 --- a/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitorTest.java +++ b/src/test/java/com/exasol/adapter/dialects/sqlserver/SqlServerSqlGenerationVisitorTest.java @@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static utils.SqlNodesCreator.*; -import java.sql.Connection; import java.util.ArrayList; import java.util.List; @@ -14,7 +13,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.Mock; import com.exasol.adapter.AdapterException; import com.exasol.adapter.AdapterProperties; @@ -24,8 +22,6 @@ class SqlServerSqlGenerationVisitorTest { private SqlServerSqlGenerationVisitor visitor; - @Mock - private Connection connectionMock; @BeforeEach void beforeEach() { @@ -47,43 +43,22 @@ void testVisitSqlSelectListSelectStar() throws AdapterException { assertSqlNodeConvertedToAsterisk(sqlSelectList, this.visitor); } - @CsvSource({ "text, NVARCHAR(4000)", // - "date, DateTime", // - "datetime2, DateTime", // - "hierarchyid, NVARCHAR(4000)", // - "geometry, VARCHAR(8000)", // - "geography, VARCHAR(8000)", // - "timestamp, DateTime", // - "xml, NVARCHAR(4000)" // - }) - @ParameterizedTest - void testVisitSqlSelectListSelectStarRequiresCast(final String typeName, final String expectedCastType) - throws AdapterException { - final SqlSelectList sqlSelectList = createSqlSelectStarListWithOneColumn( - "{\"jdbcDataType\":2009, \"typeName\":\"" + typeName + "\"}", - DataType.createVarChar(10, DataType.ExaCharset.UTF8), "test_column"); - assertThat(this.visitor.visit(sqlSelectList), equalTo("CAST([test_column] as " + expectedCastType + " )")); - } - - @CsvSource({ "varbinary", // - "binary" // - }) - @ParameterizedTest - void testVisitSqlSelectListSelectStarUnsupportedType(final String typeName) throws AdapterException { + @Test + void testVisitSqlSelectListSelectStarRequiresCast() throws AdapterException { final SqlSelectList sqlSelectList = createSqlSelectStarListWithOneColumn( - "{\"jdbcDataType\":2009, \"typeName\":\"" + typeName + "\"}", - DataType.createVarChar(10, DataType.ExaCharset.UTF8), "test_column"); - assertThat(this.visitor.visit(sqlSelectList), equalTo("'" + typeName + " NOT SUPPORTED'")); + "{\"jdbcDataType\":-155, \"typeName\":\"datetimeoffset\"}", + DataType.createVarChar(36, DataType.ExaCharset.UTF8), "test_column"); + assertThat(this.visitor.visit(sqlSelectList), equalTo("CAST([test_column] as VARCHAR(34))")); } @Test void testVisitSqlStatementSelect() throws AdapterException { final SqlStatementSelect select = (SqlStatementSelect) DialectTestData.getTestSqlNode(); assertThat(this.visitor.visit(select), // - equalTo("SELECT TOP 10 [USER_ID], COUNT([URL])" // + equalTo("SELECT TOP 10 [USER_ID], COUNT_BIG([URL])" // + " FROM [test_catalog].[test_schema].[CLICKS]" // + " WHERE 1 < [USER_ID]" // - + " GROUP BY [USER_ID] HAVING 1 < COUNT([URL])" // + + " GROUP BY [USER_ID] HAVING 1 < COUNT_BIG([URL])" // + " ORDER BY [USER_ID] NULLS LAST")); } @@ -120,7 +95,7 @@ void testVisitSqlFunctionScalarTimeBetween(final ScalarFunction scalarFunction, assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("DATEDIFF(" + expected + ",10,[test_column])")); } - @CsvSource({ "CURRENT_DATE, CAST( GETDATE() AS DATE)", // + @CsvSource({ "CURRENT_DATE, CAST(GETDATE() AS DATE)", // "CURRENT_TIMESTAMP, GETDATE()", // "SYSDATE, CAST( SYSDATETIME() AS DATE)", // "SYSTIMESTAMP, SYSDATETIME()" }) diff --git a/src/test/resources/container-license-acceptance.txt b/src/test/resources/container-license-acceptance.txt new file mode 100644 index 000000000..0591e50c6 --- /dev/null +++ b/src/test/resources/container-license-acceptance.txt @@ -0,0 +1 @@ + mcr.microsoft.com/mssql/server:2019-CU6-ubuntu-16.04 \ No newline at end of file diff --git a/src/test/resources/integration/driver/sqlserver/mssql-jdbc-8.4.0.jre8.jar b/src/test/resources/integration/driver/sqlserver/mssql-jdbc-8.4.0.jre8.jar new file mode 100644 index 000000000..a57a14641 Binary files /dev/null and b/src/test/resources/integration/driver/sqlserver/mssql-jdbc-8.4.0.jre8.jar differ diff --git a/src/test/resources/integration/driver/sqlserver/settings.cfg b/src/test/resources/integration/driver/sqlserver/settings.cfg new file mode 100644 index 000000000..bda950288 --- /dev/null +++ b/src/test/resources/integration/driver/sqlserver/settings.cfg @@ -0,0 +1,6 @@ +DRIVERNAME=SQLSERVER +JAR=mssql-jdbc-8.4.0.jre8.jar +DRIVERMAIN=com.microsoft.sqlserver.jdbc.SQLServerDriver +PREFIX=jdbc:sqlserver: +FETCHSIZE=100000 +INSERTSIZE=-1 diff --git a/src/test/resources/integration/driver/sqlserver/sqlserver.properties b/src/test/resources/integration/driver/sqlserver/sqlserver.properties new file mode 100644 index 000000000..7fa91f800 --- /dev/null +++ b/src/test/resources/integration/driver/sqlserver/sqlserver.properties @@ -0,0 +1,2 @@ +driver.name=mssql-jdbc-8.4.0.jre8.jar +driver.path=src/test/resources/integration/driver/sqlserver \ No newline at end of file