Skip to content

Commit

Permalink
Merge branch 'master' into panama-sdkv2-gdcv2
Browse files Browse the repository at this point in the history
  • Loading branch information
@aimethed
aimethed authored Nov 15, 2024
2 parents 7f8c53c + f6d3620 commit d162972
Show file tree
Hide file tree
Showing 28 changed files with 135 additions and 50 deletions.
1 change: 0 additions & 1 deletion .github/workflows/run_release_tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,4 @@ jobs:
RESULTS_LOCATION: ${{ secrets.RESULTS_LOCATION }}
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
S3_DATA_PATH: ${{ secrets.S3_DATA_PATH }}
S3_JARS_BUCKET: ${{ secrets.S3_JARS_BUCKET }}
SPILL_BUCKET: ${{ secrets.SPILL_BUCKET }}
2 changes: 1 addition & 1 deletion athena-clickhouse/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<dependency>
<groupId>com.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>0.6.5</version>
<version>0.7.1-patch1</version>
<classifier>all</classifier>
</dependency>
<dependency>
Expand Down
20 changes: 20 additions & 0 deletions ...a-hive/src/main/java/com/amazonaws/athena/connectors/cloudera/HiveQueryStringBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@
import com.amazonaws.athena.connector.lambda.domain.Split;
import com.amazonaws.athena.connectors.jdbc.manager.FederationExpressionParser;
import com.amazonaws.athena.connectors.jdbc.manager.JdbcSplitQueryBuilder;
import com.amazonaws.athena.connectors.jdbc.manager.TypeAndValue;
import com.google.common.base.Strings;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.pojo.ArrowType;

import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -56,4 +59,21 @@ protected List<String> getPartitionWhereClauses(Split split)
}
return Collections.emptyList();
}

protected String toPredicate(String columnName, String operator, Object value, ArrowType type,
List<TypeAndValue> accumulator)
{
Types.MinorType minorType = Types.getMinorTypeForArrowType(type);
if (minorType.equals(Types.MinorType.DATEMILLI) && isOperatorEquals(operator)) {
accumulator.add(new TypeAndValue(type, value));
return quote(columnName) + " " + operator + " cast(? as timestamp)";
}
// Default to parent's behavior
return super.toPredicate(columnName, operator, value, type, accumulator);
}

private boolean isOperatorEquals(String operator)
{
return operator.equals("=");
}
}
37 changes: 37 additions & 0 deletions ...ra-hive/src/test/java/com/amazonaws/athena/connectors/cloudera/HiveRecordHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.concurrent.TimeUnit;

Expand Down Expand Up @@ -136,4 +140,37 @@ public void buildSplitSql()
Mockito.verify(preparedStatement, Mockito.times(1))
.setDate(1, expectedDate);
}

@Test
public void buildSplitSqlTimestamp()
throws SQLException
{
TableName tableName = new TableName("testSchema", "testTable");

SchemaBuilder schemaBuilder = SchemaBuilder.newBuilder();
schemaBuilder.addField(FieldBuilder.newBuilder("testCol1", Types.MinorType.DATEMILLI.getType()).build());
schemaBuilder.addField(FieldBuilder.newBuilder("partition", Types.MinorType.VARCHAR.getType()).build());
Schema schema = schemaBuilder.build();

Split split = Mockito.mock(Split.class);
Mockito.when(split.getProperties()).thenReturn(Collections.singletonMap("partition", "p0"));
Mockito.when(split.getProperty(Mockito.eq("partition"))).thenReturn("p0");

DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
LocalDateTime timestamp = LocalDateTime.parse("2024-10-03 12:34:56", formatter);
ValueSet valueSet2 = getSingleValueSet(timestamp);
Constraints constraints = Mockito.mock(Constraints.class);
Mockito.when(constraints.getSummary()).thenReturn(new ImmutableMap.Builder<String, ValueSet>()
.put("testCol1", valueSet2)
.build());
PreparedStatement expectedPreparedStatement = Mockito.mock(PreparedStatement.class);
Mockito.when(this.connection.prepareStatement(nullable(String.class))).thenReturn(expectedPreparedStatement);
PreparedStatement preparedStatement = this.hiveRecordHandler.buildSplitSql(this.connection, "testCatalogName", tableName, schema, constraints, split);
Assert.assertEquals(expectedPreparedStatement, preparedStatement);
LocalDateTime timestampExp = LocalDateTime.parse("2024-10-03 12:34:56", formatter);
Timestamp expectedTimestamp = new Timestamp(timestamp.toInstant(ZoneOffset.UTC).toEpochMilli());
Assert.assertEquals(expectedPreparedStatement, preparedStatement);
Mockito.verify(preparedStatement, Mockito.times(1))
.setTimestamp(1, expectedTimestamp);
}
}
2 changes: 1 addition & 1 deletion athena-cloudwatch/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>cloudwatchlogs</artifactId>
<version>2.28.2</version>
<version>2.29.9</version>
<exclusions>
<!-- replaced with jcl-over-slf4j -->
<exclusion>
Expand Down
2 changes: 1 addition & 1 deletion athena-dynamodb/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.8.0</version>
<version>3.8.1</version>
<executions>
<execution>
<id>copy</id>
Expand Down
6 changes: 2 additions & 4 deletions ...src/main/java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchSchemaUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ private static FieldType toFieldType(Map<String, Object> mapping)
case "byte":
minorType = Types.MinorType.TINYINT;
break;
case "float":
case "half_float":
case "double":
minorType = Types.MinorType.FLOAT8;
break;
Expand All @@ -170,10 +172,6 @@ private static FieldType toFieldType(Map<String, Object> mapping)
minorType = Types.MinorType.BIGINT;
metadata.put("scaling_factor", mapping.get("scaling_factor").toString());
break;
case "float":
case "half_float":
minorType = Types.MinorType.FLOAT4;
break;
case "date":
case "date_nanos":
minorType = Types.MinorType.DATEMILLI;
Expand Down
8 changes: 4 additions & 4 deletions .../java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchMetadataHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,8 @@ public void doGetTable()
.addField(new Field("myscaled",
new FieldType(true, Types.MinorType.BIGINT.getType(), null,
ImmutableMap.of("scaling_factor", "10.0")), null))
.addField("myfloat", Types.MinorType.FLOAT4.getType())
.addField("myhalf", Types.MinorType.FLOAT4.getType())
.addField("myfloat", Types.MinorType.FLOAT8.getType())
.addField("myhalf", Types.MinorType.FLOAT8.getType())
.addField("mydatemilli", Types.MinorType.DATEMILLI.getType())
.addField("mydatenano", Types.MinorType.DATEMILLI.getType())
.addField("myboolean", Types.MinorType.BIT.getType())
Expand Down Expand Up @@ -328,10 +328,10 @@ public void doGetTable()
" \"type\" : \"double\"\n" + // type: double (FLOAT8)
" },\n" +
" \"myfloat\" : {\n" + // myfloat:
" \"type\" : \"float\"\n" + // type: float (FLOAT4)
" \"type\" : \"float\"\n" + // type: float (FLOAT8)
" },\n" +
" \"myhalf\" : {\n" + // myhalf:
" \"type\" : \"half_float\"\n" + // type: half_float (FLOAT4)
" \"type\" : \"half_float\"\n" + // type: half_float (FLOAT8)
" },\n" +
" \"myinteger\" : {\n" + // myinteger:
" \"type\" : \"integer\"\n" + // type: integer (INT)
Expand Down
8 changes: 4 additions & 4 deletions ...test/java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchSchemaUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ public void setUp()
.addField(new Field("myscaled",
new FieldType(true, Types.MinorType.BIGINT.getType(), null,
ImmutableMap.of("scaling_factor", "10.0")), null))
.addField("myfloat", Types.MinorType.FLOAT4.getType())
.addField("myhalf", Types.MinorType.FLOAT4.getType())
.addField("myfloat", Types.MinorType.FLOAT8.getType())
.addField("myhalf", Types.MinorType.FLOAT8.getType())
.addField("mydatemilli", Types.MinorType.DATEMILLI.getType())
.addField("mydatenano", Types.MinorType.DATEMILLI.getType())
.addField("myboolean", Types.MinorType.BIT.getType())
Expand Down Expand Up @@ -111,10 +111,10 @@ public void setUp()
" \"type\" : \"double\"\n" + // type: double (FLOAT8)
" },\n" +
" \"myfloat\" : {\n" + // myfloat:
" \"type\" : \"float\"\n" + // type: float (FLOAT4)
" \"type\" : \"float\"\n" + // type: float (FLOAT8)
" },\n" +
" \"myhalf\" : {\n" + // myhalf:
" \"type\" : \"half_float\"\n" + // type: half_float (FLOAT4)
" \"type\" : \"half_float\"\n" + // type: half_float (FLOAT8)
" },\n" +
" \"myinteger\" : {\n" + // myinteger:
" \"type\" : \"integer\"\n" + // type: integer (INT)
Expand Down
2 changes: 1 addition & 1 deletion athena-gcs/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>2.43.2</version>
<version>2.44.1</version>
</dependency>
<!-- Test Dependencies -->
<dependency>
Expand Down
2 changes: 1 addition & 1 deletion athena-google-bigquery/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<version>3.17.1</version>
<version>3.17.3</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
4 changes: 2 additions & 2 deletions ...dbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcSplitQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,8 +336,8 @@ else if (singleValues.size() > 1) {
return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
}

private String toPredicate(String columnName, String operator, Object value, ArrowType type,
List<TypeAndValue> accumulator)
protected String toPredicate(String columnName, String operator, Object value, ArrowType type,
List<TypeAndValue> accumulator)
{
accumulator.add(new TypeAndValue(type, value));
return quote(columnName) + " " + operator + " ?";
Expand Down
4 changes: 2 additions & 2 deletions athena-neptune/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<version>2022.47.1</version>
<properties>
<!-- make sure gremlin driver version stays within the Neptune supported range -->
<gremlinDriverVersion>3.7.2</gremlinDriverVersion>
<gremlinDriverVersion>3.7.3</gremlinDriverVersion>
<neptune.sigv4.signer.version>2.4.0</neptune.sigv4.signer.version>
</properties>
<dependencies>
Expand Down Expand Up @@ -124,7 +124,7 @@
<dependency>
<groupId>org.eclipse.rdf4j</groupId>
<artifactId>rdf4j-repository-sparql</artifactId>
<version>5.0.2</version>
<version>5.0.3</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
Expand Down
2 changes: 1 addition & 1 deletion athena-oracle/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
<version>23.5.0.24.07</version>
<version>23.6.0.24.10</version>
</dependency>
<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/rds -->
<dependency>
Expand Down
2 changes: 1 addition & 1 deletion athena-redis/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
<dependency>
<groupId>io.lettuce</groupId>
<artifactId>lettuce-core</artifactId>
<version>6.4.0.RELEASE</version>
<version>6.5.0.RELEASE</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
Expand Down
2 changes: 1 addition & 1 deletion ...hana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaQueryStringBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ else if (singleValues.size() > 1) {
return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
}

private String toPredicate(String columnName, String operator, Object value, ArrowType type,
protected String toPredicate(String columnName, String operator, Object value, ArrowType type,
List<TypeAndValue> accumulator)
{
accumulator.add(new TypeAndValue(type, value));
Expand Down
2 changes: 1 addition & 1 deletion athena-snowflake/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<dependency>
<groupId>net.snowflake</groupId>
<artifactId>snowflake-jdbc</artifactId>
<version>3.19.0</version>
<version>3.20.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/rds -->
<dependency>
Expand Down
2 changes: 1 addition & 1 deletion athena-vertica/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna-platform</artifactId>
<version>5.14.0</version>
<version>5.15.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
Expand Down
14 changes: 7 additions & 7 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<maven.compiler.release>11</maven.compiler.release>
<maven.compiler.plugin.version>3.13.0</maven.compiler.plugin.version>
<!--- to meet engine version 1.12.533-->
<aws-sdk-v2.version>2.28.9</aws-sdk-v2.version>
<aws-sdk-v2.version>2.29.9</aws-sdk-v2.version>
<aws.lambda-java-core.version>1.2.2</aws.lambda-java-core.version>
<aws.lambda-java-log4j2.version>1.6.0</aws.lambda-java-log4j2.version>
<aws-cdk.version>1.204.0</aws-cdk.version>
Expand All @@ -27,8 +27,8 @@
<assertj.version>3.26.3</assertj.version>
<testng.version>7.10.2</testng.version>
<!--- to meet engine version 2.12.6 -->
<fasterxml.jackson.version>2.18.0</fasterxml.jackson.version>
<surefire.failsafe.version>3.5.1</surefire.failsafe.version>
<fasterxml.jackson.version>2.18.1</fasterxml.jackson.version>
<surefire.failsafe.version>3.5.2</surefire.failsafe.version>
<log4j2Version>2.24.1</log4j2Version>
<apache.arrow.version>13.0.0</apache.arrow.version>
<guava.version>33.3.1-jre</guava.version>
Expand All @@ -41,10 +41,10 @@
<spark.version>3.2.1</spark.version>
<test.system.rules.version>1.19.0</test.system.rules.version>
<!-- These are mvn on purpose to not conflict with the maven.* namespace -->
<mvn.checkstyle.version>3.5.0</mvn.checkstyle.version>
<mvn.checkstyle.version>3.6.0</mvn.checkstyle.version>
<mvn.shade.plugin.version>3.6.0</mvn.shade.plugin.version>
<mvn.source.plugin.version>3.3.1</mvn.source.plugin.version>
<mvn.javadoc.plugin.version>3.10.1</mvn.javadoc.plugin.version>
<mvn.javadoc.plugin.version>3.11.1</mvn.javadoc.plugin.version>
<mvn.jar.plugin.version>3.4.2</mvn.jar.plugin.version>
<doclint>none</doclint>
</properties>
Expand All @@ -53,7 +53,7 @@
<dependency>
<groupId>org.junit</groupId>
<artifactId>junit-bom</artifactId>
<version>5.11.2</version>
<version>5.11.3</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Expand Down Expand Up @@ -232,7 +232,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.8.0</version>
<version>3.8.1</version>
</plugin>
</plugins>
</pluginManagement>
Expand Down
2 changes: 1 addition & 1 deletion tools/bump_versions/bump_connectors_version.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,5 @@
common.update_yaml(yaml_files, new_version)

# Bump the versions in the Dockerfiles
dockerfiles = glob.glob("Dockerfile")
dockerfiles = glob.glob(f"{connector}/Dockerfile")
common.update_dockerfile(dockerfiles, new_version)
6 changes: 3 additions & 3 deletions tools/bump_versions/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,13 @@ def update_yaml(yaml_files, new_version):
for yml in yaml_files:
subprocess.run(["sed", "-i", f"s/\(SemanticVersion:\s*\).*/\\1{new_version}/", yml])
subprocess.run(["sed", "-i", f"s/\(CodeUri:.*-\)[0-9]*\.[0-9]*\.[0-9]*\(-\?.*\.jar\)/\\1{new_version}\\2/", yml])
subprocess.run(["sed", "-i", f"s/\(ImageUri:.*:\)[0-9]*\.[0-9]*\.[0-9]*\(\'\)/\\1{new_version}\\2/", yml])
subprocess.run(["sed", "-i", f"s|\(athena-federation-repository-.*:\)[0-9]*\.[0-9]*\.[0-9]*\(\'\)|\\1{new_version}\\2|", yml])


def update_dockerfile(dockerfiles, new_version):
for file in dockerfiles:
subprocess.run(["sed", "-i", f"s/\(COPY\s.*-\)[0-9]*\.[0-9]*\.[0-9]*\(\.jar.*\)/\\1{new_version}\\2/", file])
subprocess.run(["sed", "-i", f"s/\(RUN\sjar\sxf.*-\)[0-9]*\.[0-9]*\.[0-9]*\(\.jar\)/\\1{new_version}\\2/", file])
subprocess.run(["sed", "-i", f"s|\(target\/.*-\)[0-9]*\.[0-9]*\.[0-9]*|\\1{new_version}|", file])
subprocess.run(["sed", "-i", f"s|\(xf\s*.*-\)[0-9]*\.[0-9]*\.[0-9]*|\\1{new_version}|", file])


def update_project_version(soup, new_version):
Expand Down
1 change: 1 addition & 0 deletions validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/dynamo-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import * as cdk from 'aws-cdk-lib';
import * as glue from '@aws-cdk/aws-glue-alpha';
import * as ddb from 'aws-cdk-lib/aws-dynamodb';
import * as iam from 'aws-cdk-lib/aws-iam';
import { Repository } from 'aws-cdk-lib/aws-ecr';
import { CfnInclude } from 'aws-cdk-lib/cloudformation-include';
import { Construct } from 'constructs';
import tpcdsJson from '../../resources/tpcds_specs.json'
Expand Down
1 change: 1 addition & 0 deletions validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/opensearch-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as glue from '@aws-cdk/aws-glue-alpha';
import * as s3 from 'aws-cdk-lib/aws-s3';
import * as iam from 'aws-cdk-lib/aws-iam';
import { Repository } from 'aws-cdk-lib/aws-ecr';
import { CfnInclude } from 'aws-cdk-lib/cloudformation-include';
const path = require('path');
import {FederationStackProps} from './stack-props'
Expand Down
1 change: 1 addition & 0 deletions validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/rds-generic-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import * as rds from 'aws-cdk-lib/aws-rds';
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as glue from '@aws-cdk/aws-glue-alpha';
import * as iam from 'aws-cdk-lib/aws-iam'
import { Repository } from 'aws-cdk-lib/aws-ecr';
import { CfnInclude } from 'aws-cdk-lib/cloudformation-include';
import { Construct } from 'constructs';
const path = require('path')
Expand Down
1 change: 1 addition & 0 deletions validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/redshift-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import * as iam from 'aws-cdk-lib/aws-iam';
import * as s3 from 'aws-cdk-lib/aws-s3';
import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
import * as ssm from 'aws-cdk-lib/aws-ssm';
import { Repository } from 'aws-cdk-lib/aws-ecr';
import { CfnInclude } from 'aws-cdk-lib/cloudformation-include';
import { Construct } from 'constructs';
const path = require('path');
Expand Down
Loading

0 comments on commit d162972

Please sign in to comment.