diff --git a/athena-aws-cmdb/athena-aws-cmdb.yaml b/athena-aws-cmdb/athena-aws-cmdb.yaml index 4365e6781d..8ead0e1db0 100644 --- a/athena-aws-cmdb/athena-aws-cmdb.yaml +++ b/athena-aws-cmdb/athena-aws-cmdb.yaml @@ -42,6 +42,8 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -53,7 +55,9 @@ Resources: spill_prefix: !Ref SpillPrefix FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-aws-cmdb:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-aws-cmdb:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with various AWS Services, making your resource inventories accessible via SQL." Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-clickhouse/athena-clickhouse.yaml b/athena-clickhouse/athena-clickhouse.yaml index 259aae7198..8f65ef1018 100644 --- a/athena-clickhouse/athena-clickhouse.yaml +++ b/athena-clickhouse/athena-clickhouse.yaml @@ -59,6 +59,8 @@ Parameters: Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -71,7 +73,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-clickhouse:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-clickhouse:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with ClickHouse using JDBC" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-cloudera-hive/Dockerfile b/athena-cloudera-hive/Dockerfile index b386d6db35..266ff4b120 100644 --- a/athena-cloudera-hive/Dockerfile +++ b/athena-cloudera-hive/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-cloudera-hive-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-cloudera-hive-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-cloudera-hive/athena-cloudera-hive.yaml b/athena-cloudera-hive/athena-cloudera-hive.yaml index 46dea219ae..2ec64cad7b 100644 --- a/athena-cloudera-hive/athena-cloudera-hive.yaml +++ b/athena-cloudera-hive/athena-cloudera-hive.yaml @@ -54,6 +54,8 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -66,7 +68,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudera-hive:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudera-hive:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.cloudera.HiveMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with Coludera Hive using JDBC" diff --git a/athena-cloudera-impala/Dockerfile b/athena-cloudera-impala/Dockerfile index 0f0736a9af..67742679e0 100644 --- a/athena-cloudera-impala/Dockerfile +++ b/athena-cloudera-impala/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-cloudera-impala-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-cloudera-impala-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-cloudera-impala/athena-cloudera-impala.yaml b/athena-cloudera-impala/athena-cloudera-impala.yaml index 5d9f9d2aab..55e1d742d7 100644 --- a/athena-cloudera-impala/athena-cloudera-impala.yaml +++ b/athena-cloudera-impala/athena-cloudera-impala.yaml @@ -59,6 +59,8 @@ Parameters: Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasLambdaEncryptionKmsKeyARN: !Not [ !Equals [ !Ref LambdaEncryptionKmsKeyARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -71,7 +73,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudera-impala:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudera-impala:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.cloudera.ImpalaMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with Cloudera Impala using JDBC" diff --git a/athena-cloudwatch-metrics/athena-cloudwatch-metrics.yaml b/athena-cloudwatch-metrics/athena-cloudwatch-metrics.yaml index 974b979e37..5f4ce7585c 100644 --- a/athena-cloudwatch-metrics/athena-cloudwatch-metrics.yaml +++ b/athena-cloudwatch-metrics/athena-cloudwatch-metrics.yaml @@ -42,6 +42,8 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -53,7 +55,9 @@ Resources: spill_prefix: !Ref SpillPrefix FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudwatch-metrics:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudwatch-metrics:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with Cloudwatch Metrics, making your metrics data accessible via SQL" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-cloudwatch/athena-cloudwatch.yaml b/athena-cloudwatch/athena-cloudwatch.yaml index 15eef67930..860dbf9c5d 100644 --- a/athena-cloudwatch/athena-cloudwatch.yaml +++ b/athena-cloudwatch/athena-cloudwatch.yaml @@ -54,7 +54,8 @@ Conditions: NotHasLambdaRole: !Equals [!Ref LambdaRole, ""] HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] CreateKMSPolicy: !And [ !Condition HasKMSKeyId, !Condition NotHasLambdaRole ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -67,7 +68,9 @@ Resources: kms_key_id: !If [HasKMSKeyId, !Ref KMSKeyId, !Ref "AWS::NoValue"] FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudwatch:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-cloudwatch:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with Cloudwatch, making your log accessible via SQL" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-datalakegen2/Dockerfile b/athena-datalakegen2/Dockerfile index 3bc46f1ded..d6667524ad 100644 --- a/athena-datalakegen2/Dockerfile +++ b/athena-datalakegen2/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-datalakegen2-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-datalakegen2-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-datalakegen2/athena-datalakegen2.yaml b/athena-datalakegen2/athena-datalakegen2.yaml index 4fe96f0e65..1318145981 100644 --- a/athena-datalakegen2/athena-datalakegen2.yaml +++ b/athena-datalakegen2/athena-datalakegen2.yaml @@ -60,6 +60,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -72,7 +74,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-datalakegen2:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-datalakegen2:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.datalakegen2.DataLakeGen2MuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with DataLake Gen2 using JDBC" diff --git a/athena-db2-as400/Dockerfile b/athena-db2-as400/Dockerfile index 7c2e7b4367..cac4944c1a 100644 --- a/athena-db2-as400/Dockerfile +++ b/athena-db2-as400/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-db2-as400-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-db2-as400-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-db2-as400/athena-db2-as400.yaml b/athena-db2-as400/athena-db2-as400.yaml index 8faa8eda20..ac6dca58a5 100644 --- a/athena-db2-as400/athena-db2-as400.yaml +++ b/athena-db2-as400/athena-db2-as400.yaml @@ -61,6 +61,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -73,7 +75,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-db2-as400:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-db2-as400:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.db2as400.Db2As400MuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with DB2 on iSeries (AS400) using JDBC" diff --git a/athena-db2/Dockerfile b/athena-db2/Dockerfile index c3172649e7..26e4d3746a 100644 --- a/athena-db2/Dockerfile +++ b/athena-db2/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-db2-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-db2-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-db2/athena-db2.yaml b/athena-db2/athena-db2.yaml index 4a7567d589..711652357a 100644 --- a/athena-db2/athena-db2.yaml +++ b/athena-db2/athena-db2.yaml @@ -61,6 +61,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -73,7 +75,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-db2:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-db2:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.db2.Db2MuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with DB2 using JDBC" diff --git a/athena-docdb/athena-docdb.yaml b/athena-docdb/athena-docdb.yaml index 588b05f52e..5b8a91261a 100644 --- a/athena-docdb/athena-docdb.yaml +++ b/athena-docdb/athena-docdb.yaml @@ -55,6 +55,8 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -67,7 +69,9 @@ Resources: default_docdb: !Ref DocDBConnectionString FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-docdb:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-docdb:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with DocumentDB, making your DocumentDB data accessible via SQL." Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-dynamodb/athena-dynamodb.yaml b/athena-dynamodb/athena-dynamodb.yaml index fe07b4c27f..ec2201408f 100644 --- a/athena-dynamodb/athena-dynamodb.yaml +++ b/athena-dynamodb/athena-dynamodb.yaml @@ -54,7 +54,8 @@ Conditions: NotHasLambdaRole: !Equals [!Ref LambdaRole, ""] HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] CreateKMSPolicy: !And [!Condition HasKMSKeyId, !Condition NotHasLambdaRole] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -67,7 +68,9 @@ Resources: kms_key_id: !If [HasKMSKeyId, !Ref KMSKeyId, !Ref "AWS::NoValue"] FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-dynamodb:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-dynamodb:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with DynamoDB, making your tables accessible via SQL" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-elasticsearch/athena-elasticsearch.yaml b/athena-elasticsearch/athena-elasticsearch.yaml index 7330324293..f99f6800c1 100644 --- a/athena-elasticsearch/athena-elasticsearch.yaml +++ b/athena-elasticsearch/athena-elasticsearch.yaml @@ -86,7 +86,8 @@ Parameters: Conditions: IsVPCAccessSelected: !Equals [!Ref IsVPCAccess, true] HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -103,7 +104,9 @@ Resources: query_scroll_timeout: !Ref QueryScrollTimeout FunctionName: !Sub "${AthenaCatalogName}" PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-elasticsearch:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-elasticsearch:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "The Elasticsearch Lambda Connector provides Athena users the ability to query data stored on Elasticsearch clusters." Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-elasticsearch/pom.xml b/athena-elasticsearch/pom.xml index 316cf40b7f..0d040efdbc 100644 --- a/athena-elasticsearch/pom.xml +++ b/athena-elasticsearch/pom.xml @@ -73,7 +73,7 @@ org.elasticsearch.client elasticsearch-rest-client - 8.15.2 + 8.15.3 diff --git a/athena-example/athena-example.yaml b/athena-example/athena-example.yaml index 8f019191ea..19230b92a6 100644 --- a/athena-example/athena-example.yaml +++ b/athena-example/athena-example.yaml @@ -42,7 +42,6 @@ Parameters: Description: "WARNING: If set to 'true' encryption for spilled data is disabled." Default: "false" Type: String - Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' diff --git a/athena-gcs/athena-gcs.yaml b/athena-gcs/athena-gcs.yaml index a12e60d369..aabe3fa4e2 100644 --- a/athena-gcs/athena-gcs.yaml +++ b/athena-gcs/athena-gcs.yaml @@ -47,7 +47,8 @@ Parameters: Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: AthenaGCSConnector: Type: 'AWS::Serverless::Function' @@ -60,7 +61,9 @@ Resources: secret_manager_gcp_creds_name: !Ref GCSSecretName FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-gcs:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-gcs:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Amazon Athena GCS Connector" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-google-bigquery/athena-google-bigquery.yaml b/athena-google-bigquery/athena-google-bigquery.yaml index f75b53334f..b113376133 100644 --- a/athena-google-bigquery/athena-google-bigquery.yaml +++ b/athena-google-bigquery/athena-google-bigquery.yaml @@ -65,6 +65,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: AthenaBigQueryConnector: Type: 'AWS::Serverless::Function' @@ -80,7 +82,9 @@ Resources: GOOGLE_APPLICATION_CREDENTIALS: '/tmp/service-account.json' FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-google-bigquery:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-google-bigquery:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with BigQuery using Google SDK" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-hbase/athena-hbase.yaml b/athena-hbase/athena-hbase.yaml index c9d70a24e0..ba51dfd827 100644 --- a/athena-hbase/athena-hbase.yaml +++ b/athena-hbase/athena-hbase.yaml @@ -70,6 +70,8 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -86,7 +88,9 @@ Resources: hbase_rpc_protection: !Ref HbaseRpcProtection FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-hbase:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-hbase:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with HBase, making your HBase data accessible via SQL" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-hortonworks-hive/Dockerfile b/athena-hortonworks-hive/Dockerfile index 20b44e9ca1..d0c666fb70 100644 --- a/athena-hortonworks-hive/Dockerfile +++ b/athena-hortonworks-hive/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-hortonworks-hive-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-hortonworks-hive-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-hortonworks-hive/athena-hortonworks-hive.yaml b/athena-hortonworks-hive/athena-hortonworks-hive.yaml index 8613bf4e83..85f8dfc206 100644 --- a/athena-hortonworks-hive/athena-hortonworks-hive.yaml +++ b/athena-hortonworks-hive/athena-hortonworks-hive.yaml @@ -58,6 +58,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -70,7 +72,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-hortonworks-hive:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-hortonworks-hive:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.hortonworks.HiveMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with Hortonworks Hive using JDBC" diff --git a/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcSplitQueryBuilder.java b/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcSplitQueryBuilder.java index 104f271017..a532d02aa2 100644 --- a/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcSplitQueryBuilder.java +++ b/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcSplitQueryBuilder.java @@ -336,7 +336,7 @@ else if (singleValues.size() > 1) { return "(" + Joiner.on(" OR ").join(disjuncts) + ")"; } - protected String toPredicate(String columnName, String operator, Object value, ArrowType type, + private String toPredicate(String columnName, String operator, Object value, ArrowType type, List accumulator) { accumulator.add(new TypeAndValue(type, value)); diff --git a/athena-kafka/athena-kafka.yaml b/athena-kafka/athena-kafka.yaml index 4b88028448..2479b64225 100644 --- a/athena-kafka/athena-kafka.yaml +++ b/athena-kafka/athena-kafka.yaml @@ -85,7 +85,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: AthenaKafkaConnector: Type: 'AWS::Serverless::Function' @@ -102,7 +103,9 @@ Resources: auth_type: !Ref AuthType FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-kafka:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-kafka:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with Kafka clusters" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-msk/athena-msk.yaml b/athena-msk/athena-msk.yaml index a6dd3b0f6a..f04269c75f 100644 --- a/athena-msk/athena-msk.yaml +++ b/athena-msk/athena-msk.yaml @@ -81,7 +81,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: AthenaMSKConnector: Type: 'AWS::Serverless::Function' @@ -97,7 +98,9 @@ Resources: auth_type: !Ref AuthType FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-msk:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-msk:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with MSK clusters" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-msk/pom.xml b/athena-msk/pom.xml index c7a37a59a0..e12aad9167 100644 --- a/athena-msk/pom.xml +++ b/athena-msk/pom.xml @@ -98,7 +98,7 @@ software.amazon.glue schema-registry-serde - 1.1.20 + 1.1.21 io.confluent diff --git a/athena-mysql/Dockerfile b/athena-mysql/Dockerfile index ca7a43e642..64e7a2115a 100644 --- a/athena-mysql/Dockerfile +++ b/athena-mysql/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-mysql-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-mysql-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-mysql/athena-mysql.yaml b/athena-mysql/athena-mysql.yaml index 0901916aee..c657e41e64 100644 --- a/athena-mysql/athena-mysql.yaml +++ b/athena-mysql/athena-mysql.yaml @@ -59,6 +59,8 @@ Parameters: Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -71,7 +73,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-mysql:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-mysql:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.mysql.MySqlMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with MySQL using JDBC" diff --git a/athena-mysql/pom.xml b/athena-mysql/pom.xml index b281bdbd80..a8c9ac7b4e 100644 --- a/athena-mysql/pom.xml +++ b/athena-mysql/pom.xml @@ -35,7 +35,7 @@ com.mysql mysql-connector-j - 9.0.0 + 9.1.0 com.google.protobuf diff --git a/athena-neptune/athena-neptune.yaml b/athena-neptune/athena-neptune.yaml index b1b92e2c9c..ebf995f804 100644 --- a/athena-neptune/athena-neptune.yaml +++ b/athena-neptune/athena-neptune.yaml @@ -77,7 +77,8 @@ Parameters: Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -97,7 +98,9 @@ Resources: enable_caseinsensitivematch: !Ref EnableCaseInsensitiveMatch FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-neptune:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-neptune:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with Neptune, making your Neptune graph data accessible via SQL." Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-oracle/Dockerfile b/athena-oracle/Dockerfile index 3cfbda8854..eaeafac1b6 100644 --- a/athena-oracle/Dockerfile +++ b/athena-oracle/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-oracle-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-oracle-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-oracle/athena-oracle.yaml b/athena-oracle/athena-oracle.yaml index dc723ea6d2..6badf23d43 100644 --- a/athena-oracle/athena-oracle.yaml +++ b/athena-oracle/athena-oracle.yaml @@ -70,6 +70,8 @@ Conditions: NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -83,7 +85,9 @@ Resources: is_FIPS_Enabled: !Ref IsFIPSEnabled FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-oracle:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-oracle:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.oracle.OracleMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with ORACLE using JDBC" diff --git a/athena-postgresql/athena-postgresql.yaml b/athena-postgresql/athena-postgresql.yaml index dccd609e78..79d2013130 100644 --- a/athena-postgresql/athena-postgresql.yaml +++ b/athena-postgresql/athena-postgresql.yaml @@ -69,6 +69,8 @@ Parameters: Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -82,7 +84,9 @@ Resources: default_scale: !Ref DefaultScale FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-postgresql:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-postgresql:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ !Sub "com.amazonaws.athena.connectors.postgresql.${CompositeHandler}" ] Description: "Enables Amazon Athena to communicate with PostgreSQL using JDBC" diff --git a/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlMetadataHandler.java b/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlMetadataHandler.java index 544c1513ca..64e3159440 100644 --- a/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlMetadataHandler.java +++ b/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlMetadataHandler.java @@ -39,6 +39,7 @@ import com.amazonaws.athena.connector.lambda.metadata.optimizations.OptimizationSubType; import com.amazonaws.athena.connector.lambda.metadata.optimizations.pushdown.ComplexExpressionPushdownSubType; import com.amazonaws.athena.connector.lambda.metadata.optimizations.pushdown.FilterPushdownSubType; +import com.amazonaws.athena.connector.lambda.metadata.optimizations.pushdown.HintsSubtype; import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionConfig; import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionInfo; import com.amazonaws.athena.connectors.jdbc.connection.GenericJdbcConnectionFactory; @@ -97,6 +98,9 @@ public class PostGreSqlMetadataHandler static final String LIST_PAGINATED_TABLES_QUERY = "SELECT a.\"TABLE_NAME\", a.\"TABLE_SCHEM\" FROM ((SELECT table_name as \"TABLE_NAME\", table_schema as \"TABLE_SCHEM\" FROM information_schema.tables WHERE table_schema = ?) UNION (SELECT matviewname as \"TABLE_NAME\", schemaname as \"TABLE_SCHEM\" from pg_catalog.pg_matviews mv where has_table_privilege(format('%I.%I', mv.schemaname, mv.matviewname), 'select') and schemaname = ?)) AS a ORDER BY a.\"TABLE_NAME\" LIMIT ? OFFSET ?"; + //Session Property Flag that hints to the engine that the data source is using none default collation + protected static final String NON_DEFAULT_COLLATE = "non_default_collate"; + /** * Instantiates handler to be used by Lambda function directly. * @@ -143,6 +147,14 @@ public GetDataSourceCapabilitiesResponse doGetDataSourceCapabilities(BlockAlloca )); jdbcQueryPassthrough.addQueryPassthroughCapabilityIfEnabled(capabilities, configOptions); + + //Provide a hint to the engine that postgresql is using default collate settings + //Which doesn't match Athena's Engine Collation; this disabling Predicate pushdown + boolean nonDefaultCollate = Boolean.valueOf(this.configOptions.getOrDefault(NON_DEFAULT_COLLATE, "false")); + if (nonDefaultCollate) { + capabilities.put(DataSourceOptimizations.DATA_SOURCE_HINTS.withSupportedSubTypes(HintsSubtype.NON_DEFAULT_COLLATE)); + } + return new GetDataSourceCapabilitiesResponse(request.getCatalogName(), capabilities.build()); } diff --git a/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlQueryStringBuilder.java b/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlQueryStringBuilder.java index 046d7049ca..65b6c772d8 100644 --- a/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlQueryStringBuilder.java +++ b/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlQueryStringBuilder.java @@ -1,7 +1,4 @@ -/*- - * #%L - * athena-postgresql - * %% +/*- #%L athena-postgresql %% * Copyright (C) 2019 Amazon Web Services * %% * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,10 +20,7 @@ import com.amazonaws.athena.connector.lambda.domain.predicate.Constraints; import com.amazonaws.athena.connectors.jdbc.manager.FederationExpressionParser; import com.amazonaws.athena.connectors.jdbc.manager.JdbcSplitQueryBuilder; -import com.amazonaws.athena.connectors.jdbc.manager.TypeAndValue; import com.google.common.base.Strings; -import org.apache.arrow.vector.types.Types; -import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Schema; import java.sql.Connection; @@ -37,8 +31,6 @@ import java.util.Objects; import java.util.stream.Collectors; -import static java.lang.String.format; - /** * Extends {@link JdbcSplitQueryBuilder} and implements PostGreSql specific SQL clauses for split. * @@ -47,13 +39,9 @@ public class PostGreSqlQueryStringBuilder extends JdbcSplitQueryBuilder { - private final java.util.Map configOptions; - private final String postgresqlCollateExperimentalFlag = "postgresql_collate_experimental_flag"; - - public PostGreSqlQueryStringBuilder(final String quoteCharacters, final FederationExpressionParser federationExpressionParser, final java.util.Map configOptions) + public PostGreSqlQueryStringBuilder(final String quoteCharacters, final FederationExpressionParser federationExpressionParser) { super(quoteCharacters, federationExpressionParser); - this.configOptions = configOptions; } @Override @@ -106,10 +94,10 @@ protected String getFromClauseWithSplit(String catalog, String schema, String ta if (PostGreSqlMetadataHandler.ALL_PARTITIONS.equals(partitionSchemaName) || PostGreSqlMetadataHandler.ALL_PARTITIONS.equals(partitionName)) { // No partitions - return format(" FROM %s ", tableName); + return String.format(" FROM %s ", tableName); } - return format(" FROM %s.%s ", quote(partitionSchemaName), quote(partitionName)); + return String.format(" FROM %s.%s ", quote(partitionSchemaName), quote(partitionName)); } @Override @@ -122,43 +110,4 @@ protected List getPartitionWhereClauses(final Split split) return Collections.emptyList(); } - - protected String toPredicate(String columnName, String operator, Object value, ArrowType type, - List accumulator) - { - if (isPostgresqlCollateExperimentalFlagEnabled()) { - Types.MinorType minorType = Types.getMinorTypeForArrowType(type); - //Only check for varchar; as it's the only collate-able type - //Only a range that is applicable - if (minorType.equals(Types.MinorType.VARCHAR) && isOperatorARange(operator)) { - accumulator.add(new TypeAndValue(type, value)); - return format("%s %s ? COLLATE \"C\"", quote(columnName), operator); - } - } - // Default to parent's behavior - return super.toPredicate(columnName, operator, value, type, accumulator); - } - - /** - * Flags to check if experimental flag to allow different collate for postgresql - * @return true if a flag is set; default otherwise to false; - */ - private boolean isPostgresqlCollateExperimentalFlagEnabled() - { - String flag = configOptions.getOrDefault(postgresqlCollateExperimentalFlag, "false"); - return flag.equalsIgnoreCase("true"); - } - - private boolean isOperatorARange(String operator) - { - switch (operator) { - case ">": - case "<": - case ">=": - case "<=": - return true; - default: - return false; - } - } } diff --git a/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandler.java b/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandler.java index 98a8cd089b..fe4837c95a 100644 --- a/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandler.java +++ b/athena-postgresql/src/main/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandler.java @@ -70,7 +70,7 @@ public PostGreSqlRecordHandler(DatabaseConnectionConfig databaseConnectionConfig { this(databaseConnectionConfig, S3Client.create(), SecretsManagerClient.create(), AthenaClient.create(), new GenericJdbcConnectionFactory(databaseConnectionConfig, PostGreSqlMetadataHandler.JDBC_PROPERTIES, new DatabaseConnectionInfo(POSTGRESQL_DRIVER_CLASS, POSTGRESQL_DEFAULT_PORT)), - new PostGreSqlQueryStringBuilder(POSTGRES_QUOTE_CHARACTER, new PostgreSqlFederationExpressionParser(POSTGRES_QUOTE_CHARACTER), configOptions), configOptions); + new PostGreSqlQueryStringBuilder(POSTGRES_QUOTE_CHARACTER, new PostgreSqlFederationExpressionParser(POSTGRES_QUOTE_CHARACTER)), configOptions); } @VisibleForTesting diff --git a/athena-postgresql/src/test/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandlerTest.java b/athena-postgresql/src/test/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandlerTest.java index b11f7f965d..e54c337d8e 100644 --- a/athena-postgresql/src/test/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandlerTest.java +++ b/athena-postgresql/src/test/java/com/amazonaws/athena/connectors/postgresql/PostGreSqlRecordHandlerTest.java @@ -56,7 +56,6 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Collections; -import java.util.Map; import java.util.concurrent.TimeUnit; import static com.amazonaws.athena.connectors.postgresql.PostGreSqlConstants.POSTGRES_NAME; @@ -76,7 +75,7 @@ public class PostGreSqlRecordHandlerTest extends TestBase private SecretsManagerClient secretsManager; private AthenaClient athena; private MockedStatic mockedPostGreSqlMetadataHandler; - private DatabaseConnectionConfig databaseConnectionConfig; + @Before public void setup() throws Exception @@ -87,8 +86,8 @@ public void setup() this.connection = Mockito.mock(Connection.class); this.jdbcConnectionFactory = Mockito.mock(JdbcConnectionFactory.class); Mockito.when(this.jdbcConnectionFactory.getConnection(nullable(JdbcCredentialProvider.class))).thenReturn(this.connection); - jdbcSplitQueryBuilder = new PostGreSqlQueryStringBuilder("\"", new PostgreSqlFederationExpressionParser("\""), Collections.emptyMap()); - databaseConnectionConfig = new DatabaseConnectionConfig("testCatalog", POSTGRES_NAME, + jdbcSplitQueryBuilder = new PostGreSqlQueryStringBuilder("\"", new PostgreSqlFederationExpressionParser("\"")); + final DatabaseConnectionConfig databaseConnectionConfig = new DatabaseConnectionConfig("testCatalog", POSTGRES_NAME, "postgres://jdbc:postgresql://hostname/user=A&password=B"); this.postGreSqlRecordHandler = new PostGreSqlRecordHandler(databaseConnectionConfig, amazonS3, secretsManager, athena, jdbcConnectionFactory, jdbcSplitQueryBuilder, com.google.common.collect.ImmutableMap.of()); @@ -230,95 +229,6 @@ public void buildSplitSqlForDateTest() logger.info("buildSplitSqlForDateTest - exit"); } - @Test - public void buildSplitSqlCollateAwareQuery() - throws SQLException - { - logger.info("buildSplitSqlCollateAwareQuery - enter"); - - TableName tableName = new TableName("testSchema", "testTable"); - - SchemaBuilder schemaBuilder = SchemaBuilder.newBuilder(); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol1", Types.MinorType.INT.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol2", Types.MinorType.VARCHAR.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol3", Types.MinorType.BIGINT.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol4", Types.MinorType.FLOAT4.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol5", Types.MinorType.SMALLINT.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol6", Types.MinorType.TINYINT.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol7", Types.MinorType.FLOAT8.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol8", Types.MinorType.BIT.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol9", new ArrowType.Decimal(8, 2)).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("testCol10", new ArrowType.Utf8()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("partition_schema_name", Types.MinorType.VARCHAR.getType()).build()); - schemaBuilder.addField(FieldBuilder.newBuilder("partition_name", Types.MinorType.VARCHAR.getType()).build()); - Schema schema = schemaBuilder.build(); - - Split split = Mockito.mock(Split.class); - Mockito.when(split.getProperties()).thenReturn(ImmutableMap.of("partition_schema_name", "s0", "partition_name", "p0")); - Mockito.when(split.getProperty(Mockito.eq(com.amazonaws.athena.connectors.postgresql.PostGreSqlMetadataHandler.BLOCK_PARTITION_SCHEMA_COLUMN_NAME))).thenReturn("s0"); - Mockito.when(split.getProperty(Mockito.eq(com.amazonaws.athena.connectors.postgresql.PostGreSqlMetadataHandler.BLOCK_PARTITION_COLUMN_NAME))).thenReturn("p0"); - - Range range1a = Mockito.mock(Range.class, Mockito.RETURNS_DEEP_STUBS); - Mockito.when(range1a.isSingleValue()).thenReturn(true); - Mockito.when(range1a.getLow().getValue()).thenReturn(1); - Range range1b = Mockito.mock(Range.class, Mockito.RETURNS_DEEP_STUBS); - Mockito.when(range1b.isSingleValue()).thenReturn(true); - Mockito.when(range1b.getLow().getValue()).thenReturn(2); - ValueSet valueSet1 = Mockito.mock(SortedRangeSet.class, Mockito.RETURNS_DEEP_STUBS); - Mockito.when(valueSet1.getRanges().getOrderedRanges()).thenReturn(ImmutableList.of(range1a, range1b)); - - ValueSet valueSet2 = getRangeSet(Marker.Bound.EXACTLY, "1", Marker.Bound.BELOW, "10"); - ValueSet valueSet3 = getRangeSet(Marker.Bound.ABOVE, 2L, Marker.Bound.EXACTLY, 20L); - ValueSet valueSet4 = getSingleValueSet(1.1F); - ValueSet valueSet5 = getSingleValueSet(1); - ValueSet valueSet6 = getSingleValueSet(0); - ValueSet valueSet7 = getSingleValueSet(1.2d); - ValueSet valueSet8 = getSingleValueSet(true); - ValueSet valueSet9 = getSingleValueSet(BigDecimal.valueOf(12.34)); - ValueSet valueSet10 = getSingleValueSet("A"); - - Constraints constraints = Mockito.mock(Constraints.class); - Mockito.when(constraints.getSummary()).thenReturn(new ImmutableMap.Builder() - .put("testCol1", valueSet1) - .put("testCol2", valueSet2) - .put("testCol3", valueSet3) - .put("testCol4", valueSet4) - .put("testCol5", valueSet5) - .put("testCol6", valueSet6) - .put("testCol7", valueSet7) - .put("testCol8", valueSet8) - .put("testCol9", valueSet9) - .put("testCol10", valueSet10) - .build()); - - String expectedSql = "SELECT \"testCol1\", \"testCol2\", \"testCol3\", \"testCol4\", \"testCol5\", \"testCol6\", \"testCol7\", \"testCol8\", \"testCol9\", RTRIM(\"testCol10\") AS \"testCol10\" FROM \"s0\".\"p0\" WHERE (\"testCol1\" IN (?,?)) AND ((\"testCol2\" >= ? COLLATE \"C\" AND \"testCol2\" < ? COLLATE \"C\")) AND ((\"testCol3\" > ? AND \"testCol3\" <= ?)) AND (\"testCol4\" = ?) AND (\"testCol5\" = ?) AND (\"testCol6\" = ?) AND (\"testCol7\" = ?) AND (\"testCol8\" = ?) AND (\"testCol9\" = ?) AND (\"testCol10\" = ?)"; - PreparedStatement expectedPreparedStatement = Mockito.mock(PreparedStatement.class); - Mockito.when(this.connection.prepareStatement(Mockito.eq(expectedSql))).thenReturn(expectedPreparedStatement); - - //Setting Collate Aware query builder flag on - Map configOptions = ImmutableMap.of("postgresql_collate_experimental_flag", "true"); - PostGreSqlQueryStringBuilder localJdbcSplitQueryBuilder = new PostGreSqlQueryStringBuilder("\"", new PostgreSqlFederationExpressionParser("\""), configOptions); - PostGreSqlRecordHandler localPostgresqlRecordHandler = new PostGreSqlRecordHandler(databaseConnectionConfig, amazonS3, secretsManager, athena, jdbcConnectionFactory, localJdbcSplitQueryBuilder, configOptions); - PreparedStatement preparedStatement = localPostgresqlRecordHandler.buildSplitSql(this.connection, "testCatalogName", tableName, schema, constraints, split); - - Assert.assertEquals(expectedPreparedStatement, preparedStatement); - Mockito.verify(preparedStatement, Mockito.times(1)).setInt(1, 1); - Mockito.verify(preparedStatement, Mockito.times(1)).setInt(2, 2); - Mockito.verify(preparedStatement, Mockito.times(1)).setString(3, "1"); - Mockito.verify(preparedStatement, Mockito.times(1)).setString(4, "10"); - Mockito.verify(preparedStatement, Mockito.times(1)).setLong(5, 2L); - Mockito.verify(preparedStatement, Mockito.times(1)).setLong(6, 20L); - Mockito.verify(preparedStatement, Mockito.times(1)).setFloat(7, 1.1F); - Mockito.verify(preparedStatement, Mockito.times(1)).setShort(8, (short) 1); - Mockito.verify(preparedStatement, Mockito.times(1)).setByte(9, (byte) 0); - Mockito.verify(preparedStatement, Mockito.times(1)).setDouble(10, 1.2d); - Mockito.verify(preparedStatement, Mockito.times(1)).setBoolean(11, true); - Mockito.verify(preparedStatement, Mockito.times(1)).setBigDecimal(12, BigDecimal.valueOf(12.34)); - Mockito.verify(preparedStatement, Mockito.times(1)).setString(13, "A"); - - logger.info("buildSplitSqlCollateAwareQuery - exit"); - } - private ValueSet getSingleValueSet(Object value) { Range range = Mockito.mock(Range.class, Mockito.RETURNS_DEEP_STUBS); Mockito.when(range.isSingleValue()).thenReturn(true); diff --git a/athena-redis/athena-redis.yaml b/athena-redis/athena-redis.yaml index c3bc541752..5d18c7ced1 100644 --- a/athena-redis/athena-redis.yaml +++ b/athena-redis/athena-redis.yaml @@ -67,6 +67,8 @@ Parameters: Type: Number Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -82,7 +84,9 @@ Resources: qpt_db_number: !Ref QPTConnectionDBNumber FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-redis:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-redis:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with Redis, making your Redis data accessible via SQL" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-redshift/Dockerfile b/athena-redshift/Dockerfile index 36d20144ab..20a2afdef3 100644 --- a/athena-redshift/Dockerfile +++ b/athena-redshift/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-redshift-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-redshift-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-redshift/athena-redshift.yaml b/athena-redshift/athena-redshift.yaml index 1b1ed8671c..b395ce646f 100644 --- a/athena-redshift/athena-redshift.yaml +++ b/athena-redshift/athena-redshift.yaml @@ -67,6 +67,8 @@ Conditions: - !Condition NotHasLambdaRole - !Condition HasKMSKeyId HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -80,7 +82,9 @@ Resources: kms_key_id: !If [HasKMSKeyId, !Ref KMSKeyId, !Ref "AWS::NoValue"] FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-redshift:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-redshift:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.redshift.RedshiftMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with Redshift using JDBC" diff --git a/athena-redshift/src/main/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandler.java b/athena-redshift/src/main/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandler.java index 16de7ec46e..0b294de160 100644 --- a/athena-redshift/src/main/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandler.java +++ b/athena-redshift/src/main/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandler.java @@ -62,7 +62,7 @@ public RedshiftRecordHandler(DatabaseConnectionConfig databaseConnectionConfig, super(databaseConnectionConfig, S3Client.create(), SecretsManagerClient.create(), AthenaClient.create(), new GenericJdbcConnectionFactory(databaseConnectionConfig, PostGreSqlMetadataHandler.JDBC_PROPERTIES, new DatabaseConnectionInfo(REDSHIFT_DRIVER_CLASS, REDSHIFT_DEFAULT_PORT)), - new PostGreSqlQueryStringBuilder(POSTGRES_QUOTE_CHARACTER, new PostgreSqlFederationExpressionParser(POSTGRES_QUOTE_CHARACTER), configOptions), configOptions); + new PostGreSqlQueryStringBuilder(POSTGRES_QUOTE_CHARACTER, new PostgreSqlFederationExpressionParser(POSTGRES_QUOTE_CHARACTER)), configOptions); } @VisibleForTesting diff --git a/athena-redshift/src/test/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandlerTest.java b/athena-redshift/src/test/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandlerTest.java index 32f1218358..c3e73e0e1f 100644 --- a/athena-redshift/src/test/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandlerTest.java +++ b/athena-redshift/src/test/java/com/amazonaws/athena/connectors/redshift/RedshiftRecordHandlerTest.java @@ -90,7 +90,7 @@ public void setup() this.connection = Mockito.mock(Connection.class); this.jdbcConnectionFactory = Mockito.mock(JdbcConnectionFactory.class); Mockito.when(this.jdbcConnectionFactory.getConnection(nullable(JdbcCredentialProvider.class))).thenReturn(this.connection); - jdbcSplitQueryBuilder = new PostGreSqlQueryStringBuilder("\"", new PostgreSqlFederationExpressionParser("\""), Collections.emptyMap()); + jdbcSplitQueryBuilder = new PostGreSqlQueryStringBuilder("\"", new PostgreSqlFederationExpressionParser("\"")); final DatabaseConnectionConfig databaseConnectionConfig = new DatabaseConnectionConfig("testCatalog", REDSHIFT_NAME, "redshift://jdbc:redshift://hostname/user=A&password=B"); diff --git a/athena-saphana/Dockerfile b/athena-saphana/Dockerfile index f87466c2b3..827a67e65c 100644 --- a/athena-saphana/Dockerfile +++ b/athena-saphana/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-saphana.zip ${LAMBDA_TASK_ROOT} RUN jar xf athena-saphana.zip # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-saphana/athena-saphana.yaml b/athena-saphana/athena-saphana.yaml index cbfbf7fd92..68ecafe0f2 100644 --- a/athena-saphana/athena-saphana.yaml +++ b/athena-saphana/athena-saphana.yaml @@ -58,6 +58,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -70,7 +72,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-saphana:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-saphana:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.saphana.SaphanaMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with Teradata using JDBC" diff --git a/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaQueryStringBuilder.java b/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaQueryStringBuilder.java index ca65ce8efe..16ec948679 100644 --- a/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaQueryStringBuilder.java +++ b/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaQueryStringBuilder.java @@ -349,4 +349,11 @@ else if (singleValues.size() > 1) { return "(" + Joiner.on(" OR ").join(disjuncts) + ")"; } + + private String toPredicate(String columnName, String operator, Object value, ArrowType type, + List accumulator) + { + accumulator.add(new TypeAndValue(type, value)); + return quote(columnName) + " " + operator + " ?"; + } } diff --git a/athena-snowflake/Dockerfile b/athena-snowflake/Dockerfile index 006be5ce85..c14408ec23 100644 --- a/athena-snowflake/Dockerfile +++ b/athena-snowflake/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-snowflake.zip ${LAMBDA_TASK_ROOT} RUN jar xf athena-snowflake.zip # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-snowflake/athena-snowflake.yaml b/athena-snowflake/athena-snowflake.yaml index 409d6788e8..7b86121621 100644 --- a/athena-snowflake/athena-snowflake.yaml +++ b/athena-snowflake/athena-snowflake.yaml @@ -58,6 +58,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -70,7 +72,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-snowflake:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-snowflake:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.snowflake.SnowflakeMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with Snowflake using JDBC" diff --git a/athena-sqlserver/Dockerfile b/athena-sqlserver/Dockerfile index da1abd4f51..2456d9e94b 100644 --- a/athena-sqlserver/Dockerfile +++ b/athena-sqlserver/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-sqlserver-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-sqlserver-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-sqlserver/athena-sqlserver.yaml b/athena-sqlserver/athena-sqlserver.yaml index 4829e10d6f..4aa9804b16 100644 --- a/athena-sqlserver/athena-sqlserver.yaml +++ b/athena-sqlserver/athena-sqlserver.yaml @@ -65,6 +65,8 @@ Conditions: NotHasLambdaRole: !Equals [!Ref LambdaRoleARN, ""] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -77,7 +79,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-sqlserver:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-sqlserver:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.sqlserver.SqlServerMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with SQLSERVER using JDBC" diff --git a/athena-synapse/Dockerfile b/athena-synapse/Dockerfile index d9bdb76676..ce59270ed0 100644 --- a/athena-synapse/Dockerfile +++ b/athena-synapse/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-synapse-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-synapse-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-synapse/athena-synapse.yaml b/athena-synapse/athena-synapse.yaml index 6fd6c697c6..1c524545f8 100644 --- a/athena-synapse/athena-synapse.yaml +++ b/athena-synapse/athena-synapse.yaml @@ -66,7 +66,8 @@ Conditions: HasPermissionsBoundary: !Not [!Equals [!Ref PermissionsBoundaryARN, ""]] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -79,7 +80,9 @@ Resources: default: !Ref DefaultConnectionString FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-synapse:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-synapse:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.synapse.SynapseMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with SYNPASE using JDBC" diff --git a/athena-teradata/Dockerfile b/athena-teradata/Dockerfile index 6dea76d17d..996da2013f 100644 --- a/athena-teradata/Dockerfile +++ b/athena-teradata/Dockerfile @@ -6,4 +6,4 @@ COPY target/athena-teradata-2022.47.1.jar ${LAMBDA_TASK_ROOT} RUN jar xf athena-teradata-2022.47.1.jar # Command can be overwritten by providing a different command in the template directly. -# No need to specify here (already defined in .yaml file because legacy and connections use different) \ No newline at end of file +# No need to specify here (already defined in .yaml file because legacy and connections use different) diff --git a/athena-teradata/athena-teradata.yaml b/athena-teradata/athena-teradata.yaml index 4f8c52c9cf..987201837b 100644 --- a/athena-teradata/athena-teradata.yaml +++ b/athena-teradata/athena-teradata.yaml @@ -62,6 +62,8 @@ Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] HasSecurityGroups: !Not [ !Equals [ !Join ["", !Ref SecurityGroupIds], "" ] ] HasSubnets: !Not [ !Equals [ !Join ["", !Ref SubnetIds], "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: JdbcConnectorConfig: Type: 'AWS::Serverless::Function' @@ -75,7 +77,9 @@ Resources: partitioncount: !Ref PartitionCount FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-teradata:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-teradata:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] ImageConfig: Command: [ "com.amazonaws.athena.connectors.teradata.TeradataMuxCompositeHandler" ] Description: "Enables Amazon Athena to communicate with Teradata using JDBC" diff --git a/athena-timestream/athena-timestream.yaml b/athena-timestream/athena-timestream.yaml index 0f48b9393b..3062d1f435 100644 --- a/athena-timestream/athena-timestream.yaml +++ b/athena-timestream/athena-timestream.yaml @@ -42,6 +42,8 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -53,7 +55,9 @@ Resources: spill_prefix: !Ref SpillPrefix FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-timestream:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-timestream:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Enables Amazon Athena to communicate with Amazon Timestream, making your time series data accessible from Athena." Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-tpcds/athena-tpcds.yaml b/athena-tpcds/athena-tpcds.yaml index 4086565f7e..ed88425b11 100644 --- a/athena-tpcds/athena-tpcds.yaml +++ b/athena-tpcds/athena-tpcds.yaml @@ -42,6 +42,8 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' @@ -53,7 +55,9 @@ Resources: spill_prefix: !Ref SpillPrefix FunctionName: !Ref AthenaCatalogName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-tpcds:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-tpcds:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "This connector enables Amazon Athena to communicate with a randomly generated TPC-DS data source." Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-udfs/athena-udfs.yaml b/athena-udfs/athena-udfs.yaml index ff428e4c0d..8a36679838 100644 --- a/athena-udfs/athena-udfs.yaml +++ b/athena-udfs/athena-udfs.yaml @@ -34,13 +34,17 @@ Parameters: Type: String Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: ConnectorConfig: Type: 'AWS::Serverless::Function' Properties: FunctionName: !Ref LambdaFunctionName PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-udfs:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-udfs:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "This connector enables Amazon Athena to leverage common UDFs made available via Lambda." Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory diff --git a/athena-vertica/athena-vertica.yaml b/athena-vertica/athena-vertica.yaml index d02c913b3a..472e70e253 100644 --- a/athena-vertica/athena-vertica.yaml +++ b/athena-vertica/athena-vertica.yaml @@ -63,7 +63,8 @@ Parameters: Conditions: HasPermissionsBoundary: !Not [ !Equals [ !Ref PermissionsBoundaryARN, "" ] ] - + IsRegionBAH: !Equals [!Ref "AWS::Region", "me-south-1"] + IsRegionHKG: !Equals [!Ref "AWS::Region", "ap-east-1"] Resources: LambdaSecurityGroup: Type: 'AWS::EC2::SecurityGroup' @@ -83,7 +84,9 @@ Resources: FunctionName: !Sub "${AthenaCatalogName}" PackageType: "Image" - ImageUri: !Sub '292517598671.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-vertica:2022.47.1' + ImageUri: !Sub + - '${Account}.dkr.ecr.${AWS::Region}.amazonaws.com/athena-federation-repository-vertica:2022.47.1' + - Account: !If [IsRegionBAH, 084828588479, !If [IsRegionHKG, 183295418215, 292517598671]] Description: "Amazon Athena Vertica Connector" Timeout: !Ref LambdaTimeout MemorySize: !Ref LambdaMemory