Merge branch 'master' into client-tags-filtering

.github/workflows/ci.yml

@@ -740,6 +740,7 @@ jobs:
           REDSHIFT_IAM_ROLES: ${{ vars.REDSHIFT_IAM_ROLES }}
           REDSHIFT_VPC_SECURITY_GROUP_IDS: ${{ vars.REDSHIFT_VPC_SECURITY_GROUP_IDS }}
           REDSHIFT_S3_TPCH_TABLES_ROOT: ${{ vars.REDSHIFT_S3_TPCH_TABLES_ROOT }}
+          REDSHIFT_S3_UNLOAD_ROOT: ${{ vars.REDSHIFT_S3_UNLOAD_ROOT }}
         if: >-
           contains(matrix.modules, 'trino-redshift') &&
           (contains(matrix.profile, 'cloud-tests') || contains(matrix.profile, 'fte-tests')) &&
@@ -752,6 +753,7 @@ jobs:
             -Dtest.redshift.jdbc.password="${REDSHIFT_PASSWORD}" \
             -Dtest.redshift.jdbc.endpoint="${REDSHIFT_ENDPOINT}:${REDSHIFT_PORT}/" \
             -Dtest.redshift.s3.tpch.tables.root="${REDSHIFT_S3_TPCH_TABLES_ROOT}" \
+            -Dtest.redshift.s3.unload.root="${REDSHIFT_S3_UNLOAD_ROOT}" \
             -Dtest.redshift.iam.role="${REDSHIFT_IAM_ROLES}" \
             -Dtest.redshift.aws.region="${AWS_REGION}" \
             -Dtest.redshift.aws.access-key="${AWS_ACCESS_KEY_ID}" \
@@ -878,7 +880,6 @@ jobs:
             - suite-7-non-generic
             - suite-hive-transactional
             - suite-azure
-            - suite-delta-lake-databricks91
             - suite-delta-lake-databricks104
             - suite-delta-lake-databricks113
             - suite-delta-lake-databricks122
@@ -919,9 +920,6 @@ jobs:
               ignore exclusion if: >-
                 ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }}
 
-            - suite: suite-delta-lake-databricks91
-              ignore exclusion if: >-
-                ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
             - suite: suite-delta-lake-databricks104
               ignore exclusion if: >-
                 ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
@@ -989,7 +987,6 @@ jobs:
           AWS_REGION: ""
           TRINO_AWS_ACCESS_KEY_ID: ""
           TRINO_AWS_SECRET_ACCESS_KEY: ""
-          DATABRICKS_91_JDBC_URL: ""
           DATABRICKS_104_JDBC_URL: ""
           DATABRICKS_113_JDBC_URL: ""
           DATABRICKS_122_JDBC_URL: ""
@@ -1067,7 +1064,6 @@ jobs:
           AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
           TRINO_AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }}
           TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
-          DATABRICKS_91_JDBC_URL: ${{ vars.DATABRICKS_91_JDBC_URL }}
           DATABRICKS_104_JDBC_URL: ${{ vars.DATABRICKS_104_JDBC_URL }}
           DATABRICKS_113_JDBC_URL: ${{ vars.DATABRICKS_113_JDBC_URL }}
           DATABRICKS_122_JDBC_URL: ${{ vars.DATABRICKS_122_JDBC_URL }}

client/trino-cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.trino</groupId>
         <artifactId>trino-root</artifactId>
-        <version>468-SNAPSHOT</version>
+        <version>469-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

client/trino-client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.trino</groupId>
         <artifactId>trino-root</artifactId>
-        <version>468-SNAPSHOT</version>
+        <version>469-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

client/trino-client/src/main/java/io/trino/client/StatementStats.java

@@ -36,14 +36,19 @@ public class StatementStats
     private final int queuedSplits;
     private final int runningSplits;
     private final int completedSplits;
+    private final long planningTimeMillis;
+    private final long analysisTimeMillis;
     private final long cpuTimeMillis;
     private final long wallTimeMillis;
     private final long queuedTimeMillis;
     private final long elapsedTimeMillis;
+    private final long finishingTimeMillis;
+    private final long physicalInputTimeMillis;
     private final long processedRows;
     private final long processedBytes;
     private final long physicalInputBytes;
     private final long physicalWrittenBytes;
+    private final long internalNetworkInputBytes;
     private final long peakMemoryBytes;
     private final long spilledBytes;
     private final StageStats rootStage;
@@ -60,14 +65,19 @@ public StatementStats(
             @JsonProperty("queuedSplits") int queuedSplits,
             @JsonProperty("runningSplits") int runningSplits,
             @JsonProperty("completedSplits") int completedSplits,
+            @JsonProperty("planningTimeMillis") long planningTimeMillis,
+            @JsonProperty("analysisTimeMillis") long analysisTimeMillis,
             @JsonProperty("cpuTimeMillis") long cpuTimeMillis,
             @JsonProperty("wallTimeMillis") long wallTimeMillis,
             @JsonProperty("queuedTimeMillis") long queuedTimeMillis,
             @JsonProperty("elapsedTimeMillis") long elapsedTimeMillis,
+            @JsonProperty("finishingTimeMillis") long finishingTimeMillis,
+            @JsonProperty("physicalInputTimeMillis") long physicalInputTimeMillis,
             @JsonProperty("processedRows") long processedRows,
             @JsonProperty("processedBytes") long processedBytes,
             @JsonProperty("physicalInputBytes") long physicalInputBytes,
             @JsonProperty("physicalWrittenBytes") long physicalWrittenBytes,
+            @JsonProperty("internalNetworkInputBytes") long internalNetworkInputBytes,
             @JsonProperty("peakMemoryBytes") long peakMemoryBytes,
             @JsonProperty("spilledBytes") long spilledBytes,
             @JsonProperty("rootStage") StageStats rootStage)
@@ -82,14 +92,19 @@ public StatementStats(
         this.queuedSplits = queuedSplits;
         this.runningSplits = runningSplits;
         this.completedSplits = completedSplits;
+        this.planningTimeMillis = planningTimeMillis;
+        this.analysisTimeMillis = analysisTimeMillis;
         this.cpuTimeMillis = cpuTimeMillis;
         this.wallTimeMillis = wallTimeMillis;
         this.queuedTimeMillis = queuedTimeMillis;
         this.elapsedTimeMillis = elapsedTimeMillis;
+        this.finishingTimeMillis = finishingTimeMillis;
+        this.physicalInputTimeMillis = physicalInputTimeMillis;
         this.processedRows = processedRows;
         this.processedBytes = processedBytes;
         this.physicalInputBytes = physicalInputBytes;
         this.physicalWrittenBytes = physicalWrittenBytes;
+        this.internalNetworkInputBytes = internalNetworkInputBytes;
         this.peakMemoryBytes = peakMemoryBytes;
         this.spilledBytes = spilledBytes;
         this.rootStage = rootStage;
@@ -155,6 +170,18 @@ public int getCompletedSplits()
         return completedSplits;
     }
 
+    @JsonProperty
+    public long getPlanningTimeMillis()
+    {
+        return planningTimeMillis;
+    }
+
+    @JsonProperty
+    public long getAnalysisTimeMillis()
+    {
+        return analysisTimeMillis;
+    }
+
     @JsonProperty
     public long getCpuTimeMillis()
     {
@@ -179,6 +206,18 @@ public long getElapsedTimeMillis()
         return elapsedTimeMillis;
     }
 
+    @JsonProperty
+    public long getFinishingTimeMillis()
+    {
+        return finishingTimeMillis;
+    }
+
+    @JsonProperty
+    public long getPhysicalInputTimeMillis()
+    {
+        return physicalInputTimeMillis;
+    }
+
     @JsonProperty
     public long getProcessedRows()
     {
@@ -203,6 +242,12 @@ public long getPhysicalWrittenBytes()
         return physicalWrittenBytes;
     }
 
+    @JsonProperty
+    public long getInternalNetworkInputBytes()
+    {
+        return internalNetworkInputBytes;
+    }
+
     @JsonProperty
     public long getPeakMemoryBytes()
     {
@@ -236,14 +281,19 @@ public String toString()
                 .add("queuedSplits", queuedSplits)
                 .add("runningSplits", runningSplits)
                 .add("completedSplits", completedSplits)
+                .add("planningTimeMillis", planningTimeMillis)
+                .add("analysisTimeMillis", analysisTimeMillis)
                 .add("cpuTimeMillis", cpuTimeMillis)
                 .add("wallTimeMillis", wallTimeMillis)
                 .add("queuedTimeMillis", queuedTimeMillis)
                 .add("elapsedTimeMillis", elapsedTimeMillis)
+                .add("finishingTimeMillis", finishingTimeMillis)
+                .add("physicalInputTimeMillis", physicalInputTimeMillis)
                 .add("processedRows", processedRows)
                 .add("processedBytes", processedBytes)
                 .add("physicalInputBytes", physicalInputBytes)
                 .add("physicalWrittenBytes", physicalWrittenBytes)
+                .add("internalNetworkInputBytes", internalNetworkInputBytes)
                 .add("peakMemoryBytes", peakMemoryBytes)
                 .add("spilledBytes", spilledBytes)
                 .add("rootStage", rootStage)
@@ -267,14 +317,19 @@ public static class Builder
         private int queuedSplits;
         private int runningSplits;
         private int completedSplits;
+        private long planningTimeMillis;
+        private long analysisTimeMillis;
         private long cpuTimeMillis;
         private long wallTimeMillis;
         private long queuedTimeMillis;
         private long elapsedTimeMillis;
+        private long finishingTimeMillis;
+        private long physicalInputTimeMillis;
         private long processedRows;
         private long processedBytes;
         private long physicalInputBytes;
         private long physicalWrittenBytes;
+        private long internalNetworkInputBytes;
         private long peakMemoryBytes;
         private long spilledBytes;
         private StageStats rootStage;
@@ -341,6 +396,18 @@ public Builder setCompletedSplits(int completedSplits)
             return this;
         }
 
+        public Builder setPlanningTimeMillis(long planningTimeMillis)
+        {
+            this.planningTimeMillis = planningTimeMillis;
+            return this;
+        }
+
+        public Builder setAnalysisTimeMillis(long analysisTimeMillis)
+        {
+            this.analysisTimeMillis = analysisTimeMillis;
+            return this;
+        }
+
         public Builder setCpuTimeMillis(long cpuTimeMillis)
         {
             this.cpuTimeMillis = cpuTimeMillis;
@@ -365,6 +432,18 @@ public Builder setElapsedTimeMillis(long elapsedTimeMillis)
             return this;
         }
 
+        public Builder setFinishingTimeMillis(long finishingTimeMillis)
+        {
+            this.finishingTimeMillis = finishingTimeMillis;
+            return this;
+        }
+
+        public Builder setPhysicalInputTimeMillis(long physicalInputTimeMillis)
+        {
+            this.physicalInputTimeMillis = physicalInputTimeMillis;
+            return this;
+        }
+
         public Builder setProcessedRows(long processedRows)
         {
             this.processedRows = processedRows;
@@ -389,6 +468,12 @@ public Builder setPhysicalWrittenBytes(long physicalWrittenBytes)
             return this;
         }
 
+        public Builder setInternalNetworkInputBytes(long internalNetworkInputBytes)
+        {
+            this.internalNetworkInputBytes = internalNetworkInputBytes;
+            return this;
+        }
+
         public Builder setPeakMemoryBytes(long peakMemoryBytes)
         {
             this.peakMemoryBytes = peakMemoryBytes;
@@ -420,14 +505,19 @@ public StatementStats build()
                     queuedSplits,
                     runningSplits,
                     completedSplits,
+                    planningTimeMillis,
+                    analysisTimeMillis,
                     cpuTimeMillis,
                     wallTimeMillis,
                     queuedTimeMillis,
                     elapsedTimeMillis,
+                    finishingTimeMillis,
+                    physicalInputTimeMillis,
                     processedRows,
                     processedBytes,
                     physicalInputBytes,
                     physicalWrittenBytes,
+                    internalNetworkInputBytes,
                     peakMemoryBytes,
                     spilledBytes,
                     rootStage);

client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java

@@ -54,45 +54,13 @@ public static OkHttpClient.Builder toHttpClientBuilder(TrinoUri uri, String user
         OkHttpClient.Builder builder = unauthenticatedClientBuilder(uri, userAgent);
         setupCookieJar(builder);
 
-        if (!uri.isUseSecureConnection()) {
-            setupInsecureSsl(builder);
-        }
-
         if (uri.hasPassword()) {
             if (!uri.isUseSecureConnection()) {
                 throw new RuntimeException("TLS/SSL is required for authentication with username and password");
             }
             builder.addNetworkInterceptor(basicAuth(uri.getRequiredUser(), uri.getPassword().orElseThrow(() -> new RuntimeException("Password expected"))));
         }
 
-        if (uri.isUseSecureConnection()) {
-            ConnectionProperties.SslVerificationMode sslVerificationMode = uri.getSslVerification();
-            if (sslVerificationMode.equals(FULL) || sslVerificationMode.equals(CA)) {
-                setupSsl(
-                        builder,
-                        uri.getSslKeyStorePath(),
-                        uri.getSslKeyStorePassword(),
-                        uri.getSslKeyStoreType(),
-                        uri.getSslUseSystemKeyStore(),
-                        uri.getSslTrustStorePath(),
-                        uri.getSslTrustStorePassword(),
-                        uri.getSslTrustStoreType(),
-                        uri.getSslUseSystemTrustStore());
-            }
-            if (sslVerificationMode.equals(FULL)) {
-                uri.getHostnameInCertificate().ifPresent(certHostname ->
-                        setupAlternateHostnameVerification(builder, certHostname));
-            }
-
-            if (sslVerificationMode.equals(CA)) {
-                builder.hostnameVerifier((hostname, session) -> true);
-            }
-
-            if (sslVerificationMode.equals(NONE)) {
-                setupInsecureSsl(builder);
-            }
-        }
-
         if (uri.getKerberosRemoteServiceName().isPresent()) {
             if (!uri.isUseSecureConnection()) {
                 throw new RuntimeException("TLS/SSL is required for Kerberos authentication");
@@ -145,7 +113,6 @@ public static OkHttpClient.Builder toHttpClientBuilder(TrinoUri uri, String user
             builder.addNetworkInterceptor(authenticator);
         }
 
-        uri.getDnsResolver().ifPresent(resolverClass -> builder.dns(instantiateDnsResolver(resolverClass, uri.getDnsResolverContext())::lookup));
         return builder;
     }
 
@@ -157,6 +124,40 @@ public static OkHttpClient.Builder unauthenticatedClientBuilder(TrinoUri uri, St
         setupHttpProxy(builder, uri.getHttpProxy());
         setupTimeouts(builder, toIntExact(uri.getTimeout().toMillis()), TimeUnit.MILLISECONDS);
         setupHttpLogging(builder, uri.getHttpLoggingLevel());
+
+        if (uri.isUseSecureConnection()) {
+            ConnectionProperties.SslVerificationMode sslVerificationMode = uri.getSslVerification();
+            if (sslVerificationMode.equals(FULL) || sslVerificationMode.equals(CA)) {
+                setupSsl(
+                        builder,
+                        uri.getSslKeyStorePath(),
+                        uri.getSslKeyStorePassword(),
+                        uri.getSslKeyStoreType(),
+                        uri.getSslUseSystemKeyStore(),
+                        uri.getSslTrustStorePath(),
+                        uri.getSslTrustStorePassword(),
+                        uri.getSslTrustStoreType(),
+                        uri.getSslUseSystemTrustStore());
+            }
+            if (sslVerificationMode.equals(FULL)) {
+                uri.getHostnameInCertificate().ifPresent(certHostname ->
+                        setupAlternateHostnameVerification(builder, certHostname));
+            }
+
+            if (sslVerificationMode.equals(CA)) {
+                builder.hostnameVerifier((hostname, session) -> true);
+            }
+
+            if (sslVerificationMode.equals(NONE)) {
+                setupInsecureSsl(builder);
+            }
+        }
+        else {
+            setupInsecureSsl(builder);
+        }
+
+        uri.getDnsResolver().ifPresent(resolverClass -> builder.dns(instantiateDnsResolver(resolverClass, uri.getDnsResolverContext())::lookup));
+
         return builder;
     }
 

client/trino-client/src/test/java/io/trino/client/TestRetry.java

@@ -143,7 +143,7 @@ private String newQueryResults(String state)
                 TypedQueryData.of(IntStream.range(0, numRecords)
                         .mapToObj(index -> Stream.of((Object) index, "a").collect(toList()))
                         .collect(toList())),
-                new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
+                new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
                 null,
                 ImmutableList.of(),
                 null,