From c95b12d0cea171311044d68209d2a8b5070a7e0b Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Tue, 10 Dec 2024 15:37:17 -0800 Subject: [PATCH 001/158] Restructure SQL routine docs Move them in appropriate folders for user-defined functions and SQL user-defined functions. Update all references so that the docs build process fully works. --- .../admin/properties-sql-environment.md | 2 +- docs/src/main/sphinx/functions.md | 2 +- docs/src/main/sphinx/functions/conditional.md | 4 +- docs/src/main/sphinx/index.md | 2 +- docs/src/main/sphinx/release/release-431.md | 2 +- docs/src/main/sphinx/release/release-436.md | 2 +- docs/src/main/sphinx/release/release-446.md | 4 +- docs/src/main/sphinx/routines.md | 25 -------- docs/src/main/sphinx/routines/begin.md | 57 ------------------- docs/src/main/sphinx/sql/create-function.md | 6 +- docs/src/main/sphinx/sql/drop-function.md | 2 +- docs/src/main/sphinx/sql/select.md | 2 +- .../main/sphinx/sql/show-create-function.md | 2 +- docs/src/main/sphinx/sql/show-functions.md | 4 +- docs/src/main/sphinx/udf.md | 14 +++++ .../main/sphinx/{routines => udf}/function.md | 7 +-- .../sphinx/{routines => udf}/introduction.md | 30 +++++----- docs/src/main/sphinx/udf/sql.md | 18 ++++++ docs/src/main/sphinx/udf/sql/begin.md | 56 ++++++++++++++++++ .../main/sphinx/{routines => udf/sql}/case.md | 7 +-- .../sphinx/{routines => udf/sql}/declare.md | 10 ++-- .../sphinx/{routines => udf/sql}/examples.md | 20 +++---- .../main/sphinx/{routines => udf/sql}/if.md | 15 +++-- .../sphinx/{routines => udf/sql}/iterate.md | 16 +++--- .../sphinx/{routines => udf/sql}/leave.md | 14 ++--- .../main/sphinx/{routines => udf/sql}/loop.md | 9 ++- .../sphinx/{routines => udf/sql}/repeat.md | 17 +++--- .../sphinx/{routines => udf/sql}/return.md | 14 ++--- .../main/sphinx/{routines => udf/sql}/set.md | 13 ++--- .../sphinx/{routines => udf/sql}/while.md | 16 +++--- 30 files changed, 193 insertions(+), 199 deletions(-) delete mode 100644 docs/src/main/sphinx/routines.md delete mode 100644 docs/src/main/sphinx/routines/begin.md create mode 100644 docs/src/main/sphinx/udf.md rename docs/src/main/sphinx/{routines => udf}/function.md (96%) rename docs/src/main/sphinx/{routines => udf}/introduction.md (93%) create mode 100644 docs/src/main/sphinx/udf/sql.md create mode 100644 docs/src/main/sphinx/udf/sql/begin.md rename docs/src/main/sphinx/{routines => udf/sql}/case.md (86%) rename docs/src/main/sphinx/{routines => udf/sql}/declare.md (80%) rename docs/src/main/sphinx/{routines => udf/sql}/examples.md (97%) rename docs/src/main/sphinx/{routines => udf/sql}/if.md (56%) rename docs/src/main/sphinx/{routines => udf/sql}/iterate.md (54%) rename docs/src/main/sphinx/{routines => udf/sql}/leave.md (62%) rename docs/src/main/sphinx/{routines => udf/sql}/loop.md (86%) rename docs/src/main/sphinx/{routines => udf/sql}/repeat.md (81%) rename docs/src/main/sphinx/{routines => udf/sql}/return.md (61%) rename docs/src/main/sphinx/{routines => udf/sql}/set.md (63%) rename docs/src/main/sphinx/{routines => udf/sql}/while.md (68%) diff --git a/docs/src/main/sphinx/admin/properties-sql-environment.md b/docs/src/main/sphinx/admin/properties-sql-environment.md index a4092cef9ace..2b1825883109 100644 --- a/docs/src/main/sphinx/admin/properties-sql-environment.md +++ b/docs/src/main/sphinx/admin/properties-sql-environment.md @@ -31,7 +31,7 @@ client overrides this default. - **Type:** [](prop-type-string) -Set the default catalog for [SQL routine](/routines) storage for all clients. +Set the default catalog for [](/udf) storage for all clients. The connector used in the catalog must support [](sql-routine-management). Any usage of a fully qualified name for a routine overrides this default. diff --git a/docs/src/main/sphinx/functions.md b/docs/src/main/sphinx/functions.md index c445861ecce0..cb931d755667 100644 --- a/docs/src/main/sphinx/functions.md +++ b/docs/src/main/sphinx/functions.md @@ -12,7 +12,7 @@ Refer to the following sections for further details: In addition, Trino supports implementation of [custom functions](/develop/functions) or [custom table functions](/develop/table-functions) provided by a plugin, and creation of -user-defined functions as [SQL routines](/routines). +[](/udf). ## Functions by name diff --git a/docs/src/main/sphinx/functions/conditional.md b/docs/src/main/sphinx/functions/conditional.md index d4086848294a..5f6f3cb21090 100644 --- a/docs/src/main/sphinx/functions/conditional.md +++ b/docs/src/main/sphinx/functions/conditional.md @@ -51,7 +51,7 @@ SELECT a, b, END ``` -SQL routines can use [`CASE` statements](/routines/case) that use a slightly +SQL routines can use [`CASE` statements](/udf/sql/case) that use a slightly different syntax from the CASE expressions. Specifically note the requirements for terminating each clause with a semicolon `;` and the usage of `END CASE`. @@ -95,7 +95,7 @@ SELECT FROM tpch.sf1.orders; ``` -SQL routines can use [`IF` statements](/routines/if) that use a slightly +SQL routines can use [`IF` statements](/udf/sql/if) that use a slightly different syntax from `IF` expressions. Specifically note the requirement for terminating each clause with a semicolon `;` and the usage of `END IF`. diff --git a/docs/src/main/sphinx/index.md b/docs/src/main/sphinx/index.md index 16e8ab72011b..24ea617a83bb 100644 --- a/docs/src/main/sphinx/index.md +++ b/docs/src/main/sphinx/index.md @@ -12,9 +12,9 @@ optimizer connector object-storage functions +udf language sql -routines develop glossary appendix diff --git a/docs/src/main/sphinx/release/release-431.md b/docs/src/main/sphinx/release/release-431.md index 5a80e2d172fb..3b3711acca84 100644 --- a/docs/src/main/sphinx/release/release-431.md +++ b/docs/src/main/sphinx/release/release-431.md @@ -2,7 +2,7 @@ ## General -* Add support for [](/routines). ({issue}`19308`) +* Add support for [](/udf/sql). ({issue}`19308`) * Add support for [](/sql/create-function) and [](/sql/drop-function) statements. ({issue}`19308`) * Add support for the `REPLACE` modifier to the `CREATE TABLE` statement. ({issue}`13180`) * Disallow a `null` offset for the {func}`lead` and {func}`lag` functions. ({issue}`19003`) diff --git a/docs/src/main/sphinx/release/release-436.md b/docs/src/main/sphinx/release/release-436.md index b1a0bab0e7c9..c6abe7357eaf 100644 --- a/docs/src/main/sphinx/release/release-436.md +++ b/docs/src/main/sphinx/release/release-436.md @@ -6,7 +6,7 @@ [](jvm-config). ({issue}`20010`) * Improve performance by not generating redundant predicates. ({issue}`16520`) * Fix query failure when invoking the `json_table` function. ({issue}`20122`) -* Fix query hang when a [SQL routine](/routines) dereferences a row field. ({issue}`19997`). +* Fix query hang when a [](/udf/sql) dereferences a row field. ({issue}`19997`). * Fix potential incorrect results when using the {func}`ST_Centroid` and {func}`ST_Buffer` functions for tiny geometries. ({issue}`20237`) diff --git a/docs/src/main/sphinx/release/release-446.md b/docs/src/main/sphinx/release/release-446.md index eadcbe190fea..05f6b0bf0f16 100644 --- a/docs/src/main/sphinx/release/release-446.md +++ b/docs/src/main/sphinx/release/release-446.md @@ -10,9 +10,9 @@ statement. ({issue}`21619`) * Fix `CREATE CATALOG` statements including quotes in catalog names. ({issue}`21399`) * Fix potential query failure when a column name ends with a `:`. ({issue}`21676`) -* Fix potential query failure when a [SQL routine](/routines) contains a label +* Fix potential query failure when a [](/udf/sql) contains a label reference in a `LEAVE`, `ITERATE`, `REPEAT`, or `WHILE` statement. ({issue}`21682`) -* Fix query failure when [SQL routines](/routines) use the `NULLIF` or `BETWEEN` +* Fix query failure when [](/udf/sql) use the `NULLIF` or `BETWEEN` functions. ({issue}`19820`) * Fix potential query failure due to worker nodes running out of memory in concurrent scenarios. ({issue}`21706`) diff --git a/docs/src/main/sphinx/routines.md b/docs/src/main/sphinx/routines.md deleted file mode 100644 index 541608d529d2..000000000000 --- a/docs/src/main/sphinx/routines.md +++ /dev/null @@ -1,25 +0,0 @@ -# SQL routines - -A SQL routine is a custom, user-defined function authored by a user of Trino in -a client and written in the SQL routine language. Routines are scalar functions -that return a single output value. More details are available in the following -sections: - -```{toctree} -:maxdepth: 1 - -Introduction -Examples -routines/begin -routines/case -routines/declare -routines/function -routines/if -routines/iterate -routines/leave -routines/loop -routines/repeat -routines/return -routines/set -routines/while -``` diff --git a/docs/src/main/sphinx/routines/begin.md b/docs/src/main/sphinx/routines/begin.md deleted file mode 100644 index c1ef143fd70a..000000000000 --- a/docs/src/main/sphinx/routines/begin.md +++ /dev/null @@ -1,57 +0,0 @@ -# BEGIN - -## Synopsis - -```text -BEGIN - [ DECLARE ... ] - statements -END -``` - -## Description - -Marks the start and end of a block in a [SQL routine](/routines/introduction). -`BEGIN` can be used wherever a statement can be used to group multiple -statements together and to declare variables local to the block. A typical use -case is as first statement within a [](/routines/function). Blocks can also be -nested. - -After the `BEGIN` keyword, you can add variable declarations using -[](/routines/declare) statements, followed by one or more statements that define -the main body of the routine, separated by `;`. The following statements can be -used: - -* [](/routines/case) -* [](/routines/if) -* [](/routines/iterate) -* [](/routines/leave) -* [](/routines/loop) -* [](/routines/repeat) -* [](/routines/return) -* [](/routines/set) -* [](/routines/while) -* Nested [](/routines/begin) blocks - -## Examples - -The following example computes the value `42`: - -```sql -FUNCTION meaning_of_life() - RETURNS tinyint - BEGIN - DECLARE a tinyint DEFAULT 6; - DECLARE b tinyint DEFAULT 7; - RETURN a * b; - END -``` - -Further examples of varying complexity that cover usage of the `BEGIN` statement -in combination with other statements are available in the [SQL routines examples -documentation](/routines/examples). - -## See also - -* [](/routines/introduction) -* [](/routines/function) diff --git a/docs/src/main/sphinx/sql/create-function.md b/docs/src/main/sphinx/sql/create-function.md index 513b64557506..4b0cb5fe67ee 100644 --- a/docs/src/main/sphinx/sql/create-function.md +++ b/docs/src/main/sphinx/sql/create-function.md @@ -10,7 +10,7 @@ CREATE [OR REPLACE] FUNCTION ## Description Create or replace a [](routine-catalog). The `routine_definition` is composed of -the usage of [](/routines/function) and nested statements. The name of the +the usage of [](/udf/function) and nested statements. The name of the routine must be fully qualified with catalog and schema location, unless the [default SQL routine storage catalog and schema](/admin/properties-sql-environment) are configured. The connector used in @@ -42,12 +42,12 @@ CREATE FUNCTION meaning_of_life() RETURNS bigint RETURN 42; Further examples of varying complexity that cover usage of the `FUNCTION` statement in combination with other statements are available in the [SQL -routines examples documentation](/routines/examples). +routines examples documentation](/udf/sql/examples). ## See also * [](/sql/drop-function) * [](/sql/show-create-function) * [](/sql/show-functions) -* [](/routines/introduction) +* [](/udf) * [](/admin/properties-sql-environment) diff --git a/docs/src/main/sphinx/sql/drop-function.md b/docs/src/main/sphinx/sql/drop-function.md index 76235b0f6de9..e956a2186752 100644 --- a/docs/src/main/sphinx/sql/drop-function.md +++ b/docs/src/main/sphinx/sql/drop-function.md @@ -47,5 +47,5 @@ DROP FUNCTION meaning_of_life(); * [](/sql/create-function) * [](/sql/show-create-function) * [](/sql/show-functions) -* [](/routines/introduction) +* [](/udf) * [](/admin/properties-sql-environment) diff --git a/docs/src/main/sphinx/sql/select.md b/docs/src/main/sphinx/sql/select.md index 5d6b93b6ed23..1a07c05b89ff 100644 --- a/docs/src/main/sphinx/sql/select.md +++ b/docs/src/main/sphinx/sql/select.md @@ -88,7 +88,7 @@ SELECT hello('Finn') || ' and ' || bye('Joe'); ``` Find further information about routines in general, inline routines, all -supported statements, and examples in [](/routines). +supported statements, and examples in [](/udf). ## WITH clause diff --git a/docs/src/main/sphinx/sql/show-create-function.md b/docs/src/main/sphinx/sql/show-create-function.md index 1015726b6de1..bac15c00b448 100644 --- a/docs/src/main/sphinx/sql/show-create-function.md +++ b/docs/src/main/sphinx/sql/show-create-function.md @@ -23,5 +23,5 @@ SHOW CREATE FUNCTION example.default.meaning_of_life; * [](/sql/create-function) * [](/sql/drop-function) * [](/sql/show-functions) -* [](/routines/introduction) +* [](/udf) * [](/admin/properties-sql-environment) diff --git a/docs/src/main/sphinx/sql/show-functions.md b/docs/src/main/sphinx/sql/show-functions.md index 9d9fe0067591..e6c3a7329161 100644 --- a/docs/src/main/sphinx/sql/show-functions.md +++ b/docs/src/main/sphinx/sql/show-functions.md @@ -10,7 +10,7 @@ SHOW FUNCTIONS [ FROM schema ] [ LIKE pattern ] List functions in `schema` or all functions in the current session path. This can include built-in functions, [functions from a custom -plugin](/develop/functions), and [SQL routines](/routines). +plugin](/develop/functions), and [](/udf). For each function returned, the following information is displayed: @@ -62,7 +62,7 @@ Example output: ## See also * [](/functions) -* [](/routines) +* [](/udf) * [](/develop/functions) * [](/sql/create-function) * [](/sql/drop-function) diff --git a/docs/src/main/sphinx/udf.md b/docs/src/main/sphinx/udf.md new file mode 100644 index 000000000000..d368d423804c --- /dev/null +++ b/docs/src/main/sphinx/udf.md @@ -0,0 +1,14 @@ +# User-defined functions + +A SQL routine is a custom, user-defined function authored by a user of Trino in +a client and written in the SQL routine language. Routines are scalar functions +that return a single output value. More details are available in the following +sections: + +```{toctree} +:titlesonly: true + +udf/introduction +udf/function +udf/sql +``` diff --git a/docs/src/main/sphinx/routines/function.md b/docs/src/main/sphinx/udf/function.md similarity index 96% rename from docs/src/main/sphinx/routines/function.md rename to docs/src/main/sphinx/udf/function.md index 21df3a95c392..72fbae1d751d 100644 --- a/docs/src/main/sphinx/routines/function.md +++ b/docs/src/main/sphinx/udf/function.md @@ -91,12 +91,11 @@ SELECT meaning_of_life(); Further examples of varying complexity that cover usage of the `FUNCTION` statement in combination with other statements are available in the [SQL -routines examples documentation](/routines/examples). +UDF examples documentation](/udf/sql/examples). ## See also -* [](/routines/introduction) -* [](/routines/begin) -* [](/routines/return) +* [](/udf) +* [](/udf/sql) * [](/sql/create-function) diff --git a/docs/src/main/sphinx/routines/introduction.md b/docs/src/main/sphinx/udf/introduction.md similarity index 93% rename from docs/src/main/sphinx/routines/introduction.md rename to docs/src/main/sphinx/udf/introduction.md index b6fb55d05631..db5d51493578 100644 --- a/docs/src/main/sphinx/routines/introduction.md +++ b/docs/src/main/sphinx/udf/introduction.md @@ -95,28 +95,28 @@ testing of your SQL routines. (routine-declaration)= ## Routine declaration -Refer to the documentation for the [](/routines/function) keyword for more +Refer to the documentation for the [](/udf/function) keyword for more details about declaring the routine overall. The routine body is composed with statements from the following list: -* [](/routines/begin) -* [](/routines/case) -* [](/routines/declare) -* [](/routines/if) -* [](/routines/iterate) -* [](/routines/leave) -* [](/routines/loop) -* [](/routines/repeat) -* [](/routines/return) -* [](/routines/set) -* [](/routines/while) +* [](/udf/sql/begin) +* [](/udf/sql/case) +* [](/udf/sql/declare) +* [](/udf/sql/if) +* [](/udf/sql/iterate) +* [](/udf/sql/leave) +* [](/udf/sql/loop) +* [](/udf/sql/repeat) +* [](/udf/sql/return) +* [](/udf/sql/set) +* [](/udf/sql/while) Statements can also use [built-in functions and operators](/functions) as well as other routines, although recursion is not supported for routines. -Find simple examples in each statement documentation, and refer to the [example -documentation](/routines/examples) for more complex use cases that combine -multiple statements. +Find simple examples in each statement documentation, and refer to the +[](/udf/sql/examples) for more complex use cases that combine multiple +statements. :::{note} User-defined functions can alternatively be written in Java and deployed as a diff --git a/docs/src/main/sphinx/udf/sql.md b/docs/src/main/sphinx/udf/sql.md new file mode 100644 index 000000000000..ae40c062f943 --- /dev/null +++ b/docs/src/main/sphinx/udf/sql.md @@ -0,0 +1,18 @@ +# SQL routines + +```{toctree} +:titlesonly: true + +sql/examples +sql/begin +sql/case +sql/declare +sql/if +sql/iterate +sql/leave +sql/loop +sql/repeat +sql/return +sql/set +sql/while +``` \ No newline at end of file diff --git a/docs/src/main/sphinx/udf/sql/begin.md b/docs/src/main/sphinx/udf/sql/begin.md new file mode 100644 index 000000000000..449b713ce004 --- /dev/null +++ b/docs/src/main/sphinx/udf/sql/begin.md @@ -0,0 +1,56 @@ +# BEGIN + +## Synopsis + +```text +BEGIN + [ DECLARE ... ] + statements +END +``` + +## Description + +Marks the start and end of a block in a [](/udf/sql). `BEGIN` can be used +wherever a statement can be used to group multiple statements together and to +declare variables local to the block. A typical use case is as first statement +within a [](/udf/function). Blocks can also be nested. + +After the `BEGIN` keyword, you can add variable declarations using +[](/udf/sql/declare) statements, followed by one or more statements that define +the main body of the routine, separated by `;`. The following statements can be +used: + +* [](/udf/sql/case) +* [](/udf/sql/if) +* [](/udf/sql/iterate) +* [](/udf/sql/leave) +* [](/udf/sql/loop) +* [](/udf/sql/repeat) +* [](/udf/sql/return) +* [](/udf/sql/set) +* [](/udf/sql/while) +* Nested [](/udf/sql/begin) blocks + +## Examples + +The following example computes the value `42`: + +```sql +FUNCTION meaning_of_life() + RETURNS tinyint + BEGIN + DECLARE a tinyint DEFAULT 6; + DECLARE b tinyint DEFAULT 7; + RETURN a * b; + END +``` + +Further examples of varying complexity that cover usage of the `BEGIN` statement +in combination with other statements are available in the [](/udf/sql/examples). + +## See also + +* [](/udf) +* [](/udf/sql) +* [](/udf/function) diff --git a/docs/src/main/sphinx/routines/case.md b/docs/src/main/sphinx/udf/sql/case.md similarity index 86% rename from docs/src/main/sphinx/routines/case.md rename to docs/src/main/sphinx/udf/sql/case.md index f7264d08b096..146699360299 100644 --- a/docs/src/main/sphinx/routines/case.md +++ b/docs/src/main/sphinx/udf/sql/case.md @@ -25,7 +25,7 @@ END ## Description The `CASE` statement is an optional construct to allow conditional processing -in [SQL routines](/routines/introduction). +in [](/udf/sql). The `WHEN` clauses are evaluated sequentially, stopping after the first match, and therefore the order of the statements is significant. The statements of the @@ -54,10 +54,9 @@ FUNCTION simple_case(a bigint) ``` Further examples of varying complexity that cover usage of the `CASE` statement -in combination with other statements are available in the [SQL routines examples -documentation](/routines/examples). +in combination with other statements are available in the [](/udf/sql/examples). ## See also -* [](/routines/introduction) +* [](/udf/sql) * [Conditional expressions using `CASE`](case-expression) diff --git a/docs/src/main/sphinx/routines/declare.md b/docs/src/main/sphinx/udf/sql/declare.md similarity index 80% rename from docs/src/main/sphinx/routines/declare.md rename to docs/src/main/sphinx/udf/sql/declare.md index 2238ca2de0c0..518b6b4552dd 100644 --- a/docs/src/main/sphinx/routines/declare.md +++ b/docs/src/main/sphinx/udf/sql/declare.md @@ -8,8 +8,8 @@ DECLARE identifier [, ...] type [ DEFAULT expression ] ## Description -Use the `DECLARE` statement directly after the [](/routines/begin) keyword in -[](/routines) to define one or more variables with an `identifier` as name. Each +Use the `DECLARE` statement directly after the [](/udf/sql/begin) keyword in +[](/udf/sql) to define one or more variables with an `identifier` as name. Each statement must specify the [data type](/language/types) of the variable with `type`. It can optionally include a default, initial value defined by an `expression`. The default value is `NULL` if not specified. @@ -43,10 +43,10 @@ DECLARE start_time timestamp(3) with time zone DEFAULT now(); ``` Further examples of varying complexity that cover usage of the `DECLARE` -statement in combination with other statements are available in the [SQL -routines examples documentation](/routines/examples). +statement in combination with other statements are available in the +[](/udf/sql/examples). ## See also -* [](/routines/introduction) +* [](/udf/sql) * [](/language/types) diff --git a/docs/src/main/sphinx/routines/examples.md b/docs/src/main/sphinx/udf/sql/examples.md similarity index 97% rename from docs/src/main/sphinx/routines/examples.md rename to docs/src/main/sphinx/udf/sql/examples.md index 40667856e53f..80b69fb7b453 100644 --- a/docs/src/main/sphinx/routines/examples.md +++ b/docs/src/main/sphinx/udf/sql/examples.md @@ -1,22 +1,20 @@ # Example SQL routines - -After learning about [SQL routines from the -introduction](/routines/introduction), the following sections show numerous -examples of valid SQL routines. The routines are suitable as [inline +After learning about [](/udf/sql), the following sections show numerous examples +of valid SQL routines. The routines are suitable as [inline routines](routine-inline) or [catalog routines](routine-catalog), after adjusting the name and the example invocations. The examples combine numerous supported statements. Refer to the specific statement documentation for further details: -* [](/routines/function) for general SQL routine declaration -* [](/routines/begin) and [](/routines/declare) for routine blocks -* [](/routines/set) for assigning values to variables -* [](/routines/return) for returning routine results -* [](/routines/case) and [](/routines/if) for conditional flows -* [](/routines/loop), [](/routines/repeat), and [](/routines/while) for looping constructs -* [](/routines/iterate) and [](/routines/leave) for flow control +* [](/udf/function) for general SQL routine declaration +* [](/udf/sql/begin) and [](/udf/sql/declare) for routine blocks +* [](/udf/sql/set) for assigning values to variables +* [](/udf/sql/return) for returning routine results +* [](/udf/sql/case) and [](/udf/sql/if) for conditional flows +* [](/udf/sql/loop), [](/udf/sql/repeat), and [](/udf/sql/while) for looping constructs +* [](/udf/sql/iterate) and [](/udf/sql/leave) for flow control A very simple routine that returns a static value without requiring any input: diff --git a/docs/src/main/sphinx/routines/if.md b/docs/src/main/sphinx/udf/sql/if.md similarity index 56% rename from docs/src/main/sphinx/routines/if.md rename to docs/src/main/sphinx/udf/sql/if.md index 264beec66b52..626b5a2512a9 100644 --- a/docs/src/main/sphinx/routines/if.md +++ b/docs/src/main/sphinx/udf/sql/if.md @@ -14,11 +14,11 @@ END IF ## Description The `IF THEN` statement is an optional construct to allow conditional processing -in [SQL routines](/routines/introduction). Each `condition` following an `IF` -or `ELSEIF` must evaluate to a boolean. The result of processing the expression -must result in a boolean `true` value to process the `statements` in the `THEN` -block. A result of `false` results in skipping the `THEN` block and moving to -evaluate the next `ELSEIF` and `ELSE` blocks in order. +in [](/udf/sql). Each `condition` following an `IF` or `ELSEIF` must evaluate +to a boolean. The result of processing the expression must result in a boolean +`true` value to process the `statements` in the `THEN` block. A result of +`false` results in skipping the `THEN` block and moving to evaluate the next +`ELSEIF` and `ELSE` blocks in order. The `ELSEIF` and `ELSE` segments are optional. @@ -39,10 +39,9 @@ FUNCTION simple_if(a bigint) ``` Further examples of varying complexity that cover usage of the `IF` statement in -combination with other statements are available in the [SQL routines examples -documentation](/routines/examples). +combination with other statements are available in the [](/udf/sql/examples). ## See also -* [](/routines/introduction) +* [](/udf/sql) * [Conditional expressions using `IF`](if-expression) diff --git a/docs/src/main/sphinx/routines/iterate.md b/docs/src/main/sphinx/udf/sql/iterate.md similarity index 54% rename from docs/src/main/sphinx/routines/iterate.md rename to docs/src/main/sphinx/udf/sql/iterate.md index 7be395881459..9ec29dc0f65d 100644 --- a/docs/src/main/sphinx/routines/iterate.md +++ b/docs/src/main/sphinx/udf/sql/iterate.md @@ -8,10 +8,10 @@ ITERATE label ## Description -The `ITERATE` statement allows processing of blocks in [SQL -routines](/routines/introduction) to move processing back to the start of a -context block. Contexts are defined by a [`label`](routine-label). If no label -is found, the functions fails with an error message. +The `ITERATE` statement allows processing of blocks in [](/udf/sql) to move +processing back to the start of a context block. Contexts are defined by a +[`label`](routine-label). If no label is found, the functions fails with an +error message. ## Examples @@ -32,10 +32,10 @@ END ``` Further examples of varying complexity that cover usage of the `ITERATE` -statement in combination with other statements are available in the [SQL -routines examples documentation](/routines/examples). +statement in combination with other statements are available in the +[](/udf/sql/examples). ## See also -* [](/routines/introduction) -* [](/routines/leave) +* [](/udf/sql) +* [](/udf/sql/leave) diff --git a/docs/src/main/sphinx/routines/leave.md b/docs/src/main/sphinx/udf/sql/leave.md similarity index 62% rename from docs/src/main/sphinx/routines/leave.md rename to docs/src/main/sphinx/udf/sql/leave.md index 196855509921..9aea557e08f5 100644 --- a/docs/src/main/sphinx/routines/leave.md +++ b/docs/src/main/sphinx/udf/sql/leave.md @@ -8,10 +8,9 @@ LEAVE label ## Description -The `LEAVE` statement allows processing of blocks in [SQL -routines](/routines/introduction) to move out of a specified context. Contexts -are defined by a [`label`](routine-label). If no label is found, the functions -fails with an error message. +The `LEAVE` statement allows processing of blocks in [](/udf/sql) to move out of +a specified context. Contexts are defined by a [`label`](routine-label). If no +label is found, the functions fails with an error message. ## Examples @@ -37,10 +36,9 @@ END ``` Further examples of varying complexity that cover usage of the `LEAVE` statement -in combination with other statements are available in the [SQL routines examples -documentation](/routines/examples). +in combination with other statements are available in the [](/udf/sql/examples). ## See also -* [](/routines/introduction) -* [](/routines/iterate) +* [](/udf/sql) +* [](/udf/sql/iterate) diff --git a/docs/src/main/sphinx/routines/loop.md b/docs/src/main/sphinx/udf/sql/loop.md similarity index 86% rename from docs/src/main/sphinx/routines/loop.md rename to docs/src/main/sphinx/udf/sql/loop.md index d38918a456bf..9517486d79de 100644 --- a/docs/src/main/sphinx/routines/loop.md +++ b/docs/src/main/sphinx/udf/sql/loop.md @@ -10,8 +10,7 @@ END LOOP ## Description -The `LOOP` statement is an optional construct in [SQL -routines](/routines/introduction) to allow processing of a block of statements +The `LOOP` statement is an optional construct in [](/udf/sql) to allow processing of a block of statements repeatedly. The block of `statements` is processed until an explicit use of `LEAVE` causes @@ -56,9 +55,9 @@ SELECT to_one_hundred(12, 3); -- 30 Further examples of varying complexity that cover usage of the `LOOP` statement in combination with other statements are available in the [SQL routines examples -documentation](/routines/examples). +documentation](/udf/sql/examples). ## See also -* [](/routines/introduction) -* [](/routines/leave) +* [](/udf/sql) +* [](/udf/sql/leave) diff --git a/docs/src/main/sphinx/routines/repeat.md b/docs/src/main/sphinx/udf/sql/repeat.md similarity index 81% rename from docs/src/main/sphinx/routines/repeat.md rename to docs/src/main/sphinx/udf/sql/repeat.md index 5d750554c172..5d71ebaa5241 100644 --- a/docs/src/main/sphinx/routines/repeat.md +++ b/docs/src/main/sphinx/udf/sql/repeat.md @@ -11,10 +11,9 @@ END REPEAT ## Description -The `REPEAT UNTIL` statement is an optional construct in [SQL -routines](/routines/introduction) to allow processing of a block of statements -as long as a condition is met. The condition is validated as a last step of each -iteration. +The `REPEAT UNTIL` statement is an optional construct in [](/udf/sql) to allow +processing of a block of statements as long as a condition is met. The condition +is validated as a last step of each iteration. The block of statements is processed at least once. After the first, and every subsequent processing the expression `condidtion` is validated. If the result is @@ -60,11 +59,11 @@ SELECT test_repeat(12); -- 13 ``` Further examples of varying complexity that cover usage of the `REPEAT` -statement in combination with other statements are available in the [SQL -routines examples documentation](/routines/examples). +statement in combination with other statements are available in the +[](/udf/sql/examples). ## See also -* [](/routines/introduction) -* [](/routines/loop) -* [](/routines/while) +* [](/udf/sql) +* [](/udf/sql/loop) +* [](/udf/sql/while) diff --git a/docs/src/main/sphinx/routines/return.md b/docs/src/main/sphinx/udf/sql/return.md similarity index 61% rename from docs/src/main/sphinx/routines/return.md rename to docs/src/main/sphinx/udf/sql/return.md index cc8bfe483989..a376bd304e1a 100644 --- a/docs/src/main/sphinx/routines/return.md +++ b/docs/src/main/sphinx/udf/sql/return.md @@ -8,9 +8,9 @@ RETURN expression ## Description -Provide the value from a [SQL routines](/routines/introduction) to the caller. -The value is the result of evaluating the expression. It can be a static value, -a declared variable or a more complex expression. +Provide the value from a [](/udf/sql) to the caller. The value is the result of +evaluating the expression. It can be a static value, a declared variable or a +more complex expression. ## Examples @@ -24,13 +24,13 @@ RETURN x; ``` Further examples of varying complexity that cover usage of the `RETURN` -statement in combination with other statements are available in the [SQL -routines examples documentation](/routines/examples). +statement in combination with other statements are available in the +[](/udf/sql/examples). All routines must contain a `RETURN` statement at the end of the top-level block in the `FUNCTION` declaration, even if it's unreachable. ## See also -* [](/routines/introduction) -* [](/routines/function) +* [](/udf/sql) +* [](/udf/function) diff --git a/docs/src/main/sphinx/routines/set.md b/docs/src/main/sphinx/udf/sql/set.md similarity index 63% rename from docs/src/main/sphinx/routines/set.md rename to docs/src/main/sphinx/udf/sql/set.md index d9c24414e049..971ac5772e59 100644 --- a/docs/src/main/sphinx/routines/set.md +++ b/docs/src/main/sphinx/udf/sql/set.md @@ -8,9 +8,9 @@ SET identifier = expression ## Description -Use the `SET` statement in [SQL routines](/routines/introduction) to assign a -value to a variable, referenced by comma-separated `identifier`s. The -value is determined by evaluating the `expression` after the `=` sign. +Use the `SET` statement in [](/udf/sql) to assign a value to a variable, +referenced by comma-separated `identifier`s. The value is determined by +evaluating the `expression` after the `=` sign. Before the assignment the variable must be defined with a `DECLARE` statement. The data type of the variable must be identical to the data type of evaluating @@ -34,10 +34,9 @@ FUNCTION one() ``` Further examples of varying complexity that cover usage of the `SET` statement -in combination with other statements are available in the [SQL routines examples -documentation](/routines/examples). +in combination with other statements are available in the [](/udf/sql/examples). ## See also -* [](/routines/introduction) -* [](/routines/declare) +* [](/udf/sql) +* [](/udf/sql/declare) diff --git a/docs/src/main/sphinx/routines/while.md b/docs/src/main/sphinx/udf/sql/while.md similarity index 68% rename from docs/src/main/sphinx/routines/while.md rename to docs/src/main/sphinx/udf/sql/while.md index 6e252482d4b8..9646968d51e8 100644 --- a/docs/src/main/sphinx/routines/while.md +++ b/docs/src/main/sphinx/udf/sql/while.md @@ -10,10 +10,9 @@ END WHILE ## Description -The `WHILE` statement is an optional construct in [SQL -routines](/routines/introduction) to allow processing of a block of statements -as long as a condition is met. The condition is validated as a first step of -each iteration. +The `WHILE` statement is an optional construct in [](/udf/sql) to allow +processing of a block of statements as long as a condition is met. The condition +is validated as a first step of each iteration. The expression that defines the `condition` is evaluated at least once. If the result is `true`, processing moves to `DO`, through following `statements` and @@ -37,11 +36,10 @@ END WHILE; ``` Further examples of varying complexity that cover usage of the `WHILE` statement -in combination with other statements are available in the [SQL routines examples -documentation](/routines/examples). +in combination with other statements are available in the [](/udf/sql/examples). ## See also -* [](/routines/introduction) -* [](/routines/loop) -* [](/routines/repeat) +* [](/udf/sql) +* [](/udf/sql/loop) +* [](/udf/sql/repeat) From fbed6f7b2e007d81250c78396e6001d0feeb9bc7 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Tue, 10 Dec 2024 16:26:04 -0800 Subject: [PATCH 002/158] Add redirectors for SQL routine change --- docs/src/main/sphinx/redirects.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/src/main/sphinx/redirects.txt b/docs/src/main/sphinx/redirects.txt index 361a63fc6989..26e0254da98a 100644 --- a/docs/src/main/sphinx/redirects.txt +++ b/docs/src/main/sphinx/redirects.txt @@ -10,3 +10,18 @@ connector/atop.md connector/removed.md connector/localfile.md connector/removed.md connector/accumulo.md connector/removed.md security/apache-ranger-access-control.md security/ranger-access-control.md +routines.md udf.md +routines/function.md udf/function.md +routines/introduction.md udf/introduction.md +routines/begin.md udf/sql/begin.md +routines/case.md udf/sql/case.md +routines/declare.md udf/sql/declare.md +routines/examples.md udf/sql/examples.md +routines/if.md udf/sql/if.md +routines/iterate.md udf/sql/iterate.md +routines/leave.md udf/sql/leave.md +routines/loop.md udf/sql/loop.md +routines/repeat.md udf/sql/repeat.md +routines/return.md udf/sql/return.md +routines/set.md udf/sql/set.md +routines/while.md udf/sql/while.md From 71f513503cbc8c614c2d33d2fb884563989d0723 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Wed, 11 Dec 2024 14:54:04 -0800 Subject: [PATCH 003/158] Reword from SQL routine to SQL UDF And generally introduce user-defined functions (UDF) as a term. --- .../admin/properties-sql-environment.md | 21 ++- docs/src/main/sphinx/client/cli.md | 3 +- docs/src/main/sphinx/client/jdbc.md | 2 +- docs/src/main/sphinx/connector/hive.md | 2 +- docs/src/main/sphinx/connector/memory.md | 2 +- docs/src/main/sphinx/functions/conditional.md | 8 +- docs/src/main/sphinx/language/sql-support.md | 6 +- docs/src/main/sphinx/release/release-431.md | 4 +- docs/src/main/sphinx/release/release-440.md | 2 +- docs/src/main/sphinx/release/release-447.md | 2 +- docs/src/main/sphinx/release/release-453.md | 2 +- docs/src/main/sphinx/release/release-454.md | 2 +- .../security/file-system-access-control.md | 4 +- docs/src/main/sphinx/sql/create-function.md | 21 ++- docs/src/main/sphinx/sql/drop-function.md | 21 ++- docs/src/main/sphinx/sql/select.md | 12 +- docs/src/main/sphinx/sql/show-functions.md | 4 +- docs/src/main/sphinx/udf.md | 16 ++- docs/src/main/sphinx/udf/function.md | 45 +++--- docs/src/main/sphinx/udf/introduction.md | 114 +++++++-------- docs/src/main/sphinx/udf/sql.md | 2 +- docs/src/main/sphinx/udf/sql/begin.md | 2 +- docs/src/main/sphinx/udf/sql/examples.md | 133 +++++++++--------- docs/src/main/sphinx/udf/sql/iterate.md | 2 +- docs/src/main/sphinx/udf/sql/leave.md | 2 +- docs/src/main/sphinx/udf/sql/loop.md | 4 +- docs/src/main/sphinx/udf/sql/repeat.md | 6 +- docs/src/main/sphinx/udf/sql/return.md | 4 +- docs/src/main/sphinx/udf/sql/while.md | 2 +- 29 files changed, 227 insertions(+), 223 deletions(-) diff --git a/docs/src/main/sphinx/admin/properties-sql-environment.md b/docs/src/main/sphinx/admin/properties-sql-environment.md index 2b1825883109..d31ce3f13990 100644 --- a/docs/src/main/sphinx/admin/properties-sql-environment.md +++ b/docs/src/main/sphinx/admin/properties-sql-environment.md @@ -31,15 +31,14 @@ client overrides this default. - **Type:** [](prop-type-string) -Set the default catalog for [](/udf) storage for all clients. -The connector used in the catalog must support [](sql-routine-management). Any -usage of a fully qualified name for a routine overrides this default. +Set the default catalog for [](/udf) storage for all clients. The connector used +in the catalog must support [](udf-management). Any usage of a fully qualified +name for a UDF overrides this default. -The default catalog and schema for SQL routine storage must be configured -together, and the resulting entry must be set as part of the path. For example, -the following example section for [](config-properties) uses the `functions` -schema in the `brain` catalog for routine storage, and adds it as the only entry -on the path: +The default catalog and schema for UDF storage must be configured together, and +the resulting entry must be set as part of the path. For example, the following +section for [](config-properties) uses the `functions` schema in the `brain` +catalog for UDF storage, and adds it as the only entry on the path: ```properties sql.default-function-catalog=brain @@ -51,9 +50,9 @@ sql.path=brain.default - **Type:** [](prop-type-string) -Set the default schema for SQL routine storage for all clients. Must be set to a -schema name that is valid for the default function catalog. Any usage of a fully -qualified name for a routine overrides this default. +Set the default schema for UDF storage for all clients. Must be set to a schema +name that is valid for the default function catalog. Any usage of a fully +qualified name for a UDF overrides this default. ## `sql.path` diff --git a/docs/src/main/sphinx/client/cli.md b/docs/src/main/sphinx/client/cli.md index 8b7a0ee17613..439d43558d82 100644 --- a/docs/src/main/sphinx/client/cli.md +++ b/docs/src/main/sphinx/client/cli.md @@ -181,8 +181,7 @@ mode: - Do not show query processing progress. * - `--path` - Set the default [SQL path](/sql/set-path) for the session. Useful for - setting a catalog and schema location for [catalog - routines](routine-catalog). + setting a catalog and schema location for [](udf-catalog). * - `--password` - Prompts for a password. Use if your Trino server requires password authentication. You can set the `TRINO_PASSWORD` environment variable with diff --git a/docs/src/main/sphinx/client/jdbc.md b/docs/src/main/sphinx/client/jdbc.md index 427f688000f3..bcf7af7a0182 100644 --- a/docs/src/main/sphinx/client/jdbc.md +++ b/docs/src/main/sphinx/client/jdbc.md @@ -142,7 +142,7 @@ may not be specified using both methods. - Client tags for selecting resource groups. Example: `abc,xyz` * - `path` - Set the default [SQL path](/sql/set-path) for the session. Useful for - setting a catalog and schema location for [catalog routines](routine-catalog). + setting a catalog and schema location for [](udf-catalog). * - `traceToken` - Trace token for correlating requests across systems. * - `source` diff --git a/docs/src/main/sphinx/connector/hive.md b/docs/src/main/sphinx/connector/hive.md index cf58c3a000d0..b642004c06a0 100644 --- a/docs/src/main/sphinx/connector/hive.md +++ b/docs/src/main/sphinx/connector/hive.md @@ -403,7 +403,7 @@ configured object storage system and metadata stores: - {ref}`sql-view-management`; see also {ref}`Hive-specific view management ` -- [](sql-routine-management) +- [](udf-management) - {ref}`sql-security-operations`: see also {ref}`SQL standard-based authorization for object storage ` diff --git a/docs/src/main/sphinx/connector/memory.md b/docs/src/main/sphinx/connector/memory.md index 2a7377a1dea7..23bc958abf80 100644 --- a/docs/src/main/sphinx/connector/memory.md +++ b/docs/src/main/sphinx/connector/memory.md @@ -69,7 +69,7 @@ statements, the connector supports the following features: - {doc}`/sql/alter-schema` - {doc}`/sql/comment` - [](sql-view-management) -- [](sql-routine-management) +- [](udf-management) ### TRUNCATE and DROP TABLE diff --git a/docs/src/main/sphinx/functions/conditional.md b/docs/src/main/sphinx/functions/conditional.md index 5f6f3cb21090..6aad6280f331 100644 --- a/docs/src/main/sphinx/functions/conditional.md +++ b/docs/src/main/sphinx/functions/conditional.md @@ -51,7 +51,7 @@ SELECT a, b, END ``` -SQL routines can use [`CASE` statements](/udf/sql/case) that use a slightly +SQL UDFs can use [`CASE` statements](/udf/sql/case) that use a slightly different syntax from the CASE expressions. Specifically note the requirements for terminating each clause with a semicolon `;` and the usage of `END CASE`. @@ -95,9 +95,9 @@ SELECT FROM tpch.sf1.orders; ``` -SQL routines can use [`IF` statements](/udf/sql/if) that use a slightly -different syntax from `IF` expressions. Specifically note the requirement -for terminating each clause with a semicolon `;` and the usage of `END IF`. +SQL UDFs can use [`IF` statements](/udf/sql/if) that use a slightly different +syntax from `IF` expressions. Specifically note the requirement for terminating +each clause with a semicolon `;` and the usage of `END IF`. (coalesce-function)= ## COALESCE diff --git a/docs/src/main/sphinx/language/sql-support.md b/docs/src/main/sphinx/language/sql-support.md index 767db2af93cf..8dc3b693d51d 100644 --- a/docs/src/main/sphinx/language/sql-support.md +++ b/docs/src/main/sphinx/language/sql-support.md @@ -118,10 +118,10 @@ connector to connector: - {doc}`/sql/drop-materialized-view` - {doc}`/sql/refresh-materialized-view` -(sql-routine-management)= -### Routine management +(udf-management)= +### User-defined function management -The following statements are used to manage [catalog routines](routine-catalog): +The following statements are used to manage [](udf-catalog): - [](/sql/create-function) - [](/sql/drop-function) diff --git a/docs/src/main/sphinx/release/release-431.md b/docs/src/main/sphinx/release/release-431.md index 3b3711acca84..068cc79657ba 100644 --- a/docs/src/main/sphinx/release/release-431.md +++ b/docs/src/main/sphinx/release/release-431.md @@ -27,7 +27,7 @@ ## Hive connector -* Add support for [SQL routine management](sql-routine-management). ({issue}`19308`) +* Add support for [](udf-management). ({issue}`19308`) * Replace the `hive.metastore-timeout` Hive metastore configuration property with the `hive.metastore.thrift.client.connect-timeout` and `hive.metastore.thrift.client.read-timeout` properties. ({issue}`19390`) @@ -50,7 +50,7 @@ ## Memory connector -* Add support for [SQL routine management](sql-routine-management). ({issue}`19308`) +* Add support for [](udf-management). ({issue}`19308`) ## SPI diff --git a/docs/src/main/sphinx/release/release-440.md b/docs/src/main/sphinx/release/release-440.md index 2a79828a8e5f..5490782be738 100644 --- a/docs/src/main/sphinx/release/release-440.md +++ b/docs/src/main/sphinx/release/release-440.md @@ -12,7 +12,7 @@ * Fix query failure when a check constraint is null. ({issue}`20906`) * Fix query failure for aggregations over `CASE` expressions when the input evaluation could throw an error. ({issue}`20652`) -* Fix incorrect behavior of the else clause in a SQL routines with a single +* Fix incorrect behavior of the else clause in a SQL UDFs with a single if/end condition. ({issue}`20926`) * Fix the `ALTER TABLE EXECUTE optimize` queries failing due to exceeding the open writer limit. ({issue}`20871`) diff --git a/docs/src/main/sphinx/release/release-447.md b/docs/src/main/sphinx/release/release-447.md index d9e675f7e6b7..b1aee4fd4d31 100644 --- a/docs/src/main/sphinx/release/release-447.md +++ b/docs/src/main/sphinx/release/release-447.md @@ -13,7 +13,7 @@ ## CLI -* Fix incorrect error location markers for SQL routines causing CLI to print +* Fix incorrect error location markers for SQL UDFs causing the CLI to print exceptions. ({issue}`21357`) ## Delta Lake connector diff --git a/docs/src/main/sphinx/release/release-453.md b/docs/src/main/sphinx/release/release-453.md index ecf70c8eea80..e5c6e63cdf1d 100644 --- a/docs/src/main/sphinx/release/release-453.md +++ b/docs/src/main/sphinx/release/release-453.md @@ -70,7 +70,7 @@ `hive.metastore.glue.use-web-identity-token-credentials-provider` configuration property. ({issue}`15267`) * Fix failure to read Hive tables migrated to Iceberg with Apache Spark. ({issue}`11338`) -* Fix failure for `CREATE FUNCTION` with SQL routine storage in Glue when +* Fix failure for `CREATE FUNCTION` with SQL UDF storage in Glue when `hive.metastore.glue.catalogid` is set. ({issue}`22717`) ## Hudi connector diff --git a/docs/src/main/sphinx/release/release-454.md b/docs/src/main/sphinx/release/release-454.md index 7c66071bd7ec..6aa22f305814 100644 --- a/docs/src/main/sphinx/release/release-454.md +++ b/docs/src/main/sphinx/release/release-454.md @@ -23,7 +23,7 @@ ## Web UI -* Add information about which tables and routines have been referenced by a +* Add information about which tables and UDFs have been referenced by a query. ({issue}`20843`) ## JDBC driver diff --git a/docs/src/main/sphinx/security/file-system-access-control.md b/docs/src/main/sphinx/security/file-system-access-control.md index 55644dfc12b5..7651f438a33a 100644 --- a/docs/src/main/sphinx/security/file-system-access-control.md +++ b/docs/src/main/sphinx/security/file-system-access-control.md @@ -142,11 +142,11 @@ Permissions required for executing functions: * - `CREATE FUNCTION` - `all` - `ownership` - - Not all connectors support [catalog routines](routine-catalog). + - Not all connectors support [](udf-catalog). * - `DROP FUNCTION` - `all` - `ownership` - - Not all connectors support [catalog routines](routine-catalog). + - Not all connectors support [](udf-catalog). ::: (system-file-auth-visibility)= diff --git a/docs/src/main/sphinx/sql/create-function.md b/docs/src/main/sphinx/sql/create-function.md index 4b0cb5fe67ee..b5ace2dde8a0 100644 --- a/docs/src/main/sphinx/sql/create-function.md +++ b/docs/src/main/sphinx/sql/create-function.md @@ -4,24 +4,23 @@ ```text CREATE [OR REPLACE] FUNCTION - routine_definition + udf_definition ``` ## Description -Create or replace a [](routine-catalog). The `routine_definition` is composed of -the usage of [](/udf/function) and nested statements. The name of the -routine must be fully qualified with catalog and schema location, unless the -[default SQL routine storage catalog and -schema](/admin/properties-sql-environment) are configured. The connector used in -the catalog must support routine storage. +Create or replace a [](udf-catalog). The `udf_definition` is composed of the +usage of [](/udf/function) and nested statements. The name of the UDF must be +fully qualified with catalog and schema location, unless the [default UDF +storage catalog and schema](/admin/properties-sql-environment) are configured. +The connector used in the catalog must support UDF storage. -The optional `OR REPLACE` clause causes the routine to be replaced if it already +The optional `OR REPLACE` clause causes the UDF to be replaced if it already exists rather than raising an error. ## Examples -The following example creates the `meaning_of_life` routine in the `default` +The following example creates the `meaning_of_life` UDF in the `default` schema of the `example` catalog: ```sql @@ -32,7 +31,7 @@ CREATE FUNCTION example.default.meaning_of_life() END; ``` -If the [default catalog and schema for routine +If the [default catalog and schema for UDF storage](/admin/properties-sql-environment) is configured, you can use the following more compact syntax: @@ -42,7 +41,7 @@ CREATE FUNCTION meaning_of_life() RETURNS bigint RETURN 42; Further examples of varying complexity that cover usage of the `FUNCTION` statement in combination with other statements are available in the [SQL -routines examples documentation](/udf/sql/examples). +UDF examples documentation](/udf/sql/examples). ## See also diff --git a/docs/src/main/sphinx/sql/drop-function.md b/docs/src/main/sphinx/sql/drop-function.md index e956a2186752..0183facb80f1 100644 --- a/docs/src/main/sphinx/sql/drop-function.md +++ b/docs/src/main/sphinx/sql/drop-function.md @@ -3,38 +3,37 @@ ## Synopsis ```text -DROP FUNCTION [ IF EXISTS ] routine_name ( [ [ parameter_name ] data_type [, ...] ] ) +DROP FUNCTION [ IF EXISTS ] udf_name ( [ [ parameter_name ] data_type [, ...] ] ) ``` ## Description -Removes a [](routine-catalog). The value of `routine_name` -must be fully qualified with catalog and schema location of the routine, unless -the [default SQL routine storage catalog and -schema](/admin/properties-sql-environment) are configured. +Removes a [catalog UDF](udf-catalog). The value of `udf_name` must be fully +qualified with catalog and schema location of the UDF, unless the [default UDF storage catalog and schema](/admin/properties-sql-environment) are +configured. -The `data_type`s must be included for routines that use parameters to ensure the -routine with the correct name and parameter signature is removed. +The `data_type`s must be included for UDFs that use parameters to ensure the UDF +with the correct name and parameter signature is removed. The optional `IF EXISTS` clause causes the error to be suppressed if the function does not exist. ## Examples -The following example removes the `meaning_of_life` routine in the `default` -schema of the `example` catalog: +The following example removes the `meaning_of_life` UDF in the `default` schema +of the `example` catalog: ```sql DROP FUNCTION example.default.meaning_of_life(); ``` -If the routine uses a input parameter, the type must be added: +If the UDF uses a input parameter, the type must be added: ```sql DROP FUNCTION multiply_by_two(bigint); ``` -If the [default catalog and schema for routine +If the [default catalog and schema for UDF storage](/admin/properties-sql-environment) is configured, you can use the following more compact syntax: diff --git a/docs/src/main/sphinx/sql/select.md b/docs/src/main/sphinx/sql/select.md index 1a07c05b89ff..82938f816871 100644 --- a/docs/src/main/sphinx/sql/select.md +++ b/docs/src/main/sphinx/sql/select.md @@ -3,7 +3,7 @@ ## Synopsis ```text -[ WITH FUNCTION sql_routines ] +[ WITH FUNCTION udf ] [ WITH [ RECURSIVE ] with_query [, ...] ] SELECT [ ALL | DISTINCT ] select_expression [, ...] [ FROM from_item [, ...] ] @@ -70,10 +70,10 @@ Retrieve rows from zero or more tables. ## WITH FUNCTION clause -The `WITH FUNCTION` clause allows you to define a list of inline SQL routines -that are available for use in the rest of the query. +The `WITH FUNCTION` clause allows you to define a list of [](udf-inline) that +are available for use in the rest of the query. -The following example declares and uses two inline routines: +The following example declares and uses two inline UDFs: ```sql WITH @@ -87,8 +87,8 @@ SELECT hello('Finn') || ' and ' || bye('Joe'); -- Hello Finn! and Bye Joe! ``` -Find further information about routines in general, inline routines, all -supported statements, and examples in [](/udf). +Find further information about UDFs in general, inline UDFs, all supported +statements, and examples in [](/udf). ## WITH clause diff --git a/docs/src/main/sphinx/sql/show-functions.md b/docs/src/main/sphinx/sql/show-functions.md index e6c3a7329161..41136258bda7 100644 --- a/docs/src/main/sphinx/sql/show-functions.md +++ b/docs/src/main/sphinx/sql/show-functions.md @@ -30,8 +30,8 @@ filter the results to the desired subset. ## Examples -List all SQL routines and plugin functions in the `default` schema of the -`example` catalog: +List all UDFs and plugin functions in the `default` schema of the `example` +catalog: ```sql SHOW FUNCTIONS FROM example.default; diff --git a/docs/src/main/sphinx/udf.md b/docs/src/main/sphinx/udf.md index d368d423804c..4c865781c228 100644 --- a/docs/src/main/sphinx/udf.md +++ b/docs/src/main/sphinx/udf.md @@ -1,9 +1,17 @@ # User-defined functions -A SQL routine is a custom, user-defined function authored by a user of Trino in -a client and written in the SQL routine language. Routines are scalar functions -that return a single output value. More details are available in the following -sections: +A user-defined function (UDF) is a custom function authored by a user of Trino +in a client application. UDFs are scalar functions that return a single output +value, similar to [built-in functions](/functions). + +[Declare the UDF](udf-declaration) with a `FUNCTION` definition using the +supported statements. A UDF can be declared and used as an [inline +UDF](udf-inline) or declared as a [catalog UDF](udf-catalog) and used +repeatedly. + +UDFs are defined and written using the [SQL routine language](/udf/sql). + +More details are available in the following sections: ```{toctree} :titlesonly: true diff --git a/docs/src/main/sphinx/udf/function.md b/docs/src/main/sphinx/udf/function.md index 72fbae1d751d..a8d5b6425dc9 100644 --- a/docs/src/main/sphinx/udf/function.md +++ b/docs/src/main/sphinx/udf/function.md @@ -16,51 +16,50 @@ FUNCTION name ( [ parameter_name data_type [, ...] ] ) ## Description -Declare a SQL routine. +Declare a [user-defined function](/udf). -The `name` of the routine. [Inline routines](routine-inline) can use a simple -string. [Catalog routines](routine-catalog) must qualify the name of the catalog -and schema, delimited by `.`, to store the routine or rely on the [default -catalog and schema for routine storage](/admin/properties-sql-environment). +The `name` of the UDF. [](udf-inline) can use a simple string. [](udf-catalog) +must qualify the name of the catalog and schema, delimited by `.`, to store the +UDF or rely on the [default catalog and schema for UDF +storage](/admin/properties-sql-environment). The list of parameters is a comma-separated list of names `parameter_name` and data types `data_type`, see [data type](/language/types). An empty list, specified as `()` is also valid. The `type` value after the `RETURNS` keyword identifies the [data -type](/language/types) of the routine output. +type](/language/types) of the UDF output. -The optional `LANGUAGE` characteristic identifies the language used for the -routine definition with `language`. Only `SQL` is supported. +The optional `LANGUAGE` characteristic identifies the language used for the UDF +definition with `language`. Only `SQL` is supported. The optional `DETERMINISTIC` or `NOT DETERMINISTIC` characteristic declares that -the routine is deterministic. This means that repeated routine calls with -identical input parameters yield the same result. For SQL language routines, a -routine is non-deterministic if it calls any non-deterministic routines and -[functions](/functions). By default, routines are assume to have a deterministic -behavior. +the UDF is deterministic. This means that repeated UDF calls with identical +input parameters yield the same result. A UDF is non-deterministic if it calls +any non-deterministic UDFs and [functions](/functions). By default, UDFs are +assumed to have a deterministic behavior. -The optional `RETURNS NULL ON NULL INPUT` characteristic declares that the -routine returns a `NULL` value when any of the input parameters are `NULL`. -The routine is not invoked with a `NULL` input value. +The optional `RETURNS NULL ON NULL INPUT` characteristic declares that the UDF +returns a `NULL` value when any of the input parameters are `NULL`. The UDF is +not invoked with a `NULL` input value. -The `CALLED ON NULL INPUT` characteristic declares that the routine is invoked -with `NULL` input parameter values. +The `CALLED ON NULL INPUT` characteristic declares that the UDF is invoked with +`NULL` input parameter values. The `RETURNS NULL ON NULL INPUT` and `CALLED ON NULL INPUT` characteristics are mutually exclusive, with `CALLED ON NULL INPUT` as the default. The security declaration of `SECURITY INVOKER` or `SECURITY DEFINER` is only -valid for catalog routines. It sets the mode for processing the routine with the -permissions of the user who calls the routine (`INVOKER`) or the user who -created the routine (`DEFINER`). +valid for catalog UDFs. It sets the mode for processing the UDF with the +permissions of the user who calls the UDF (`INVOKER`) or the user who created +the UDF (`DEFINER`). The `COMMENT` characteristic can be used to provide information about the function to other users as `description`. The information is accessible with [](/sql/show-functions). -The body of the routine can either be a simple single `RETURN` statement with an -expression, or compound list of `statements` in a `BEGIN` block. Routines must +The body of the UDF can either be a simple single `RETURN` statement with an +expression, or compound list of `statements` in a `BEGIN` block. UDF must contain a `RETURN` statement at the end of the top-level block, even if it's unreachable. diff --git a/docs/src/main/sphinx/udf/introduction.md b/docs/src/main/sphinx/udf/introduction.md index db5d51493578..71585bd9f3b2 100644 --- a/docs/src/main/sphinx/udf/introduction.md +++ b/docs/src/main/sphinx/udf/introduction.md @@ -1,19 +1,22 @@ -# Introduction to SQL routines +# Introduction to user-defined functions -A SQL routine is a custom, user-defined function authored by a user of Trino and -written in the SQL routine language. Routines are scalar functions that return a -single output value, similar to [built-in functions](/functions). +A user-defined function (UDF) is a custom function authored by a user of Trino +in a client application. UDFs are scalar functions that return a single output +value, similar to [built-in functions](/functions). -[Declare the routine](routine-declaration) with a `FUNCTION` definition using -the supported SQL routine statements. A routine can be declared and used as an -[inline routine](routine-inline) or declared as a [catalog -routine](routine-catalog) and used repeatedly. +[Declare the UDF](udf-declaration) with a `FUNCTION` definition using the +supported statements. A UDF can be declared and used as an [inline +UDF](udf-inline) or declared as a [catalog UDF](udf-catalog) and used +repeatedly. -(routine-inline)= -## Inline routines +UDFs are defined and written using the [SQL routine language](/udf/sql). -An inline routine declares and uses the routine within a query processing -context. The routine is declared in a `WITH` block before the query: +(udf-inline)= +## Inline user-defined functions + +An inline user-defined function (inline UDF) declares and uses the UDF within a +query processing context. The UDF is declared in a `WITH` block before the +query: ```sql WITH @@ -23,16 +26,16 @@ WITH SELECT abc(21); ``` -Inline routine names must follow SQL identifier naming conventions, and cannot +Inline UDF names must follow SQL identifier naming conventions, and cannot contain `.` characters. -The routine declaration is only valid within the context of the query. A -separate later invocation of the routine is not possible. If this is desired, -use a [catalog routine](routine-catalog). +The UDF declaration is only valid within the context of the query. A separate +later invocation of the UDF is not possible. If this is desired, use a [catalog +UDF](udf-catalog). -Multiple inline routine declarations are comma-separated, and can include -routines calling each other, as long as a called routine is declared before -the first invocation. +Multiple inline UDF declarations are comma-separated, and can include UDFs +calling each other, as long as a called UDF is declared before the first +invocation. ```sql WITH @@ -45,7 +48,7 @@ WITH SELECT xyz(21); ``` -Note that inline routines can mask and override the meaning of a built-in function: +Note that inline UDFs can mask and override the meaning of a built-in function: ```sql WITH @@ -55,48 +58,47 @@ WITH SELECT abs(-10); -- -20, not 10! ``` -(routine-catalog)= -## Catalog routines +(udf-catalog)= +## Catalog user-defined functions -You can store a routine in the context of a catalog, if the connector used in -the catalog supports routine storage. The following connectors support catalog -routine storage: +You can store a UDF in the context of a catalog, if the connector used in the +catalog supports UDF storage. The following connectors support catalog UDF +storage: * [](/connector/hive) * [](/connector/memory) In this scenario, the following commands can be used: -* [](/sql/create-function) to create and store a routine. -* [](/sql/drop-function) to remove a routine. -* [](/sql/show-functions) to display a list of routines in a catalog. +* [](/sql/create-function) to create and store a UDF. +* [](/sql/drop-function) to remove a UDF. +* [](/sql/show-functions) to display a list of UDFs in a catalog. -Catalog routines must use a name that combines the catalog name and schema name -with the routine name, such as `example.default.power` for the `power` routine -in the `default` schema of the `example` catalog. +Catalog UDFs must use a name that combines the catalog name and schema name with +the UDF name, such as `example.default.power` for the `power` UDF in the +`default` schema of the `example` catalog. Invocation must use the fully qualified name, such as `example.default.power`. -(routine-sql-environment)= -## SQL environment configuration +(udf-sql-environment)= +## SQL environment configuration for UDFs Configuration of the `sql.default-function-catalog` and `sql.default-function-schema` [](/admin/properties-sql-environment) allows you -to set the default storage for SQL routines. The catalog and schema must be -added to the `sql.path` as well. This enables users to call SQL routines and -perform all [SQL routine management](sql-routine-management) without specifying -the full path to the routine. +to set the default storage for UDFs. The catalog and schema must be added to the +`sql.path` as well. This enables users to call UDFs and perform all +[](udf-management) without specifying the full path to the UDF. :::{note} Use the [](/connector/memory) in a catalog for simple storing and -testing of your SQL routines. +testing of your UDFs. ::: -(routine-declaration)= -## Routine declaration +(udf-declaration)= +## UDF declaration Refer to the documentation for the [](/udf/function) keyword for more -details about declaring the routine overall. The routine body is composed with +details about declaring the UDF overall. The UDF body is composed with statements from the following list: * [](/udf/sql/begin) @@ -112,7 +114,7 @@ statements from the following list: * [](/udf/sql/while) Statements can also use [built-in functions and operators](/functions) as well -as other routines, although recursion is not supported for routines. +as other UDFs, although recursion is not supported for UDFs. Find simple examples in each statement documentation, and refer to the [](/udf/sql/examples) for more complex use cases that combine multiple @@ -123,10 +125,10 @@ User-defined functions can alternatively be written in Java and deployed as a plugin. Details are available in the [developer guide](/develop/functions). ::: -(routine-label)= +(udf-sql-label)= ## Labels -Routines can contain labels as markers for a specific block in the declaration +SQL UDFs can contain labels as markers for a specific block in the declaration before the following keywords: * `CASE` @@ -158,12 +160,12 @@ nested blocks and labels. ## Recommendations -Processing routines can potentially be resource intensive on the cluster in +Processing UDFs can potentially be resource intensive on the cluster in terms of memory and processing. Take the following considerations into account -when writing and running SQL routines: +when writing and running SQL UDFs: -* Some checks for the runtime behavior of routines are in place. For example, - routines that take longer to process than a hardcoded threshold are +* Some checks for the runtime behavior of UDF are in place. For example, + UDFs that take longer to process than a hardcoded threshold are automatically terminated. * Avoid creation of arrays in a looping construct. Each iteration creates a separate new array with all items and copies the data for each modification, @@ -173,20 +175,20 @@ when writing and running SQL routines: separate new string and copying the old string for each modification, leaving the prior string in memory for automated clean up later. Use a [lambda expression](/functions/lambda) instead of the loop. -* Most routines should declare the `RETURNS NULL ON NULL INPUT` characteristics +* Most UDFs should declare the `RETURNS NULL ON NULL INPUT` characteristics unless the code has some special handling for null values. You must declare this explicitly since `CALLED ON NULL INPUT` is the default characteristic. ## Limitations -The following limitations apply to SQL routines. +The following limitations apply to SQL UDFs. -* Routines must be declared before they are referenced. +* SQL UDFs must be declared before they are referenced. * Recursion cannot be declared or processed. * Mutual recursion can not be declared or processed. -* Queries cannot be processed in a routine. +* Queries cannot be processed in a SQL UDF. -Specifically this means that routines can not use `SELECT` queries to retrieve -data or any other queries to process data within the routine. Instead queries -can use routines to process data. Routines only work on data provided as input -values and only provide output data from the `RETURN` statement. +Specifically this means that SQL UDFs can not use `SELECT` queries to retrieve +data or any other queries to process data within the UDF. Instead queries can +use UDFs to process data. UDFs only work on data provided as input values and +only provide output data from the `RETURN` statement. diff --git a/docs/src/main/sphinx/udf/sql.md b/docs/src/main/sphinx/udf/sql.md index ae40c062f943..018b7acf66ea 100644 --- a/docs/src/main/sphinx/udf/sql.md +++ b/docs/src/main/sphinx/udf/sql.md @@ -1,4 +1,4 @@ -# SQL routines +# SQL user-defined functions ```{toctree} :titlesonly: true diff --git a/docs/src/main/sphinx/udf/sql/begin.md b/docs/src/main/sphinx/udf/sql/begin.md index 449b713ce004..7e850383193e 100644 --- a/docs/src/main/sphinx/udf/sql/begin.md +++ b/docs/src/main/sphinx/udf/sql/begin.md @@ -18,7 +18,7 @@ within a [](/udf/function). Blocks can also be nested. After the `BEGIN` keyword, you can add variable declarations using [](/udf/sql/declare) statements, followed by one or more statements that define -the main body of the routine, separated by `;`. The following statements can be +the main body of the SQL UDF, separated by `;`. The following statements can be used: * [](/udf/sql/case) diff --git a/docs/src/main/sphinx/udf/sql/examples.md b/docs/src/main/sphinx/udf/sql/examples.md index 80b69fb7b453..95a87da63f77 100644 --- a/docs/src/main/sphinx/udf/sql/examples.md +++ b/docs/src/main/sphinx/udf/sql/examples.md @@ -1,22 +1,21 @@ -# Example SQL routines +# Example SQL UDFs After learning about [](/udf/sql), the following sections show numerous examples -of valid SQL routines. The routines are suitable as [inline -routines](routine-inline) or [catalog routines](routine-catalog), after -adjusting the name and the example invocations. +of valid SQL UDFs. The UDFs are suitable as [](udf-inline) or [](udf-catalog), +after adjusting the name and the example invocations. The examples combine numerous supported statements. Refer to the specific statement documentation for further details: -* [](/udf/function) for general SQL routine declaration -* [](/udf/sql/begin) and [](/udf/sql/declare) for routine blocks +* [](/udf/function) for general UDF declaration +* [](/udf/sql/begin) and [](/udf/sql/declare) for SQL UDF blocks * [](/udf/sql/set) for assigning values to variables -* [](/udf/sql/return) for returning routine results +* [](/udf/sql/return) for returning results * [](/udf/sql/case) and [](/udf/sql/if) for conditional flows * [](/udf/sql/loop), [](/udf/sql/repeat), and [](/udf/sql/while) for looping constructs * [](/udf/sql/iterate) and [](/udf/sql/leave) for flow control -A very simple routine that returns a static value without requiring any input: +A very simple SQL UDF that returns a static value without requiring any input: ```sql FUNCTION answer() @@ -24,10 +23,10 @@ RETURNS BIGINT RETURN 42 ``` -## Inline and catalog routines +## Inline and catalog UDFs -A full example of this routine as inline routine and usage in a string -concatenation with a cast: +A full example of this UDF as inline UDF and usage in a string concatenation +with a cast: ```sql WITH @@ -38,8 +37,8 @@ SELECT 'The answer is ' || CAST(answer() as varchar); -- The answer is 42 ``` -Provided the catalog `example` supports routine storage in the `default` schema, -you can use the following: +Provided the catalog `example` supports UDF storage in the `default` schema, you +can use the following: ```sql USE example.default; @@ -48,8 +47,8 @@ CREATE FUNCTION example.default.answer() RETURN 42; ``` -With the routine stored in the catalog, you can run the routine multiple times -without repeated definition: +With the UDF stored in the catalog, you can run the UDF multiple times without +repeated definition: ```sql SELECT example.default.answer() + 1; -- 43 @@ -57,14 +56,14 @@ SELECT 'The answer is' || CAST(example.default.answer() as varchar); -- The answ ``` Alternatively, you can configure the SQL environment in the -[](config-properties) to a catalog and schema that support SQL routine storage: +[](config-properties) to a catalog and schema that support UDF storage: ```properties sql.default-function-catalog=example sql.default-function-schema=default ``` -Now you can manage SQL routines without the full path: +Now you can manage UDFs without the full path: ```sql CREATE FUNCTION answer() @@ -72,7 +71,7 @@ CREATE FUNCTION answer() RETURN 42; ``` -SQL routine invocation works without the full path: +UDF invocation works without the full path: ```sql SELECT answer() + 5; -- 47 @@ -80,7 +79,7 @@ SELECT answer() + 5; -- 47 ## Declaration examples -The result of calling the routine `answer()` is always identical, so you can +The result of calling the UDF `answer()` is always identical, so you can declare it as deterministic, and add some other information: ```sql @@ -92,10 +91,10 @@ COMMENT 'Provide the answer to the question about life, the universe, and everyt RETURN 42 ``` -The comment and other information about the routine is visible in the output of +The comment and other information about the UDF is visible in the output of [](/sql/show-functions). -A simple routine that returns a greeting back to the input string `fullname` +A simple UDF that returns a greeting back to the input string `fullname` concatenating two strings and the input value: ```sql @@ -110,7 +109,7 @@ Following is an example invocation: SELECT hello('Jane Doe'); -- Hello, Jane Doe! ``` -A first example routine, that uses multiple statements in a `BEGIN` block. It +A first example UDF, that uses multiple statements in a `BEGIN` block. It calculates the result of a multiplication of the input integer with `99`. The `bigint` data type is used for all variables and values. The value of integer `99` is cast to `bigint` in the default value assignment for the variable `x`: @@ -132,7 +131,7 @@ SELECT times_ninety_nine(CAST(2 as bigint)); -- 198 ## Conditional flows -A first example of conditional flow control in a routine using the `CASE` +A first example of conditional flow control in a SQL UDF using the `CASE` statement. The simple `bigint` input value is compared to a number of values: ```sql @@ -163,7 +162,7 @@ SELECT simple_case(100); -- other (from else clause) SELECT simple_case(null); -- null .. but really?? ``` -A second example of a routine with a `CASE` statement, this time with two +A second example of a SQL UDF with a `CASE` statement, this time with two parameters, showcasing the importance of the order of the conditions: ```sql @@ -198,15 +197,15 @@ SELECT simple_case(null,null); -- null .. but really?? ## Fibonacci example -This routine calculates the `n`-th value in the Fibonacci series, in which each -number is the sum of the two preceding ones. The two initial values are set -to `1` as the defaults for `a` and `b`. The routine uses an `IF` statement -condition to return `1` for all input values of `2` or less. The `WHILE` block -then starts to calculate each number in the series, starting with `a=1` and -`b=1` and iterates until it reaches the `n`-th position. In each iteration is -sets `a` and `b` for the preceding to values, so it can calculate the sum, and -finally return it. Note that processing the routine takes longer and longer with -higher `n` values, and the result is deterministic: +This SQL UDF calculates the `n`-th value in the Fibonacci series, in which each +number is the sum of the two preceding ones. The two initial values are set to +`1` as the defaults for `a` and `b`. The UDF uses an `IF` statement condition to +return `1` for all input values of `2` or less. The `WHILE` block then starts to +calculate each number in the series, starting with `a=1` and `b=1` and iterates +until it reaches the `n`-th position. In each iteration it sets `a` and `b` for +the preceding to values, so it can calculate the sum, and finally return it. +Note that processing the UDF takes longer and longer with higher `n` values, and +the result is deterministic: ```sql FUNCTION fib(n bigint) @@ -244,13 +243,13 @@ SELECT fib(8); -- 21 ## Labels and loops -This routine uses the `top` label to name the `WHILE` block, and then controls +This SQL UDF uses the `top` label to name the `WHILE` block, and then controls the flow with conditional statements, `ITERATE`, and `LEAVE`. For the values of `a=1` and `a=2` in the first two iterations of the loop the `ITERATE` call moves the flow up to `top` before `b` is ever increased. Then `b` is increased for the values `a=3`, `a=4`, `a=5`, `a=6`, and `a=7`, resulting in `b=5`. The `LEAVE` call then causes the exit of the block before a is increased further to `10` and -therefore the result of the routine is `5`: +therefore the result of the UDF is `5`: ```sql FUNCTION labels() @@ -271,9 +270,9 @@ BEGIN END ``` -This routine implements calculating the `n` to the power of `p` by repeated +This SQL UDF implements calculating the `n` to the power of `p` by repeated multiplication and keeping track of the number of multiplications performed. -Note that this routine does not return the correct `0` for `p=0` since the `top` +Note that this SQL UDF does not return the correct `0` for `p=0` since the `top` block is merely escaped and the value of `n` is returned. The same incorrect behavior happens for negative values of `p`: @@ -303,7 +302,7 @@ SELECT power(3, 0); -- 3, which is wrong SELECT power(3, -2); -- 3, which is wrong ``` -This routine returns `7` as a result of the increase of `b` in the loop from +This SQL UDF returns `7` as a result of the increase of `b` in the loop from `a=3` to `a=10`: ```sql @@ -324,7 +323,7 @@ BEGIN END ``` -This routine returns `2` and shows that labels can be repeated and label usage +This SQL UDF returns `2` and shows that labels can be repeated and label usage within a block refers to the label of that block: ```sql @@ -344,10 +343,10 @@ BEGIN END ``` -## Routines and built-in functions +## SQL UDFs and built-in functions -This routine show that multiple data types and built-in functions like -`length()` and `cardinality()` can be used in a routine. The two nested `BEGIN` +This SQL UDF shows that multiple data types and built-in functions like +`length()` and `cardinality()` can be used in a UDF. The two nested `BEGIN` blocks also show how variable names are local within these blocks `x`, but the global `r` from the top-level block can be accessed in the nested blocks: @@ -370,13 +369,13 @@ END ## Optional parameter example -Routines can invoke other routines and other functions. The full signature of a -routine is composed of the routine name and parameters, and determines the exact -routine to use. You can declare multiple routines with the same name, but with a -different number of arguments or different argument types. One example use case -is to implement an optional parameter. +UDFs can invoke other UDFs and other functions. The full signature of a UDF is +composed of the UDF name and parameters, and determines the exact UDF to use. +You can declare multiple UDFs with the same name, but with a different number of +arguments or different argument types. One example use case is to implement an +optional parameter. -The following routine truncates a string to the specified length including three +The following SQL UDF truncates a string to the specified length including three dots at the end of the output: ```sql @@ -399,8 +398,8 @@ SELECT dots('A short string',15); -- A short string ``` -If you want to provide a routine with the same name, but without the parameter -for length, you can create another routine that invokes the preceding routine: +If you want to provide a UDF with the same name, but without the parameter +for length, you can create another UDF that invokes the preceding UDF: ```sql FUNCTION dots(input varchar) @@ -408,7 +407,7 @@ RETURNS varchar RETURN dots(input, 15); ``` -You can now use both routines. When the length parameter is omitted, the default +You can now use both UDFs. When the length parameter is omitted, the default value from the second declaration is used. ```sql @@ -422,18 +421,18 @@ SELECT dots('A long string that will be shortened',20); ## Date string parsing example -This example routine parses a date string of type `VARCHAR` into `TIMESTAMP WITH +This example SQL UDF parses a date string of type `VARCHAR` into `TIMESTAMP WITH TIME ZONE`. Date strings are commonly represented by ISO 8601 standard, such as `2023-12-01`, `2023-12-01T23`. Date strings are also often represented in the `YYYYmmdd` and `YYYYmmddHH` format, such as `20230101` and `2023010123`. Hive tables can use this format to represent day and hourly partitions, for example `/day=20230101`, `/hour=2023010123`. -This routine parses date strings in a best-effort fashion and can be used as a +This UDF parses date strings in a best-effort fashion and can be used as a replacement for date string manipulation functions such as `date`, `date_parse`, `from_iso8601_date`, and `from_iso8601_timestamp`. -Note that the routine defaults the time value to `00:00:00.000` and the time +Note that the UDF defaults the time value to `00:00:00.000` and the time zone to the session time zone: ```sql @@ -473,7 +472,7 @@ SELECT human_readable_seconds(134823); -- 1 day, 13 hours, 27 minutes, 3 seconds ``` -The example routine `hrd` formats a number of days into a human readable text +The example SQL UDF `hrd` formats a number of days into a human readable text that provides the approximate number of years and months: ```sql @@ -521,16 +520,16 @@ SELECT hrd(1100); -- About 3 years SELECT hrd(5000); -- About 13 years and 8 months ``` -Improvements of the routine could include the following modifications: +Improvements of the SQL UDF could include the following modifications: * Take into account that one month equals 30.4375 days. * Take into account that one year equals 365.25 days. * Add weeks to the output. -* Expand to cover decades, centuries, and millenia. +* Expand to cover decades, centuries, and millennia. ## Truncating long strings -This example routine `strtrunc` truncates strings longer than 60 characters, +This example SQL UDF `strtrunc` truncates strings longer than 60 characters, leaving the first 30 and the last 25 characters, and cutting out extra characters in the middle: @@ -548,10 +547,10 @@ The preceding declaration is very compact and consists of only one complex statement with a [`CASE` expression](case-expression) and multiple function calls. It can therefore define the complete logic in the `RETURN` clause. -The following statement shows the same capability within the routine itself. +The following statement shows the same capability within the SQL UDF itself. Note the duplicate `RETURN` inside and outside the `CASE` statement and the -required `END CASE;`. The second `RETURN` statement is required, because a -routine must end with a `RETURN` statement. As a result the `ELSE` clause can be +required `END CASE;`. The second `RETURN` statement is required, because a SQL +UDF must end with a `RETURN` statement. As a result the `ELSE` clause can be omitted: ```sql @@ -599,8 +598,8 @@ FROM data ORDER BY data.value; ``` -The preceding query produces the following output with all variants of the -routine: +The preceding query produces the following output with all variants of the SQL +UDF: ``` value | truncated @@ -621,7 +620,7 @@ A possible improvement is to introduce parameters for the total length. ## Formatting bytes Trino includes a built-in `format_number()` function. However, it is using units -that do not work well with bytes. The following `format_data_size` routine can +that do not work well with bytes. The following `format_data_size` SQL UDF can format large values of bytes into a human readable string: ```sql @@ -743,7 +742,7 @@ The preceding query produces the following output: Trino already has a built-in `bar()` [color function](/functions/color), but it is using ANSI escape codes to output colors, and thus is only usable for -displaying results in a terminal. The following example shows a similar routine +displaying results in a terminal. The following example shows a similar SQL UDF that only uses ASCII characters: ```sql @@ -816,7 +815,7 @@ The preceding query produces the following output: 3.1 | 0.0416 | ▋ ``` -It is also possible to draw more compacted charts. Following is a routine +It is also possible to draw more compacted charts. Following is a SQL UDF drawing vertical bars: ```sql @@ -896,7 +895,7 @@ Trino already has a built-in [aggregate function](/functions/aggregate) called values. It returns a map with values as keys and number of occurrences as values. Maps are not ordered, so when displayed, the entries can change places on subsequent runs of the same query, and readers must still compare all -frequencies to find the one most frequent value. The following is a routine that +frequencies to find the one most frequent value. The following is a SQL UDF that returns ordered results as a string: ```sql diff --git a/docs/src/main/sphinx/udf/sql/iterate.md b/docs/src/main/sphinx/udf/sql/iterate.md index 9ec29dc0f65d..59b67f2710ca 100644 --- a/docs/src/main/sphinx/udf/sql/iterate.md +++ b/docs/src/main/sphinx/udf/sql/iterate.md @@ -10,7 +10,7 @@ ITERATE label The `ITERATE` statement allows processing of blocks in [](/udf/sql) to move processing back to the start of a context block. Contexts are defined by a -[`label`](routine-label). If no label is found, the functions fails with an +[`label`](udf-sql-label). If no label is found, the functions fails with an error message. ## Examples diff --git a/docs/src/main/sphinx/udf/sql/leave.md b/docs/src/main/sphinx/udf/sql/leave.md index 9aea557e08f5..7697a3fc3fa7 100644 --- a/docs/src/main/sphinx/udf/sql/leave.md +++ b/docs/src/main/sphinx/udf/sql/leave.md @@ -9,7 +9,7 @@ LEAVE label ## Description The `LEAVE` statement allows processing of blocks in [](/udf/sql) to move out of -a specified context. Contexts are defined by a [`label`](routine-label). If no +a specified context. Contexts are defined by a [`label`](udf-sql-label). If no label is found, the functions fails with an error message. ## Examples diff --git a/docs/src/main/sphinx/udf/sql/loop.md b/docs/src/main/sphinx/udf/sql/loop.md index 9517486d79de..339cea3b50b4 100644 --- a/docs/src/main/sphinx/udf/sql/loop.md +++ b/docs/src/main/sphinx/udf/sql/loop.md @@ -19,7 +19,7 @@ of processing from the beginning starts. `LEAVE` statements are typically wrapped in an `IF` statement that declares a condition to stop the loop. The optional `label` before the `LOOP` keyword can be used to [name the -block](routine-label). +block](udf-sql-label). ## Examples @@ -54,7 +54,7 @@ SELECT to_one_hundred(12, 3); -- 30 ``` Further examples of varying complexity that cover usage of the `LOOP` statement -in combination with other statements are available in the [SQL routines examples +in combination with other statements are available in the [SQL UDF examples documentation](/udf/sql/examples). ## See also diff --git a/docs/src/main/sphinx/udf/sql/repeat.md b/docs/src/main/sphinx/udf/sql/repeat.md index 5d71ebaa5241..d75eebfec5ca 100644 --- a/docs/src/main/sphinx/udf/sql/repeat.md +++ b/docs/src/main/sphinx/udf/sql/repeat.md @@ -21,7 +21,7 @@ subsequent processing the expression `condidtion` is validated. If the result is the function. If the result is `false`, the statements are processed again. The optional `label` before the `REPEAT` keyword can be used to [name the -block](routine-label). +block](udf-sql-label). Note that a `WHILE` statement is very similar, with the difference that for `REPEAT` the statements are processed at least once, and for `WHILE` blocks the @@ -29,7 +29,7 @@ statements might not be processed at all. ## Examples -The following routine shows a routine with a `REPEAT` statement that runs until +The following SQL UDF shows a UDF with a `REPEAT` statement that runs until the value of `a` is greater or equal to `10`. ```sql @@ -45,7 +45,7 @@ FUNCTION test_repeat(a bigint) ``` Since `a` is also the input value and it is increased before the check the -routine always returns `10` for input values of `9` or less, and the input value +UDF always returns `10` for input values of `9` or less, and the input value + 1 for all higher values. Following are a couple of example invocations with result and explanation: diff --git a/docs/src/main/sphinx/udf/sql/return.md b/docs/src/main/sphinx/udf/sql/return.md index a376bd304e1a..2044b17466b0 100644 --- a/docs/src/main/sphinx/udf/sql/return.md +++ b/docs/src/main/sphinx/udf/sql/return.md @@ -27,8 +27,8 @@ Further examples of varying complexity that cover usage of the `RETURN` statement in combination with other statements are available in the [](/udf/sql/examples). -All routines must contain a `RETURN` statement at the end of the top-level -block in the `FUNCTION` declaration, even if it's unreachable. +All SQL UDFs must contain a `RETURN` statement at the end of the top-level block +in the `FUNCTION` declaration, even if it's unreachable. ## See also diff --git a/docs/src/main/sphinx/udf/sql/while.md b/docs/src/main/sphinx/udf/sql/while.md index 9646968d51e8..74f4b143659e 100644 --- a/docs/src/main/sphinx/udf/sql/while.md +++ b/docs/src/main/sphinx/udf/sql/while.md @@ -20,7 +20,7 @@ back to `WHILE` and the `condition`. If the result is `false`, processing moves to `END WHILE` and continues with the next statement in the function. The optional `label` before the `WHILE` keyword can be used to [name the -block](routine-label). +block](udf-sql-label). Note that a `WHILE` statement is very similar, with the difference that for `REPEAT` the statements are processed at least once, and for `WHILE` blocks the From a1fcbce84eff4dc6760e81332b81215498da299b Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Thu, 12 Dec 2024 13:40:37 -0800 Subject: [PATCH 004/158] Move SQL UDF content Into the separate page, and adjust the generic content to be suitable for any UDF language. --- docs/src/main/sphinx/udf.md | 8 +- docs/src/main/sphinx/udf/introduction.md | 109 ++++++----------------- docs/src/main/sphinx/udf/sql.md | 77 +++++++++++++++- 3 files changed, 105 insertions(+), 89 deletions(-) diff --git a/docs/src/main/sphinx/udf.md b/docs/src/main/sphinx/udf.md index 4c865781c228..ac715cc029a9 100644 --- a/docs/src/main/sphinx/udf.md +++ b/docs/src/main/sphinx/udf.md @@ -4,17 +4,11 @@ A user-defined function (UDF) is a custom function authored by a user of Trino in a client application. UDFs are scalar functions that return a single output value, similar to [built-in functions](/functions). -[Declare the UDF](udf-declaration) with a `FUNCTION` definition using the -supported statements. A UDF can be declared and used as an [inline -UDF](udf-inline) or declared as a [catalog UDF](udf-catalog) and used -repeatedly. - -UDFs are defined and written using the [SQL routine language](/udf/sql). - More details are available in the following sections: ```{toctree} :titlesonly: true +:maxdepth: 1 udf/introduction udf/function diff --git a/docs/src/main/sphinx/udf/introduction.md b/docs/src/main/sphinx/udf/introduction.md index 71585bd9f3b2..4056b707088a 100644 --- a/docs/src/main/sphinx/udf/introduction.md +++ b/docs/src/main/sphinx/udf/introduction.md @@ -1,16 +1,25 @@ -# Introduction to user-defined functions +# Introduction to UDFs A user-defined function (UDF) is a custom function authored by a user of Trino in a client application. UDFs are scalar functions that return a single output value, similar to [built-in functions](/functions). -[Declare the UDF](udf-declaration) with a `FUNCTION` definition using the -supported statements. A UDF can be declared and used as an [inline -UDF](udf-inline) or declared as a [catalog UDF](udf-catalog) and used -repeatedly. - UDFs are defined and written using the [SQL routine language](/udf/sql). +:::{note} +User-defined functions can alternatively be written in Java and deployed as a +plugin. Details are available in the [developer guide](/develop/functions). +::: + +(udf-declaration)= +## UDF declaration + +Declare the UDF with a [](/udf/function) keyword using the supported statements +for [](/udf/sql). + +A UDF can be declared and used as an [inline UDF](udf-inline) or declared as a +[catalog UDF](udf-catalog) and used repeatedly. + (udf-inline)= ## Inline user-defined functions @@ -20,10 +29,11 @@ query: ```sql WITH - FUNCTION abc(x integer) + FUNCTION doubleup(x integer) RETURNS integer RETURN x * 2 -SELECT abc(21); +SELECT doubleup(21); +-- 42 ``` Inline UDF names must follow SQL identifier naming conventions, and cannot @@ -39,13 +49,14 @@ invocation. ```sql WITH - FUNCTION abc(x integer) + FUNCTION doubleup(x integer) RETURNS integer RETURN x * 2, - FUNCTION xyz(x integer) + FUNCTION doubleupplusone(x integer) RETURNS integer - RETURN abc(x) + 1 -SELECT xyz(21); + RETURN doubleup(x) + 1 +SELECT doubleupplusone(21); +-- 43 ``` Note that inline UDFs can mask and override the meaning of a built-in function: @@ -94,75 +105,11 @@ Use the [](/connector/memory) in a catalog for simple storing and testing of your UDFs. ::: -(udf-declaration)= -## UDF declaration - -Refer to the documentation for the [](/udf/function) keyword for more -details about declaring the UDF overall. The UDF body is composed with -statements from the following list: - -* [](/udf/sql/begin) -* [](/udf/sql/case) -* [](/udf/sql/declare) -* [](/udf/sql/if) -* [](/udf/sql/iterate) -* [](/udf/sql/leave) -* [](/udf/sql/loop) -* [](/udf/sql/repeat) -* [](/udf/sql/return) -* [](/udf/sql/set) -* [](/udf/sql/while) - -Statements can also use [built-in functions and operators](/functions) as well -as other UDFs, although recursion is not supported for UDFs. - -Find simple examples in each statement documentation, and refer to the -[](/udf/sql/examples) for more complex use cases that combine multiple -statements. - -:::{note} -User-defined functions can alternatively be written in Java and deployed as a -plugin. Details are available in the [developer guide](/develop/functions). -::: - -(udf-sql-label)= -## Labels - -SQL UDFs can contain labels as markers for a specific block in the declaration -before the following keywords: - -* `CASE` -* `IF` -* `LOOP` -* `REPEAT` -* `WHILE` - -The label is used to name the block to continue processing with the `ITERATE` -statement or exit the block with the `LEAVE` statement. This flow control is -supported for nested blocks, allowing to continue or exit an outer block, not -just the innermost block. For example, the following snippet uses the label -`top` to name the complete block from `REPEAT` to `END REPEAT`: - -```sql -top: REPEAT - SET a = a + 1; - IF a <= 3 THEN - ITERATE top; - END IF; - SET b = b + 1; - UNTIL a >= 10 -END REPEAT; -``` - -Labels can be used with the `ITERATE` and `LEAVE` statements to continue -processing the block or leave the block. This flow control is also supported for -nested blocks and labels. - ## Recommendations Processing UDFs can potentially be resource intensive on the cluster in terms of memory and processing. Take the following considerations into account -when writing and running SQL UDFs: +when writing and running UDFs: * Some checks for the runtime behavior of UDF are in place. For example, UDFs that take longer to process than a hardcoded threshold are @@ -181,14 +128,14 @@ when writing and running SQL UDFs: ## Limitations -The following limitations apply to SQL UDFs. +The following limitations apply to UDFs. -* SQL UDFs must be declared before they are referenced. +* UDFs must be declared before they are referenced. * Recursion cannot be declared or processed. * Mutual recursion can not be declared or processed. -* Queries cannot be processed in a SQL UDF. +* Queries cannot be processed in a UDF. -Specifically this means that SQL UDFs can not use `SELECT` queries to retrieve +Specifically this means that UDFs can not use `SELECT` queries to retrieve data or any other queries to process data within the UDF. Instead queries can use UDFs to process data. UDFs only work on data provided as input values and only provide output data from the `RETURN` statement. diff --git a/docs/src/main/sphinx/udf/sql.md b/docs/src/main/sphinx/udf/sql.md index 018b7acf66ea..85e05d9ec1d1 100644 --- a/docs/src/main/sphinx/udf/sql.md +++ b/docs/src/main/sphinx/udf/sql.md @@ -1,7 +1,30 @@ # SQL user-defined functions +A SQL user-defined function, also known as SQL routine, is a [user-defined +function](/udf) that uses the SQL routine language and statements for the +definition of the function. + +## SQL UDF declaration + +Declare a SQL UDF using the [](/udf/function) keyword and the following +statements can be used in addition to [built-in functions and +operators](/functions) and other UDFs: + +* [](/udf/sql/begin) +* [](/udf/sql/case) +* [](/udf/sql/declare) +* [](/udf/sql/if) +* [](/udf/sql/iterate) +* [](/udf/sql/leave) +* [](/udf/sql/loop) +* [](/udf/sql/repeat) +* [](/udf/sql/return) +* [](/udf/sql/set) +* [](/udf/sql/while) + ```{toctree} :titlesonly: true +:hidden: sql/examples sql/begin @@ -15,4 +38,56 @@ sql/repeat sql/return sql/set sql/while -``` \ No newline at end of file +``` + +A minimal example declares the UDF `doubleup` that returns the input integer +value `x` multiplied by two. The example shows declaration as [](udf-inline) and +invocation with the value 21 to yield the result 42: + +```sql +WITH + FUNCTION doubleup(x integer) + RETURNS integer + RETURN x * 2 +SELECT doubleup(21); +-- 42 +``` + +The same UDF can also be declared as [](udf-catalog). + +Find simple examples in each statement documentation, and refer to the +[](/udf/sql/examples) for more complex use cases that combine multiple +statements. + +(udf-sql-label)= +## Labels + +SQL UDFs can contain labels as markers for a specific block in the declaration +before the following keywords: + +* `CASE` +* `IF` +* `LOOP` +* `REPEAT` +* `WHILE` + +The label is used to name the block to continue processing with the `ITERATE` +statement or exit the block with the `LEAVE` statement. This flow control is +supported for nested blocks, allowing to continue or exit an outer block, not +just the innermost block. For example, the following snippet uses the label +`top` to name the complete block from `REPEAT` to `END REPEAT`: + +```sql +top: REPEAT + SET a = a + 1; + IF a <= 3 THEN + ITERATE top; + END IF; + SET b = b + 1; + UNTIL a >= 10 +END REPEAT; +``` + +Labels can be used with the `ITERATE` and `LEAVE` statements to continue +processing the block or leave the block. This flow control is also supported for +nested blocks and labels. From bcb9f6fd777059810e96aafc9c9a20b4c9b283a0 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Mon, 16 Dec 2024 09:35:37 -0800 Subject: [PATCH 005/158] Move SQL UDF content Into the separate page, and adjust the generic content to be suitable for any UDF language. --- docs/src/main/sphinx/udf/introduction.md | 29 ++++++------------------ docs/src/main/sphinx/udf/sql.md | 14 ++++++++++++ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/src/main/sphinx/udf/introduction.md b/docs/src/main/sphinx/udf/introduction.md index 4056b707088a..88189ccb3f52 100644 --- a/docs/src/main/sphinx/udf/introduction.md +++ b/docs/src/main/sphinx/udf/introduction.md @@ -7,18 +7,16 @@ value, similar to [built-in functions](/functions). UDFs are defined and written using the [SQL routine language](/udf/sql). :::{note} -User-defined functions can alternatively be written in Java and deployed as a +Custom functions can alternatively be written in Java and deployed as a plugin. Details are available in the [developer guide](/develop/functions). ::: (udf-declaration)= ## UDF declaration -Declare the UDF with a [](/udf/function) keyword using the supported statements -for [](/udf/sql). - -A UDF can be declared and used as an [inline UDF](udf-inline) or declared as a -[catalog UDF](udf-catalog) and used repeatedly. +A UDF can be declared as an [inline UDF](udf-inline) to be used in the current +query, or declared as a [catalog UDF](udf-catalog) to be used in any future +query. (udf-inline)= ## Inline user-defined functions @@ -111,9 +109,9 @@ Processing UDFs can potentially be resource intensive on the cluster in terms of memory and processing. Take the following considerations into account when writing and running UDFs: -* Some checks for the runtime behavior of UDF are in place. For example, - UDFs that take longer to process than a hardcoded threshold are - automatically terminated. +* Some checks for the runtime behavior of queries, and therefore UDF processing, + are in place. For example, if a query takes longer to process than a hardcoded + threshold, processing is automatically terminated. * Avoid creation of arrays in a looping construct. Each iteration creates a separate new array with all items and copies the data for each modification, leaving the prior array in memory for automated clean up later. Use a [lambda @@ -126,16 +124,3 @@ when writing and running UDFs: unless the code has some special handling for null values. You must declare this explicitly since `CALLED ON NULL INPUT` is the default characteristic. -## Limitations - -The following limitations apply to UDFs. - -* UDFs must be declared before they are referenced. -* Recursion cannot be declared or processed. -* Mutual recursion can not be declared or processed. -* Queries cannot be processed in a UDF. - -Specifically this means that UDFs can not use `SELECT` queries to retrieve -data or any other queries to process data within the UDF. Instead queries can -use UDFs to process data. UDFs only work on data provided as input values and -only provide output data from the `RETURN` statement. diff --git a/docs/src/main/sphinx/udf/sql.md b/docs/src/main/sphinx/udf/sql.md index 85e05d9ec1d1..df70207075b4 100644 --- a/docs/src/main/sphinx/udf/sql.md +++ b/docs/src/main/sphinx/udf/sql.md @@ -91,3 +91,17 @@ END REPEAT; Labels can be used with the `ITERATE` and `LEAVE` statements to continue processing the block or leave the block. This flow control is also supported for nested blocks and labels. + +## Limitations + +The following limitations apply to SQL UDFs. + +* UDFs must be declared before they are referenced. +* Recursion cannot be declared or processed. +* Mutual recursion can not be declared or processed. +* Queries cannot be processed in a UDF. + +Specifically this means that UDFs can not use `SELECT` queries to retrieve +data or any other queries to process data within the UDF. Instead queries can +use UDFs to process data. UDFs only work on data provided as input values and +only provide output data from the `RETURN` statement. \ No newline at end of file From 893fc429a6367b8f6de135c77d65a4d9ad9eb777 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Mon, 16 Dec 2024 09:36:50 -0800 Subject: [PATCH 006/158] Add docs for Python UDFs --- docs/src/main/sphinx/udf.md | 1 + docs/src/main/sphinx/udf/function.md | 29 +++- docs/src/main/sphinx/udf/introduction.md | 5 +- docs/src/main/sphinx/udf/python.md | 181 ++++++++++++++++++++ docs/src/main/sphinx/udf/python/examples.md | 67 ++++++++ 5 files changed, 273 insertions(+), 10 deletions(-) create mode 100644 docs/src/main/sphinx/udf/python.md create mode 100644 docs/src/main/sphinx/udf/python/examples.md diff --git a/docs/src/main/sphinx/udf.md b/docs/src/main/sphinx/udf.md index ac715cc029a9..4e8c54a61420 100644 --- a/docs/src/main/sphinx/udf.md +++ b/docs/src/main/sphinx/udf.md @@ -13,4 +13,5 @@ More details are available in the following sections: udf/introduction udf/function udf/sql +udf/python ``` diff --git a/docs/src/main/sphinx/udf/function.md b/docs/src/main/sphinx/udf/function.md index a8d5b6425dc9..f5564ab574af 100644 --- a/docs/src/main/sphinx/udf/function.md +++ b/docs/src/main/sphinx/udf/function.md @@ -11,7 +11,8 @@ FUNCTION name ( [ parameter_name data_type [, ...] ] ) [ CALLED ON NULL INPUT ] [ SECURITY { DEFINER | INVOKER } ] [ COMMENT description] - statements + [ WITH ( property_name = expression [, ...] ) ] + { statements | AS definition } ``` ## Description @@ -31,7 +32,9 @@ The `type` value after the `RETURNS` keyword identifies the [data type](/language/types) of the UDF output. The optional `LANGUAGE` characteristic identifies the language used for the UDF -definition with `language`. Only `SQL` is supported. +definition with `language`. The `SQL` and `PYTHON` languages are supported by +default. Additional languages may be supported via a language engine plugin. +If not specified, the default language is `SQL`. The optional `DETERMINISTIC` or `NOT DETERMINISTIC` characteristic declares that the UDF is deterministic. This means that repeated UDF calls with identical @@ -58,10 +61,18 @@ The `COMMENT` characteristic can be used to provide information about the function to other users as `description`. The information is accessible with [](/sql/show-functions). -The body of the UDF can either be a simple single `RETURN` statement with an -expression, or compound list of `statements` in a `BEGIN` block. UDF must -contain a `RETURN` statement at the end of the top-level block, even if it's -unreachable. +The optional `WITH` clause can be used to specify properties for the function. +The available properties vary based on the function language. For +[](/udf/python), the `handler` property specifies the name of the Python +function to invoke. + +For SQL UDFs the body of the UDF can either be a simple single `RETURN` +statement with an expression, or compound list of `statements` in a `BEGIN` +block. UDF must contain a `RETURN` statement at the end of the top-level block, +even if it's unreachable. + +For UDFs in other languages, the `definition` is enclosed in a `$$`-quoted +string. ## Examples @@ -89,12 +100,14 @@ SELECT meaning_of_life(); ``` Further examples of varying complexity that cover usage of the `FUNCTION` -statement in combination with other statements are available in the [SQL -UDF examples documentation](/udf/sql/examples). +statement in combination with other statements are available in the [SQL UDF +documentation](/udf/sql/examples) and the [Python UDF +documentation](/udf/python). ## See also * [](/udf) * [](/udf/sql) +* [](/udf/python) * [](/sql/create-function) diff --git a/docs/src/main/sphinx/udf/introduction.md b/docs/src/main/sphinx/udf/introduction.md index 88189ccb3f52..da479dc491f1 100644 --- a/docs/src/main/sphinx/udf/introduction.md +++ b/docs/src/main/sphinx/udf/introduction.md @@ -4,8 +4,6 @@ A user-defined function (UDF) is a custom function authored by a user of Trino in a client application. UDFs are scalar functions that return a single output value, similar to [built-in functions](/functions). -UDFs are defined and written using the [SQL routine language](/udf/sql). - :::{note} Custom functions can alternatively be written in Java and deployed as a plugin. Details are available in the [developer guide](/develop/functions). @@ -14,6 +12,9 @@ plugin. Details are available in the [developer guide](/develop/functions). (udf-declaration)= ## UDF declaration +Declare the UDF with the SQL [](/udf/function) keyword and the supported +statements for [](/udf/sql) or [](/udf/python). + A UDF can be declared as an [inline UDF](udf-inline) to be used in the current query, or declared as a [catalog UDF](udf-catalog) to be used in any future query. diff --git a/docs/src/main/sphinx/udf/python.md b/docs/src/main/sphinx/udf/python.md new file mode 100644 index 000000000000..32e7dd229e6d --- /dev/null +++ b/docs/src/main/sphinx/udf/python.md @@ -0,0 +1,181 @@ +# Python user-defined functions + +A Python user-defined function is a [user-defined function](/udf) that uses the +[Python programming language and statements](python-udf-lang) for the definition +of the function. + +:::{warning} +Python user-defined functions are an experimental feature. +::: + +## Python UDF declaration + +Declare a Python UDF as [inline](udf-inline) or [catalog UDF](udf-catalog) with +the following steps: + +* Use the [](/udf/function) keyword to declare the UDF name and parameters. +* Add the `RETURNS` declaration to specify the data type of the result. +* Set the `LANGUAGE` to `PYTHON`. +* Declare the name of the Python function to call with the `handler` property in + the `WITH` block. +* Use `$$` to enclose the Python code after the `AS` keyword. +* Add the function from the handler property and ensure it returns the declared + data type. +* Expand your Python code section to implement the function using the available + [Python language](python-udf-lang). + +The following snippet shows pseudo-code: + +```text + FUNCTION python_udf_name(input_parameter data_type) + RETURNS result_data_type + LANGUAGE PYTHON + WITH (handler = 'python_function') + AS $$ + ... + def python_function(input): + return ... + ... + $$ +``` + +A minimal example declares the UDF `doubleup` that returns the input integer +value `x` multiplied by two. The example shows declaration as [](udf-inline) and +invocation with the value `21` to yield the result `42`. + +Set the language to `PYTHON` to override the default `SQL` for [](/udf/sql). +The Python code is enclosed with ``$$` and must use valid formatting. + +```text +WITH + FUNCTION doubleup(x integer) + RETURNS integer + LANGUAGE PYTHON + WITH (handler = 'twice') + AS $$ + def twice(a): + return a * 2 + $$ +SELECT doubleup(21); +-- 42 +``` + +The same UDF can also be declared as [](udf-catalog). + +Refer to the [](/udf/python/examples) for more complex use cases and examples. + +```{toctree} +:titlesonly: true +:hidden: + +/udf/python/examples +``` + +(python-udf-lang)= +## Python language details + +The Trino Python UDF integrations uses Python 3.13.0 in a sandboxed environment. +Python code runs within a WebAssembly (WASM) runtime within the Java virtual +machine running Trino. + +Python language rules including indents must be observed. + +Python UDFs therefore only have access to the Python language and core libraries +included in the sandboxed runtime. Access to external resources with network or +file system operations is not supported. Usage of other Python libraries as well +as command line tools or package managers is not supported. + +The following libraries are explicitly removed from the runtime and therefore +not available within a Python UDF: + +* `bdb` +* `concurrent` +* `curses` +* `ensurepip` +* `doctest` +* `idlelib` +* `multiprocessing` +* `pdb` +* `pydoc` +* `socketserver*` +* `sqlite3` +* `ssl` +* `subprocess*` +* `tkinter` +* `turtle*` +* `unittest` +* `venv` +* `webbrowser*` +* `wsgiref` +* `xmlrpc` + +## Type mapping + +The following table shows supported Trino types and their corresponding Python +types for input and output values of a Python UDF: + +:::{list-table} File system support properties +:widths: 50, 50 +:header-rows: 1 + +* - Trino type + - Python type +* - row + - tuple +* - array + - list +* - map + - dict +* - boolean + - bool +* - tinyint + - int +* - smallint + - int +* - integer + - int +* - bigint + - int +* - real + - float +* - double + - float +* - decimal + - decimal.Decimal +* - varchar + - str +* - varbinary + - bytes +* - date + - datetime.date +* - time + - datetime.time +* - time with time zone + - datetime.time with datetime.tzinfo +* - timestamp + - datetime.datetime +* - timestamp with time zone + - datetime.datetime with datetime.tzinfo 1 +* - interval year to month + - int as the number of months +* - interval day to second + - datetime.timedelta +* - json + - str +* - uuid + - uuid.UUID +* - ipaddress + - ipaddress.IPv4Address or ipaddress.IPv6Address + +::: + +### Date and time + +Python datetime objects only support microsecond precision. Trino argument +values with greater precision arerounded when converted to Python values, and +Python return values are rounded if the Trino return type has less than +microsecond precision. + +Only fixed offset time zones are supported. Timestamps with political time zones +have the zone converted to the zone's offset for the timestamp's instant. + diff --git a/docs/src/main/sphinx/udf/python/examples.md b/docs/src/main/sphinx/udf/python/examples.md new file mode 100644 index 000000000000..f0b8d7584b1e --- /dev/null +++ b/docs/src/main/sphinx/udf/python/examples.md @@ -0,0 +1,67 @@ +# Example Python UDFs + +After learning about [](/udf/python), the following sections show examples +of valid Python UDFs. + +## XOR + +The following example implements a `xor` function for a logical Exclusive OR +operation on two boolean input parameters and tests it with two invocations: + +```text +WITH FUNCTION xor(a boolean, b boolean) +RETURNS boolean +LANGUAGE PYTHON +WITH (handler = 'bool_xor') +AS $$ +import operator +def bool_xor(a, b): + return operator.xor(a, b) +$$ +SELECT xor(true, false), xor(false, true); +``` + +Result of the query: + +``` + true | true +``` + +## reverse_words + +The following example uses a more elaborate Python script to reverse the +characters in each word of the input string `s` of type `varchar` and tests the +function. + +```text +WITH FUNCTION reverse_words(s varchar) +RETURNS varchar +LANGUAGE PYTHON +WITH (handler = 'reverse_words') +AS $$ +import re + +def reverse(s): + str = "" + for i in s: + str = i + str + return str + +pattern = re.compile(r"\w+[.,'!?\"]\w*") + +def process_word(word): + # Reverse only words without non-letter signs + return word if pattern.match(word) else reverse(word) + +def reverse_words(payload): + text_words = payload.split(' ') + return ' '.join([process_word(w) for w in text_words]) +$$ +SELECT reverse_words('Civic, level, dna racecar era semordnilap'); +``` + +Result of the query: + +``` +Civic, level, and racecar are palindromes +``` \ No newline at end of file From f03c4aaf5bbc283c2efad1d4e27d9c846d490e8b Mon Sep 17 00:00:00 2001 From: chenjian2664 Date: Tue, 17 Dec 2024 15:10:10 +0800 Subject: [PATCH 007/158] Remove unnecessary annotation in Kudu connector test --- .../test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java index e1c7579e9c01..73de900f28ba 100644 --- a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java +++ b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java @@ -1047,7 +1047,6 @@ protected Optional filterDataMappingSmokeTestData(DataMapp return Optional.of(dataMappingTestSetup); } - @Test @Override protected TestTable createTableWithDefaultColumns() { From b9879e40b1258100f7f915d363ae6789e7f322a8 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Mon, 16 Dec 2024 20:51:14 -0800 Subject: [PATCH 008/158] Update to oryd/hydra:v1.11.10 for OAuth testing Fix build issues on newer OS/hardware. Use latest 1.x release since 2.x causes container start issues without further changes. --- .../server/security/oauth2/TestingHydraIdentityProvider.java | 2 +- .../product/launcher/env/common/HydraIdentityProvider.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java b/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java index 9d910dae3ce3..c16fad53035b 100644 --- a/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java +++ b/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java @@ -71,7 +71,7 @@ public class TestingHydraIdentityProvider implements AutoCloseable { - private static final String HYDRA_IMAGE = "oryd/hydra:v1.10.6"; + private static final String HYDRA_IMAGE = "oryd/hydra:v1.11.10"; private static final String ISSUER = "https://localhost:4444/"; private static final String DSN = "postgres://hydra:mysecretpassword@database:5432/hydra?sslmode=disable"; diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java index bfb9237365ca..3af695f4b646 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java @@ -36,7 +36,7 @@ public class HydraIdentityProvider private static final int TTL_ACCESS_TOKEN_IN_SECONDS = 5; private static final int TTL_REFRESH_TOKEN_IN_SECONDS = 15; - private static final String HYDRA_IMAGE = "oryd/hydra:v1.10.6"; + private static final String HYDRA_IMAGE = "oryd/hydra:v1.11.10"; private static final String DSN = "postgres://hydra:mysecretpassword@hydra-db:5432/hydra?sslmode=disable"; private final PortBinder binder; private final DockerFiles.ResourceProvider configDir; From 547cc430e7bcc7d633560f2e8eff69ee647bebdf Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Tue, 17 Dec 2024 17:12:17 +0100 Subject: [PATCH 009/158] Update airbase to 206 and airlift to 293 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 356eda7ae09c..bcc4fe98b417 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.airlift airbase - 205 + 206 io.trino @@ -184,7 +184,7 @@ ${air.test.jvm.additional-arguments.default} - 292 + 293 2.9.6 4.13.2 1.12.0 @@ -2038,7 +2038,7 @@ org.apache.logging.log4j log4j-api - 2.24.2 + 2.24.3 From df3bd95ff1beec00a0fe8ea6865c75cc95fd1187 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Tue, 17 Dec 2024 17:15:52 +0100 Subject: [PATCH 010/158] Update AWS SDK v2 to 2.29.35 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bcc4fe98b417..c1eea4360f18 100644 --- a/pom.xml +++ b/pom.xml @@ -311,7 +311,7 @@ software.amazon.awssdk bom - 2.29.34 + 2.29.35 pom import From 8ac15ee7e293d926b030862c825538be77150424 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Tue, 17 Dec 2024 17:16:50 +0100 Subject: [PATCH 011/158] Update netty to 4.1.116.Final --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c1eea4360f18..540ee3a66e9f 100644 --- a/pom.xml +++ b/pom.xml @@ -271,7 +271,7 @@ io.netty netty-bom - 4.1.115.Final + 4.1.116.Final pom import From a87e255925341c1d49bf893fd5d0071b9f474ebf Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Tue, 17 Dec 2024 17:19:01 +0100 Subject: [PATCH 012/158] Update gcs connector to hadoop3-2.2.26 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 540ee3a66e9f..e660ec2d052f 100644 --- a/pom.xml +++ b/pom.xml @@ -522,7 +522,7 @@ com.google.cloud.bigdataoss gcs-connector - hadoop3-2.2.25 + hadoop3-2.2.26 shaded From 6cad052ba85d876b8d8b00a7edf6f3b7325f5733 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Mon, 16 Dec 2024 12:57:53 -0800 Subject: [PATCH 013/158] Add example for inline and catalog Python UDF --- docs/src/main/sphinx/udf/python/examples.md | 90 +++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/docs/src/main/sphinx/udf/python/examples.md b/docs/src/main/sphinx/udf/python/examples.md index f0b8d7584b1e..19b28729b86c 100644 --- a/docs/src/main/sphinx/udf/python/examples.md +++ b/docs/src/main/sphinx/udf/python/examples.md @@ -3,6 +3,96 @@ After learning about [](/udf/python), the following sections show examples of valid Python UDFs. +The UDFs are suitable as [](udf-inline) or [](udf-catalog), +after adjusting the name and the example invocations. + +## Inline and catalog Python UDFs + +The following section shows the differences in usage with inline and catalog +UDFs with a simple Python UDF example. The same pattern applies to all other +following sections. + +A very simple Python UDF that returns the static int value `42` without +requiring any input: + +```text +FUNCTION answer() +LANGUAGE PYTHON +RETURNS int +WITH (handler='theanswer') +AS $$ +def theanswer(): + return 42 +$$ +``` + +A full example of this UDF as inline UDF and usage in a string concatenation +with a cast: + +```text +WITH + FUNCTION answer() + RETURNS int + LANGUAGE PYTHON + WITH (handler='theanswer') + AS $$ + def theanswer(): + return 42 + $$ +SELECT 'The answer is ' || CAST(answer() as varchar); +-- The answer is 42 +``` + +Provided the catalog `example` supports UDF storage in the `default` schema, you +can use the following: + +```text +CREATE FUNCTION example.default.answer() + RETURNS int + LANGUAGE PYTHON + WITH (handler='theanswer') + AS $$ + def theanswer(): + return 42 + $$; +``` + +With the UDF stored in the catalog, you can run the UDF multiple times without +repeated definition: + +```sql +SELECT example.default.answer() + 1; -- 43 +SELECT 'The answer is ' || CAST(example.default.answer() as varchar); -- The answer is 42 +``` + +Alternatively, you can configure the SQL PATH in the [](config-properties) to a +catalog and schema that support UDF storage: + +```properties +sql.default-function-catalog=example +sql.default-function-schema=default +sql.path=example.default +``` + +Now you can manage UDFs without the full path: + +```text +CREATE FUNCTION answer() + RETURNS int + LANGUAGE PYTHON + WITH (handler='theanswer') + AS $$ + def theanswer(): + return 42 + $$; +``` + +UDF invocation works without the full path: + +```sql +SELECT answer() + 5; -- 47 +``` + ## XOR The following example implements a `xor` function for a logical Exclusive OR From d9615016912399dbf4019d06c6c124ad83be61df Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Mon, 16 Dec 2024 12:58:24 -0800 Subject: [PATCH 014/158] Improve inline and catalog SQL UDF docs --- docs/src/main/sphinx/udf/sql/examples.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/src/main/sphinx/udf/sql/examples.md b/docs/src/main/sphinx/udf/sql/examples.md index 95a87da63f77..2ffd63a45fab 100644 --- a/docs/src/main/sphinx/udf/sql/examples.md +++ b/docs/src/main/sphinx/udf/sql/examples.md @@ -15,6 +15,12 @@ statement documentation for further details: * [](/udf/sql/loop), [](/udf/sql/repeat), and [](/udf/sql/while) for looping constructs * [](/udf/sql/iterate) and [](/udf/sql/leave) for flow control +## Inline and catalog UDFs + +The following section shows the differences in usage with inline and catalog +UDFs with a simple SQL UDF example. The same pattern applies to all other +following sections. + A very simple SQL UDF that returns a static value without requiring any input: ```sql @@ -23,8 +29,6 @@ RETURNS BIGINT RETURN 42 ``` -## Inline and catalog UDFs - A full example of this UDF as inline UDF and usage in a string concatenation with a cast: @@ -41,7 +45,6 @@ Provided the catalog `example` supports UDF storage in the `default` schema, you can use the following: ```sql -USE example.default; CREATE FUNCTION example.default.answer() RETURNS BIGINT RETURN 42; @@ -52,15 +55,16 @@ repeated definition: ```sql SELECT example.default.answer() + 1; -- 43 -SELECT 'The answer is' || CAST(example.default.answer() as varchar); -- The answer is 42 +SELECT 'The answer is ' || CAST(example.default.answer() as varchar); -- The answer is 42 ``` -Alternatively, you can configure the SQL environment in the -[](config-properties) to a catalog and schema that support UDF storage: +Alternatively, you can configure the SQL PATH in the [](config-properties) to a +catalog and schema that support UDF storage: ```properties sql.default-function-catalog=example sql.default-function-schema=default +sql.path=example.default ``` Now you can manage UDFs without the full path: From 0c9c6c377f2c05b9e33c1f99a35d526740da835f Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Mon, 9 Dec 2024 11:56:21 -0800 Subject: [PATCH 015/158] Add Trino 468 release notes --- docs/src/main/sphinx/release.md | 1 + docs/src/main/sphinx/release/release-468.md | 51 +++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 docs/src/main/sphinx/release/release-468.md diff --git a/docs/src/main/sphinx/release.md b/docs/src/main/sphinx/release.md index 2d083b797a47..c2bac1c08263 100644 --- a/docs/src/main/sphinx/release.md +++ b/docs/src/main/sphinx/release.md @@ -6,6 +6,7 @@ ```{toctree} :maxdepth: 1 +release/release-468 release/release-467 release/release-466 release/release-465 diff --git a/docs/src/main/sphinx/release/release-468.md b/docs/src/main/sphinx/release/release-468.md new file mode 100644 index 000000000000..601d41728fba --- /dev/null +++ b/docs/src/main/sphinx/release/release-468.md @@ -0,0 +1,51 @@ +# Release 468 (17 Dec 2024) + +## General + +* Add experimental support for [](/udf/python). ({issue}`24378`) +* Add cluster overview to the [](/admin/preview-web-interface). ({issue}`23600`) +* Add new node states `DRAINING` and `DRAINED` to make it possible to reactivate + a draining worker node. ({issue}`24444 `) + +## BigQuery connector + +* Improve performance when reading external + [BigLake](https://cloud.google.com/bigquery/docs/biglake-intro) tables. ({issue}`21016`) + +## Delta Lake connector + +* {{breaking}} Reduce coordinator memory usage for the Delta table metadata + cache and enable configuration `delta.metadata.cache-max-retained-size` to + control memory usage. Remove the configuration property + `delta.metadata.cache-size` and increase the default for + `delta.metadata.cache-ttl` to `30m`. ({issue}`24432`) + +## Hive connector + +* Enable mismatched bucket execution optimization by default. This can be + disabled with `hive.optimize-mismatched-bucket-count` configuration property + and the `optimize_mismatched_bucket_count` session property. ({issue}`23432`) +* Improve performance by deactivating bucket execution when not useful in query + processing. ({issue}`23432`) + +## Iceberg connector + +* Improve performance when running a join or aggregation on a bucketed table + with bucketed execution. This can be deactivated with the + `iceberg.bucket-execution` configuration property and the + `bucket_execution_enabled` session property. ({issue}`23432`) +* Deprecate the `iceberg.materialized-views.storage-schema` configuration + property. ({issue}`24398`) +* {{breaking}} Rename the `partitions` column in the `$manifests` metadata table + to `partition_summaries`. ({issue}`24103`) +* Avoid excessive resource usage on coordinator when reading Iceberg system + tables. ({issue}`24396`) + +## PostgreSQL connector + +* Add support for non-transactional [MERGE statements](/sql/merge). ({issue}`23034`) + +## SPI + +* Add partitioning push down, which a connector can use to activate optional + partitioning or choose between multiple partitioning strategies. ({issue}`23432`) From cb7a9cc1cf4e9ea26ccaa966e8a421a76c273954 Mon Sep 17 00:00:00 2001 From: Martin Traverso Date: Tue, 17 Dec 2024 21:29:19 +0000 Subject: [PATCH 016/158] [maven-release-plugin] prepare release 468 --- client/trino-cli/pom.xml | 2 +- client/trino-client/pom.xml | 2 +- client/trino-jdbc/pom.xml | 2 +- core/trino-grammar/pom.xml | 2 +- core/trino-main/pom.xml | 2 +- core/trino-parser/pom.xml | 2 +- core/trino-server-main/pom.xml | 2 +- core/trino-server-rpm/pom.xml | 2 +- core/trino-server/pom.xml | 2 +- core/trino-spi/pom.xml | 2 +- core/trino-web-ui/pom.xml | 2 +- docs/pom.xml | 2 +- lib/trino-array/pom.xml | 2 +- lib/trino-cache/pom.xml | 2 +- lib/trino-filesystem-alluxio/pom.xml | 2 +- lib/trino-filesystem-azure/pom.xml | 2 +- lib/trino-filesystem-cache-alluxio/pom.xml | 2 +- lib/trino-filesystem-gcs/pom.xml | 2 +- lib/trino-filesystem-manager/pom.xml | 2 +- lib/trino-filesystem-s3/pom.xml | 2 +- lib/trino-filesystem/pom.xml | 2 +- lib/trino-geospatial-toolkit/pom.xml | 2 +- lib/trino-hdfs/pom.xml | 2 +- lib/trino-hive-formats/pom.xml | 2 +- lib/trino-matching/pom.xml | 2 +- lib/trino-memory-context/pom.xml | 2 +- lib/trino-metastore/pom.xml | 2 +- lib/trino-orc/pom.xml | 2 +- lib/trino-parquet/pom.xml | 2 +- lib/trino-plugin-toolkit/pom.xml | 2 +- lib/trino-record-decoder/pom.xml | 2 +- plugin/trino-base-jdbc/pom.xml | 2 +- plugin/trino-bigquery/pom.xml | 2 +- plugin/trino-blackhole/pom.xml | 2 +- plugin/trino-cassandra/pom.xml | 2 +- plugin/trino-clickhouse/pom.xml | 2 +- plugin/trino-delta-lake/pom.xml | 2 +- plugin/trino-druid/pom.xml | 2 +- plugin/trino-elasticsearch/pom.xml | 2 +- plugin/trino-example-http/pom.xml | 2 +- plugin/trino-example-jdbc/pom.xml | 2 +- plugin/trino-exasol/pom.xml | 2 +- plugin/trino-exchange-filesystem/pom.xml | 2 +- plugin/trino-exchange-hdfs/pom.xml | 2 +- plugin/trino-faker/pom.xml | 2 +- plugin/trino-functions-python/pom.xml | 2 +- plugin/trino-geospatial/pom.xml | 2 +- plugin/trino-google-sheets/pom.xml | 2 +- plugin/trino-hive/pom.xml | 2 +- plugin/trino-http-event-listener/pom.xml | 2 +- plugin/trino-http-server-event-listener/pom.xml | 2 +- plugin/trino-hudi/pom.xml | 2 +- plugin/trino-iceberg/pom.xml | 2 +- plugin/trino-ignite/pom.xml | 2 +- plugin/trino-jmx/pom.xml | 2 +- plugin/trino-kafka-event-listener/pom.xml | 2 +- plugin/trino-kafka/pom.xml | 2 +- plugin/trino-kinesis/pom.xml | 2 +- plugin/trino-kudu/pom.xml | 2 +- plugin/trino-mariadb/pom.xml | 2 +- plugin/trino-memory/pom.xml | 2 +- plugin/trino-ml/pom.xml | 2 +- plugin/trino-mongodb/pom.xml | 2 +- plugin/trino-mysql-event-listener/pom.xml | 2 +- plugin/trino-mysql/pom.xml | 2 +- plugin/trino-opa/pom.xml | 2 +- plugin/trino-openlineage/pom.xml | 2 +- plugin/trino-opensearch/pom.xml | 2 +- plugin/trino-oracle/pom.xml | 2 +- plugin/trino-password-authenticators/pom.xml | 2 +- plugin/trino-phoenix5/pom.xml | 2 +- plugin/trino-pinot/pom.xml | 2 +- plugin/trino-postgresql/pom.xml | 2 +- plugin/trino-prometheus/pom.xml | 2 +- plugin/trino-ranger/pom.xml | 2 +- plugin/trino-redis/pom.xml | 2 +- plugin/trino-redshift/pom.xml | 2 +- plugin/trino-resource-group-managers/pom.xml | 2 +- plugin/trino-session-property-managers/pom.xml | 2 +- plugin/trino-singlestore/pom.xml | 2 +- plugin/trino-snowflake/pom.xml | 2 +- plugin/trino-spooling-filesystem/pom.xml | 2 +- plugin/trino-sqlserver/pom.xml | 2 +- plugin/trino-teradata-functions/pom.xml | 2 +- plugin/trino-thrift-api/pom.xml | 2 +- plugin/trino-thrift-testing-server/pom.xml | 2 +- plugin/trino-thrift/pom.xml | 2 +- plugin/trino-tpcds/pom.xml | 2 +- plugin/trino-tpch/pom.xml | 2 +- plugin/trino-vertica/pom.xml | 2 +- pom.xml | 6 +++--- service/trino-proxy/pom.xml | 2 +- service/trino-verifier/pom.xml | 2 +- testing/trino-benchmark-queries/pom.xml | 2 +- testing/trino-benchto-benchmarks/pom.xml | 2 +- testing/trino-faulttolerant-tests/pom.xml | 2 +- testing/trino-plugin-reader/pom.xml | 2 +- testing/trino-product-tests-groups/pom.xml | 2 +- testing/trino-product-tests-launcher/pom.xml | 2 +- testing/trino-product-tests/pom.xml | 2 +- testing/trino-server-dev/pom.xml | 2 +- testing/trino-test-jdbc-compatibility-old-driver/pom.xml | 4 ++-- testing/trino-test-jdbc-compatibility-old-server/pom.xml | 2 +- testing/trino-testing-containers/pom.xml | 2 +- testing/trino-testing-kafka/pom.xml | 2 +- testing/trino-testing-resources/pom.xml | 2 +- testing/trino-testing-services/pom.xml | 2 +- testing/trino-testing/pom.xml | 2 +- testing/trino-tests/pom.xml | 2 +- 109 files changed, 112 insertions(+), 112 deletions(-) diff --git a/client/trino-cli/pom.xml b/client/trino-cli/pom.xml index 94878e86e7d2..efe41a26cac2 100644 --- a/client/trino-cli/pom.xml +++ b/client/trino-cli/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/client/trino-client/pom.xml b/client/trino-client/pom.xml index 5e663b0af009..ed6a74bf7d44 100644 --- a/client/trino-client/pom.xml +++ b/client/trino-client/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/client/trino-jdbc/pom.xml b/client/trino-jdbc/pom.xml index 19b74cc7804d..719c5152cb30 100644 --- a/client/trino-jdbc/pom.xml +++ b/client/trino-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-grammar/pom.xml b/core/trino-grammar/pom.xml index 5e4790b63bb0..2f3c770f1607 100644 --- a/core/trino-grammar/pom.xml +++ b/core/trino-grammar/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index 116f9749924f..6d599e90c938 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-parser/pom.xml b/core/trino-parser/pom.xml index fbf13b76bc9c..7330d67c0d17 100644 --- a/core/trino-parser/pom.xml +++ b/core/trino-parser/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-server-main/pom.xml b/core/trino-server-main/pom.xml index b7f011bca4e4..8b3359c51343 100644 --- a/core/trino-server-main/pom.xml +++ b/core/trino-server-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-server-rpm/pom.xml b/core/trino-server-rpm/pom.xml index 82c2b0828f92..7210b7937569 100644 --- a/core/trino-server-rpm/pom.xml +++ b/core/trino-server-rpm/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-server/pom.xml b/core/trino-server/pom.xml index 1fb9b5ba82c5..0513120d2c98 100644 --- a/core/trino-server/pom.xml +++ b/core/trino-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index 57fb4b73f050..bc23756ad41a 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/core/trino-web-ui/pom.xml b/core/trino-web-ui/pom.xml index 9f21b8edf7d3..0e032708c15e 100644 --- a/core/trino-web-ui/pom.xml +++ b/core/trino-web-ui/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/docs/pom.xml b/docs/pom.xml index 55aa7e62b4b9..94f7a392c4dc 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 trino-docs diff --git a/lib/trino-array/pom.xml b/lib/trino-array/pom.xml index 9cd91d5fa696..c7fb5d57bd67 100644 --- a/lib/trino-array/pom.xml +++ b/lib/trino-array/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-cache/pom.xml b/lib/trino-cache/pom.xml index 0ae4a50d484e..dc6f59bfe7fb 100644 --- a/lib/trino-cache/pom.xml +++ b/lib/trino-cache/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-filesystem-alluxio/pom.xml b/lib/trino-filesystem-alluxio/pom.xml index db27a0db7c9e..c755f0624355 100644 --- a/lib/trino-filesystem-alluxio/pom.xml +++ b/lib/trino-filesystem-alluxio/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-filesystem-azure/pom.xml b/lib/trino-filesystem-azure/pom.xml index 8fad433ce06f..c6a88366ff48 100644 --- a/lib/trino-filesystem-azure/pom.xml +++ b/lib/trino-filesystem-azure/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-filesystem-cache-alluxio/pom.xml b/lib/trino-filesystem-cache-alluxio/pom.xml index 93a491906c03..269eaaa9038c 100644 --- a/lib/trino-filesystem-cache-alluxio/pom.xml +++ b/lib/trino-filesystem-cache-alluxio/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-filesystem-gcs/pom.xml b/lib/trino-filesystem-gcs/pom.xml index e06cf563005e..405667b46f15 100644 --- a/lib/trino-filesystem-gcs/pom.xml +++ b/lib/trino-filesystem-gcs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-filesystem-manager/pom.xml b/lib/trino-filesystem-manager/pom.xml index bc27cd26088b..6bfa8fa39726 100644 --- a/lib/trino-filesystem-manager/pom.xml +++ b/lib/trino-filesystem-manager/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-filesystem-s3/pom.xml b/lib/trino-filesystem-s3/pom.xml index 882a0005422c..f2e0e54e74b1 100644 --- a/lib/trino-filesystem-s3/pom.xml +++ b/lib/trino-filesystem-s3/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-filesystem/pom.xml b/lib/trino-filesystem/pom.xml index 91ec951a6488..e30d68915cce 100644 --- a/lib/trino-filesystem/pom.xml +++ b/lib/trino-filesystem/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-geospatial-toolkit/pom.xml b/lib/trino-geospatial-toolkit/pom.xml index d3a4dacea661..6169df15b844 100644 --- a/lib/trino-geospatial-toolkit/pom.xml +++ b/lib/trino-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-hdfs/pom.xml b/lib/trino-hdfs/pom.xml index da3817d881ec..c1bc583554f1 100644 --- a/lib/trino-hdfs/pom.xml +++ b/lib/trino-hdfs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-hive-formats/pom.xml b/lib/trino-hive-formats/pom.xml index 5b202a849289..f5ebec6d06c6 100644 --- a/lib/trino-hive-formats/pom.xml +++ b/lib/trino-hive-formats/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-matching/pom.xml b/lib/trino-matching/pom.xml index 92175857a135..644704a40421 100644 --- a/lib/trino-matching/pom.xml +++ b/lib/trino-matching/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-memory-context/pom.xml b/lib/trino-memory-context/pom.xml index db57396b9f95..047ef17993f5 100644 --- a/lib/trino-memory-context/pom.xml +++ b/lib/trino-memory-context/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-metastore/pom.xml b/lib/trino-metastore/pom.xml index 65411b9cbfef..ebc63b5cd781 100644 --- a/lib/trino-metastore/pom.xml +++ b/lib/trino-metastore/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-orc/pom.xml b/lib/trino-orc/pom.xml index 37b7e911e8ad..84a022273ebc 100644 --- a/lib/trino-orc/pom.xml +++ b/lib/trino-orc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-parquet/pom.xml b/lib/trino-parquet/pom.xml index 3c5924e4d3f7..1a04dc86c91e 100644 --- a/lib/trino-parquet/pom.xml +++ b/lib/trino-parquet/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-plugin-toolkit/pom.xml b/lib/trino-plugin-toolkit/pom.xml index 210030ac6160..ba1faf4e90d6 100644 --- a/lib/trino-plugin-toolkit/pom.xml +++ b/lib/trino-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/lib/trino-record-decoder/pom.xml b/lib/trino-record-decoder/pom.xml index da3957d2241c..1047479f03ee 100644 --- a/lib/trino-record-decoder/pom.xml +++ b/lib/trino-record-decoder/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-base-jdbc/pom.xml b/plugin/trino-base-jdbc/pom.xml index 8251f803705a..70248555cd87 100644 --- a/plugin/trino-base-jdbc/pom.xml +++ b/plugin/trino-base-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-bigquery/pom.xml b/plugin/trino-bigquery/pom.xml index 3273806736df..345d937700e4 100644 --- a/plugin/trino-bigquery/pom.xml +++ b/plugin/trino-bigquery/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-blackhole/pom.xml b/plugin/trino-blackhole/pom.xml index 09ac4505dba2..0c3498a6c5ad 100644 --- a/plugin/trino-blackhole/pom.xml +++ b/plugin/trino-blackhole/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-cassandra/pom.xml b/plugin/trino-cassandra/pom.xml index 3e8fe2860f73..f7646cdf658b 100644 --- a/plugin/trino-cassandra/pom.xml +++ b/plugin/trino-cassandra/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml index 675473440ecb..c99bc6914d09 100644 --- a/plugin/trino-clickhouse/pom.xml +++ b/plugin/trino-clickhouse/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-delta-lake/pom.xml b/plugin/trino-delta-lake/pom.xml index 17f738dc01e4..66c2886f0cc7 100644 --- a/plugin/trino-delta-lake/pom.xml +++ b/plugin/trino-delta-lake/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-druid/pom.xml b/plugin/trino-druid/pom.xml index 90dfae22c2a5..d538636510fc 100644 --- a/plugin/trino-druid/pom.xml +++ b/plugin/trino-druid/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-elasticsearch/pom.xml b/plugin/trino-elasticsearch/pom.xml index e6ff418e58ba..00e5bca9beed 100644 --- a/plugin/trino-elasticsearch/pom.xml +++ b/plugin/trino-elasticsearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-example-http/pom.xml b/plugin/trino-example-http/pom.xml index ac0f88877bb0..3c4917c70f2e 100644 --- a/plugin/trino-example-http/pom.xml +++ b/plugin/trino-example-http/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-example-jdbc/pom.xml b/plugin/trino-example-jdbc/pom.xml index c8f65d9df5e4..991b14405d91 100644 --- a/plugin/trino-example-jdbc/pom.xml +++ b/plugin/trino-example-jdbc/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-exasol/pom.xml b/plugin/trino-exasol/pom.xml index ddc7816163f3..5e1c50a953c2 100644 --- a/plugin/trino-exasol/pom.xml +++ b/plugin/trino-exasol/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-exchange-filesystem/pom.xml b/plugin/trino-exchange-filesystem/pom.xml index 73b73eb6fd59..c1fcee1becf1 100644 --- a/plugin/trino-exchange-filesystem/pom.xml +++ b/plugin/trino-exchange-filesystem/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-exchange-hdfs/pom.xml b/plugin/trino-exchange-hdfs/pom.xml index 12f0cf3513bc..f5781edf6dbe 100644 --- a/plugin/trino-exchange-hdfs/pom.xml +++ b/plugin/trino-exchange-hdfs/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-faker/pom.xml b/plugin/trino-faker/pom.xml index c0d228bead17..9af9084c4474 100644 --- a/plugin/trino-faker/pom.xml +++ b/plugin/trino-faker/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-functions-python/pom.xml b/plugin/trino-functions-python/pom.xml index 83dd5a661c94..f67018af8f59 100644 --- a/plugin/trino-functions-python/pom.xml +++ b/plugin/trino-functions-python/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-geospatial/pom.xml b/plugin/trino-geospatial/pom.xml index 379b37d6b7d3..9f79f6995595 100644 --- a/plugin/trino-geospatial/pom.xml +++ b/plugin/trino-geospatial/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-google-sheets/pom.xml b/plugin/trino-google-sheets/pom.xml index 8f534bd58bfd..d29ad025a921 100644 --- a/plugin/trino-google-sheets/pom.xml +++ b/plugin/trino-google-sheets/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml index a163377c0638..beea0a99d5dd 100644 --- a/plugin/trino-hive/pom.xml +++ b/plugin/trino-hive/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-http-event-listener/pom.xml b/plugin/trino-http-event-listener/pom.xml index c21ce86410a6..d92f1fa5d7df 100644 --- a/plugin/trino-http-event-listener/pom.xml +++ b/plugin/trino-http-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-http-server-event-listener/pom.xml b/plugin/trino-http-server-event-listener/pom.xml index 29dc7f087d69..f6c24bd646ff 100644 --- a/plugin/trino-http-server-event-listener/pom.xml +++ b/plugin/trino-http-server-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-hudi/pom.xml b/plugin/trino-hudi/pom.xml index ed968a22aa45..72667b42d300 100644 --- a/plugin/trino-hudi/pom.xml +++ b/plugin/trino-hudi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index 8849c30793ab..0e77542dd47e 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-ignite/pom.xml b/plugin/trino-ignite/pom.xml index 54b205c4fa3f..2d07bb987e9b 100644 --- a/plugin/trino-ignite/pom.xml +++ b/plugin/trino-ignite/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-jmx/pom.xml b/plugin/trino-jmx/pom.xml index f0571da2c96e..1750cfe00a34 100644 --- a/plugin/trino-jmx/pom.xml +++ b/plugin/trino-jmx/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-kafka-event-listener/pom.xml b/plugin/trino-kafka-event-listener/pom.xml index 78f3612fe0b1..e6776b5ad670 100644 --- a/plugin/trino-kafka-event-listener/pom.xml +++ b/plugin/trino-kafka-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-kafka/pom.xml b/plugin/trino-kafka/pom.xml index 497e4b224f01..d5242443a170 100644 --- a/plugin/trino-kafka/pom.xml +++ b/plugin/trino-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-kinesis/pom.xml b/plugin/trino-kinesis/pom.xml index 9d171338ff8c..4b67922dfa27 100644 --- a/plugin/trino-kinesis/pom.xml +++ b/plugin/trino-kinesis/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-kudu/pom.xml b/plugin/trino-kudu/pom.xml index a88965bd4975..a2f8c3c97720 100644 --- a/plugin/trino-kudu/pom.xml +++ b/plugin/trino-kudu/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-mariadb/pom.xml b/plugin/trino-mariadb/pom.xml index f17bf53033ba..05ef446ae28a 100644 --- a/plugin/trino-mariadb/pom.xml +++ b/plugin/trino-mariadb/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-memory/pom.xml b/plugin/trino-memory/pom.xml index 4d642b657707..91d9a987ab25 100644 --- a/plugin/trino-memory/pom.xml +++ b/plugin/trino-memory/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-ml/pom.xml b/plugin/trino-ml/pom.xml index 6023d4b10550..8d4f5f91f29b 100644 --- a/plugin/trino-ml/pom.xml +++ b/plugin/trino-ml/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-mongodb/pom.xml b/plugin/trino-mongodb/pom.xml index c2d57fb6ceaa..d2edea082f26 100644 --- a/plugin/trino-mongodb/pom.xml +++ b/plugin/trino-mongodb/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-mysql-event-listener/pom.xml b/plugin/trino-mysql-event-listener/pom.xml index fab9d3498f2a..e83deb6ca0b6 100644 --- a/plugin/trino-mysql-event-listener/pom.xml +++ b/plugin/trino-mysql-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-mysql/pom.xml b/plugin/trino-mysql/pom.xml index c54961744c11..63af742d3b44 100644 --- a/plugin/trino-mysql/pom.xml +++ b/plugin/trino-mysql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-opa/pom.xml b/plugin/trino-opa/pom.xml index 331e73f8369d..020a1d3f0520 100644 --- a/plugin/trino-opa/pom.xml +++ b/plugin/trino-opa/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-openlineage/pom.xml b/plugin/trino-openlineage/pom.xml index 4faaed1daccb..ab9e7c9c2456 100644 --- a/plugin/trino-openlineage/pom.xml +++ b/plugin/trino-openlineage/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-opensearch/pom.xml b/plugin/trino-opensearch/pom.xml index f9d066bec3f2..a48fffe4c947 100644 --- a/plugin/trino-opensearch/pom.xml +++ b/plugin/trino-opensearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-oracle/pom.xml b/plugin/trino-oracle/pom.xml index 0d9e6086753f..d8982e8eaa57 100644 --- a/plugin/trino-oracle/pom.xml +++ b/plugin/trino-oracle/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-password-authenticators/pom.xml b/plugin/trino-password-authenticators/pom.xml index 3d386230d167..78f1f63e06c2 100644 --- a/plugin/trino-password-authenticators/pom.xml +++ b/plugin/trino-password-authenticators/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-phoenix5/pom.xml b/plugin/trino-phoenix5/pom.xml index 3b9fb9f5617d..9270526f2714 100644 --- a/plugin/trino-phoenix5/pom.xml +++ b/plugin/trino-phoenix5/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-pinot/pom.xml b/plugin/trino-pinot/pom.xml index e745a06616aa..2625eefc964d 100755 --- a/plugin/trino-pinot/pom.xml +++ b/plugin/trino-pinot/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-postgresql/pom.xml b/plugin/trino-postgresql/pom.xml index 465cf9c92ba1..a28e5f1fbec9 100644 --- a/plugin/trino-postgresql/pom.xml +++ b/plugin/trino-postgresql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-prometheus/pom.xml b/plugin/trino-prometheus/pom.xml index f8e11f295707..f4512e62cda6 100644 --- a/plugin/trino-prometheus/pom.xml +++ b/plugin/trino-prometheus/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-ranger/pom.xml b/plugin/trino-ranger/pom.xml index 2f60db528737..9c84aec7bb68 100644 --- a/plugin/trino-ranger/pom.xml +++ b/plugin/trino-ranger/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-redis/pom.xml b/plugin/trino-redis/pom.xml index 7722bb80f16c..fc428242c466 100644 --- a/plugin/trino-redis/pom.xml +++ b/plugin/trino-redis/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-redshift/pom.xml b/plugin/trino-redshift/pom.xml index a18cfb70890d..48b00ae943f7 100644 --- a/plugin/trino-redshift/pom.xml +++ b/plugin/trino-redshift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-resource-group-managers/pom.xml b/plugin/trino-resource-group-managers/pom.xml index 8da2752169e6..2052e9d26e59 100644 --- a/plugin/trino-resource-group-managers/pom.xml +++ b/plugin/trino-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-session-property-managers/pom.xml b/plugin/trino-session-property-managers/pom.xml index af958eaa9ead..21d005d43df2 100644 --- a/plugin/trino-session-property-managers/pom.xml +++ b/plugin/trino-session-property-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-singlestore/pom.xml b/plugin/trino-singlestore/pom.xml index bf777aa166e4..9609d847a6ce 100644 --- a/plugin/trino-singlestore/pom.xml +++ b/plugin/trino-singlestore/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-snowflake/pom.xml b/plugin/trino-snowflake/pom.xml index ff7f3b6cba6f..c7db0e7598d0 100644 --- a/plugin/trino-snowflake/pom.xml +++ b/plugin/trino-snowflake/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-spooling-filesystem/pom.xml b/plugin/trino-spooling-filesystem/pom.xml index 280761c86450..d56a86692758 100644 --- a/plugin/trino-spooling-filesystem/pom.xml +++ b/plugin/trino-spooling-filesystem/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-sqlserver/pom.xml b/plugin/trino-sqlserver/pom.xml index 33ff0e4c224e..75ec3cf7e701 100644 --- a/plugin/trino-sqlserver/pom.xml +++ b/plugin/trino-sqlserver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-teradata-functions/pom.xml b/plugin/trino-teradata-functions/pom.xml index bcec10794f54..aecf423da3d8 100644 --- a/plugin/trino-teradata-functions/pom.xml +++ b/plugin/trino-teradata-functions/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-thrift-api/pom.xml b/plugin/trino-thrift-api/pom.xml index ac22279fcf77..3c67875c4cd3 100644 --- a/plugin/trino-thrift-api/pom.xml +++ b/plugin/trino-thrift-api/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-thrift-testing-server/pom.xml b/plugin/trino-thrift-testing-server/pom.xml index 263e2d938447..34597323c9b5 100644 --- a/plugin/trino-thrift-testing-server/pom.xml +++ b/plugin/trino-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-thrift/pom.xml b/plugin/trino-thrift/pom.xml index 1bca2e060feb..4d9978671053 100644 --- a/plugin/trino-thrift/pom.xml +++ b/plugin/trino-thrift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-tpcds/pom.xml b/plugin/trino-tpcds/pom.xml index 55c1c4622a9c..45cfcbcca5ec 100644 --- a/plugin/trino-tpcds/pom.xml +++ b/plugin/trino-tpcds/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-tpch/pom.xml b/plugin/trino-tpch/pom.xml index 614b7b2cd7cd..38dcddff2523 100644 --- a/plugin/trino-tpch/pom.xml +++ b/plugin/trino-tpch/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/plugin/trino-vertica/pom.xml b/plugin/trino-vertica/pom.xml index f741ebf9e8cf..a266b550fbe7 100644 --- a/plugin/trino-vertica/pom.xml +++ b/plugin/trino-vertica/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/pom.xml b/pom.xml index e660ec2d052f..6dc7857afa42 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 pom ${project.artifactId} @@ -141,14 +141,14 @@ scm:git:git://github.com/trinodb/trino.git scm:git:git@github.com:trinodb/trino.git - HEAD + 468 https://github.com/trinodb/trino 23 - 2024-12-06T21:31:48Z + 2024-12-17T21:11:27Z ERROR diff --git a/service/trino-proxy/pom.xml b/service/trino-proxy/pom.xml index 536e2a331dd0..339d1c421d56 100644 --- a/service/trino-proxy/pom.xml +++ b/service/trino-proxy/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/service/trino-verifier/pom.xml b/service/trino-verifier/pom.xml index 3f5d74b4bc8c..1f7f8a55efa3 100644 --- a/service/trino-verifier/pom.xml +++ b/service/trino-verifier/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-benchmark-queries/pom.xml b/testing/trino-benchmark-queries/pom.xml index 51defaa2a581..89badbdba159 100644 --- a/testing/trino-benchmark-queries/pom.xml +++ b/testing/trino-benchmark-queries/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-benchto-benchmarks/pom.xml b/testing/trino-benchto-benchmarks/pom.xml index d84a4535521b..affb22b83ff9 100644 --- a/testing/trino-benchto-benchmarks/pom.xml +++ b/testing/trino-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-faulttolerant-tests/pom.xml b/testing/trino-faulttolerant-tests/pom.xml index de6f1fb27f72..34df652131b6 100644 --- a/testing/trino-faulttolerant-tests/pom.xml +++ b/testing/trino-faulttolerant-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-plugin-reader/pom.xml b/testing/trino-plugin-reader/pom.xml index 23e4477fa9c2..165e4fcde3c9 100644 --- a/testing/trino-plugin-reader/pom.xml +++ b/testing/trino-plugin-reader/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-product-tests-groups/pom.xml b/testing/trino-product-tests-groups/pom.xml index f8f3a7416358..bdd7a493a5bc 100644 --- a/testing/trino-product-tests-groups/pom.xml +++ b/testing/trino-product-tests-groups/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-product-tests-launcher/pom.xml b/testing/trino-product-tests-launcher/pom.xml index 5fb09e66521f..2502f66c137c 100644 --- a/testing/trino-product-tests-launcher/pom.xml +++ b/testing/trino-product-tests-launcher/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index 0c39ece9f7be..625ebcc0c563 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-server-dev/pom.xml b/testing/trino-server-dev/pom.xml index c7d57af2e149..b9dd6eeff081 100644 --- a/testing/trino-server-dev/pom.xml +++ b/testing/trino-server-dev/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml index 90af66fba5b2..02acab740bb4 100644 --- a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml @@ -14,7 +14,7 @@ - 468-SNAPSHOT + 468 diff --git a/testing/trino-test-jdbc-compatibility-old-server/pom.xml b/testing/trino-test-jdbc-compatibility-old-server/pom.xml index ac6f6ccd1e2d..e31ac5173f8f 100644 --- a/testing/trino-test-jdbc-compatibility-old-server/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-testing-containers/pom.xml b/testing/trino-testing-containers/pom.xml index ab8dad0dae4d..3e34379295c9 100644 --- a/testing/trino-testing-containers/pom.xml +++ b/testing/trino-testing-containers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-testing-kafka/pom.xml b/testing/trino-testing-kafka/pom.xml index 25fe5d7fa499..60f01d878ed3 100644 --- a/testing/trino-testing-kafka/pom.xml +++ b/testing/trino-testing-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-testing-resources/pom.xml b/testing/trino-testing-resources/pom.xml index 924468815c7c..8f4d85073387 100644 --- a/testing/trino-testing-resources/pom.xml +++ b/testing/trino-testing-resources/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-testing-services/pom.xml b/testing/trino-testing-services/pom.xml index a58acd49b3f1..f10100725660 100644 --- a/testing/trino-testing-services/pom.xml +++ b/testing/trino-testing-services/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-testing/pom.xml b/testing/trino-testing/pom.xml index 0a2f9d1c1786..8847e6edc394 100644 --- a/testing/trino-testing/pom.xml +++ b/testing/trino-testing/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml diff --git a/testing/trino-tests/pom.xml b/testing/trino-tests/pom.xml index 11ab4ad04bec..94d96a730298 100644 --- a/testing/trino-tests/pom.xml +++ b/testing/trino-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468-SNAPSHOT + 468 ../../pom.xml From e1574bc709ced8f77548ab2fec33722ef63f63f7 Mon Sep 17 00:00:00 2001 From: Martin Traverso Date: Tue, 17 Dec 2024 21:29:19 +0000 Subject: [PATCH 017/158] [maven-release-plugin] prepare for next development iteration --- client/trino-cli/pom.xml | 2 +- client/trino-client/pom.xml | 2 +- client/trino-jdbc/pom.xml | 2 +- core/trino-grammar/pom.xml | 2 +- core/trino-main/pom.xml | 2 +- core/trino-parser/pom.xml | 2 +- core/trino-server-main/pom.xml | 2 +- core/trino-server-rpm/pom.xml | 2 +- core/trino-server/pom.xml | 2 +- core/trino-spi/pom.xml | 2 +- core/trino-web-ui/pom.xml | 2 +- docs/pom.xml | 2 +- lib/trino-array/pom.xml | 2 +- lib/trino-cache/pom.xml | 2 +- lib/trino-filesystem-alluxio/pom.xml | 2 +- lib/trino-filesystem-azure/pom.xml | 2 +- lib/trino-filesystem-cache-alluxio/pom.xml | 2 +- lib/trino-filesystem-gcs/pom.xml | 2 +- lib/trino-filesystem-manager/pom.xml | 2 +- lib/trino-filesystem-s3/pom.xml | 2 +- lib/trino-filesystem/pom.xml | 2 +- lib/trino-geospatial-toolkit/pom.xml | 2 +- lib/trino-hdfs/pom.xml | 2 +- lib/trino-hive-formats/pom.xml | 2 +- lib/trino-matching/pom.xml | 2 +- lib/trino-memory-context/pom.xml | 2 +- lib/trino-metastore/pom.xml | 2 +- lib/trino-orc/pom.xml | 2 +- lib/trino-parquet/pom.xml | 2 +- lib/trino-plugin-toolkit/pom.xml | 2 +- lib/trino-record-decoder/pom.xml | 2 +- plugin/trino-base-jdbc/pom.xml | 2 +- plugin/trino-bigquery/pom.xml | 2 +- plugin/trino-blackhole/pom.xml | 2 +- plugin/trino-cassandra/pom.xml | 2 +- plugin/trino-clickhouse/pom.xml | 2 +- plugin/trino-delta-lake/pom.xml | 2 +- plugin/trino-druid/pom.xml | 2 +- plugin/trino-elasticsearch/pom.xml | 2 +- plugin/trino-example-http/pom.xml | 2 +- plugin/trino-example-jdbc/pom.xml | 2 +- plugin/trino-exasol/pom.xml | 2 +- plugin/trino-exchange-filesystem/pom.xml | 2 +- plugin/trino-exchange-hdfs/pom.xml | 2 +- plugin/trino-faker/pom.xml | 2 +- plugin/trino-functions-python/pom.xml | 2 +- plugin/trino-geospatial/pom.xml | 2 +- plugin/trino-google-sheets/pom.xml | 2 +- plugin/trino-hive/pom.xml | 2 +- plugin/trino-http-event-listener/pom.xml | 2 +- plugin/trino-http-server-event-listener/pom.xml | 2 +- plugin/trino-hudi/pom.xml | 2 +- plugin/trino-iceberg/pom.xml | 2 +- plugin/trino-ignite/pom.xml | 2 +- plugin/trino-jmx/pom.xml | 2 +- plugin/trino-kafka-event-listener/pom.xml | 2 +- plugin/trino-kafka/pom.xml | 2 +- plugin/trino-kinesis/pom.xml | 2 +- plugin/trino-kudu/pom.xml | 2 +- plugin/trino-mariadb/pom.xml | 2 +- plugin/trino-memory/pom.xml | 2 +- plugin/trino-ml/pom.xml | 2 +- plugin/trino-mongodb/pom.xml | 2 +- plugin/trino-mysql-event-listener/pom.xml | 2 +- plugin/trino-mysql/pom.xml | 2 +- plugin/trino-opa/pom.xml | 2 +- plugin/trino-openlineage/pom.xml | 2 +- plugin/trino-opensearch/pom.xml | 2 +- plugin/trino-oracle/pom.xml | 2 +- plugin/trino-password-authenticators/pom.xml | 2 +- plugin/trino-phoenix5/pom.xml | 2 +- plugin/trino-pinot/pom.xml | 2 +- plugin/trino-postgresql/pom.xml | 2 +- plugin/trino-prometheus/pom.xml | 2 +- plugin/trino-ranger/pom.xml | 2 +- plugin/trino-redis/pom.xml | 2 +- plugin/trino-redshift/pom.xml | 2 +- plugin/trino-resource-group-managers/pom.xml | 2 +- plugin/trino-session-property-managers/pom.xml | 2 +- plugin/trino-singlestore/pom.xml | 2 +- plugin/trino-snowflake/pom.xml | 2 +- plugin/trino-spooling-filesystem/pom.xml | 2 +- plugin/trino-sqlserver/pom.xml | 2 +- plugin/trino-teradata-functions/pom.xml | 2 +- plugin/trino-thrift-api/pom.xml | 2 +- plugin/trino-thrift-testing-server/pom.xml | 2 +- plugin/trino-thrift/pom.xml | 2 +- plugin/trino-tpcds/pom.xml | 2 +- plugin/trino-tpch/pom.xml | 2 +- plugin/trino-vertica/pom.xml | 2 +- pom.xml | 6 +++--- service/trino-proxy/pom.xml | 2 +- service/trino-verifier/pom.xml | 2 +- testing/trino-benchmark-queries/pom.xml | 2 +- testing/trino-benchto-benchmarks/pom.xml | 2 +- testing/trino-faulttolerant-tests/pom.xml | 2 +- testing/trino-plugin-reader/pom.xml | 2 +- testing/trino-product-tests-groups/pom.xml | 2 +- testing/trino-product-tests-launcher/pom.xml | 2 +- testing/trino-product-tests/pom.xml | 2 +- testing/trino-server-dev/pom.xml | 2 +- testing/trino-test-jdbc-compatibility-old-driver/pom.xml | 4 ++-- testing/trino-test-jdbc-compatibility-old-server/pom.xml | 2 +- testing/trino-testing-containers/pom.xml | 2 +- testing/trino-testing-kafka/pom.xml | 2 +- testing/trino-testing-resources/pom.xml | 2 +- testing/trino-testing-services/pom.xml | 2 +- testing/trino-testing/pom.xml | 2 +- testing/trino-tests/pom.xml | 2 +- 109 files changed, 112 insertions(+), 112 deletions(-) diff --git a/client/trino-cli/pom.xml b/client/trino-cli/pom.xml index efe41a26cac2..d5948c669b92 100644 --- a/client/trino-cli/pom.xml +++ b/client/trino-cli/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/client/trino-client/pom.xml b/client/trino-client/pom.xml index ed6a74bf7d44..3b37b0fc7aef 100644 --- a/client/trino-client/pom.xml +++ b/client/trino-client/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/client/trino-jdbc/pom.xml b/client/trino-jdbc/pom.xml index 719c5152cb30..1f6688a2fdeb 100644 --- a/client/trino-jdbc/pom.xml +++ b/client/trino-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-grammar/pom.xml b/core/trino-grammar/pom.xml index 2f3c770f1607..42b3bed260eb 100644 --- a/core/trino-grammar/pom.xml +++ b/core/trino-grammar/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index 6d599e90c938..75ae527e0385 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-parser/pom.xml b/core/trino-parser/pom.xml index 7330d67c0d17..55a765673a7c 100644 --- a/core/trino-parser/pom.xml +++ b/core/trino-parser/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-server-main/pom.xml b/core/trino-server-main/pom.xml index 8b3359c51343..62098b8116e3 100644 --- a/core/trino-server-main/pom.xml +++ b/core/trino-server-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-server-rpm/pom.xml b/core/trino-server-rpm/pom.xml index 7210b7937569..ef18e071218f 100644 --- a/core/trino-server-rpm/pom.xml +++ b/core/trino-server-rpm/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-server/pom.xml b/core/trino-server/pom.xml index 0513120d2c98..1b611ff11e6b 100644 --- a/core/trino-server/pom.xml +++ b/core/trino-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index bc23756ad41a..da80d6442716 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/core/trino-web-ui/pom.xml b/core/trino-web-ui/pom.xml index 0e032708c15e..a8ef1e605f99 100644 --- a/core/trino-web-ui/pom.xml +++ b/core/trino-web-ui/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/docs/pom.xml b/docs/pom.xml index 94f7a392c4dc..8ad50977d6df 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT trino-docs diff --git a/lib/trino-array/pom.xml b/lib/trino-array/pom.xml index c7fb5d57bd67..770af98fada8 100644 --- a/lib/trino-array/pom.xml +++ b/lib/trino-array/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-cache/pom.xml b/lib/trino-cache/pom.xml index dc6f59bfe7fb..49d3b1c68de9 100644 --- a/lib/trino-cache/pom.xml +++ b/lib/trino-cache/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-alluxio/pom.xml b/lib/trino-filesystem-alluxio/pom.xml index c755f0624355..7adc50ffd260 100644 --- a/lib/trino-filesystem-alluxio/pom.xml +++ b/lib/trino-filesystem-alluxio/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-azure/pom.xml b/lib/trino-filesystem-azure/pom.xml index c6a88366ff48..8cc67d7b6144 100644 --- a/lib/trino-filesystem-azure/pom.xml +++ b/lib/trino-filesystem-azure/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-cache-alluxio/pom.xml b/lib/trino-filesystem-cache-alluxio/pom.xml index 269eaaa9038c..f66de3c15c38 100644 --- a/lib/trino-filesystem-cache-alluxio/pom.xml +++ b/lib/trino-filesystem-cache-alluxio/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-gcs/pom.xml b/lib/trino-filesystem-gcs/pom.xml index 405667b46f15..e7c399254847 100644 --- a/lib/trino-filesystem-gcs/pom.xml +++ b/lib/trino-filesystem-gcs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-manager/pom.xml b/lib/trino-filesystem-manager/pom.xml index 6bfa8fa39726..c12f2086d9d5 100644 --- a/lib/trino-filesystem-manager/pom.xml +++ b/lib/trino-filesystem-manager/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-s3/pom.xml b/lib/trino-filesystem-s3/pom.xml index f2e0e54e74b1..30d9336b5f62 100644 --- a/lib/trino-filesystem-s3/pom.xml +++ b/lib/trino-filesystem-s3/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem/pom.xml b/lib/trino-filesystem/pom.xml index e30d68915cce..21a156e8b538 100644 --- a/lib/trino-filesystem/pom.xml +++ b/lib/trino-filesystem/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-geospatial-toolkit/pom.xml b/lib/trino-geospatial-toolkit/pom.xml index 6169df15b844..645db171e803 100644 --- a/lib/trino-geospatial-toolkit/pom.xml +++ b/lib/trino-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-hdfs/pom.xml b/lib/trino-hdfs/pom.xml index c1bc583554f1..ce397e67bb26 100644 --- a/lib/trino-hdfs/pom.xml +++ b/lib/trino-hdfs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-hive-formats/pom.xml b/lib/trino-hive-formats/pom.xml index f5ebec6d06c6..44bf90e72f63 100644 --- a/lib/trino-hive-formats/pom.xml +++ b/lib/trino-hive-formats/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-matching/pom.xml b/lib/trino-matching/pom.xml index 644704a40421..df6407fdf319 100644 --- a/lib/trino-matching/pom.xml +++ b/lib/trino-matching/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-memory-context/pom.xml b/lib/trino-memory-context/pom.xml index 047ef17993f5..e598ec5a2220 100644 --- a/lib/trino-memory-context/pom.xml +++ b/lib/trino-memory-context/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-metastore/pom.xml b/lib/trino-metastore/pom.xml index ebc63b5cd781..c0549ec5c888 100644 --- a/lib/trino-metastore/pom.xml +++ b/lib/trino-metastore/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-orc/pom.xml b/lib/trino-orc/pom.xml index 84a022273ebc..c3a4a125421a 100644 --- a/lib/trino-orc/pom.xml +++ b/lib/trino-orc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-parquet/pom.xml b/lib/trino-parquet/pom.xml index 1a04dc86c91e..5a25b76485aa 100644 --- a/lib/trino-parquet/pom.xml +++ b/lib/trino-parquet/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-plugin-toolkit/pom.xml b/lib/trino-plugin-toolkit/pom.xml index ba1faf4e90d6..44008bb79400 100644 --- a/lib/trino-plugin-toolkit/pom.xml +++ b/lib/trino-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/lib/trino-record-decoder/pom.xml b/lib/trino-record-decoder/pom.xml index 1047479f03ee..7a5278dc8120 100644 --- a/lib/trino-record-decoder/pom.xml +++ b/lib/trino-record-decoder/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-base-jdbc/pom.xml b/plugin/trino-base-jdbc/pom.xml index 70248555cd87..d116c088cc21 100644 --- a/plugin/trino-base-jdbc/pom.xml +++ b/plugin/trino-base-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-bigquery/pom.xml b/plugin/trino-bigquery/pom.xml index 345d937700e4..9f5f027897db 100644 --- a/plugin/trino-bigquery/pom.xml +++ b/plugin/trino-bigquery/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-blackhole/pom.xml b/plugin/trino-blackhole/pom.xml index 0c3498a6c5ad..95203943a402 100644 --- a/plugin/trino-blackhole/pom.xml +++ b/plugin/trino-blackhole/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-cassandra/pom.xml b/plugin/trino-cassandra/pom.xml index f7646cdf658b..a5d02719359b 100644 --- a/plugin/trino-cassandra/pom.xml +++ b/plugin/trino-cassandra/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml index c99bc6914d09..860d259d6fb0 100644 --- a/plugin/trino-clickhouse/pom.xml +++ b/plugin/trino-clickhouse/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-delta-lake/pom.xml b/plugin/trino-delta-lake/pom.xml index 66c2886f0cc7..fd35145fc80a 100644 --- a/plugin/trino-delta-lake/pom.xml +++ b/plugin/trino-delta-lake/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-druid/pom.xml b/plugin/trino-druid/pom.xml index d538636510fc..4a2f5a765657 100644 --- a/plugin/trino-druid/pom.xml +++ b/plugin/trino-druid/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-elasticsearch/pom.xml b/plugin/trino-elasticsearch/pom.xml index 00e5bca9beed..b27ecb50a245 100644 --- a/plugin/trino-elasticsearch/pom.xml +++ b/plugin/trino-elasticsearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-example-http/pom.xml b/plugin/trino-example-http/pom.xml index 3c4917c70f2e..59e3d0c47ae7 100644 --- a/plugin/trino-example-http/pom.xml +++ b/plugin/trino-example-http/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-example-jdbc/pom.xml b/plugin/trino-example-jdbc/pom.xml index 991b14405d91..2e861e37d6a0 100644 --- a/plugin/trino-example-jdbc/pom.xml +++ b/plugin/trino-example-jdbc/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-exasol/pom.xml b/plugin/trino-exasol/pom.xml index 5e1c50a953c2..f8398dcba998 100644 --- a/plugin/trino-exasol/pom.xml +++ b/plugin/trino-exasol/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-exchange-filesystem/pom.xml b/plugin/trino-exchange-filesystem/pom.xml index c1fcee1becf1..b807dad33bb5 100644 --- a/plugin/trino-exchange-filesystem/pom.xml +++ b/plugin/trino-exchange-filesystem/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-exchange-hdfs/pom.xml b/plugin/trino-exchange-hdfs/pom.xml index f5781edf6dbe..b42d917a91fa 100644 --- a/plugin/trino-exchange-hdfs/pom.xml +++ b/plugin/trino-exchange-hdfs/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-faker/pom.xml b/plugin/trino-faker/pom.xml index 9af9084c4474..b9f8f35e05b0 100644 --- a/plugin/trino-faker/pom.xml +++ b/plugin/trino-faker/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-functions-python/pom.xml b/plugin/trino-functions-python/pom.xml index f67018af8f59..3f641714f5db 100644 --- a/plugin/trino-functions-python/pom.xml +++ b/plugin/trino-functions-python/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-geospatial/pom.xml b/plugin/trino-geospatial/pom.xml index 9f79f6995595..bea306914efc 100644 --- a/plugin/trino-geospatial/pom.xml +++ b/plugin/trino-geospatial/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-google-sheets/pom.xml b/plugin/trino-google-sheets/pom.xml index d29ad025a921..32a9754376da 100644 --- a/plugin/trino-google-sheets/pom.xml +++ b/plugin/trino-google-sheets/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml index beea0a99d5dd..fdda2afcc447 100644 --- a/plugin/trino-hive/pom.xml +++ b/plugin/trino-hive/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-http-event-listener/pom.xml b/plugin/trino-http-event-listener/pom.xml index d92f1fa5d7df..cc1044569593 100644 --- a/plugin/trino-http-event-listener/pom.xml +++ b/plugin/trino-http-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-http-server-event-listener/pom.xml b/plugin/trino-http-server-event-listener/pom.xml index f6c24bd646ff..a77b7996c059 100644 --- a/plugin/trino-http-server-event-listener/pom.xml +++ b/plugin/trino-http-server-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-hudi/pom.xml b/plugin/trino-hudi/pom.xml index 72667b42d300..f8eb77ab75cf 100644 --- a/plugin/trino-hudi/pom.xml +++ b/plugin/trino-hudi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index 0e77542dd47e..f07debe57d23 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-ignite/pom.xml b/plugin/trino-ignite/pom.xml index 2d07bb987e9b..320ba396ceed 100644 --- a/plugin/trino-ignite/pom.xml +++ b/plugin/trino-ignite/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-jmx/pom.xml b/plugin/trino-jmx/pom.xml index 1750cfe00a34..9f6b5404064d 100644 --- a/plugin/trino-jmx/pom.xml +++ b/plugin/trino-jmx/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kafka-event-listener/pom.xml b/plugin/trino-kafka-event-listener/pom.xml index e6776b5ad670..95147ca0724a 100644 --- a/plugin/trino-kafka-event-listener/pom.xml +++ b/plugin/trino-kafka-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kafka/pom.xml b/plugin/trino-kafka/pom.xml index d5242443a170..fa18036d378f 100644 --- a/plugin/trino-kafka/pom.xml +++ b/plugin/trino-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kinesis/pom.xml b/plugin/trino-kinesis/pom.xml index 4b67922dfa27..99f3beb33985 100644 --- a/plugin/trino-kinesis/pom.xml +++ b/plugin/trino-kinesis/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kudu/pom.xml b/plugin/trino-kudu/pom.xml index a2f8c3c97720..cb119e0db3b1 100644 --- a/plugin/trino-kudu/pom.xml +++ b/plugin/trino-kudu/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-mariadb/pom.xml b/plugin/trino-mariadb/pom.xml index 05ef446ae28a..3aac1b576a39 100644 --- a/plugin/trino-mariadb/pom.xml +++ b/plugin/trino-mariadb/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-memory/pom.xml b/plugin/trino-memory/pom.xml index 91d9a987ab25..1e52e1771cce 100644 --- a/plugin/trino-memory/pom.xml +++ b/plugin/trino-memory/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-ml/pom.xml b/plugin/trino-ml/pom.xml index 8d4f5f91f29b..71bb45b39476 100644 --- a/plugin/trino-ml/pom.xml +++ b/plugin/trino-ml/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-mongodb/pom.xml b/plugin/trino-mongodb/pom.xml index d2edea082f26..e9ffb478f544 100644 --- a/plugin/trino-mongodb/pom.xml +++ b/plugin/trino-mongodb/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-mysql-event-listener/pom.xml b/plugin/trino-mysql-event-listener/pom.xml index e83deb6ca0b6..94da473f2bc4 100644 --- a/plugin/trino-mysql-event-listener/pom.xml +++ b/plugin/trino-mysql-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-mysql/pom.xml b/plugin/trino-mysql/pom.xml index 63af742d3b44..7ae932343e81 100644 --- a/plugin/trino-mysql/pom.xml +++ b/plugin/trino-mysql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-opa/pom.xml b/plugin/trino-opa/pom.xml index 020a1d3f0520..77f8d21b8559 100644 --- a/plugin/trino-opa/pom.xml +++ b/plugin/trino-opa/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-openlineage/pom.xml b/plugin/trino-openlineage/pom.xml index ab9e7c9c2456..dfb0963dc7fa 100644 --- a/plugin/trino-openlineage/pom.xml +++ b/plugin/trino-openlineage/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-opensearch/pom.xml b/plugin/trino-opensearch/pom.xml index a48fffe4c947..a174da0fea2b 100644 --- a/plugin/trino-opensearch/pom.xml +++ b/plugin/trino-opensearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-oracle/pom.xml b/plugin/trino-oracle/pom.xml index d8982e8eaa57..c8f0865e7225 100644 --- a/plugin/trino-oracle/pom.xml +++ b/plugin/trino-oracle/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-password-authenticators/pom.xml b/plugin/trino-password-authenticators/pom.xml index 78f1f63e06c2..a1b94adf0727 100644 --- a/plugin/trino-password-authenticators/pom.xml +++ b/plugin/trino-password-authenticators/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-phoenix5/pom.xml b/plugin/trino-phoenix5/pom.xml index 9270526f2714..dd25843aed77 100644 --- a/plugin/trino-phoenix5/pom.xml +++ b/plugin/trino-phoenix5/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-pinot/pom.xml b/plugin/trino-pinot/pom.xml index 2625eefc964d..baa74b10aca9 100755 --- a/plugin/trino-pinot/pom.xml +++ b/plugin/trino-pinot/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-postgresql/pom.xml b/plugin/trino-postgresql/pom.xml index a28e5f1fbec9..d7105256f301 100644 --- a/plugin/trino-postgresql/pom.xml +++ b/plugin/trino-postgresql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-prometheus/pom.xml b/plugin/trino-prometheus/pom.xml index f4512e62cda6..5cf75975de3e 100644 --- a/plugin/trino-prometheus/pom.xml +++ b/plugin/trino-prometheus/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-ranger/pom.xml b/plugin/trino-ranger/pom.xml index 9c84aec7bb68..52de64f55b64 100644 --- a/plugin/trino-ranger/pom.xml +++ b/plugin/trino-ranger/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-redis/pom.xml b/plugin/trino-redis/pom.xml index fc428242c466..c7e119b77ac1 100644 --- a/plugin/trino-redis/pom.xml +++ b/plugin/trino-redis/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-redshift/pom.xml b/plugin/trino-redshift/pom.xml index 48b00ae943f7..e755eaa2ae92 100644 --- a/plugin/trino-redshift/pom.xml +++ b/plugin/trino-redshift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-resource-group-managers/pom.xml b/plugin/trino-resource-group-managers/pom.xml index 2052e9d26e59..94f1b8862ffe 100644 --- a/plugin/trino-resource-group-managers/pom.xml +++ b/plugin/trino-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-session-property-managers/pom.xml b/plugin/trino-session-property-managers/pom.xml index 21d005d43df2..8103062282a9 100644 --- a/plugin/trino-session-property-managers/pom.xml +++ b/plugin/trino-session-property-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-singlestore/pom.xml b/plugin/trino-singlestore/pom.xml index 9609d847a6ce..80f6c73bce1e 100644 --- a/plugin/trino-singlestore/pom.xml +++ b/plugin/trino-singlestore/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-snowflake/pom.xml b/plugin/trino-snowflake/pom.xml index c7db0e7598d0..e9efdfe0c427 100644 --- a/plugin/trino-snowflake/pom.xml +++ b/plugin/trino-snowflake/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-spooling-filesystem/pom.xml b/plugin/trino-spooling-filesystem/pom.xml index d56a86692758..bd05fa96b7dc 100644 --- a/plugin/trino-spooling-filesystem/pom.xml +++ b/plugin/trino-spooling-filesystem/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-sqlserver/pom.xml b/plugin/trino-sqlserver/pom.xml index 75ec3cf7e701..2cb41eb6fa90 100644 --- a/plugin/trino-sqlserver/pom.xml +++ b/plugin/trino-sqlserver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-teradata-functions/pom.xml b/plugin/trino-teradata-functions/pom.xml index aecf423da3d8..61b82f22590f 100644 --- a/plugin/trino-teradata-functions/pom.xml +++ b/plugin/trino-teradata-functions/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift-api/pom.xml b/plugin/trino-thrift-api/pom.xml index 3c67875c4cd3..32fc9779ec76 100644 --- a/plugin/trino-thrift-api/pom.xml +++ b/plugin/trino-thrift-api/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift-testing-server/pom.xml b/plugin/trino-thrift-testing-server/pom.xml index 34597323c9b5..37399a9f7cb9 100644 --- a/plugin/trino-thrift-testing-server/pom.xml +++ b/plugin/trino-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift/pom.xml b/plugin/trino-thrift/pom.xml index 4d9978671053..c28c1d49459f 100644 --- a/plugin/trino-thrift/pom.xml +++ b/plugin/trino-thrift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-tpcds/pom.xml b/plugin/trino-tpcds/pom.xml index 45cfcbcca5ec..f9aa6929d6c0 100644 --- a/plugin/trino-tpcds/pom.xml +++ b/plugin/trino-tpcds/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-tpch/pom.xml b/plugin/trino-tpch/pom.xml index 38dcddff2523..36067fff8132 100644 --- a/plugin/trino-tpch/pom.xml +++ b/plugin/trino-tpch/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-vertica/pom.xml b/plugin/trino-vertica/pom.xml index a266b550fbe7..a0ee11b9c890 100644 --- a/plugin/trino-vertica/pom.xml +++ b/plugin/trino-vertica/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 6dc7857afa42..ac740d07d7c3 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT pom ${project.artifactId} @@ -141,14 +141,14 @@ scm:git:git://github.com/trinodb/trino.git scm:git:git@github.com:trinodb/trino.git - 468 + HEAD https://github.com/trinodb/trino 23 - 2024-12-17T21:11:27Z + 2024-12-17T21:29:19Z ERROR diff --git a/service/trino-proxy/pom.xml b/service/trino-proxy/pom.xml index 339d1c421d56..ca569928a9b5 100644 --- a/service/trino-proxy/pom.xml +++ b/service/trino-proxy/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/service/trino-verifier/pom.xml b/service/trino-verifier/pom.xml index 1f7f8a55efa3..fb9151af5b0f 100644 --- a/service/trino-verifier/pom.xml +++ b/service/trino-verifier/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-benchmark-queries/pom.xml b/testing/trino-benchmark-queries/pom.xml index 89badbdba159..a5f08759cad6 100644 --- a/testing/trino-benchmark-queries/pom.xml +++ b/testing/trino-benchmark-queries/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-benchto-benchmarks/pom.xml b/testing/trino-benchto-benchmarks/pom.xml index affb22b83ff9..32443cd48126 100644 --- a/testing/trino-benchto-benchmarks/pom.xml +++ b/testing/trino-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-faulttolerant-tests/pom.xml b/testing/trino-faulttolerant-tests/pom.xml index 34df652131b6..12a6f242568c 100644 --- a/testing/trino-faulttolerant-tests/pom.xml +++ b/testing/trino-faulttolerant-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-plugin-reader/pom.xml b/testing/trino-plugin-reader/pom.xml index 165e4fcde3c9..1aa84a197865 100644 --- a/testing/trino-plugin-reader/pom.xml +++ b/testing/trino-plugin-reader/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-product-tests-groups/pom.xml b/testing/trino-product-tests-groups/pom.xml index bdd7a493a5bc..ee707c5d7a67 100644 --- a/testing/trino-product-tests-groups/pom.xml +++ b/testing/trino-product-tests-groups/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-product-tests-launcher/pom.xml b/testing/trino-product-tests-launcher/pom.xml index 2502f66c137c..9d4e65e5a002 100644 --- a/testing/trino-product-tests-launcher/pom.xml +++ b/testing/trino-product-tests-launcher/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index 625ebcc0c563..b524d674de83 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-server-dev/pom.xml b/testing/trino-server-dev/pom.xml index b9dd6eeff081..3ca183c4f022 100644 --- a/testing/trino-server-dev/pom.xml +++ b/testing/trino-server-dev/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml index 02acab740bb4..ca0371e655aa 100644 --- a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml @@ -14,7 +14,7 @@ - 468 + 469-SNAPSHOT diff --git a/testing/trino-test-jdbc-compatibility-old-server/pom.xml b/testing/trino-test-jdbc-compatibility-old-server/pom.xml index e31ac5173f8f..ba993b65a10d 100644 --- a/testing/trino-test-jdbc-compatibility-old-server/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing-containers/pom.xml b/testing/trino-testing-containers/pom.xml index 3e34379295c9..90c73d3d3d89 100644 --- a/testing/trino-testing-containers/pom.xml +++ b/testing/trino-testing-containers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing-kafka/pom.xml b/testing/trino-testing-kafka/pom.xml index 60f01d878ed3..55a6483baa7a 100644 --- a/testing/trino-testing-kafka/pom.xml +++ b/testing/trino-testing-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing-resources/pom.xml b/testing/trino-testing-resources/pom.xml index 8f4d85073387..28fadb11c6f8 100644 --- a/testing/trino-testing-resources/pom.xml +++ b/testing/trino-testing-resources/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing-services/pom.xml b/testing/trino-testing-services/pom.xml index f10100725660..37b9d5d8f600 100644 --- a/testing/trino-testing-services/pom.xml +++ b/testing/trino-testing-services/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing/pom.xml b/testing/trino-testing/pom.xml index 8847e6edc394..438992b4764d 100644 --- a/testing/trino-testing/pom.xml +++ b/testing/trino-testing/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml diff --git a/testing/trino-tests/pom.xml b/testing/trino-tests/pom.xml index 94d96a730298..1a218a5be589 100644 --- a/testing/trino-tests/pom.xml +++ b/testing/trino-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 468 + 469-SNAPSHOT ../../pom.xml From 1c5e6213a863cb5851bb59f9b7c7fff63e82b528 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Tue, 17 Dec 2024 22:29:50 +0100 Subject: [PATCH 018/158] Do not require Java presence for RPM installation This allows for custom JDK to be used when running Trino with launcher --jvm-dir argument. --- .../trino-server-rpm/src/main/rpm/postinstall | 10 --- core/trino-server-rpm/src/main/rpm/preinstall | 68 ------------------- .../java/io/trino/server/rpm/ServerIT.java | 2 + 3 files changed, 2 insertions(+), 78 deletions(-) diff --git a/core/trino-server-rpm/src/main/rpm/postinstall b/core/trino-server-rpm/src/main/rpm/postinstall index 3764fa7455c3..f0261473f117 100644 --- a/core/trino-server-rpm/src/main/rpm/postinstall +++ b/core/trino-server-rpm/src/main/rpm/postinstall @@ -10,16 +10,6 @@ install --directory --mode=755 /var/log/trino # Populate node.id from uuidgen by replacing template with the node uuid sed -i "s/\$(uuid-generated-nodeid)/$(cat /proc/sys/kernel/random/uuid)/g" /etc/trino/node.properties -# read /tmp/trino-rpm-install-java-home created during pre-install and save JAVA_HOME in env.sh at Trino config location -if [ -r /tmp/trino-rpm-install-java-home ]; then - JAVA_HOME=$(cat /tmp/trino-rpm-install-java-home) - target=/etc/trino/env.sh - sed -i "/^#JAVA_HOME=$/d" $target - if ! grep -q '^JAVA_HOME=' $target >/dev/null; then - echo "JAVA_HOME=$JAVA_HOME" >> $target - fi -fi - chown -R trino:trino /var/lib/trino chown -R trino:trino /var/log/trino chown -R trino:trino /etc/trino diff --git a/core/trino-server-rpm/src/main/rpm/preinstall b/core/trino-server-rpm/src/main/rpm/preinstall index d0a7e90d2c21..4ac7fc7a6fa7 100644 --- a/core/trino-server-rpm/src/main/rpm/preinstall +++ b/core/trino-server-rpm/src/main/rpm/preinstall @@ -1,72 +1,4 @@ # Pre installation script -# Ensure that the proper version of Java exists on the system - -java_version() { - # The one argument is the location of java (either $JAVA_HOME or a potential - # candidate for JAVA_HOME. - JAVA="$1"/bin/java - "$JAVA" -version 2>&1 | grep "\(java\|openjdk\) version" | awk '{ print substr($3, 2, length($3)-2); }' -} - -check_if_correct_java_version() { - - # If the string is empty return non-zero code. We don't want false positives if /bin/java is - # a valid java version because that will leave JAVA_HOME unset and the init.d scripts will - # use the default java version, which may not be the correct version. - if [ -z "$1" ]; then - return 1 - fi - - # The one argument is the location of java (either $JAVA_HOME or a potential - # candidate for JAVA_HOME). - JAVA_VERSION=$(java_version "$1") - JAVA_MAJOR=$(echo "$JAVA_VERSION" | cut -d'-' -f1 | cut -d'.' -f1) - if [ "$JAVA_MAJOR" -ge "23" ]; then - echo "$1" >/tmp/trino-rpm-install-java-home - return 0 - else - return 1 - fi -} - -# if Java version of $JAVA_HOME is not correct, then try to find it again below -if ! check_if_correct_java_version "$JAVA_HOME"; then - java_found=false - for candidate in \ - /usr/lib/jvm/java-23-* \ - /usr/lib/jvm/zulu-23 \ - /usr/lib/jvm/temurin-23 \ - /usr/lib/jvm/temurin-23-* \ - /usr/lib/jvm/default-java \ - /usr/java/default \ - / \ - /usr; do - if [ -e "$candidate"/bin/java ]; then - if check_if_correct_java_version "$candidate"; then - java_found=true - break - fi - fi - done -fi - -# if no appropriate java found -if [ "$java_found" = false ]; then - cat 1>&2 </dev/null || /usr/sbin/groupadd -r trino getent passwd trino >/dev/null || /usr/sbin/useradd --comment "Trino" -s /sbin/nologin -g trino -r -d /var/lib/trino trino diff --git a/core/trino-server-rpm/src/test/java/io/trino/server/rpm/ServerIT.java b/core/trino-server-rpm/src/test/java/io/trino/server/rpm/ServerIT.java index 93e89aebfef5..946d3734883b 100644 --- a/core/trino-server-rpm/src/test/java/io/trino/server/rpm/ServerIT.java +++ b/core/trino-server-rpm/src/test/java/io/trino/server/rpm/ServerIT.java @@ -109,6 +109,7 @@ private void testInstall(String temurinReleaseName, String javaHome, String expe .withCommand("sh", "-xeuc", command) .withCreateContainerCmdModifier(modifier -> modifier .withHostConfig(modifier.getHostConfig().withInit(true))) + .withEnv("JAVA_HOME", javaHome) .waitingFor(forLogMessage(".*SERVER STARTED.*", 1).withStartupTimeout(Duration.ofMinutes(5))) .start(); QueryRunner queryRunner = new QueryRunner(container.getHost(), container.getMappedPort(8080)); @@ -137,6 +138,7 @@ private void testUninstall(String temurinReleaseName, String javaHome) try (GenericContainer container = new GenericContainer<>(BASE_IMAGE)) { container.withFileSystemBind(rpmHostPath, rpm, BindMode.READ_ONLY) .withCommand("sh", "-xeuc", installAndStartTrino) + .withEnv("JAVA_HOME", javaHome) .withCreateContainerCmdModifier(modifier -> modifier .withHostConfig(modifier.getHostConfig().withInit(true))) .waitingFor(forLogMessage(".*SERVER STARTED.*", 1).withStartupTimeout(Duration.ofMinutes(5))) From 82055816d6420465851f8a2ca8d3722ce161379d Mon Sep 17 00:00:00 2001 From: chenjian2664 Date: Wed, 11 Dec 2024 22:14:47 +0800 Subject: [PATCH 019/158] Allow left side as update target in join pushdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow left side as update target when try to pushdown join into table scan. The change prevent the pushdown join into the table scan instead of throwing exception Co-Authored-By: Łukasz Osipiuk --- .../iterative/rule/PushJoinIntoTableScan.java | 5 +++-- .../trino/plugin/kudu/TestKuduConnectorTest.java | 14 ++++++++++++++ .../mariadb/BaseMariaDbFailureRecoveryTest.java | 11 ----------- .../plugin/mysql/BaseMySqlFailureRecoveryTest.java | 11 ----------- .../oracle/BaseOracleFailureRecoveryTest.java | 11 ----------- .../BasePostgresFailureRecoveryTest.java | 2 +- .../redshift/BaseRedshiftFailureRecoveryTest.java | 11 ----------- .../BaseSqlServerFailureRecoveryTest.java | 11 ----------- .../java/io/trino/testing/BaseConnectorTest.java | 12 ++++++++++++ 9 files changed, 30 insertions(+), 58 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushJoinIntoTableScan.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushJoinIntoTableScan.java index e6a71664f953..37f0054ae67d 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushJoinIntoTableScan.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushJoinIntoTableScan.java @@ -46,7 +46,6 @@ import java.util.Map; import java.util.Optional; -import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.trino.SystemSessionProperties.isAllowPushdownIntoConnectors; @@ -103,7 +102,9 @@ public Result apply(JoinNode joinNode, Captures captures, Context context) TableScanNode left = captures.get(LEFT_TABLE_SCAN); TableScanNode right = captures.get(RIGHT_TABLE_SCAN); - verify(!left.isUpdateTarget() && !right.isUpdateTarget(), "Unexpected Join over for-update table scan"); + if (left.isUpdateTarget() && !right.isUpdateTarget()) { + return Result.empty(); + } Expression effectiveFilter = getEffectiveFilter(joinNode); ConnectorExpressionTranslation translation = ConnectorExpressionTranslator.translateConjuncts( diff --git a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java index 73de900f28ba..6150f160a19d 100644 --- a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java +++ b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java @@ -995,6 +995,20 @@ public void testUpdateRowConcurrently() abort("Kudu doesn't support concurrent update of different columns in a row"); } + @Test + @Override + protected void testUpdateWithSubquery() + { + withTableName("test_update_with_subquery", tableName -> { + createTableForWrites("CREATE TABLE %s " + ORDER_COLUMNS, tableName, Optional.empty()); + assertUpdate("INSERT INTO " + tableName + " SELECT * FROM orders", 15000); + + assertQuery("SELECT count(*) FROM " + tableName + " WHERE shippriority = 101 AND custkey = (SELECT min(custkey) FROM customer)", "VALUES 0"); + assertUpdate("UPDATE " + tableName + " SET shippriority = 101 WHERE custkey = (SELECT min(custkey) FROM customer)", 9); + assertQuery("SELECT count(*) FROM " + tableName + " WHERE shippriority = 101 AND custkey = (SELECT min(custkey) FROM customer)", "VALUES 9"); + }); + } + @Test @Override public void testCreateTableWithTableComment() diff --git a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbFailureRecoveryTest.java b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbFailureRecoveryTest.java index e8eb5128ba7d..dcde5b95b11d 100644 --- a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbFailureRecoveryTest.java +++ b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbFailureRecoveryTest.java @@ -26,9 +26,6 @@ import java.util.Map; import java.util.Optional; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - public abstract class BaseMariaDbFailureRecoveryTest extends BaseJdbcFailureRecoveryTest { @@ -55,14 +52,6 @@ protected QueryRunner createQueryRunner(List> requiredTpchTables, M .build(); } - @Test - @Override - protected void testUpdateWithSubquery() - { - assertThatThrownBy(super::testUpdateWithSubquery).hasMessageContaining("Unexpected Join over for-update table scan"); - abort("skipped"); - } - @Test @Override protected void testUpdate() diff --git a/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlFailureRecoveryTest.java b/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlFailureRecoveryTest.java index 1b474e83f11c..494e04deead5 100644 --- a/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlFailureRecoveryTest.java +++ b/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlFailureRecoveryTest.java @@ -26,9 +26,6 @@ import java.util.Map; import java.util.Optional; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - public abstract class BaseMySqlFailureRecoveryTest extends BaseJdbcFailureRecoveryTest { @@ -58,14 +55,6 @@ protected QueryRunner createQueryRunner( .build(); } - @Test - @Override - protected void testUpdateWithSubquery() - { - assertThatThrownBy(super::testUpdateWithSubquery).hasMessageContaining("Unexpected Join over for-update table scan"); - abort("skipped"); - } - @Test @Override protected void testUpdate() diff --git a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleFailureRecoveryTest.java b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleFailureRecoveryTest.java index 6836fcce8859..490dff0c94eb 100644 --- a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleFailureRecoveryTest.java +++ b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleFailureRecoveryTest.java @@ -26,9 +26,6 @@ import java.util.Map; import java.util.Optional; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - public abstract class BaseOracleFailureRecoveryTest extends BaseJdbcFailureRecoveryTest { @@ -59,14 +56,6 @@ protected QueryRunner createQueryRunner( .build(); } - @Test - @Override - protected void testUpdateWithSubquery() - { - assertThatThrownBy(super::testUpdateWithSubquery).hasMessageContaining("Unexpected Join over for-update table scan"); - abort("skipped"); - } - @Test @Override protected void testUpdate() diff --git a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/BasePostgresFailureRecoveryTest.java b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/BasePostgresFailureRecoveryTest.java index c5f49addd7db..53b2211c17d4 100644 --- a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/BasePostgresFailureRecoveryTest.java +++ b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/BasePostgresFailureRecoveryTest.java @@ -74,7 +74,7 @@ protected void testDeleteWithSubquery() @Override protected void testUpdateWithSubquery() { - assertThatThrownBy(super::testUpdateWithSubquery).hasMessageContaining("Unexpected Join over for-update table scan"); + assertThatThrownBy(super::testUpdateWithSubquery).hasMessageContaining("Non-transactional MERGE is disabled"); abort("skipped"); } diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/BaseRedshiftFailureRecoveryTest.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/BaseRedshiftFailureRecoveryTest.java index e38fa4cf3d78..2856863c5b3a 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/BaseRedshiftFailureRecoveryTest.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/BaseRedshiftFailureRecoveryTest.java @@ -26,9 +26,6 @@ import java.util.Map; import java.util.Optional; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - public abstract class BaseRedshiftFailureRecoveryTest extends BaseJdbcFailureRecoveryTest { @@ -58,14 +55,6 @@ protected QueryRunner createQueryRunner( .build(); } - @Test - @Override - protected void testUpdateWithSubquery() - { - assertThatThrownBy(super::testUpdateWithSubquery).hasMessageContaining("Unexpected Join over for-update table scan"); - abort("skipped"); - } - @Test @Override protected void testUpdate() diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerFailureRecoveryTest.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerFailureRecoveryTest.java index 4dbce2a01c03..ac8d2967479a 100644 --- a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerFailureRecoveryTest.java +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerFailureRecoveryTest.java @@ -26,9 +26,6 @@ import java.util.Map; import java.util.Optional; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - public abstract class BaseSqlServerFailureRecoveryTest extends BaseJdbcFailureRecoveryTest { @@ -58,14 +55,6 @@ protected QueryRunner createQueryRunner( .build(); } - @Test - @Override - protected void testUpdateWithSubquery() - { - assertThatThrownBy(super::testUpdateWithSubquery).hasMessageContaining("Unexpected Join over for-update table scan"); - abort("skipped"); - } - @Test @Override protected void testUpdate() diff --git a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java index e53c808fa9bf..c9fbbc3bffa3 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java @@ -6801,6 +6801,18 @@ public void testMergeAllColumnsReversed() assertUpdate("DROP TABLE " + targetTable); } + @Test + protected void testUpdateWithSubquery() + { + skipTestUnless(hasBehavior(SUPPORTS_MERGE)); + + try (TestTable table = createTestTableForWrites("test_update_with_subquery", " AS SELECT * FROM orders", "orderkey")) { + assertQuery("SELECT count(*) FROM " + table.getName() + " WHERE shippriority = 101 AND custkey = (SELECT min(custkey) FROM customer)", "VALUES 0"); + assertUpdate("UPDATE " + table.getName() + " SET shippriority = 101 WHERE custkey = (SELECT min(custkey) FROM customer)", 9); + assertQuery("SELECT count(*) FROM " + table.getName() + " WHERE shippriority = 101 AND custkey = (SELECT min(custkey) FROM customer)", "VALUES 9"); + } + } + private void verifyUnsupportedTypeException(Throwable exception, String trinoTypeName) { String typeNameBase = trinoTypeName.replaceFirst("\\(.*", ""); From 465f48aa3a3798380a585a49a3f289000c7800bb Mon Sep 17 00:00:00 2001 From: Raunaq Morarka Date: Wed, 18 Dec 2024 13:28:10 +0530 Subject: [PATCH 020/158] Bind filesystem cache metrics per catalog --- lib/trino-filesystem-cache-alluxio/pom.xml | 11 +++++------ .../alluxio/AlluxioFileSystemCacheModule.java | 6 +++++- .../trino/tests/product/utils/CachingTestUtils.java | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/trino-filesystem-cache-alluxio/pom.xml b/lib/trino-filesystem-cache-alluxio/pom.xml index f66de3c15c38..f169d718b470 100644 --- a/lib/trino-filesystem-cache-alluxio/pom.xml +++ b/lib/trino-filesystem-cache-alluxio/pom.xml @@ -63,6 +63,11 @@ trino-filesystem + + io.trino + trino-spi + + jakarta.annotation jakarta.annotation-api @@ -112,12 +117,6 @@ runtime - - io.trino - trino-spi - runtime - - io.airlift junit-extensions diff --git a/lib/trino-filesystem-cache-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystemCacheModule.java b/lib/trino-filesystem-cache-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystemCacheModule.java index d4acfe96d4c8..4d063e05e5d9 100644 --- a/lib/trino-filesystem-cache-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystemCacheModule.java +++ b/lib/trino-filesystem-cache-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystemCacheModule.java @@ -16,11 +16,13 @@ import alluxio.metrics.MetricsConfig; import alluxio.metrics.MetricsSystem; import com.google.inject.Binder; +import com.google.inject.Provider; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.filesystem.cache.CachingHostAddressProvider; import io.trino.filesystem.cache.ConsistentHashingHostAddressProvider; import io.trino.filesystem.cache.ConsistentHashingHostAddressProviderConfig; import io.trino.filesystem.cache.TrinoFileSystemCache; +import io.trino.spi.catalog.CatalogName; import java.util.Properties; @@ -45,7 +47,9 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(AlluxioFileSystemCacheConfig.class); configBinder(binder).bindConfig(ConsistentHashingHostAddressProviderConfig.class); binder.bind(AlluxioCacheStats.class).in(SINGLETON); - newExporter(binder).export(AlluxioCacheStats.class).as(generator -> generator.generatedNameOf(AlluxioCacheStats.class)); + Provider catalogName = binder.getProvider(CatalogName.class); + newExporter(binder).export(AlluxioCacheStats.class) + .as(generator -> generator.generatedNameOf(AlluxioCacheStats.class, catalogName.get().toString())); if (isCoordinator) { newOptionalBinder(binder, CachingHostAddressProvider.class).setBinding().to(ConsistentHashingHostAddressProvider.class).in(SINGLETON); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/CachingTestUtils.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/CachingTestUtils.java index a1ce1d3a8c95..3e28296ac6e9 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/CachingTestUtils.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/utils/CachingTestUtils.java @@ -27,7 +27,7 @@ public static CacheStats getCacheStats(String catalog) QueryResult queryResult = onTrino().executeQuery("SELECT " + " sum(\"cachereads.alltime.count\") as cachereads, " + " sum(\"externalreads.alltime.count\") as externalreads " + - "FROM jmx.current.\"io.trino.filesystem.alluxio:name=" + catalog + ",type=alluxiocachestats\";"); + "FROM jmx.current.\"io.trino.filesystem.alluxio:catalog=" + catalog + ",name=" + catalog + ",type=alluxiocachestats\";"); double cacheReads = (Double) getOnlyElement(queryResult.rows()) .get(queryResult.tryFindColumnIndex("cachereads").get() - 1); From 9015414aa35824c0bb948836545311890138ad74 Mon Sep 17 00:00:00 2001 From: James Petty Date: Wed, 18 Dec 2024 15:26:55 -0500 Subject: [PATCH 021/158] Cleanup InformationSchemaPageSource projection Avoids unnecessary Integer boxing and Block[] allocations in InformationSchemaPageSource by using Page#getColumns. --- .../InformationSchemaPageSource.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java index 3c6e16f10a6e..a28449bfe5ad 100644 --- a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java +++ b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java @@ -22,7 +22,6 @@ import io.trino.security.AccessControl; import io.trino.spi.Page; import io.trino.spi.PageBuilder; -import io.trino.spi.block.Block; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorPageSource; @@ -141,18 +140,12 @@ public InformationSchemaPageSource( .boxed() .collect(toImmutableMap(i -> columnMetadata.get(i).getName(), Function.identity())); - List channels = columns.stream() + int[] channels = columns.stream() .map(columnHandle -> (InformationSchemaColumnHandle) columnHandle) - .map(columnHandle -> columnNameToChannel.get(columnHandle.columnName())) - .collect(toImmutableList()); + .mapToInt(columnHandle -> columnNameToChannel.get(columnHandle.columnName())) + .toArray(); - projection = page -> { - Block[] blocks = new Block[channels.size()]; - for (int i = 0; i < blocks.length; i++) { - blocks[i] = page.getBlock(channels.get(i)); - } - return new Page(page.getPositionCount(), blocks); - }; + projection = page -> page.getColumns(channels); } @Override From 065b490097a3a7de355f8bf31d832c13033bd618 Mon Sep 17 00:00:00 2001 From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:50:50 -0500 Subject: [PATCH 022/158] Update docker image version to 107 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ac740d07d7c3..134ea6c48ef0 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ 1.12.780 4.17.0 7.7.1 - 106 + 107 1.22 11.1.0 1.15.1 From ea6c825cc24432f2dbc2ed396e0013277a4cca2d Mon Sep 17 00:00:00 2001 From: Mayank Vadariya Date: Fri, 13 Dec 2024 17:20:22 -0500 Subject: [PATCH 023/158] Rename HiveMinioDataLake to Hive3MinioDataLake --- .../plugin/deltalake/BaseDeltaFailureRecoveryTest.java | 4 ++-- .../deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java | 6 +++--- .../plugin/deltalake/BaseDeltaLakeCompatibility.java | 6 +++--- .../io/trino/plugin/deltalake/DeltaLakeQueryRunner.java | 4 ++-- .../java/io/trino/plugin/deltalake/SparkDeltaLake.java | 6 +++--- .../deltalake/TestDeltaLakeCreateTableStatistics.java | 4 ++-- .../io/trino/plugin/deltalake/TestDeltaLakeDelete.java | 6 +++--- .../plugin/deltalake/TestDeltaLakeDynamicFiltering.java | 6 +++--- .../TestDeltaLakeFlushMetadataCacheProcedure.java | 4 ++-- .../TestDeltaLakeSharedHiveMetastoreWithViews.java | 6 +++--- .../io/trino/plugin/deltalake/TestDeltaLakeUpdate.java | 4 ++-- .../io/trino/plugin/deltalake/TestPredicatePushdown.java | 6 +++--- .../java/io/trino/plugin/hive/TestHive3OnDataLake.java | 6 +++--- .../plugin/hive/TestHiveAnalyzeCorruptStatistics.java | 6 +++--- .../hive/TestHiveCustomCatalogConnectorSmokeTest.java | 4 ++-- .../plugin/hive/TestHiveQueryFailureRecoveryTest.java | 6 +++--- .../plugin/hive/TestHiveTaskFailureRecoveryTest.java | 6 +++--- .../{HiveMinioDataLake.java => Hive3MinioDataLake.java} | 8 ++++---- .../hive/metastore/thrift/TestHiveMetastoreCatalogs.java | 6 +++--- .../java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java | 8 ++++---- .../io/trino/plugin/hive/s3/TestHiveS3MinioQueries.java | 6 +++--- .../test/java/io/trino/plugin/hudi/HudiQueryRunner.java | 6 +++--- .../plugin/hudi/TestHudiMinioConnectorSmokeTest.java | 4 ++-- .../iceberg/BaseIcebergMinioConnectorSmokeTest.java | 6 +++--- .../java/io/trino/plugin/iceberg/IcebergQueryRunner.java | 4 ++-- .../hms/TestTrinoHiveCatalogWithHiveMetastore.java | 6 +++--- .../delta/TestDeltaFaultTolerantExecutionTest.java | 4 ++-- 27 files changed, 74 insertions(+), 74 deletions(-) rename plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/{HiveMinioDataLake.java => Hive3MinioDataLake.java} (94%) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaFailureRecoveryTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaFailureRecoveryTest.java index d189e96fa4e8..88f7ee2411aa 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaFailureRecoveryTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaFailureRecoveryTest.java @@ -17,7 +17,7 @@ import io.trino.operator.RetryPolicy; import io.trino.plugin.exchange.filesystem.FileSystemExchangePlugin; import io.trino.plugin.exchange.filesystem.containers.MinioStorage; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.spi.ErrorType; import io.trino.testing.BaseFailureRecoveryTest; import io.trino.testing.QueryRunner; @@ -59,7 +59,7 @@ protected QueryRunner createQueryRunner( Module failureInjectionModule) throws Exception { - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); MinioStorage minioStorage = closeAfterClass(new MinioStorage("test-exchange-spooling-" + randomNameSuffix())); minioStorage.start(); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java index e7b1529a8f1d..8fd2de505c58 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeAwsConnectorSmokeTest.java @@ -13,8 +13,8 @@ */ package io.trino.plugin.deltalake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.TestInstance; @@ -29,12 +29,12 @@ public abstract class BaseDeltaLakeAwsConnectorSmokeTest extends BaseDeltaLakeConnectorSmokeTest { - protected HiveMinioDataLake hiveMinioDataLake; + protected Hive3MinioDataLake hiveMinioDataLake; @Override protected HiveHadoop createHiveHadoop() { - hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); return hiveMinioDataLake.getHiveHadoop(); // closed by superclass } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeCompatibility.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeCompatibility.java index 9c9cde68f35a..4c352a590b57 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeCompatibility.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeCompatibility.java @@ -13,7 +13,7 @@ */ package io.trino.plugin.deltalake; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import io.trino.tpch.TpchTable; @@ -32,7 +32,7 @@ public abstract class BaseDeltaLakeCompatibility { protected final String bucketName; protected final String resourcePath; - protected HiveMinioDataLake hiveMinioDataLake; + protected Hive3MinioDataLake hiveMinioDataLake; public BaseDeltaLakeCompatibility(String resourcePath) { @@ -44,7 +44,7 @@ public BaseDeltaLakeCompatibility(String resourcePath) protected QueryRunner createQueryRunner() throws Exception { - hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); QueryRunner queryRunner = DeltaLakeQueryRunner.builder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/DeltaLakeQueryRunner.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/DeltaLakeQueryRunner.java index 01df1213f3d5..5c2ecb1d36f5 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/DeltaLakeQueryRunner.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/DeltaLakeQueryRunner.java @@ -19,8 +19,8 @@ import io.airlift.log.Level; import io.airlift.log.Logger; import io.airlift.log.Logging; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.tpch.TpchPlugin; import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; @@ -259,7 +259,7 @@ public static void main(String[] args) { String bucketName = "test-bucket"; - HiveMinioDataLake hiveMinioDataLake = new HiveMinioDataLake(bucketName); + Hive3MinioDataLake hiveMinioDataLake = new Hive3MinioDataLake(bucketName); hiveMinioDataLake.start(); QueryRunner queryRunner = builder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/SparkDeltaLake.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/SparkDeltaLake.java index 6ac5e396d839..9742c853c5ef 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/SparkDeltaLake.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/SparkDeltaLake.java @@ -14,8 +14,8 @@ package io.trino.plugin.deltalake; import io.trino.plugin.base.util.AutoCloseableCloser; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.testing.containers.Minio; import org.testcontainers.containers.GenericContainer; @@ -26,11 +26,11 @@ public final class SparkDeltaLake implements AutoCloseable { private final AutoCloseableCloser closer = AutoCloseableCloser.create(); - private final HiveMinioDataLake hiveMinio; + private final Hive3MinioDataLake hiveMinio; public SparkDeltaLake(String bucketName) { - hiveMinio = closer.register(new HiveMinioDataLake(bucketName)); + hiveMinio = closer.register(new Hive3MinioDataLake(bucketName)); hiveMinio.start(); closer.register(new GenericContainer<>("ghcr.io/trinodb/testing/spark3-delta:" + getDockerImagesVersion())) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeCreateTableStatistics.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeCreateTableStatistics.java index 1263d32ce54b..1bace334c40c 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeCreateTableStatistics.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeCreateTableStatistics.java @@ -18,7 +18,7 @@ import io.trino.plugin.deltalake.transactionlog.AddFileEntry; import io.trino.plugin.deltalake.transactionlog.TransactionLogAccess; import io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeFileStatistics; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.spi.type.DateType; import io.trino.spi.type.DecimalType; import io.trino.spi.type.DoubleType; @@ -68,7 +68,7 @@ protected QueryRunner createQueryRunner() throws Exception { this.bucketName = "delta-test-create-table-statistics-" + randomNameSuffix(); - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); return DeltaLakeQueryRunner.builder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDelete.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDelete.java index d616b3ec2045..f854110453ed 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDelete.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDelete.java @@ -14,7 +14,7 @@ package io.trino.plugin.deltalake; import com.google.common.collect.ImmutableSet; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.Test; @@ -31,13 +31,13 @@ public class TestDeltaLakeDelete extends AbstractTestQueryFramework { private final String bucketName = "test-delta-lake-connector-test-" + randomNameSuffix(); - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; @Override protected QueryRunner createQueryRunner() throws Exception { - hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); return DeltaLakeQueryRunner.builder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDynamicFiltering.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDynamicFiltering.java index ab566111784d..4bfe9a7cd8f6 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDynamicFiltering.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeDynamicFiltering.java @@ -22,7 +22,7 @@ import io.trino.metadata.QualifiedObjectName; import io.trino.metadata.Split; import io.trino.metadata.TableHandle; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.security.AllowAllAccessControl; import io.trino.spi.QueryId; import io.trino.spi.connector.ColumnHandle; @@ -60,14 +60,14 @@ public class TestDeltaLakeDynamicFiltering extends AbstractTestQueryFramework { private final String bucketName = "delta-lake-test-dynamic-filtering-" + randomNameSuffix(); - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; @Override protected QueryRunner createQueryRunner() throws Exception { verify(new DynamicFilterConfig().isEnableDynamicFiltering(), "this class assumes dynamic filtering is enabled by default"); - hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); QueryRunner queryRunner = DeltaLakeQueryRunner.builder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java index 61d73d098800..68531e7e93c2 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java @@ -14,7 +14,7 @@ package io.trino.plugin.deltalake; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; @@ -39,7 +39,7 @@ public class TestDeltaLakeFlushMetadataCacheProcedure protected QueryRunner createQueryRunner() throws Exception { - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName, HIVE3_IMAGE)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName, HIVE3_IMAGE)); hiveMinioDataLake.start(); metastore = new BridgingHiveMetastore( testingThriftHiveMetastoreBuilder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java index bef79413f6fd..31b8cd778d82 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java @@ -15,7 +15,7 @@ import com.google.common.collect.ImmutableMap; import io.trino.plugin.hive.TestingHivePlugin; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.AfterAll; @@ -35,14 +35,14 @@ public class TestDeltaLakeSharedHiveMetastoreWithViews extends AbstractTestQueryFramework { private final String bucketName = "delta-lake-shared-hive-with-views-" + randomNameSuffix(); - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; private String schema; @Override protected QueryRunner createQueryRunner() throws Exception { - this.hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + this.hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); this.hiveMinioDataLake.start(); QueryRunner queryRunner = DeltaLakeQueryRunner.builder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeUpdate.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeUpdate.java index 857f15b0c2c0..6c1cab6c0982 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeUpdate.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeUpdate.java @@ -13,7 +13,7 @@ */ package io.trino.plugin.deltalake; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.Test; @@ -35,7 +35,7 @@ public TestDeltaLakeUpdate() protected QueryRunner createQueryRunner() throws Exception { - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); return DeltaLakeQueryRunner.builder() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestPredicatePushdown.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestPredicatePushdown.java index 0d6bf3610430..e6b909b93aa0 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestPredicatePushdown.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestPredicatePushdown.java @@ -16,7 +16,7 @@ import com.google.common.collect.ContiguousSet; import io.trino.Session; import io.trino.operator.OperatorStats; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.spi.QueryId; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.MaterializedResult; @@ -48,13 +48,13 @@ public class TestPredicatePushdown */ private final TableResource testTable = new TableResource("custkey_15rowgroups"); - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; @Override protected QueryRunner createQueryRunner() throws Exception { - hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); return DeltaLakeQueryRunner.builder() diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java index 558a45059a5e..13b4b7682002 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java @@ -25,8 +25,8 @@ import io.trino.metastore.PartitionStatistics; import io.trino.metastore.PartitionWithStatistics; import io.trino.metastore.Table; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.s3.S3HiveQueryRunner; import io.trino.spi.connector.SchemaTableName; @@ -80,7 +80,7 @@ public class TestHive3OnDataLake private static final DataSize HIVE_S3_STREAMING_PART_SIZE = DataSize.of(5, MEGABYTE); private String bucketName; - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; private HiveMetastore metastoreClient; @Override @@ -89,7 +89,7 @@ protected QueryRunner createQueryRunner() { this.bucketName = "test-hive-insert-overwrite-" + randomNameSuffix(); this.hiveMinioDataLake = closeAfterClass( - new HiveMinioDataLake(bucketName, HiveHadoop.HIVE3_IMAGE)); + new Hive3MinioDataLake(bucketName, HiveHadoop.HIVE3_IMAGE)); this.hiveMinioDataLake.start(); this.metastoreClient = new BridgingHiveMetastore( testingThriftHiveMetastoreBuilder() diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java index f986576683df..0f5fc357ea67 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java @@ -14,7 +14,7 @@ package io.trino.plugin.hive; import io.airlift.units.Duration; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.s3.S3HiveQueryRunner; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; @@ -28,13 +28,13 @@ public class TestHiveAnalyzeCorruptStatistics extends AbstractTestQueryFramework { - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; @Override protected QueryRunner createQueryRunner() throws Exception { - hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake("test-analyze")); + hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake("test-analyze")); hiveMinioDataLake.start(); return S3HiveQueryRunner.builder(hiveMinioDataLake) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java index 35433d3610da..24cb40038a98 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java @@ -15,8 +15,8 @@ import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.PrincipalType; import io.trino.testing.BaseConnectorSmokeTest; @@ -49,7 +49,7 @@ protected QueryRunner createQueryRunner() throws Exception { String bucketName = "test-hive-metastore-catalog-smoke-test-" + randomNameSuffix(); - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName, HiveHadoop.HIVE3_IMAGE)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName, HiveHadoop.HIVE3_IMAGE)); hiveMinioDataLake.start(); // Inserting into metastore's database directly because the Hive does not expose a way to create a custom catalog diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveQueryFailureRecoveryTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveQueryFailureRecoveryTest.java index 54b05e04f833..fd36b81605ec 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveQueryFailureRecoveryTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveQueryFailureRecoveryTest.java @@ -17,7 +17,7 @@ import io.trino.operator.RetryPolicy; import io.trino.plugin.exchange.filesystem.FileSystemExchangePlugin; import io.trino.plugin.exchange.filesystem.containers.MinioStorage; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.s3.S3HiveQueryRunner; import io.trino.testing.QueryRunner; import io.trino.tpch.TpchTable; @@ -43,7 +43,7 @@ public TestHiveQueryFailureRecoveryTest() super(RetryPolicy.QUERY); } - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; private MinioStorage minioStorage; @Override @@ -55,7 +55,7 @@ protected QueryRunner createQueryRunner( throws Exception { String bucketName = "test-hive-insert-overwrite-" + randomNameSuffix(); // randomizing bucket name to ensure cached TrinoS3FileSystem objects are not reused - this.hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + this.hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); this.minioStorage = closeAfterClass(new MinioStorage("test-exchange-spooling-" + randomNameSuffix())); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveTaskFailureRecoveryTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveTaskFailureRecoveryTest.java index a74f29d305d8..cca5e7a009c0 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveTaskFailureRecoveryTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveTaskFailureRecoveryTest.java @@ -17,7 +17,7 @@ import io.trino.operator.RetryPolicy; import io.trino.plugin.exchange.filesystem.FileSystemExchangePlugin; import io.trino.plugin.exchange.filesystem.containers.MinioStorage; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.s3.S3HiveQueryRunner; import io.trino.testing.QueryRunner; import io.trino.tpch.TpchTable; @@ -43,7 +43,7 @@ public TestHiveTaskFailureRecoveryTest() super(RetryPolicy.TASK); } - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; private MinioStorage minioStorage; @Override @@ -55,7 +55,7 @@ protected QueryRunner createQueryRunner( throws Exception { String bucketName = "test-hive-insert-overwrite-" + randomNameSuffix(); // randomizing bucket name to ensure cached TrinoS3FileSystem objects are not reused - this.hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + this.hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); this.minioStorage = closeAfterClass(new MinioStorage("test-exchange-spooling-" + randomNameSuffix())); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive3MinioDataLake.java similarity index 94% rename from plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java rename to plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive3MinioDataLake.java index 0ce79c2ff6a6..0ec3034e08b5 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive3MinioDataLake.java @@ -29,7 +29,7 @@ import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.Network.newNetwork; -public class HiveMinioDataLake +public class Hive3MinioDataLake implements AutoCloseable { /** @@ -49,17 +49,17 @@ public class HiveMinioDataLake private State state = State.INITIAL; private MinioClient minioClient; - public HiveMinioDataLake(String bucketName) + public Hive3MinioDataLake(String bucketName) { this(bucketName, HiveHadoop.HIVE3_IMAGE); } - public HiveMinioDataLake(String bucketName, String hiveHadoopImage) + public Hive3MinioDataLake(String bucketName, String hiveHadoopImage) { this(bucketName, ImmutableMap.of("/etc/hadoop/conf/core-site.xml", getPathFromClassPathResource("hive_minio_datalake/hive-core-site.xml")), hiveHadoopImage); } - public HiveMinioDataLake(String bucketName, Map hiveHadoopFilesToMount, String hiveHadoopImage) + public Hive3MinioDataLake(String bucketName, Map hiveHadoopFilesToMount, String hiveHadoopImage) { this.bucketName = requireNonNull(bucketName, "bucketName is null"); network = closer.register(newNetwork()); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java index 1fa641e51955..cb6736f4b6bc 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java @@ -18,8 +18,8 @@ import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; import io.trino.plugin.hive.HiveQueryRunner; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.PrincipalType; import io.trino.testing.AbstractTestQueryFramework; @@ -52,7 +52,7 @@ protected QueryRunner createQueryRunner() throws Exception { this.bucketName = "test-hive-metastore-catalogs-" + randomNameSuffix(); - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName, HiveHadoop.HIVE3_IMAGE)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName, HiveHadoop.HIVE3_IMAGE)); hiveMinioDataLake.start(); QueryRunner queryRunner = HiveQueryRunner.builder() @@ -75,7 +75,7 @@ protected QueryRunner createQueryRunner() return queryRunner; } - private static Map buildHiveProperties(HiveMinioDataLake hiveMinioDataLake) + private static Map buildHiveProperties(Hive3MinioDataLake hiveMinioDataLake) { return ImmutableMap.builder() .put("hive.metastore", "thrift") diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java index 9df3a85934b3..450acfda9c5f 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java @@ -19,7 +19,7 @@ import io.airlift.log.Logging; import io.airlift.units.Duration; import io.trino.plugin.hive.HiveQueryRunner; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.TestingTokenAwareMetastoreClientFactory; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreConfig; @@ -47,7 +47,7 @@ public final class S3HiveQueryRunner private S3HiveQueryRunner() {} public static QueryRunner create( - HiveMinioDataLake hiveMinioDataLake, + Hive3MinioDataLake hiveMinioDataLake, Map additionalHiveProperties) throws Exception { @@ -56,7 +56,7 @@ public static QueryRunner create( .build(); } - public static Builder builder(HiveMinioDataLake hiveMinioDataLake) + public static Builder builder(Hive3MinioDataLake hiveMinioDataLake) { return builder() .setHiveMetastoreEndpoint(hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint()) @@ -173,7 +173,7 @@ public DistributedQueryRunner build() public static void main(String[] args) throws Exception { - HiveMinioDataLake hiveMinioDataLake = new HiveMinioDataLake("tpch"); + Hive3MinioDataLake hiveMinioDataLake = new Hive3MinioDataLake("tpch"); hiveMinioDataLake.start(); QueryRunner queryRunner = S3HiveQueryRunner.builder(hiveMinioDataLake) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/TestHiveS3MinioQueries.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/TestHiveS3MinioQueries.java index 8c0fbaf5b091..8c8d6c172a1a 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/TestHiveS3MinioQueries.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/TestHiveS3MinioQueries.java @@ -14,7 +14,7 @@ package io.trino.plugin.hive.s3; import com.google.common.collect.ImmutableMap; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.Test; @@ -31,7 +31,7 @@ public class TestHiveS3MinioQueries extends AbstractTestQueryFramework { - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; private String bucketName; @Override @@ -39,7 +39,7 @@ protected QueryRunner createQueryRunner() throws Exception { this.bucketName = "test-hive-minio-queries-" + randomNameSuffix(); - this.hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + this.hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); this.hiveMinioDataLake.start(); return S3HiveQueryRunner.builder(hiveMinioDataLake) diff --git a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java index 8f817c21fabf..915ca2b9ead2 100644 --- a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java +++ b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java @@ -20,7 +20,7 @@ import io.trino.filesystem.Location; import io.trino.metastore.Database; import io.trino.plugin.base.util.Closables; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hudi.testing.HudiTablesInitializer; import io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer; @@ -56,7 +56,7 @@ public static Builder builder() return new Builder("local:///"); } - public static Builder builder(HiveMinioDataLake hiveMinioDataLake) + public static Builder builder(Hive3MinioDataLake hiveMinioDataLake) { return new Builder("s3://" + hiveMinioDataLake.getBucketName() + "/") .addConnectorProperty("fs.hadoop.enabled", "false") @@ -157,7 +157,7 @@ public static void main(String[] args) Logging.initialize(); Logger log = Logger.get(HudiMinioQueryRunnerMain.class); - HiveMinioDataLake hiveMinioDataLake = new HiveMinioDataLake("test-bucket"); + Hive3MinioDataLake hiveMinioDataLake = new Hive3MinioDataLake("test-bucket"); hiveMinioDataLake.start(); QueryRunner queryRunner = builder(hiveMinioDataLake) .addCoordinatorProperty("http-server.http.port", "8080") diff --git a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java index 069fb05c0523..5fa1332073cd 100644 --- a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java +++ b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiMinioConnectorSmokeTest.java @@ -13,7 +13,7 @@ */ package io.trino.plugin.hudi; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hudi.testing.TpchHudiTablesInitializer; import io.trino.testing.QueryRunner; @@ -29,7 +29,7 @@ protected QueryRunner createQueryRunner() throws Exception { String bucketName = "test-hudi-connector-" + randomNameSuffix(); - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName, HIVE3_IMAGE)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName, HIVE3_IMAGE)); hiveMinioDataLake.start(); hiveMinioDataLake.getMinioClient().ensureBucketExists(bucketName); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java index 4fdeb620b748..03aa104bd33f 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java @@ -17,7 +17,7 @@ import io.minio.messages.Event; import io.trino.Session; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.testing.QueryRunner; import io.trino.testing.minio.MinioClient; @@ -50,7 +50,7 @@ public abstract class BaseIcebergMinioConnectorSmokeTest private final String schemaName; private final String bucketName; - private HiveMinioDataLake hiveMinioDataLake; + private Hive3MinioDataLake hiveMinioDataLake; protected BaseIcebergMinioConnectorSmokeTest(FileFormat format) { @@ -63,7 +63,7 @@ protected BaseIcebergMinioConnectorSmokeTest(FileFormat format) protected QueryRunner createQueryRunner() throws Exception { - this.hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + this.hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); this.hiveMinioDataLake.start(); return IcebergQueryRunner.builder() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java index 698658916e7e..ff7664af0ee0 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java @@ -21,8 +21,8 @@ import io.airlift.log.Logger; import io.airlift.log.Logging; import io.trino.plugin.exchange.filesystem.FileSystemExchangePlugin; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.iceberg.catalog.jdbc.TestingIcebergJdbcServer; import io.trino.plugin.iceberg.catalog.rest.TestingPolarisCatalog; import io.trino.plugin.iceberg.containers.NessieContainer; @@ -319,7 +319,7 @@ public static void main(String[] args) { String bucketName = "test-bucket"; @SuppressWarnings("resource") - HiveMinioDataLake hiveMinioDataLake = new HiveMinioDataLake(bucketName); + Hive3MinioDataLake hiveMinioDataLake = new Hive3MinioDataLake(bucketName); hiveMinioDataLake.start(); @SuppressWarnings("resource") diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index 587f63d2accf..56b1876bc779 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -33,7 +33,7 @@ import io.trino.metastore.TableInfo; import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.ThriftMetastore; @@ -89,7 +89,7 @@ public class TestTrinoHiveCatalogWithHiveMetastore private AutoCloseableCloser closer = AutoCloseableCloser.create(); // Use MinIO for storage, since HDFS is hard to get working in a unit test - private HiveMinioDataLake dataLake; + private Hive3MinioDataLake dataLake; private TrinoFileSystem fileSystem; private String bucketName; @@ -97,7 +97,7 @@ public class TestTrinoHiveCatalogWithHiveMetastore public void setUp() { bucketName = "test-hive-catalog-with-hms-" + randomNameSuffix(); - dataLake = closer.register(new HiveMinioDataLake(bucketName, HIVE3_IMAGE)); + dataLake = closer.register(new Hive3MinioDataLake(bucketName, HIVE3_IMAGE)); dataLake.start(); } diff --git a/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/delta/TestDeltaFaultTolerantExecutionTest.java b/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/delta/TestDeltaFaultTolerantExecutionTest.java index 0aeee0ac7346..bcffb9655fb2 100644 --- a/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/delta/TestDeltaFaultTolerantExecutionTest.java +++ b/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/delta/TestDeltaFaultTolerantExecutionTest.java @@ -17,7 +17,7 @@ import io.trino.plugin.deltalake.DeltaLakeQueryRunner; import io.trino.plugin.exchange.filesystem.FileSystemExchangePlugin; import io.trino.plugin.exchange.filesystem.containers.MinioStorage; -import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.testing.FaultTolerantExecutionConnectorTestHelper; import io.trino.testing.QueryRunner; @@ -38,7 +38,7 @@ public TestDeltaFaultTolerantExecutionTest() protected QueryRunner createQueryRunner() throws Exception { - HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new HiveMinioDataLake(bucketName)); + Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); hiveMinioDataLake.start(); MinioStorage minioStorage = closeAfterClass(new MinioStorage(bucketName)); minioStorage.start(); From 171f210ebc364de8951a5211f351b290e5d1b824 Mon Sep 17 00:00:00 2001 From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:27:34 -0500 Subject: [PATCH 024/158] Extract HiveMinioDataLake class --- ...tDeltaLakeFlushMetadataCacheProcedure.java | 2 +- ...DeltaLakeSharedHiveMetastoreWithViews.java | 6 +- ...stHiveCustomCatalogConnectorSmokeTest.java | 2 +- .../hive/containers/Hive3MinioDataLake.java | 108 ++------------ .../hive/containers/HiveMinioDataLake.java | 135 ++++++++++++++++++ .../thrift/TestHiveMetastoreCatalogs.java | 2 +- .../plugin/hive/s3/S3HiveQueryRunner.java | 2 +- .../BaseIcebergMinioConnectorSmokeTest.java | 6 +- ...TestTrinoHiveCatalogWithHiveMetastore.java | 2 +- 9 files changed, 159 insertions(+), 106 deletions(-) create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java index 68531e7e93c2..615c6ab50f81 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java @@ -43,7 +43,7 @@ protected QueryRunner createQueryRunner() hiveMinioDataLake.start(); metastore = new BridgingHiveMetastore( testingThriftHiveMetastoreBuilder() - .metastoreClient(hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint()) + .metastoreClient(hiveMinioDataLake.getHiveMetastoreEndpoint()) .build(this::closeAfterClass)); return DeltaLakeQueryRunner.builder("default") diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java index 31b8cd778d82..bcd9da3bd217 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java @@ -54,7 +54,7 @@ protected QueryRunner createQueryRunner() queryRunner.installPlugin(new TestingHivePlugin(queryRunner.getCoordinator().getBaseDataDir().resolve("hive_data"))); queryRunner.createCatalog("hive", "hive", ImmutableMap.builder() .put("hive.metastore", "thrift") - .put("hive.metastore.uri", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint().toString()) + .put("hive.metastore.uri", hiveMinioDataLake.getHiveMetastoreEndpoint().toString()) .put("fs.hadoop.enabled", "false") .put("fs.native-s3.enabled", "true") .put("s3.aws-access-key", MINIO_ACCESS_KEY) @@ -67,7 +67,7 @@ protected QueryRunner createQueryRunner() schema = queryRunner.getDefaultSession().getSchema().orElseThrow(); queryRunner.execute("CREATE TABLE hive." + schema + ".hive_table (a_integer integer)"); - hiveMinioDataLake.getHiveHadoop().runOnHive("CREATE VIEW " + schema + ".hive_view AS SELECT * FROM " + schema + ".hive_table"); + hiveMinioDataLake.runOnHive("CREATE VIEW " + schema + ".hive_view AS SELECT * FROM " + schema + ".hive_table"); queryRunner.execute("CREATE TABLE delta." + schema + ".delta_table (a_varchar varchar)"); return queryRunner; @@ -82,7 +82,7 @@ protected QueryRunner createQueryRunner() public void cleanup() { assertQuerySucceeds("DROP TABLE IF EXISTS hive." + schema + ".hive_table"); - hiveMinioDataLake.getHiveHadoop().runOnHive("DROP VIEW IF EXISTS " + schema + ".hive_view"); + hiveMinioDataLake.runOnHive("DROP VIEW IF EXISTS " + schema + ".hive_view"); assertQuerySucceeds("DROP TABLE IF EXISTS delta." + schema + ".delta_table"); assertQuerySucceeds("DROP SCHEMA IF EXISTS hive." + schema); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java index 24cb40038a98..6d55f9cd915d 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java @@ -57,7 +57,7 @@ protected QueryRunner createQueryRunner() QueryRunner queryRunner = HiveQueryRunner.builder() .addHiveProperty("hive.metastore", "thrift") - .addHiveProperty("hive.metastore.uri", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint().toString()) + .addHiveProperty("hive.metastore.uri", hiveMinioDataLake.getHiveMetastoreEndpoint().toString()) .addHiveProperty("hive.metastore.thrift.catalog-name", HIVE_CUSTOM_CATALOG) .addHiveProperty("fs.hadoop.enabled", "false") .addHiveProperty("fs.native-s3.enabled", "true") diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive3MinioDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive3MinioDataLake.java index 0ec3034e08b5..840e18d55bf9 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive3MinioDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive3MinioDataLake.java @@ -14,41 +14,18 @@ package io.trino.plugin.hive.containers; import com.google.common.collect.ImmutableMap; -import io.trino.plugin.base.util.AutoCloseableCloser; -import io.trino.testing.containers.Minio; -import io.trino.testing.minio.MinioClient; -import org.testcontainers.containers.Network; -import java.util.List; +import java.net.URI; import java.util.Map; -import static com.google.common.base.Preconditions.checkState; -import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; -import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; +import static io.trino.plugin.hive.containers.HiveMinioDataLake.State.STARTED; import static io.trino.testing.containers.TestContainers.getPathFromClassPathResource; -import static java.util.Objects.requireNonNull; -import static org.testcontainers.containers.Network.newNetwork; public class Hive3MinioDataLake - implements AutoCloseable + extends HiveMinioDataLake { - /** - * In S3 this region is implicitly the default one. In Minio, however, - * if we set an empty region, it will accept any. - * So setting it by default to `us-east-1` simulates S3 better - */ - public static final String MINIO_DEFAULT_REGION = "us-east-1"; - - private final String bucketName; - private final Minio minio; private final HiveHadoop hiveHadoop; - private final AutoCloseableCloser closer = AutoCloseableCloser.create(); - private final Network network; - - private State state = State.INITIAL; - private MinioClient minioClient; - public Hive3MinioDataLake(String bucketName) { this(bucketName, HiveHadoop.HIVE3_IMAGE); @@ -61,18 +38,7 @@ public Hive3MinioDataLake(String bucketName, String hiveHadoopImage) public Hive3MinioDataLake(String bucketName, Map hiveHadoopFilesToMount, String hiveHadoopImage) { - this.bucketName = requireNonNull(bucketName, "bucketName is null"); - network = closer.register(newNetwork()); - this.minio = closer.register( - Minio.builder() - .withNetwork(network) - .withEnvVars(ImmutableMap.builder() - .put("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY) - .put("MINIO_SECRET_KEY", MINIO_SECRET_KEY) - .put("MINIO_REGION", MINIO_DEFAULT_REGION) - .buildOrThrow()) - .build()); - + super(bucketName); HiveHadoop.Builder hiveHadoopBuilder = HiveHadoop.builder() .withImage(hiveHadoopImage) .withNetwork(network) @@ -80,77 +46,29 @@ public Hive3MinioDataLake(String bucketName, Map hiveHadoopFiles this.hiveHadoop = closer.register(hiveHadoopBuilder.build()); } + @Override public void start() { - checkState(state == State.INITIAL, "Already started: %s", state); - state = State.STARTING; - minio.start(); + super.start(); hiveHadoop.start(); - minioClient = closer.register(minio.createMinioClient()); - minio.createBucket(bucketName); - state = State.STARTED; - } - - public void stop() - throws Exception - { - closer.close(); - state = State.STOPPED; + state = STARTED; } - public Network getNetwork() - { - return network; - } - - public MinioClient getMinioClient() - { - checkState(state == State.STARTED, "Can't provide client when MinIO state is: %s", state); - return minioClient; - } - - public void copyResources(String resourcePath, String target) - { - minio.copyResources(resourcePath, bucketName, target); - } - - public void writeFile(byte[] contents, String target) - { - minio.writeFile(contents, bucketName, target); - } - - public List listFiles(String targetDirectory) - { - return getMinioClient().listObjects(getBucketName(), targetDirectory); - } - - public Minio getMinio() + @Override + public String runOnHive(String sql) { - return minio; + return hiveHadoop.runOnHive(sql); } + @Override public HiveHadoop getHiveHadoop() { return hiveHadoop; } - public String getBucketName() - { - return bucketName; - } - @Override - public void close() - throws Exception - { - stop(); - } - - private enum State + public URI getHiveMetastoreEndpoint() { - INITIAL, - STARTING, - STARTED, - STOPPED, + return hiveHadoop.getHiveMetastoreEndpoint(); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java new file mode 100644 index 000000000000..19bbc35d32a4 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.containers; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.base.util.AutoCloseableCloser; +import io.trino.testing.containers.Minio; +import io.trino.testing.minio.MinioClient; +import org.testcontainers.containers.Network; + +import java.net.URI; +import java.util.List; + +import static com.google.common.base.Preconditions.checkState; +import static io.trino.plugin.hive.containers.HiveMinioDataLake.State.INITIAL; +import static io.trino.plugin.hive.containers.HiveMinioDataLake.State.STARTED; +import static io.trino.plugin.hive.containers.HiveMinioDataLake.State.STARTING; +import static io.trino.plugin.hive.containers.HiveMinioDataLake.State.STOPPED; +import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; +import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.containers.Network.newNetwork; + +public abstract class HiveMinioDataLake + implements AutoCloseable +{ + private static final String MINIO_DEFAULT_REGION = "us-east-1"; + + private final String bucketName; + private final Minio minio; + private MinioClient minioClient; + + protected final AutoCloseableCloser closer = AutoCloseableCloser.create(); + protected final Network network; + protected State state = INITIAL; + + public HiveMinioDataLake(String bucketName) + { + this.bucketName = requireNonNull(bucketName, "bucketName is null"); + this.network = closer.register(newNetwork()); + this.minio = closer.register( + Minio.builder() + .withNetwork(network) + .withEnvVars(ImmutableMap.builder() + .put("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY) + .put("MINIO_SECRET_KEY", MINIO_SECRET_KEY) + .put("MINIO_REGION", MINIO_DEFAULT_REGION) + .buildOrThrow()) + .build()); + } + + public void start() + { + checkState(state == INITIAL, "Already started: %s", state); + state = STARTING; + minio.start(); + minioClient = closer.register(minio.createMinioClient()); + minio.createBucket(bucketName); + } + + public void stop() + throws Exception + { + closer.close(); + state = STOPPED; + } + + public Network getNetwork() + { + return network; + } + + public MinioClient getMinioClient() + { + checkState(state == STARTED, "Can't provide client when MinIO state is: %s", state); + return minioClient; + } + + public void copyResources(String resourcePath, String target) + { + minio.copyResources(resourcePath, bucketName, target); + } + + public void writeFile(byte[] contents, String target) + { + minio.writeFile(contents, bucketName, target); + } + + public List listFiles(String targetDirectory) + { + return getMinioClient().listObjects(getBucketName(), targetDirectory); + } + + public Minio getMinio() + { + return minio; + } + + public abstract String runOnHive(String sql); + + public abstract HiveHadoop getHiveHadoop(); + + public abstract URI getHiveMetastoreEndpoint(); + + public String getBucketName() + { + return bucketName; + } + + @Override + public void close() + throws Exception + { + stop(); + } + + protected enum State + { + INITIAL, + STARTING, + STARTED, + STOPPED, + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java index cb6736f4b6bc..23a18de4e7e5 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java @@ -79,7 +79,7 @@ private static Map buildHiveProperties(Hive3MinioDataLake hiveMi { return ImmutableMap.builder() .put("hive.metastore", "thrift") - .put("hive.metastore.uri", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint().toString()) + .put("hive.metastore.uri", hiveMinioDataLake.getHiveMetastoreEndpoint().toString()) .put("fs.hadoop.enabled", "false") .put("fs.native-s3.enabled", "true") .put("s3.path-style-access", "true") diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java index 450acfda9c5f..17724fc599ab 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java @@ -59,7 +59,7 @@ public static QueryRunner create( public static Builder builder(Hive3MinioDataLake hiveMinioDataLake) { return builder() - .setHiveMetastoreEndpoint(hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint()) + .setHiveMetastoreEndpoint(hiveMinioDataLake.getHiveMetastoreEndpoint()) .setS3Endpoint("http://" + hiveMinioDataLake.getMinio().getMinioApiEndpoint()) .setS3Region(MINIO_REGION) .setS3AccessKey(MINIO_ACCESS_KEY) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java index 03aa104bd33f..3144bd56bb02 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java @@ -71,7 +71,7 @@ protected QueryRunner createQueryRunner() ImmutableMap.builder() .put("iceberg.file-format", format.name()) .put("iceberg.catalog.type", "HIVE_METASTORE") - .put("hive.metastore.uri", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint().toString()) + .put("hive.metastore.uri", hiveMinioDataLake.getHiveMetastoreEndpoint().toString()) .put("hive.metastore.thrift.client.read-timeout", "1m") // read timed out sometimes happens with the default timeout .put("fs.hadoop.enabled", "false") .put("fs.native-s3.enabled", "true") @@ -258,7 +258,7 @@ protected void dropTableFromMetastore(String tableName) { HiveMetastore metastore = new BridgingHiveMetastore( testingThriftHiveMetastoreBuilder() - .metastoreClient(hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint()) + .metastoreClient(hiveMinioDataLake.getHiveMetastoreEndpoint()) .build(this::closeAfterClass)); metastore.dropTable(schemaName, tableName, false); assertThat(metastore.getTable(schemaName, tableName)).isEmpty(); @@ -269,7 +269,7 @@ protected String getMetadataLocation(String tableName) { HiveMetastore metastore = new BridgingHiveMetastore( testingThriftHiveMetastoreBuilder() - .metastoreClient(hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint()) + .metastoreClient(hiveMinioDataLake.getHiveMetastoreEndpoint()) .build(this::closeAfterClass)); return metastore .getTable(schemaName, tableName).orElseThrow() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index 56b1876bc779..3085162c4777 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -130,7 +130,7 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) .thriftMetastoreConfig(new ThriftMetastoreConfig() // Read timed out sometimes happens with the default timeout .setReadTimeout(new Duration(1, MINUTES))) - .metastoreClient(dataLake.getHiveHadoop().getHiveMetastoreEndpoint()) + .metastoreClient(dataLake.getHiveMetastoreEndpoint()) .build(closer::register); CachingHiveMetastore metastore = createPerTransactionCache(new BridgingHiveMetastore(thriftMetastore), 1000); fileSystem = fileSystemFactory.create(SESSION); From 3cb0e60f42dee7e3ffc4b3d27aff2c53aec7bf8f Mon Sep 17 00:00:00 2001 From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:58:04 -0500 Subject: [PATCH 025/158] Extract BaseTestHiveOnDataLake to reuse it across Hive3/4 test --- .../{TestHive3OnDataLake.java => BaseTestHiveOnDataLake.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename plugin/trino-hive/src/test/java/io/trino/plugin/hive/{TestHive3OnDataLake.java => BaseTestHiveOnDataLake.java} (99%) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java similarity index 99% rename from plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java rename to plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java index 13b4b7682002..0874e9ee7a13 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java @@ -73,7 +73,7 @@ import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @TestInstance(PER_CLASS) -public class TestHive3OnDataLake +public class BaseTestHiveOnDataLake extends AbstractTestQueryFramework { private static final String HIVE_TEST_SCHEMA = "hive_datalake"; From 125472b9ddfe7b26eaace503b299a795591cd97d Mon Sep 17 00:00:00 2001 From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:51:24 -0500 Subject: [PATCH 026/158] Add TestHive3OnDataLake test --- .../plugin/hive/BaseTestHiveOnDataLake.java | 75 +++++++++++-------- .../plugin/hive/TestHive3OnDataLake.java | 33 ++++++++ .../plugin/hive/s3/S3HiveQueryRunner.java | 3 +- 3 files changed, 78 insertions(+), 33 deletions(-) create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java index 0874e9ee7a13..59df61051c61 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java @@ -25,8 +25,7 @@ import io.trino.metastore.PartitionStatistics; import io.trino.metastore.PartitionWithStatistics; import io.trino.metastore.Table; -import io.trino.plugin.hive.containers.Hive3MinioDataLake; -import io.trino.plugin.hive.containers.HiveHadoop; +import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.s3.S3HiveQueryRunner; import io.trino.spi.connector.SchemaTableName; @@ -36,6 +35,7 @@ import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import io.trino.testing.minio.MinioClient; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -73,27 +73,31 @@ import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @TestInstance(PER_CLASS) -public class BaseTestHiveOnDataLake +abstract class BaseTestHiveOnDataLake extends AbstractTestQueryFramework { private static final String HIVE_TEST_SCHEMA = "hive_datalake"; private static final DataSize HIVE_S3_STREAMING_PART_SIZE = DataSize.of(5, MEGABYTE); - private String bucketName; - private Hive3MinioDataLake hiveMinioDataLake; + private final HiveMinioDataLake hiveMinioDataLake; + private final String bucketName; + private HiveMetastore metastoreClient; + public BaseTestHiveOnDataLake(String bucketName, HiveMinioDataLake hiveMinioDataLake) + { + this.bucketName = bucketName; + this.hiveMinioDataLake = hiveMinioDataLake; + } + @Override protected QueryRunner createQueryRunner() throws Exception { - this.bucketName = "test-hive-insert-overwrite-" + randomNameSuffix(); - this.hiveMinioDataLake = closeAfterClass( - new Hive3MinioDataLake(bucketName, HiveHadoop.HIVE3_IMAGE)); this.hiveMinioDataLake.start(); this.metastoreClient = new BridgingHiveMetastore( testingThriftHiveMetastoreBuilder() - .metastoreClient(this.hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint()) + .metastoreClient(hiveMinioDataLake.getHiveMetastoreEndpoint()) .build(this::closeAfterClass)); return S3HiveQueryRunner.builder(hiveMinioDataLake) .addExtraProperty("sql.path", "hive.functions") @@ -124,6 +128,13 @@ public void setUp() computeActual("CREATE SCHEMA hive.functions"); } + @AfterAll + public void destroy() + throws Exception + { + hiveMinioDataLake.close(); + } + @Test public void testInsertOverwriteInTransaction() { @@ -456,7 +467,7 @@ public void testEnumPartitionProjectionOnVarcharColumnWithWhitespace() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short name\\.type[ |]+enum[ |]+") @@ -533,7 +544,7 @@ public void testEnumPartitionProjectionOnVarcharColumnWithStorageLocationTemplat " partition_projection_location_template='" + storageFormat + "' " + ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(schemaName, tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+storage\\.location\\.template[ |]+" + quote(storageFormat) + "[ |]+") @@ -553,7 +564,7 @@ public void testEnumPartitionProjectionOnVarcharColumnWithStorageLocationTemplat this.bucketName, HIVE_TEST_SCHEMA, tableName); - hiveMinioDataLake.getHiveHadoop().runOnHive( + hiveMinioDataLake.runOnHive( "CREATE TABLE " + getHiveTestTableName(tableName) + " ( " + " name varchar(25), " + " comment varchar(152), " + @@ -628,7 +639,7 @@ public void testEnumPartitionProjectionOnVarcharColumn() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -685,7 +696,7 @@ public void testIntegerPartitionProjectionOnVarcharColumnWithDigitsAlignCreatedO " partition_projection_enabled=true " + ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -700,7 +711,7 @@ public void testIntegerPartitionProjectionOnVarcharColumnWithDigitsAlignCreatedO public void testIntegerPartitionProjectionOnVarcharColumnWithDigitsAlignCreatedOnHive() { String tableName = "nation_" + randomNameSuffix(); - hiveMinioDataLake.getHiveHadoop().runOnHive( + hiveMinioDataLake.runOnHive( "CREATE TABLE " + getHiveTestTableName(tableName) + " ( " + " name varchar(25), " + " comment varchar(152), " + @@ -783,7 +794,7 @@ public void testIntegerPartitionProjectionOnIntegerColumnWithInterval() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -843,7 +854,7 @@ public void testIntegerPartitionProjectionOnIntegerColumnWithDefaults() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -903,7 +914,7 @@ public void testDatePartitionProjectionOnDateColumnWithDefaults() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -980,7 +991,7 @@ public void testDatePartitionProjectionOnTimestampColumnWithInterval() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -1054,7 +1065,7 @@ public void testDatePartitionProjectionOnTimestampColumnWithIntervalExpressionCr " partition_projection_enabled=true " + ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -1071,7 +1082,7 @@ public void testDatePartitionProjectionOnTimestampColumnWithIntervalExpressionCr { String tableName = getRandomTestTableName(); String dateProjectionFormat = "yyyy-MM-dd HH:mm:ss"; - hiveMinioDataLake.getHiveHadoop().runOnHive( + hiveMinioDataLake.runOnHive( "CREATE TABLE " + getHiveTestTableName(tableName) + " ( " + " name varchar(25), " + " comment varchar(152), " + @@ -1153,7 +1164,7 @@ public void testDatePartitionProjectionOnVarcharColumnWithHoursInterval() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -1227,7 +1238,7 @@ public void testDatePartitionProjectionOnVarcharColumnWithDaysInterval() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -1300,7 +1311,7 @@ public void testDatePartitionProjectionOnVarcharColumnWithIntervalExpression() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -1404,7 +1415,7 @@ public void testInjectedPartitionProjectionOnVarcharColumn() ")"); assertThat( - hiveMinioDataLake.getHiveHadoop() + hiveMinioDataLake .runOnHive("SHOW TBLPROPERTIES " + getHiveTestTableName(tableName))) .containsPattern("[ |]+projection\\.enabled[ |]+true[ |]+") .containsPattern("[ |]+projection\\.short_name1\\.type[ |]+enum[ |]+") @@ -1643,7 +1654,7 @@ public void testPartitionProjectionIgnore() String fullyQualifiedTestTableName = getFullyQualifiedTestTableName(tableName); // Create corrupted configuration - hiveMinioDataLake.getHiveHadoop().runOnHive( + hiveMinioDataLake.runOnHive( "CREATE TABLE " + hiveTestTableName + " ( " + " name varchar(25) " + ") PARTITIONED BY (" + @@ -1662,7 +1673,7 @@ public void testPartitionProjectionIgnore() "or '^\\s*NOW\\s*(([+-])\\s*([0-9]+)\\s*(DAY|HOUR|MINUTE|SECOND)S?\\s*)?$' that are sequential: Unparseable date: \"2001-01-01\""); // Append kill switch table property to ignore Partition Projection properties - hiveMinioDataLake.getHiveHadoop().runOnHive( + hiveMinioDataLake.runOnHive( "ALTER TABLE " + hiveTestTableName + " SET TBLPROPERTIES ( 'trino.partition_projection.ignore'='TRUE' )"); // Flush cache to get new definition computeActual("CALL system.flush_metadata_cache(schema_name => '" + HIVE_TEST_SCHEMA + "', table_name => '" + tableName + "')"); @@ -1966,9 +1977,9 @@ public void testUnsupportedDropSchemaCascadeWithNonHiveTable() String schemaName = "test_unsupported_drop_schema_cascade_" + randomNameSuffix(); String icebergTableName = "test_dummy_iceberg_table" + randomNameSuffix(); - hiveMinioDataLake.getHiveHadoop().runOnHive("CREATE DATABASE %2$s LOCATION 's3a://%1$s/%2$s'".formatted(bucketName, schemaName)); + hiveMinioDataLake.runOnHive("CREATE DATABASE %2$s LOCATION 's3a://%1$s/%2$s'".formatted(bucketName, schemaName)); try { - hiveMinioDataLake.getHiveHadoop().runOnHive("CREATE TABLE " + schemaName + "." + icebergTableName + " TBLPROPERTIES ('table_type'='iceberg') AS SELECT 1 a"); + hiveMinioDataLake.runOnHive("CREATE TABLE " + schemaName + "." + icebergTableName + " TBLPROPERTIES ('table_type'='iceberg') AS SELECT 1 a"); assertQueryFails("DROP SCHEMA " + schemaName + " CASCADE", "\\QCannot query Iceberg table '%s.%s'".formatted(schemaName, icebergTableName)); @@ -1977,7 +1988,7 @@ public void testUnsupportedDropSchemaCascadeWithNonHiveTable() assertThat(hiveMinioDataLake.getMinioClient().listObjects(bucketName, schemaName).stream()).isNotEmpty(); } finally { - hiveMinioDataLake.getHiveHadoop().runOnHive("DROP DATABASE IF EXISTS " + schemaName + " CASCADE"); + hiveMinioDataLake.runOnHive("DROP DATABASE IF EXISTS " + schemaName + " CASCADE"); } } @@ -1986,12 +1997,12 @@ public void testUnsupportedCommentOnHiveView() { String viewName = HIVE_TEST_SCHEMA + ".test_unsupported_comment_on_hive_view_" + randomNameSuffix(); - hiveMinioDataLake.getHiveHadoop().runOnHive("CREATE VIEW " + viewName + " AS SELECT 1 x"); + hiveMinioDataLake.runOnHive("CREATE VIEW " + viewName + " AS SELECT 1 x"); try { assertQueryFails("COMMENT ON COLUMN " + viewName + ".x IS NULL", "Hive views are not supported.*"); } finally { - hiveMinioDataLake.getHiveHadoop().runOnHive("DROP VIEW " + viewName); + hiveMinioDataLake.runOnHive("DROP VIEW " + viewName); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java new file mode 100644 index 000000000000..75efa817c54b --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive3OnDataLake.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive; + +import io.trino.plugin.hive.containers.Hive3MinioDataLake; +import io.trino.plugin.hive.containers.HiveHadoop; +import org.junit.jupiter.api.TestInstance; + +import static io.trino.testing.TestingNames.randomNameSuffix; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +class TestHive3OnDataLake + extends BaseTestHiveOnDataLake +{ + private static final String BUCKET_NAME = "test-hive-insert-overwrite-" + randomNameSuffix(); + + public TestHive3OnDataLake() + { + super(BUCKET_NAME, new Hive3MinioDataLake(BUCKET_NAME, HiveHadoop.HIVE3_IMAGE)); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java index 17724fc599ab..7149cc189110 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java @@ -20,6 +20,7 @@ import io.airlift.units.Duration; import io.trino.plugin.hive.HiveQueryRunner; import io.trino.plugin.hive.containers.Hive3MinioDataLake; +import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.TestingTokenAwareMetastoreClientFactory; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreConfig; @@ -56,7 +57,7 @@ public static QueryRunner create( .build(); } - public static Builder builder(Hive3MinioDataLake hiveMinioDataLake) + public static Builder builder(HiveMinioDataLake hiveMinioDataLake) { return builder() .setHiveMetastoreEndpoint(hiveMinioDataLake.getHiveMetastoreEndpoint()) From 72f47961e9ceeb06319f6bb27d9ed4e8504f50b3 Mon Sep 17 00:00:00 2001 From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:20:17 -0500 Subject: [PATCH 027/158] Add S3 Hive4 query runner --- .../hive/containers/Hive4HiveServer.java | 95 ++++++++++++++++++ .../hive/containers/Hive4Metastore.java | 96 +++++++++++++++++++ .../hive/containers/Hive4MinioDataLake.java | 92 ++++++++++++++++++ .../plugin/hive/s3/S3HiveQueryRunner.java | 21 ++++ .../hive_minio_datalake/hive4-hive-site.xml | 68 +++++++++++++ 5 files changed, 372 insertions(+) create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4HiveServer.java create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4Metastore.java create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4MinioDataLake.java create mode 100644 plugin/trino-hive/src/test/resources/hive_minio_datalake/hive4-hive-site.xml diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4HiveServer.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4HiveServer.java new file mode 100644 index 000000000000..96af084ca48c --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4HiveServer.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.containers; + +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import io.trino.testing.containers.BaseTestContainer; +import org.testcontainers.containers.Network; + +import java.net.URI; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.trino.plugin.hive.containers.Hive4Metastore.HIVE4_IMAGE; + +public class Hive4HiveServer + extends BaseTestContainer +{ + public static final int HIVE_SERVER_PORT = 10000; + + private static final Logger log = Logger.get(Hive4HiveServer.class); + private static final String HOST_NAME = "hiveserver2"; + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + extends BaseTestContainer.Builder + { + private Builder() + { + this.image = HIVE4_IMAGE; + this.hostName = HOST_NAME; + this.exposePorts = ImmutableSet.of(HIVE_SERVER_PORT); + } + + @Override + public Hive4HiveServer build() + { + return new Hive4HiveServer(image, hostName, exposePorts, filesToMount, envVars, network, startupRetryLimit); + } + } + + private Hive4HiveServer( + String image, + String hostName, + Set ports, + Map filesToMount, + Map envVars, + Optional network, + int startupRetryLimit) + { + super( + image, + hostName, + ports, + filesToMount, + envVars, + network, + startupRetryLimit); + } + + @Override + public void start() + { + super.start(); + log.info("Hive container started with addresses for hive server: %s", getHiveServerEndpoint()); + } + + public String runOnHive(String query) + { + return executeInContainerFailOnError("beeline", "-u", "jdbc:hive2://localhost:%s/default".formatted(HIVE_SERVER_PORT), "-n", "hive", "-e", query); + } + + public URI getHiveServerEndpoint() + { + HostAndPort address = getMappedHostAndPortForExposedPort(HIVE_SERVER_PORT); + return URI.create(address.getHost() + ":" + address.getPort()); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4Metastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4Metastore.java new file mode 100644 index 000000000000..c7d01ca36345 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4Metastore.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.containers; + +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import io.trino.testing.containers.BaseTestContainer; +import org.testcontainers.containers.Network; + +import java.net.URI; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.trino.testing.TestingProperties.getDockerImagesVersion; + +public class Hive4Metastore + extends BaseTestContainer +{ + public static final String HIVE4_IMAGE = "ghcr.io/trinodb/testing/hive4.0-hive:" + getDockerImagesVersion(); + public static final int HIVE_METASTORE_PORT = 9083; + + private static final Logger log = Logger.get(HiveHadoop.class); + private static final String HOST_NAME = "metastore"; + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + extends BaseTestContainer.Builder + { + private Builder() + { + this.image = HIVE4_IMAGE; + this.hostName = HOST_NAME; + this.exposePorts = ImmutableSet.of(HIVE_METASTORE_PORT); + } + + @Override + public Hive4Metastore build() + { + return new Hive4Metastore(image, hostName, exposePorts, filesToMount, envVars, network, startupRetryLimit); + } + } + + private Hive4Metastore( + String image, + String hostName, + Set ports, + Map filesToMount, + Map envVars, + Optional network, + int startupRetryLimit) + { + super( + image, + hostName, + ports, + filesToMount, + envVars, + network, + startupRetryLimit); + } + + @Override + public void start() + { + super.start(); + log.info("Hive container started with addresses for metastore: %s", getHiveMetastoreEndpoint()); + } + + public URI getHiveMetastoreEndpoint() + { + HostAndPort address = getMappedHostAndPortForExposedPort(HIVE_METASTORE_PORT); + return URI.create("thrift://" + address.getHost() + ":" + address.getPort()); + } + + public URI getInternalHiveMetastoreEndpoint() + { + return URI.create("thrift://" + HOST_NAME + ":" + HIVE_METASTORE_PORT); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4MinioDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4MinioDataLake.java new file mode 100644 index 000000000000..e58ea118889b --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/Hive4MinioDataLake.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.containers; + +import com.google.common.collect.ImmutableMap; + +import java.net.URI; +import java.util.Map; +import java.util.Set; + +import static io.trino.plugin.hive.containers.Hive4HiveServer.HIVE_SERVER_PORT; +import static io.trino.plugin.hive.containers.Hive4Metastore.HIVE4_IMAGE; +import static io.trino.plugin.hive.containers.HiveMinioDataLake.State.STARTED; +import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; +import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; +import static io.trino.testing.containers.TestContainers.getPathFromClassPathResource; + +public class Hive4MinioDataLake + extends HiveMinioDataLake +{ + private final Hive4HiveServer hiveServer; + private final Hive4Metastore hiveMetastore; + + public Hive4MinioDataLake(String bucketName) + { + super(bucketName); + String hiveImage = HIVE4_IMAGE; + Map hiveFilesToMount = ImmutableMap.of("/opt/hive/conf/hive-site.xml", getPathFromClassPathResource("hive_minio_datalake/hive4-hive-site.xml")); + // Separate hms and hiveserver(below) is created as standalone hiveserver doesn't expose embedded hms. https://github.com/apache/hive/blob/a1420ed816c315d98be7ebf05cdc3ba139a68643/packaging/src/docker/README.md?plain=1#L46. + // Run standalone metastore https://github.com/apache/hive/blob/a1420ed816c315d98be7ebf05cdc3ba139a68643/packaging/src/docker/README.md?plain=1#L105 + Hive4Metastore.Builder metastorebuilder = Hive4Metastore.builder() + .withImage(hiveImage) + .withEnvVars(Map.of("SERVICE_NAME", "metastore")) + .withNetwork(network) + .withExposePorts(Set.of(Hive4Metastore.HIVE_METASTORE_PORT)) + .withFilesToMount(hiveFilesToMount); + this.hiveMetastore = closer.register(metastorebuilder.build()); + + // Run hive server connecting to remote(above) metastore https://github.com/apache/hive/blob/a1420ed816c315d98be7ebf05cdc3ba139a68643/packaging/src/docker/README.md?plain=1#L139-L143 + Hive4HiveServer.Builder hiveHadoopBuilder = Hive4HiveServer.builder() + .withImage(hiveImage) + .withEnvVars(Map.of( + "SERVICE_NAME", "hiveserver2", + "HIVE_SERVER2_THRIFT_PORT", String.valueOf(HIVE_SERVER_PORT), + "SERVICE_OPTS", "-Xmx1G -Dhive.metastore.uris=%s".formatted(hiveMetastore.getInternalHiveMetastoreEndpoint()), + "IS_RESUME", "true", + "AWS_ACCESS_KEY_ID", MINIO_ACCESS_KEY, + "AWS_SECRET_KEY", MINIO_SECRET_KEY)) + .withNetwork(network) + .withExposePorts(Set.of(HIVE_SERVER_PORT)) + .withFilesToMount(hiveFilesToMount); + this.hiveServer = closer.register(hiveHadoopBuilder.build()); + } + + @Override + public void start() + { + super.start(); + hiveMetastore.start(); + hiveServer.start(); + state = STARTED; + } + + @Override + public String runOnHive(String sql) + { + return hiveServer.runOnHive(sql); + } + + @Override + public HiveHadoop getHiveHadoop() + { + throw new UnsupportedOperationException(); + } + + @Override + public URI getHiveMetastoreEndpoint() + { + return hiveMetastore.getHiveMetastoreEndpoint(); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java index 7149cc189110..ea246e127f5d 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java @@ -20,6 +20,7 @@ import io.airlift.units.Duration; import io.trino.plugin.hive.HiveQueryRunner; import io.trino.plugin.hive.containers.Hive3MinioDataLake; +import io.trino.plugin.hive.containers.Hive4MinioDataLake; import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.TestingTokenAwareMetastoreClientFactory; @@ -187,4 +188,24 @@ public static void main(String[] args) log.info("======== SERVER STARTED ========"); log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); } + + public static class S3Hive4QueryRunner + { + public static void main(String[] args) + throws Exception + { + Hive4MinioDataLake hiveMinioDataLake = new Hive4MinioDataLake("tpch"); + hiveMinioDataLake.start(); + + QueryRunner queryRunner = S3HiveQueryRunner.builder(hiveMinioDataLake) + .addCoordinatorProperty("http-server.http.port", "8080") + .setHiveProperties(ImmutableMap.of("hive.security", "allow-all")) + .setSkipTimezoneSetup(true) + .setInitialTables(TpchTable.getTables()) + .build(); + Logger log = Logger.get(S3Hive4QueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } + } } diff --git a/plugin/trino-hive/src/test/resources/hive_minio_datalake/hive4-hive-site.xml b/plugin/trino-hive/src/test/resources/hive_minio_datalake/hive4-hive-site.xml new file mode 100644 index 000000000000..eec52b5f6ac5 --- /dev/null +++ b/plugin/trino-hive/src/test/resources/hive_minio_datalake/hive4-hive-site.xml @@ -0,0 +1,68 @@ + + + + hive.server2.enable.doAs + false + + + hive.tez.exec.inplace.progress + false + + + hive.exec.scratchdir + /opt/hive/scratch_dir + + + hive.user.install.directory + /opt/hive/install_dir + + + tez.runtime.optimize.local.fetch + true + + + hive.exec.submit.local.task.via.child + false + + + mapreduce.framework.name + local + + + hive.metastore.warehouse.dir + /opt/hive/data/warehouse + + + metastore.metastore.event.db.notification.api.auth + false + + + + + hive.users.in.admin.role + hive + + + + + fs.s3a.access.key + accesskey + + + fs.s3a.secret.key + secretkey + + + fs.s3a.endpoint + http://minio:4566 + + + fs.s3a.path.style.access + true + + + fs.s3.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + From 6590bfc72800003c74c7f668c6e1d82c28bb51f2 Mon Sep 17 00:00:00 2001 From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:20:40 -0500 Subject: [PATCH 028/158] Add TestHive4OnDataLake test --- .../plugin/hive/TestHive4OnDataLake.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive4OnDataLake.java diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive4OnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive4OnDataLake.java new file mode 100644 index 000000000000..379de81fcc91 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive4OnDataLake.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive; + +import io.trino.plugin.hive.containers.Hive4MinioDataLake; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +import static io.trino.testing.TestingNames.randomNameSuffix; +import static org.junit.jupiter.api.Assumptions.abort; + +@Execution(ExecutionMode.SAME_THREAD) // TODO Make custom hive4 image to support running queries concurrently +class TestHive4OnDataLake + extends BaseTestHiveOnDataLake +{ + private static final String BUCKET_NAME = "test-hive-insert-overwrite-" + randomNameSuffix(); + + public TestHive4OnDataLake() + { + super(BUCKET_NAME, new Hive4MinioDataLake(BUCKET_NAME)); + } + + @Override + @Test + public void testSyncPartitionOnBucketRoot() + { + // https://github.com/trinodb/trino/issues/24453 + abort("Fails with `location must not be root path`"); + } + + @Override + @Test + public void testUnpartitionedTableExternalLocationOnTopOfTheBucket() + { + // https://github.com/trinodb/trino/issues/24453 + abort("Fails with `location must not be root path`"); + } + + @Override + @Test + public void testPartitionedTableExternalLocationOnTopOfTheBucket() + { + // https://github.com/trinodb/trino/issues/24453 + abort("Fails with `location must not be root path`"); + } + + @Override + @Test + public void testInsertOverwritePartitionedAndBucketedAcidTable() + { + // https://github.com/trinodb/trino/issues/24454 + abort("Fails with `Processor has no capabilities, cannot create an ACID table`"); + } +} From 1bd8b3aba4695c2be0d80e0701de81105d906d10 Mon Sep 17 00:00:00 2001 From: Mayank Vadariya <48036907+mayankvadariya@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:21:07 -0500 Subject: [PATCH 029/158] Add TestTrinoHive4CatalogWithHiveMetastore test --- ...estTrinoHive4CatalogWithHiveMetastore.java | 27 +++++++++++++++++++ ...TestTrinoHiveCatalogWithHiveMetastore.java | 14 +++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHive4CatalogWithHiveMetastore.java diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHive4CatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHive4CatalogWithHiveMetastore.java new file mode 100644 index 000000000000..7c4f0ae1e245 --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHive4CatalogWithHiveMetastore.java @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.hms; + +import io.trino.plugin.hive.containers.Hive4MinioDataLake; +import io.trino.plugin.hive.containers.HiveMinioDataLake; + +public class TestTrinoHive4CatalogWithHiveMetastore + extends TestTrinoHiveCatalogWithHiveMetastore +{ + @Override + HiveMinioDataLake hiveMinioDataLake() + { + return new Hive4MinioDataLake(bucketName); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index 3085162c4777..01f4b8a4c3ed 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -34,6 +34,7 @@ import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.containers.Hive3MinioDataLake; +import io.trino.plugin.hive.containers.HiveMinioDataLake; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.ThriftMetastore; @@ -87,17 +88,22 @@ public class TestTrinoHiveCatalogWithHiveMetastore { private static final Logger LOG = Logger.get(TestTrinoHiveCatalogWithHiveMetastore.class); - private AutoCloseableCloser closer = AutoCloseableCloser.create(); + private final AutoCloseableCloser closer = AutoCloseableCloser.create(); // Use MinIO for storage, since HDFS is hard to get working in a unit test - private Hive3MinioDataLake dataLake; + private HiveMinioDataLake dataLake; private TrinoFileSystem fileSystem; - private String bucketName; + protected String bucketName; + + HiveMinioDataLake hiveMinioDataLake() + { + return new Hive3MinioDataLake(bucketName, HIVE3_IMAGE); + } @BeforeAll public void setUp() { bucketName = "test-hive-catalog-with-hms-" + randomNameSuffix(); - dataLake = closer.register(new Hive3MinioDataLake(bucketName, HIVE3_IMAGE)); + dataLake = closer.register(hiveMinioDataLake()); dataLake.start(); } From a6faaa9b8fab9da1bcb323fc49531cfa565a0f41 Mon Sep 17 00:00:00 2001 From: Vikash Kumar Date: Tue, 17 Dec 2024 11:41:28 +0530 Subject: [PATCH 030/158] Extract requireEnv in a util class SystemEnvironmentUtils --- .../env/environment/EnvMultinodeAzure.java | 6 +--- .../env/environment/EnvMultinodeGcs.java | 6 +--- .../environment/EnvMultinodeSnowflake.java | 6 +--- .../trino/testing/SystemEnvironmentUtils.java | 29 +++++++++++++++++++ 4 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAzure.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAzure.java index d31e6892a939..c46ef68e7356 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAzure.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAzure.java @@ -34,6 +34,7 @@ import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermissions; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; @@ -203,9 +204,4 @@ private Path getTemptoConfiguration(String schema) throw new UncheckedIOException(e); } } - - private static String requireEnv(String variable) - { - return requireNonNull(System.getenv(variable), () -> "environment variable not set: " + variable); - } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java index 3fab018afb8c..19091983dd4d 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java @@ -37,6 +37,7 @@ import java.util.Base64; import java.util.UUID; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; @@ -188,9 +189,4 @@ private Path getHiveSiteOverrideXml(String gcpStorageBucket) throw new UncheckedIOException(e); } } - - private static String requireEnv(String variable) - { - return requireNonNull(System.getenv(variable), () -> "environment variable not set: " + variable); - } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeSnowflake.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeSnowflake.java index ac5694a0b835..ab1933931999 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeSnowflake.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeSnowflake.java @@ -20,6 +20,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.launcher.env.EnvironmentContainers.isTrinoContainer; import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_TRINO_JVM_CONFIG; import static java.util.Objects.requireNonNull; @@ -57,9 +58,4 @@ public void extendEnvironment(Environment.Builder builder) builder.addConnector("snowflake", forHostPath(configDir.getPath("snowflake.properties"))); } - - private static String requireEnv(String variable) - { - return requireNonNull(System.getenv(variable), () -> "environment variable not set: " + variable); - } } diff --git a/testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java b/testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java new file mode 100644 index 000000000000..ef8a8a4d305a --- /dev/null +++ b/testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.testing; + +import static java.util.Objects.requireNonNull; + +public final class SystemEnvironmentUtils +{ + private SystemEnvironmentUtils() {} + + /** + * Get the named environment variable, throwing an exception if it is not set. + */ + public static String requireEnv(String variable) + { + return requireNonNull(System.getenv(variable), () -> "environment variable not set: " + variable); + } +} From 2efe63341ccbcdf60fd0e23c306dc84d3578337f Mon Sep 17 00:00:00 2001 From: Vikash Kumar Date: Tue, 17 Dec 2024 14:20:24 +0530 Subject: [PATCH 031/158] Remove unnecessay requirenment of password in test in TestSalesforceBasicAuthenticator#createAuthenticatedPrincipalRealBadPassword --- .../salesforce/TestSalesforceBasicAuthenticator.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java b/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java index 5689cb9b2438..2834e4875979 100644 --- a/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java +++ b/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java @@ -273,9 +273,8 @@ public void createAuthenticatedPrincipalRealBadPassword() fail("Must set SALESFORCE_TEST_ORG environment variable."); } String username = System.getenv("SALESFORCE_TEST_USERNAME"); - String password = System.getenv("SALESFORCE_TEST_PASSWORD"); - if (emptyToNull(username) == null || emptyToNull(password) == null) { - fail("Must set SALESFORCE_TEST_USERNAME and SALESFORCE_TEST_PASSWORD environment variables."); + if (emptyToNull(username) == null) { + fail("Must set SALESFORCE_TEST_USERNAME environment variable."); } SalesforceConfig config = new SalesforceConfig() From 503329aa95526628714c66ae8766996db2dfa0db Mon Sep 17 00:00:00 2001 From: Vikash Kumar Date: Tue, 17 Dec 2024 12:55:13 +0530 Subject: [PATCH 032/158] Use SystemEnvironmentUtils#requireEnv --- lib/trino-filesystem-azure/pom.xml | 6 +++ .../azure/AbstractTestAzureFileSystem.java | 5 --- .../azure/TestAzureFileSystemGen2Flat.java | 3 +- .../TestAzureFileSystemGen2Hierarchical.java | 3 +- .../TestAzureFileSystemOAuthGen2Flat.java | 9 +++-- ...tAzureFileSystemOAuthGen2Hierarchical.java | 9 +++-- .../gcs/AbstractTestGcsFileSystem.java | 6 --- .../filesystem/gcs/TestGcsFileSystem.java | 3 +- lib/trino-filesystem-s3/pom.xml | 6 +++ .../filesystem/s3/TestS3FileSystemAwsS3.java | 15 +++---- ...stS3FileSystemAwsS3WithSseCustomerKey.java | 15 +++---- .../hdfs/s3/TestTrinoS3FileSystemAwsS3.java | 6 +-- ...eWithCustomLocationUsingGlueMetastore.java | 4 +- ...ltaLakeRegisterTableProcedureWithGlue.java | 4 +- .../glue/TestDeltaLakeViewsGlueMetastore.java | 4 +- .../glue/TestDeltaS3AndGlueMetastoreTest.java | 4 +- .../hive/TestHiveS3AndGlueMetastoreTest.java | 4 +- .../metastore/thrift/TestUnityMetastore.java | 8 ++-- ...tIcebergGlueCatalogConnectorSmokeTest.java | 4 +- .../TestIcebergS3AndGlueMetastoreTest.java | 4 +- .../TestSalesforceBasicAuthenticator.java | 39 +++++-------------- ...AbstractSinglenodeDeltaLakeDatabricks.java | 9 +++-- .../EnvMultinodeDatabricksHttpHms.java | 26 +++++-------- .../EnvSinglenodeDeltaLakeDatabricks104.java | 4 +- .../EnvSinglenodeDeltaLakeDatabricks113.java | 4 +- .../EnvSinglenodeDeltaLakeDatabricks122.java | 4 +- .../EnvSinglenodeDeltaLakeDatabricks133.java | 4 +- .../EnvSinglenodeDeltaLakeDatabricks143.java | 4 +- .../EnvSinglenodeDeltaLakeDatabricks154.java | 4 +- .../EnvSinglenodeDeltaLakeDatabricks91.java | 4 +- .../deltalake/BaseTestDeltaLakeS3Storage.java | 4 +- .../product/deltalake/S3ClientFactory.java | 4 +- .../TestDeltaLakeAlluxioCaching.java | 4 +- .../product/deltalake/TestDeltaLakeAzure.java | 6 +-- ...DeltaLakeDatabricksUnityCompatibility.java | 6 +-- .../hive/TestAbfsSyncPartitionMetadata.java | 6 +-- .../product/hive/TestAzureBlobFileSystem.java | 6 +-- .../tests/product/hive/TestHiveAzure.java | 6 +-- .../TestHiveDatabricksUnityCompatibility.java | 6 +-- .../hudi/TestHudiSparkCompatibility.java | 4 +- .../iceberg/TestIcebergAlluxioCaching.java | 4 +- .../product/iceberg/TestIcebergAzure.java | 6 +-- 42 files changed, 128 insertions(+), 158 deletions(-) diff --git a/lib/trino-filesystem-azure/pom.xml b/lib/trino-filesystem-azure/pom.xml index 8cc67d7b6144..e67100cd4db4 100644 --- a/lib/trino-filesystem-azure/pom.xml +++ b/lib/trino-filesystem-azure/pom.xml @@ -178,6 +178,12 @@ test + + io.trino + trino-testing-services + test + + org.assertj assertj-core diff --git a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/AbstractTestAzureFileSystem.java b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/AbstractTestAzureFileSystem.java index 5b5afa9dc3c4..5d87efbc5623 100644 --- a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/AbstractTestAzureFileSystem.java +++ b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/AbstractTestAzureFileSystem.java @@ -53,11 +53,6 @@ public abstract class AbstractTestAzureFileSystem { private final EncryptionKey key = EncryptionKey.randomAes256(); - protected static String getRequiredEnvironmentVariable(String name) - { - return requireNonNull(System.getenv(name), "Environment variable not set: " + name); - } - protected enum AccountKind { HIERARCHICAL, FLAT diff --git a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Flat.java b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Flat.java index cc3233764284..def79fb20435 100644 --- a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Flat.java +++ b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Flat.java @@ -20,6 +20,7 @@ import java.io.IOException; import static io.trino.filesystem.azure.AbstractTestAzureFileSystem.AccountKind.FLAT; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestInstance(Lifecycle.PER_CLASS) class TestAzureFileSystemGen2Flat @@ -29,6 +30,6 @@ class TestAzureFileSystemGen2Flat void setup() throws IOException { - initializeWithAccessKey(getRequiredEnvironmentVariable("ABFS_FLAT_ACCOUNT"), getRequiredEnvironmentVariable("ABFS_FLAT_ACCESS_KEY"), FLAT); + initializeWithAccessKey(requireEnv("ABFS_FLAT_ACCOUNT"), requireEnv("ABFS_FLAT_ACCESS_KEY"), FLAT); } } diff --git a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Hierarchical.java b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Hierarchical.java index 5efb95e40ec1..d8423e00c2b5 100644 --- a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Hierarchical.java +++ b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemGen2Hierarchical.java @@ -20,6 +20,7 @@ import java.io.IOException; import static io.trino.filesystem.azure.AbstractTestAzureFileSystem.AccountKind.HIERARCHICAL; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestInstance(Lifecycle.PER_CLASS) class TestAzureFileSystemGen2Hierarchical @@ -29,6 +30,6 @@ class TestAzureFileSystemGen2Hierarchical void setup() throws IOException { - initializeWithAccessKey(getRequiredEnvironmentVariable("ABFS_HIERARCHICAL_ACCOUNT"), getRequiredEnvironmentVariable("ABFS_HIERARCHICAL_ACCESS_KEY"), HIERARCHICAL); + initializeWithAccessKey(requireEnv("ABFS_HIERARCHICAL_ACCOUNT"), requireEnv("ABFS_HIERARCHICAL_ACCESS_KEY"), HIERARCHICAL); } } diff --git a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Flat.java b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Flat.java index 82c3ac4efb55..ff70e01f44c5 100644 --- a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Flat.java +++ b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Flat.java @@ -20,6 +20,7 @@ import java.io.IOException; import static io.trino.filesystem.azure.AbstractTestAzureFileSystem.AccountKind.FLAT; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestInstance(Lifecycle.PER_CLASS) public class TestAzureFileSystemOAuthGen2Flat @@ -29,10 +30,10 @@ public class TestAzureFileSystemOAuthGen2Flat void setup() throws IOException { - String account = getRequiredEnvironmentVariable("ABFS_FLAT_ACCOUNT"); - String tenantId = getRequiredEnvironmentVariable("ABFS_OAUTH_TENANT_ID"); - String clientId = getRequiredEnvironmentVariable("ABFS_OAUTH_CLIENT_ID"); - String clientSecret = getRequiredEnvironmentVariable("ABFS_OAUTH_CLIENT_SECRET"); + String account = requireEnv("ABFS_FLAT_ACCOUNT"); + String tenantId = requireEnv("ABFS_OAUTH_TENANT_ID"); + String clientId = requireEnv("ABFS_OAUTH_CLIENT_ID"); + String clientSecret = requireEnv("ABFS_OAUTH_CLIENT_SECRET"); initializeWithOAuth(account, tenantId, clientId, clientSecret, FLAT); } } diff --git a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Hierarchical.java b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Hierarchical.java index 0276a6542b25..1ac28bed2d84 100644 --- a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Hierarchical.java +++ b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemOAuthGen2Hierarchical.java @@ -20,6 +20,7 @@ import java.io.IOException; import static io.trino.filesystem.azure.AbstractTestAzureFileSystem.AccountKind.HIERARCHICAL; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestInstance(Lifecycle.PER_CLASS) public class TestAzureFileSystemOAuthGen2Hierarchical @@ -29,10 +30,10 @@ public class TestAzureFileSystemOAuthGen2Hierarchical void setup() throws IOException { - String account = getRequiredEnvironmentVariable("ABFS_HIERARCHICAL_ACCOUNT"); - String tenantId = getRequiredEnvironmentVariable("ABFS_OAUTH_TENANT_ID"); - String clientId = getRequiredEnvironmentVariable("ABFS_OAUTH_CLIENT_ID"); - String clientSecret = getRequiredEnvironmentVariable("ABFS_OAUTH_CLIENT_SECRET"); + String account = requireEnv("ABFS_HIERARCHICAL_ACCOUNT"); + String tenantId = requireEnv("ABFS_OAUTH_TENANT_ID"); + String clientId = requireEnv("ABFS_OAUTH_CLIENT_ID"); + String clientSecret = requireEnv("ABFS_OAUTH_CLIENT_SECRET"); initializeWithOAuth(account, tenantId, clientId, clientSecret, HIERARCHICAL); } } diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java index d17b1d64571a..128f634fb3f1 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java @@ -35,7 +35,6 @@ import static com.google.cloud.storage.Storage.BlobTargetOption.doesNotExist; import static io.trino.filesystem.encryption.EncryptionKey.randomAes256; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -48,11 +47,6 @@ public abstract class AbstractTestGcsFileSystem private Storage storage; private GcsFileSystemFactory gcsFileSystemFactory; - protected static String getRequiredEnvironmentVariable(String name) - { - return requireNonNull(System.getenv(name), "Environment variable not set: " + name); - } - protected void initialize(String gcpCredentialKey) throws IOException { diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystem.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystem.java index 177e1390de20..0213d77c32af 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystem.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystem.java @@ -20,6 +20,7 @@ import java.io.IOException; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static java.nio.charset.StandardCharsets.UTF_8; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -30,7 +31,7 @@ public class TestGcsFileSystem void setup() throws IOException { - initialize(getRequiredEnvironmentVariable("GCP_CREDENTIALS_KEY")); + initialize(requireEnv("GCP_CREDENTIALS_KEY")); } @Test diff --git a/lib/trino-filesystem-s3/pom.xml b/lib/trino-filesystem-s3/pom.xml index 30d9336b5f62..4d767d05c5ca 100644 --- a/lib/trino-filesystem-s3/pom.xml +++ b/lib/trino-filesystem-s3/pom.xml @@ -239,6 +239,12 @@ test + + io.trino + trino-testing-services + test + + org.assertj assertj-core diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3.java index 200732d4499c..5d578543aedb 100644 --- a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3.java +++ b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3.java @@ -29,7 +29,7 @@ import java.util.List; import static com.google.common.collect.Iterables.getOnlyElement; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static org.assertj.core.api.Assertions.assertThat; public class TestS3FileSystemAwsS3 @@ -43,10 +43,10 @@ public class TestS3FileSystemAwsS3 @Override protected void initEnvironment() { - accessKey = environmentVariable("AWS_ACCESS_KEY_ID"); - secretKey = environmentVariable("AWS_SECRET_ACCESS_KEY"); - region = environmentVariable("AWS_REGION"); - bucket = environmentVariable("EMPTY_S3_BUCKET"); + accessKey = requireEnv("AWS_ACCESS_KEY_ID"); + secretKey = requireEnv("AWS_SECRET_ACCESS_KEY"); + region = requireEnv("AWS_REGION"); + bucket = requireEnv("EMPTY_S3_BUCKET"); } @Override @@ -75,11 +75,6 @@ protected S3FileSystemFactory createS3FileSystemFactory() .setStreamingPartSize(DataSize.valueOf("5.5MB")), new S3FileSystemStats()); } - private static String environmentVariable(String name) - { - return requireNonNull(System.getenv(name), "Environment variable not set: " + name); - } - @Test void testS3FileIteratorFileEntryTags() throws IOException diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3WithSseCustomerKey.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3WithSseCustomerKey.java index 7dcdaba12f8e..3de274317995 100644 --- a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3WithSseCustomerKey.java +++ b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3WithSseCustomerKey.java @@ -33,7 +33,7 @@ import java.util.function.Function; import static io.trino.filesystem.s3.S3FileSystemConfig.S3SseType.CUSTOMER; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; public class TestS3FileSystemAwsS3WithSseCustomerKey extends AbstractTestS3FileSystem @@ -49,10 +49,10 @@ public class TestS3FileSystemAwsS3WithSseCustomerKey @Override protected void initEnvironment() { - accessKey = environmentVariable("AWS_ACCESS_KEY_ID"); - secretKey = environmentVariable("AWS_SECRET_ACCESS_KEY"); - region = environmentVariable("AWS_REGION"); - bucket = environmentVariable("EMPTY_S3_BUCKET"); + accessKey = requireEnv("AWS_ACCESS_KEY_ID"); + secretKey = requireEnv("AWS_SECRET_ACCESS_KEY"); + region = requireEnv("AWS_REGION"); + bucket = requireEnv("EMPTY_S3_BUCKET"); s3SseCustomerKey = S3SseCustomerKey.onAes256(CUSTOMER_KEY); } @@ -109,11 +109,6 @@ protected S3FileSystemFactory createS3FileSystemFactory() new S3FileSystemStats()); } - private static String environmentVariable(String name) - { - return requireNonNull(System.getenv(name), "Environment variable not set: " + name); - } - private static String generateCustomerKey() { try { diff --git a/lib/trino-hdfs/src/test/java/io/trino/hdfs/s3/TestTrinoS3FileSystemAwsS3.java b/lib/trino-hdfs/src/test/java/io/trino/hdfs/s3/TestTrinoS3FileSystemAwsS3.java index 413836858627..69b5e96bd53f 100644 --- a/lib/trino-hdfs/src/test/java/io/trino/hdfs/s3/TestTrinoS3FileSystemAwsS3.java +++ b/lib/trino-hdfs/src/test/java/io/trino/hdfs/s3/TestTrinoS3FileSystemAwsS3.java @@ -15,7 +15,7 @@ import org.apache.hadoop.conf.Configuration; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; /** * Tests file system operations on AWS S3 storage. @@ -31,8 +31,8 @@ public class TestTrinoS3FileSystemAwsS3 public TestTrinoS3FileSystemAwsS3() { - bucketName = requireNonNull(System.getenv("S3_BUCKET"), "Environment S3_BUCKET was not set"); - s3Endpoint = requireNonNull(System.getenv("S3_BUCKET_ENDPOINT"), "Environment S3_BUCKET_ENDPOINT was not set"); + bucketName = requireEnv("S3_BUCKET"); + s3Endpoint = requireEnv("S3_BUCKET_ENDPOINT"); } @Override diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java index 8deee1b2a95e..37f0fdfab526 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java @@ -24,8 +24,8 @@ import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingGlueHiveMetastore; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; -import static java.util.Objects.requireNonNull; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @TestInstance(PER_CLASS) @@ -45,7 +45,7 @@ protected QueryRunner createQueryRunner() schema = "test_tables_with_custom_location" + randomNameSuffix(); return DeltaLakeQueryRunner.builder(schema) .addDeltaProperty("hive.metastore", "glue") - .addDeltaProperty("hive.metastore.glue.region", requireNonNull(System.getenv("AWS_REGION"), "AWS_REGION is null")) + .addDeltaProperty("hive.metastore.glue.region", requireEnv("AWS_REGION")) .addDeltaProperty("hive.metastore.glue.default-warehouse-dir", warehouseDir.toUri().toString()) .build(); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRegisterTableProcedureWithGlue.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRegisterTableProcedureWithGlue.java index d190234723a6..d67d92431355 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRegisterTableProcedureWithGlue.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeRegisterTableProcedureWithGlue.java @@ -25,8 +25,8 @@ import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingGlueHiveMetastore; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; -import static java.util.Objects.requireNonNull; public class TestDeltaLakeRegisterTableProcedureWithGlue extends BaseDeltaLakeRegisterTableProcedureTest @@ -44,7 +44,7 @@ protected QueryRunner createQueryRunner() schema = "test_delta_lake_register_table" + randomNameSuffix(); return DeltaLakeQueryRunner.builder(schema) .addDeltaProperty("hive.metastore", "glue") - .addDeltaProperty("hive.metastore.glue.region", requireNonNull(System.getenv("AWS_REGION"), "AWS_REGION is null")) + .addDeltaProperty("hive.metastore.glue.region", requireEnv("AWS_REGION")) .addDeltaProperty("hive.metastore.glue.default-warehouse-dir", warehouseDir.toUri().toString()) .addDeltaProperty("delta.unique-table-location", "true") .addDeltaProperty("delta.register-table-procedure.enabled", "true") diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java index f5b694bbf34c..beeebb49ec41 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java @@ -30,9 +30,9 @@ import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingGlueHiveMetastore; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @TestInstance(PER_CLASS) @@ -52,7 +52,7 @@ protected QueryRunner createQueryRunner() schema = "test_delta_lake_glue_views_" + randomNameSuffix(); return DeltaLakeQueryRunner.builder(schema) .addDeltaProperty("hive.metastore", "glue") - .addDeltaProperty("hive.metastore.glue.region", requireNonNull(System.getenv("AWS_REGION"), "AWS_REGION is null")) + .addDeltaProperty("hive.metastore.glue.region", requireEnv("AWS_REGION")) .addDeltaProperty("hive.metastore.glue.default-warehouse-dir", warehouseDir.toUri().toString()) .build(); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaS3AndGlueMetastoreTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaS3AndGlueMetastoreTest.java index 865857a32851..c66ddbb7dbfa 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaS3AndGlueMetastoreTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaS3AndGlueMetastoreTest.java @@ -25,7 +25,7 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingGlueHiveMetastore; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static org.assertj.core.api.Assertions.assertThat; public class TestDeltaS3AndGlueMetastoreTest @@ -33,7 +33,7 @@ public class TestDeltaS3AndGlueMetastoreTest { public TestDeltaS3AndGlueMetastoreTest() { - super("partitioned_by", "location", requireNonNull(System.getenv("S3_BUCKET"), "Environment variable not set: S3_BUCKET")); + super("partitioned_by", "location", requireEnv("S3_BUCKET")); } @Override diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java index b602f01cb00d..cf5076c377d9 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java @@ -33,10 +33,10 @@ import static io.trino.plugin.hive.TestingHiveUtils.getConnectorService; import static io.trino.spi.security.SelectedRole.Type.ROLE; import static io.trino.testing.MaterializedResult.resultBuilder; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -45,7 +45,7 @@ public class TestHiveS3AndGlueMetastoreTest { public TestHiveS3AndGlueMetastoreTest() { - super("partitioned_by", "external_location", requireNonNull(System.getenv("S3_BUCKET"), "Environment S3_BUCKET was not set")); + super("partitioned_by", "external_location", requireEnv("S3_BUCKET")); } @Override diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestUnityMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestUnityMetastore.java index 16afc1bad2b6..c3799770bd74 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestUnityMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestUnityMetastore.java @@ -27,7 +27,7 @@ import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder; import static io.trino.plugin.hive.metastore.thrift.ThriftHttpMetastoreConfig.AuthenticationMode.BEARER; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static org.assertj.core.api.Assertions.assertThat; final class TestUnityMetastore @@ -36,9 +36,9 @@ final class TestUnityMetastore void test() throws Exception { - String databricksHost = requireNonNull(System.getenv("DATABRICKS_HOST"), "Environment variable not set: DATABRICKS_HOST"); - String databricksToken = requireNonNull(System.getenv("DATABRICKS_TOKEN"), "Environment variable not set: DATABRICKS_TOKEN"); - String databricksCatalogName = requireNonNull(System.getenv("DATABRICKS_UNITY_CATALOG_NAME"), "Environment variable not set: DATABRICKS_UNITY_CATALOG_NAME"); + String databricksHost = requireEnv("DATABRICKS_HOST"); + String databricksToken = requireEnv("DATABRICKS_TOKEN"); + String databricksCatalogName = requireEnv("DATABRICKS_UNITY_CATALOG_NAME"); URI metastoreUri = URI.create("https://%s:443/api/2.0/unity-hms-proxy/metadata" .formatted(databricksHost)); ThriftHttpMetastoreConfig config = new ThriftHttpMetastoreConfig() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java index 9dec58090118..d7adfe595aa4 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java @@ -57,10 +57,10 @@ import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableType; import static io.trino.plugin.iceberg.IcebergTestUtils.checkParquetFileSorting; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingConnectorSession.SESSION; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -82,7 +82,7 @@ public class TestIcebergGlueCatalogConnectorSmokeTest public TestIcebergGlueCatalogConnectorSmokeTest() { super(FileFormat.PARQUET); - this.bucketName = requireNonNull(System.getenv("S3_BUCKET"), "Environment S3_BUCKET was not set"); + this.bucketName = requireEnv("S3_BUCKET"); this.schemaName = "test_iceberg_smoke_" + randomNameSuffix(); glueClient = AWSGlueAsyncClientBuilder.defaultClient(); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergS3AndGlueMetastoreTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergS3AndGlueMetastoreTest.java index 044f41f71ef1..48f2b0d1f94a 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergS3AndGlueMetastoreTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergS3AndGlueMetastoreTest.java @@ -28,8 +28,8 @@ import java.util.stream.Collectors; import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingGlueHiveMetastore; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; public class TestIcebergS3AndGlueMetastoreTest @@ -37,7 +37,7 @@ public class TestIcebergS3AndGlueMetastoreTest { public TestIcebergS3AndGlueMetastoreTest() { - super("partitioning", "location", requireNonNull(System.getenv("S3_BUCKET"), "Environment S3_BUCKET was not set")); + super("partitioning", "location", requireEnv("S3_BUCKET")); } @Override diff --git a/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java b/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java index 2834e4875979..bb40dac86d77 100644 --- a/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java +++ b/plugin/trino-password-authenticators/src/test/java/io/trino/plugin/password/salesforce/TestSalesforceBasicAuthenticator.java @@ -24,14 +24,13 @@ import java.security.Principal; import java.util.concurrent.TimeUnit; -import static com.google.common.base.Strings.emptyToNull; import static com.google.common.net.MediaType.ANY_TEXT_TYPE; import static io.airlift.http.client.HttpStatus.OK; import static io.airlift.http.client.testing.TestingResponse.mockResponse; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Fail.fail; import static org.junit.jupiter.api.Assumptions.abort; public class TestSalesforceBasicAuthenticator @@ -182,15 +181,9 @@ public void createAuthenticatedPrincipalRealSuccess() abort("Skipping real tests."); } - String org = System.getenv("SALESFORCE_TEST_ORG"); - if (emptyToNull(org) == null) { - fail("Must set SALESFORCE_TEST_ORG environment variable."); - } - String username = System.getenv("SALESFORCE_TEST_USERNAME"); - String password = System.getenv("SALESFORCE_TEST_PASSWORD"); - if (emptyToNull(username) == null || emptyToNull(password) == null) { - fail("Must set SALESFORCE_TEST_USERNAME and SALESFORCE_TEST_PASSWORD environment variables."); - } + String org = requireEnv("SALESFORCE_TEST_ORG"); + String username = requireEnv("SALESFORCE_TEST_USERNAME"); + String password = requireEnv("SALESFORCE_TEST_PASSWORD"); SalesforceConfig config = new SalesforceConfig() .setAllowedOrganizations(org); @@ -213,11 +206,8 @@ public void createAuthenticatedPrincipalRealWrongOrg() abort("Skipping real tests."); } - String username = System.getenv("SALESFORCE_TEST_USERNAME"); - String password = System.getenv("SALESFORCE_TEST_PASSWORD"); - if (emptyToNull(username) == null || emptyToNull(password) == null) { - fail("Must set SALESFORCE_TEST_USERNAME and SALESFORCE_TEST_PASSWORD environment variables."); - } + String username = requireEnv("SALESFORCE_TEST_USERNAME"); + String password = requireEnv("SALESFORCE_TEST_PASSWORD"); String org = "NotMyOrg"; SalesforceConfig config = new SalesforceConfig() @@ -240,11 +230,8 @@ public void createAuthenticatedPrincipalRealAllOrgs() abort("Skipping real tests."); } - String username = System.getenv("SALESFORCE_TEST_USERNAME"); - String password = System.getenv("SALESFORCE_TEST_PASSWORD"); - if (emptyToNull(username) == null || emptyToNull(password) == null) { - fail("Must set SALESFORCE_TEST_USERNAME and SALESFORCE_TEST_PASSWORD environment variables."); - } + String username = requireEnv("SALESFORCE_TEST_USERNAME"); + String password = requireEnv("SALESFORCE_TEST_PASSWORD"); SalesforceConfig config = new SalesforceConfig() .setAllowedOrganizations("all"); @@ -268,14 +255,8 @@ public void createAuthenticatedPrincipalRealBadPassword() abort("Skipping real tests."); } - String org = System.getenv("SALESFORCE_TEST_ORG"); - if (emptyToNull(org) == null) { - fail("Must set SALESFORCE_TEST_ORG environment variable."); - } - String username = System.getenv("SALESFORCE_TEST_USERNAME"); - if (emptyToNull(username) == null) { - fail("Must set SALESFORCE_TEST_USERNAME environment variable."); - } + String org = requireEnv("SALESFORCE_TEST_ORG"); + String username = requireEnv("SALESFORCE_TEST_USERNAME"); SalesforceConfig config = new SalesforceConfig() .setAllowedOrganizations(org); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java index ef7666f5cbb1..651a9a3edc6a 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java @@ -21,6 +21,7 @@ import java.io.File; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; @@ -51,10 +52,10 @@ public AbstractSinglenodeDeltaLakeDatabricks(Standard standard, DockerFiles dock public void extendEnvironment(Environment.Builder builder) { String databricksTestJdbcUrl = databricksTestJdbcUrl(); - String databricksTestLogin = requireNonNull(System.getenv("DATABRICKS_LOGIN"), "Environment DATABRICKS_LOGIN was not set"); - String databricksTestToken = requireNonNull(System.getenv("DATABRICKS_TOKEN"), "Environment DATABRICKS_TOKEN was not set"); - String awsRegion = requireNonNull(System.getenv("AWS_REGION"), "Environment AWS_REGION was not set"); - String s3Bucket = requireNonNull(System.getenv("S3_BUCKET"), "Environment S3_BUCKET was not set"); + String databricksTestLogin = requireEnv("DATABRICKS_LOGIN"); + String databricksTestToken = requireEnv("DATABRICKS_TOKEN"); + String awsRegion = requireEnv("AWS_REGION"); + String s3Bucket = requireEnv("S3_BUCKET"); DockerFiles.ResourceProvider configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/singlenode-delta-lake-databricks"); builder.configureContainer(COORDINATOR, dockerContainer -> exportAWSCredentials(dockerContainer) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeDatabricksHttpHms.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeDatabricksHttpHms.java index 7a5317b0195c..ce0c8b48115b 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeDatabricksHttpHms.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeDatabricksHttpHms.java @@ -23,6 +23,7 @@ import java.io.File; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; import static io.trino.tests.product.launcher.env.EnvironmentContainers.isTrinoContainer; @@ -50,18 +51,18 @@ public EnvMultinodeDatabricksHttpHms(StandardMultinode standardMultinode, Docker @Override public void extendEnvironment(Environment.Builder builder) { - String databricksTestJdbcUrl = requireNonNull(getEnvVariable("DATABRICKS_UNITY_JDBC_URL"), "Environment DATABRICKS_UNITY_JDBC_URL was not set"); - String databricksTestLogin = requireNonNull(getEnvVariable("DATABRICKS_LOGIN"), "Environment DATABRICKS_LOGIN was not set"); - String databricksTestToken = requireNonNull(getEnvVariable("DATABRICKS_TOKEN"), "Environment DATABRICKS_TOKEN was not set"); - String awsRegion = requireNonNull(getEnvVariable("AWS_REGION"), "Environment AWS_REGION was not set"); + String databricksTestJdbcUrl = requireEnv("DATABRICKS_UNITY_JDBC_URL"); + String databricksTestLogin = requireEnv("DATABRICKS_LOGIN"); + String databricksTestToken = requireEnv("DATABRICKS_TOKEN"); + String awsRegion = requireEnv("AWS_REGION"); builder.configureContainers(container -> { if (isTrinoContainer(container.getLogicalName())) { exportAwsCredentials(container) .withEnv("AWS_REGION", awsRegion) .withEnv("DATABRICKS_TOKEN", databricksTestToken) - .withEnv("DATABRICKS_HOST", getEnvVariable("DATABRICKS_HOST")) - .withEnv("DATABRICKS_UNITY_CATALOG_NAME", getEnvVariable("DATABRICKS_UNITY_CATALOG_NAME")); + .withEnv("DATABRICKS_HOST", requireEnv("DATABRICKS_HOST")) + .withEnv("DATABRICKS_UNITY_CATALOG_NAME", requireEnv("DATABRICKS_UNITY_CATALOG_NAME")); } }); @@ -69,8 +70,8 @@ public void extendEnvironment(Environment.Builder builder) .withEnv("DATABRICKS_JDBC_URL", databricksTestJdbcUrl) .withEnv("DATABRICKS_LOGIN", databricksTestLogin) .withEnv("DATABRICKS_TOKEN", databricksTestToken) - .withEnv("DATABRICKS_UNITY_CATALOG_NAME", getEnvVariable("DATABRICKS_UNITY_CATALOG_NAME")) - .withEnv("DATABRICKS_UNITY_EXTERNAL_LOCATION", getEnvVariable("DATABRICKS_UNITY_EXTERNAL_LOCATION")) + .withEnv("DATABRICKS_UNITY_CATALOG_NAME", requireEnv("DATABRICKS_UNITY_CATALOG_NAME")) + .withEnv("DATABRICKS_UNITY_EXTERNAL_LOCATION", requireEnv("DATABRICKS_UNITY_EXTERNAL_LOCATION")) .withCopyFileToContainer( forHostPath(DATABRICKS_JDBC_PROVIDER.getAbsolutePath()), "/docker/jdbc/databricks-jdbc.jar")); @@ -83,15 +84,6 @@ public void extendEnvironment(Environment.Builder builder) configureTempto(builder, configDir); } - private static String getEnvVariable(String name) - { - String credentialValue = System.getenv(name); - if (credentialValue == null) { - throw new IllegalStateException(format("Environment variable %s not set", name)); - } - return credentialValue; - } - private DockerContainer exportAwsCredentials(DockerContainer container) { container = exportAwsCredential(container, "TRINO_AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY_ID", true); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks104.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks104.java index 65d8196523c3..cf204d8d18ab 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks104.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks104.java @@ -18,7 +18,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestsEnvironment public class EnvSinglenodeDeltaLakeDatabricks104 @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks104(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_104_JDBC_URL"), "Environment DATABRICKS_104_JDBC_URL was not set"); + return requireEnv("DATABRICKS_104_JDBC_URL"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks113.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks113.java index d8248cec2d60..54ea27443fbe 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks113.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks113.java @@ -18,7 +18,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestsEnvironment public class EnvSinglenodeDeltaLakeDatabricks113 @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks113(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_113_JDBC_URL"), "Environment DATABRICKS_113_JDBC_URL was not set"); + return requireEnv("DATABRICKS_113_JDBC_URL"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks122.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks122.java index 0435b226bfb1..586fd0f7ee24 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks122.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks122.java @@ -18,7 +18,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestsEnvironment public class EnvSinglenodeDeltaLakeDatabricks122 @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks122(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_122_JDBC_URL"), "Environment DATABRICKS_122_JDBC_URL was not set"); + return requireEnv("DATABRICKS_122_JDBC_URL"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java index e054cec92822..b1da087a8653 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java @@ -18,7 +18,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestsEnvironment public class EnvSinglenodeDeltaLakeDatabricks133 @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks133(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_133_JDBC_URL"), "Environment DATABRICKS_133_JDBC_URL was not set"); + return requireEnv("DATABRICKS_133_JDBC_URL"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java index 06b70041a58b..bec3c502b8b6 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java @@ -18,7 +18,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestsEnvironment public class EnvSinglenodeDeltaLakeDatabricks143 @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks143(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_143_JDBC_URL"), "Environment DATABRICKS_143_JDBC_URL was not set"); + return requireEnv("DATABRICKS_143_JDBC_URL"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java index ea427a21c71b..37bb6d41eeb7 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java @@ -18,7 +18,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestsEnvironment public class EnvSinglenodeDeltaLakeDatabricks154 @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks154(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_154_JDBC_URL"), "Environment DATABRICKS_154_JDBC_URL was not set"); + return requireEnv("DATABRICKS_154_JDBC_URL"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java index 0f9b61613109..cf85d7118b53 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java @@ -18,7 +18,7 @@ import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; @TestsEnvironment public class EnvSinglenodeDeltaLakeDatabricks91 @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks91(Standard standard, DockerFiles dockerF @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_91_JDBC_URL"), "Environment DATABRICKS_91_JDBC_URL was not set"); + return requireEnv("DATABRICKS_91_JDBC_URL"); } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeS3Storage.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeS3Storage.java index a0ec14e63776..5feaca425845 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeS3Storage.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeS3Storage.java @@ -16,7 +16,7 @@ import io.trino.tempto.BeforeMethodWithContext; import io.trino.tempto.ProductTest; -import static java.util.Objects.requireNonNull; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; public abstract class BaseTestDeltaLakeS3Storage extends ProductTest @@ -26,6 +26,6 @@ public abstract class BaseTestDeltaLakeS3Storage @BeforeMethodWithContext public void setUp() { - bucketName = requireNonNull(System.getenv("S3_BUCKET"), "Environment variable not set: S3_BUCKET"); + bucketName = requireEnv("S3_BUCKET"); } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/S3ClientFactory.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/S3ClientFactory.java index c81171488413..ca372999c91a 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/S3ClientFactory.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/S3ClientFactory.java @@ -25,9 +25,9 @@ import com.amazonaws.services.s3.model.Region; import io.trino.testing.minio.MinioClient; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.minio.MinioClient.DEFAULT_MINIO_ACCESS_KEY; import static io.trino.testing.minio.MinioClient.DEFAULT_MINIO_SECRET_KEY; -import static java.util.Objects.requireNonNull; final class S3ClientFactory { @@ -45,7 +45,7 @@ public AmazonS3 createS3Client(String serverType) private AmazonS3 createAwsS3Client() { - String region = requireNonNull(System.getenv("AWS_REGION"), "AWS_REGION is null"); + String region = requireEnv("AWS_REGION"); return AmazonS3Client.builder().withRegion(region).build(); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlluxioCaching.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlluxioCaching.java index ce75707232ec..ee3e8ee772f3 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlluxioCaching.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlluxioCaching.java @@ -19,12 +19,12 @@ import io.trino.tests.product.utils.CachingTestUtils.CacheStats; import org.testng.annotations.Test; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.TestGroups.DELTA_LAKE_ALLUXIO_CACHING; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.utils.CachingTestUtils.getCacheStats; import static io.trino.tests.product.utils.QueryAssertions.assertEventually; import static io.trino.tests.product.utils.QueryExecutors.onTrino; -import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; @@ -36,7 +36,7 @@ public class TestDeltaLakeAlluxioCaching @BeforeMethodWithContext public void setUp() { - bucketName = requireNonNull(System.getenv("S3_BUCKET"), "Environment variable not set: S3_BUCKET"); + bucketName = requireEnv("S3_BUCKET"); } @Test(groups = {DELTA_LAKE_ALLUXIO_CACHING, PROFILE_SPECIFIC_TESTS}) diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAzure.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAzure.java index 97c8816c9465..115dbe334887 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAzure.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAzure.java @@ -19,10 +19,10 @@ import io.trino.tests.product.BaseTestTableFormats; import org.testng.annotations.Test; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.TestGroups.DELTA_LAKE_AZURE; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; public class TestDeltaLakeAzure extends BaseTestTableFormats @@ -35,8 +35,8 @@ public class TestDeltaLakeAzure @BeforeMethodWithContext public void setUp() { - String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); - String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); + String container = requireEnv("ABFS_CONTAINER"); + String account = requireEnv("ABFS_ACCOUNT"); schemaLocation = format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUnityCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUnityCompatibility.java index 5c9cec3d33ef..8dbb1fdf9e32 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUnityCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUnityCompatibility.java @@ -25,6 +25,7 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DATABRICKS_UNITY_HTTP_HMS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; @@ -34,7 +35,6 @@ import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; public class TestDeltaLakeDatabricksUnityCompatibility @@ -47,8 +47,8 @@ public class TestDeltaLakeDatabricksUnityCompatibility @BeforeMethodWithContext public void setUp() { - unityCatalogName = requireNonNull(System.getenv("DATABRICKS_UNITY_CATALOG_NAME"), "Environment variable not set: DATABRICKS_UNITY_CATALOG_NAME"); - externalLocationPath = requireNonNull(System.getenv("DATABRICKS_UNITY_EXTERNAL_LOCATION"), "Environment variable not set: DATABRICKS_UNITY_EXTERNAL_LOCATION"); + unityCatalogName = requireEnv("DATABRICKS_UNITY_CATALOG_NAME"); + externalLocationPath = requireEnv("DATABRICKS_UNITY_EXTERNAL_LOCATION"); onDelta().executeQuery(format("CREATE SCHEMA %s.%s", unityCatalogName, schemaName)); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAbfsSyncPartitionMetadata.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAbfsSyncPartitionMetadata.java index 20b1a0339730..37ef181ba5c6 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAbfsSyncPartitionMetadata.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAbfsSyncPartitionMetadata.java @@ -21,13 +21,13 @@ import org.testng.annotations.Test; import static com.google.common.base.Preconditions.checkArgument; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.TestGroups.AZURE; import static io.trino.tests.product.utils.HadoopTestUtils.RETRYABLE_FAILURES_ISSUES; import static io.trino.tests.product.utils.HadoopTestUtils.RETRYABLE_FAILURES_MATCH; import static io.trino.tests.product.utils.QueryExecutors.onHive; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.apache.parquet.Strings.isNullOrEmpty; public class TestAbfsSyncPartitionMetadata @@ -53,8 +53,8 @@ public void tearDown() @Override protected String schemaLocation() { - String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); - String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); + String container = requireEnv("ABFS_CONTAINER"); + String account = requireEnv("ABFS_ACCOUNT"); return format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAzureBlobFileSystem.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAzureBlobFileSystem.java index e816b3428542..090da1d5d554 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAzureBlobFileSystem.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAzureBlobFileSystem.java @@ -26,6 +26,7 @@ import java.util.List; import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.AZURE; import static io.trino.tests.product.utils.HadoopTestUtils.RETRYABLE_FAILURES_ISSUES; @@ -33,7 +34,6 @@ import static io.trino.tests.product.utils.QueryExecutors.onHive; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; public class TestAzureBlobFileSystem @@ -47,8 +47,8 @@ public class TestAzureBlobFileSystem @BeforeMethodWithContext public void setUp() { - String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); - String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); + String container = requireEnv("ABFS_CONTAINER"); + String account = requireEnv("ABFS_ACCOUNT"); schemaLocation = format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); onHive().executeQuery("dfs -rm -f -r " + schemaLocation); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveAzure.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveAzure.java index b8586917cc91..0b163e47e2af 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveAzure.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveAzure.java @@ -19,10 +19,10 @@ import io.trino.tests.product.BaseTestTableFormats; import org.testng.annotations.Test; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.TestGroups.AZURE; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; public class TestHiveAzure extends BaseTestTableFormats @@ -35,8 +35,8 @@ public class TestHiveAzure @BeforeMethodWithContext public void setUp() { - String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); - String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); + String container = requireEnv("ABFS_CONTAINER"); + String account = requireEnv("ABFS_ACCOUNT"); schemaLocation = format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveDatabricksUnityCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveDatabricksUnityCompatibility.java index a3bd6e9579b0..84cb24502b8e 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveDatabricksUnityCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveDatabricksUnityCompatibility.java @@ -20,6 +20,7 @@ import org.testng.annotations.Test; import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DATABRICKS_UNITY_HTTP_HMS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; @@ -28,7 +29,6 @@ import static io.trino.tests.product.utils.QueryExecutors.onDelta; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; public class TestHiveDatabricksUnityCompatibility @@ -41,8 +41,8 @@ public class TestHiveDatabricksUnityCompatibility @BeforeMethodWithContext public void setUp() { - unityCatalogName = requireNonNull(System.getenv("DATABRICKS_UNITY_CATALOG_NAME"), "Environment variable not set: DATABRICKS_UNITY_CATALOG_NAME"); - externalLocationPath = requireNonNull(System.getenv("DATABRICKS_UNITY_EXTERNAL_LOCATION"), "Environment variable not set: DATABRICKS_UNITY_EXTERNAL_LOCATION"); + unityCatalogName = requireEnv("DATABRICKS_UNITY_CATALOG_NAME"); + externalLocationPath = requireEnv("DATABRICKS_UNITY_EXTERNAL_LOCATION"); String schemaLocation = format("%s/%s", externalLocationPath, schemaName); onDelta().executeQuery("CREATE SCHEMA " + unityCatalogName + "." + schemaName + " MANAGED LOCATION '" + schemaLocation + "'"); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hudi/TestHudiSparkCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hudi/TestHudiSparkCompatibility.java index 91c0702c0068..c42783245eaf 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hudi/TestHudiSparkCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hudi/TestHudiSparkCompatibility.java @@ -23,6 +23,7 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.HIVE_HUDI_REDIRECTIONS; import static io.trino.tests.product.TestGroups.HUDI; @@ -30,7 +31,6 @@ import static io.trino.tests.product.utils.QueryExecutors.onHudi; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; public class TestHudiSparkCompatibility @@ -44,7 +44,7 @@ public class TestHudiSparkCompatibility @BeforeMethodWithContext public void setUp() { - bucketName = requireNonNull(System.getenv("S3_BUCKET"), "Environment variable not set: S3_BUCKET"); + bucketName = requireEnv("S3_BUCKET"); } @Test(groups = {HUDI, PROFILE_SPECIFIC_TESTS}) diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAlluxioCaching.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAlluxioCaching.java index 4fa903cc1731..8ee6e3a9e424 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAlluxioCaching.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAlluxioCaching.java @@ -19,12 +19,12 @@ import io.trino.tests.product.utils.CachingTestUtils.CacheStats; import org.testng.annotations.Test; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.TestGroups.ICEBERG_ALLUXIO_CACHING; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.utils.CachingTestUtils.getCacheStats; import static io.trino.tests.product.utils.QueryAssertions.assertEventually; import static io.trino.tests.product.utils.QueryExecutors.onTrino; -import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; @@ -36,7 +36,7 @@ public class TestIcebergAlluxioCaching @BeforeMethodWithContext public void setUp() { - bucketName = requireNonNull(System.getenv("S3_BUCKET"), "Environment variable not set: S3_BUCKET"); + bucketName = requireEnv("S3_BUCKET"); } @Test(groups = {ICEBERG_ALLUXIO_CACHING, PROFILE_SPECIFIC_TESTS}) diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAzure.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAzure.java index e689e1dcb84a..6fe56c3dfb6f 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAzure.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAzure.java @@ -19,10 +19,10 @@ import io.trino.tests.product.BaseTestTableFormats; import org.testng.annotations.Test; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.tests.product.TestGroups.ICEBERG_AZURE; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; public class TestIcebergAzure extends BaseTestTableFormats @@ -35,8 +35,8 @@ public class TestIcebergAzure @BeforeMethodWithContext public void setUp() { - String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); - String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); + String container = requireEnv("ABFS_CONTAINER"); + String account = requireEnv("ABFS_ACCOUNT"); schemaLocation = format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); } From a99d96ece15113d4edf204f5a5345b26f5c9721f Mon Sep 17 00:00:00 2001 From: Vikash Kumar Date: Tue, 17 Dec 2024 13:42:21 +0530 Subject: [PATCH 033/158] Add and use SystemEnvironmentUtils#isEnvSet method --- .../java/io/trino/tests/product/launcher/cli/TestRun.java | 3 ++- .../java/io/trino/testing/containers/TestContainers.java | 3 ++- .../main/java/io/trino/testing/SystemEnvironmentUtils.java | 5 +++++ .../testing/services/junit/LogTestDurationListener.java | 3 ++- .../io/trino/testng/services/FlakyTestRetryAnalyzer.java | 3 ++- .../io/trino/testng/services/LogTestDurationListener.java | 3 ++- .../io/trino/testng/services/ProgressLoggingListener.java | 3 ++- 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java index 27c05f8bbc68..7d938caa9dc4 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java @@ -53,6 +53,7 @@ import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; import static io.trino.tests.product.launcher.env.DockerContainer.cleanOrCreateHostPath; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; import static io.trino.tests.product.launcher.env.EnvironmentListener.getStandardListeners; @@ -335,7 +336,7 @@ private Environment getEnvironment() unsafelyExposePort(container, 5007); // debug port } - if (System.getenv("CONTINUOUS_INTEGRATION") != null) { + if (isEnvSet("CONTINUOUS_INTEGRATION")) { container.withEnv("CONTINUOUS_INTEGRATION", "true"); } diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/TestContainers.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/TestContainers.java index 03330b35f54c..9f9e0886fd34 100644 --- a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/TestContainers.java +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/TestContainers.java @@ -31,6 +31,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.padEnd; import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; import static io.trino.testing.containers.ConditionalPullPolicy.TESTCONTAINERS_NEVER_PULL; import static java.lang.Boolean.parseBoolean; import static java.lang.System.getenv; @@ -72,7 +73,7 @@ public static String getPathFromClassPathResource(String resourcePath) public static void exposeFixedPorts(GenericContainer container) { - checkState(System.getenv("CONTINUOUS_INTEGRATION") == null, "" + + checkState(isEnvSet("CONTINUOUS_INTEGRATION"), "" + "Exposing fixed ports should not be used in regular test code. This could break parallel test execution. " + "This method is supposed to be invoked from local development helpers only e.g. QueryRunner.main(), " + "hence it should never run on CI"); diff --git a/testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java b/testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java index ef8a8a4d305a..0f866033cf56 100644 --- a/testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java +++ b/testing/trino-testing-services/src/main/java/io/trino/testing/SystemEnvironmentUtils.java @@ -26,4 +26,9 @@ public static String requireEnv(String variable) { return requireNonNull(System.getenv(variable), () -> "environment variable not set: " + variable); } + + public static boolean isEnvSet(String variable) + { + return System.getenv(variable) != null; + } } diff --git a/testing/trino-testing-services/src/main/java/io/trino/testing/services/junit/LogTestDurationListener.java b/testing/trino-testing-services/src/main/java/io/trino/testing/services/junit/LogTestDurationListener.java index 33127d251bb2..8263d6196b3c 100644 --- a/testing/trino-testing-services/src/main/java/io/trino/testing/services/junit/LogTestDurationListener.java +++ b/testing/trino-testing-services/src/main/java/io/trino/testing/services/junit/LogTestDurationListener.java @@ -37,6 +37,7 @@ import static com.google.common.base.Throwables.getStackTraceAsString; import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.Duration.nanosSince; +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; import static io.trino.testing.services.junit.Listeners.reportListenerFailure; import static java.lang.String.format; import static java.lang.management.ManagementFactory.getThreadMXBean; @@ -73,7 +74,7 @@ private static boolean isEnabled() if (System.getProperty("LogTestDurationListener.enabled") != null) { return Boolean.getBoolean("LogTestDurationListener.enabled"); } - if (System.getenv("CONTINUOUS_INTEGRATION") != null) { + if (isEnvSet("CONTINUOUS_INTEGRATION")) { return true; } // For local development, logging durations is not typically useful. diff --git a/testing/trino-testing-services/src/main/java/io/trino/testng/services/FlakyTestRetryAnalyzer.java b/testing/trino-testing-services/src/main/java/io/trino/testng/services/FlakyTestRetryAnalyzer.java index 849403bea407..8ff39a4f6a98 100644 --- a/testing/trino-testing-services/src/main/java/io/trino/testng/services/FlakyTestRetryAnalyzer.java +++ b/testing/trino-testing-services/src/main/java/io/trino/testng/services/FlakyTestRetryAnalyzer.java @@ -28,6 +28,7 @@ import java.util.regex.Pattern; import static com.google.common.base.Throwables.getStackTraceAsString; +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; import static java.lang.String.format; public class FlakyTestRetryAnalyzer @@ -54,7 +55,7 @@ public boolean retry(ITestResult result) Optional enabledSystemPropertyValue = Optional.ofNullable(System.getProperty(ENABLED_SYSTEM_PROPERTY)); if (!enabledSystemPropertyValue.map(Boolean::parseBoolean) - .orElseGet(() -> System.getenv("CONTINUOUS_INTEGRATION") != null)) { + .orElseGet(() -> isEnvSet("CONTINUOUS_INTEGRATION"))) { log.info( "FlakyTestRetryAnalyzer not enabled: " + "CONTINUOUS_INTEGRATION environment is not detected or " + diff --git a/testing/trino-testing-services/src/main/java/io/trino/testng/services/LogTestDurationListener.java b/testing/trino-testing-services/src/main/java/io/trino/testng/services/LogTestDurationListener.java index f5e0e4d726b7..25a933487715 100644 --- a/testing/trino-testing-services/src/main/java/io/trino/testng/services/LogTestDurationListener.java +++ b/testing/trino-testing-services/src/main/java/io/trino/testng/services/LogTestDurationListener.java @@ -39,6 +39,7 @@ import static com.google.common.base.Throwables.getStackTraceAsString; import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.Duration.nanosSince; +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; import static io.trino.testng.services.Listeners.formatTestName; import static io.trino.testng.services.Listeners.reportListenerFailure; import static java.lang.String.format; @@ -79,7 +80,7 @@ private static boolean isEnabled() if (System.getProperty("LogTestDurationListener.enabled") != null) { return Boolean.getBoolean("LogTestDurationListener.enabled"); } - if (System.getenv("CONTINUOUS_INTEGRATION") != null) { + if (isEnvSet("CONTINUOUS_INTEGRATION")) { return true; } // LogTestDurationListener does not support concurrent invocations of same test method diff --git a/testing/trino-testing-services/src/main/java/io/trino/testng/services/ProgressLoggingListener.java b/testing/trino-testing-services/src/main/java/io/trino/testng/services/ProgressLoggingListener.java index 3f61b2496044..8a0011e699ee 100644 --- a/testing/trino-testing-services/src/main/java/io/trino/testng/services/ProgressLoggingListener.java +++ b/testing/trino-testing-services/src/main/java/io/trino/testng/services/ProgressLoggingListener.java @@ -25,6 +25,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; import static io.trino.testng.services.Listeners.formatTestName; import static java.lang.String.format; @@ -47,7 +48,7 @@ private static boolean isEnabled() if (System.getProperty("ProgressLoggingListener.enabled") != null) { return Boolean.getBoolean("ProgressLoggingListener.enabled"); } - if (System.getenv("CONTINUOUS_INTEGRATION") != null) { + if (isEnvSet("CONTINUOUS_INTEGRATION")) { return true; } // most often not useful for local development From 938944023ecfe2f51e243fb4d2825bbeb801a4f4 Mon Sep 17 00:00:00 2001 From: Slawomir Pajak Date: Mon, 16 Dec 2024 13:00:37 +0100 Subject: [PATCH 034/158] Fix misspelling --- .../java/io/trino/plugin/deltalake/TestSplitPruning.java | 2 +- .../java/io/trino/plugin/iceberg/ExpressionConverter.java | 4 ++-- .../main/java/io/trino/plugin/iceberg/IcebergMetadata.java | 6 +++--- .../java/io/trino/plugin/iceberg/IcebergSplitSource.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java index fe3acd47323a..6360b306440b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestSplitPruning.java @@ -253,7 +253,7 @@ public void testPartitionPruningWithExpression() /** * Test that partition filter that cannot be converted to a {@link io.trino.spi.predicate.Domain} - * gets applied (and not forgotten) when there is another, Domain-convertable filter. + * gets applied (and not forgotten) when there is another, Domain-convertible filter. *

* In the past, that caused a significant decrease in the connector's performance. */ diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/ExpressionConverter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/ExpressionConverter.java index eb647f10fe3d..f63fc00bb9c8 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/ExpressionConverter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/ExpressionConverter.java @@ -52,7 +52,7 @@ public final class ExpressionConverter { private ExpressionConverter() {} - public static boolean isConvertableToIcebergExpression(Domain domain) + public static boolean isConvertibleToIcebergExpression(Domain domain) { if (isStructuralType(domain.getType())) { // structural types cannot be used to filter a table scan in Iceberg library. @@ -81,7 +81,7 @@ public static Expression toIcebergExpression(TupleDomain tu IcebergColumnHandle columnHandle = entry.getKey(); checkArgument(!isMetadataColumnId(columnHandle.getId()), "Constraint on an unexpected column %s", columnHandle); Domain domain = entry.getValue(); - checkArgument(isConvertableToIcebergExpression(domain), "Unexpected not convertable domain on column %s: %s", columnHandle, domain); + checkArgument(isConvertibleToIcebergExpression(domain), "Unexpected not convertible domain on column %s: %s", columnHandle, domain); conjuncts.add(toIcebergExpression(columnHandle.getQualifiedName(), columnHandle.getType(), domain)); } return and(conjuncts); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 2a6af35d5c1d..7e51d89d68f3 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -237,7 +237,7 @@ import static io.trino.plugin.hive.util.HiveUtil.isHudiTable; import static io.trino.plugin.hive.util.HiveUtil.isIcebergTable; import static io.trino.plugin.iceberg.ColumnIdentity.createColumnIdentity; -import static io.trino.plugin.iceberg.ExpressionConverter.isConvertableToIcebergExpression; +import static io.trino.plugin.iceberg.ExpressionConverter.isConvertibleToIcebergExpression; import static io.trino.plugin.iceberg.ExpressionConverter.toIcebergExpression; import static io.trino.plugin.iceberg.IcebergAnalyzeProperties.getColumnNames; import static io.trino.plugin.iceberg.IcebergColumnHandle.TRINO_MERGE_PARTITION_DATA; @@ -2867,7 +2867,7 @@ private void finishWrite(ConnectorSession session, IcebergTableHandle table, Col RowDelta rowDelta = transaction.newRowDelta(); table.getSnapshotId().map(icebergTable::snapshot).ifPresent(s -> rowDelta.validateFromSnapshot(s.snapshotId())); TupleDomain dataColumnPredicate = table.getEnforcedPredicate().filter((column, domain) -> !isMetadataColumnId(column.getId())); - TupleDomain convertibleUnenforcedPredicate = table.getUnenforcedPredicate().filter((_, domain) -> isConvertableToIcebergExpression(domain)); + TupleDomain convertibleUnenforcedPredicate = table.getUnenforcedPredicate().filter((_, domain) -> isConvertibleToIcebergExpression(domain)); TupleDomain effectivePredicate = dataColumnPredicate.intersect(convertibleUnenforcedPredicate); if (!effectivePredicate.isAll()) { rowDelta.conflictDetectionFilter(toIcebergExpression(effectivePredicate)); @@ -3112,7 +3112,7 @@ public Optional> applyFilter(C Map newUnenforced = new LinkedHashMap<>(); Map domains = predicate.getDomains().orElseThrow(() -> new VerifyException("No domains")); domains.forEach((columnHandle, domain) -> { - if (!isConvertableToIcebergExpression(domain)) { + if (!isConvertibleToIcebergExpression(domain)) { unsupported.put(columnHandle, domain); } else if (canEnforceColumnConstraintInSpecs(typeManager.getTypeOperators(), icebergTable, partitionSpecIds, columnHandle, domain)) { diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitSource.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitSource.java index 1d7dd3fa8222..878add85ff2d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitSource.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitSource.java @@ -90,7 +90,7 @@ import static io.airlift.slice.Slices.utf8Slice; import static io.trino.cache.CacheUtils.uncheckedCacheGet; import static io.trino.cache.SafeCaches.buildNonEvictableCache; -import static io.trino.plugin.iceberg.ExpressionConverter.isConvertableToIcebergExpression; +import static io.trino.plugin.iceberg.ExpressionConverter.isConvertibleToIcebergExpression; import static io.trino.plugin.iceberg.ExpressionConverter.toIcebergExpression; import static io.trino.plugin.iceberg.IcebergExceptions.translateMetadataException; import static io.trino.plugin.iceberg.IcebergMetadataColumn.isMetadataColumnId; @@ -256,7 +256,7 @@ private synchronized ConnectorSplitBatch getNextBatchInternal(int maxSize) if (fileScanIterable == null) { this.pushedDownDynamicFilterPredicate = dynamicFilter.getCurrentPredicate() .transformKeys(IcebergColumnHandle.class::cast) - .filter((columnHandle, domain) -> isConvertableToIcebergExpression(domain)); + .filter((columnHandle, domain) -> isConvertibleToIcebergExpression(domain)); TupleDomain effectivePredicate = TupleDomain.intersect( ImmutableList.of(dataColumnPredicate, tableHandle.getUnenforcedPredicate(), pushedDownDynamicFilterPredicate)); From 9217692698b006aabad145026918c776ae54a9fd Mon Sep 17 00:00:00 2001 From: Slawomir Pajak Date: Mon, 16 Dec 2024 15:22:15 +0100 Subject: [PATCH 035/158] Test concurrent update without partition --- .../iceberg/TestIcebergLocalConcurrentWrites.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java index 65253a37868d..903fff33da5e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java @@ -393,13 +393,22 @@ void testConcurrentNonOverlappingUpdate() @Test void testConcurrentOverlappingUpdate() throws Exception + { + testConcurrentOverlappingUpdate(false); + testConcurrentOverlappingUpdate(true); + } + + private void testConcurrentOverlappingUpdate(boolean partitioned) + throws Exception { int threads = 3; CyclicBarrier barrier = new CyclicBarrier(threads); ExecutorService executor = newFixedThreadPool(threads); String tableName = "test_concurrent_overlapping_updates_table_" + randomNameSuffix(); - assertUpdate("CREATE TABLE " + tableName + " (a, part) WITH (partitioning = ARRAY['part']) AS VALUES (1, 10), (11, 20), (21, 30), (31, 40)", 4); + assertUpdate("CREATE TABLE " + tableName + " (a, part) " + + (partitioned ? " WITH (partitioning = ARRAY['part'])" : "") + + " AS VALUES (1, 10), (11, 20), (21, 30), (31, 40)", 4); try { List> futures = IntStream.range(0, threads) From 7f0fa941ec3f70741afea84a40b24249bade8e88 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 18 Dec 2024 07:21:58 +0900 Subject: [PATCH 036/158] Extract KuduColumnProperties from KuduTableProperties --- .../trino/plugin/kudu/KuduClientSession.java | 7 +- .../io/trino/plugin/kudu/KuduConnector.java | 6 +- .../io/trino/plugin/kudu/KuduMetadata.java | 15 +- .../java/io/trino/plugin/kudu/KuduModule.java | 2 + .../kudu/properties/KuduColumnProperties.java | 151 ++++++++++++++++++ .../kudu/properties/KuduTableProperties.java | 117 -------------- 6 files changed, 170 insertions(+), 128 deletions(-) create mode 100644 plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduColumnProperties.java diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java index eec1ac45dd8a..139c9e7b132c 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduClientSession.java @@ -19,6 +19,7 @@ import io.airlift.slice.Slices; import io.trino.plugin.kudu.properties.ColumnDesign; import io.trino.plugin.kudu.properties.HashPartitionDefinition; +import io.trino.plugin.kudu.properties.KuduColumnProperties; import io.trino.plugin.kudu.properties.KuduTableProperties; import io.trino.plugin.kudu.properties.PartitionDesign; import io.trino.plugin.kudu.properties.RangePartition; @@ -416,7 +417,7 @@ private Schema buildSchema(List columns) private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata) { String name = columnMetadata.getName(); - ColumnDesign design = KuduTableProperties.getColumnDesign(columnMetadata.getProperties()); + ColumnDesign design = KuduColumnProperties.getColumnDesign(columnMetadata.getProperties()); Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType()); ColumnSchemaBuilder builder = new ColumnSchemaBuilder(name, ktype); builder.key(design.isPrimaryKey()).nullable(design.isNullable()); @@ -440,7 +441,7 @@ private void setCompression(String name, ColumnSchemaBuilder builder, ColumnDesi { if (design.getCompression() != null) { try { - CompressionAlgorithm algorithm = KuduTableProperties.lookupCompression(design.getCompression()); + CompressionAlgorithm algorithm = KuduColumnProperties.lookupCompression(design.getCompression()); builder.compressionAlgorithm(algorithm); } catch (IllegalArgumentException e) { @@ -453,7 +454,7 @@ private void setEncoding(String name, ColumnSchemaBuilder builder, ColumnDesign { if (design.getEncoding() != null) { try { - Encoding encoding = KuduTableProperties.lookupEncoding(design.getEncoding()); + Encoding encoding = KuduColumnProperties.lookupEncoding(design.getEncoding()); builder.encoding(encoding); } catch (IllegalArgumentException e) { diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduConnector.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduConnector.java index 062533db4502..d798e1923b86 100755 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduConnector.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduConnector.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import io.airlift.bootstrap.LifeCycleManager; +import io.trino.plugin.kudu.properties.KuduColumnProperties; import io.trino.plugin.kudu.properties.KuduTableProperties; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorMetadata; @@ -44,6 +45,7 @@ public class KuduConnector private final ConnectorSplitManager splitManager; private final ConnectorPageSourceProvider pageSourceProvider; private final KuduTableProperties tableProperties; + private final KuduColumnProperties columnProperties; private final ConnectorPageSinkProvider pageSinkProvider; private final Set procedures; private final ConnectorNodePartitioningProvider nodePartitioningProvider; @@ -55,6 +57,7 @@ public KuduConnector( KuduMetadata metadata, ConnectorSplitManager splitManager, KuduTableProperties tableProperties, + KuduColumnProperties columnProperties, ConnectorPageSourceProvider pageSourceProvider, ConnectorPageSinkProvider pageSinkProvider, Set procedures, @@ -66,6 +69,7 @@ public KuduConnector( this.splitManager = requireNonNull(splitManager, "splitManager is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); this.tableProperties = requireNonNull(tableProperties, "tableProperties is null"); + this.columnProperties = requireNonNull(columnProperties, "columnProperties is null"); this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); this.procedures = ImmutableSet.copyOf(requireNonNull(procedures, "procedures is null")); this.nodePartitioningProvider = requireNonNull(nodePartitioningProvider, "nodePartitioningProvider is null"); @@ -112,7 +116,7 @@ public List> getTableProperties() @Override public List> getColumnProperties() { - return tableProperties.getColumnProperties(); + return columnProperties.getColumnProperties(); } @Override diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduMetadata.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduMetadata.java index 511a424be7b1..49d5bb64cc54 100755 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduMetadata.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduMetadata.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import io.airlift.slice.Slice; +import io.trino.plugin.kudu.properties.KuduColumnProperties; import io.trino.plugin.kudu.properties.KuduTableProperties; import io.trino.plugin.kudu.properties.PartitionDesign; import io.trino.spi.TrinoException; @@ -139,24 +140,24 @@ private ColumnMetadata getColumnMetadata(ColumnSchema column) Map properties = new LinkedHashMap<>(); StringBuilder extra = new StringBuilder(); if (column.isKey()) { - properties.put(KuduTableProperties.PRIMARY_KEY, true); + properties.put(KuduColumnProperties.PRIMARY_KEY, true); extra.append("primary_key, "); } if (column.isNullable()) { - properties.put(KuduTableProperties.NULLABLE, true); + properties.put(KuduColumnProperties.NULLABLE, true); extra.append("nullable, "); } - String encoding = KuduTableProperties.lookupEncodingString(column.getEncoding()); + String encoding = KuduColumnProperties.lookupEncodingString(column.getEncoding()); if (column.getEncoding() != ColumnSchema.Encoding.AUTO_ENCODING) { - properties.put(KuduTableProperties.ENCODING, encoding); + properties.put(KuduColumnProperties.ENCODING, encoding); } extra.append("encoding=").append(encoding).append(", "); - String compression = KuduTableProperties.lookupCompressionString(column.getCompressionAlgorithm()); + String compression = KuduColumnProperties.lookupCompressionString(column.getCompressionAlgorithm()); if (column.getCompressionAlgorithm() != ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION) { - properties.put(KuduTableProperties.COMPRESSION, compression); + properties.put(KuduColumnProperties.COMPRESSION, compression); } extra.append("compression=").append(compression); @@ -374,7 +375,7 @@ public ConnectorOutputTableHandle beginCreateTable( String rowId = ROW_ID; List copy = new ArrayList<>(tableMetadata.getColumns()); Map columnProperties = new HashMap<>(); - columnProperties.put(KuduTableProperties.PRIMARY_KEY, true); + columnProperties.put(KuduColumnProperties.PRIMARY_KEY, true); copy.add(0, ColumnMetadata.builder() .setName(rowId) .setType(VarcharType.VARCHAR) diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java index 6dd9a977e643..628f21274fd7 100755 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduModule.java @@ -21,6 +21,7 @@ import io.trino.plugin.base.classloader.ClassLoaderSafeNodePartitioningProvider; import io.trino.plugin.base.classloader.ForClassLoaderSafe; import io.trino.plugin.kudu.procedures.RangePartitionProcedures; +import io.trino.plugin.kudu.properties.KuduColumnProperties; import io.trino.plugin.kudu.properties.KuduTableProperties; import io.trino.spi.connector.ConnectorNodePartitioningProvider; import io.trino.spi.connector.ConnectorPageSinkProvider; @@ -50,6 +51,7 @@ protected void setup(Binder binder) binder.bind(KuduConnector.class).in(Scopes.SINGLETON); binder.bind(KuduMetadata.class).in(Scopes.SINGLETON); binder.bind(KuduTableProperties.class).in(Scopes.SINGLETON); + binder.bind(KuduColumnProperties.class).in(Scopes.SINGLETON); binder.bind(ConnectorSplitManager.class).to(KuduSplitManager.class).in(Scopes.SINGLETON); binder.bind(ConnectorPageSourceProvider.class).to(KuduPageSourceProvider.class) .in(Scopes.SINGLETON); diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduColumnProperties.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduColumnProperties.java new file mode 100644 index 000000000000..778d160276bc --- /dev/null +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduColumnProperties.java @@ -0,0 +1,151 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kudu.properties; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.spi.session.PropertyMetadata; +import org.apache.kudu.ColumnSchema; + +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static io.trino.spi.session.PropertyMetadata.booleanProperty; +import static io.trino.spi.session.PropertyMetadata.stringProperty; +import static java.util.Objects.requireNonNull; + +public final class KuduColumnProperties +{ + public static final String PRIMARY_KEY = "primary_key"; + public static final String NULLABLE = "nullable"; + public static final String ENCODING = "encoding"; + public static final String COMPRESSION = "compression"; + + private final List> columnProperties; + + @Inject + public KuduColumnProperties() + { + columnProperties = ImmutableList.>builder() + .add(booleanProperty( + PRIMARY_KEY, + "If column belongs to primary key", + false, + false)) + .add(booleanProperty( + NULLABLE, + "If column can be set to null", + false, + false)) + .add(stringProperty( + ENCODING, + "Optional specification of the column encoding. Otherwise default encoding is applied.", + null, + false)) + .add(stringProperty( + COMPRESSION, + "Optional specification of the column compression. Otherwise default compression is applied.", + null, + false)) + .build(); + } + + public List> getColumnProperties() + { + return columnProperties; + } + + public static ColumnDesign getColumnDesign(Map columnProperties) + { + requireNonNull(columnProperties); + if (columnProperties.isEmpty()) { + return ColumnDesign.DEFAULT; + } + + ColumnDesign design = new ColumnDesign(); + Boolean key = (Boolean) columnProperties.get(PRIMARY_KEY); + if (key != null) { + design.setPrimaryKey(key); + } + + Boolean nullable = (Boolean) columnProperties.get(NULLABLE); + if (nullable != null) { + design.setNullable(nullable); + } + + String encoding = (String) columnProperties.get(ENCODING); + if (encoding != null) { + design.setEncoding(encoding); + } + + String compression = (String) columnProperties.get(COMPRESSION); + if (compression != null) { + design.setCompression(compression); + } + return design; + } + + public static ColumnSchema.CompressionAlgorithm lookupCompression(String compression) + { + return switch (compression.toLowerCase(Locale.ENGLISH)) { + case "default", "default_compression" -> ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION; + case "no", "no_compression" -> ColumnSchema.CompressionAlgorithm.NO_COMPRESSION; + case "lz4" -> ColumnSchema.CompressionAlgorithm.LZ4; + case "snappy" -> ColumnSchema.CompressionAlgorithm.SNAPPY; + case "zlib" -> ColumnSchema.CompressionAlgorithm.ZLIB; + default -> throw new IllegalArgumentException(); + }; + } + + public static String lookupCompressionString(ColumnSchema.CompressionAlgorithm algorithm) + { + return switch (algorithm) { + case DEFAULT_COMPRESSION -> "default"; + case NO_COMPRESSION -> "no"; + case LZ4 -> "lz4"; + case SNAPPY -> "snappy"; + case ZLIB -> "zlib"; + default -> "unknown"; + }; + } + + public static ColumnSchema.Encoding lookupEncoding(String encoding) + { + return switch (encoding.toLowerCase(Locale.ENGLISH)) { + case "auto", "auto_encoding" -> ColumnSchema.Encoding.AUTO_ENCODING; + case "bitshuffle", "bit_shuffle" -> ColumnSchema.Encoding.BIT_SHUFFLE; + case "dictionary", "dict_encoding" -> ColumnSchema.Encoding.DICT_ENCODING; + case "plain", "plain_encoding" -> ColumnSchema.Encoding.PLAIN_ENCODING; + case "prefix", "prefix_encoding" -> ColumnSchema.Encoding.PREFIX_ENCODING; + case "runlength", "run_length", "run length", "rle" -> ColumnSchema.Encoding.RLE; + case "group_varint" -> ColumnSchema.Encoding.GROUP_VARINT; + default -> throw new IllegalArgumentException(); + }; + } + + public static String lookupEncodingString(ColumnSchema.Encoding encoding) + { + return switch (encoding) { + case AUTO_ENCODING -> "auto"; + case BIT_SHUFFLE -> "bitshuffle"; + case DICT_ENCODING -> "dictionary"; + case PLAIN_ENCODING -> "plain"; + case PREFIX_ENCODING -> "prefix"; + case RLE -> "runlength"; + case GROUP_VARINT -> "group_varint"; + default -> "unknown"; + }; + } +} diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduTableProperties.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduTableProperties.java index 909a3f7cc86f..588bc0f9b985 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduTableProperties.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/properties/KuduTableProperties.java @@ -36,14 +36,12 @@ import java.util.Base64; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.plugin.base.util.JsonUtils.parseJson; import static io.trino.spi.StandardErrorCode.GENERIC_USER_ERROR; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; import static io.trino.spi.session.PropertyMetadata.integerProperty; import static io.trino.spi.session.PropertyMetadata.stringProperty; import static io.trino.spi.type.VarcharType.VARCHAR; @@ -59,10 +57,6 @@ public final class KuduTableProperties public static final String PARTITION_BY_RANGE_COLUMNS = "partition_by_range_columns"; public static final String RANGE_PARTITIONS = "range_partitions"; public static final String NUM_REPLICAS = "number_of_replicas"; - public static final String PRIMARY_KEY = "primary_key"; - public static final String NULLABLE = "nullable"; - public static final String ENCODING = "encoding"; - public static final String COMPRESSION = "compression"; private static final ObjectMapper mapper = new ObjectMapper(); @@ -70,8 +64,6 @@ public final class KuduTableProperties private final List> tableProperties; - private final List> columnProperties; - @Inject public KuduTableProperties() { @@ -129,28 +121,6 @@ public KuduTableProperties() "Initial range partitions as JSON", null, false)); - - columnProperties = ImmutableList.of( - booleanProperty( - PRIMARY_KEY, - "If column belongs to primary key", - false, - false), - booleanProperty( - NULLABLE, - "If column can be set to null", - false, - false), - stringProperty( - ENCODING, - "Optional specification of the column encoding. Otherwise default encoding is applied.", - null, - false), - stringProperty( - COMPRESSION, - "Optional specification of the column compression. Otherwise default compression is applied.", - null, - false)); } public List> getTableProperties() @@ -158,11 +128,6 @@ public List> getTableProperties() return tableProperties; } - public List> getColumnProperties() - { - return columnProperties; - } - public static PartitionDesign getPartitionDesign(Map tableProperties) { requireNonNull(tableProperties); @@ -196,36 +161,6 @@ else if (!hashColumns2.isEmpty()) { return design; } - public static ColumnDesign getColumnDesign(Map columnProperties) - { - requireNonNull(columnProperties); - if (columnProperties.isEmpty()) { - return ColumnDesign.DEFAULT; - } - - ColumnDesign design = new ColumnDesign(); - Boolean key = (Boolean) columnProperties.get(PRIMARY_KEY); - if (key != null) { - design.setPrimaryKey(key); - } - - Boolean nullable = (Boolean) columnProperties.get(NULLABLE); - if (nullable != null) { - design.setNullable(nullable); - } - - String encoding = (String) columnProperties.get(ENCODING); - if (encoding != null) { - design.setEncoding(encoding); - } - - String compression = (String) columnProperties.get(COMPRESSION); - if (compression != null) { - design.setCompression(compression); - } - return design; - } - private static HashPartitionDefinition getHashPartitionDefinition(Map tableProperties, List columns, String bucketPropertyName) { Integer hashBuckets = (Integer) tableProperties.get(bucketPropertyName); @@ -542,56 +477,4 @@ private static void handleInvalidValue(String name, Type type, Object obj) { throw new IllegalStateException("Invalid value " + obj + " for column " + name + " of type " + type); } - - public static ColumnSchema.CompressionAlgorithm lookupCompression(String compression) - { - return switch (compression.toLowerCase(Locale.ENGLISH)) { - case "default", "default_compression" -> ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION; - case "no", "no_compression" -> ColumnSchema.CompressionAlgorithm.NO_COMPRESSION; - case "lz4" -> ColumnSchema.CompressionAlgorithm.LZ4; - case "snappy" -> ColumnSchema.CompressionAlgorithm.SNAPPY; - case "zlib" -> ColumnSchema.CompressionAlgorithm.ZLIB; - default -> throw new IllegalArgumentException(); - }; - } - - public static String lookupCompressionString(ColumnSchema.CompressionAlgorithm algorithm) - { - return switch (algorithm) { - case DEFAULT_COMPRESSION -> "default"; - case NO_COMPRESSION -> "no"; - case LZ4 -> "lz4"; - case SNAPPY -> "snappy"; - case ZLIB -> "zlib"; - default -> "unknown"; - }; - } - - public static ColumnSchema.Encoding lookupEncoding(String encoding) - { - return switch (encoding.toLowerCase(Locale.ENGLISH)) { - case "auto", "auto_encoding" -> ColumnSchema.Encoding.AUTO_ENCODING; - case "bitshuffle", "bit_shuffle" -> ColumnSchema.Encoding.BIT_SHUFFLE; - case "dictionary", "dict_encoding" -> ColumnSchema.Encoding.DICT_ENCODING; - case "plain", "plain_encoding" -> ColumnSchema.Encoding.PLAIN_ENCODING; - case "prefix", "prefix_encoding" -> ColumnSchema.Encoding.PREFIX_ENCODING; - case "runlength", "run_length", "run length", "rle" -> ColumnSchema.Encoding.RLE; - case "group_varint" -> ColumnSchema.Encoding.GROUP_VARINT; - default -> throw new IllegalArgumentException(); - }; - } - - public static String lookupEncodingString(ColumnSchema.Encoding encoding) - { - return switch (encoding) { - case AUTO_ENCODING -> "auto"; - case BIT_SHUFFLE -> "bitshuffle"; - case DICT_ENCODING -> "dictionary"; - case PLAIN_ENCODING -> "plain"; - case PREFIX_ENCODING -> "prefix"; - case RLE -> "runlength"; - case GROUP_VARINT -> "group_varint"; - default -> "unknown"; - }; - } } From 6c52253c3411d2e117e2f97cf4cc3f57a43501bb Mon Sep 17 00:00:00 2001 From: Alex Jo Date: Wed, 18 Dec 2024 14:18:55 -0700 Subject: [PATCH 037/158] Fix S3InputStream's handling of large skips When the skip(n) method is called the MAX_SKIP_BYTES check is skipped, resulting in the call potentially blocking for a long time. Instead of delegating to the underlying stream, set the nextReadPosition value. This allows the next read to decide if it is best to keep the existing s3 object stream or open a new one. This behavior matches the implementations for Azure and GCS. --- .../java/io/trino/filesystem/s3/S3InputStream.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputStream.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputStream.java index 6bcbc6e128ee..0122cf1093a8 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputStream.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputStream.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.InterruptedIOException; +import static java.lang.Math.clamp; import static java.lang.Math.max; import static java.util.Objects.requireNonNull; @@ -126,14 +127,10 @@ public long skip(long n) throws IOException { ensureOpen(); - seekStream(false); - return reconnectStreamIfNecessary(() -> { - long skip = doSkip(n); - streamPosition += skip; - nextReadPosition += skip; - return skip; - }); + long skipSize = clamp(n, 0, length != null ? length - nextReadPosition : Integer.MAX_VALUE); + nextReadPosition += skipSize; + return skipSize; } @Override From a54bc17d717b44827aad65a5ae19a0e55be3b5a9 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Thu, 19 Dec 2024 14:55:32 +0100 Subject: [PATCH 038/158] Update google cloud SDK to 26.52.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 134ea6c48ef0..330dac5a343a 100644 --- a/pom.xml +++ b/pom.xml @@ -197,7 +197,7 @@ 1.15.1 v22.11.0 10.9.0 - 1.45.1 + 1.45.3 5.3.1 1.7.1 5.15.0 @@ -239,7 +239,7 @@ com.google.cloud libraries-bom - 26.51.0 + 26.52.0 pom import From c1b608c28a10eb50c123fda119987db33b973eaa Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Thu, 19 Dec 2024 14:56:50 +0100 Subject: [PATCH 039/158] Update AWS SDK v2 to 2.29.37 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 330dac5a343a..2b17d8681c88 100644 --- a/pom.xml +++ b/pom.xml @@ -311,7 +311,7 @@ software.amazon.awssdk bom - 2.29.35 + 2.29.37 pom import From d7aba15ecd0dbf201f1f2ba14593c6618a4e05ca Mon Sep 17 00:00:00 2001 From: mduggan-starburst Date: Thu, 19 Dec 2024 13:14:24 -0500 Subject: [PATCH 040/158] Use QUERY_EXCEEDED_COMPILER_LIMIT error code --- .../src/main/java/io/trino/util/CompilerUtils.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/trino-main/src/main/java/io/trino/util/CompilerUtils.java b/core/trino-main/src/main/java/io/trino/util/CompilerUtils.java index 4b371a5dd8b9..931f50750dd0 100644 --- a/core/trino-main/src/main/java/io/trino/util/CompilerUtils.java +++ b/core/trino-main/src/main/java/io/trino/util/CompilerUtils.java @@ -17,6 +17,8 @@ import io.airlift.bytecode.DynamicClassLoader; import io.airlift.bytecode.ParameterizedType; import io.airlift.log.Logger; +import io.trino.spi.TrinoException; +import org.objectweb.asm.MethodTooLargeException; import java.lang.invoke.MethodHandle; import java.time.Instant; @@ -28,6 +30,7 @@ import static io.airlift.bytecode.BytecodeUtils.toJavaIdentifierString; import static io.airlift.bytecode.ClassGenerator.classGenerator; import static io.airlift.bytecode.ParameterizedType.typeFromJavaClassName; +import static io.trino.spi.StandardErrorCode.QUERY_EXCEEDED_COMPILER_LIMIT; import static java.time.ZoneOffset.UTC; public final class CompilerUtils @@ -78,6 +81,11 @@ public static Class defineClass(ClassDefinition classDefinition public static Class defineClass(ClassDefinition classDefinition, Class superType, DynamicClassLoader classLoader) { log.debug("Defining class: %s", classDefinition.getName()); - return classGenerator(classLoader).defineClass(classDefinition, superType); + try { + return classGenerator(classLoader).defineClass(classDefinition, superType); + } + catch (MethodTooLargeException e) { + throw new TrinoException(QUERY_EXCEEDED_COMPILER_LIMIT, "Query exceeded maximum method size.", e); + } } } From bb9d6483aebe7182ff201356d522be089ad9a146 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Thu, 19 Dec 2024 08:17:10 +0900 Subject: [PATCH 041/158] Include deletion vector when filtering active add entries in Delta --- .../transactionlog/DeletionVectorEntry.java | 10 +++++++++ .../transactionlog/TransactionLogAccess.java | 21 ++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeletionVectorEntry.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeletionVectorEntry.java index cf23159250f1..b1c0c2805787 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeletionVectorEntry.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeletionVectorEntry.java @@ -34,6 +34,16 @@ public record DeletionVectorEntry(String storageType, String pathOrInlineDv, Opt requireNonNull(offset, "offset is null"); } + // https://github.com/delta-io/delta/blob/34f02d8/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java#L167-L174 + public String uniqueId() + { + String uniqueFileId = storageType + pathOrInlineDv; + if (offset.isPresent()) { + return uniqueFileId + "@" + offset; + } + return uniqueFileId; + } + public long getRetainedSizeInBytes() { return INSTANCE_SIZE diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java index 27ecdbe7cb58..a2b76fa3e4dc 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java @@ -429,21 +429,23 @@ public static ImmutableList columnsWithStats(List activeAddEntries(Stream checkpointEntries, List transactions) { - Map activeJsonEntries = new LinkedHashMap<>(); - HashSet removedFiles = new HashSet<>(); + Map activeJsonEntries = new LinkedHashMap<>(); + HashSet removedFiles = new HashSet<>(); // The json entries containing the last few entries in the log need to be applied on top of the parquet snapshot: // - Any files which have been removed need to be excluded // - Any files with newer add actions need to be updated with the most recent metadata transactions.forEach(transaction -> { - Map addFilesInTransaction = new LinkedHashMap<>(); - Set removedFilesInTransaction = new HashSet<>(); + Map addFilesInTransaction = new LinkedHashMap<>(); + Set removedFilesInTransaction = new HashSet<>(); transaction.transactionEntries().forEach(deltaLakeTransactionLogEntry -> { if (deltaLakeTransactionLogEntry.getAdd() != null) { - addFilesInTransaction.put(deltaLakeTransactionLogEntry.getAdd().getPath(), deltaLakeTransactionLogEntry.getAdd()); + AddFileEntry add = deltaLakeTransactionLogEntry.getAdd(); + addFilesInTransaction.put(new FileEntryKey(add.getPath(), add.getDeletionVector().map(DeletionVectorEntry::uniqueId)), add); } else if (deltaLakeTransactionLogEntry.getRemove() != null) { - removedFilesInTransaction.add(deltaLakeTransactionLogEntry.getRemove().path()); + RemoveFileEntry remove = deltaLakeTransactionLogEntry.getRemove(); + removedFilesInTransaction.add(new FileEntryKey(remove.path(), remove.deletionVector().map(DeletionVectorEntry::uniqueId))); } }); @@ -456,11 +458,16 @@ else if (deltaLakeTransactionLogEntry.getRemove() != null) { Stream filteredCheckpointEntries = checkpointEntries .map(DeltaLakeTransactionLogEntry::getAdd) .filter(Objects::nonNull) - .filter(addEntry -> !removedFiles.contains(addEntry.getPath()) && !activeJsonEntries.containsKey(addEntry.getPath())); + .filter(addEntry -> { + FileEntryKey key = new FileEntryKey(addEntry.getPath(), addEntry.getDeletionVector().map(DeletionVectorEntry::uniqueId)); + return !removedFiles.contains(key) && !activeJsonEntries.containsKey(key); + }); return Stream.concat(filteredCheckpointEntries, activeJsonEntries.values().stream()); } + private record FileEntryKey(String path, Optional deletionVectorId) {} + public Stream getRemoveEntries(ConnectorSession session, TableSnapshot tableSnapshot) { return getEntries( From 92c1fdd1cca1266aa77275b80d0e2abcf7898ffd Mon Sep 17 00:00:00 2001 From: chenjian2664 Date: Tue, 17 Dec 2024 15:25:02 +0800 Subject: [PATCH 042/158] Add nonnull check for directExchangeClientSupplier Refactor the PlanTester to pass the nonnull `directExchangeClientSupplier`. Also add nonnull check for the `sourceId`, `serdeFactory` in `ExchangeOperator` --- .../src/main/java/io/trino/operator/ExchangeOperator.java | 6 +++--- .../java/io/trino/sql/planner/LocalExecutionPlanner.java | 2 +- .../src/main/java/io/trino/testing/PlanTester.java | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java b/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java index 2c1c42b8e0df..ad44dc66b8f5 100644 --- a/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java @@ -70,9 +70,9 @@ public ExchangeOperatorFactory( ExchangeManagerRegistry exchangeManagerRegistry) { this.operatorId = operatorId; - this.sourceId = sourceId; - this.directExchangeClientSupplier = directExchangeClientSupplier; - this.serdeFactory = serdeFactory; + this.sourceId = requireNonNull(sourceId, "sourceId is null"); + this.directExchangeClientSupplier = requireNonNull(directExchangeClientSupplier, "directExchangeClientSupplier is null"); + this.serdeFactory = requireNonNull(serdeFactory, "serdeFactory is null"); this.retryPolicy = requireNonNull(retryPolicy, "retryPolicy is null"); this.exchangeManagerRegistry = requireNonNull(exchangeManagerRegistry, "exchangeManagerRegistry is null"); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java index 2ec18558dd32..cbdf2c3203ac 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java @@ -508,7 +508,7 @@ public LocalExecutionPlanner( this.pageSourceManager = requireNonNull(pageSourceManager, "pageSourceManager is null"); this.indexManager = requireNonNull(indexManager, "indexManager is null"); this.nodePartitioningManager = requireNonNull(nodePartitioningManager, "nodePartitioningManager is null"); - this.directExchangeClientSupplier = directExchangeClientSupplier; + this.directExchangeClientSupplier = requireNonNull(directExchangeClientSupplier, "directExchangeClientSupplier is null"); this.pageSinkManager = requireNonNull(pageSinkManager, "pageSinkManager is null"); this.expressionCompiler = requireNonNull(expressionCompiler, "expressionCompiler is null"); this.pageFunctionCompiler = requireNonNull(pageFunctionCompiler, "pageFunctionCompiler is null"); diff --git a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java index 1b1a226da4e9..28a3992337d0 100644 --- a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java +++ b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java @@ -745,7 +745,9 @@ private List createDrivers(Session session, @Language("SQL") String sql) indexManager, nodePartitioningManager, pageSinkManager, - null, + (_, _, _, _, _, _) -> { + throw new UnsupportedOperationException(); + }, expressionCompiler, pageFunctionCompiler, joinFilterFunctionCompiler, From bbf15b5b630e40a899ca39b738548c5bcdd5a8ee Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Tue, 19 Nov 2024 19:06:59 +0900 Subject: [PATCH 043/158] Make FilesTable.toJson method package-private --- .../io/trino/plugin/iceberg/FilesTable.java | 140 +++++++++--------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/FilesTable.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/FilesTable.java index 715941c12f1f..319a1376b209 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/FilesTable.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/FilesTable.java @@ -356,80 +356,12 @@ private List getRecord(StructLike structLike) columns.add(structLike.get(columnNameToPosition.get(SORT_ORDER_ID_COLUMN_NAME), Integer.class)); ReadableMetricsStruct readableMetrics = structLike.get(columnNameToPosition.get(READABLE_METRICS_COLUMN_NAME), ReadableMetricsStruct.class); - columns.add(toJson(readableMetrics)); + columns.add(toJson(readableMetrics, primitiveFields)); checkArgument(columns.size() == types.size(), "Expected %s types in row, but got %s values", types.size(), columns.size()); return columns; } - private String toJson(ReadableMetricsStruct readableMetrics) - { - StringWriter writer = new StringWriter(); - try { - JsonGenerator generator = JSON_FACTORY.createGenerator(writer); - generator.writeStartObject(); - - for (int i = 0; i < readableMetrics.size(); i++) { - Types.NestedField field = primitiveFields.get(i); - generator.writeFieldName(field.name()); - - generator.writeStartObject(); - ReadableColMetricsStruct columnMetrics = readableMetrics.get(i, ReadableColMetricsStruct.class); - - generator.writeFieldName("column_size"); - Long columnSize = columnMetrics.get(0, Long.class); - if (columnSize == null) { - generator.writeNull(); - } - else { - generator.writeNumber(columnSize); - } - - generator.writeFieldName("value_count"); - Long valueCount = columnMetrics.get(1, Long.class); - if (valueCount == null) { - generator.writeNull(); - } - else { - generator.writeNumber(valueCount); - } - - generator.writeFieldName("null_value_count"); - Long nullValueCount = columnMetrics.get(2, Long.class); - if (nullValueCount == null) { - generator.writeNull(); - } - else { - generator.writeNumber(nullValueCount); - } - - generator.writeFieldName("nan_value_count"); - Long nanValueCount = columnMetrics.get(3, Long.class); - if (nanValueCount == null) { - generator.writeNull(); - } - else { - generator.writeNumber(nanValueCount); - } - - generator.writeFieldName("lower_bound"); - SingleValueParser.toJson(field.type(), columnMetrics.get(4, Object.class), generator); - - generator.writeFieldName("upper_bound"); - SingleValueParser.toJson(field.type(), columnMetrics.get(5, Object.class), generator); - - generator.writeEndObject(); - } - - generator.writeEndObject(); - generator.flush(); - return writer.toString(); - } - catch (IOException e) { - throw new UncheckedIOException("JSON conversion failed for: " + readableMetrics, e); - } - } - private SqlMap getIntegerBigintSqlMap(Map value) { if (value == null) { @@ -506,7 +438,75 @@ private static Slice toVarbinarySlice(ByteBuffer value) } } - private static Map getIcebergIdToTypeMapping(Schema schema) + static String toJson(ReadableMetricsStruct readableMetrics, List primitiveFields) + { + StringWriter writer = new StringWriter(); + try { + JsonGenerator generator = JSON_FACTORY.createGenerator(writer); + generator.writeStartObject(); + + for (int i = 0; i < readableMetrics.size(); i++) { + Types.NestedField field = primitiveFields.get(i); + generator.writeFieldName(field.name()); + + generator.writeStartObject(); + ReadableColMetricsStruct columnMetrics = readableMetrics.get(i, ReadableColMetricsStruct.class); + + generator.writeFieldName("column_size"); + Long columnSize = columnMetrics.get(0, Long.class); + if (columnSize == null) { + generator.writeNull(); + } + else { + generator.writeNumber(columnSize); + } + + generator.writeFieldName("value_count"); + Long valueCount = columnMetrics.get(1, Long.class); + if (valueCount == null) { + generator.writeNull(); + } + else { + generator.writeNumber(valueCount); + } + + generator.writeFieldName("null_value_count"); + Long nullValueCount = columnMetrics.get(2, Long.class); + if (nullValueCount == null) { + generator.writeNull(); + } + else { + generator.writeNumber(nullValueCount); + } + + generator.writeFieldName("nan_value_count"); + Long nanValueCount = columnMetrics.get(3, Long.class); + if (nanValueCount == null) { + generator.writeNull(); + } + else { + generator.writeNumber(nanValueCount); + } + + generator.writeFieldName("lower_bound"); + SingleValueParser.toJson(field.type(), columnMetrics.get(4, Object.class), generator); + + generator.writeFieldName("upper_bound"); + SingleValueParser.toJson(field.type(), columnMetrics.get(5, Object.class), generator); + + generator.writeEndObject(); + } + + generator.writeEndObject(); + generator.flush(); + return writer.toString(); + } + catch (IOException e) { + throw new UncheckedIOException("JSON conversion failed for: " + readableMetrics, e); + } + } + + static Map getIcebergIdToTypeMapping(Schema schema) { ImmutableMap.Builder icebergIdToTypeMapping = ImmutableMap.builder(); for (Types.NestedField field : schema.columns()) { From 4db417903df4f295f2e214d810404017e84406dc Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Tue, 19 Nov 2024 19:07:16 +0900 Subject: [PATCH 044/158] Add $entries metadata table to Iceberg --- .../io/trino/plugin/iceberg/EntriesTable.java | 266 ++++++++++++++++++ .../trino/plugin/iceberg/IcebergMetadata.java | 1 + .../io/trino/plugin/iceberg/TableType.java | 1 + .../iceberg/BaseIcebergSystemTables.java | 100 ++++++- .../TestIcebergMetastoreAccessOperations.java | 9 +- ...estIcebergGlueCatalogAccessOperations.java | 9 +- 6 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java new file mode 100644 index 000000000000..0c8d54384192 --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java @@ -0,0 +1,266 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import com.google.common.collect.ImmutableList; +import io.trino.plugin.iceberg.util.PageListBuilder; +import io.trino.spi.block.ArrayBlockBuilder; +import io.trino.spi.block.MapBlockBuilder; +import io.trino.spi.block.RowBlockBuilder; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.RowType; +import io.trino.spi.type.TimeZoneKey; +import io.trino.spi.type.TypeManager; +import io.trino.spi.type.TypeSignature; +import jakarta.annotation.Nullable; +import org.apache.iceberg.MetricsUtil.ReadableMetricsStruct; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.Table; +import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructProjection; + +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutorService; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.slice.Slices.wrappedHeapBuffer; +import static io.trino.plugin.iceberg.FilesTable.getIcebergIdToTypeMapping; +import static io.trino.plugin.iceberg.IcebergTypes.convertIcebergValueToTrino; +import static io.trino.plugin.iceberg.IcebergUtil.getPartitionColumnType; +import static io.trino.plugin.iceberg.IcebergUtil.partitionTypes; +import static io.trino.plugin.iceberg.IcebergUtil.primitiveFieldTypes; +import static io.trino.plugin.iceberg.PartitionsTable.getAllPartitionFields; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.StandardTypes.JSON; +import static io.trino.spi.type.TypeSignature.mapType; +import static io.trino.spi.type.TypeUtils.writeNativeValue; +import static io.trino.spi.type.VarbinaryType.VARBINARY; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static java.util.Objects.requireNonNull; +import static org.apache.iceberg.MetadataTableType.ENTRIES; + +// https://iceberg.apache.org/docs/latest/spark-queries/#entries +public class EntriesTable + extends BaseSystemTable +{ + private final Map idToTypeMapping; + private final List primitiveFields; + private final Optional partitionColumn; + private final List partitionTypes; + + public EntriesTable(TypeManager typeManager, SchemaTableName tableName, Table icebergTable, ExecutorService executor) + { + super( + requireNonNull(icebergTable, "icebergTable is null"), + new ConnectorTableMetadata( + requireNonNull(tableName, "tableName is null"), + columns(requireNonNull(typeManager, "typeManager is null"), icebergTable)), + ENTRIES, + executor); + idToTypeMapping = getIcebergIdToTypeMapping(icebergTable.schema()); + primitiveFields = IcebergUtil.primitiveFields(icebergTable.schema()).stream() + .sorted(Comparator.comparing(Types.NestedField::name)) + .collect(toImmutableList()); + List partitionFields = getAllPartitionFields(icebergTable); + partitionColumn = getPartitionColumnType(partitionFields, icebergTable.schema(), typeManager); + partitionTypes = partitionTypes(partitionFields, primitiveFieldTypes(icebergTable.schema())); + } + + private static List columns(TypeManager typeManager, Table icebergTable) + { + return ImmutableList.builder() + .add(new ColumnMetadata("status", INTEGER)) + .add(new ColumnMetadata("snapshot_id", BIGINT)) + .add(new ColumnMetadata("sequence_number", BIGINT)) + .add(new ColumnMetadata("file_sequence_number", BIGINT)) + .add(new ColumnMetadata("data_file", RowType.from(dataFileFieldMetadata(typeManager, icebergTable)))) + .add(new ColumnMetadata("readable_metrics", typeManager.getType(new TypeSignature(JSON)))) + .build(); + } + + private static List dataFileFieldMetadata(TypeManager typeManager, Table icebergTable) + { + List partitionFields = getAllPartitionFields(icebergTable); + Optional partitionColumnType = getPartitionColumnType(partitionFields, icebergTable.schema(), typeManager); + + ImmutableList.Builder fields = ImmutableList.builder(); + fields.add(new RowType.Field(Optional.of("content"), INTEGER)); + fields.add(new RowType.Field(Optional.of("file_path"), VARCHAR)); + fields.add(new RowType.Field(Optional.of("file_format"), VARCHAR)); + fields.add(new RowType.Field(Optional.of("spec_id"), INTEGER)); + partitionColumnType.ifPresent(type -> fields.add(new RowType.Field(Optional.of("partition"), type.rowType()))); + fields.add(new RowType.Field(Optional.of("record_count"), BIGINT)); + fields.add(new RowType.Field(Optional.of("file_size_in_bytes"), BIGINT)); + fields.add(new RowType.Field(Optional.of("column_sizes"), typeManager.getType(mapType(INTEGER.getTypeSignature(), BIGINT.getTypeSignature())))); + fields.add(new RowType.Field(Optional.of("value_counts"), typeManager.getType(mapType(INTEGER.getTypeSignature(), BIGINT.getTypeSignature())))); + fields.add(new RowType.Field(Optional.of("null_value_counts"), typeManager.getType(mapType(INTEGER.getTypeSignature(), BIGINT.getTypeSignature())))); + fields.add(new RowType.Field(Optional.of("nan_value_counts"), typeManager.getType(mapType(INTEGER.getTypeSignature(), BIGINT.getTypeSignature())))); + fields.add(new RowType.Field(Optional.of("lower_bounds"), typeManager.getType(mapType(INTEGER.getTypeSignature(), VARCHAR.getTypeSignature())))); + fields.add(new RowType.Field(Optional.of("upper_bounds"), typeManager.getType(mapType(INTEGER.getTypeSignature(), VARCHAR.getTypeSignature())))); + fields.add(new RowType.Field(Optional.of("key_metadata"), VARBINARY)); + fields.add(new RowType.Field(Optional.of("split_offsets"), new ArrayType(BIGINT))); + fields.add(new RowType.Field(Optional.of("equality_ids"), new ArrayType(INTEGER))); + fields.add(new RowType.Field(Optional.of("sort_order_id"), INTEGER)); + return fields.build(); + } + + @Override + protected void addRow(PageListBuilder pagesBuilder, Row row, TimeZoneKey timeZoneKey) + { + pagesBuilder.beginRow(); + pagesBuilder.appendInteger(row.get("status", Integer.class)); + pagesBuilder.appendBigint(row.get("snapshot_id", Long.class)); + pagesBuilder.appendBigint(row.get("sequence_number", Long.class)); + pagesBuilder.appendBigint(row.get("file_sequence_number", Long.class)); + StructProjection dataFile = row.get("data_file", StructProjection.class); + appendDataFile((RowBlockBuilder) pagesBuilder.nextColumn(), dataFile); + ReadableMetricsStruct readableMetrics = row.get("readable_metrics", ReadableMetricsStruct.class); + String readableMetricsJson = FilesTable.toJson(readableMetrics, primitiveFields); + pagesBuilder.appendVarchar(readableMetricsJson); + pagesBuilder.endRow(); + } + + private void appendDataFile(RowBlockBuilder blockBuilder, StructProjection dataFile) + { + blockBuilder.buildEntry(fieldBuilders -> { + Integer content = dataFile.get(0, Integer.class); + INTEGER.writeLong(fieldBuilders.get(0), content); + + String filePath = dataFile.get(1, String.class); + VARCHAR.writeString(fieldBuilders.get(1), filePath); + + String fileFormat = dataFile.get(2, String.class); + VARCHAR.writeString(fieldBuilders.get(2), fileFormat); + + Integer specId = dataFile.get(3, Integer.class); + INTEGER.writeLong(fieldBuilders.get(3), Long.valueOf(specId)); + + partitionColumn.ifPresent(type -> { + StructProjection partition = dataFile.get(4, StructProjection.class); + RowBlockBuilder partitionBlockBuilder = (RowBlockBuilder) fieldBuilders.get(4); + partitionBlockBuilder.buildEntry(partitionBuilder -> { + for (int i = 0; i < type.rowType().getFields().size(); i++) { + Type icebergType = partitionTypes.get(i); + io.trino.spi.type.Type trinoType = type.rowType().getFields().get(i).getType(); + Object value = null; + Integer fieldId = type.fieldIds().get(i); + if (fieldId != null) { + value = convertIcebergValueToTrino(icebergType, partition.get(i, icebergType.typeId().javaClass())); + } + writeNativeValue(trinoType, partitionBuilder.get(i), value); + } + }); + }); + + int position = partitionColumn.isEmpty() ? 4 : 5; + Long recordCount = dataFile.get(position, Long.class); + BIGINT.writeLong(fieldBuilders.get(position), recordCount); + + Long fileSizeInBytes = dataFile.get(++position, Long.class); + BIGINT.writeLong(fieldBuilders.get(position), fileSizeInBytes); + + //noinspection unchecked + Map columnSizes = dataFile.get(++position, Map.class); + appendIntegerBigintMap((MapBlockBuilder) fieldBuilders.get(position), columnSizes); + + //noinspection unchecked + Map valueCounts = dataFile.get(++position, Map.class); + appendIntegerBigintMap((MapBlockBuilder) fieldBuilders.get(position), valueCounts); + + //noinspection unchecked + Map nullValueCounts = dataFile.get(++position, Map.class); + appendIntegerBigintMap((MapBlockBuilder) fieldBuilders.get(position), nullValueCounts); + + //noinspection unchecked + Map nanValueCounts = dataFile.get(++position, Map.class); + appendIntegerBigintMap((MapBlockBuilder) fieldBuilders.get(position), nanValueCounts); + + //noinspection unchecked + Map lowerBounds = dataFile.get(++position, Map.class); + appendIntegerVarcharMap((MapBlockBuilder) fieldBuilders.get(position), lowerBounds); + + //noinspection unchecked + Map upperBounds = dataFile.get(++position, Map.class); + appendIntegerVarcharMap((MapBlockBuilder) fieldBuilders.get(position), upperBounds); + + ByteBuffer keyMetadata = dataFile.get(++position, ByteBuffer.class); + if (keyMetadata == null) { + fieldBuilders.get(position).appendNull(); + } + else { + VARBINARY.writeSlice(fieldBuilders.get(position), wrappedHeapBuffer(keyMetadata)); + } + + //noinspection unchecked + List splitOffsets = dataFile.get(++position, List.class); + appendBigintArray((ArrayBlockBuilder) fieldBuilders.get(position), splitOffsets); + + //noinspection unchecked + List equalityIds = dataFile.get(++position, List.class); + appendBigintArray((ArrayBlockBuilder) fieldBuilders.get(position), equalityIds); + + Integer sortOrderId = dataFile.get(++position, Integer.class); + INTEGER.writeLong(fieldBuilders.get(position), Long.valueOf(sortOrderId)); + }); + } + + public static void appendBigintArray(ArrayBlockBuilder blockBuilder, @Nullable List values) + { + if (values == null) { + blockBuilder.appendNull(); + return; + } + blockBuilder.buildEntry(elementBuilder -> { + for (Long value : values) { + BIGINT.writeLong(elementBuilder, value); + } + }); + } + + private static void appendIntegerBigintMap(MapBlockBuilder blockBuilder, @Nullable Map values) + { + if (values == null) { + blockBuilder.appendNull(); + return; + } + blockBuilder.buildEntry((keyBuilder, valueBuilder) -> values.forEach((key, value) -> { + INTEGER.writeLong(keyBuilder, key); + BIGINT.writeLong(valueBuilder, value); + })); + } + + private void appendIntegerVarcharMap(MapBlockBuilder blockBuilder, @Nullable Map values) + { + if (values == null) { + blockBuilder.appendNull(); + return; + } + blockBuilder.buildEntry((keyBuilder, valueBuilder) -> values.forEach((key, value) -> { + Type type = idToTypeMapping.get(key); + INTEGER.writeLong(keyBuilder, key); + VARCHAR.writeString(valueBuilder, Transforms.identity().toHumanString(type, Conversions.fromByteBuffer(type, value))); + })); + } +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 7e51d89d68f3..6107360a40ad 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -700,6 +700,7 @@ private Optional getRawSystemTable(ConnectorSession session, Schema case ALL_MANIFESTS -> Optional.of(new AllManifestsTable(tableName, table, executor)); case MANIFESTS -> Optional.of(new ManifestsTable(tableName, table, getCurrentSnapshotId(table))); case FILES -> Optional.of(new FilesTable(tableName, typeManager, table, getCurrentSnapshotId(table), executor)); + case ENTRIES -> Optional.of(new EntriesTable(typeManager, tableName, table, executor)); case PROPERTIES -> Optional.of(new PropertiesTable(tableName, table)); case REFS -> Optional.of(new RefsTable(tableName, table, executor)); }; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java index 2ba845acc379..ab39c3504b6f 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java @@ -23,6 +23,7 @@ public enum TableType MANIFESTS, PARTITIONS, FILES, + ENTRIES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE, diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java index 7212f3a3aa1d..e6fbf00ca54e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java @@ -15,13 +15,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.HiveMetastore; import io.trino.spi.type.ArrayType; import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; import io.trino.testing.MaterializedResult; import io.trino.testing.MaterializedRow; import io.trino.testing.QueryRunner; import io.trino.testing.sql.TestTable; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.FileContent; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -37,9 +43,12 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.trino.plugin.iceberg.IcebergFileFormat.ORC; import static io.trino.plugin.iceberg.IcebergFileFormat.PARQUET; +import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; +import static io.trino.plugin.iceberg.IcebergTestUtils.getHiveMetastore; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.testing.MaterializedResult.DEFAULT_PRECISION; import static io.trino.testing.MaterializedResult.resultBuilder; +import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -49,6 +58,8 @@ public abstract class BaseIcebergSystemTables extends AbstractTestQueryFramework { private final IcebergFileFormat format; + private HiveMetastore metastore; + private TrinoFileSystemFactory fileSystemFactory; protected BaseIcebergSystemTables(IcebergFileFormat format) { @@ -59,9 +70,12 @@ protected BaseIcebergSystemTables(IcebergFileFormat format) protected QueryRunner createQueryRunner() throws Exception { - return IcebergQueryRunner.builder() + DistributedQueryRunner queryRunner = IcebergQueryRunner.builder() .setIcebergProperties(ImmutableMap.of("iceberg.file-format", format.name())) .build(); + metastore = getHiveMetastore(queryRunner); + fileSystemFactory = getFileSystemFactory(queryRunner); + return queryRunner; } @BeforeAll @@ -549,6 +563,85 @@ public void testFilesTableWithDelete() assertUpdate("DROP TABLE IF EXISTS test_schema.test_table_with_delete"); } + @Test + void testEntriesTable() + { + try (TestTable table = new TestTable(getQueryRunner()::execute, "test_entries", "AS SELECT 1 id, DATE '2014-01-01' dt")) { + assertQuery("SHOW COLUMNS FROM \"" + table.getName() + "$entries\"", + "VALUES ('status', 'integer', '', '')," + + "('snapshot_id', 'bigint', '', '')," + + "('sequence_number', 'bigint', '', '')," + + "('file_sequence_number', 'bigint', '', '')," + + "('data_file', 'row(content integer, file_path varchar, file_format varchar, spec_id integer, record_count bigint, file_size_in_bytes bigint, " + + "column_sizes map(integer, bigint), value_counts map(integer, bigint), null_value_counts map(integer, bigint), nan_value_counts map(integer, bigint), " + + "lower_bounds map(integer, varchar), upper_bounds map(integer, varchar), key_metadata varbinary, split_offsets array(bigint), " + + "equality_ids array(integer), sort_order_id integer)', '', '')," + + "('readable_metrics', 'json', '', '')"); + + Table icebergTable = loadTable(table.getName()); + Snapshot snapshot = icebergTable.currentSnapshot(); + long snapshotId = snapshot.snapshotId(); + long sequenceNumber = snapshot.sequenceNumber(); + + assertThat(computeScalar("SELECT status FROM \"" + table.getName() + "$entries\"")) + .isEqualTo(1); + assertThat(computeScalar("SELECT snapshot_id FROM \"" + table.getName() + "$entries\"")) + .isEqualTo(snapshotId); + assertThat(computeScalar("SELECT sequence_number FROM \"" + table.getName() + "$entries\"")) + .isEqualTo(sequenceNumber); + assertThat(computeScalar("SELECT file_sequence_number FROM \"" + table.getName() + "$entries\"")) + .isEqualTo(1L); + + MaterializedRow dataFile = (MaterializedRow) computeScalar("SELECT data_file FROM \"" + table.getName() + "$entries\""); + assertThat(dataFile.getFieldCount()).isEqualTo(16); + assertThat(dataFile.getField(0)).isEqualTo(0); // content + assertThat((String) dataFile.getField(1)).endsWith(format.toString().toLowerCase(ENGLISH)); // file_path + assertThat(dataFile.getField(2)).isEqualTo(format.toString()); // file_format + assertThat(dataFile.getField(3)).isEqualTo(0); // spec_id + assertThat(dataFile.getField(4)).isEqualTo(1L); // record_count + assertThat((long) dataFile.getField(5)).isPositive(); // file_size_in_bytes + assertThat(dataFile.getField(6)).isEqualTo(value(Map.of(1, 36L, 2, 36L), null)); // column_sizes + assertThat(dataFile.getField(7)).isEqualTo(Map.of(1, 1L, 2, 1L)); // value_counts + assertThat(dataFile.getField(8)).isEqualTo(Map.of(1, 0L, 2, 0L)); // null_value_counts + assertThat(dataFile.getField(9)).isEqualTo(value(Map.of(), null)); // nan_value_counts + assertThat(dataFile.getField(10)).isEqualTo(Map.of(1, "1", 2, "2014-01-01")); // lower_bounds + assertThat(dataFile.getField(11)).isEqualTo(Map.of(1, "1", 2, "2014-01-01")); // upper_bounds + assertThat(dataFile.getField(12)).isNull(); // key_metadata + assertThat(dataFile.getField(13)).isEqualTo(List.of(value(4L, 3L))); // split_offsets + assertThat(dataFile.getField(14)).isNull(); // equality_ids + assertThat(dataFile.getField(15)).isEqualTo(0); // sort_order_id + + assertThat(computeScalar("SELECT readable_metrics FROM \"" + table.getName() + "$entries\"")) + .isEqualTo("{" + + "\"dt\":{\"column_size\":" + value(36, null) + ",\"value_count\":1,\"null_value_count\":0,\"nan_value_count\":null,\"lower_bound\":\"2014-01-01\",\"upper_bound\":\"2014-01-01\"}," + + "\"id\":{\"column_size\":" + value(36, null) + ",\"value_count\":1,\"null_value_count\":0,\"nan_value_count\":null,\"lower_bound\":1,\"upper_bound\":1}" + + "}"); + } + } + + @Test + void testEntriesPartitionTable() + { + try (TestTable table = new TestTable( + getQueryRunner()::execute, + "test_entries_partition", + "WITH (partitioning = ARRAY['dt']) AS SELECT 1 id, DATE '2014-01-01' dt")) { + assertQuery("SHOW COLUMNS FROM \"" + table.getName() + "$entries\"", + "VALUES ('status', 'integer', '', '')," + + "('snapshot_id', 'bigint', '', '')," + + "('sequence_number', 'bigint', '', '')," + + "('file_sequence_number', 'bigint', '', '')," + + "('data_file', 'row(content integer, file_path varchar, file_format varchar, spec_id integer, partition row(dt date), record_count bigint, file_size_in_bytes bigint, " + + "column_sizes map(integer, bigint), value_counts map(integer, bigint), null_value_counts map(integer, bigint), nan_value_counts map(integer, bigint), " + + "lower_bounds map(integer, varchar), upper_bounds map(integer, varchar), key_metadata varbinary, split_offsets array(bigint), " + + "equality_ids array(integer), sort_order_id integer)', '', '')," + + "('readable_metrics', 'json', '', '')"); + + assertThat(query("SELECT data_file.partition FROM \"" + table.getName() + "$entries\"")) + .matches("SELECT CAST(ROW(DATE '2014-01-01') AS ROW(dt date))"); + } + } + private Long nanCount(long value) { // Parquet does not have nan count metrics @@ -565,4 +658,9 @@ private Object value(Object parquet, Object orc) { return format == PARQUET ? parquet : orc; } + + private BaseTable loadTable(String tableName) + { + return IcebergTestUtils.loadTable(tableName, metastore, fileSystemFactory, "hive", "tpch"); + } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java index 23a3b809d513..401e68e38c90 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java @@ -38,6 +38,7 @@ import static io.trino.plugin.iceberg.IcebergSessionProperties.COLLECT_EXTENDED_STATISTICS_ON_WRITE; import static io.trino.plugin.iceberg.TableType.ALL_MANIFESTS; import static io.trino.plugin.iceberg.TableType.DATA; +import static io.trino.plugin.iceberg.TableType.ENTRIES; import static io.trino.plugin.iceberg.TableType.FILES; import static io.trino.plugin.iceberg.TableType.HISTORY; import static io.trino.plugin.iceberg.TableType.MANIFESTS; @@ -338,6 +339,12 @@ public void testSelectSystemTable() .addCopies(GET_TABLE, 1) .build()); + // select from $entries + assertMetastoreInvocations("SELECT * FROM \"test_select_snapshots$entries\"", + ImmutableMultiset.builder() + .addCopies(GET_TABLE, 1) + .build()); + // select from $properties assertMetastoreInvocations("SELECT * FROM \"test_select_snapshots$properties\"", ImmutableMultiset.builder() @@ -349,7 +356,7 @@ public void testSelectSystemTable() // This test should get updated if a new system table is added. assertThat(TableType.values()) - .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); + .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, ENTRIES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); } @Test diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java index 4a063672dd85..e116e95fab0f 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java @@ -52,6 +52,7 @@ import static io.trino.plugin.iceberg.IcebergSessionProperties.COLLECT_EXTENDED_STATISTICS_ON_WRITE; import static io.trino.plugin.iceberg.TableType.ALL_MANIFESTS; import static io.trino.plugin.iceberg.TableType.DATA; +import static io.trino.plugin.iceberg.TableType.ENTRIES; import static io.trino.plugin.iceberg.TableType.FILES; import static io.trino.plugin.iceberg.TableType.HISTORY; import static io.trino.plugin.iceberg.TableType.MANIFESTS; @@ -470,6 +471,12 @@ public void testSelectSystemTable() .add(GET_TABLE) .build()); + // select from $entries + assertGlueMetastoreApiInvocations("SELECT * FROM \"test_select_snapshots$entries\"", + ImmutableMultiset.builder() + .add(GET_TABLE) + .build()); + // select from $properties assertGlueMetastoreApiInvocations("SELECT * FROM \"test_select_snapshots$properties\"", ImmutableMultiset.builder() @@ -487,7 +494,7 @@ public void testSelectSystemTable() // This test should get updated if a new system table is added. assertThat(TableType.values()) - .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); + .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, ENTRIES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); } finally { getQueryRunner().execute("DROP TABLE IF EXISTS test_select_snapshots"); From 01f7433effac09dcfab8661efb04bc081bd7912f Mon Sep 17 00:00:00 2001 From: Slawomir Pajak Date: Fri, 20 Dec 2024 14:09:32 +0100 Subject: [PATCH 045/158] Run Iceberg concurrent tests multiple times --- .../TestIcebergLocalConcurrentWrites.java | 59 ++++++++++++------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java index 903fff33da5e..951e8b7c1533 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java @@ -19,7 +19,7 @@ import io.trino.testing.MaterializedResult; import io.trino.testing.QueryRunner; import io.trino.testing.sql.TestTable; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.TestInstance; import java.util.List; @@ -52,7 +52,8 @@ protected QueryRunner createQueryRunner() return IcebergQueryRunner.builder().build(); } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentInserts() throws Exception { @@ -101,7 +102,8 @@ private void testConcurrentInserts(boolean partitioned) } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentInsertsSelectingFromTheSameTable() throws Exception { @@ -148,7 +150,8 @@ private void testConcurrentInsertsSelectingFromTheSameTable(boolean partitioned) } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentInsertsSelectingFromTheSameVersionedTable() throws Exception { @@ -197,7 +200,8 @@ private void testConcurrentInsertsSelectingFromTheSameVersionedTable(boolean par } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentDelete() throws Exception { @@ -238,7 +242,8 @@ void testConcurrentDelete() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentDeleteFromTheSamePartition() throws Exception { @@ -273,7 +278,8 @@ void testConcurrentDeleteFromTheSamePartition() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentTruncate() throws Exception { @@ -308,7 +314,8 @@ void testConcurrentTruncate() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentTruncateAndInserts() throws Exception { @@ -349,7 +356,8 @@ void testConcurrentTruncateAndInserts() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentNonOverlappingUpdate() throws Exception { @@ -390,7 +398,8 @@ void testConcurrentNonOverlappingUpdate() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentOverlappingUpdate() throws Exception { @@ -455,7 +464,8 @@ private void testConcurrentOverlappingUpdate(boolean partitioned) } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentNonOverlappingUpdateOnNestedPartition() throws Exception { @@ -502,7 +512,8 @@ void testConcurrentNonOverlappingUpdateOnNestedPartition() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentDeleteAndInserts() throws Exception { @@ -579,7 +590,8 @@ void testConcurrentDeleteAndInserts() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentUpdateAndInserts() throws Exception { @@ -657,7 +669,8 @@ void testConcurrentUpdateAndInserts() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentMergeAndInserts() throws Exception { @@ -737,7 +750,8 @@ void testConcurrentMergeAndInserts() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentDeleteAndDeletePushdownAndInsert() throws Exception { @@ -781,7 +795,8 @@ void testConcurrentDeleteAndDeletePushdownAndInsert() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentUpdateWithPartitionTransformation() throws Exception { @@ -824,7 +839,8 @@ void testConcurrentUpdateWithPartitionTransformation() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentUpdateWithNestedPartitionTransformation() throws Exception { @@ -867,7 +883,8 @@ void testConcurrentUpdateWithNestedPartitionTransformation() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentUpdateWithMultiplePartitionTransformation() throws Exception { @@ -924,7 +941,8 @@ void testConcurrentUpdateWithMultiplePartitionTransformation() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentUpdateWithOverlappingPartitionTransformation() throws Exception { @@ -985,7 +1003,8 @@ void testConcurrentUpdateWithOverlappingPartitionTransformation() } } - @Test + // Repeat test with invocationCount for better test coverage, since the tested aspect is inherently non-deterministic. + @RepeatedTest(3) void testConcurrentUpdateWithEnforcedAndUnenforcedPartitions() throws Exception { From 5f84a49c80f1253e2d563d7fe8258153711f3ddc Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Fri, 20 Dec 2024 10:33:09 -0800 Subject: [PATCH 046/158] Update client driver and application sections --- docs/src/main/sphinx/client.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/src/main/sphinx/client.md b/docs/src/main/sphinx/client.md index 502bf3552520..98837c87f999 100644 --- a/docs/src/main/sphinx/client.md +++ b/docs/src/main/sphinx/client.md @@ -15,9 +15,10 @@ following client drivers: * [trino-go-client](https://github.com/trinodb/trino-go-client) * [trino-js-client](https://github.com/trinodb/trino-js-client) * [trino-python-client](https://github.com/trinodb/trino-python-client) +* [trino-csharp-client](https://github.com/trinodb/trino-csharp-client) Other communities and vendors provide [other client -drivers](https://trino.io/ecosystem/client.html). +drivers](https://trino.io/ecosystem/client-driver#other-client-drivers). ## Client applications @@ -26,11 +27,12 @@ run queries with Trino. You can inspect the results, perform analytics with further queries, and create visualizations. Client applications typically use a client driver. -The Trino project maintains the [Trino command line interface](/client/cli) as a -client application. +The Trino project maintains the [Trino command line interface](/client/cli) and +the [Trino Grafana Data Source Plugin](https://github.com/trinodb/grafana-trino) +as a client application. Other communities and vendors provide [numerous other client -applications](https://trino.io/ecosystem/client.html) +applications](https://trino.io/ecosystem/client-application#other-client-applications) ## Client protocol From a401e3776bf0b399b14f5b903306e07adc747902 Mon Sep 17 00:00:00 2001 From: okumin Date: Sat, 21 Dec 2024 13:43:00 +0900 Subject: [PATCH 047/158] Document sort direction and null order in Iceberg --- docs/src/main/sphinx/connector/iceberg.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/src/main/sphinx/connector/iceberg.md b/docs/src/main/sphinx/connector/iceberg.md index 6ec6cd7bd4e0..42cb12927220 100644 --- a/docs/src/main/sphinx/connector/iceberg.md +++ b/docs/src/main/sphinx/connector/iceberg.md @@ -1502,6 +1502,18 @@ CREATE TABLE example.customers.orders ( WITH (sorted_by = ARRAY['order_date']) ``` +You can explicitly configure sort directions or null ordering in the following way: + +``` +CREATE TABLE example.customers.orders ( + order_id BIGINT, + order_date DATE, + account_number BIGINT, + customer VARCHAR, + country VARCHAR) +WITH (sorted_by = ARRAY['order_date DESC NULLS FIRST', 'order_id ASC NULLS LAST']) +``` + Sorting can be combined with partitioning on the same column. For example: ``` From 715eb7bc8e10b62fca64f3583b4509f4dfbaca98 Mon Sep 17 00:00:00 2001 From: Shoham Yamin Date: Fri, 20 Dec 2024 06:09:42 +0200 Subject: [PATCH 048/158] Rename executor to icebergScanExecutor --- .../trino/plugin/iceberg/IcebergMetadata.java | 20 +++++++++---------- .../iceberg/IcebergMetadataFactory.java | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 6107360a40ad..61bb1fa43c67 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -406,7 +406,7 @@ public class IcebergMetadata private final Optional metastoreFactory; private final boolean addFilesProcedureEnabled; private final Predicate allowedExtraProperties; - private final ExecutorService executor; + private final ExecutorService icebergScanExecutor; private final Map> tableStatisticsCache = new ConcurrentHashMap<>(); @@ -423,7 +423,7 @@ public IcebergMetadata( Optional metastoreFactory, boolean addFilesProcedureEnabled, Predicate allowedExtraProperties, - ExecutorService executor) + ExecutorService icebergScanExecutor) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.trinoCatalogHandle = requireNonNull(trinoCatalogHandle, "trinoCatalogHandle is null"); @@ -434,7 +434,7 @@ public IcebergMetadata( this.metastoreFactory = requireNonNull(metastoreFactory, "metastoreFactory is null"); this.addFilesProcedureEnabled = addFilesProcedureEnabled; this.allowedExtraProperties = requireNonNull(allowedExtraProperties, "allowedExtraProperties is null"); - this.executor = requireNonNull(executor, "executor is null"); + this.icebergScanExecutor = requireNonNull(icebergScanExecutor, "icebergScanExecutor is null"); } @Override @@ -694,15 +694,15 @@ private Optional getRawSystemTable(ConnectorSession session, Schema return switch (tableType) { case DATA, MATERIALIZED_VIEW_STORAGE -> throw new VerifyException("Unexpected table type: " + tableType); // Handled above. case HISTORY -> Optional.of(new HistoryTable(tableName, table)); - case METADATA_LOG_ENTRIES -> Optional.of(new MetadataLogEntriesTable(tableName, table, executor)); - case SNAPSHOTS -> Optional.of(new SnapshotsTable(tableName, typeManager, table, executor)); - case PARTITIONS -> Optional.of(new PartitionsTable(tableName, typeManager, table, getCurrentSnapshotId(table), executor)); - case ALL_MANIFESTS -> Optional.of(new AllManifestsTable(tableName, table, executor)); + case METADATA_LOG_ENTRIES -> Optional.of(new MetadataLogEntriesTable(tableName, table, icebergScanExecutor)); + case SNAPSHOTS -> Optional.of(new SnapshotsTable(tableName, typeManager, table, icebergScanExecutor)); + case PARTITIONS -> Optional.of(new PartitionsTable(tableName, typeManager, table, getCurrentSnapshotId(table), icebergScanExecutor)); + case ALL_MANIFESTS -> Optional.of(new AllManifestsTable(tableName, table, icebergScanExecutor)); case MANIFESTS -> Optional.of(new ManifestsTable(tableName, table, getCurrentSnapshotId(table))); - case FILES -> Optional.of(new FilesTable(tableName, typeManager, table, getCurrentSnapshotId(table), executor)); - case ENTRIES -> Optional.of(new EntriesTable(typeManager, tableName, table, executor)); + case FILES -> Optional.of(new FilesTable(tableName, typeManager, table, getCurrentSnapshotId(table), icebergScanExecutor)); + case ENTRIES -> Optional.of(new EntriesTable(typeManager, tableName, table, icebergScanExecutor)); case PROPERTIES -> Optional.of(new PropertiesTable(tableName, table)); - case REFS -> Optional.of(new RefsTable(tableName, table, executor)); + case REFS -> Optional.of(new RefsTable(tableName, table, icebergScanExecutor)); }; } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java index 023952f6e780..6b9e28ae5ce2 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java @@ -41,7 +41,7 @@ public class IcebergMetadataFactory private final Optional metastoreFactory; private final boolean addFilesProcedureEnabled; private final Predicate allowedExtraProperties; - private final ExecutorService executor; + private final ExecutorService icebergScanExecutor; @Inject public IcebergMetadataFactory( @@ -52,7 +52,7 @@ public IcebergMetadataFactory( IcebergFileSystemFactory fileSystemFactory, TableStatisticsWriter tableStatisticsWriter, @RawHiveMetastoreFactory Optional metastoreFactory, - @ForIcebergScanPlanning ExecutorService executor, + @ForIcebergScanPlanning ExecutorService icebergScanExecutor, IcebergConfig config) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); @@ -62,7 +62,7 @@ public IcebergMetadataFactory( this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.tableStatisticsWriter = requireNonNull(tableStatisticsWriter, "tableStatisticsWriter is null"); this.metastoreFactory = requireNonNull(metastoreFactory, "metastoreFactory is null"); - this.executor = requireNonNull(executor, "executor is null"); + this.icebergScanExecutor = requireNonNull(icebergScanExecutor, "icebergScanExecutor is null"); this.addFilesProcedureEnabled = config.isAddFilesProcedureEnabled(); if (config.getAllowedExtraProperties().equals(ImmutableList.of("*"))) { this.allowedExtraProperties = _ -> true; @@ -84,6 +84,6 @@ public IcebergMetadata create(ConnectorIdentity identity) metastoreFactory, addFilesProcedureEnabled, allowedExtraProperties, - executor); + icebergScanExecutor); } } From 4f480877d01f28fa68665af19f41f217c4156201 Mon Sep 17 00:00:00 2001 From: Shoham Yamin Date: Fri, 20 Dec 2024 11:06:12 +0200 Subject: [PATCH 049/158] Improve performance when listing columns in Iceberg --- .../trino/plugin/iceberg/IcebergMetadata.java | 58 +++++++++++++------ .../iceberg/IcebergMetadataFactory.java | 15 ++++- .../iceberg/catalog/BaseTrinoCatalogTest.java | 4 +- .../catalog/glue/TestTrinoGlueCatalog.java | 3 +- .../nessie/TestTrinoNessieCatalog.java | 4 +- .../catalog/rest/TestTrinoRestCatalog.java | 4 +- .../snowflake/TestTrinoSnowflakeCatalog.java | 4 +- 7 files changed, 69 insertions(+), 23 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 61bb1fa43c67..0f92016badcf 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -199,7 +199,10 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -228,6 +231,7 @@ import static io.trino.plugin.base.filter.UtcConstraintExtractor.extractTupleDomain; import static io.trino.plugin.base.projection.ApplyProjectionUtil.extractSupportedProjectedColumns; import static io.trino.plugin.base.projection.ApplyProjectionUtil.replaceWithNewVariables; +import static io.trino.plugin.base.util.ExecutorUtil.processWithAdditionalThreads; import static io.trino.plugin.base.util.Procedures.checkProcedureArgument; import static io.trino.plugin.hive.HiveMetadata.TRANSACTIONAL; import static io.trino.plugin.hive.HiveTimestampPrecision.DEFAULT_PRECISION; @@ -407,6 +411,7 @@ public class IcebergMetadata private final boolean addFilesProcedureEnabled; private final Predicate allowedExtraProperties; private final ExecutorService icebergScanExecutor; + private final Executor metadataFetchingExecutor; private final Map> tableStatisticsCache = new ConcurrentHashMap<>(); @@ -423,7 +428,8 @@ public IcebergMetadata( Optional metastoreFactory, boolean addFilesProcedureEnabled, Predicate allowedExtraProperties, - ExecutorService icebergScanExecutor) + ExecutorService icebergScanExecutor, + Executor metadataFetchingExecutor) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.trinoCatalogHandle = requireNonNull(trinoCatalogHandle, "trinoCatalogHandle is null"); @@ -435,6 +441,7 @@ public IcebergMetadata( this.addFilesProcedureEnabled = addFilesProcedureEnabled; this.allowedExtraProperties = requireNonNull(allowedExtraProperties, "allowedExtraProperties is null"); this.icebergScanExecutor = requireNonNull(icebergScanExecutor, "icebergScanExecutor is null"); + this.metadataFetchingExecutor = requireNonNull(metadataFetchingExecutor, "metadataFetchingExecutor is null"); } @Override @@ -982,23 +989,40 @@ public Iterator streamTableColumns(ConnectorSession sessio tableMetadatas.add(TableColumnsMetadata.forTable(tableName, columns)); }); - for (SchemaTableName tableName : remainingTables) { - try { - Table icebergTable = catalog.loadTable(session, tableName); - List columns = getColumnMetadatas(icebergTable.schema(), typeManager); - tableMetadatas.add(TableColumnsMetadata.forTable(tableName, columns)); - } - catch (TableNotFoundException e) { - // Table disappeared during listing operation - } - catch (UnknownTableTypeException e) { - // Skip unsupported table type in case that the table redirects are not enabled - } - catch (RuntimeException e) { - // Table can be being removed and this may cause all sorts of exceptions. Log, because we're catching broadly. - log.warn(e, "Failed to access metadata of table %s during streaming table columns for %s", tableName, prefix); - } + List>> tasks = remainingTables.stream() + .map(tableName -> (Callable>) () -> { + try { + Table icebergTable = catalog.loadTable(session, tableName); + List columns = getColumnMetadatas(icebergTable.schema(), typeManager); + return Optional.of(TableColumnsMetadata.forTable(tableName, columns)); + } + catch (TableNotFoundException e) { + // Table disappeared during listing operation + return Optional.empty(); + } + catch (UnknownTableTypeException e) { + // Skip unsupported table type in case that the table redirects are not enabled + return Optional.empty(); + } + catch (RuntimeException e) { + // Table can be being removed and this may cause all sorts of exceptions. Log, because we're catching broadly. + log.warn(e, "Failed to access metadata of table %s during streaming table columns for %s", tableName, prefix); + return Optional.empty(); + } + }) + .collect(toImmutableList()); + + try { + List taskResults = processWithAdditionalThreads(tasks, metadataFetchingExecutor).stream() + .flatMap(Optional::stream) // Flatten the Optionals into a stream + .collect(toImmutableList()); + + tableMetadatas.addAll(taskResults); + } + catch (ExecutionException e) { + throw new RuntimeException(e.getCause()); } + return tableMetadatas.build(); }) .flatMap(List::stream) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java index 6b9e28ae5ce2..6b451c241b31 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; +import io.airlift.concurrent.BoundedExecutor; import io.airlift.json.JsonCodec; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; @@ -25,9 +26,11 @@ import io.trino.spi.type.TypeManager; import java.util.Optional; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.function.Predicate; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.Objects.requireNonNull; public class IcebergMetadataFactory @@ -42,6 +45,7 @@ public class IcebergMetadataFactory private final boolean addFilesProcedureEnabled; private final Predicate allowedExtraProperties; private final ExecutorService icebergScanExecutor; + private final Executor metadataFetchingExecutor; @Inject public IcebergMetadataFactory( @@ -53,6 +57,7 @@ public IcebergMetadataFactory( TableStatisticsWriter tableStatisticsWriter, @RawHiveMetastoreFactory Optional metastoreFactory, @ForIcebergScanPlanning ExecutorService icebergScanExecutor, + @ForIcebergMetadata ExecutorService metadataExecutorService, IcebergConfig config) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); @@ -70,6 +75,13 @@ public IcebergMetadataFactory( else { this.allowedExtraProperties = ImmutableSet.copyOf(requireNonNull(config.getAllowedExtraProperties(), "allowedExtraProperties is null"))::contains; } + + if (config.getMetadataParallelism() == 1) { + this.metadataFetchingExecutor = directExecutor(); + } + else { + this.metadataFetchingExecutor = new BoundedExecutor(metadataExecutorService, config.getMetadataParallelism()); + } } public IcebergMetadata create(ConnectorIdentity identity) @@ -84,6 +96,7 @@ public IcebergMetadata create(ConnectorIdentity identity) metastoreFactory, addFilesProcedureEnabled, allowedExtraProperties, - icebergScanExecutor); + icebergScanExecutor, + metadataFetchingExecutor); } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java index 82254efdb85a..a8aa117f9cde 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java @@ -48,6 +48,7 @@ import java.util.Optional; import java.util.UUID; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static io.airlift.json.JsonCodec.jsonCodec; import static io.trino.metastore.TableInfo.ExtendedRelationType.TABLE; @@ -122,7 +123,8 @@ public void testNonLowercaseNamespace() Optional.empty(), false, _ -> false, - newDirectExecutorService()); + newDirectExecutorService(), + directExecutor()); assertThat(icebergMetadata.schemaExists(SESSION, namespace)).as("icebergMetadata.schemaExists(namespace)") .isFalse(); assertThat(icebergMetadata.schemaExists(SESSION, schema)).as("icebergMetadata.schemaExists(schema)") diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java index ca22eb99f0b8..80ee1628b561 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java @@ -141,7 +141,8 @@ public void testNonLowercaseGlueDatabase() Optional.empty(), false, _ -> false, - newDirectExecutorService()); + newDirectExecutorService(), + directExecutor()); assertThat(icebergMetadata.schemaExists(SESSION, databaseName)).as("icebergMetadata.schemaExists(databaseName)") .isFalse(); assertThat(icebergMetadata.schemaExists(SESSION, trinoSchemaName)).as("icebergMetadata.schemaExists(trinoSchemaName)") diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java index 2046ec69b861..ce0089125172 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java @@ -46,6 +46,7 @@ import java.util.Map; import java.util.Optional; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static io.airlift.json.JsonCodec.jsonCodec; import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; @@ -191,7 +192,8 @@ public void testNonLowercaseNamespace() Optional.empty(), false, _ -> false, - newDirectExecutorService()); + newDirectExecutorService(), + directExecutor()); assertThat(icebergMetadata.schemaExists(SESSION, namespace)).as("icebergMetadata.schemaExists(namespace)") .isTrue(); assertThat(icebergMetadata.schemaExists(SESSION, schema)).as("icebergMetadata.schemaExists(schema)") diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java index 6e75f3aa8b78..df3ad2a5400c 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java @@ -45,6 +45,7 @@ import java.util.Map; import java.util.Optional; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static io.airlift.json.JsonCodec.jsonCodec; import static io.trino.metastore.TableInfo.ExtendedRelationType.OTHER_VIEW; @@ -129,7 +130,8 @@ public void testNonLowercaseNamespace() Optional.empty(), false, _ -> false, - newDirectExecutorService()); + newDirectExecutorService(), + directExecutor()); assertThat(icebergMetadata.schemaExists(SESSION, namespace)).as("icebergMetadata.schemaExists(namespace)") .isTrue(); assertThat(icebergMetadata.schemaExists(SESSION, schema)).as("icebergMetadata.schemaExists(schema)") diff --git a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java index d445512ed2ae..936c3e251342 100644 --- a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java +++ b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java @@ -58,6 +58,7 @@ import java.util.Map; import java.util.Optional; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static io.airlift.json.JsonCodec.jsonCodec; import static io.trino.plugin.iceberg.catalog.snowflake.TestIcebergSnowflakeCatalogConnectorSmokeTest.S3_ACCESS_KEY; @@ -225,7 +226,8 @@ public void testNonLowercaseNamespace() Optional.empty(), false, _ -> false, - newDirectExecutorService()); + newDirectExecutorService(), + directExecutor()); assertThat(icebergMetadata.schemaExists(SESSION, namespace)).as("icebergMetadata.schemaExists(namespace)") .isTrue(); assertThat(icebergMetadata.schemaExists(SESSION, schema)).as("icebergMetadata.schemaExists(schema)") From 85b25ceaaec2c8cc13e1f18b6424c4d87e37ea8a Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Sat, 21 Dec 2024 08:42:28 +0900 Subject: [PATCH 050/158] Remove support for Databricks 9.1 LTS --- .github/workflows/ci.yml | 6 --- plugin/trino-delta-lake/README.md | 4 +- .../plugin/deltalake/TestDeltaLakeBasic.java | 9 ++++ .../resources/databricks91/region/README.md | 13 ++++++ .../_delta_log/00000000000000000000.json | 3 ++ .../_delta_log/00000000000000000001.json | 3 ++ ...404f-b99f-d749d1c7419e-c000.snappy.parquet | Bin 0 -> 1408 bytes ...4dc0-a2d6-4f2a6760e3b9-c000.snappy.parquet | Bin 0 -> 1446 bytes .../io/trino/tests/product/TestGroups.java | 2 +- .../EnvSinglenodeDeltaLakeDatabricks91.java | 38 ---------------- .../suites/SuiteDeltaLakeDatabricks104.java | 6 ++- .../suites/SuiteDeltaLakeDatabricks91.java | 42 ------------------ .../TestDeltaLakeAlterTableCompatibility.java | 4 +- .../TestDeltaLakeCaseInsensitiveMapping.java | 4 +- ...tDeltaLakeChangeDataFeedCompatibility.java | 12 ++--- .../TestDeltaLakeCloneTableCompatibility.java | 4 +- .../TestDeltaLakeColumnMappingMode.java | 9 ++-- ...akeDatabricksCreateTableCompatibility.java | 7 +++ .../TestDeltaLakeDeleteCompatibility.java | 6 +-- ...TestDeltaLakeSystemTableCompatibility.java | 4 +- 20 files changed, 63 insertions(+), 113 deletions(-) create mode 100644 plugin/trino-delta-lake/src/test/resources/databricks91/region/README.md create mode 100644 plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000000.json create mode 100644 plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000001.json create mode 100644 plugin/trino-delta-lake/src/test/resources/databricks91/region/part-00000-3a487bbf-c1d3-404f-b99f-d749d1c7419e-c000.snappy.parquet create mode 100644 plugin/trino-delta-lake/src/test/resources/databricks91/region/part-00001-87059848-17f8-4dc0-a2d6-4f2a6760e3b9-c000.snappy.parquet delete mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java delete mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks91.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dd65babf608..086fc01298ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -878,7 +878,6 @@ jobs: - suite-7-non-generic - suite-hive-transactional - suite-azure - - suite-delta-lake-databricks91 - suite-delta-lake-databricks104 - suite-delta-lake-databricks113 - suite-delta-lake-databricks122 @@ -919,9 +918,6 @@ jobs: ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }} - - suite: suite-delta-lake-databricks91 - ignore exclusion if: >- - ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} - suite: suite-delta-lake-databricks104 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} @@ -989,7 +985,6 @@ jobs: AWS_REGION: "" TRINO_AWS_ACCESS_KEY_ID: "" TRINO_AWS_SECRET_ACCESS_KEY: "" - DATABRICKS_91_JDBC_URL: "" DATABRICKS_104_JDBC_URL: "" DATABRICKS_113_JDBC_URL: "" DATABRICKS_122_JDBC_URL: "" @@ -1067,7 +1062,6 @@ jobs: AWS_REGION: ${{ vars.TRINO_AWS_REGION }} TRINO_AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }} TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }} - DATABRICKS_91_JDBC_URL: ${{ vars.DATABRICKS_91_JDBC_URL }} DATABRICKS_104_JDBC_URL: ${{ vars.DATABRICKS_104_JDBC_URL }} DATABRICKS_113_JDBC_URL: ${{ vars.DATABRICKS_113_JDBC_URL }} DATABRICKS_122_JDBC_URL: ${{ vars.DATABRICKS_122_JDBC_URL }} diff --git a/plugin/trino-delta-lake/README.md b/plugin/trino-delta-lake/README.md index 9806a769da36..561635493fce 100644 --- a/plugin/trino-delta-lake/README.md +++ b/plugin/trino-delta-lake/README.md @@ -163,7 +163,7 @@ for setting up your Databricks personal access token. ### Test the functionality of the Databricks Delta Lake product test environment -Run the following command for spinning up the Databricks 9.1 Delta Lake product test +Run the following command for spinning up the Databricks 15.4 Delta Lake product test environment for Trino: ``` @@ -171,7 +171,7 @@ env S3_BUCKET=my-s3-bucket \ AWS_REGION=us-east-2 \ TRINO_AWS_SECRET_ACCESS_KEY=xxx \ TRINO_AWS_ACCESS_KEY_ID=xxx \ - DATABRICKS_91_JDBC_URL='xxx' \ + DATABRICKS_154_JDBC_URL='xxx' \ DATABRICKS_LOGIN=token \ DATABRICKS_TOKEN=xxx \ testing/bin/ptl env up --environment singlenode-delta-lake-databricks91 diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java index 8d1db3984a13..cb6de42b7954 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java @@ -123,6 +123,7 @@ public class TestDeltaLakeBasic new ResourceTable("stats_with_minmax_nulls", "deltalake/stats_with_minmax_nulls"), new ResourceTable("no_column_stats", "databricks73/no_column_stats"), new ResourceTable("liquid_clustering", "deltalake/liquid_clustering"), + new ResourceTable("region_91_lts", "databricks91/region"), new ResourceTable("timestamp_ntz", "databricks131/timestamp_ntz"), new ResourceTable("timestamp_ntz_partition", "databricks131/timestamp_ntz_partition"), new ResourceTable("uniform_hudi", "deltalake/uniform_hudi"), @@ -206,6 +207,14 @@ public void testSimpleQueries() } } + @Test + void testDatabricks91() + { + assertThat(query("SELECT * FROM region_91_lts")) + .skippingTypesCheck() // name and comment columns are unbounded varchar in Delta Lake and bounded varchar in TPCH + .matches("SELECT * FROM tpch.tiny.region"); + } + @Test public void testNoColumnStats() { diff --git a/plugin/trino-delta-lake/src/test/resources/databricks91/region/README.md b/plugin/trino-delta-lake/src/test/resources/databricks91/region/README.md new file mode 100644 index 000000000000..cba2f72da1d1 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks91/region/README.md @@ -0,0 +1,13 @@ +Data generated using Databricks 9.1: + +```sql +CREATE TABLE default.region (regionkey bigint, name string, comment string) +USING DELTA LOCATION 's3://trino-ci-test/default/region'; + +INSERT INTO default.region VALUES +(0, 'AFRICA', 'lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to '), +(1, 'AMERICA', 'hs use ironic, even requests. s'), +(2, 'ASIA', 'ges. thinly even pinto beans ca'), +(3, 'EUROPE', 'ly final courts cajole furiously final excuse'), +(4, 'MIDDLE EAST', 'uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl'); +``` diff --git a/plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000000.json b/plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..d70cd47d56a5 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"2e681e6c-a691-4e01-b6f5-7be057fba57e","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"regionkey\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"comment\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1734911306140}} +{"commitInfo":{"timestamp":1734911306279,"userId":"7853186923043731","userName":"yuya.ebihara@starburstdata.com","operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{}"},"notebook":{"notebookId":"1841155838656679"},"clusterId":"0705-100855-ka5jat1a","isolationLevel":"SnapshotIsolation","isBlindAppend":true,"operationMetrics":{}}} diff --git a/plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000001.json b/plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000001.json new file mode 100644 index 000000000000..e6ab37a1df3f --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks91/region/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"add":{"path":"part-00000-3a487bbf-c1d3-404f-b99f-d749d1c7419e-c000.snappy.parquet","partitionValues":{},"size":1408,"modificationTime":1734911311000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"regionkey\":0,\"name\":\"AFRICA\",\"comment\":\"hs use ironic, even requests. s\"},\"maxValues\":{\"regionkey\":1,\"name\":\"AMERICA\",\"comment\":\"lar deposits. blithely final pac�\"},\"nullCount\":{\"regionkey\":0,\"name\":0,\"comment\":0}}","tags":{"INSERTION_TIME":"1734911311000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} +{"add":{"path":"part-00001-87059848-17f8-4dc0-a2d6-4f2a6760e3b9-c000.snappy.parquet","partitionValues":{},"size":1446,"modificationTime":1734911311000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"regionkey\":2,\"name\":\"ASIA\",\"comment\":\"ges. thinly even pinto beans ca\"},\"maxValues\":{\"regionkey\":4,\"name\":\"MIDDLE EAST\",\"comment\":\"uickly special accounts cajole c�\"},\"nullCount\":{\"regionkey\":0,\"name\":0,\"comment\":0}}","tags":{"INSERTION_TIME":"1734911311000001","OPTIMIZE_TARGET_SIZE":"268435456"}}} +{"commitInfo":{"timestamp":1734911310660,"userId":"7853186923043731","userName":"yuya.ebihara@starburstdata.com","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1841155838656679"},"clusterId":"0705-100855-ka5jat1a","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"5","numOutputBytes":"2854"}}} diff --git a/plugin/trino-delta-lake/src/test/resources/databricks91/region/part-00000-3a487bbf-c1d3-404f-b99f-d749d1c7419e-c000.snappy.parquet b/plugin/trino-delta-lake/src/test/resources/databricks91/region/part-00000-3a487bbf-c1d3-404f-b99f-d749d1c7419e-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1a09077fdf47adf74c404b71f0e87aeda71f192b GIT binary patch literal 1408 zcmd5+zi-n(6h6CY5=6UkmMtRnWYj1{sZ}R!nnYQm0-{nuRaLPdgf5p$V&WgE&u$e} z9a$L|m>8ItIv|8VEQmj$D}t5Ih!qB&V<#4C`@Vbk-sjtFZkZV2B3{C8 zJ{`VK=Wq(bG|uMKtc2oA%_ak+pu#y{VOH6e;W;%m29BwqRl#-q^T+egmrtZCsE}#i zxw*Bz)>PBYThSYcS9`HW8piR*Of-sdAoNkjuVnoF`0G3AHz{FLpS1bF6ILJ$(z30f z$L%5MST426fU-XAazPk<Q@Llc#Z>%67^B2;$^>{~7^sh`i*#~q5(2&d*{^%88{^4vZjB0K|e%B2pUM{$s0p5t&g zfLKXsIjxb;LL0Odl+J3Wm{?X)%4m?1HnQk?M5WFpUyY7tSEWj%|Pgz^{SfdwYh%1KOqmRli94dN3St zy{?PE4_TlW^^V2uw$SVM_w~pTFf}m>%qD2tbGs0^p>5Na4Ip6iqCOVZ>-z@>#fiX3 zXTpV*;K#Z+JF());ujNmaDddgGo^^Dc>|7vZOh?NLGLA%S#*4ol#-Z+xn!7&QH@%@ z#rmQ~3a!wx+k|y0tXe8}sw=GAs?^%cHEyn~micm(m&z4$xyn|TYP?!n&>A^};u}4N JqjD3@-d_@Ke`){# literal 0 HcmV?d00001 diff --git a/plugin/trino-delta-lake/src/test/resources/databricks91/region/part-00001-87059848-17f8-4dc0-a2d6-4f2a6760e3b9-c000.snappy.parquet b/plugin/trino-delta-lake/src/test/resources/databricks91/region/part-00001-87059848-17f8-4dc0-a2d6-4f2a6760e3b9-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dce5997a2d5d32367fcd4c445eaf827bad189dc2 GIT binary patch literal 1446 zcmd6nL2DCH5Xax9$u`m+eakL{c3Ej`AzhlJO_PvIpaBcDmbQ8kk(bRRS-aUy_r0yg zTJR(uJbLiz)q{8wJc)>ipTL8N-g*$hOV7?G+cwgQ?adI_H~aqcn|U*{Z>_qXrG!>! znSTBL?M*U8Wi(})OsUB@8T`~_ELtgK`jof0IdL(RjEqXMBvnQt^`SokSLr(4fB)*a ze9SM9>9MQJOG_&?y;j}WRO8i+kI4NT`p_4W@>0UR%IQNk3^<4#0}k`< zGkzqL4~g_aEM)v=^E8yd@;Rx9nWo{<>JT3>JEy~8#2Zr*eLJr?PZPtea?|DMzYW29ZlhJ@UsDK z?GbhIL@5T7@$ly;N;#InaH}r3Hw~t2v literal 0 HcmV?d00001 diff --git a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java index b31a26f78247..ca159621c163 100644 --- a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java @@ -102,7 +102,7 @@ public final class TestGroups public static final String DELTA_LAKE_DATABRICKS_133 = "delta-lake-databricks-133"; public static final String DELTA_LAKE_DATABRICKS_143 = "delta-lake-databricks-143"; public static final String DATABRICKS_UNITY_HTTP_HMS = "databricks-unity-http-hms"; - public static final String DELTA_LAKE_EXCLUDE_91 = "delta-lake-exclude-91"; + public static final String DELTA_LAKE_EXCLUDE_104 = "delta-lake-exclude-104"; public static final String DELTA_LAKE_ALLUXIO_CACHING = "delta-lake-alluxio-caching"; public static final String HUDI = "hudi"; public static final String PARQUET = "parquet"; diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java deleted file mode 100644 index cf85d7118b53..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.env.environment; - -import com.google.inject.Inject; -import io.trino.tests.product.launcher.docker.DockerFiles; -import io.trino.tests.product.launcher.env.common.Standard; -import io.trino.tests.product.launcher.env.common.TestsEnvironment; - -import static io.trino.testing.SystemEnvironmentUtils.requireEnv; - -@TestsEnvironment -public class EnvSinglenodeDeltaLakeDatabricks91 - extends AbstractSinglenodeDeltaLakeDatabricks -{ - @Inject - public EnvSinglenodeDeltaLakeDatabricks91(Standard standard, DockerFiles dockerFiles) - { - super(standard, dockerFiles); - } - - @Override - String databricksTestJdbcUrl() - { - return requireEnv("DATABRICKS_91_JDBC_URL"); - } -} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks104.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks104.java index 5ea5c74566fe..decb318190d4 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks104.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks104.java @@ -22,7 +22,8 @@ import java.util.List; import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; public class SuiteDeltaLakeDatabricks104 @@ -33,7 +34,8 @@ public List getTestRuns(EnvironmentConfig config) { return ImmutableList.of( testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks104.class) - .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS_104) + .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS) + .withExcludedGroups(DELTA_LAKE_EXCLUDE_104) .withExcludedTests(getExcludedTests()) .build()); } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks91.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks91.java deleted file mode 100644 index c510d8b2d5ee..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks91.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.suite.suites; - -import com.google.common.collect.ImmutableList; -import io.trino.tests.product.launcher.env.EnvironmentConfig; -import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks91; -import io.trino.tests.product.launcher.suite.SuiteDeltaLakeDatabricks; -import io.trino.tests.product.launcher.suite.SuiteTestRun; - -import java.util.List; - -import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; -import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; - -public class SuiteDeltaLakeDatabricks91 - extends SuiteDeltaLakeDatabricks -{ - @Override - public List getTestRuns(EnvironmentConfig config) - { - return ImmutableList.of( - testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks91.class) - .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS) - .withExcludedGroups(DELTA_LAKE_EXCLUDE_91) - .withExcludedTests(getExcludedTests()) - .build()); - } -} diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java index c904549cc8ad..bc8d7edd1f24 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java @@ -25,7 +25,7 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION; @@ -244,7 +244,7 @@ public void testTrinoPreservesReaderAndWriterVersions() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_91, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoPreservesTableFeature() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java index 41049876761c..d0df70939f0c 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java @@ -29,7 +29,7 @@ import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION; @@ -192,7 +192,7 @@ public void testGeneratedColumnWithNonLowerCaseColumnName() } // Exclude 10.4 because it throws MISSING_COLUMN when executing INSERT statement - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumnWithNonLowerCaseColumnName() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java index f4633503d194..b06653a2e9c8 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java @@ -36,7 +36,7 @@ import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -64,7 +64,7 @@ public void setup() s3Client = new S3ClientFactory().createS3Client(s3ServerType); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdateTableWithCdf(String columnMappingMode) { @@ -100,7 +100,7 @@ public void testUpdateTableWithCdf(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdateTableWithChangeDataFeedWriterFeature() { @@ -201,7 +201,7 @@ public void testUpdateCdfTableWithNonLowercaseColumn(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdatePartitionedTableWithCdf(String columnMappingMode) { @@ -387,7 +387,7 @@ public void testUpdateTableWithCdfEnabledAfterTableIsAlreadyCreated() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeleteFromTableWithCdf(String columnMappingMode) { @@ -533,7 +533,7 @@ public void testMergeDeleteIntoTableWithCdfEnabled(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testMergeMixedDeleteAndUpdateIntoTableWithCdfEnabled() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java index e5e7428f954c..d773c66a6547 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java @@ -28,7 +28,7 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -241,7 +241,7 @@ public void testReadFromSchemaChangedShallowCloneTable() testReadSchemaChangedCloneTable("SHALLOW", false); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testReadFromSchemaChangedDeepCloneTable() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java index 5b8c89476177..03d89222d0a7 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java @@ -32,7 +32,6 @@ import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -56,7 +55,7 @@ public class TestDeltaLakeColumnMappingMode extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testColumnMappingModeNone() { @@ -235,7 +234,7 @@ private void testColumnMappingModeReaderAndWriterVersion(Consumer create onTrino().executeQuery("DROP TABLE delta.default." + tableName); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoColumnMappingMode(String mode) { @@ -249,7 +248,7 @@ public void testTrinoColumnMappingMode(String mode) ")")); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeltaColumnMappingMode(String mode) { @@ -666,7 +665,7 @@ public void testShowStatsOnPartitionedForColumnMappingModeId() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testProjectionPushdownDmlWithColumnMappingMode(String mode) { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java index a6406a1fef84..09dad521a2d9 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java @@ -401,6 +401,13 @@ public void testCreateTableWithAllPartitionColumns() private String getDatabricksDefaultTableProperties() { + if (databricksRuntimeVersion.equals(DATABRICKS_104_RUNTIME_VERSION)) { + return "TBLPROPERTIES (\n" + + " 'Type' = 'EXTERNAL',\n" + + " 'delta.enableDeletionVectors' = 'false',\n" + + " 'delta.minReaderVersion' = '1',\n" + + " 'delta.minWriterVersion' = '2')\n"; + } if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { return "TBLPROPERTIES (\n" + " 'delta.enableDeletionVectors' = 'false',\n" + diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java index b022ad716b42..ef24c834f3a8 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java @@ -31,7 +31,7 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -208,7 +208,7 @@ public void testTrinoDeletionVectors() } // Databricks 12.1 and OSS Delta 2.4.0 added support for deletion vectors - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeletionVectors(String mode) { @@ -495,7 +495,7 @@ public void testDeletionVectorsAcrossAddFile(boolean partitioned) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeletionVectorsTruncateTable() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java index 723319275880..63417308ebb7 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java @@ -24,7 +24,7 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -38,7 +38,7 @@ public class TestDeltaLakeSystemTableCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_91, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTablePropertiesCaseSensitivity() { From 9dc0a9c25627be33ee5a6ce22720e1bfb3efa6e6 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 23 Dec 2024 17:03:36 +0900 Subject: [PATCH 051/158] Pin openpolicyagent/opa version as 0.70.0 --- .../src/test/java/io/trino/plugin/opa/OpaContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-opa/src/test/java/io/trino/plugin/opa/OpaContainer.java b/plugin/trino-opa/src/test/java/io/trino/plugin/opa/OpaContainer.java index 6adbd04fe5a4..7ab583215939 100644 --- a/plugin/trino-opa/src/test/java/io/trino/plugin/opa/OpaContainer.java +++ b/plugin/trino-opa/src/test/java/io/trino/plugin/opa/OpaContainer.java @@ -38,7 +38,7 @@ public class OpaContainer public OpaContainer() { - this.container = new GenericContainer<>(DockerImageName.parse("openpolicyagent/opa:latest")) + this.container = new GenericContainer<>(DockerImageName.parse("openpolicyagent/opa:0.70.0")) .withCommand("run", "--server", "--addr", ":%d".formatted(OPA_PORT), "--set", "decision_logs.console=true") .withExposedPorts(OPA_PORT) .waitingFor(Wait.forListeningPort()); From 21306cd6a57f1d1cd81bf7447e79581325a3785e Mon Sep 17 00:00:00 2001 From: lukasz-stec Date: Wed, 18 Dec 2024 18:24:39 +0100 Subject: [PATCH 052/158] Add tests for dropMaterializedView The dropMaterializedView on TrinoHiveCatalog that uses FileHiveMetastore works only if unique table locations are enabled. --- ...TestTrinoHiveCatalogWithFileMetastore.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java index a23952cc3167..28c7beb01b61 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java @@ -13,6 +13,9 @@ */ package io.trino.plugin.iceberg.catalog.file; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.log.Logger; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.local.LocalFileSystemFactory; @@ -24,9 +27,15 @@ import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalog; import io.trino.spi.catalog.CatalogName; +import io.trino.spi.connector.ConnectorMaterializedViewDefinition; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.security.PrincipalType; +import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.type.TestingTypeManager; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.parallel.Execution; @@ -34,12 +43,19 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.hive.metastore.file.TestingFileHiveMetastore.createTestingFileHiveMetastore; +import static io.trino.plugin.iceberg.IcebergFileFormat.PARQUET; +import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.testing.TestingConnectorSession.SESSION; +import static io.trino.testing.TestingNames.randomNameSuffix; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; @@ -48,6 +64,8 @@ public class TestTrinoHiveCatalogWithFileMetastore extends BaseTrinoCatalogTest { + private static final Logger log = Logger.get(TestTrinoHiveCatalogWithFileMetastore.class); + private Path tempDir; private TrinoFileSystemFactory fileSystemFactory; private HiveMetastore metastore; @@ -87,4 +105,53 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) new IcebergConfig().isHideMaterializedViewStorageTable(), directExecutor()); } + + @Test + @Disabled + public void testDropMaterializedView() + { + testDropMaterializedView(false); + } + + @Test + public void testDropMaterializedViewWithUniqueTableLocation() + { + testDropMaterializedView(true); + } + + private void testDropMaterializedView(boolean useUniqueTableLocations) + { + TrinoCatalog catalog = createTrinoCatalog(useUniqueTableLocations); + String namespace = "test_create_mv_" + randomNameSuffix(); + String materializedViewName = "materialized_view_name"; + try { + catalog.createNamespace(SESSION, namespace, defaultNamespaceProperties(namespace), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser())); + catalog.createMaterializedView( + SESSION, + new SchemaTableName(namespace, materializedViewName), + new ConnectorMaterializedViewDefinition( + "SELECT * FROM tpch.tiny.nation", + Optional.empty(), + Optional.of("catalog_name"), + Optional.of("schema_name"), + ImmutableList.of(new ConnectorMaterializedViewDefinition.Column("col1", INTEGER.getTypeId(), Optional.empty())), + Optional.empty(), + Optional.empty(), + Optional.empty(), + ImmutableList.of()), + ImmutableMap.of(FILE_FORMAT_PROPERTY, PARQUET, FORMAT_VERSION_PROPERTY, 1), + false, + false); + + catalog.dropMaterializedView(SESSION, new SchemaTableName(namespace, materializedViewName)); + } + finally { + try { + catalog.dropNamespace(SESSION, namespace); + } + catch (Exception e) { + log.warn("Failed to clean up namespace: %s", namespace); + } + } + } } From 8dfa5ab1acb4f9f3553b370786e183dd59cd9b3a Mon Sep 17 00:00:00 2001 From: lukasz-stec Date: Wed, 18 Dec 2024 18:49:09 +0100 Subject: [PATCH 053/158] Replace testView method override with getViewType --- .../iceberg/catalog/BaseTrinoCatalogTest.java | 8 +- .../catalog/rest/TestTrinoRestCatalog.java | 73 ++----------------- 2 files changed, 13 insertions(+), 68 deletions(-) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java index a8aa117f9cde..4f8b355798da 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; import io.trino.metastore.TableInfo; +import io.trino.metastore.TableInfo.ExtendedRelationType; import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.iceberg.CommitTaskData; @@ -364,7 +365,7 @@ public void testView() catalog.createNamespace(SESSION, namespace, defaultNamespaceProperties(namespace), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser())); catalog.createView(SESSION, schemaTableName, viewDefinition, false); - assertThat(catalog.listTables(SESSION, Optional.of(namespace)).stream()).contains(new TableInfo(schemaTableName, TRINO_VIEW)); + assertThat(catalog.listTables(SESSION, Optional.of(namespace)).stream()).contains(new TableInfo(schemaTableName, getViewType())); Map views = catalog.getViews(SESSION, Optional.of(schemaTableName.getSchemaName())); assertThat(views).hasSize(1); @@ -393,6 +394,11 @@ public void testView() } } + protected ExtendedRelationType getViewType() + { + return TRINO_VIEW; + } + @Test public void testListTables() throws Exception diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java index df3ad2a5400c..398ee371ad1b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java @@ -13,9 +13,7 @@ */ package io.trino.plugin.iceberg.catalog.rest; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; import io.trino.cache.EvictableCacheBuilder; import io.trino.metastore.TableInfo; import io.trino.plugin.hive.NodeVersion; @@ -27,12 +25,9 @@ import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.CatalogHandle; import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorViewDefinition; -import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.PrincipalType; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.type.TestingTypeManager; -import io.trino.spi.type.VarcharType; import org.apache.iceberg.exceptions.BadRequestException; import org.apache.iceberg.rest.DelegatingRestSessionCatalog; import org.apache.iceberg.rest.RESTSessionCatalog; @@ -40,8 +35,6 @@ import org.junit.jupiter.api.Test; import java.io.File; -import java.io.IOException; -import java.nio.file.Path; import java.util.Map; import java.util.Optional; @@ -62,8 +55,6 @@ public class TestTrinoRestCatalog extends BaseTrinoCatalogTest { - private static final Logger LOG = Logger.get(TestTrinoRestCatalog.class); - @Override protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) { @@ -145,64 +136,6 @@ public void testNonLowercaseNamespace() } } - @Test - @Override - public void testView() - throws IOException - { - TrinoCatalog catalog = createTrinoCatalog(false); - Path tmpDirectory = java.nio.file.Files.createTempDirectory("iceberg_catalog_test_create_view_"); - tmpDirectory.toFile().deleteOnExit(); - - String namespace = "test_create_view_" + randomNameSuffix(); - String viewName = "viewName"; - String renamedViewName = "renamedViewName"; - SchemaTableName schemaTableName = new SchemaTableName(namespace, viewName); - SchemaTableName renamedSchemaTableName = new SchemaTableName(namespace, renamedViewName); - ConnectorViewDefinition viewDefinition = new ConnectorViewDefinition( - "SELECT name FROM local.tiny.nation", - Optional.empty(), - Optional.empty(), - ImmutableList.of( - new ConnectorViewDefinition.ViewColumn("name", VarcharType.createUnboundedVarcharType().getTypeId(), Optional.empty())), - Optional.empty(), - Optional.of(SESSION.getUser()), - false, - ImmutableList.of()); - - try { - catalog.createNamespace(SESSION, namespace, ImmutableMap.of(), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser())); - catalog.createView(SESSION, schemaTableName, viewDefinition, false); - - assertThat(catalog.listTables(SESSION, Optional.of(namespace)).stream()).contains(new TableInfo(schemaTableName, OTHER_VIEW)); - - Map views = catalog.getViews(SESSION, Optional.of(schemaTableName.getSchemaName())); - assertThat(views).hasSize(1); - assertViewDefinition(views.get(schemaTableName), viewDefinition); - assertViewDefinition(catalog.getView(SESSION, schemaTableName).orElseThrow(), viewDefinition); - - catalog.renameView(SESSION, schemaTableName, renamedSchemaTableName); - assertThat(catalog.listTables(SESSION, Optional.of(namespace)).stream().map(TableInfo::tableName).toList()).doesNotContain(schemaTableName); - views = catalog.getViews(SESSION, Optional.of(schemaTableName.getSchemaName())); - assertThat(views).hasSize(1); - assertViewDefinition(views.get(renamedSchemaTableName), viewDefinition); - assertViewDefinition(catalog.getView(SESSION, renamedSchemaTableName).orElseThrow(), viewDefinition); - assertThat(catalog.getView(SESSION, schemaTableName)).isEmpty(); - - catalog.dropView(SESSION, renamedSchemaTableName); - assertThat(catalog.listTables(SESSION, Optional.empty()).stream().map(TableInfo::tableName).toList()) - .doesNotContain(renamedSchemaTableName); - } - finally { - try { - catalog.dropNamespace(SESSION, namespace); - } - catch (Exception e) { - LOG.warn("Failed to clean up namespace: %s", namespace); - } - } - } - @Test public void testPrefix() { @@ -220,4 +153,10 @@ public void testPrefix() .as("should fail as the prefix dev is not implemented for the current endpoint") .hasMessageContaining("Malformed request: No route for request: POST v1/dev/namespaces"); } + + @Override + protected TableInfo.ExtendedRelationType getViewType() + { + return OTHER_VIEW; + } } From 563ba351da6340109ace5c4c8624fa2355107b32 Mon Sep 17 00:00:00 2001 From: lukasz-stec Date: Wed, 18 Dec 2024 18:27:58 +0100 Subject: [PATCH 054/158] Extend testListTables with other relation types TrinoCatalog.listTables returns not only iceberg tables but also other relations like views, materialized view or non-iceberg tables. --- .../iceberg/catalog/BaseTrinoCatalogTest.java | 90 ++++++++++++++++++- ...TestTrinoHiveCatalogWithFileMetastore.java | 9 ++ ...TestTrinoHiveCatalogWithHiveMetastore.java | 54 ++++++++++- 3 files changed, 150 insertions(+), 3 deletions(-) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java index 4f8b355798da..1cf15d256401 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java @@ -21,10 +21,12 @@ import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.iceberg.CommitTaskData; +import io.trino.plugin.iceberg.IcebergFileFormat; import io.trino.plugin.iceberg.IcebergMetadata; import io.trino.plugin.iceberg.TableStatisticsWriter; import io.trino.spi.TrinoException; import io.trino.spi.connector.CatalogHandle; +import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorViewDefinition; @@ -44,6 +46,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -53,10 +56,15 @@ import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static io.airlift.json.JsonCodec.jsonCodec; import static io.trino.metastore.TableInfo.ExtendedRelationType.TABLE; +import static io.trino.metastore.TableInfo.ExtendedRelationType.TRINO_MATERIALIZED_VIEW; import static io.trino.metastore.TableInfo.ExtendedRelationType.TRINO_VIEW; import static io.trino.plugin.hive.HiveErrorCode.HIVE_DATABASE_LOCATION_ERROR; import static io.trino.plugin.iceberg.IcebergSchemaProperties.LOCATION_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; import static io.trino.plugin.iceberg.IcebergUtil.quotedTableName; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.sql.planner.TestingPlannerContext.PLANNER_CONTEXT; import static io.trino.testing.TestingConnectorSession.SESSION; import static io.trino.testing.TestingNames.randomNameSuffix; @@ -438,8 +446,60 @@ public void testListTables() .commitTransaction(); closer.register(() -> catalog.dropTable(SESSION, table2)); + ImmutableList.Builder allTables = ImmutableList.builder() + .add(new TableInfo(table1, TABLE)) + .add(new TableInfo(table2, TABLE)); + + SchemaTableName view = new SchemaTableName(ns2, "view"); + try { + catalog.createView( + SESSION, + view, + new ConnectorViewDefinition( + "SELECT name FROM local.tiny.nation", + Optional.empty(), + Optional.empty(), + ImmutableList.of( + new ConnectorViewDefinition.ViewColumn("name", VarcharType.createUnboundedVarcharType().getTypeId(), Optional.empty())), + Optional.empty(), + Optional.of(SESSION.getUser()), + false, + ImmutableList.of()), + false); + closer.register(() -> catalog.dropView(SESSION, view)); + allTables.add(new TableInfo(view, getViewType())); + } + catch (TrinoException e) { + assertThat(e.getErrorCode()).isEqualTo(NOT_SUPPORTED.toErrorCode()); + } + + try { + SchemaTableName materializedView = new SchemaTableName(ns2, "mv"); + catalog.createMaterializedView( + SESSION, + materializedView, + someMaterializedView(), + ImmutableMap.of( + FILE_FORMAT_PROPERTY, IcebergFileFormat.PARQUET, + FORMAT_VERSION_PROPERTY, 1), + false, + false); + closer.register(() -> catalog.dropMaterializedView(SESSION, materializedView)); + allTables.add(new TableInfo(materializedView, TRINO_MATERIALIZED_VIEW)); + } + catch (TrinoException e) { + assertThat(e.getErrorCode()).isEqualTo(NOT_SUPPORTED.toErrorCode()); + } + + createExternalIcebergTable(catalog, ns2, closer).ifPresent(table -> { + allTables.add(new TableInfo(table, TABLE)); + }); + createExternalNonIcebergTable(catalog, ns2, closer).ifPresent(table -> { + allTables.add(new TableInfo(table, TABLE)); + }); + // No namespace provided, all tables across all namespaces should be returned - assertThat(catalog.listTables(SESSION, Optional.empty())).containsAll(ImmutableList.of(new TableInfo(table1, TABLE), new TableInfo(table2, TABLE))); + assertThat(catalog.listTables(SESSION, Optional.empty())).containsAll(allTables.build()); // Namespace is provided and exists assertThat(catalog.listTables(SESSION, Optional.of(ns1))).containsExactly(new TableInfo(table1, TABLE)); // Namespace is provided and does not exist @@ -447,6 +507,18 @@ public void testListTables() } } + protected Optional createExternalIcebergTable(TrinoCatalog catalog, String namespace, AutoCloseableCloser closer) + throws Exception + { + return Optional.empty(); + } + + protected Optional createExternalNonIcebergTable(TrinoCatalog catalog, String namespace, AutoCloseableCloser closer) + throws Exception + { + return Optional.empty(); + } + protected void assertViewDefinition(ConnectorViewDefinition actualView, ConnectorViewDefinition expectedView) { assertThat(actualView.getOriginalSql()).isEqualTo(expectedView.getOriginalSql()); @@ -460,7 +532,7 @@ protected void assertViewDefinition(ConnectorViewDefinition actualView, Connecto assertThat(actualView.isRunAsInvoker()).isEqualTo(expectedView.isRunAsInvoker()); } - private String arbitraryTableLocation(TrinoCatalog catalog, ConnectorSession session, SchemaTableName schemaTableName) + protected String arbitraryTableLocation(TrinoCatalog catalog, ConnectorSession session, SchemaTableName schemaTableName) throws Exception { try { @@ -481,4 +553,18 @@ private void assertViewColumnDefinition(ConnectorViewDefinition.ViewColumn actua assertThat(actualViewColumn.getName()).isEqualTo(expectedViewColumn.getName()); assertThat(actualViewColumn.getType()).isEqualTo(expectedViewColumn.getType()); } + + private static ConnectorMaterializedViewDefinition someMaterializedView() + { + return new ConnectorMaterializedViewDefinition( + "select 1", + Optional.empty(), + Optional.empty(), + Optional.empty(), + ImmutableList.of(new ConnectorMaterializedViewDefinition.Column("test", BIGINT.getTypeId(), Optional.empty())), + Optional.of(Duration.ZERO), + Optional.empty(), + Optional.of("owner"), + ImmutableList.of()); + } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java index 28c7beb01b61..c8641e0773f0 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java @@ -56,6 +56,7 @@ import static io.trino.spi.type.IntegerType.INTEGER; import static io.trino.testing.TestingConnectorSession.SESSION; import static io.trino.testing.TestingNames.randomNameSuffix; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; @@ -154,4 +155,12 @@ private void testDropMaterializedView(boolean useUniqueTableLocations) } } } + + @Test + @Override + public void testListTables() + { + // the test actually works but when cleanup up the materialized view the error is thrown + assertThatThrownBy(super::testListTables).hasMessageMatching("Table 'ns2.*.mv' not found"); + } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index 01f4b8a4c3ed..c0f80a947eeb 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -30,6 +30,7 @@ import io.trino.hdfs.authentication.NoHdfsAuthentication; import io.trino.hdfs.s3.HiveS3Config; import io.trino.hdfs.s3.TrinoS3ConfigurationInitializer; +import io.trino.metastore.Table; import io.trino.metastore.TableInfo; import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.plugin.hive.TrinoViewHiveMetastore; @@ -51,6 +52,10 @@ import io.trino.spi.security.PrincipalType; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.type.TestingTypeManager; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -65,6 +70,7 @@ import static com.google.common.base.Verify.verify; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static io.trino.metastore.PrincipalPrivileges.NO_PRIVILEGES; import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder; import static io.trino.plugin.hive.containers.HiveHadoop.HIVE3_IMAGE; import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; @@ -76,7 +82,10 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; +import static java.util.Locale.ENGLISH; import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE; +import static org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; @@ -92,6 +101,7 @@ public class TestTrinoHiveCatalogWithHiveMetastore // Use MinIO for storage, since HDFS is hard to get working in a unit test private HiveMinioDataLake dataLake; private TrinoFileSystem fileSystem; + private CachingHiveMetastore metastore; protected String bucketName; HiveMinioDataLake hiveMinioDataLake() @@ -138,7 +148,7 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) .setReadTimeout(new Duration(1, MINUTES))) .metastoreClient(dataLake.getHiveMetastoreEndpoint()) .build(closer::register); - CachingHiveMetastore metastore = createPerTransactionCache(new BridgingHiveMetastore(thriftMetastore), 1000); + metastore = createPerTransactionCache(new BridgingHiveMetastore(thriftMetastore), 1000); fileSystem = fileSystemFactory.create(SESSION); return new TrinoHiveCatalog( @@ -235,6 +245,48 @@ public void testCreateMaterializedView() } } + @Override + protected Optional createExternalIcebergTable(TrinoCatalog catalog, String namespace, AutoCloseableCloser closer) + throws Exception + { + // simulate iceberg table created by spark with lowercase table type + return createTableWithTableType(catalog, namespace, closer, "lowercase_type", Optional.of(ICEBERG_TABLE_TYPE_VALUE.toLowerCase(ENGLISH))); + } + + @Override + protected Optional createExternalNonIcebergTable(TrinoCatalog catalog, String namespace, AutoCloseableCloser closer) + throws Exception + { + return createTableWithTableType(catalog, namespace, closer, "non_iceberg_table", Optional.empty()); + } + + private Optional createTableWithTableType(TrinoCatalog catalog, String namespace, AutoCloseableCloser closer, String tableName, Optional tableType) + throws Exception + { + SchemaTableName lowerCaseTableTypeTable = new SchemaTableName(namespace, tableName); + catalog.newCreateTableTransaction( + SESSION, + lowerCaseTableTypeTable, + new Schema(Types.NestedField.of(1, true, "col1", Types.LongType.get())), + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + arbitraryTableLocation(catalog, SESSION, lowerCaseTableTypeTable), + ImmutableMap.of()) + .commitTransaction(); + + Table metastoreTable = metastore.getTable(namespace, tableName).get(); + + metastore.replaceTable( + namespace, + tableName, + Table.builder(metastoreTable) + .setParameter(TABLE_TYPE_PROP, tableType) + .build(), + NO_PRIVILEGES); + closer.register(() -> metastore.dropTable(namespace, tableName, true)); + return Optional.of(lowerCaseTableTypeTable); + } + @Override protected Map defaultNamespaceProperties(String namespaceName) { From 8cd060d86ae335ab3cf6c8547931a32e188e9725 Mon Sep 17 00:00:00 2001 From: lukasz-stec Date: Fri, 13 Dec 2024 11:19:01 +0100 Subject: [PATCH 055/158] Add HiveMetastore.getTableNamesWithParameters --- .../io/trino/metastore/HiveMetastore.java | 6 +++ .../tracing/TracingHiveMetastore.java | 15 +++++++ .../metastore/cache/CachingHiveMetastore.java | 38 ++++++++++++++++ .../cache/SharedHiveMetastoreCache.java | 7 +++ .../metastore/file/FileHiveMetastore.java | 17 +++++-- .../metastore/glue/GlueHiveMetastore.java | 16 ++++++- .../metastore/glue/v1/GlueHiveMetastore.java | 19 ++++++++ .../thrift/BridgingHiveMetastore.java | 7 +++ .../DefaultThriftMetastoreClientFactory.java | 2 + .../FailureAwareThriftMetastoreClient.java | 8 ++++ .../HttpThriftMetastoreClientFactory.java | 2 + .../metastore/thrift/ThriftHiveMetastore.java | 24 ++++++++++ .../thrift/ThriftHiveMetastoreClient.java | 45 +++++++++++++++++++ .../metastore/thrift/ThriftMetastore.java | 2 + .../thrift/ThriftMetastoreClient.java | 4 ++ .../plugin/hive/TestHiveMetadataListing.java | 7 +++ .../thrift/MockThriftMetastoreClient.java | 7 +++ .../thrift/TestThriftHiveMetastoreClient.java | 1 + 18 files changed, 222 insertions(+), 5 deletions(-) diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java index 8d06dadf5d6b..ff2f44ee0946 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java @@ -13,6 +13,7 @@ */ package io.trino.metastore; +import com.google.common.collect.ImmutableSet; import io.trino.metastore.HivePrivilegeInfo.HivePrivilege; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaTableName; @@ -67,6 +68,11 @@ default boolean useSparkTableStatistics() List getTables(String databaseName); + /** + * @param parameterValues is using ImmutableSet to mark that this api does not support filtering by null parameter value. + */ + List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues); + void createDatabase(Database database); void dropDatabase(String databaseName, boolean deleteData); diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java b/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java index e8c6f56631ca..4549a69af5cc 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java @@ -13,6 +13,7 @@ */ package io.trino.metastore.tracing; +import com.google.common.collect.ImmutableSet; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; import io.trino.metastore.AcidOperation; @@ -164,6 +165,20 @@ public List getTables(String databaseName) }); } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + Span span = tracer.spanBuilder("HiveMetastore.getTableNamesWithParameters") + .setAttribute(SCHEMA, databaseName) + .setAttribute(TABLE, parameterKey) + .startSpan(); + return withTracing(span, () -> { + List tables = delegate.getTableNamesWithParameters(databaseName, parameterKey, parameterValues); + span.setAttribute(TABLE_RESPONSE_COUNT, tables.size()); + return tables; + }); + } + @Override public void createDatabase(Database database) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java index bcee3aa656d5..f3cf6ee5db16 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java @@ -123,6 +123,7 @@ public enum ObjectType private final LoadingCache> tableCache; private final LoadingCache> tablesCacheNew; private final Cache>> tableColumnStatisticsCache; + private final LoadingCache> tableNamesWithParametersCache; private final Cache>> partitionStatisticsCache; private final Cache>> partitionCache; private final LoadingCache>> partitionFilterCache; @@ -206,6 +207,7 @@ private CachingHiveMetastore( tablesCacheNew = cacheFactory.buildCache(this::loadTablesNew); tableColumnStatisticsCache = statsCacheFactory.buildCache(this::refreshTableColumnStatistics); tableCache = cacheFactory.buildCache(this::loadTable); + tableNamesWithParametersCache = cacheFactory.buildCache(this::loadTablesMatchingParameter); tablePrivilegesCache = cacheFactory.buildCache(key -> loadTablePrivileges(key.database(), key.table(), key.owner(), key.principal())); rolesCache = cacheFactory.buildCache(_ -> loadRoles()); roleGrantsCache = cacheFactory.buildCache(this::loadRoleGrants); @@ -223,6 +225,7 @@ public void flushCache() tablesCacheNew.invalidateAll(); databaseCache.invalidateAll(); tableCache.invalidateAll(); + tableNamesWithParametersCache.invalidateAll(); partitionCache.invalidateAll(); partitionFilterCache.invalidateAll(); tablePrivilegesCache.invalidateAll(); @@ -565,6 +568,18 @@ private List loadTablesNew(String databaseName) return delegate.getTables(databaseName); } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + TablesWithParameterCacheKey key = new TablesWithParameterCacheKey(databaseName, parameterKey, parameterValues); + return get(tableNamesWithParametersCache, key); + } + + private List loadTablesMatchingParameter(TablesWithParameterCacheKey key) + { + return delegate.getTableNamesWithParameters(key.databaseName(), key.parameterKey(), key.parameterValues()); + } + @Override public void createDatabase(Database database) { @@ -733,6 +748,7 @@ public void invalidateTable(String databaseName, String tableName) HiveTableName hiveTableName = new HiveTableName(databaseName, tableName); tableCache.invalidate(hiveTableName); tablesCacheNew.invalidate(databaseName); + tableNamesWithParametersCache.invalidateAll(); invalidateAllIf(tablePrivilegesCache, userTableKey -> userTableKey.matches(databaseName, tableName)); tableColumnStatisticsCache.invalidate(hiveTableName); invalidatePartitionCache(databaseName, tableName); @@ -1153,6 +1169,16 @@ private static Cache> buildBulkCache( return cacheBuilder.build(); } + record TablesWithParameterCacheKey(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + TablesWithParameterCacheKey + { + requireNonNull(databaseName, "databaseName is null"); + requireNonNull(parameterKey, "parameterKey is null"); + requireNonNull(parameterValues, "parameterValues is null"); + } + } + record UserTableKey(Optional principal, String database, String table, Optional owner) { UserTableKey @@ -1201,6 +1227,13 @@ public CacheStatsMBean getTableNamesStats() return new CacheStatsMBean(tablesCacheNew); } + @Managed + @Nested + public CacheStatsMBean getTableWithParameterStats() + { + return new CacheStatsMBean(tableNamesWithParametersCache); + } + @Managed @Nested public CacheStatsMBean getTableColumnStatisticsStats() @@ -1275,6 +1308,11 @@ LoadingCache> getTableCache() return tableCache; } + LoadingCache> getTableNamesWithParametersCache() + { + return tableNamesWithParametersCache; + } + public LoadingCache> getTablesCacheNew() { return tablesCacheNew; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java index b8c50af89a55..364d25a02047 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java @@ -261,6 +261,13 @@ public AggregateCacheStatsMBean getTablesStats() return new AggregateCacheStatsMBean(CachingHiveMetastore::getTablesCacheNew); } + @Managed + @Nested + public AggregateCacheStatsMBean getTableWithParameterStats() + { + return new AggregateCacheStatsMBean(CachingHiveMetastore::getTableNamesWithParametersCache); + } + @Managed @Nested public AggregateCacheStatsMBean getTableColumnStatisticsCache() diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java index 89343de0c71a..7f6efda69f4b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java @@ -83,6 +83,7 @@ import java.util.OptionalLong; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -177,7 +178,7 @@ public FileHiveMetastore(NodeVersion nodeVersion, TrinoFileSystemFactory fileSys listTablesCache = EvictableCacheBuilder.newBuilder() .expireAfterWrite(10, SECONDS) - .build(CacheLoader.from(this::doListAllTables)); + .build(CacheLoader.from(databaseName -> doListAllTables(databaseName, _ -> true))); } @Override @@ -532,7 +533,16 @@ private List listAllTables(String databaseName) return listTablesCache.getUnchecked(databaseName); } - private synchronized List doListAllTables(String databaseName) + @Override + public synchronized List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + requireNonNull(parameterKey, "parameterKey is null"); + return doListAllTables(databaseName, table -> parameterValues.contains(table.getParameters().get(parameterKey))).stream() + .map(tableInfo -> tableInfo.tableName().getTableName()) + .collect(toImmutableList()); + } + + private synchronized List doListAllTables(String databaseName, Predicate tableMetadataPredicate) { requireNonNull(databaseName, "databaseName is null"); @@ -557,7 +567,8 @@ private synchronized List doListAllTables(String databaseName) Location schemaFileLocation = subdirectory.appendPath(TRINO_SCHEMA_FILE_NAME_SUFFIX); readFile("table schema", schemaFileLocation, tableCodec).ifPresent(tableMetadata -> { checkVersion(tableMetadata.getWriterVersion()); - if (hideDeltaLakeTables && DELTA_LAKE_PROVIDER.equals(tableMetadata.getParameters().get(SPARK_TABLE_PROVIDER_KEY))) { + if ((hideDeltaLakeTables && DELTA_LAKE_PROVIDER.equals(tableMetadata.getParameters().get(SPARK_TABLE_PROVIDER_KEY))) + || !tableMetadataPredicate.test(tableMetadata)) { return; } tables.add(new TableInfo( diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java index 59e05ac8c334..eee2a234a016 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java @@ -412,10 +412,21 @@ public void setDatabaseOwner(String databaseName, HivePrincipal principal) @Override public List getTables(String databaseName) { - return glueCache.getTables(databaseName, cacheTable -> getTablesInternal(cacheTable, databaseName)); + return glueCache.getTables(databaseName, cacheTable -> getTablesInternal(cacheTable, databaseName, _ -> true)); } - private List getTablesInternal(Consumer cacheTable, String databaseName) + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + return getTablesInternal( + _ -> {}, + databaseName, + table -> table.parameters() != null && parameterValues.contains(table.parameters().get(parameterKey))).stream() + .map(tableInfo -> tableInfo.tableName().getTableName()) + .collect(toImmutableList()); + } + + private List getTablesInternal(Consumer
cacheTable, String databaseName, Predicate filter) { try { ImmutableList glueTables = stats.getGetTables() @@ -425,6 +436,7 @@ private List getTablesInternal(Consumer
cacheTable, String dat .map(GetTablesResponse::tableList) .flatMap(List::stream)) .filter(tableVisibilityFilter) + .filter(filter) .collect(toImmutableList()); // Store only valid tables in cache diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java index a988769b58ee..5734df27bd79 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java @@ -374,6 +374,25 @@ public List getTables(String databaseName) } } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + try { + return getGlueTables(databaseName) + .filter(tableFilter) + .filter(table -> parameterValues.contains(getTableParameters(table).get(parameterKey))) + .map(com.amazonaws.services.glue.model.Table::getName) + .collect(toImmutableList()); + } + catch (EntityNotFoundException | AccessDeniedException e) { + // database does not exist or permission denied + return ImmutableList.of(); + } + catch (AmazonServiceException e) { + throw new TrinoException(HIVE_METASTORE_ERROR, e); + } + } + @Override public Optional
getTable(String databaseName, String tableName) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java index 826a6caf0c31..7d1efdafe889 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import io.trino.hive.thrift.metastore.FieldSchema; import io.trino.metastore.AcidOperation; import io.trino.metastore.AcidTransactionOwner; @@ -152,6 +153,12 @@ public List getTables(String databaseName) .collect(toImmutableList()); } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + return delegate.getTableNamesWithParameters(databaseName, parameterKey, parameterValues); + } + @Override public void createDatabase(Database database) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/DefaultThriftMetastoreClientFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/DefaultThriftMetastoreClientFactory.java index ec0d33783f82..a01a87b9e69f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/DefaultThriftMetastoreClientFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/DefaultThriftMetastoreClientFactory.java @@ -49,6 +49,7 @@ public class DefaultThriftMetastoreClientFactory private final MetastoreSupportsDateStatistics metastoreSupportsDateStatistics = new MetastoreSupportsDateStatistics(); private final AtomicInteger chosenGetTableAlternative = new AtomicInteger(Integer.MAX_VALUE); + private final AtomicInteger chosenTableParamAlternative = new AtomicInteger(Integer.MAX_VALUE); private final AtomicInteger chosenAlterTransactionalTableAlternative = new AtomicInteger(Integer.MAX_VALUE); private final AtomicInteger chosenAlterPartitionsAlternative = new AtomicInteger(Integer.MAX_VALUE); @@ -115,6 +116,7 @@ protected ThriftMetastoreClient create(TransportSupplier transportSupplier, Stri metastoreSupportsDateStatistics, true, chosenGetTableAlternative, + chosenTableParamAlternative, chosenAlterTransactionalTableAlternative, chosenAlterPartitionsAlternative); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/FailureAwareThriftMetastoreClient.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/FailureAwareThriftMetastoreClient.java index ed57071e3389..0d9ee63183ab 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/FailureAwareThriftMetastoreClient.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/FailureAwareThriftMetastoreClient.java @@ -37,6 +37,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import static java.util.Objects.requireNonNull; @@ -92,6 +93,13 @@ public List getTableMeta(String databaseName) return runWithHandle(() -> delegate.getTableMeta(databaseName)); } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, Set parameterValues) + throws TException + { + return runWithHandle(() -> delegate.getTableNamesWithParameters(databaseName, parameterKey, parameterValues)); + } + @Override public void createDatabase(Database database) throws TException diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/HttpThriftMetastoreClientFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/HttpThriftMetastoreClientFactory.java index 52b0df41ca1b..ddd696fa9d54 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/HttpThriftMetastoreClientFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/HttpThriftMetastoreClientFactory.java @@ -57,6 +57,7 @@ public class HttpThriftMetastoreClientFactory private final OpenTelemetry openTelemetry; private final AtomicInteger chosenGetTableAlternative = new AtomicInteger(Integer.MAX_VALUE); + private final AtomicInteger chosenGetTableParamAlternative = new AtomicInteger(Integer.MAX_VALUE); private final AtomicInteger chosenAlterTransactionalTableAlternative = new AtomicInteger(Integer.MAX_VALUE); private final AtomicInteger chosenAlterPartitionsAlternative = new AtomicInteger(Integer.MAX_VALUE); @@ -85,6 +86,7 @@ public ThriftMetastoreClient create(URI uri, Optional delegationToken) new MetastoreSupportsDateStatistics(), false, chosenGetTableAlternative, + chosenGetTableParamAlternative, chosenAlterTransactionalTableAlternative, chosenAlterPartitionsAlternative); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java index a7908d8e7de0..7b07889050cd 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java @@ -268,6 +268,30 @@ public List getTables(String databaseName) } } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, Set parameterValues) + { + try { + return retry() + .stopOn(NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getTableNamesWithParameters", () -> { + try (ThriftMetastoreClient client = createMetastoreClient()) { + return client.getTableNamesWithParameters(databaseName, parameterKey, parameterValues); + } + }); + } + catch (NoSuchObjectException e) { + return ImmutableList.of(); + } + catch (TException e) { + throw new TrinoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + throw propagate(e); + } + } + @Override public Optional
getTable(String databaseName, String tableName) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java index acf253025ef5..97c7a5fb3ead 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java @@ -78,8 +78,10 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; +import java.util.regex.Pattern; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Throwables.throwIfInstanceOf; @@ -90,6 +92,7 @@ import static com.google.common.reflect.Reflection.newProxy; import static io.trino.hive.thrift.metastore.GrantRevokeType.GRANT; import static io.trino.hive.thrift.metastore.GrantRevokeType.REVOKE; +import static io.trino.hive.thrift.metastore.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS; import static io.trino.metastore.TableInfo.PRESTO_VIEW_COMMENT; import static io.trino.plugin.hive.TableType.VIRTUAL_VIEW; import static io.trino.plugin.hive.metastore.thrift.MetastoreSupportsDateStatistics.DateStatisticsSupport.NOT_SUPPORTED; @@ -99,6 +102,7 @@ import static io.trino.plugin.hive.metastore.thrift.TxnUtils.createValidTxnWriteIdList; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; import static org.apache.thrift.TApplicationException.UNKNOWN_METHOD; public class ThriftHiveMetastoreClient @@ -109,6 +113,9 @@ public class ThriftHiveMetastoreClient private static final String CATALOG_DB_SEPARATOR = "#"; private static final String DB_EMPTY_MARKER = "!"; + private static final Pattern TABLE_PARAMETER_SAFE_KEY_PATTERN = Pattern.compile("^[a-zA-Z_]+$"); + private static final Pattern TABLE_PARAMETER_SAFE_VALUE_PATTERN = Pattern.compile("^[a-zA-Z0-9\\s]*$"); + private final TransportSupplier transportSupplier; private TTransport transport; protected ThriftHiveMetastore.Iface client; @@ -117,6 +124,7 @@ public class ThriftHiveMetastoreClient private final MetastoreSupportsDateStatistics metastoreSupportsDateStatistics; private final boolean metastoreSupportsTableMeta; private final AtomicInteger chosenGetTableAlternative; + private final AtomicInteger chosenTableParamAlternative; private final AtomicInteger chosenAlterTransactionalTableAlternative; private final AtomicInteger chosenAlterPartitionsAlternative; private final Optional catalogName; @@ -128,6 +136,7 @@ public ThriftHiveMetastoreClient( MetastoreSupportsDateStatistics metastoreSupportsDateStatistics, boolean metastoreSupportsTableMeta, AtomicInteger chosenGetTableAlternative, + AtomicInteger chosenTableParamAlternative, AtomicInteger chosenAlterTransactionalTableAlternative, AtomicInteger chosenAlterPartitionsAlternative) throws TTransportException @@ -137,6 +146,7 @@ public ThriftHiveMetastoreClient( this.metastoreSupportsDateStatistics = requireNonNull(metastoreSupportsDateStatistics, "metastoreSupportsDateStatistics is null"); this.metastoreSupportsTableMeta = metastoreSupportsTableMeta; this.chosenGetTableAlternative = requireNonNull(chosenGetTableAlternative, "chosenGetTableAlternative is null"); + this.chosenTableParamAlternative = requireNonNull(chosenTableParamAlternative, "chosenTableParamAlternative is null"); this.chosenAlterTransactionalTableAlternative = requireNonNull(chosenAlterTransactionalTableAlternative, "chosenAlterTransactionalTableAlternative is null"); this.chosenAlterPartitionsAlternative = requireNonNull(chosenAlterPartitionsAlternative, "chosenAlterPartitionsAlternative is null"); this.catalogName = requireNonNull(catalogName, "catalogName is null"); @@ -210,6 +220,41 @@ public List getTableMeta(String databaseName) return client.getTableMeta(prependCatalogToDbName(catalogName, databaseName), "*", ImmutableList.of()); } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, Set parameterValues) + throws TException + { + checkArgument(TABLE_PARAMETER_SAFE_KEY_PATTERN.matcher(parameterKey).matches(), "Parameter key contains invalid characters: '%s'", parameterKey); + /* + * The parameter value is restricted to have only alphanumeric characters so that it's safe + * to be used against HMS. When using with a LIKE operator, the HMS may want the parameter + * value to follow a Java regex pattern or an SQL pattern. And it's hard to predict the + * HMS's behavior from outside. Also, by restricting parameter values, we avoid the problem + * of how to quote them when passing within the filter string. + */ + for (String parameterValue : parameterValues) { + checkArgument(TABLE_PARAMETER_SAFE_VALUE_PATTERN.matcher(parameterValue).matches(), "Parameter value contains invalid characters: '%s'", parameterValue); + } + /* + * Thrift call `get_table_names_by_filter` may be translated by Metastore to an SQL query against Metastore database. + * Hive 2.3 on some databases uses CLOB for table parameter value column and some databases disallow `=` predicate over + * CLOB values. At the same time, they allow `LIKE` predicates over them. + */ + String filterWithEquals = parameterValues.stream() + .map(parameterValue -> HIVE_FILTER_FIELD_PARAMS + parameterKey + " = \"" + parameterValue + "\"") + .collect(joining(" or ")); + + String filterWithLike = parameterValues.stream() + .map(parameterValue -> HIVE_FILTER_FIELD_PARAMS + parameterKey + " LIKE \"" + parameterValue + "\"") + .collect(joining(" or ")); + + return alternativeCall( + ThriftHiveMetastoreClient::defaultIsValidExceptionalResponse, + chosenTableParamAlternative, + () -> client.getTableNamesByFilter(databaseName, filterWithEquals, (short) -1), + () -> client.getTableNamesByFilter(databaseName, filterWithLike, (short) -1)); + } + @Override public void createDatabase(Database database) throws TException diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastore.java index ebcdbf5359d4..6f9cf7a04869 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastore.java @@ -65,6 +65,8 @@ public sealed interface ThriftMetastore List getTables(String databaseName); + List getTableNamesWithParameters(String databaseName, String parameterKey, Set parameterValues); + Optional getDatabase(String databaseName); void addPartitions(String databaseName, String tableName, List partitions); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreClient.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreClient.java index b9c39b54a333..7fe9ef73f04f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreClient.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreClient.java @@ -37,6 +37,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; public interface ThriftMetastoreClient extends Closeable @@ -53,6 +54,9 @@ Database getDatabase(String databaseName) List getTableMeta(String databaseName) throws TException; + List getTableNamesWithParameters(String databaseName, String parameterKey, Set parameterValues) + throws TException; + void createDatabase(Database database) throws TException; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMetadataListing.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMetadataListing.java index bb96fefa3cad..af734b91ec22 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMetadataListing.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMetadataListing.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import io.trino.metastore.Column; import io.trino.metastore.Database; import io.trino.metastore.HiveBucketProperty; @@ -244,6 +245,12 @@ public List getTables(String databaseName) .build(); } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, ImmutableSet parameterValues) + { + throw new UnsupportedOperationException(); + } + @Override public Optional
getTable(String databaseName, String tableName) { diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/MockThriftMetastoreClient.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/MockThriftMetastoreClient.java index 5503de707e43..080f685864d4 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/MockThriftMetastoreClient.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/MockThriftMetastoreClient.java @@ -49,6 +49,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -164,6 +165,12 @@ public List getTableMeta(String databaseName) return ImmutableList.of(new TableMeta(TEST_DATABASE, TEST_TABLE, MANAGED_TABLE.name())); } + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, Set parameterValues) + { + throw new UnsupportedOperationException(); + } + @Override public Database getDatabase(String name) throws TException diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestThriftHiveMetastoreClient.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestThriftHiveMetastoreClient.java index c469374be025..6faa578b65ab 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestThriftHiveMetastoreClient.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestThriftHiveMetastoreClient.java @@ -50,6 +50,7 @@ public void testAlternativeCall() true, new AtomicInteger(), new AtomicInteger(), + new AtomicInteger(), new AtomicInteger()); assertThat(connectionCount.get()).isEqualTo(1); From bc385e70622d0a54837bcd8253c7770c13f8519d Mon Sep 17 00:00:00 2001 From: lukasz-stec Date: Fri, 13 Dec 2024 11:22:11 +0100 Subject: [PATCH 056/158] Add TrinoCatalog.listIcebergTables --- .../plugin/iceberg/catalog/TrinoCatalog.java | 2 ++ .../catalog/glue/TrinoGlueCatalog.java | 17 ++++++++++++++ .../iceberg/catalog/hms/TrinoHiveCatalog.java | 23 +++++++++++++++++++ .../catalog/jdbc/TrinoJdbcCatalog.java | 21 +++++++++++++++++ .../catalog/nessie/TrinoNessieCatalog.java | 8 +++++++ .../catalog/rest/TrinoRestCatalog.java | 15 ++++++++++++ .../snowflake/TrinoSnowflakeCatalog.java | 9 ++++++++ .../iceberg/catalog/BaseTrinoCatalogTest.java | 7 ++++++ 8 files changed, 102 insertions(+) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java index 005760040b1b..c7370d65d810 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java @@ -83,6 +83,8 @@ default Optional getNamespaceSeparator() List listTables(ConnectorSession session, Optional namespace); + List listIcebergTables(ConnectorSession session, Optional namespace); + default List listViews(ConnectorSession session, Optional namespace) { return listTables(session, namespace).stream() diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index 7b5e4cb67d1b..6c8a3dfa44bb 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -373,9 +373,26 @@ public void renameNamespace(ConnectorSession session, String source, String targ @Override public List listTables(ConnectorSession session, Optional namespace) + { + return listTables(session, namespace, _ -> true); + } + + @Override + public List listIcebergTables(ConnectorSession session, Optional namespace) + { + return listTables(session, namespace, table -> isIcebergTable(getTableParameters(table))).stream() + .map(TableInfo::tableName) + .collect(toImmutableList()); + } + + private List listTables( + ConnectorSession session, + Optional namespace, + Predicate tablePredicate) { List>> tasks = listNamespaces(session, namespace).stream() .map(glueNamespace -> (Callable>) () -> getGlueTablesWithExceptionHandling(glueNamespace) + .filter(tablePredicate) .map(table -> mapToTableInfo(glueNamespace, table)) .collect(toImmutableList())) .collect(toImmutableList()); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java index b450063dba6f..908d7cc6dde9 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java @@ -16,6 +16,7 @@ import com.google.common.cache.Cache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.UncheckedExecutionException; import io.airlift.log.Logger; import io.trino.cache.EvictableCacheBuilder; @@ -375,6 +376,28 @@ public List listTables(ConnectorSession session, Optional nam } } + @Override + public List listIcebergTables(ConnectorSession session, Optional namespace) + { + List>> tasks = listNamespaces(session, namespace).stream() + .map(schema -> (Callable>) () -> metastore.getTableNamesWithParameters(schema, TABLE_TYPE_PROP, ImmutableSet.of( + // Get tables with parameter table_type set to "ICEBERG" or "iceberg". This is required because + // Trino uses lowercase value whereas Spark and Flink use uppercase. + ICEBERG_TABLE_TYPE_VALUE.toLowerCase(ENGLISH), + ICEBERG_TABLE_TYPE_VALUE.toUpperCase(ENGLISH))).stream() + .map(tableName -> new SchemaTableName(schema, tableName)) + .collect(toImmutableList())) + .collect(toImmutableList()); + try { + return processWithAdditionalThreads(tasks, metadataFetchingExecutor).stream() + .flatMap(Collection::stream) + .collect(toImmutableList()); + } + catch (ExecutionException e) { + throw new RuntimeException(e.getCause()); + } + } + @Override public Optional> streamRelationColumns( ConnectorSession session, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalog.java index 1d8daf1effd1..0e5c401c8ba5 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalog.java @@ -60,6 +60,7 @@ import org.apache.iceberg.view.ViewVersion; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -200,6 +201,26 @@ public List listTables(ConnectorSession session, Optional nam return ImmutableList.copyOf(tablesListBuilder.values()); } + @Override + public List listIcebergTables(ConnectorSession session, Optional namespace) + { + List namespaces = listNamespaces(session, namespace); + + // Build as a set and convert to list for removing duplicate entries due to case difference + Set tablesListBuilder = new HashSet<>(); + for (String schemaName : namespaces) { + try { + listTableIdentifiers(schemaName, () -> jdbcCatalog.listTables(Namespace.of(schemaName))).stream() + .map(tableId -> SchemaTableName.schemaTableName(schemaName, tableId.name())) + .forEach(tablesListBuilder::add); + } + catch (NoSuchNamespaceException e) { + // Namespace may have been deleted + } + } + return ImmutableList.copyOf(tablesListBuilder); + } + @Override public List listViews(ConnectorSession session, Optional namespace) { diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java index 240e41f4cdd1..ed7be861fd9f 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java @@ -170,6 +170,14 @@ public List listTables(ConnectorSession session, Optional nam .collect(toImmutableList()); } + @Override + public List listIcebergTables(ConnectorSession session, Optional namespace) + { + return listTables(session, namespace).stream() + .map(TableInfo::tableName) + .collect(toImmutableList()); + } + @Override public Optional> streamRelationColumns( ConnectorSession session, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java index 40037baed704..d72f75d3f334 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java @@ -261,6 +261,21 @@ public List listTables(ConnectorSession session, Optional nam return tables.build(); } + @Override + public List listIcebergTables(ConnectorSession session, Optional namespace) + { + SessionContext sessionContext = convert(session); + List namespaces = listNamespaces(session, namespace); + + ImmutableList.Builder tables = ImmutableList.builder(); + for (Namespace restNamespace : namespaces) { + listTableIdentifiers(restNamespace, () -> restSessionCatalog.listTables(sessionContext, toRemoteNamespace(session, restNamespace))).stream() + .map(id -> SchemaTableName.schemaTableName(toSchemaName(id.namespace()), id.name())) + .forEach(tables::add); + } + return tables.build(); + } + @Override public List listViews(ConnectorSession session, Optional namespace) { diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/snowflake/TrinoSnowflakeCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/snowflake/TrinoSnowflakeCatalog.java index fb901d4890b0..2f58396e6c69 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/snowflake/TrinoSnowflakeCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/snowflake/TrinoSnowflakeCatalog.java @@ -58,6 +58,7 @@ import java.util.stream.Stream; import static com.google.common.base.Throwables.throwIfUnchecked; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.cache.CacheUtils.uncheckedCacheGet; import static io.trino.plugin.iceberg.IcebergUtil.getIcebergTableWithMetadata; import static io.trino.plugin.iceberg.IcebergUtil.quotedTableName; @@ -171,6 +172,14 @@ public List listTables(ConnectorSession session, Optional nam .toList(); } + @Override + public List listIcebergTables(ConnectorSession session, Optional namespace) + { + return listTables(session, namespace).stream() + .map(TableInfo::tableName) + .collect(toImmutableList()); + } + @Override public Optional> streamRelationColumns( ConnectorSession session, diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java index 1cf15d256401..e2ae7af4c66b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java @@ -450,6 +450,9 @@ public void testListTables() .add(new TableInfo(table1, TABLE)) .add(new TableInfo(table2, TABLE)); + ImmutableList.Builder icebergTables = ImmutableList.builder() + .add(table1) + .add(table2); SchemaTableName view = new SchemaTableName(ns2, "view"); try { catalog.createView( @@ -493,6 +496,7 @@ public void testListTables() createExternalIcebergTable(catalog, ns2, closer).ifPresent(table -> { allTables.add(new TableInfo(table, TABLE)); + icebergTables.add(table); }); createExternalNonIcebergTable(catalog, ns2, closer).ifPresent(table -> { allTables.add(new TableInfo(table, TABLE)); @@ -500,10 +504,13 @@ public void testListTables() // No namespace provided, all tables across all namespaces should be returned assertThat(catalog.listTables(SESSION, Optional.empty())).containsAll(allTables.build()); + assertThat(catalog.listIcebergTables(SESSION, Optional.empty())).containsAll(icebergTables.build()); // Namespace is provided and exists assertThat(catalog.listTables(SESSION, Optional.of(ns1))).containsExactly(new TableInfo(table1, TABLE)); + assertThat(catalog.listIcebergTables(SESSION, Optional.of(ns1))).containsExactly(table1); // Namespace is provided and does not exist assertThat(catalog.listTables(SESSION, Optional.of("non_existing"))).isEmpty(); + assertThat(catalog.listIcebergTables(SESSION, Optional.of("non_existing"))).isEmpty(); } } From 5ce80becb339a75c05e332ca5aa47cd63672a7ce Mon Sep 17 00:00:00 2001 From: lukasz-stec Date: Fri, 13 Dec 2024 11:29:00 +0100 Subject: [PATCH 057/158] Add system.iceberg_tables table function Add the ability to list only iceberg tables from the iceberg catalog. Before this change, there was no way to list only iceberg tables. The SHOW TABLES statement, information_schema.tables, and jdbc.tables will all return all tables that exist in the underlying metastore, even if the table cannot be handled in any way by the iceberg connector. This can happen if other connectors like hive or delta, use the same metastore, catalog, and schema to store its tables. The function accepts an optional parameter with the schema name. Sample statements: SELECT * FROM TABLE(iceberg.system.iceberg_tables()); SELECT * FROM TABLE(iceberg.system.iceberg_tables(SCHEMA_NAME => 'test')); --- .../trino/plugin/iceberg/IcebergModule.java | 5 +- .../plugin/iceberg/IcebergSplitManager.java | 4 + .../functions/IcebergFunctionProvider.java | 30 +++ .../tables/IcebergTablesFunction.java | 129 +++++++++++++ .../tables/IcebergTablesFunctionProvider.java | 42 ++++ .../BaseIcebergConnectorSmokeTest.java | 52 +++++ .../BaseIcebergMinioConnectorSmokeTest.java | 30 +++ .../iceberg/BaseSharedMetastoreTest.java | 7 + .../TestSharedHiveThriftMetastore.java | 181 ++++++++++++++++++ ...ergUnityRestCatalogConnectorSmokeTest.java | 12 +- 10 files changed, 489 insertions(+), 3 deletions(-) create mode 100644 plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunction.java create mode 100644 plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunctionProvider.java create mode 100644 plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestSharedHiveThriftMetastore.java diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index 0992e51d2375..386f7dd221b9 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -42,6 +42,7 @@ import io.trino.plugin.iceberg.functions.IcebergFunctionProvider; import io.trino.plugin.iceberg.functions.tablechanges.TableChangesFunctionProcessorProviderFactory; import io.trino.plugin.iceberg.functions.tablechanges.TableChangesFunctionProvider; +import io.trino.plugin.iceberg.functions.tables.IcebergTablesFunctionProvider; import io.trino.plugin.iceberg.procedure.AddFilesTableFromTableProcedure; import io.trino.plugin.iceberg.procedure.AddFilesTableProcedure; import io.trino.plugin.iceberg.procedure.DropExtendedStatsTableProcedure; @@ -135,7 +136,9 @@ public void configure(Binder binder) tableProcedures.addBinding().toProvider(AddFilesTableProcedure.class).in(Scopes.SINGLETON); tableProcedures.addBinding().toProvider(AddFilesTableFromTableProcedure.class).in(Scopes.SINGLETON); - newSetBinder(binder, ConnectorTableFunction.class).addBinding().toProvider(TableChangesFunctionProvider.class).in(Scopes.SINGLETON); + Multibinder tableFunctions = newSetBinder(binder, ConnectorTableFunction.class); + tableFunctions.addBinding().toProvider(TableChangesFunctionProvider.class).in(Scopes.SINGLETON); + tableFunctions.addBinding().toProvider(IcebergTablesFunctionProvider.class).in(Scopes.SINGLETON); binder.bind(FunctionProvider.class).to(IcebergFunctionProvider.class).in(Scopes.SINGLETON); binder.bind(TableChangesFunctionProcessorProviderFactory.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitManager.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitManager.java index f23983a4a2c3..0cb4d8d849f3 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitManager.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSplitManager.java @@ -21,6 +21,7 @@ import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitSource; import io.trino.plugin.iceberg.functions.tablechanges.TableChangesFunctionHandle; import io.trino.plugin.iceberg.functions.tablechanges.TableChangesSplitSource; +import io.trino.plugin.iceberg.functions.tables.IcebergTablesFunction.IcebergTables; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorSplitManager; import io.trino.spi.connector.ConnectorSplitSource; @@ -157,6 +158,9 @@ public ConnectorSplitSource getSplits( .toSnapshot(functionHandle.endSnapshotId())); return new ClassLoaderSafeConnectorSplitSource(tableChangesSplitSource, IcebergSplitManager.class.getClassLoader()); } + if (function instanceof IcebergTables icebergTables) { + return new ClassLoaderSafeConnectorSplitSource(new FixedSplitSource(icebergTables), IcebergSplitManager.class.getClassLoader()); + } throw new IllegalStateException("Unknown table function: " + function); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/IcebergFunctionProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/IcebergFunctionProvider.java index 07326a42bcef..b23b543a3cbf 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/IcebergFunctionProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/IcebergFunctionProvider.java @@ -14,12 +14,20 @@ package io.trino.plugin.iceberg.functions; import com.google.inject.Inject; +import io.trino.plugin.base.classloader.ClassLoaderSafeTableFunctionProcessorProvider; import io.trino.plugin.base.classloader.ClassLoaderSafeTableFunctionProcessorProviderFactory; +import io.trino.plugin.base.classloader.ClassLoaderSafeTableFunctionSplitProcessor; import io.trino.plugin.iceberg.functions.tablechanges.TableChangesFunctionHandle; import io.trino.plugin.iceberg.functions.tablechanges.TableChangesFunctionProcessorProviderFactory; +import io.trino.plugin.iceberg.functions.tables.IcebergTablesFunction; +import io.trino.spi.classloader.ThreadContextClassLoader; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplit; import io.trino.spi.function.FunctionProvider; import io.trino.spi.function.table.ConnectorTableFunctionHandle; +import io.trino.spi.function.table.TableFunctionProcessorProvider; import io.trino.spi.function.table.TableFunctionProcessorProviderFactory; +import io.trino.spi.function.table.TableFunctionSplitProcessor; import static java.util.Objects.requireNonNull; @@ -40,6 +48,28 @@ public TableFunctionProcessorProviderFactory getTableFunctionProcessorProviderFa if (functionHandle instanceof TableChangesFunctionHandle) { return new ClassLoaderSafeTableFunctionProcessorProviderFactory(tableChangesFunctionProcessorProviderFactory, getClass().getClassLoader()); } + if (functionHandle instanceof IcebergTablesFunction.IcebergTables) { + ClassLoader classLoader = getClass().getClassLoader(); + return new TableFunctionProcessorProviderFactory() + { + @Override + public TableFunctionProcessorProvider createTableFunctionProcessorProvider() + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return new ClassLoaderSafeTableFunctionProcessorProvider(new TableFunctionProcessorProvider() + { + @Override + public TableFunctionSplitProcessor getSplitProcessor(ConnectorSession session, ConnectorTableFunctionHandle handle, ConnectorSplit split) + { + return new ClassLoaderSafeTableFunctionSplitProcessor( + new IcebergTablesFunction.IcebergTablesProcessor(((IcebergTablesFunction.IcebergTables) split).tables()), + getClass().getClassLoader()); + } + }, classLoader); + } + } + }; + } throw new UnsupportedOperationException("Unsupported function: " + functionHandle); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunction.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunction.java new file mode 100644 index 000000000000..345c527a6ecf --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunction.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.functions.tables; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.trino.plugin.iceberg.catalog.TrinoCatalog; +import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; +import io.trino.spi.Page; +import io.trino.spi.block.VariableWidthBlockBuilder; +import io.trino.spi.connector.ConnectorAccessControl; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.function.table.AbstractConnectorTableFunction; +import io.trino.spi.function.table.Argument; +import io.trino.spi.function.table.ConnectorTableFunctionHandle; +import io.trino.spi.function.table.Descriptor; +import io.trino.spi.function.table.ScalarArgument; +import io.trino.spi.function.table.ScalarArgumentSpecification; +import io.trino.spi.function.table.TableFunctionAnalysis; +import io.trino.spi.function.table.TableFunctionProcessorState; +import io.trino.spi.function.table.TableFunctionSplitProcessor; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.trino.spi.function.table.ReturnTypeSpecification.GenericTable.GENERIC_TABLE; +import static io.trino.spi.function.table.TableFunctionProcessorState.Finished.FINISHED; +import static io.trino.spi.function.table.TableFunctionProcessorState.Processed.produced; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static java.util.Objects.requireNonNull; + +public class IcebergTablesFunction + extends AbstractConnectorTableFunction +{ + private static final String FUNCTION_NAME = "iceberg_tables"; + private static final String SCHEMA_NAME_VAR_NAME = "SCHEMA_NAME"; + + private final TrinoCatalogFactory trinoCatalogFactory; + + public IcebergTablesFunction(TrinoCatalogFactory trinoCatalogFactory) + { + super( + "system", + FUNCTION_NAME, + ImmutableList.of( + ScalarArgumentSpecification.builder() + .name(SCHEMA_NAME_VAR_NAME) + .type(VARCHAR) + .defaultValue(null) + .build()), + GENERIC_TABLE); + this.trinoCatalogFactory = requireNonNull(trinoCatalogFactory, "trinoCatalogFactory is null"); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments, ConnectorAccessControl accessControl) + { + ScalarArgument argument = (ScalarArgument) getOnlyElement(arguments.values()); + Optional schemaFilter = Optional.ofNullable(((Slice) argument.getValue())).map(Slice::toStringUtf8); + + TrinoCatalog catalog = trinoCatalogFactory.create(session.getIdentity()); + List tables = catalog.listIcebergTables(session, schemaFilter); + Set filtered = accessControl.filterTables(null, ImmutableSet.copyOf(tables)); + return TableFunctionAnalysis.builder() + .returnedType(new Descriptor(ImmutableList.of( + new Descriptor.Field("table_schema", Optional.of(VARCHAR)), + new Descriptor.Field("table_name", Optional.of(VARCHAR))))) + .handle(new IcebergTables(filtered)) + .build(); + } + + public record IcebergTables(Collection tables) + implements ConnectorTableFunctionHandle, ConnectorSplit + { + public IcebergTables + { + requireNonNull(tables, "tables is null"); + } + } + + public static class IcebergTablesProcessor + implements TableFunctionSplitProcessor + { + private final Collection tables; + private boolean finished; + + public IcebergTablesProcessor(Collection tables) + { + this.tables = requireNonNull(tables, "tables is null"); + } + + @Override + public TableFunctionProcessorState process() + { + if (finished) { + return FINISHED; + } + + VariableWidthBlockBuilder schema = VARCHAR.createBlockBuilder(null, tables.size()); + VariableWidthBlockBuilder tableName = VARCHAR.createBlockBuilder(null, tables.size()); + for (SchemaTableName table : tables) { + schema.writeEntry(Slices.utf8Slice(table.getSchemaName())); + tableName.writeEntry(Slices.utf8Slice(table.getTableName())); + } + finished = true; + return produced(new Page(tables.size(), schema.build(), tableName.build())); + } + } +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunctionProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunctionProvider.java new file mode 100644 index 000000000000..a0d281423f04 --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/functions/tables/IcebergTablesFunctionProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.functions.tables; + +import com.google.inject.Inject; +import com.google.inject.Provider; +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorTableFunction; +import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; +import io.trino.spi.function.table.ConnectorTableFunction; + +import static java.util.Objects.requireNonNull; + +public class IcebergTablesFunctionProvider + implements Provider +{ + private final TrinoCatalogFactory trinoCatalogFactory; + + @Inject + public IcebergTablesFunctionProvider(TrinoCatalogFactory trinoCatalogFactory) + { + this.trinoCatalogFactory = requireNonNull(trinoCatalogFactory, "trinoCatalogFactory is null"); + } + + @Override + public ConnectorTableFunction get() + { + return new ClassLoaderSafeConnectorTableFunction( + new IcebergTablesFunction(trinoCatalogFactory), + getClass().getClassLoader()); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java index c9fef25627a5..745894d2d2ff 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java @@ -22,6 +22,7 @@ import io.trino.filesystem.TrinoFileSystem; import io.trino.plugin.iceberg.fileio.ForwardingFileIo; import io.trino.testing.BaseConnectorSmokeTest; +import io.trino.testing.QueryRunner; import io.trino.testing.TestingConnectorBehavior; import io.trino.testing.sql.TestTable; import org.apache.iceberg.FileFormat; @@ -61,6 +62,7 @@ import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.joining; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -835,6 +837,56 @@ public void testCreateOrReplaceWithTableChangesFunction() } } + @Test + public void testIcebergTablesFunction() + throws Exception + { + String schemaName = getSession().getSchema().orElseThrow(); + String firstSchema = "first_schema_" + randomNameSuffix(); + String secondSchema = "second_schema_" + randomNameSuffix(); + String firstSchemaLocation = schemaPath().replaceAll(schemaName, firstSchema); + String secondSchemaLocation = schemaPath().replaceAll(schemaName, secondSchema); + assertQuerySucceeds("CREATE SCHEMA " + firstSchema + " WITH (location = '%s')".formatted(firstSchemaLocation)); + assertQuerySucceeds("CREATE SCHEMA " + secondSchema + " WITH (location = '%s')".formatted(secondSchemaLocation)); + QueryRunner queryRunner = getQueryRunner(); + Session firstSchemaSession = Session.builder(queryRunner.getDefaultSession()).setSchema(firstSchema).build(); + Session secondSchemaSession = Session.builder(queryRunner.getDefaultSession()).setSchema(secondSchema).build(); + + try (TestTable _ = new TestTable( + sql -> getQueryRunner().execute(firstSchemaSession, sql), + "first_schema_table1_", + "(id int)"); + TestTable _ = new TestTable( + sql -> getQueryRunner().execute(firstSchemaSession, sql), + "first_schema_table2_", + "(id int)"); + TestTable secondSchemaTable = new TestTable( + sql -> queryRunner.execute(secondSchemaSession, sql), + "second_schema_table_", + "(id int)"); + AutoCloseable _ = createAdditionalTables(firstSchema)) { + String firstSchemaTablesValues = "VALUES " + getQueryRunner() + .execute("SELECT table_schema, table_name FROM iceberg.information_schema.tables WHERE table_schema='%s'".formatted(firstSchema)) + .getMaterializedRows().stream() + .map(row -> "('%s', '%s')".formatted(row.getField(0), row.getField(1))) + .collect(joining(", ")); + String bothSchemasTablesValues = firstSchemaTablesValues + ", ('%s', '%s')".formatted(secondSchema, secondSchemaTable.getName()); + assertQuery("SELECT * FROM TABLE(iceberg.system.iceberg_tables(SCHEMA_NAME => '%s'))".formatted(firstSchema), firstSchemaTablesValues); + assertQuery("SELECT * FROM TABLE(iceberg.system.iceberg_tables(null)) WHERE table_schema = '%s'".formatted(firstSchema), firstSchemaTablesValues); + assertQuery("SELECT * FROM TABLE(iceberg.system.iceberg_tables()) WHERE table_schema in ('%s', '%s')".formatted(firstSchema, secondSchema), bothSchemasTablesValues); + assertQuery("SELECT * FROM TABLE(iceberg.system.iceberg_tables(null)) WHERE table_schema in ('%s', '%s')".formatted(firstSchema, secondSchema), bothSchemasTablesValues); + } + finally { + assertQuerySucceeds("DROP SCHEMA " + firstSchema); + assertQuerySucceeds("DROP SCHEMA " + secondSchema); + } + } + + protected AutoCloseable createAdditionalTables(String schema) + { + return () -> {}; + } + private long getMostRecentSnapshotId(String tableName) { return (long) Iterables.getOnlyElement(getQueryRunner().execute(format("SELECT snapshot_id FROM \"%s$snapshots\" ORDER BY committed_at DESC LIMIT 1", tableName)) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java index 3144bd56bb02..91474fa6fa8c 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java @@ -13,10 +13,14 @@ */ package io.trino.plugin.iceberg; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.minio.messages.Event; import io.trino.Session; +import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveType; +import io.trino.metastore.Table; import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.testing.QueryRunner; @@ -28,18 +32,25 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.trino.metastore.PrincipalPrivileges.NO_PRIVILEGES; +import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder; +import static io.trino.plugin.iceberg.IcebergTestUtils.getHiveMetastore; +import static io.trino.plugin.iceberg.catalog.AbstractIcebergTableOperations.ICEBERG_METASTORE_STORAGE_FORMAT; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; import static io.trino.testing.containers.Minio.MINIO_REGION; import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; import static java.lang.String.format; import static java.util.Locale.ENGLISH; +import static org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE; +import static org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; @@ -233,6 +244,25 @@ public void testPathContainsSpecialCharacter() assertUpdate("DROP TABLE " + tableName); } + @Override + protected AutoCloseable createAdditionalTables(String schema) + { + HiveMetastore metastore = getHiveMetastore(getQueryRunner()); + // simulate iceberg table created by spark with lowercase table type + Table lowerCaseTableType = io.trino.metastore.Table.builder() + .setDatabaseName(schema) + .setTableName("lowercase_type_" + randomNameSuffix()) + .setOwner(Optional.empty()) + .setDataColumns(ImmutableList.of(new Column("id", HiveType.HIVE_STRING, Optional.empty(), ImmutableMap.of()))) + .setTableType(EXTERNAL_TABLE.name()) + .withStorage(storage -> storage.setStorageFormat(ICEBERG_METASTORE_STORAGE_FORMAT)) + .setParameter("EXTERNAL", "TRUE") + .setParameter(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toLowerCase(ENGLISH)) + .build(); + metastore.createTable(lowerCaseTableType, NO_PRIVILEGES); + return () -> metastore.dropTable(lowerCaseTableType.getDatabaseName(), lowerCaseTableType.getTableName(), true); + } + private String onMetastore(@Language("SQL") String sql) { return hiveMinioDataLake.getHiveHadoop().runOnMetastore(sql); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseSharedMetastoreTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseSharedMetastoreTest.java index d059623c8294..b755bcea9859 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseSharedMetastoreTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseSharedMetastoreTest.java @@ -152,6 +152,13 @@ public void testShowSchemas() assertThat(showCreateIcebergWithRedirectionsSchema).isEqualTo(getExpectedIcebergCreateSchema("iceberg_with_redirections")); } + @Test + public void testIcebergTablesFunction() + { + assertQuery("SELECT * FROM TABLE(iceberg.system.iceberg_tables(SCHEMA_NAME => '%s'))".formatted(tpchSchema), "VALUES ('%s', 'nation')".formatted(tpchSchema)); + assertQuery("SELECT * FROM TABLE(iceberg_with_redirections.system.iceberg_tables(SCHEMA_NAME => '%s'))".formatted(tpchSchema), "VALUES ('%s', 'nation')".formatted(tpchSchema)); + } + @Test public void testTimeTravelWithRedirection() throws InterruptedException diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestSharedHiveThriftMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestSharedHiveThriftMetastore.java new file mode 100644 index 000000000000..6228db8ed7c6 --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestSharedHiveThriftMetastore.java @@ -0,0 +1,181 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.Session; +import io.trino.plugin.hive.TestingHivePlugin; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; +import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.tpch.TpchPlugin; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import io.trino.tpch.TpchTable; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; + +import java.nio.file.Path; +import java.util.Map; + +import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; +import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.trino.testing.QueryAssertions.copyTpchTables; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; +import static io.trino.testing.containers.Minio.MINIO_REGION; +import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; +import static java.lang.String.format; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; + +@TestInstance(PER_CLASS) +@Execution(CONCURRENT) +public class TestSharedHiveThriftMetastore + extends BaseSharedMetastoreTest +{ + private static final String HIVE_CATALOG = "hive"; + private String bucketName; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + bucketName = "test-iceberg-shared-metastore" + randomNameSuffix(); + HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName)); + hiveMinioDataLake.start(); + + Session icebergSession = testSessionBuilder() + .setCatalog(ICEBERG_CATALOG) + .setSchema(tpchSchema) + .build(); + Session hiveSession = testSessionBuilder() + .setCatalog(HIVE_CATALOG) + .setSchema(tpchSchema) + .build(); + + QueryRunner queryRunner = DistributedQueryRunner.builder(icebergSession).build(); + + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch"); + + Path dataDirectory = queryRunner.getCoordinator().getBaseDataDir().resolve("iceberg_data"); + dataDirectory.toFile().deleteOnExit(); + + queryRunner.installPlugin(new IcebergPlugin()); + queryRunner.createCatalog( + ICEBERG_CATALOG, + "iceberg", + ImmutableMap.builder() + .put("iceberg.catalog.type", "HIVE_METASTORE") + .put("hive.metastore.uri", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint().toString()) + .put("hive.metastore.thrift.client.read-timeout", "1m") // read timed out sometimes happens with the default timeout + .put("fs.hadoop.enabled", "false") + .put("fs.native-s3.enabled", "true") + .put("s3.aws-access-key", MINIO_ACCESS_KEY) + .put("s3.aws-secret-key", MINIO_SECRET_KEY) + .put("s3.region", MINIO_REGION) + .put("s3.endpoint", hiveMinioDataLake.getMinio().getMinioAddress()) + .put("s3.path-style-access", "true") + .put("s3.streaming.part-size", "5MB") // minimize memory usage + .put("s3.max-connections", "2") // verify no leaks + .put("iceberg.register-table-procedure.enabled", "true") + .put("iceberg.writer-sort-buffer-size", "1MB") + .buildOrThrow()); + queryRunner.createCatalog( + "iceberg_with_redirections", + "iceberg", + ImmutableMap.builder() + .put("iceberg.catalog.type", "HIVE_METASTORE") + .put("hive.metastore.uri", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint().toString()) + .put("hive.metastore.thrift.client.read-timeout", "1m") // read timed out sometimes happens with the default timeout + .put("fs.hadoop.enabled", "false") + .put("fs.native-s3.enabled", "true") + .put("s3.aws-access-key", MINIO_ACCESS_KEY) + .put("s3.aws-secret-key", MINIO_SECRET_KEY) + .put("s3.region", MINIO_REGION) + .put("s3.endpoint", hiveMinioDataLake.getMinio().getMinioAddress()) + .put("s3.path-style-access", "true") + .put("s3.streaming.part-size", "5MB") // minimize memory usage + .put("s3.max-connections", "2") // verify no leaks + .put("iceberg.register-table-procedure.enabled", "true") + .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.hive-catalog-name", "hive") + .buildOrThrow()); + + queryRunner.installPlugin(new TestingHivePlugin(dataDirectory)); + Map hiveProperties = ImmutableMap.builder() + .put("hive.metastore", "thrift") + .put("hive.metastore.uri", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint().toString()) + .put("fs.hadoop.enabled", "false") + .put("fs.native-s3.enabled", "true") + .put("s3.aws-access-key", MINIO_ACCESS_KEY) + .put("s3.aws-secret-key", MINIO_SECRET_KEY) + .put("s3.region", MINIO_REGION) + .put("s3.endpoint", hiveMinioDataLake.getMinio().getMinioAddress()) + .put("s3.path-style-access", "true") + .put("s3.streaming.part-size", "5MB") + .put("hive.max-partitions-per-scan", "1000") + .put("hive.max-partitions-for-eager-load", "1000") + .put("hive.security", "allow-all") + .buildOrThrow(); + queryRunner.createCatalog(HIVE_CATALOG, "hive", hiveProperties); + queryRunner.createCatalog( + "hive_with_redirections", + "hive", + ImmutableMap.builder() + .putAll(hiveProperties).put("hive.iceberg-catalog-name", "iceberg") + .buildOrThrow()); + + queryRunner.execute("CREATE SCHEMA " + tpchSchema + " WITH (location = 's3://" + bucketName + "/" + tpchSchema + "')"); + copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, icebergSession, ImmutableList.of(TpchTable.NATION)); + copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, hiveSession, ImmutableList.of(TpchTable.REGION)); + queryRunner.execute("CREATE SCHEMA " + testSchema + " WITH (location = 's3://" + bucketName + "/" + testSchema + "')"); + + return queryRunner; + } + + @AfterAll + public void cleanup() + { + assertQuerySucceeds("DROP TABLE IF EXISTS hive." + tpchSchema + ".region"); + assertQuerySucceeds("DROP TABLE IF EXISTS iceberg." + tpchSchema + ".nation"); + assertQuerySucceeds("DROP SCHEMA IF EXISTS hive." + tpchSchema); + assertQuerySucceeds("DROP SCHEMA IF EXISTS hive." + testSchema); + } + + @Override + protected String getExpectedHiveCreateSchema(String catalogName) + { + return """ + CREATE SCHEMA %s.%s + WITH ( + location = 's3://%s/%s' + )""" + .formatted(catalogName, tpchSchema, bucketName, tpchSchema); + } + + @Override + protected String getExpectedIcebergCreateSchema(String catalogName) + { + String expectedIcebergCreateSchema = "CREATE SCHEMA %s.%s\n" + + "AUTHORIZATION USER user\n" + + "WITH (\n" + + " location = 's3://%s/%s'\n" + + ")"; + return format(expectedIcebergCreateSchema, catalogName, tpchSchema, bucketName, tpchSchema); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java index a6cdc1793d2b..01f87a4b3d73 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java @@ -95,7 +95,7 @@ protected String getMetadataLocation(String tableName) @Override protected String schemaPath() { - return format("%s/%s", warehouseLocation, getSession().getSchema()); + return format("%s/%s", warehouseLocation, getSession().getSchema().orElseThrow()); } @Override @@ -470,7 +470,7 @@ public void testDropTableWithMissingDataFile() public void testDropTableWithNonExistentTableLocation() { assertThatThrownBy(super::testDropTableWithNonExistentTableLocation) - .hasMessageContaining("Access Denied"); + .hasStackTraceContaining("Access Denied"); } @Test @@ -520,4 +520,12 @@ public void testTruncateTable() assertThatThrownBy(super::testTruncateTable) .hasMessageContaining("Access Denied"); } + + @Test + @Override + public void testIcebergTablesFunction() + { + assertThatThrownBy(super::testIcebergTablesFunction) + .hasStackTraceContaining("Access Denied"); + } } From 975e0622d094d74aa37a8106668236c025e2fc75 Mon Sep 17 00:00:00 2001 From: lukasz-stec Date: Mon, 23 Dec 2024 11:34:31 +0100 Subject: [PATCH 058/158] Fix failures in iceberg cloud tests --- .../iceberg/catalog/BaseTrinoCatalogTest.java | 21 ++++++++++++++++- .../catalog/glue/TestTrinoGlueCatalog.java | 23 +++++++++++++++++++ ...ergSnowflakeCatalogConnectorSmokeTest.java | 8 +++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java index e2ae7af4c66b..acc0c3943fb9 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java @@ -478,8 +478,9 @@ public void testListTables() try { SchemaTableName materializedView = new SchemaTableName(ns2, "mv"); - catalog.createMaterializedView( + createMaterializedView( SESSION, + catalog, materializedView, someMaterializedView(), ImmutableMap.of( @@ -514,6 +515,24 @@ public void testListTables() } } + protected void createMaterializedView( + ConnectorSession session, + TrinoCatalog catalog, + SchemaTableName materializedView, + ConnectorMaterializedViewDefinition materializedViewDefinition, + Map properties, + boolean replace, + boolean ignoreExisting) + { + catalog.createMaterializedView( + session, + materializedView, + materializedViewDefinition, + properties, + replace, + ignoreExisting); + } + protected Optional createExternalIcebergTable(TrinoCatalog catalog, String namespace, AutoCloseableCloser closer) throws Exception { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java index 80ee1628b561..4e30c6a7d8c4 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java @@ -35,6 +35,7 @@ import io.trino.spi.connector.CatalogHandle; import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.MaterializedViewNotFoundException; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.PrincipalType; @@ -257,4 +258,26 @@ public void testDefaultLocation() } } } + + @Override + protected void createMaterializedView( + ConnectorSession session, + TrinoCatalog catalog, + SchemaTableName materializedView, + ConnectorMaterializedViewDefinition materializedViewDefinition, + Map properties, + boolean replace, + boolean ignoreExisting) + { + catalog.createMaterializedView( + session, + materializedView, + materializedViewDefinition, + ImmutableMap.builder() + .putAll(properties) + .put(LOCATION_PROPERTY, "file:///tmp/a/path/" + materializedView.getTableName()) + .buildOrThrow(), + replace, + ignoreExisting); + } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/snowflake/TestIcebergSnowflakeCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/snowflake/TestIcebergSnowflakeCatalogConnectorSmokeTest.java index 394e4c0b32a9..2488a1c683b5 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/snowflake/TestIcebergSnowflakeCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/snowflake/TestIcebergSnowflakeCatalogConnectorSmokeTest.java @@ -684,6 +684,14 @@ public void testSnowflakeNativeTable() .hasRootCauseMessage("SQL compilation error:\ninvalid parameter 'table ? is not a Snowflake iceberg table'"); } + @Test + @Override + public void testIcebergTablesFunction() + { + assertThatThrownBy(super::testIcebergTablesFunction) + .hasMessageContaining("schemaPath is not supported for Iceberg snowflake catalog"); + } + @Override protected boolean isFileSorted(Location path, String sortColumnName) { From 4dd2d295bf95f22fafb4e688b404102570e6ae62 Mon Sep 17 00:00:00 2001 From: chenjian2664 Date: Wed, 11 Dec 2024 16:09:28 +0800 Subject: [PATCH 059/158] Support MERGE for Ignite connector --- .../io/trino/plugin/ignite/IgniteClient.java | 23 +++++++++ .../plugin/ignite/IgniteMergeTableHandle.java | 40 +++++++++++++++ .../trino/plugin/ignite/IgniteMetadata.java | 49 +++++++++++++++++++ .../ignite/TestIgniteConnectorTest.java | 13 +++++ 4 files changed, 125 insertions(+) create mode 100644 plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMergeTableHandle.java diff --git a/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteClient.java b/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteClient.java index 34c8183d7886..887db854e4e2 100644 --- a/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteClient.java +++ b/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteClient.java @@ -81,10 +81,13 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.BiFunction; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.plugin.ignite.IgniteTableProperties.PRIMARY_KEY_PROPERTY; import static io.trino.plugin.jdbc.ColumnMapping.longMapping; import static io.trino.plugin.jdbc.DecimalConfig.DecimalMapping.ALLOW_OVERFLOW; @@ -466,6 +469,26 @@ public boolean isLimitGuaranteed(ConnectorSession session) return true; } + @Override + public boolean supportsMerge() + { + return true; + } + + @Override + public List getPrimaryKeys(ConnectorSession session, RemoteTableName remoteTableName) + { + JdbcTableHandle plainTable = new JdbcTableHandle(remoteTableName.getSchemaTableName(), remoteTableName, Optional.empty()); + Map tableProperties = getTableProperties(session, plainTable); + Set primaryKey = ImmutableSet.copyOf(IgniteTableProperties.getPrimaryKey(tableProperties)); + List primaryKeys = getColumns(session, remoteTableName.getSchemaTableName(), remoteTableName) + .stream() + .filter(columnHandle -> primaryKey.contains(columnHandle.getColumnName().toLowerCase(ENGLISH))) + .collect(toImmutableList()); + verify(!primaryKeys.isEmpty(), "Ignite primary keys is empty"); + return primaryKeys; + } + @Override public boolean supportsTopN(ConnectorSession session, JdbcTableHandle handle, List sortOrder) { diff --git a/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMergeTableHandle.java b/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMergeTableHandle.java new file mode 100644 index 000000000000..7f027ed4ee5e --- /dev/null +++ b/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMergeTableHandle.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ignite; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.plugin.jdbc.JdbcColumnHandle; +import io.trino.plugin.jdbc.JdbcMergeTableHandle; +import io.trino.plugin.jdbc.JdbcTableHandle; +import io.trino.spi.connector.ColumnHandle; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class IgniteMergeTableHandle + extends JdbcMergeTableHandle +{ + @JsonCreator + public IgniteMergeTableHandle( + @JsonProperty("tableHandle") JdbcTableHandle tableHandle, + @JsonProperty("outputTableHandle") IgniteOutputTableHandle outputTableHandle, + @JsonProperty("primaryKeys") List primaryKeys, + @JsonProperty("dataColumns") List dataColumns, + @JsonProperty("updateCaseColumns") Map> updateCaseColumns) + { + super(tableHandle, outputTableHandle, primaryKeys, dataColumns, updateCaseColumns); + } +} diff --git a/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMetadata.java b/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMetadata.java index de60474d001d..283226d8506a 100644 --- a/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMetadata.java +++ b/plugin/trino-ignite/src/main/java/io/trino/plugin/ignite/IgniteMetadata.java @@ -18,6 +18,7 @@ import io.trino.plugin.jdbc.DefaultJdbcMetadata; import io.trino.plugin.jdbc.JdbcClient; import io.trino.plugin.jdbc.JdbcColumnHandle; +import io.trino.plugin.jdbc.JdbcMergeTableHandle; import io.trino.plugin.jdbc.JdbcNamedRelationHandle; import io.trino.plugin.jdbc.JdbcQueryEventListener; import io.trino.plugin.jdbc.JdbcTableHandle; @@ -28,6 +29,7 @@ import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorInsertTableHandle; +import io.trino.spi.connector.ConnectorMergeTableHandle; import io.trino.spi.connector.ConnectorOutputMetadata; import io.trino.spi.connector.ConnectorOutputTableHandle; import io.trino.spi.connector.ConnectorSession; @@ -44,9 +46,11 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.plugin.jdbc.JdbcMetadata.getColumns; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; @@ -100,6 +104,9 @@ public ConnectorInsertTableHandle beginInsert(ConnectorSession session, Connecto ImmutableList.Builder columnJdbcTypeHandles = ImmutableList.builder(); for (ColumnHandle column : columns) { JdbcColumnHandle columnHandle = (JdbcColumnHandle) column; + if (IGNITE_DUMMY_ID.equalsIgnoreCase(columnHandle.getColumnName())) { + continue; + } columnNames.add(columnHandle.getColumnName()); columnTypes.add(columnHandle.getColumnType()); columnJdbcTypeHandles.add(columnHandle.getJdbcTypeHandle()); @@ -125,6 +132,48 @@ public Optional finishInsert( return Optional.empty(); } + @Override + public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateColumnHandles, RetryMode retryMode) + { + JdbcTableHandle handle = (JdbcTableHandle) tableHandle; + JdbcMergeTableHandle mergeTableHandle = (JdbcMergeTableHandle) super.beginMerge(session, tableHandle, updateColumnHandles, retryMode); + + List primaryKeys = mergeTableHandle.getPrimaryKeys(); + List columns = igniteClient.getColumns(session, + handle.getRequiredNamedRelation().getSchemaTableName(), + handle.getRequiredNamedRelation().getRemoteTableName()).stream() + .filter(column -> !IGNITE_DUMMY_ID.equalsIgnoreCase(column.getColumnName())) + .collect(toImmutableList()); + + for (Collection updateColumns : updateColumnHandles.values()) { + for (ColumnHandle column : updateColumns) { + checkArgument(columns.contains(column), "the update column not found in the target table"); + checkArgument(!primaryKeys.contains(column), "Ignite does not allow update primary key"); + } + } + + if (handle.getColumns().isPresent()) { + handle = new JdbcTableHandle( + handle.getRelationHandle(), + handle.getConstraint(), + handle.getConstraintExpressions(), + handle.getSortOrder(), + handle.getLimit(), + Optional.of(columns), + handle.getOtherReferencedTables(), + handle.getNextSyntheticColumnId(), + handle.getAuthorization(), + handle.getUpdateAssignments()); + } + + return new IgniteMergeTableHandle( + handle, + (IgniteOutputTableHandle) mergeTableHandle.getOutputTableHandle(), + primaryKeys, + columns, + mergeTableHandle.getUpdateCaseColumns()); + } + @Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) { diff --git a/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java b/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java index a563121ed76a..c4ee09cdd966 100644 --- a/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java +++ b/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java @@ -14,6 +14,7 @@ package io.trino.plugin.ignite; import com.google.common.collect.ImmutableList; +import io.trino.Session; import io.trino.plugin.jdbc.BaseJdbcConnectorTest; import io.trino.sql.planner.plan.FilterNode; import io.trino.sql.planner.plan.TableScanNode; @@ -29,6 +30,7 @@ import java.util.Optional; import static com.google.common.base.Strings.nullToEmpty; +import static io.trino.plugin.jdbc.JdbcWriteSessionProperties.NON_TRANSACTIONAL_MERGE; import static io.trino.sql.planner.assertions.PlanMatchPattern.node; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; @@ -52,6 +54,15 @@ protected QueryRunner createQueryRunner() .build(); } + @Override + protected Session getSession() + { + Session session = super.getSession(); + return Session.builder(session) + .setCatalogSessionProperty(session.getCatalog().orElseThrow(), NON_TRANSACTIONAL_MERGE, "true") + .build(); + } + @Override protected SqlExecutor onRemoteDatabase() { @@ -64,7 +75,9 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) return switch (connectorBehavior) { case SUPPORTS_AGGREGATION_PUSHDOWN, SUPPORTS_JOIN_PUSHDOWN, + SUPPORTS_MERGE, SUPPORTS_PREDICATE_EXPRESSION_PUSHDOWN_WITH_LIKE, + SUPPORTS_ROW_LEVEL_UPDATE, SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR -> true; case SUPPORTS_ADD_COLUMN_NOT_NULL_CONSTRAINT, SUPPORTS_ADD_COLUMN_WITH_COMMENT, From f9aff2356e364ead651ef33432d9f671a5fe6ddc Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 23 Dec 2024 21:32:25 +0900 Subject: [PATCH 060/158] Remove unused delta-lake-databricks-104 test group --- .../src/main/java/io/trino/tests/product/TestGroups.java | 1 - .../deltalake/TestDeltaLakeCaseInsensitiveMapping.java | 3 +-- .../TestDeltaLakeChangeDataFeedCompatibility.java | 3 +-- .../deltalake/TestDeltaLakeCheckpointsCompatibility.java | 3 +-- .../product/deltalake/TestDeltaLakeColumnMappingMode.java | 5 ++--- .../TestDeltaLakeCreateTableAsSelectCompatibility.java | 3 +-- .../TestDeltaLakeIdentityColumnCompatibility.java | 7 +++---- .../deltalake/TestDeltaLakeInsertCompatibility.java | 5 ++--- .../deltalake/TestDeltaLakeSelectCompatibility.java | 3 +-- .../deltalake/TestDeltaLakeUpdateCompatibility.java | 3 +-- .../TestDeltaLakeWriteDatabricksCompatibility.java | 4 ++-- 11 files changed, 15 insertions(+), 25 deletions(-) diff --git a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java index ca159621c163..11c13365cdb9 100644 --- a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java @@ -96,7 +96,6 @@ public final class TestGroups public static final String DELTA_LAKE_AZURE = "delta-lake-azure"; public static final String DELTA_LAKE_GCS = "delta-lake-gcs"; public static final String DELTA_LAKE_DATABRICKS = "delta-lake-databricks"; - public static final String DELTA_LAKE_DATABRICKS_104 = "delta-lake-databricks-104"; public static final String DELTA_LAKE_DATABRICKS_113 = "delta-lake-databricks-113"; public static final String DELTA_LAKE_DATABRICKS_122 = "delta-lake-databricks-122"; public static final String DELTA_LAKE_DATABRICKS_133 = "delta-lake-databricks-133"; diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java index d0df70939f0c..30971019b12a 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; @@ -76,7 +75,7 @@ public void testNonLowercaseColumnNames() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testNonLowercaseFieldNames() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java index b06653a2e9c8..fca2c054dbcf 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java @@ -31,7 +31,6 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; @@ -533,7 +532,7 @@ public void testMergeDeleteIntoTableWithCdfEnabled(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testMergeMixedDeleteAndUpdateIntoTableWithCdfEnabled() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java index b7a88824e179..530a409e2402 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java @@ -38,7 +38,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; @@ -279,7 +278,7 @@ private void trinoUsesCheckpointInterval(String deltaTableProperties) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDatabricksUsesCheckpointInterval() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java index 03d89222d0a7..965eda2d4df6 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java @@ -27,7 +27,6 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; @@ -55,7 +54,7 @@ public class TestDeltaLakeColumnMappingMode extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testColumnMappingModeNone() { @@ -234,7 +233,7 @@ private void testColumnMappingModeReaderAndWriterVersion(Consumer create onTrino().executeQuery("DROP TABLE delta.default." + tableName); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoColumnMappingMode(String mode) { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java index 73adc6255973..6dec47308100 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java @@ -25,7 +25,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; @@ -44,7 +43,7 @@ public class TestDeltaLakeCreateTableAsSelectCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testPrestoTypesWithDatabricks() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java index 4bf5648133a2..a0cf50cd03e4 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java @@ -21,7 +21,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -38,7 +37,7 @@ public class TestDeltaLakeIdentityColumnCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumn() { @@ -153,7 +152,7 @@ public void testDropIdentityColumn(String mode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testVacuumProcedureWithIdentityColumn() { @@ -184,7 +183,7 @@ public void testVacuumProcedureWithIdentityColumn() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumnCheckpointInterval() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java index 61c6a6fb16ce..25bb045d3c61 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java @@ -31,7 +31,6 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; @@ -59,7 +58,7 @@ public void setup() databricksRuntimeVersion = getDatabricksRuntimeVersion(); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testInsertCompatibility() { @@ -95,7 +94,7 @@ public void testInsertCompatibility() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testPartitionedInsertCompatibility() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java index e14e757ac7d6..3a5a65f57132 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; @@ -42,7 +41,7 @@ public class TestDeltaLakeSelectCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testPartitionedSelectSpecialCharacters() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java index e1a187982d0b..37ebaaa6e6bd 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; @@ -39,7 +38,7 @@ public class TestDeltaLakeUpdateCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdatesFromDatabricks() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java index bbf752da65df..6e08f9a20a9d 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java @@ -35,7 +35,7 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_122_RUNTIME_VERSION; @@ -323,7 +323,7 @@ public void testInsertingIntoDatabricksTableWithAddedNotNullConstraint() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoVacuumRemoveChangeDataFeedFiles() { From 47720cbf0441262469ce5844cd89a8c3974690d1 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Fri, 20 Dec 2024 17:06:48 +0900 Subject: [PATCH 061/158] Add $all_entries metadata table to Iceberg --- .../io/trino/plugin/iceberg/EntriesTable.java | 9 ++++++-- .../trino/plugin/iceberg/IcebergMetadata.java | 5 ++++- .../io/trino/plugin/iceberg/TableType.java | 1 + .../iceberg/BaseIcebergSystemTables.java | 21 +++++++++++++++++++ .../TestIcebergMetastoreAccessOperations.java | 9 +++++++- ...estIcebergGlueCatalogAccessOperations.java | 9 +++++++- 6 files changed, 49 insertions(+), 5 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java index 0c8d54384192..9c0560800826 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/EntriesTable.java @@ -27,6 +27,7 @@ import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; import jakarta.annotation.Nullable; +import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetricsUtil.ReadableMetricsStruct; import org.apache.iceberg.PartitionField; import org.apache.iceberg.Table; @@ -43,6 +44,7 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.wrappedHeapBuffer; import static io.trino.plugin.iceberg.FilesTable.getIcebergIdToTypeMapping; @@ -59,8 +61,10 @@ import static io.trino.spi.type.VarbinaryType.VARBINARY; import static io.trino.spi.type.VarcharType.VARCHAR; import static java.util.Objects.requireNonNull; +import static org.apache.iceberg.MetadataTableType.ALL_ENTRIES; import static org.apache.iceberg.MetadataTableType.ENTRIES; +// https://iceberg.apache.org/docs/latest/spark-queries/#all-entries // https://iceberg.apache.org/docs/latest/spark-queries/#entries public class EntriesTable extends BaseSystemTable @@ -70,15 +74,16 @@ public class EntriesTable private final Optional partitionColumn; private final List partitionTypes; - public EntriesTable(TypeManager typeManager, SchemaTableName tableName, Table icebergTable, ExecutorService executor) + public EntriesTable(TypeManager typeManager, SchemaTableName tableName, Table icebergTable, MetadataTableType metadataTableType, ExecutorService executor) { super( requireNonNull(icebergTable, "icebergTable is null"), new ConnectorTableMetadata( requireNonNull(tableName, "tableName is null"), columns(requireNonNull(typeManager, "typeManager is null"), icebergTable)), - ENTRIES, + metadataTableType, executor); + checkArgument(metadataTableType == ALL_ENTRIES || metadataTableType == ENTRIES, "Unexpected metadata table type: %s", metadataTableType); idToTypeMapping = getIcebergIdToTypeMapping(icebergTable.schema()); primitiveFields = IcebergUtil.primitiveFields(icebergTable.schema()).stream() .sorted(Comparator.comparing(Types.NestedField::name)) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 0f92016badcf..73afe67c3a2c 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -358,6 +358,8 @@ import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; import static java.util.stream.Collectors.joining; +import static org.apache.iceberg.MetadataTableType.ALL_ENTRIES; +import static org.apache.iceberg.MetadataTableType.ENTRIES; import static org.apache.iceberg.ReachableFileUtil.metadataFileLocations; import static org.apache.iceberg.ReachableFileUtil.statisticsFilesLocations; import static org.apache.iceberg.SnapshotSummary.DELETED_RECORDS_PROP; @@ -707,7 +709,8 @@ private Optional getRawSystemTable(ConnectorSession session, Schema case ALL_MANIFESTS -> Optional.of(new AllManifestsTable(tableName, table, icebergScanExecutor)); case MANIFESTS -> Optional.of(new ManifestsTable(tableName, table, getCurrentSnapshotId(table))); case FILES -> Optional.of(new FilesTable(tableName, typeManager, table, getCurrentSnapshotId(table), icebergScanExecutor)); - case ENTRIES -> Optional.of(new EntriesTable(typeManager, tableName, table, icebergScanExecutor)); + case ALL_ENTRIES -> Optional.of(new EntriesTable(typeManager, tableName, table, ALL_ENTRIES, icebergScanExecutor)); + case ENTRIES -> Optional.of(new EntriesTable(typeManager, tableName, table, ENTRIES, icebergScanExecutor)); case PROPERTIES -> Optional.of(new PropertiesTable(tableName, table)); case REFS -> Optional.of(new RefsTable(tableName, table, icebergScanExecutor)); }; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java index ab39c3504b6f..14dcbe856e82 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableType.java @@ -23,6 +23,7 @@ public enum TableType MANIFESTS, PARTITIONS, FILES, + ALL_ENTRIES, ENTRIES, PROPERTIES, REFS, diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java index e6fbf00ca54e..ff5b78fc48f7 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java @@ -563,6 +563,27 @@ public void testFilesTableWithDelete() assertUpdate("DROP TABLE IF EXISTS test_schema.test_table_with_delete"); } + @Test + void testAllEntriesTable() + { + try (TestTable table = new TestTable(getQueryRunner()::execute, "test_all_entries", "AS SELECT 1 id, DATE '2014-01-01' dt")) { + assertThat(query("DESCRIBE \"" + table.getName() + "$all_entries\"")) + .matches("DESCRIBE \"" + table.getName() + "$entries\""); + + assertThat(query("SELECT * FROM \"" + table.getName() + "$all_entries\"")) + .matches("SELECT * FROM \"" + table.getName() + "$entries\""); + + assertUpdate("DELETE FROM " + table.getName(), 1); + + assertThat(computeActual("SELECT status FROM \"" + table.getName() + "$all_entries\"").getOnlyColumnAsSet()) + .containsExactly(1, 2); + assertThat(computeActual("SELECT status FROM \"" + table.getName() + "$entries\"").getOnlyColumnAsSet()) + .containsExactly(2); + assertThat(query("SELECT * FROM \"" + table.getName() + "$all_entries\" WHERE status = 2")) + .matches("SELECT * FROM \"" + table.getName() + "$entries\""); + } + } + @Test void testEntriesTable() { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java index 401e68e38c90..c9ffe5d571bf 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetastoreAccessOperations.java @@ -36,6 +36,7 @@ import static io.trino.plugin.hive.metastore.MetastoreMethod.GET_TABLES; import static io.trino.plugin.hive.metastore.MetastoreMethod.REPLACE_TABLE; import static io.trino.plugin.iceberg.IcebergSessionProperties.COLLECT_EXTENDED_STATISTICS_ON_WRITE; +import static io.trino.plugin.iceberg.TableType.ALL_ENTRIES; import static io.trino.plugin.iceberg.TableType.ALL_MANIFESTS; import static io.trino.plugin.iceberg.TableType.DATA; import static io.trino.plugin.iceberg.TableType.ENTRIES; @@ -339,6 +340,12 @@ public void testSelectSystemTable() .addCopies(GET_TABLE, 1) .build()); + // select from $all_entries + assertMetastoreInvocations("SELECT * FROM \"test_select_snapshots$all_entries\"", + ImmutableMultiset.builder() + .addCopies(GET_TABLE, 1) + .build()); + // select from $entries assertMetastoreInvocations("SELECT * FROM \"test_select_snapshots$entries\"", ImmutableMultiset.builder() @@ -356,7 +363,7 @@ public void testSelectSystemTable() // This test should get updated if a new system table is added. assertThat(TableType.values()) - .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, ENTRIES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); + .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, ALL_ENTRIES, ENTRIES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); } @Test diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java index e116e95fab0f..29109e31d2ce 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java @@ -50,6 +50,7 @@ import static io.trino.plugin.hive.metastore.glue.GlueMetastoreMethod.GET_TABLES; import static io.trino.plugin.hive.metastore.glue.GlueMetastoreMethod.UPDATE_TABLE; import static io.trino.plugin.iceberg.IcebergSessionProperties.COLLECT_EXTENDED_STATISTICS_ON_WRITE; +import static io.trino.plugin.iceberg.TableType.ALL_ENTRIES; import static io.trino.plugin.iceberg.TableType.ALL_MANIFESTS; import static io.trino.plugin.iceberg.TableType.DATA; import static io.trino.plugin.iceberg.TableType.ENTRIES; @@ -471,6 +472,12 @@ public void testSelectSystemTable() .add(GET_TABLE) .build()); + // select from $all_entries + assertGlueMetastoreApiInvocations("SELECT * FROM \"test_select_snapshots$all_entries\"", + ImmutableMultiset.builder() + .add(GET_TABLE) + .build()); + // select from $entries assertGlueMetastoreApiInvocations("SELECT * FROM \"test_select_snapshots$entries\"", ImmutableMultiset.builder() @@ -494,7 +501,7 @@ public void testSelectSystemTable() // This test should get updated if a new system table is added. assertThat(TableType.values()) - .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, ENTRIES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); + .containsExactly(DATA, HISTORY, METADATA_LOG_ENTRIES, SNAPSHOTS, ALL_MANIFESTS, MANIFESTS, PARTITIONS, FILES, ALL_ENTRIES, ENTRIES, PROPERTIES, REFS, MATERIALIZED_VIEW_STORAGE); } finally { getQueryRunner().execute("DROP TABLE IF EXISTS test_select_snapshots"); From 9c05d059e385a9c849c6a46b1b8a173611e38104 Mon Sep 17 00:00:00 2001 From: 7hong Date: Wed, 11 Dec 2024 17:56:15 +0800 Subject: [PATCH 062/158] Delete the oldest tracked version metadata files after commit --- .../AbstractIcebergTableOperations.java | 2 + .../BaseIcebergConnectorSmokeTest.java | 51 ++++++++++++++++++- .../BaseIcebergMinioConnectorSmokeTest.java | 1 + .../plugin/iceberg/IcebergTestUtils.java | 24 +++++++++ .../TestIcebergAbfsConnectorSmokeTest.java | 1 + .../TestIcebergConnectorSmokeTest.java | 3 +- .../TestIcebergGcsConnectorSmokeTest.java | 1 + .../trino/plugin/iceberg/TestIcebergV2.java | 38 ++++++++++++++ ...tIcebergGlueCatalogConnectorSmokeTest.java | 3 +- ...tIcebergJdbcCatalogConnectorSmokeTest.java | 1 + ...cebergNessieCatalogConnectorSmokeTest.java | 3 +- ...ebergPolarisCatalogConnectorSmokeTest.java | 1 + ...alogNestedNamespaceConnectorSmokeTest.java | 1 + ...ergTrinoRestCatalogConnectorSmokeTest.java | 1 + ...ergUnityRestCatalogConnectorSmokeTest.java | 8 +++ ...gVendingRestCatalogConnectorSmokeTest.java | 1 + 16 files changed, 136 insertions(+), 4 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java index bfe408978e7b..498a718b567a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java @@ -63,6 +63,7 @@ import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; import static org.apache.iceberg.BaseMetastoreTableOperations.METADATA_LOCATION_PROP; +import static org.apache.iceberg.CatalogUtil.deleteRemovedMetadataFiles; import static org.apache.iceberg.TableMetadataParser.getFileExtension; import static org.apache.iceberg.TableProperties.METADATA_COMPRESSION; import static org.apache.iceberg.TableProperties.METADATA_COMPRESSION_DEFAULT; @@ -174,6 +175,7 @@ public void commit(@Nullable TableMetadata base, TableMetadata metadata) } else { commitToExistingTable(base, metadata); + deleteRemovedMetadataFiles(fileIo, base, metadata); } shouldRefresh = true; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java index 745894d2d2ff..91f31976d3ce 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java @@ -38,23 +38,32 @@ import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; +import java.io.IOException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; +import static io.trino.plugin.iceberg.IcebergTestUtils.getMetadataFileAndUpdatedMillis; import static io.trino.plugin.iceberg.IcebergTestUtils.withSmallRowGroups; import static io.trino.testing.TestingAccessControlManager.TestingPrivilegeType.DROP_TABLE; import static io.trino.testing.TestingAccessControlManager.privilege; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_CREATE_TABLE; import static io.trino.testing.TestingConnectorSession.SESSION; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; @@ -887,6 +896,39 @@ protected AutoCloseable createAdditionalTables(String schema) return () -> {}; } + @Test + public void testMetadataDeleteAfterCommitEnabled() + throws IOException + { + if (!hasBehavior(SUPPORTS_CREATE_TABLE)) { + return; + } + + int metadataPreviousVersionCount = 5; + String tableName = "test_metadata_delete_after_commit_enabled" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + "(_bigint BIGINT, _varchar VARCHAR)"); + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES extra_properties = MAP(ARRAY['write.metadata.delete-after-commit.enabled'], ARRAY['true'])"); + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES extra_properties = MAP(ARRAY['write.metadata.previous-versions-max'], ARRAY['" + metadataPreviousVersionCount + "'])"); + String tableLocation = getTableLocation(tableName); + + Map historyMetadataFiles = getMetadataFileAndUpdatedMillis(fileSystem, tableLocation); + for (int i = 0; i < 10; i++) { + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'a')", 1); + Map metadataFiles = getMetadataFileAndUpdatedMillis(fileSystem, tableLocation); + historyMetadataFiles.putAll(metadataFiles); + assertThat(metadataFiles.size()).isLessThanOrEqualTo(1 + metadataPreviousVersionCount); + Set expectMetadataFiles = historyMetadataFiles + .entrySet() + .stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(metadataPreviousVersionCount + 1) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + assertThat(metadataFiles.keySet()).containsAll(expectMetadataFiles); + } + assertUpdate("DROP TABLE " + tableName); + } + private long getMostRecentSnapshotId(String tableName) { return (long) Iterables.getOnlyElement(getQueryRunner().execute(format("SELECT snapshot_id FROM \"%s$snapshots\" ORDER BY committed_at DESC LIMIT 1", tableName)) @@ -901,7 +943,14 @@ private ZonedDateTime getSnapshotTime(String tableName, long snapshotId) protected String getTableLocation(String tableName) { - return (String) computeScalar("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*/[^/]*$', '') FROM " + tableName); + Pattern locationPattern = Pattern.compile(".*location = '(.*?)'.*", Pattern.DOTALL); + Matcher m = locationPattern.matcher((String) computeActual("SHOW CREATE TABLE " + tableName).getOnlyValue()); + if (m.find()) { + String location = m.group(1); + verify(!m.find(), "Unexpected second match"); + return location; + } + throw new IllegalStateException("Location not found in SHOW CREATE TABLE result"); } protected abstract void dropTableFromMetastore(String tableName); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java index 91474fa6fa8c..e18067d047be 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioConnectorSmokeTest.java @@ -95,6 +95,7 @@ protected QueryRunner createQueryRunner() .put("s3.max-connections", "2") // verify no leaks .put("iceberg.register-table-procedure.enabled", "true") .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .putAll(getAdditionalIcebergProperties()) .buildOrThrow()) .setSchemaInitializer( diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java index 9d70f271f4a2..702cd94ff027 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java @@ -15,6 +15,8 @@ import io.airlift.slice.Slice; import io.trino.Session; +import io.trino.filesystem.FileEntry; +import io.trino.filesystem.FileIterator; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; @@ -40,15 +42,20 @@ import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.file.FileMetastoreTableOperationsProvider; import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalog; +import io.trino.plugin.iceberg.fileio.ForwardingInputFile; import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.type.TestingTypeManager; import io.trino.testing.QueryRunner; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -60,6 +67,8 @@ import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; import static io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable; +import static io.trino.plugin.iceberg.util.FileOperationUtils.FileType.METADATA_JSON; +import static io.trino.plugin.iceberg.util.FileOperationUtils.FileType.fromFilePath; import static io.trino.testing.TestingConnectorSession.SESSION; public final class IcebergTestUtils @@ -190,4 +199,19 @@ public static BaseTable loadTable(String tableName, directExecutor()); return (BaseTable) loadIcebergTable(catalog, tableOperationsProvider, SESSION, new SchemaTableName(schemaName, tableName)); } + + public static Map getMetadataFileAndUpdatedMillis(TrinoFileSystem trinoFileSystem, String tableLocation) + throws IOException + { + FileIterator fileIterator = trinoFileSystem.listFiles(Location.of(tableLocation + "/metadata")); + Map metadataFiles = new HashMap<>(); + while (fileIterator.hasNext()) { + FileEntry entry = fileIterator.next(); + if (fromFilePath(entry.location().path()) == METADATA_JSON) { + TableMetadata tableMetadata = TableMetadataParser.read(null, new ForwardingInputFile(trinoFileSystem.newInputFile(entry.location()))); + metadataFiles.put(entry.location().path(), tableMetadata.lastUpdatedMillis()); + } + } + return metadataFiles; + } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergAbfsConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergAbfsConnectorSmokeTest.java index f627f76e2ddf..e8aed5b10a32 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergAbfsConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergAbfsConnectorSmokeTest.java @@ -92,6 +92,7 @@ protected QueryRunner createQueryRunner() .put("azure.access-key", accessKey) .put("iceberg.register-table-procedure.enabled", "true") .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .buildOrThrow()) .setSchemaInitializer( SchemaInitializer.builder() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java index b4b8dbfbd57e..bcc365bff98b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java @@ -55,7 +55,8 @@ protected QueryRunner createQueryRunner() .setIcebergProperties(ImmutableMap.of( "iceberg.file-format", format.name(), "iceberg.register-table-procedure.enabled", "true", - "iceberg.writer-sort-buffer-size", "1MB")) + "iceberg.writer-sort-buffer-size", "1MB", + "iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max")) .build(); metastore = getHiveMetastore(queryRunner); return queryRunner; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java index 41dd4bebbb97..8cdd3303ab48 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergGcsConnectorSmokeTest.java @@ -101,6 +101,7 @@ protected QueryRunner createQueryRunner() .put("iceberg.file-format", format.name()) .put("iceberg.register-table-procedure.enabled", "true") .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .buildOrThrow()) .setSchemaInitializer( SchemaInitializer.builder() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index 67aa3ab38a62..a5e59ec83610 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -20,6 +20,7 @@ import io.trino.filesystem.FileEntry; import io.trino.filesystem.FileIterator; import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; @@ -74,6 +75,7 @@ import org.junit.jupiter.api.TestInstance; import java.io.Closeable; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -96,6 +98,7 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; import static io.trino.plugin.iceberg.IcebergTestUtils.getHiveMetastore; +import static io.trino.plugin.iceberg.IcebergTestUtils.getMetadataFileAndUpdatedMillis; import static io.trino.plugin.iceberg.util.EqualityDeleteUtils.writeEqualityDeleteForTable; import static io.trino.plugin.iceberg.util.EqualityDeleteUtils.writeEqualityDeleteForTableWithSchema; import static io.trino.spi.type.BigintType.BIGINT; @@ -113,6 +116,8 @@ import static org.apache.iceberg.FileFormat.ORC; import static org.apache.iceberg.FileFormat.PARQUET; import static org.apache.iceberg.TableProperties.DEFAULT_NAME_MAPPING; +import static org.apache.iceberg.TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED; +import static org.apache.iceberg.TableProperties.METADATA_PREVIOUS_VERSIONS_MAX; import static org.apache.iceberg.TableProperties.SPLIT_SIZE; import static org.apache.iceberg.mapping.NameMappingParser.toJson; import static org.assertj.core.api.Assertions.assertThat; @@ -1516,6 +1521,39 @@ void testEnvironmentContext() } } + @Test + public void testMetadataDeleteAfterCommitEnabled() + throws IOException + { + int metadataPreviousVersionCount = 5; + String tableName = "test_metadata_delete_after_commit_enabled" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + "(_bigint BIGINT, _varchar VARCHAR)"); + BaseTable icebergTable = loadTable(tableName); + String location = icebergTable.location(); + icebergTable.updateProperties() + .set(METADATA_DELETE_AFTER_COMMIT_ENABLED, "true") + .set(METADATA_PREVIOUS_VERSIONS_MAX, String.valueOf(metadataPreviousVersionCount)) + .commit(); + + TrinoFileSystem trinoFileSystem = fileSystemFactory.create(SESSION); + Map historyMetadataFiles = getMetadataFileAndUpdatedMillis(trinoFileSystem, location); + for (int i = 0; i < 10; i++) { + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'a')", 1); + Map metadataFiles = getMetadataFileAndUpdatedMillis(trinoFileSystem, location); + historyMetadataFiles.putAll(metadataFiles); + assertThat(metadataFiles.size()).isLessThanOrEqualTo(1 + metadataPreviousVersionCount); + Set expectMetadataFiles = historyMetadataFiles + .entrySet() + .stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(metadataPreviousVersionCount + 1) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + assertThat(metadataFiles.keySet()).containsAll(expectMetadataFiles); + } + assertUpdate("DROP TABLE " + tableName); + } + private void testHighlyNestedFieldPartitioningWithTimestampTransform(String partitioning, String partitionDirectoryRegex, Set expectedPartitionDirectories) { String tableName = "test_highly_nested_field_partitioning_with_timestamp_transform_" + randomNameSuffix(); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java index d7adfe595aa4..8b0c017155c2 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java @@ -102,7 +102,8 @@ protected QueryRunner createQueryRunner() "iceberg.catalog.type", "glue", "hive.metastore.glue.default-warehouse-dir", schemaPath(), "iceberg.register-table-procedure.enabled", "true", - "iceberg.writer-sort-buffer-size", "1MB")) + "iceberg.writer-sort-buffer-size", "1MB", + "iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max")) .setSchemaInitializer( SchemaInitializer.builder() .withClonedTpchTables(REQUIRED_TPCH_TABLES) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestIcebergJdbcCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestIcebergJdbcCatalogConnectorSmokeTest.java index c78160d77d58..e422a0f98dc0 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestIcebergJdbcCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestIcebergJdbcCatalogConnectorSmokeTest.java @@ -108,6 +108,7 @@ protected QueryRunner createQueryRunner() .put("iceberg.jdbc-catalog.catalog-name", "tpch") .put("iceberg.register-table-procedure.enabled", "true") .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .put("iceberg.jdbc-catalog.default-warehouse-dir", warehouseLocation.getAbsolutePath()) .put("iceberg.jdbc-catalog.retryable-status-codes", "57P01,57P05") .buildOrThrow()) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogConnectorSmokeTest.java index 5ad8d045335c..16e49abd4ae4 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogConnectorSmokeTest.java @@ -103,7 +103,8 @@ protected QueryRunner createQueryRunner() "iceberg.catalog.type", "nessie", "iceberg.nessie-catalog.uri", nessieContainer.getRestApiUri(), "iceberg.nessie-catalog.default-warehouse-dir", tempDir.toString(), - "iceberg.writer-sort-buffer-size", "1MB")) + "iceberg.writer-sort-buffer-size", "1MB", + "iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max")) .setSchemaInitializer( SchemaInitializer.builder() .withClonedTpchTables(ImmutableList.>builder() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java index 9c578e94921f..0ccb0d3d4b4e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java @@ -83,6 +83,7 @@ protected QueryRunner createQueryRunner() .addIcebergProperty("iceberg.file-format", format.name()) .addIcebergProperty("iceberg.register-table-procedure.enabled", "true") .addIcebergProperty("iceberg.writer-sort-buffer-size", "1MB") + .addIcebergProperty("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .addIcebergProperty("iceberg.catalog.type", "rest") .addIcebergProperty("iceberg.rest-catalog.nested-namespace-enabled", "true") .addIcebergProperty("iceberg.rest-catalog.uri", polarisCatalog.restUri() + "/api/catalog") diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogNestedNamespaceConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogNestedNamespaceConnectorSmokeTest.java index a6dea1e1a757..0aa27a61f532 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogNestedNamespaceConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogNestedNamespaceConnectorSmokeTest.java @@ -104,6 +104,7 @@ protected QueryRunner createQueryRunner() .put("iceberg.rest-catalog.uri", testServer.getBaseUrl().toString()) .put("iceberg.register-table-procedure.enabled", "true") .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .buildOrThrow(); Map nestedNamespaceEnabled = ImmutableMap.builder() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java index 831e945240fa..8236cd43c19c 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java @@ -102,6 +102,7 @@ protected QueryRunner createQueryRunner() .put("iceberg.rest-catalog.uri", testServer.getBaseUrl().toString()) .put("iceberg.register-table-procedure.enabled", "true") .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .buildOrThrow()) .setInitialTables(REQUIRED_TPCH_TABLES) .build(); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java index 01f87a4b3d73..f665307bc265 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java @@ -528,4 +528,12 @@ public void testIcebergTablesFunction() assertThatThrownBy(super::testIcebergTablesFunction) .hasStackTraceContaining("Access Denied"); } + + @Test + @Override + public void testMetadataDeleteAfterCommitEnabled() + { + assertThatThrownBy(super::testMetadataDeleteAfterCommitEnabled) + .hasStackTraceContaining("Access Denied"); + } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java index c2efa55daa8d..acf12607fa13 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java @@ -120,6 +120,7 @@ protected QueryRunner createQueryRunner() .put("iceberg.rest-catalog.uri", "http://" + restCatalogBackendContainer.getRestCatalogEndpoint()) .put("iceberg.rest-catalog.vended-credentials-enabled", "true") .put("iceberg.writer-sort-buffer-size", "1MB") + .put("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") .put("fs.hadoop.enabled", "false") .put("fs.native-s3.enabled", "true") .put("s3.region", MINIO_REGION) From 881ca99da0c17a6bebb211656a2df147189a1fcd Mon Sep 17 00:00:00 2001 From: chenjian2664 Date: Sat, 21 Dec 2024 22:28:05 +0800 Subject: [PATCH 063/158] Using airlift log in SimulationController --- .../executor/timesharing/SimulationController.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/trino-main/src/test/java/io/trino/execution/executor/timesharing/SimulationController.java b/core/trino-main/src/test/java/io/trino/execution/executor/timesharing/SimulationController.java index 990a351170e9..eb268d329241 100644 --- a/core/trino-main/src/test/java/io/trino/execution/executor/timesharing/SimulationController.java +++ b/core/trino-main/src/test/java/io/trino/execution/executor/timesharing/SimulationController.java @@ -16,6 +16,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimaps; +import io.airlift.log.Logger; import io.trino.execution.StageId; import io.trino.execution.TaskId; import io.trino.execution.executor.timesharing.SimulationTask.IntermediateTask; @@ -35,6 +36,8 @@ class SimulationController { + private static final Logger log = Logger.get(SimulationController.class); + private static final int DEFAULT_MIN_SPLITS_PER_TASK = 3; private final TimeSharingTaskExecutor taskExecutor; @@ -63,7 +66,7 @@ public synchronized void addTaskSpecification(TaskSpecification spec) public synchronized void clearPendingQueue() { - System.out.println("Clearing pending queue.."); + log.info("Clearing pending queue.."); clearPendingQueue.set(true); } @@ -116,7 +119,7 @@ private synchronized void scheduleSplitsForRunningTasks() return; } - System.out.println("Cleared pending queue."); + log.info("Clearing pending queue."); clearPendingQueue.set(false); } @@ -151,9 +154,7 @@ private synchronized void replaceCompletedTasks() if (specification.getTotalTasks().isPresent() && specificationEnabled.get(specification) && specification.getTotalTasks().getAsInt() <= completedTasks.get(specification).size() + runningTasks.get(specification).size()) { - System.out.println(); - System.out.println(specification.getName() + " disabled for reaching target count " + specification.getTotalTasks()); - System.out.println(); + log.info("\n%s disabled for reaching target count %s\n", specification.getName(), specification.getTotalTasks()); disableSpecification(specification); continue; } From f8882175cdda4c3daec8c867324d6cfb0f071273 Mon Sep 17 00:00:00 2001 From: Raunaq Morarka Date: Fri, 13 Dec 2024 11:33:45 +0530 Subject: [PATCH 064/158] Stream large transaction log jsons instead of storing in-memory Operations fetching metadata and protocol entries can skip reading the rest of the json file after those entries are found --- .../deltalake/BaseTransactionsTable.java | 10 +- .../deltalake/DeltaLakeCommitSummary.java | 47 +++-- .../plugin/deltalake/DeltaLakeConfig.java | 15 ++ .../deltalake/DeltaLakeHistoryTable.java | 14 +- .../plugin/deltalake/DeltaLakeMetadata.java | 26 +-- .../deltalake/DeltaLakeTransactionsTable.java | 6 +- .../tablechanges/TableChangesSplitSource.java | 9 +- .../deltalake/procedure/VacuumProcedure.java | 42 ++-- .../transactionlog/TableSnapshot.java | 40 +++- .../deltalake/transactionlog/Transaction.java | 11 +- .../transactionlog/TransactionLogAccess.java | 157 +++++++------- .../transactionlog/TransactionLogEntries.java | 176 ++++++++++++++++ .../checkpoint/CheckpointWriterManager.java | 2 +- .../MetadataAndProtocolEntries.java | 54 +++++ .../checkpoint/TransactionLogTail.java | 50 ++--- ...LakeAlluxioCacheMutableTransactionLog.java | 4 +- .../deltalake/TestDeltaLakeAnalyze.java | 5 +- .../plugin/deltalake/TestDeltaLakeBasic.java | 44 ++-- .../deltalake/TestDeltaLakeColumnMapping.java | 5 +- .../plugin/deltalake/TestDeltaLakeConfig.java | 3 + .../TestDeltaLakeFileOperations.java | 199 ++++++++++++++---- .../deltalake/TestTransactionLogAccess.java | 122 ++++++----- .../transactionlog/TestTableSnapshot.java | 19 +- .../checkpoint/TestTransactionLogTail.java | 11 +- .../BaseTestDeltaLakeMinioReads.java | 2 +- 25 files changed, 751 insertions(+), 322 deletions(-) create mode 100644 plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogEntries.java create mode 100644 plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/MetadataAndProtocolEntries.java diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/BaseTransactionsTable.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/BaseTransactionsTable.java index f33fb4e84a05..387d1f0d0885 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/BaseTransactionsTable.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/BaseTransactionsTable.java @@ -14,12 +14,13 @@ package io.trino.plugin.deltalake; import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; -import io.trino.plugin.deltalake.transactionlog.DeltaLakeTransactionLogEntry; import io.trino.plugin.deltalake.transactionlog.TableSnapshot; import io.trino.plugin.deltalake.transactionlog.Transaction; import io.trino.plugin.deltalake.transactionlog.TransactionLogAccess; +import io.trino.plugin.deltalake.transactionlog.TransactionLogEntries; import io.trino.plugin.deltalake.util.PageListBuilder; import io.trino.spi.Page; import io.trino.spi.TrinoException; @@ -144,7 +145,7 @@ public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHand PageListBuilder pagesBuilder = PageListBuilder.forTable(tableMetadata); try { List transactions = loadNewTailBackward(fileSystem, tableLocation, startVersionExclusive, endVersionInclusive.get()).reversed(); - return new FixedPageSource(buildPages(session, pagesBuilder, transactions)); + return new FixedPageSource(buildPages(session, pagesBuilder, transactions, fileSystem)); } catch (TrinoException e) { throw e; @@ -170,7 +171,7 @@ private static List loadNewTailBackward( boolean endOfHead = false; while (!endOfHead) { - Optional> results = getEntriesFromJson(entryNumber, transactionLogDir, fileSystem); + Optional results = getEntriesFromJson(entryNumber, transactionLogDir, fileSystem, DataSize.of(0, DataSize.Unit.BYTE)); if (results.isPresent()) { transactionsBuilder.add(new Transaction(version, results.get())); version = entryNumber; @@ -187,5 +188,6 @@ private static List loadNewTailBackward( return transactionsBuilder.build(); } - protected abstract List buildPages(ConnectorSession session, PageListBuilder pagesBuilder, List transactions); + protected abstract List buildPages(ConnectorSession session, PageListBuilder pagesBuilder, List transactions, TrinoFileSystem fileSystem) + throws IOException; } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeCommitSummary.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeCommitSummary.java index 30d8423c9bc7..fddc6afac65c 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeCommitSummary.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeCommitSummary.java @@ -15,15 +15,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import io.trino.filesystem.TrinoFileSystem; import io.trino.plugin.deltalake.transactionlog.CommitInfoEntry; import io.trino.plugin.deltalake.transactionlog.DeltaLakeTransactionLogEntry; import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; +import io.trino.plugin.deltalake.transactionlog.TransactionLogEntries; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.canonicalizePartitionValues; import static java.util.Objects.requireNonNull; @@ -38,7 +42,7 @@ public class DeltaLakeCommitSummary private final Set>> addedFilesCanonicalPartitionValues; private final Optional isBlindAppend; - public DeltaLakeCommitSummary(long version, List transactionLogEntries) + public DeltaLakeCommitSummary(long version, TransactionLogEntries transactionLogEntries, TrinoFileSystem fileSystem) { requireNonNull(transactionLogEntries, "transactionLogEntries is null"); ImmutableList.Builder metadataUpdatesBuilder = ImmutableList.builder(); @@ -48,26 +52,29 @@ public DeltaLakeCommitSummary(long version, List t ImmutableSet.Builder>> removedFilesCanonicalPartitionValuesBuilder = ImmutableSet.builder(); boolean containsRemoveFileWithoutPartitionValues = false; - for (DeltaLakeTransactionLogEntry transactionLogEntry : transactionLogEntries) { - if (transactionLogEntry.getMetaData() != null) { - metadataUpdatesBuilder.add(transactionLogEntry.getMetaData()); - } - else if (transactionLogEntry.getProtocol() != null) { - optionalProtocol = Optional.of(transactionLogEntry.getProtocol()); - } - else if (transactionLogEntry.getCommitInfo() != null) { - optionalCommitInfo = Optional.of(transactionLogEntry.getCommitInfo()); - } - else if (transactionLogEntry.getAdd() != null) { - addedFilesCanonicalPartitionValuesBuilder.add(transactionLogEntry.getAdd().getCanonicalPartitionValues()); - } - else if (transactionLogEntry.getRemove() != null) { - Map partitionValues = transactionLogEntry.getRemove().partitionValues(); - if (partitionValues == null) { - containsRemoveFileWithoutPartitionValues = true; + try (Stream logEntryStream = transactionLogEntries.getEntries(fileSystem)) { + for (Iterator it = logEntryStream.iterator(); it.hasNext(); ) { + DeltaLakeTransactionLogEntry transactionLogEntry = it.next(); + if (transactionLogEntry.getMetaData() != null) { + metadataUpdatesBuilder.add(transactionLogEntry.getMetaData()); + } + else if (transactionLogEntry.getProtocol() != null) { + optionalProtocol = Optional.of(transactionLogEntry.getProtocol()); + } + else if (transactionLogEntry.getCommitInfo() != null) { + optionalCommitInfo = Optional.of(transactionLogEntry.getCommitInfo()); + } + else if (transactionLogEntry.getAdd() != null) { + addedFilesCanonicalPartitionValuesBuilder.add(transactionLogEntry.getAdd().getCanonicalPartitionValues()); } - else { - removedFilesCanonicalPartitionValuesBuilder.add(canonicalizePartitionValues(partitionValues)); + else if (transactionLogEntry.getRemove() != null) { + Map partitionValues = transactionLogEntry.getRemove().partitionValues(); + if (partitionValues == null) { + containsRemoveFileWithoutPartitionValues = true; + } + else { + removedFilesCanonicalPartitionValuesBuilder.add(canonicalizePartitionValues(partitionValues)); + } } } } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java index 870703998c4c..e02a6f5d8cce 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java @@ -50,6 +50,7 @@ public class DeltaLakeConfig { public static final String EXTENDED_STATISTICS_ENABLED = "delta.extended-statistics.enabled"; public static final String VACUUM_MIN_RETENTION = "delta.vacuum.min-retention"; + public static final DataSize DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE = DataSize.of(16, MEGABYTE); // Runtime.getRuntime().maxMemory() is not 100% stable and may return slightly different value over JVM lifetime. We use // constant so default configuration for cache size is stable. @@ -60,6 +61,7 @@ public class DeltaLakeConfig private Duration metadataCacheTtl = new Duration(30, TimeUnit.MINUTES); private DataSize metadataCacheMaxRetainedSize = DEFAULT_METADATA_CACHE_MAX_RETAINED_SIZE; + private DataSize transactionLogMaxCachedFileSize = DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE; private DataSize dataFileCacheSize = DEFAULT_DATA_FILE_CACHE_SIZE; private Duration dataFileCacheTtl = new Duration(30, TimeUnit.MINUTES); private int domainCompactionThreshold = 1000; @@ -121,6 +123,19 @@ public DeltaLakeConfig setMetadataCacheMaxRetainedSize(DataSize metadataCacheMax return this; } + public DataSize getTransactionLogMaxCachedFileSize() + { + return transactionLogMaxCachedFileSize; + } + + @Config("delta.transaction-log.max-cached-file-size") + @ConfigDescription("Maximum size of delta transaction log file that will be cached in memory") + public DeltaLakeConfig setTransactionLogMaxCachedFileSize(DataSize transactionLogMaxCachedFileSize) + { + this.transactionLogMaxCachedFileSize = transactionLogMaxCachedFileSize; + return this; + } + public DataSize getDataFileCacheSize() { return dataFileCacheSize; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeHistoryTable.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeHistoryTable.java index 3c5050dc1bf4..7f048eb4d1cd 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeHistoryTable.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeHistoryTable.java @@ -14,6 +14,7 @@ package io.trino.plugin.deltalake; import com.google.common.collect.ImmutableList; +import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.deltalake.transactionlog.CommitInfoEntry; import io.trino.plugin.deltalake.transactionlog.DeltaLakeTransactionLogEntry; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Objects; +import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.spi.type.BigintType.BIGINT; @@ -74,13 +76,15 @@ public DeltaLakeHistoryTable( } @Override - protected List buildPages(ConnectorSession session, PageListBuilder pagesBuilder, List transactions) + protected List buildPages(ConnectorSession session, PageListBuilder pagesBuilder, List transactions, TrinoFileSystem fileSystem) { - List commitInfoEntries = transactions.stream() - .flatMap(transaction -> transaction.transactionEntries().stream()) + List commitInfoEntries; + try (Stream commitStream = transactions.stream() + .flatMap(transaction -> transaction.transactionEntries().getEntries(fileSystem)) .map(DeltaLakeTransactionLogEntry::getCommitInfo) - .filter(Objects::nonNull) - .collect(toImmutableList()); + .filter(Objects::nonNull)) { + commitInfoEntries = commitStream.collect(toImmutableList()); + } TimeZoneKey timeZoneKey = session.getTimeZoneKey(); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java index 0fd44423d423..658ecec123cf 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java @@ -69,13 +69,14 @@ import io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport; import io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.ColumnMappingMode; import io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.UnsupportedTypeException; -import io.trino.plugin.deltalake.transactionlog.DeltaLakeTransactionLogEntry; import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; import io.trino.plugin.deltalake.transactionlog.RemoveFileEntry; import io.trino.plugin.deltalake.transactionlog.TableSnapshot; import io.trino.plugin.deltalake.transactionlog.TransactionLogAccess; +import io.trino.plugin.deltalake.transactionlog.TransactionLogEntries; import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointWriterManager; +import io.trino.plugin.deltalake.transactionlog.checkpoint.MetadataAndProtocolEntries; import io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeFileStatistics; import io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeJsonFileStatistics; import io.trino.plugin.deltalake.transactionlog.writer.TransactionConflictException; @@ -197,6 +198,7 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Sets.difference; import static com.google.common.primitives.Ints.max; +import static io.airlift.units.DataSize.Unit.BYTE; import static io.trino.filesystem.Locations.appendPath; import static io.trino.filesystem.Locations.areDirectoryLocationsEquivalent; import static io.trino.hive.formats.HiveClassNames.HIVE_SEQUENCEFILE_OUTPUT_FORMAT_CLASS; @@ -290,8 +292,6 @@ import static io.trino.plugin.deltalake.transactionlog.TransactionLogParser.getMandatoryCurrentVersion; import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.getTransactionLogDir; import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.getTransactionLogJsonEntryPath; -import static io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator.EntryType.METADATA; -import static io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator.EntryType.PROTOCOL; import static io.trino.plugin.deltalake.transactionlog.checkpoint.TransactionLogTail.getEntriesFromJson; import static io.trino.plugin.hive.HiveMetadata.TRINO_QUERY_ID_NAME; import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; @@ -630,15 +630,9 @@ public LocatedTableHandle getTableHandle( TrinoFileSystem fileSystem = fileSystemFactory.create(session); TableSnapshot tableSnapshot = getSnapshot(session, tableName, tableLocation, endVersion.map(version -> getVersion(fileSystem, tableLocation, version))); - Map, Object> logEntries; + MetadataAndProtocolEntries logEntries; try { - logEntries = transactionLogAccess.getTransactionLogEntries( - session, - tableSnapshot, - ImmutableSet.of(METADATA, PROTOCOL), - entryStream -> entryStream - .filter(entry -> entry.getMetaData() != null || entry.getProtocol() != null) - .map(entry -> firstNonNull(entry.getMetaData(), entry.getProtocol()))); + logEntries = transactionLogAccess.getMetadataAndProtocolEntry(session, tableSnapshot); } catch (TrinoException e) { if (e.getErrorCode().equals(DELTA_LAKE_INVALID_SCHEMA.toErrorCode())) { @@ -646,11 +640,11 @@ public LocatedTableHandle getTableHandle( } throw e; } - MetadataEntry metadataEntry = (MetadataEntry) logEntries.get(MetadataEntry.class); + MetadataEntry metadataEntry = logEntries.metadata().orElse(null); if (metadataEntry == null) { return new CorruptedDeltaLakeTableHandle(tableName, managed, tableLocation, new TrinoException(DELTA_LAKE_INVALID_SCHEMA, "Metadata not found in transaction log for " + tableSnapshot.getTable())); } - ProtocolEntry protocolEntry = (ProtocolEntry) logEntries.get(ProtocolEntry.class); + ProtocolEntry protocolEntry = logEntries.protocol().orElse(null); if (protocolEntry == null) { return new CorruptedDeltaLakeTableHandle(tableName, managed, tableLocation, new TrinoException(DELTA_LAKE_INVALID_SCHEMA, "Protocol not found in transaction log for " + tableSnapshot.getTable())); } @@ -2267,16 +2261,16 @@ private void checkForConcurrentTransactionConflicts( if (currentVersion > readVersionValue) { String transactionLogDirectory = getTransactionLogDir(tableLocation); for (long version = readVersionValue + 1; version <= currentVersion; version++) { - List transactionLogEntries; + TransactionLogEntries transactionLogEntries; try { long finalVersion = version; - transactionLogEntries = getEntriesFromJson(version, transactionLogDirectory, fileSystem) + transactionLogEntries = getEntriesFromJson(version, transactionLogDirectory, fileSystem, DataSize.of(0, BYTE)) .orElseThrow(() -> new TrinoException(DELTA_LAKE_BAD_DATA, "Delta Lake log entries are missing for version " + finalVersion)); } catch (IOException e) { throw new TrinoException(DELTA_LAKE_FILESYSTEM_ERROR, "Failed to access table metadata", e); } - DeltaLakeCommitSummary commitSummary = new DeltaLakeCommitSummary(version, transactionLogEntries); + DeltaLakeCommitSummary commitSummary = new DeltaLakeCommitSummary(version, transactionLogEntries, fileSystem); checkNoMetadataUpdates(commitSummary); checkNoProtocolUpdates(commitSummary); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeTransactionsTable.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeTransactionsTable.java index b35ce6074618..194481f9abe9 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeTransactionsTable.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeTransactionsTable.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import io.airlift.json.JsonCodec; +import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.deltalake.transactionlog.DeltaLakeTransactionLogEntry; import io.trino.plugin.deltalake.transactionlog.Transaction; @@ -62,12 +63,13 @@ public DeltaLakeTransactionsTable( } @Override - protected List buildPages(ConnectorSession session, PageListBuilder pagesBuilder, List transactions) + protected List buildPages(ConnectorSession session, PageListBuilder pagesBuilder, List transactions, TrinoFileSystem fileSystem) { for (Transaction transaction : transactions) { pagesBuilder.beginRow(); pagesBuilder.appendBigint(transaction.transactionId()); - pagesBuilder.appendVarchar(TRANSACTION_LOG_ENTRIES_CODEC.toJson(transaction.transactionEntries())); + pagesBuilder.appendVarchar(TRANSACTION_LOG_ENTRIES_CODEC.toJson( + transaction.transactionEntries().getEntriesList(fileSystem))); pagesBuilder.endRow(); } return pagesBuilder.build(); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesSplitSource.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesSplitSource.java index 084fe36f0f45..e9056e42ec83 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesSplitSource.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesSplitSource.java @@ -39,6 +39,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; +import static io.trino.plugin.deltalake.DeltaLakeConfig.DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE; import static io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_BAD_DATA; import static io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_FILESYSTEM_ERROR; import static io.trino.plugin.deltalake.functions.tablechanges.TableChangesFileType.CDF_FILE; @@ -74,11 +75,9 @@ private Stream prepareSplits(long currentVersion, long tableRead .boxed() .flatMap(version -> { try { - List entries = getEntriesFromJson(version, transactionLogDir, fileSystem) - .orElseThrow(() -> new TrinoException(DELTA_LAKE_BAD_DATA, "Delta Lake log entries are missing for version " + version)); - if (entries.isEmpty()) { - return ImmutableList.of().stream(); - } + List entries = getEntriesFromJson(version, transactionLogDir, fileSystem, DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE) + .orElseThrow(() -> new TrinoException(DELTA_LAKE_BAD_DATA, "Delta Lake log entries are missing for version " + version)) + .getEntriesList(fileSystem); List commitInfoEntries = entries.stream() .map(DeltaLakeTransactionLogEntry::getCommitInfo) .filter(Objects::nonNull) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/VacuumProcedure.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/VacuumProcedure.java index bc27669c87f7..e84a5d700c29 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/VacuumProcedure.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/VacuumProcedure.java @@ -190,8 +190,7 @@ private void doVacuum( checkUnsupportedUniversalFormat(handle.getMetadataEntry()); - TableSnapshot tableSnapshot = metadata.getSnapshot(session, tableName, handle.getLocation(), Optional.of(handle.getReadVersion())); - ProtocolEntry protocolEntry = transactionLogAccess.getProtocolEntry(session, tableSnapshot); + ProtocolEntry protocolEntry = handle.getProtocolEntry(); if (protocolEntry.minWriterVersion() > MAX_WRITER_VERSION) { throw new TrinoException(NOT_SUPPORTED, "Cannot execute vacuum procedure with %d writer version".formatted(protocolEntry.minWriterVersion())); } @@ -205,6 +204,7 @@ private void doVacuum( throw new TrinoException(NOT_SUPPORTED, "Cannot execute vacuum procedure with %s writer features".formatted(DELETION_VECTORS_FEATURE_NAME)); } + TableSnapshot tableSnapshot = metadata.getSnapshot(session, tableName, handle.getLocation(), Optional.of(handle.getReadVersion())); String tableLocation = tableSnapshot.getTableLocation(); String transactionLogDir = getTransactionLogDir(tableLocation); TrinoFileSystem fileSystem = fileSystemFactory.create(session); @@ -222,24 +222,26 @@ private void doVacuum( handle.getProtocolEntry(), TupleDomain.all(), alwaysFalse())) { - retainedPaths = Stream.concat( - activeAddEntries - // paths can be absolute as well in case of shallow-cloned tables, and they shouldn't be deleted as part of vacuum because according to - // delta-protocol absolute paths are inherited from base table and the vacuum procedure should only list and delete local file references - .map(AddFileEntry::getPath), - transactionLogAccess.getJsonEntries( - fileSystem, - transactionLogDir, - // discard oldest "recent" snapshot, since we take RemoveFileEntry only, to identify files that are no longer - // active files, but still needed to read a "recent" snapshot - recentVersions.stream().sorted(naturalOrder()) - .skip(1) - .collect(toImmutableList())) - .map(DeltaLakeTransactionLogEntry::getRemove) - .filter(Objects::nonNull) - .map(RemoveFileEntry::path)) - .peek(path -> checkState(!path.startsWith(tableLocation), "Unexpected absolute path in transaction log: %s", path)) - .collect(toImmutableSet()); + try (Stream pathEntries = Stream.concat( + activeAddEntries + // paths can be absolute as well in case of shallow-cloned tables, and they shouldn't be deleted as part of vacuum because according to + // delta-protocol absolute paths are inherited from base table and the vacuum procedure should only list and delete local file references + .map(AddFileEntry::getPath), + transactionLogAccess.getJsonEntries( + fileSystem, + transactionLogDir, + // discard oldest "recent" snapshot, since we take RemoveFileEntry only, to identify files that are no longer + // active files, but still needed to read a "recent" snapshot + recentVersions.stream().sorted(naturalOrder()) + .skip(1) + .collect(toImmutableList())) + .map(DeltaLakeTransactionLogEntry::getRemove) + .filter(Objects::nonNull) + .map(RemoveFileEntry::path))) { + retainedPaths = pathEntries + .peek(path -> checkState(!path.startsWith(tableLocation), "Unexpected absolute path in transaction log: %s", path)) + .collect(toImmutableSet()); + } } log.debug( diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java index 203861cfe011..16e5bd1d29ee 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import io.airlift.units.DataSize; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoInputFile; @@ -70,10 +71,12 @@ public class TableSnapshot private final TransactionLogTail logTail; private final String tableLocation; private final ParquetReaderOptions parquetReaderOptions; + private final DataSize transactionLogMaxCachedFileSize; private final boolean checkpointRowStatisticsWritingEnabled; private final int domainCompactionThreshold; private Optional cachedMetadata = Optional.empty(); + private Optional cachedProtocol = Optional.empty(); private TableSnapshot( SchemaTableName table, @@ -82,7 +85,8 @@ private TableSnapshot( String tableLocation, ParquetReaderOptions parquetReaderOptions, boolean checkpointRowStatisticsWritingEnabled, - int domainCompactionThreshold) + int domainCompactionThreshold, + DataSize transactionLogMaxCachedFileSize) { this.table = requireNonNull(table, "table is null"); this.lastCheckpoint = requireNonNull(lastCheckpoint, "lastCheckpoint is null"); @@ -91,6 +95,7 @@ private TableSnapshot( this.parquetReaderOptions = requireNonNull(parquetReaderOptions, "parquetReaderOptions is null"); this.checkpointRowStatisticsWritingEnabled = checkpointRowStatisticsWritingEnabled; this.domainCompactionThreshold = domainCompactionThreshold; + this.transactionLogMaxCachedFileSize = requireNonNull(transactionLogMaxCachedFileSize, "transactionLogMaxCachedFileSize is null"); } public static TableSnapshot load( @@ -101,11 +106,12 @@ public static TableSnapshot load( ParquetReaderOptions parquetReaderOptions, boolean checkpointRowStatisticsWritingEnabled, int domainCompactionThreshold, + DataSize transactionLogMaxCachedFileSize, Optional endVersion) throws IOException { Optional lastCheckpointVersion = lastCheckpoint.map(LastCheckpoint::version); - TransactionLogTail transactionLogTail = TransactionLogTail.loadNewTail(fileSystem, tableLocation, lastCheckpointVersion, endVersion); + TransactionLogTail transactionLogTail = TransactionLogTail.loadNewTail(fileSystem, tableLocation, lastCheckpointVersion, endVersion, transactionLogMaxCachedFileSize); return new TableSnapshot( table, @@ -114,7 +120,8 @@ public static TableSnapshot load( tableLocation, parquetReaderOptions, checkpointRowStatisticsWritingEnabled, - domainCompactionThreshold); + domainCompactionThreshold, + transactionLogMaxCachedFileSize); } public Optional getUpdatedSnapshot(TrinoFileSystem fileSystem, Optional toVersion) @@ -136,12 +143,13 @@ public Optional getUpdatedSnapshot(TrinoFileSystem fileSystem, Op parquetReaderOptions, checkpointRowStatisticsWritingEnabled, domainCompactionThreshold, + transactionLogMaxCachedFileSize, Optional.empty())); } } } - Optional updatedLogTail = logTail.getUpdatedTail(fileSystem, tableLocation, toVersion); + Optional updatedLogTail = logTail.getUpdatedTail(fileSystem, tableLocation, toVersion, transactionLogMaxCachedFileSize); return updatedLogTail.map(transactionLogTail -> new TableSnapshot( table, lastCheckpoint, @@ -149,7 +157,8 @@ public Optional getUpdatedSnapshot(TrinoFileSystem fileSystem, Op tableLocation, parquetReaderOptions, checkpointRowStatisticsWritingEnabled, - domainCompactionThreshold)); + domainCompactionThreshold, + transactionLogMaxCachedFileSize)); } public long getVersion() @@ -167,6 +176,11 @@ public Optional getCachedMetadata() return cachedMetadata; } + public Optional getCachedProtocol() + { + return cachedProtocol; + } + public String getTableLocation() { return tableLocation; @@ -177,9 +191,14 @@ public void setCachedMetadata(Optional cachedMetadata) this.cachedMetadata = cachedMetadata; } - public List getJsonTransactionLogEntries() + public void setCachedProtocol(Optional cachedProtocol) + { + this.cachedProtocol = cachedProtocol; + } + + public List getJsonTransactionLogEntries(TrinoFileSystem fileSystem) { - return logTail.getFileEntries(); + return logTail.getFileEntries(fileSystem); } public List getTransactions() @@ -194,7 +213,8 @@ public long getRetainedSizeInBytes() + table.getRetainedSizeInBytes() + logTail.getRetainedSizeInBytes() + estimatedSizeOf(tableLocation) - + sizeOf(cachedMetadata, MetadataEntry::getRetainedSizeInBytes); + + sizeOf(cachedMetadata, MetadataEntry::getRetainedSizeInBytes) + + sizeOf(cachedProtocol, ProtocolEntry::getRetainedSizeInBytes); } public Stream getCheckpointTransactionLogEntries( @@ -360,7 +380,9 @@ private Stream getV2CheckpointEntries( { if (checkpointFile.location().fileName().endsWith(".json")) { try { - return getEntriesFromJson(checkpoint.version(), checkpointFile).stream().flatMap(List::stream); + return getEntriesFromJson(checkpoint.version(), checkpointFile, transactionLogMaxCachedFileSize) + .stream() + .flatMap(logEntries -> logEntries.getEntries(fileSystem)); } catch (IOException e) { throw new TrinoException(DELTA_LAKE_FILESYSTEM_ERROR, format("Unexpected IO exception occurred while reading the entries of the file: %s for the table %s", checkpoint, table), e); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/Transaction.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/Transaction.java index ebfa2b305d4f..d5c3b0b8dcf9 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/Transaction.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/Transaction.java @@ -13,30 +13,25 @@ */ package io.trino.plugin.deltalake.transactionlog; -import com.google.common.collect.ImmutableList; - -import java.util.List; - import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.slice.SizeOf.SIZE_OF_LONG; -import static io.airlift.slice.SizeOf.estimatedSizeOf; import static io.airlift.slice.SizeOf.instanceSize; import static java.util.Objects.requireNonNull; -public record Transaction(long transactionId, List transactionEntries) +public record Transaction(long transactionId, TransactionLogEntries transactionEntries) { private static final int INSTANCE_SIZE = instanceSize(Transaction.class); public Transaction { checkArgument(transactionId >= 0, "transactionId must be >= 0"); - transactionEntries = ImmutableList.copyOf(requireNonNull(transactionEntries, "transactionEntries is null")); + requireNonNull(transactionEntries, "transactionEntries is null"); } public long getRetainedSizeInBytes() { return INSTANCE_SIZE + SIZE_OF_LONG - + estimatedSizeOf(transactionEntries, DeltaLakeTransactionLogEntry::getRetainedSizeInBytes); + + transactionEntries.getRetainedSizeInBytes(); } } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java index a2b76fa3e4dc..19a730e9efe8 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogAccess.java @@ -20,6 +20,7 @@ import com.google.common.primitives.Ints; import com.google.common.util.concurrent.UncheckedExecutionException; import com.google.inject.Inject; +import io.airlift.units.DataSize; import io.trino.cache.CacheStatsMBean; import io.trino.cache.EvictableCacheBuilder; import io.trino.filesystem.FileEntry; @@ -37,6 +38,7 @@ import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator; import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointSchemaManager; import io.trino.plugin.deltalake.transactionlog.checkpoint.LastCheckpoint; +import io.trino.plugin.deltalake.transactionlog.checkpoint.MetadataAndProtocolEntries; import io.trino.plugin.deltalake.transactionlog.checkpoint.TransactionLogTail; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.spi.TrinoException; @@ -56,7 +58,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.time.Instant; -import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; @@ -92,7 +93,6 @@ import static io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator.EntryType.ADD; import static io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator.EntryType.METADATA; import static io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator.EntryType.PROTOCOL; -import static io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator.EntryType.REMOVE; import static io.trino.plugin.deltalake.transactionlog.checkpoint.TransactionLogTail.getEntriesFromJson; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -110,6 +110,7 @@ public class TransactionLogAccess private final ParquetReaderOptions parquetReaderOptions; private final boolean checkpointRowStatisticsWritingEnabled; private final int domainCompactionThreshold; + private final DataSize transactionLogMaxCachedFileSize; private final Cache tableSnapshots; private final Cache activeDataFileCache; @@ -130,6 +131,7 @@ public TransactionLogAccess( this.parquetReaderOptions = parquetReaderConfig.toParquetReaderOptions().withBloomFilter(false); this.checkpointRowStatisticsWritingEnabled = deltaLakeConfig.isCheckpointRowStatisticsWritingEnabled(); this.domainCompactionThreshold = deltaLakeConfig.getDomainCompactionThreshold(); + this.transactionLogMaxCachedFileSize = deltaLakeConfig.getTransactionLogMaxCachedFileSize(); tableSnapshots = EvictableCacheBuilder.newBuilder() .weigher((Weigher) (key, value) -> Ints.saturatedCast(key.getRetainedSizeInBytes() + value.getRetainedSizeInBytes())) @@ -184,6 +186,7 @@ public TableSnapshot loadSnapshot(ConnectorSession session, SchemaTableName tabl parquetReaderOptions, checkpointRowStatisticsWritingEnabled, domainCompactionThreshold, + transactionLogMaxCachedFileSize, endVersion)); } catch (UncheckedExecutionException | ExecutionException e) { @@ -215,6 +218,7 @@ private TableSnapshot loadSnapshotForTimeTravel(TrinoFileSystem fileSystem, Sche parquetReaderOptions, checkpointRowStatisticsWritingEnabled, domainCompactionThreshold, + transactionLogMaxCachedFileSize, Optional.of(endVersion)); } @@ -299,12 +303,21 @@ public void invalidateCache(SchemaTableName schemaTableName, Optional ta public MetadataEntry getMetadataEntry(ConnectorSession session, TableSnapshot tableSnapshot) { if (tableSnapshot.getCachedMetadata().isEmpty()) { + TrinoFileSystem fileSystem = fileSystemFactory.create(session); try (Stream metadataEntries = getEntries( session, tableSnapshot, - METADATA, - entryStream -> entryStream.map(DeltaLakeTransactionLogEntry::getMetaData).filter(Objects::nonNull), - fileSystemFactory.create(session), + ImmutableSet.of(METADATA), + (checkpointStream, jsonTransactions) -> + Stream.concat( + checkpointStream + .map(DeltaLakeTransactionLogEntry::getMetaData) + .filter(Objects::nonNull), + jsonTransactions.stream() + .map(transaction -> transaction.transactionEntries().getMetadataAndProtocol(fileSystem)) + .filter(entry -> entry.metadata().isPresent()) + .map(entry -> entry.metadata().get())), + fileSystem, fileFormatDataSourceStats)) { // Get last entry in the stream tableSnapshot.setCachedMetadata(metadataEntries.reduce((first, second) -> second)); @@ -391,17 +404,18 @@ public Stream loadActiveFiles( Predicate addStatsMinMaxColumnFilter) { List transactions = tableSnapshot.getTransactions(); + TrinoFileSystem fileSystem = fileSystemFactory.create(session); try (Stream checkpointEntries = tableSnapshot.getCheckpointTransactionLogEntries( session, ImmutableSet.of(ADD), checkpointSchemaManager, typeManager, - fileSystemFactory.create(session), + fileSystem, fileFormatDataSourceStats, Optional.of(new MetadataAndProtocolEntry(metadataEntry, protocolEntry)), partitionConstraint, Optional.of(addStatsMinMaxColumnFilter))) { - return activeAddEntries(checkpointEntries, transactions) + return activeAddEntries(checkpointEntries, transactions, fileSystem) .filter(partitionConstraint.isAll() ? addAction -> true : addAction -> partitionMatchesPredicate(addAction.getCanonicalPartitionValues(), partitionConstraint.getDomains().orElseThrow())); @@ -427,7 +441,7 @@ public static ImmutableList columnsWithStats(List activeAddEntries(Stream checkpointEntries, List transactions) + private Stream activeAddEntries(Stream checkpointEntries, List transactions, TrinoFileSystem fileSystem) { Map activeJsonEntries = new LinkedHashMap<>(); HashSet removedFiles = new HashSet<>(); @@ -438,16 +452,18 @@ private Stream activeAddEntries(Stream { Map addFilesInTransaction = new LinkedHashMap<>(); Set removedFilesInTransaction = new HashSet<>(); - transaction.transactionEntries().forEach(deltaLakeTransactionLogEntry -> { - if (deltaLakeTransactionLogEntry.getAdd() != null) { - AddFileEntry add = deltaLakeTransactionLogEntry.getAdd(); - addFilesInTransaction.put(new FileEntryKey(add.getPath(), add.getDeletionVector().map(DeletionVectorEntry::uniqueId)), add); - } - else if (deltaLakeTransactionLogEntry.getRemove() != null) { - RemoveFileEntry remove = deltaLakeTransactionLogEntry.getRemove(); - removedFilesInTransaction.add(new FileEntryKey(remove.path(), remove.deletionVector().map(DeletionVectorEntry::uniqueId))); - } - }); + try (Stream entries = transaction.transactionEntries().getEntries(fileSystem)) { + entries.forEach(deltaLakeTransactionLogEntry -> { + if (deltaLakeTransactionLogEntry.getAdd() != null) { + AddFileEntry add = deltaLakeTransactionLogEntry.getAdd(); + addFilesInTransaction.put(new FileEntryKey(add.getPath(), add.getDeletionVector().map(DeletionVectorEntry::uniqueId)), add); + } + else if (deltaLakeTransactionLogEntry.getRemove() != null) { + RemoveFileEntry remove = deltaLakeTransactionLogEntry.getRemove(); + removedFilesInTransaction.add(new FileEntryKey(remove.path(), remove.deletionVector().map(DeletionVectorEntry::uniqueId))); + } + }); + } // Process 'remove' entries first because deletion vectors register both 'add' and 'remove' entries and the 'add' entry should be kept removedFiles.addAll(removedFilesInTransaction); @@ -468,50 +484,62 @@ else if (deltaLakeTransactionLogEntry.getRemove() != null) { private record FileEntryKey(String path, Optional deletionVectorId) {} - public Stream getRemoveEntries(ConnectorSession session, TableSnapshot tableSnapshot) - { - return getEntries( - session, - tableSnapshot, - REMOVE, - entryStream -> entryStream.map(DeltaLakeTransactionLogEntry::getRemove).filter(Objects::nonNull), - fileSystemFactory.create(session), - fileFormatDataSourceStats); - } - - public Map, Object> getTransactionLogEntries( - ConnectorSession session, - TableSnapshot tableSnapshot, - Set entryTypes, - Function, Stream> entryMapper) + public MetadataAndProtocolEntries getMetadataAndProtocolEntry(ConnectorSession session, TableSnapshot tableSnapshot) { - try (Stream entries = getEntries( - session, - tableSnapshot, - entryTypes, - (checkpointStream, jsonStream) -> entryMapper.apply(Stream.concat(checkpointStream, jsonStream.stream().map(Transaction::transactionEntries).flatMap(Collection::stream))), - fileSystemFactory.create(session), - fileFormatDataSourceStats)) { - return entries.collect(toImmutableMap(Object::getClass, Function.identity(), (first, second) -> second)); + if (tableSnapshot.getCachedMetadata().isEmpty() || tableSnapshot.getCachedProtocol().isEmpty()) { + TrinoFileSystem fileSystem = fileSystemFactory.create(session); + try (Stream entries = getEntries( + session, + tableSnapshot, + ImmutableSet.of(METADATA, PROTOCOL), + (checkpointStream, jsonTransactions) -> + Stream.concat( + checkpointStream + .filter(entry -> entry.getMetaData() != null || entry.getProtocol() != null) + .map(entry -> new MetadataAndProtocolEntries(entry.getMetaData(), entry.getProtocol())), + jsonTransactions.stream() + .map(transaction -> transaction.transactionEntries().getMetadataAndProtocol(fileSystem))), + fileSystem, + fileFormatDataSourceStats)) { + Map, Object> logEntries = entries + .flatMap(MetadataAndProtocolEntries::stream) + .collect(toImmutableMap(Object::getClass, Function.identity(), (_, second) -> second)); + tableSnapshot.setCachedMetadata(Optional.ofNullable((MetadataEntry) logEntries.get(MetadataEntry.class))); + tableSnapshot.setCachedProtocol(Optional.ofNullable((ProtocolEntry) logEntries.get(ProtocolEntry.class))); + } } + return new MetadataAndProtocolEntries(tableSnapshot.getCachedMetadata(), tableSnapshot.getCachedProtocol()); } public ProtocolEntry getProtocolEntry(ConnectorSession session, TableSnapshot tableSnapshot) { - try (Stream protocolEntries = getProtocolEntries(session, tableSnapshot)) { - return protocolEntries.reduce((first, second) -> second) - .orElseThrow(() -> new TrinoException(DELTA_LAKE_INVALID_SCHEMA, "Protocol entry not found in transaction log for table " + tableSnapshot.getTable())); + if (tableSnapshot.getCachedProtocol().isEmpty()) { + try (Stream protocolEntries = getProtocolEntries(session, tableSnapshot)) { + // Get last entry in the stream + tableSnapshot.setCachedProtocol(protocolEntries.reduce((first, second) -> second)); + } } + return tableSnapshot.getCachedProtocol() + .orElseThrow(() -> new TrinoException(DELTA_LAKE_INVALID_SCHEMA, "Protocol entry not found in transaction log for table " + tableSnapshot.getTable())); } public Stream getProtocolEntries(ConnectorSession session, TableSnapshot tableSnapshot) { + TrinoFileSystem fileSystem = fileSystemFactory.create(session); return getEntries( session, tableSnapshot, - PROTOCOL, - entryStream -> entryStream.map(DeltaLakeTransactionLogEntry::getProtocol).filter(Objects::nonNull), - fileSystemFactory.create(session), + ImmutableSet.of(PROTOCOL), + (checkpointStream, jsonTransactions) -> + Stream.concat( + checkpointStream + .map(DeltaLakeTransactionLogEntry::getProtocol) + .filter(Objects::nonNull), + jsonTransactions.stream() + .map(transaction -> transaction.transactionEntries().getMetadataAndProtocol(fileSystem)) + .filter(entry -> entry.protocol().isPresent()) + .map(entry -> entry.protocol().get())), + fileSystem, fileFormatDataSourceStats); } @@ -550,34 +578,13 @@ private Stream getEntries( } } - /** - * Convenience method for accessors which don't need to separate out the checkpoint entries from the json entries. - */ - private Stream getEntries( - ConnectorSession session, - TableSnapshot tableSnapshot, - CheckpointEntryIterator.EntryType entryType, - Function, Stream> entryMapper, - TrinoFileSystem fileSystem, - FileFormatDataSourceStats stats) - { - return getEntries( - session, - tableSnapshot, - ImmutableSet.of(entryType), - (checkpointStream, jsonStream) -> entryMapper.apply(Stream.concat(checkpointStream, jsonStream.stream().map(Transaction::transactionEntries).flatMap(Collection::stream))), - fileSystem, - stats); - } - public Stream getJsonEntries(TrinoFileSystem fileSystem, String transactionLogDir, List forVersions) { return forVersions.stream() .flatMap(version -> { try { - Optional> entriesFromJson = getEntriesFromJson(version, transactionLogDir, fileSystem); - //noinspection SimplifyOptionalCallChains - return entriesFromJson.map(List::stream) + Optional entriesFromJson = getEntriesFromJson(version, transactionLogDir, fileSystem, transactionLogMaxCachedFileSize); + return entriesFromJson.map(entries -> entries.getEntries(fileSystem)) // transaction log does not exist. Might have been expired. .orElseGet(Stream::of); } @@ -614,17 +621,17 @@ public List getPastTableVersions(TrinoFileSystem fileSystem, String transa return result.build(); } - private static List getJsonEntries(long startVersion, long endVersion, TableSnapshot tableSnapshot, TrinoFileSystem fileSystem) + private List getJsonEntries(long startVersion, long endVersion, TableSnapshot tableSnapshot, TrinoFileSystem fileSystem) throws IOException { Optional lastCheckpointVersion = tableSnapshot.getLastCheckpointVersion(); if (lastCheckpointVersion.isPresent() && startVersion < lastCheckpointVersion.get()) { return ImmutableList.builder() - .addAll(TransactionLogTail.loadNewTail(fileSystem, tableSnapshot.getTableLocation(), Optional.of(startVersion), lastCheckpointVersion).getFileEntries()) - .addAll(tableSnapshot.getJsonTransactionLogEntries()) + .addAll(TransactionLogTail.loadNewTail(fileSystem, tableSnapshot.getTableLocation(), Optional.of(startVersion), lastCheckpointVersion, transactionLogMaxCachedFileSize).getFileEntries(fileSystem)) + .addAll(tableSnapshot.getJsonTransactionLogEntries(fileSystem)) .build(); } - return TransactionLogTail.loadNewTail(fileSystem, tableSnapshot.getTableLocation(), Optional.of(startVersion), Optional.of(endVersion)).getFileEntries(); + return TransactionLogTail.loadNewTail(fileSystem, tableSnapshot.getTableLocation(), Optional.of(startVersion), Optional.of(endVersion), transactionLogMaxCachedFileSize).getFileEntries(fileSystem); } public static String canonicalizeColumnName(String columnName) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogEntries.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogEntries.java new file mode 100644 index 000000000000..55038dbe9c52 --- /dev/null +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TransactionLogEntries.java @@ -0,0 +1,176 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.deltalake.transactionlog; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoInputFile; +import io.trino.filesystem.TrinoInputStream; +import io.trino.plugin.deltalake.transactionlog.checkpoint.MetadataAndProtocolEntries; +import io.trino.spi.TrinoException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Streams.stream; +import static io.airlift.slice.SizeOf.SIZE_OF_LONG; +import static io.airlift.slice.SizeOf.estimatedSizeOf; +import static io.airlift.slice.SizeOf.instanceSize; +import static io.airlift.slice.SizeOf.sizeOf; +import static io.trino.plugin.deltalake.transactionlog.TransactionLogParser.parseJson; +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static java.nio.charset.StandardCharsets.UTF_8; + +public final class TransactionLogEntries +{ + private static final int INSTANCE_SIZE = instanceSize(TransactionLogEntries.class); + private static final int JSON_LOG_ENTRY_READ_BUFFER_SIZE = 1024 * 1024; + + private final long entryNumber; + private final Location transactionLogFilePath; + + private final Optional> cachedEntries; + + public TransactionLogEntries(long entryNumber, TrinoInputFile inputFile, DataSize maxCachedFileSize) + { + this.entryNumber = entryNumber; + this.transactionLogFilePath = inputFile.location(); + try { + if (inputFile.length() > maxCachedFileSize.toBytes()) { + this.cachedEntries = Optional.empty(); + } + else { + this.cachedEntries = Optional.of(ImmutableList.copyOf(new TransactionLogEntryIterator(entryNumber, inputFile))); + } + } + catch (IOException e) { + throw new TrinoException(GENERIC_INTERNAL_ERROR, "Error while reading from transaction entry iterator for the file %s".formatted(transactionLogFilePath)); + } + } + + /** + * Returns a stream of DeltaLakeTransactionLogEntry + * Caller has the responsibility to close this stream as it potentially holds an open file + */ + public Stream getEntries(TrinoFileSystem fileSystem) + { + if (cachedEntries.isPresent()) { + return cachedEntries.get().stream(); + } + TransactionLogEntryIterator iterator = new TransactionLogEntryIterator(entryNumber, fileSystem.newInputFile(transactionLogFilePath)); + return stream(iterator).onClose(iterator::close); + } + + public List getEntriesList(TrinoFileSystem fileSystem) + { + try (Stream jsonStream = getEntries(fileSystem)) { + return jsonStream.collect(toImmutableList()); + } + } + + public MetadataAndProtocolEntries getMetadataAndProtocol(TrinoFileSystem fileSystem) + { + // There can be at most one metadata and protocol entry per transaction log + // We use that stop reading from file when a metadata and protocol entry are found + try (Stream logEntryStream = getEntries(fileSystem)) { + Optional metadataEntry = Optional.empty(); + Optional protocolEntry = Optional.empty(); + for (Iterator it = logEntryStream.iterator(); it.hasNext(); ) { + DeltaLakeTransactionLogEntry transactionLogEntry = it.next(); + if (transactionLogEntry.getMetaData() != null) { + metadataEntry = Optional.of(transactionLogEntry.getMetaData()); + } + else if (transactionLogEntry.getProtocol() != null) { + protocolEntry = Optional.of(transactionLogEntry.getProtocol()); + } + + if (protocolEntry.isPresent() && metadataEntry.isPresent()) { + break; + } + } + return new MetadataAndProtocolEntries(metadataEntry, protocolEntry); + } + } + + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + SIZE_OF_LONG + + estimatedSizeOf(transactionLogFilePath.path()) + + sizeOf(cachedEntries, entries -> estimatedSizeOf(entries, DeltaLakeTransactionLogEntry::getRetainedSizeInBytes)); + } + + private static final class TransactionLogEntryIterator + extends AbstractIterator + { + private final long entryNumber; + private final Location location; + private final BufferedReader reader; + + public TransactionLogEntryIterator(long entryNumber, TrinoInputFile inputFile) + { + this.entryNumber = entryNumber; + this.location = inputFile.location(); + TrinoInputStream inputStream; + try { + inputStream = inputFile.newStream(); + } + catch (Exception e) { + throw new TrinoException(GENERIC_INTERNAL_ERROR, "Error while initializing the transaction entry iterator for the file %s".formatted(inputFile.location())); + } + this.reader = new BufferedReader(new InputStreamReader(inputStream, UTF_8), JSON_LOG_ENTRY_READ_BUFFER_SIZE); + } + + @Override + protected DeltaLakeTransactionLogEntry computeNext() + { + String line; + try { + line = reader.readLine(); + } + catch (IOException e) { + throw new TrinoException(GENERIC_INTERNAL_ERROR, "Error while reading from transaction entry iterator for the file %s".formatted(location)); + } + if (line == null) { + close(); + return endOfData(); + } + DeltaLakeTransactionLogEntry deltaLakeTransactionLogEntry = parseJson(line); + if (deltaLakeTransactionLogEntry.getCommitInfo() != null && deltaLakeTransactionLogEntry.getCommitInfo().version() == 0L) { + // In case that the commit info version is missing, use the version from the transaction log file name + deltaLakeTransactionLogEntry = deltaLakeTransactionLogEntry.withCommitInfo(deltaLakeTransactionLogEntry.getCommitInfo().withVersion(entryNumber)); + } + return deltaLakeTransactionLogEntry; + } + + public void close() + { + try { + reader.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java index 5403ad33628d..a1d9050047f6 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java @@ -147,7 +147,7 @@ public void writeCheckpoint(ConnectorSession session, TableSnapshot snapshot) } } - snapshot.getJsonTransactionLogEntries() + snapshot.getJsonTransactionLogEntries(fileSystem) .forEach(checkpointBuilder::addLogEntry); Location transactionLogDir = Location.of(getTransactionLogDir(snapshot.getTableLocation())); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/MetadataAndProtocolEntries.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/MetadataAndProtocolEntries.java new file mode 100644 index 000000000000..d4f1fdbd743f --- /dev/null +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/MetadataAndProtocolEntries.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.deltalake.transactionlog.checkpoint; + +import io.trino.plugin.deltalake.transactionlog.MetadataEntry; +import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; + +import java.util.Optional; +import java.util.stream.Stream; + +import static io.airlift.slice.SizeOf.instanceSize; +import static io.airlift.slice.SizeOf.sizeOf; + +public record MetadataAndProtocolEntries(Optional metadata, Optional protocol) +{ + private static final int INSTANCE_SIZE = instanceSize(MetadataAndProtocolEntries.class); + + public MetadataAndProtocolEntries(MetadataEntry metadata, ProtocolEntry protocol) + { + this(Optional.ofNullable(metadata), Optional.ofNullable(protocol)); + } + + public Stream stream() + { + if (metadata.isPresent() && protocol.isPresent()) { + return Stream.of(metadata.get(), protocol.get()); + } + if (metadata.isPresent()) { + return Stream.of(metadata.get()); + } + if (protocol.isPresent()) { + return Stream.of(protocol.get()); + } + return Stream.of(); + } + + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + sizeOf(metadata, MetadataEntry::getRetainedSizeInBytes) + + sizeOf(protocol, ProtocolEntry::getRetainedSizeInBytes); + } +} diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TransactionLogTail.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TransactionLogTail.java index ec3e8c3b973a..6f63a9efa013 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TransactionLogTail.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TransactionLogTail.java @@ -14,18 +14,17 @@ package io.trino.plugin.deltalake.transactionlog.checkpoint; import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoInputFile; import io.trino.plugin.deltalake.transactionlog.DeltaLakeTransactionLogEntry; import io.trino.plugin.deltalake.transactionlog.MissingTransactionLogException; import io.trino.plugin.deltalake.transactionlog.Transaction; +import io.trino.plugin.deltalake.transactionlog.TransactionLogEntries; -import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Collection; import java.util.List; import java.util.Optional; @@ -34,16 +33,13 @@ import static io.airlift.slice.SizeOf.SIZE_OF_LONG; import static io.airlift.slice.SizeOf.estimatedSizeOf; import static io.airlift.slice.SizeOf.instanceSize; -import static io.trino.plugin.deltalake.transactionlog.TransactionLogParser.parseJson; import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.getTransactionLogDir; import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.getTransactionLogJsonEntryPath; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; public class TransactionLogTail { private static final int INSTANCE_SIZE = instanceSize(TransactionLogTail.class); - private static final int JSON_LOG_ENTRY_READ_BUFFER_SIZE = 1024 * 1024; private final List entries; private final long version; @@ -59,7 +55,8 @@ public static TransactionLogTail loadNewTail( TrinoFileSystem fileSystem, String tableLocation, Optional startVersion, - Optional endVersion) + Optional endVersion, + DataSize transactionLogMaxCachedFileSize) throws IOException { ImmutableList.Builder entriesBuilder = ImmutableList.builder(); @@ -74,11 +71,10 @@ public static TransactionLogTail loadNewTail( checkArgument(endVersion.isEmpty() || entryNumber <= endVersion.get(), "Invalid start/end versions: %s, %s", startVersion, endVersion); String transactionLogDir = getTransactionLogDir(tableLocation); - Optional> results; boolean endOfTail = false; while (!endOfTail) { - results = getEntriesFromJson(entryNumber, transactionLogDir, fileSystem); + Optional results = getEntriesFromJson(entryNumber, transactionLogDir, fileSystem, transactionLogMaxCachedFileSize); if (results.isPresent()) { entriesBuilder.add(new Transaction(entryNumber, results.get())); version = entryNumber; @@ -99,11 +95,11 @@ public static TransactionLogTail loadNewTail( return new TransactionLogTail(entriesBuilder.build(), version); } - public Optional getUpdatedTail(TrinoFileSystem fileSystem, String tableLocation, Optional endVersion) + public Optional getUpdatedTail(TrinoFileSystem fileSystem, String tableLocation, Optional endVersion, DataSize transactionLogMaxCachedFileSize) throws IOException { checkArgument(endVersion.isEmpty() || endVersion.get() > version, "Invalid endVersion, expected higher than %s, but got %s", version, endVersion); - TransactionLogTail newTail = loadNewTail(fileSystem, tableLocation, Optional.of(version), endVersion); + TransactionLogTail newTail = loadNewTail(fileSystem, tableLocation, Optional.of(version), endVersion, transactionLogMaxCachedFileSize); if (newTail.version == version) { return Optional.empty(); } @@ -115,42 +111,32 @@ public Optional getUpdatedTail(TrinoFileSystem fileSystem, S newTail.version)); } - public static Optional> getEntriesFromJson(long entryNumber, String transactionLogDir, TrinoFileSystem fileSystem) + public static Optional getEntriesFromJson(long entryNumber, String transactionLogDir, TrinoFileSystem fileSystem, DataSize transactionLogMaxCachedFileSize) throws IOException { Location transactionLogFilePath = getTransactionLogJsonEntryPath(transactionLogDir, entryNumber); TrinoInputFile inputFile = fileSystem.newInputFile(transactionLogFilePath); - return getEntriesFromJson(entryNumber, inputFile); + return getEntriesFromJson(entryNumber, inputFile, transactionLogMaxCachedFileSize); } - public static Optional> getEntriesFromJson(long entryNumber, TrinoInputFile inputFile) + public static Optional getEntriesFromJson(long entryNumber, TrinoInputFile inputFile, DataSize transactionLogMaxCachedFileSize) throws IOException { - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(inputFile.newStream(), UTF_8), - JSON_LOG_ENTRY_READ_BUFFER_SIZE)) { - ImmutableList.Builder resultsBuilder = ImmutableList.builder(); - String line = reader.readLine(); - while (line != null) { - DeltaLakeTransactionLogEntry deltaLakeTransactionLogEntry = parseJson(line); - if (deltaLakeTransactionLogEntry.getCommitInfo() != null && deltaLakeTransactionLogEntry.getCommitInfo().version() == 0L) { - // In case that the commit info version is missing, use the version from the transaction log file name - deltaLakeTransactionLogEntry = deltaLakeTransactionLogEntry.withCommitInfo(deltaLakeTransactionLogEntry.getCommitInfo().withVersion(entryNumber)); - } - resultsBuilder.add(deltaLakeTransactionLogEntry); - line = reader.readLine(); - } - - return Optional.of(resultsBuilder.build()); + try { + inputFile.length(); // File length is cached and used in TransactionLogEntries } catch (FileNotFoundException e) { return Optional.empty(); // end of tail } + return Optional.of(new TransactionLogEntries(entryNumber, inputFile, transactionLogMaxCachedFileSize)); } - public List getFileEntries() + public List getFileEntries(TrinoFileSystem fileSystem) { - return entries.stream().map(Transaction::transactionEntries).flatMap(Collection::stream).collect(toImmutableList()); + return entries.stream() + .map(Transaction::transactionEntries) + .flatMap(logEntries -> logEntries.getEntriesList(fileSystem).stream()) + .collect(toImmutableList()); } public List getTransactions() diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java index 697543903322..910c3657acfd 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java @@ -77,7 +77,7 @@ public void testTableDataCachedWhileTransactionLogNotCached() ImmutableMultiset.builder() .addCopies(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000002.checkpoint.parquet"), 2) .addCopies(new CacheFileSystemTraceUtils.CacheOperation("Input.readTail", "00000000000000000002.checkpoint.parquet"), 2) - .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "00000000000000000003.json")) + .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "_last_checkpoint")) .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 220)) .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 220)) @@ -91,7 +91,7 @@ public void testTableDataCachedWhileTransactionLogNotCached() ImmutableMultiset.builder() .addCopies(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000002.checkpoint.parquet"), 2) .addCopies(new CacheFileSystemTraceUtils.CacheOperation("Input.readTail", "00000000000000000002.checkpoint.parquet"), 2) - .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "00000000000000000003.json")) + .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "_last_checkpoint")) .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 220)) .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 220)) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java index bf5ccba08a1d..ac3f29b7413e 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java @@ -42,6 +42,7 @@ import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.testing.Closeables.closeAllSuppress; +import static io.trino.plugin.deltalake.DeltaLakeConfig.DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.EXTENDED_STATISTICS_COLLECT_ON_WRITE; import static io.trino.plugin.deltalake.DeltaTestingConnectorSession.SESSION; import static io.trino.plugin.deltalake.TestingDeltaLakeUtils.copyDirectoryContents; @@ -1009,7 +1010,9 @@ public void testNoColumnStatsMixedCase() """); // Version 3 should be created with recalculated statistics. - List transactionLogAfterUpdate = getEntriesFromJson(3, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow(); + List transactionLogAfterUpdate = getEntriesFromJson(3, tableLocation + "/_delta_log", FILE_SYSTEM, DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE) + .orElseThrow() + .getEntriesList(FILE_SYSTEM); assertThat(transactionLogAfterUpdate).hasSize(2); AddFileEntry updateAddFileEntry = transactionLogAfterUpdate.get(1).getAdd(); DeltaLakeFileStatistics updateStats = updateAddFileEntry.getStats().orElseThrow(); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java index cb6de42b7954..f02b6c6e10e2 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java @@ -39,6 +39,7 @@ import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointSchemaManager; +import io.trino.plugin.deltalake.transactionlog.checkpoint.TransactionLogTail; import io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeFileStatistics; import io.trino.plugin.hive.parquet.TrinoParquetDataSource; import io.trino.spi.Page; @@ -85,11 +86,11 @@ import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.parquet.ParquetTestUtils.createParquetReader; +import static io.trino.plugin.deltalake.DeltaLakeConfig.DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE; import static io.trino.plugin.deltalake.DeltaTestingConnectorSession.SESSION; import static io.trino.plugin.deltalake.TestingDeltaLakeUtils.copyDirectoryContents; import static io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.extractPartitionColumns; import static io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.getColumnsMetadata; -import static io.trino.plugin.deltalake.transactionlog.checkpoint.TransactionLogTail.getEntriesFromJson; import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS; import static io.trino.spi.type.DateTimeEncoding.packDateTimeWithZone; @@ -158,6 +159,7 @@ protected QueryRunner createQueryRunner() .addDeltaProperty("hive.metastore.catalog.dir", catalogDir.toUri().toString()) .addDeltaProperty("delta.register-table-procedure.enabled", "true") .addDeltaProperty("delta.enable-non-concurrent-writes", "true") + .addDeltaProperty("delta.transaction-log.max-cached-file-size", "0B") // Tests json log streaming code path .build(); } @@ -440,7 +442,7 @@ private void testOptimizeWithColumnMappingMode(String columnMappingMode) "ALTER TABLE " + tableName + " EXECUTE OPTIMIZE"); // Verify 'add' entry contains the expected physical name in the stats - List transactionLog = getEntriesFromJson(4, tableLocation.resolve("_delta_log").toString(), FILE_SYSTEM).orElseThrow(); + List transactionLog = getEntriesFromJson(4, tableLocation.resolve("_delta_log").toString()); assertThat(transactionLog).hasSize(5); assertThat(transactionLog.get(0).getCommitInfo()).isNotNull(); assertThat(transactionLog.get(1).getRemove()).isNotNull(); @@ -704,7 +706,7 @@ public void testStatisticsWithColumnCaseSensitivity() assertUpdate("INSERT INTO " + tableName + " VALUES (10, 1), (20, 1), (null, 1)", 3); - List transactionLog = getEntriesFromJson(1, tableLocation.resolve("_delta_log").toString(), FILE_SYSTEM).orElseThrow(); + List transactionLog = getEntriesFromJson(1, tableLocation.resolve("_delta_log").toString()); assertThat(transactionLog).hasSize(2); AddFileEntry addFileEntry = transactionLog.get(1).getAdd(); DeltaLakeFileStatistics stats = addFileEntry.getStats().orElseThrow(); @@ -714,7 +716,7 @@ public void testStatisticsWithColumnCaseSensitivity() assertUpdate("UPDATE " + tableName + " SET upper_case = upper_case + 10", 3); - List transactionLogAfterUpdate = getEntriesFromJson(2, tableLocation.resolve("_delta_log").toString(), FILE_SYSTEM).orElseThrow(); + List transactionLogAfterUpdate = getEntriesFromJson(2, tableLocation.resolve("_delta_log").toString()); assertThat(transactionLogAfterUpdate).hasSize(3); AddFileEntry updateAddFileEntry = transactionLogAfterUpdate.get(2).getAdd(); DeltaLakeFileStatistics updateStats = updateAddFileEntry.getStats().orElseThrow(); @@ -868,7 +870,7 @@ private void testTrinoCreateTableWithTimestampNtz(ZoneId sessionZone, Consumer transactionLogs = getEntriesFromJson(0, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow(); + List transactionLogs = getEntriesFromJson(0, tableLocation + "/_delta_log"); ProtocolEntry protocolEntry = transactionLogs.get(1).getProtocol(); assertThat(protocolEntry).isNotNull(); assertThat(protocolEntry.minReaderVersion()).isEqualTo(3); @@ -1044,7 +1046,7 @@ private void testTimestampNtzPartitioned(ZoneId sessionZone) ('part', null, 8.0, 0.1111111111111111, null, null, null), (null, null, null, null, 9.0, null, null) """); - List transactionLogs = getEntriesFromJson(2, tableLocation.resolve("_delta_log").toString(), FILE_SYSTEM).orElseThrow(); + List transactionLogs = getEntriesFromJson(2, tableLocation.resolve("_delta_log").toString()); assertThat(transactionLogs).hasSize(2); AddFileEntry addFileEntry = transactionLogs.get(1).getAdd(); assertThat(addFileEntry).isNotNull(); @@ -1066,7 +1068,7 @@ public void testAddTimestampNtzColumn() assertQuery("SELECT * FROM " + tableName, "VALUES (1, TIMESTAMP '2023-01-02 03:04:05.123456')"); String tableLocation = getTableLocation(tableName); - List transactionLogsByCreateTable = getEntriesFromJson(0, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow(); + List transactionLogsByCreateTable = getEntriesFromJson(0, tableLocation + "/_delta_log"); ProtocolEntry protocolEntryByCreateTable = transactionLogsByCreateTable.get(1).getProtocol(); assertThat(protocolEntryByCreateTable).isNotNull(); assertThat(protocolEntryByCreateTable.minReaderVersion()).isEqualTo(1); @@ -1074,7 +1076,7 @@ public void testAddTimestampNtzColumn() assertThat(protocolEntryByCreateTable.readerFeatures()).isEmpty(); assertThat(protocolEntryByCreateTable.writerFeatures()).isEmpty(); - List transactionLogsByAddColumn = getEntriesFromJson(1, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow(); + List transactionLogsByAddColumn = getEntriesFromJson(1, tableLocation + "/_delta_log"); ProtocolEntry protocolEntryByAddColumn = transactionLogsByAddColumn.get(1).getProtocol(); assertThat(protocolEntryByAddColumn).isNotNull(); assertThat(protocolEntryByAddColumn.minReaderVersion()).isEqualTo(3); @@ -1099,7 +1101,7 @@ public void testIdentityColumns() assertUpdate("CALL system.register_table(CURRENT_SCHEMA, '%s', '%s')".formatted(tableName, tableLocation.toUri())); assertQueryReturnsEmptyResult("SELECT * FROM " + tableName); - List transactionLog = getEntriesFromJson(0, tableLocation.resolve("_delta_log").toString(), FILE_SYSTEM).orElseThrow(); + List transactionLog = getEntriesFromJson(0, tableLocation.resolve("_delta_log").toString()); assertThat(transactionLog).hasSize(3); MetadataEntry metadataEntry = transactionLog.get(2).getMetaData(); assertThat(getColumnsMetadata(metadataEntry).get("b")) @@ -1111,7 +1113,7 @@ public void testIdentityColumns() // Verify a column operation preserves delta.identity.* column properties assertUpdate("COMMENT ON COLUMN " + tableName + ".b IS 'test column comment'"); - List transactionLogAfterComment = getEntriesFromJson(1, tableLocation.resolve("_delta_log").toString(), FILE_SYSTEM).orElseThrow(); + List transactionLogAfterComment = getEntriesFromJson(1, tableLocation.resolve("_delta_log").toString()); assertThat(transactionLogAfterComment).hasSize(3); MetadataEntry commentMetadataEntry = transactionLogAfterComment.get(2).getMetaData(); assertThat(getColumnsMetadata(commentMetadataEntry).get("b")) @@ -1202,7 +1204,7 @@ private void testDeletionVectorsEnabledCreateTable(String tableDefinition) .contains("deletion_vectors_enabled = true"); String tableLocation = getTableLocation(table.getName()); - List transactionLogs = getEntriesFromJson(0, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow(); + List transactionLogs = getEntriesFromJson(0, tableLocation + "/_delta_log"); assertThat(transactionLogs.get(1).getProtocol()) .isEqualTo(new ProtocolEntry(3, 7, Optional.of(Set.of("deletionVectors")), Optional.of(Set.of("deletionVectors")))); @@ -1235,7 +1237,7 @@ private void testDeletionVectorsDisabledCreateTable(String tableDefinition) .doesNotContain("deletion_vectors_enabled"); String tableLocation = getTableLocation(table.getName()); - List transactionLogs = getEntriesFromJson(0, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow(); + List transactionLogs = getEntriesFromJson(0, tableLocation + "/_delta_log"); assertThat(transactionLogs.get(1).getProtocol()) .isEqualTo(new ProtocolEntry(1, 2, Optional.empty(), Optional.empty())); @@ -1304,8 +1306,8 @@ public void testDeletionVectorsAllRows() assertUpdate("DELETE FROM " + tableName + " WHERE a != 999", 1); // 'remove' entry should have the same deletion vector as the previous operation when deleting all rows - DeletionVectorEntry deletionVector = getEntriesFromJson(2, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow().get(2).getAdd().getDeletionVector().orElseThrow(); - assertThat(getEntriesFromJson(3, tableLocation + "/_delta_log", FILE_SYSTEM).orElseThrow().get(1).getRemove().deletionVector().orElseThrow()) + DeletionVectorEntry deletionVector = getEntriesFromJson(2, tableLocation + "/_delta_log").get(2).getAdd().getDeletionVector().orElseThrow(); + assertThat(getEntriesFromJson(3, tableLocation + "/_delta_log").get(1).getRemove().deletionVector().orElseThrow()) .isEqualTo(deletionVector); assertUpdate("INSERT INTO " + tableName + " VALUES (3, 31), (3, 32)", 2); @@ -2257,8 +2259,7 @@ public void testUnsupportedWriterVersion() private static MetadataEntry loadMetadataEntry(long entryNumber, Path tableLocation) throws IOException { - TrinoFileSystem fileSystem = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS).create(SESSION); - DeltaLakeTransactionLogEntry transactionLog = getEntriesFromJson(entryNumber, tableLocation.resolve("_delta_log").toString(), fileSystem).orElseThrow().stream() + DeltaLakeTransactionLogEntry transactionLog = getEntriesFromJson(entryNumber, tableLocation.resolve("_delta_log").toString()).stream() .filter(log -> log.getMetaData() != null) .collect(onlyElement()); return transactionLog.getMetaData(); @@ -2267,8 +2268,7 @@ private static MetadataEntry loadMetadataEntry(long entryNumber, Path tableLocat private static ProtocolEntry loadProtocolEntry(long entryNumber, Path tableLocation) throws IOException { - TrinoFileSystem fileSystem = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS).create(SESSION); - DeltaLakeTransactionLogEntry transactionLog = getEntriesFromJson(entryNumber, tableLocation.resolve("_delta_log").toString(), fileSystem).orElseThrow().stream() + DeltaLakeTransactionLogEntry transactionLog = getEntriesFromJson(entryNumber, tableLocation.resolve("_delta_log").toString()).stream() .filter(log -> log.getProtocol() != null) .collect(onlyElement()); return transactionLog.getProtocol(); @@ -2285,4 +2285,12 @@ private String getTableLocation(String tableName) } throw new IllegalStateException("Location not found in SHOW CREATE TABLE result"); } + + private static List getEntriesFromJson(long entryNumber, String transactionLogDir) + throws IOException + { + return TransactionLogTail.getEntriesFromJson(entryNumber, transactionLogDir, FILE_SYSTEM, DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE) + .orElseThrow() + .getEntriesList(FILE_SYSTEM); + } } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeColumnMapping.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeColumnMapping.java index 6746c8f2a522..9d2447df488d 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeColumnMapping.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeColumnMapping.java @@ -37,6 +37,7 @@ import static com.google.common.collect.MoreCollectors.onlyElement; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.trino.plugin.deltalake.DeltaLakeConfig.DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE; import static io.trino.plugin.deltalake.DeltaTestingConnectorSession.SESSION; import static io.trino.plugin.deltalake.transactionlog.checkpoint.TransactionLogTail.getEntriesFromJson; import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; @@ -144,7 +145,9 @@ private static MetadataEntry loadMetadataEntry(long entryNumber, Path tableLocat throws IOException { TrinoFileSystem fileSystem = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS).create(SESSION); - DeltaLakeTransactionLogEntry transactionLog = getEntriesFromJson(entryNumber, tableLocation.resolve("_delta_log").toString(), fileSystem).orElseThrow().stream() + DeltaLakeTransactionLogEntry transactionLog = getEntriesFromJson(entryNumber, tableLocation.resolve("_delta_log").toString(), fileSystem, DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE) + .orElseThrow() + .getEntriesList(fileSystem).stream() .filter(log -> log.getMetaData() != null) .collect(onlyElement()); return transactionLog.getMetaData(); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java index 3ef6e8c0a500..78750f664d20 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java @@ -44,6 +44,7 @@ public void testDefaults() .setDataFileCacheTtl(new Duration(30, MINUTES)) .setMetadataCacheTtl(new Duration(30, TimeUnit.MINUTES)) .setMetadataCacheMaxRetainedSize(DeltaLakeConfig.DEFAULT_METADATA_CACHE_MAX_RETAINED_SIZE) + .setTransactionLogMaxCachedFileSize(DeltaLakeConfig.DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE) .setDomainCompactionThreshold(1000) .setMaxSplitsPerSecond(Integer.MAX_VALUE) .setMaxOutstandingSplits(1_000) @@ -84,6 +85,7 @@ public void testExplicitPropertyMappings() Map properties = ImmutableMap.builder() .put("delta.metadata.cache-ttl", "10m") .put("delta.metadata.cache-max-retained-size", "1GB") + .put("delta.transaction-log.max-cached-file-size", "1MB") .put("delta.metadata.live-files.cache-size", "0 MB") .put("delta.metadata.live-files.cache-ttl", "60m") .put("delta.domain-compaction-threshold", "500") @@ -125,6 +127,7 @@ public void testExplicitPropertyMappings() .setDataFileCacheTtl(new Duration(60, MINUTES)) .setMetadataCacheTtl(new Duration(10, TimeUnit.MINUTES)) .setMetadataCacheMaxRetainedSize(DataSize.of(1, GIGABYTE)) + .setTransactionLogMaxCachedFileSize(DataSize.of(1, MEGABYTE)) .setDomainCompactionThreshold(500) .setMaxOutstandingSplits(200) .setMaxSplitsPerSecond(10) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java index 7460919d0bfa..38e348644f4e 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java @@ -137,8 +137,9 @@ public void testCreateOrReplaceTable() assertFileSystemAccesses("CREATE OR REPLACE TABLE test_create_or_replace (id VARCHAR, age INT)", ImmutableMultiset.builder() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "OutputFile.createOrOverwrite")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.exists")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.exists")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) @@ -167,8 +168,9 @@ public void testCreateOrReplaceTableAsSelect() .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "OutputFile.createOrOverwrite")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "OutputFile.createOrOverwrite")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.exists")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(DATA, "no partition", "OutputFile.create")) @@ -196,7 +198,10 @@ public void testReadUnpartitionedTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 2) .build()); @@ -209,7 +214,10 @@ public void testReadUnpartitionedTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 2) .build()); @@ -233,7 +241,7 @@ public void testReadTableCheckpointInterval() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .addCopies(new FileOperation(CHECKPOINT, "00000000000000000002.checkpoint.parquet", "InputFile.length"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query .addCopies(new FileOperation(CHECKPOINT, "00000000000000000002.checkpoint.parquet", "InputFile.newInput"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 2) .build()); @@ -256,7 +264,7 @@ public void testReadPartitionTableWithCheckpointFiltering() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .addCopies(new FileOperation(CHECKPOINT, "00000000000000000002.checkpoint.parquet", "InputFile.length"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query .addCopies(new FileOperation(CHECKPOINT, "00000000000000000002.checkpoint.parquet", "InputFile.newInput"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .addCopies(new FileOperation(DATA, "key=p1/", "InputFile.newInput"), 2) .addCopies(new FileOperation(DATA, "key=p2/", "InputFile.newInput"), 2) .build()); @@ -283,7 +291,10 @@ public void testReadWholePartition() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .addCopies(new FileOperation(DATA, "key=p1/", "InputFile.newInput"), 2) .addCopies(new FileOperation(DATA, "key=p2/", "InputFile.newInput"), 2) .build()); @@ -297,7 +308,10 @@ public void testReadWholePartition() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .addCopies(new FileOperation(DATA, "key=p1/", "InputFile.newInput"), 2) .addCopies(new FileOperation(DATA, "key=p2/", "InputFile.newInput"), 2) @@ -312,7 +326,10 @@ public void testReadWholePartition() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .build()); @@ -325,7 +342,10 @@ public void testReadWholePartition() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .build()); // Read partition and synthetic columns @@ -337,7 +357,10 @@ public void testReadWholePartition() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .build()); @@ -350,7 +373,10 @@ public void testReadWholePartition() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .build()); assertUpdate("DROP TABLE test_read_part_key"); @@ -383,7 +409,9 @@ public void testReadWholePartitionSplittableFile() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .build()); @@ -396,7 +424,9 @@ public void testReadWholePartitionSplittableFile() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) .build()); assertUpdate("DROP TABLE test_read_whole_splittable_file"); @@ -417,7 +447,8 @@ public void testSelfJoin() ImmutableMultiset.builder() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 2) .build()); @@ -438,6 +469,7 @@ public void testSelectFromVersionedTable() ImmutableMultiset.builder() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.exists")) .build()); assertFileSystemAccesses("SELECT * FROM " + tableName + " FOR VERSION AS OF 1", @@ -445,6 +477,8 @@ public void testSelectFromVersionedTable() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.exists")) .add(new FileOperation(DATA, "no partition", "InputFile.newInput")) .build()); @@ -454,6 +488,9 @@ public void testSelectFromVersionedTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.exists")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 2) .build()); @@ -469,6 +506,15 @@ public void testSelectFromVersionedTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000006.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000007.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000008.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000006.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000007.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000008.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000008.json", "InputFile.exists")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 8) .build()); @@ -480,6 +526,9 @@ public void testSelectFromVersionedTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000011.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000012.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000013.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000011.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000012.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000013.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000013.json", "InputFile.exists")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 13) .build()); @@ -499,6 +548,9 @@ public void testSelectFromVersionedTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000021.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000022.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000023.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000021.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000022.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000023.json", "InputFile.length")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000023.json", "InputFile.exists")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 23) .build()); @@ -527,7 +579,10 @@ public void testDeleteWholePartition() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.exists")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "OutputFile.createOrOverwrite")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .build()); assertUpdate("DROP TABLE test_delete_part_key"); @@ -553,7 +608,10 @@ public void testDeleteWholeTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.exists")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "OutputFile.createOrOverwrite")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .build()); assertUpdate("DROP TABLE test_delete_whole_table"); @@ -578,7 +636,11 @@ public void testDeleteWithNonPartitionFilter() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.exists")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "OutputFile.createOrOverwrite")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length")) .addCopies(new FileOperation(DATA, "key=domain1/", "InputFile.newInput"), 2) .add(new FileOperation(DATA, "key=domain1/", "InputFile.length")) .add(new FileOperation(DATA, "key=domain1/", "OutputFile.create")) @@ -603,7 +665,12 @@ public void testHistorySystemTable() .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream"), 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream"), 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream"), 2) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.newStream")) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length"), 2) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .build()); @@ -614,7 +681,12 @@ public void testHistorySystemTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream"), 2) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length"), 2) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .build()); @@ -625,7 +697,12 @@ public void testHistorySystemTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream"), 2) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length"), 2) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .build()); @@ -636,7 +713,12 @@ public void testHistorySystemTable() .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream"), 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream"), 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream"), 2) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length"), 2) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .build()); @@ -647,7 +729,12 @@ public void testHistorySystemTable() .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream"), 2) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length"), 2) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .build()); @@ -658,7 +745,12 @@ public void testHistorySystemTable() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .build()); } @@ -688,7 +780,14 @@ public void testTableChangesFileSystemAccess() .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.newStream"), 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.newStream"), 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000006.json", "InputFile.newStream"), 2) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000007.json", "InputFile.newStream")) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000004.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000005.json", "InputFile.length"), 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000006.json", "InputFile.length"), 2) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000007.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(CDF_DATA, "key=domain1/", "InputFile.newInput")) .addCopies(new FileOperation(CDF_DATA, "key=domain2/", "InputFile.newInput"), cdfFilesForDomain2) @@ -738,9 +837,12 @@ private void testInformationSchemaColumns(boolean removeCachedProperties) ImmutableMultiset.builder() .addCopies(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream"), tables * 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream"), tables * 2) - .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream"), tables * 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream"), tables) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream"), tables) - .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream"), tables) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length"), tables * 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length"), tables * 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length"), tables) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length"), tables) .build() : ImmutableMultiset.builder() .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.exists"), tables) @@ -774,7 +876,10 @@ private void testInformationSchemaColumns(boolean removeCachedProperties) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .build()); // Pointed lookup with LIKE predicate (as if unintentional) @@ -793,7 +898,10 @@ private void testInformationSchemaColumns(boolean removeCachedProperties) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .build()); for (int i = 0; i < tables; i++) { @@ -842,9 +950,12 @@ private void testSystemMetadataTableComments(boolean removeCachedProperties) ImmutableMultiset.builder() .addCopies(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream"), tables * 2) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream"), tables * 2) - .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream"), tables * 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream"), tables) .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream"), tables) - .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream"), tables) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length"), tables * 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length"), tables * 2) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length"), tables) + .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length"), tables) .build() : ImmutableMultiset.builder() .addCopies(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.exists"), tables) @@ -887,7 +998,10 @@ private void testSystemMetadataTableComments(boolean removeCachedProperties) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .build()); // Pointed lookup with LIKE predicate (as if unintentional) @@ -929,7 +1043,8 @@ public void testReadMultipartCheckpoint() .addCopies(new FileOperation(CHECKPOINT, "00000000000000000006.checkpoint.0000000002.0000000002.parquet", "InputFile.length"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query .addCopies(new FileOperation(CHECKPOINT, "00000000000000000006.checkpoint.0000000002.0000000002.parquet", "InputFile.newInput"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000007.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000008.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000007.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000008.json", "InputFile.length")) .addCopies(new FileOperation(DATA, "no partition", "InputFile.newInput"), 7) .build()); } @@ -947,7 +1062,7 @@ public void testV2CheckpointJson() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.73a4ddb8-2bfc-40d8-b09f-1b6a0abdfb04.json", "InputFile.length")) .add(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.73a4ddb8-2bfc-40d8-b09f-1b6a0abdfb04.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail .build()); assertFileSystemAccesses("SELECT * FROM " + tableName, @@ -957,7 +1072,7 @@ public void testV2CheckpointJson() .addCopies(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.73a4ddb8-2bfc-40d8-b09f-1b6a0abdfb04.json", "InputFile.newStream"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query .add(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.0000000001.0000000001.90cf4e21-dbaa-41d6-8ae5-6709cfbfbfe0.parquet", "InputFile.length")) .add(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.0000000001.0000000001.90cf4e21-dbaa-41d6-8ae5-6709cfbfbfe0.parquet", "InputFile.newInput")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail .add(new FileOperation(DATA, "no partition", "InputFile.newInput")) .build()); @@ -977,7 +1092,7 @@ public void testV2CheckpointParquet() .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .addCopies(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.156b3304-76b2-49c3-a9a1-626f07df27c9.parquet", "InputFile.length"), 2) .add(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.156b3304-76b2-49c3-a9a1-626f07df27c9.parquet", "InputFile.newInput")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail .build()); assertFileSystemAccesses("SELECT * FROM " + tableName, @@ -987,7 +1102,7 @@ public void testV2CheckpointParquet() .addCopies(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.156b3304-76b2-49c3-a9a1-626f07df27c9.parquet", "InputFile.newInput"), 2) // TODO (https://github.com/trinodb/trino/issues/18916) should be checked once per query .add(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.0000000001.0000000001.03288d7e-af16-44ed-829c-196064a71812.parquet", "InputFile.length")) .add(new FileOperation(CHECKPOINT, "00000000000000000001.checkpoint.0000000001.0000000001.03288d7e-af16-44ed-829c-196064a71812.parquet", "InputFile.newInput")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) // TransactionLogTail.getEntriesFromJson access non-existing file as end of tail .add(new FileOperation(DATA, "no partition", "InputFile.newInput")) .build()); @@ -1006,7 +1121,10 @@ public void testDeletionVectors() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) .add(new FileOperation(DELETION_VECTOR, "deletion_vector_a52eda8c-0a57-4636-814b-9c165388f7ca.bin", "InputFile.newInput")) .add(new FileOperation(DATA, "no partition", "InputFile.newInput")) @@ -1017,7 +1135,10 @@ public void testDeletionVectors() .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.newStream")) .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.newStream")) - .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.newStream")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000000.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000001.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000002.json", "InputFile.length")) + .add(new FileOperation(TRANSACTION_LOG_JSON, "00000000000000000003.json", "InputFile.length")) .add(new FileOperation(STARBURST_EXTENDED_STATS_JSON, "extendeded_stats.json", "InputFile.newStream")) .add(new FileOperation(TRINO_EXTENDED_STATS_JSON, "extended_stats.json", "InputFile.newStream")) .add(new FileOperation(LAST_CHECKPOINT, "_last_checkpoint", "InputFile.newStream")) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestTransactionLogAccess.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestTransactionLogAccess.java index 3593fc783ef7..cba08151ff37 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestTransactionLogAccess.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestTransactionLogAccess.java @@ -27,10 +27,10 @@ import io.trino.plugin.deltalake.transactionlog.AddFileEntry; import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; -import io.trino.plugin.deltalake.transactionlog.RemoveFileEntry; import io.trino.plugin.deltalake.transactionlog.TableSnapshot; import io.trino.plugin.deltalake.transactionlog.TransactionLogAccess; import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointSchemaManager; +import io.trino.plugin.deltalake.transactionlog.checkpoint.MetadataAndProtocolEntries; import io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeFileStatistics; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hive.parquet.ParquetWriterConfig; @@ -103,14 +103,6 @@ public class TestTransactionLogAccess "age=28/part-00000-40dd1707-1d42-4328-a59a-21f5c945fe60.c000.snappy.parquet", "age=29/part-00000-3794c463-cb0c-4beb-8d07-7cc1e3b5920f.c000.snappy.parquet"); - private static final Set EXPECTED_REMOVE_ENTRIES = ImmutableSet.of( - new RemoveFileEntry("age=30/part-00000-7e43a3c3-ea26-4ae7-8eac-8f60cbb4df03.c000.snappy.parquet", null, 1579190163932L, false, Optional.empty()), - new RemoveFileEntry("age=30/part-00000-72a56c23-01ba-483a-9062-dd0accc86599.c000.snappy.parquet", null, 1579190163932L, false, Optional.empty()), - new RemoveFileEntry("age=42/part-00000-951068bd-bcf4-4094-bb94-536f3c41d31f.c000.snappy.parquet", null, 1579190155406L, false, Optional.empty()), - new RemoveFileEntry("age=25/part-00000-609e34b1-5466-4dbc-a780-2708166e7adb.c000.snappy.parquet", null, 1579190163932L, false, Optional.empty()), - new RemoveFileEntry("age=42/part-00000-6aed618a-2beb-4edd-8466-653e67a9b380.c000.snappy.parquet", null, 1579190155406L, false, Optional.empty()), - new RemoveFileEntry("age=42/part-00000-b82d8859-84a0-4f05-872c-206b07dd54f0.c000.snappy.parquet", null, 1579190163932L, false, Optional.empty())); - private final TestingTelemetry testingTelemetry = TestingTelemetry.create("transaction-log-access"); private final TracingFileSystemFactory tracingFileSystemFactory = new TracingFileSystemFactory(testingTelemetry.getTracer(), new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS)); @@ -322,18 +314,6 @@ public void testAddRemoveAdd() } } - @Test - public void testGetRemoveEntries() - throws Exception - { - setupTransactionLogAccessFromResources("person", "databricks73/person"); - - try (Stream removeEntries = transactionLogAccess.getRemoveEntries(SESSION, tableSnapshot)) { - Set removedEntries = removeEntries.collect(Collectors.toSet()); - assertThat(removedEntries).isEqualTo(EXPECTED_REMOVE_ENTRIES); - } - } - @Test public void testAllGetMetadataEntry() throws Exception @@ -359,6 +339,35 @@ private void testAllGetMetadataEntry(String tableName, String resourcePath) assertThat(format.provider()).isEqualTo("parquet"); } + @Test + void testGetMetadataAndProtocolEntry() + throws Exception + { + testGetMetadataAndProtocolEntry("person", "databricks73/person"); + testGetMetadataAndProtocolEntry("person_without_last_checkpoint", "databricks73/person_without_last_checkpoint"); + testGetMetadataAndProtocolEntry("person_without_old_jsons", "databricks73/person_without_old_jsons"); + testGetMetadataAndProtocolEntry("person_without_checkpoints", "databricks73/person_without_checkpoints"); + } + + private void testGetMetadataAndProtocolEntry(String tableName, String resourcePath) + throws Exception + { + setupTransactionLogAccessFromResources(tableName, resourcePath); + + transactionLogAccess.getMetadataAndProtocolEntry(SESSION, tableSnapshot); + MetadataAndProtocolEntries logEntries = transactionLogAccess.getMetadataAndProtocolEntry(SESSION, tableSnapshot); + + MetadataEntry metadataEntry = logEntries.metadata().orElseThrow(); + assertThat(metadataEntry.getOriginalPartitionColumns()).containsOnly("age"); + MetadataEntry.Format format = metadataEntry.getFormat(); + assertThat(format.options().keySet()).isEmpty(); + assertThat(format.provider()).isEqualTo("parquet"); + + ProtocolEntry protocolEntry = logEntries.protocol().orElseThrow(); + assertThat(protocolEntry.minReaderVersion()).isEqualTo(1); + assertThat(protocolEntry.minWriterVersion()).isEqualTo(2); + } + @Test public void testAllGetActiveAddEntries() throws Exception @@ -384,29 +393,6 @@ private void testAllGetActiveAddEntries(String tableName, String resourcePath) } } - @Test - public void testAllGetRemoveEntries() - throws Exception - { - testAllGetRemoveEntries("person", "databricks73/person"); - testAllGetRemoveEntries("person_without_last_checkpoint", "databricks73/person_without_last_checkpoint"); - testAllGetRemoveEntries("person_without_old_jsons", "databricks73/person_without_old_jsons"); - testAllGetRemoveEntries("person_without_checkpoints", "databricks73/person_without_checkpoints"); - } - - private void testAllGetRemoveEntries(String tableName, String resourcePath) - throws Exception - { - setupTransactionLogAccessFromResources(tableName, resourcePath); - - try (Stream removeEntries = transactionLogAccess.getRemoveEntries(SESSION, tableSnapshot)) { - Set removedPaths = removeEntries.map(RemoveFileEntry::path).collect(Collectors.toSet()); - Set expectedPaths = EXPECTED_REMOVE_ENTRIES.stream().map(RemoveFileEntry::path).collect(Collectors.toSet()); - - assertThat(removedPaths).isEqualTo(expectedPaths); - } - } - @Test public void testAllGetProtocolEntries() throws Exception @@ -597,7 +583,8 @@ public void testIncrementalCacheUpdates() .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.newInput")) .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) - .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) .build()); copyTransactionLogEntry(12, 14, resourceDir, transactionLogDir); @@ -615,7 +602,9 @@ public void testIncrementalCacheUpdates() .add(new FileOperation("_last_checkpoint", "InputFile.newStream")) .addCopies(new FileOperation("00000000000000000012.json", "InputFile.newStream"), 2) .addCopies(new FileOperation("00000000000000000013.json", "InputFile.newStream"), 2) - .add(new FileOperation("00000000000000000014.json", "InputFile.newStream")) + .addCopies(new FileOperation("00000000000000000012.json", "InputFile.length"), 2) + .addCopies(new FileOperation("00000000000000000013.json", "InputFile.length"), 2) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .build()); } @@ -777,7 +766,10 @@ public void testTableSnapshotsCacheDisabled() .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000013.json", "InputFile.newStream")) - .add(new FileOperation("00000000000000000014.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) + .add(new FileOperation("00000000000000000013.json", "InputFile.length")) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .build()); // With the transaction log cache disabled, when loading the snapshot again, all the needed files will be opened again @@ -790,7 +782,10 @@ public void testTableSnapshotsCacheDisabled() .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000013.json", "InputFile.newStream")) - .add(new FileOperation("00000000000000000014.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) + .add(new FileOperation("00000000000000000013.json", "InputFile.length")) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .build()); } @@ -826,7 +821,10 @@ public void testTableSnapshotsActiveDataFilesCache() .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000013.json", "InputFile.newStream")) - .add(new FileOperation("00000000000000000014.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) + .add(new FileOperation("00000000000000000013.json", "InputFile.length")) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.newInput")) .build()); @@ -866,7 +864,10 @@ public void testFlushSnapshotAndActiveFileCache() .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000013.json", "InputFile.newStream")) - .add(new FileOperation("00000000000000000014.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) + .add(new FileOperation("00000000000000000013.json", "InputFile.length")) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.newInput")) .build()); @@ -885,7 +886,10 @@ public void testFlushSnapshotAndActiveFileCache() .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000013.json", "InputFile.newStream")) - .add(new FileOperation("00000000000000000014.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) + .add(new FileOperation("00000000000000000013.json", "InputFile.length")) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.newInput")) .build()); @@ -916,7 +920,10 @@ public void testTableSnapshotsActiveDataFilesCacheDisabled() .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000013.json", "InputFile.newStream")) - .add(new FileOperation("00000000000000000014.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) + .add(new FileOperation("00000000000000000013.json", "InputFile.length")) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.newInput")) .build()); @@ -968,6 +975,16 @@ public void testLoadSnapshotWithEndVersion() .add(new FileOperation("00000000000000000007.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000008.json", "InputFile.newStream")) .add(new FileOperation("00000000000000000009.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000000.json", "InputFile.length")) + .add(new FileOperation("00000000000000000001.json", "InputFile.length")) + .add(new FileOperation("00000000000000000002.json", "InputFile.length")) + .add(new FileOperation("00000000000000000003.json", "InputFile.length")) + .add(new FileOperation("00000000000000000004.json", "InputFile.length")) + .add(new FileOperation("00000000000000000005.json", "InputFile.length")) + .add(new FileOperation("00000000000000000006.json", "InputFile.length")) + .add(new FileOperation("00000000000000000007.json", "InputFile.length")) + .add(new FileOperation("00000000000000000008.json", "InputFile.length")) + .add(new FileOperation("00000000000000000009.json", "InputFile.length")) .build()); setupTransactionLogAccess("person", getClass().getClassLoader().getResource("databricks73/person").toString(), new DeltaLakeConfig(), Optional.of(10L)); @@ -999,6 +1016,7 @@ public void testLoadSnapshotWithEndVersion() .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.length")) .add(new FileOperation("00000000000000000010.checkpoint.parquet", "InputFile.newInput")) .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) .build()); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/TestTableSnapshot.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/TestTableSnapshot.java index 130c788225d6..0960f74ffd23 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/TestTableSnapshot.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/TestTableSnapshot.java @@ -46,6 +46,7 @@ import static com.google.common.base.Predicates.alwaysTrue; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.filesystem.tracing.FileSystemAttributes.FILE_LOCATION; +import static io.trino.plugin.deltalake.DeltaLakeConfig.DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE; import static io.trino.plugin.deltalake.transactionlog.TableSnapshot.MetadataAndProtocolEntry; import static io.trino.plugin.deltalake.transactionlog.TableSnapshot.load; import static io.trino.plugin.deltalake.transactionlog.TransactionLogParser.readLastCheckpoint; @@ -98,19 +99,23 @@ public void testOnlyReadsTrailingJsonFiles() parquetReaderOptions, true, domainCompactionThreshold, + DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE, Optional.empty())); }, ImmutableMultiset.builder() - .addCopies(new FileOperation("_last_checkpoint", "InputFile.newStream"), 1) - .addCopies(new FileOperation("00000000000000000011.json", "InputFile.newStream"), 1) - .addCopies(new FileOperation("00000000000000000012.json", "InputFile.newStream"), 1) - .addCopies(new FileOperation("00000000000000000013.json", "InputFile.newStream"), 1) - .addCopies(new FileOperation("00000000000000000014.json", "InputFile.newStream"), 1) + .add(new FileOperation("_last_checkpoint", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000012.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000013.json", "InputFile.newStream")) + .add(new FileOperation("00000000000000000011.json", "InputFile.length")) + .add(new FileOperation("00000000000000000012.json", "InputFile.length")) + .add(new FileOperation("00000000000000000013.json", "InputFile.length")) + .add(new FileOperation("00000000000000000014.json", "InputFile.length")) .build()); assertFileSystemAccesses( () -> { - tableSnapshot.get().getJsonTransactionLogEntries().forEach(entry -> {}); + tableSnapshot.get().getJsonTransactionLogEntries(trackingFileSystem).forEach(entry -> {}); }, ImmutableMultiset.of()); } @@ -129,6 +134,7 @@ public void readsCheckpointFile() parquetReaderOptions, true, domainCompactionThreshold, + DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE, Optional.empty()); TestingConnectorContext context = new TestingConnectorContext(); TypeManager typeManager = context.getTypeManager(); @@ -257,6 +263,7 @@ public void testMaxTransactionId() parquetReaderOptions, true, domainCompactionThreshold, + DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE, Optional.empty()); assertThat(tableSnapshot.getVersion()).isEqualTo(13L); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TestTransactionLogTail.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TestTransactionLogTail.java index b42a58cb04a7..d4610bd756f5 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TestTransactionLogTail.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/checkpoint/TestTransactionLogTail.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Optional; +import static io.trino.plugin.deltalake.DeltaLakeConfig.DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE; import static io.trino.plugin.deltalake.DeltaTestingConnectorSession.SESSION; import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS; @@ -49,17 +50,17 @@ private List updateJsonTransactionLogTails(String throws Exception { TrinoFileSystem fileSystem = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS).create(SESSION); - TransactionLogTail transactionLogTail = TransactionLogTail.loadNewTail(fileSystem, tableLocation, Optional.of(10L), Optional.of(12L)); - Optional updatedLogTail = transactionLogTail.getUpdatedTail(fileSystem, tableLocation, Optional.empty()); + TransactionLogTail transactionLogTail = TransactionLogTail.loadNewTail(fileSystem, tableLocation, Optional.of(10L), Optional.of(12L), DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE); + Optional updatedLogTail = transactionLogTail.getUpdatedTail(fileSystem, tableLocation, Optional.empty(), DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE); assertThat(updatedLogTail).isPresent(); - return updatedLogTail.get().getFileEntries(); + return updatedLogTail.get().getFileEntries(fileSystem); } private List readJsonTransactionLogTails(String tableLocation) throws Exception { TrinoFileSystem fileSystem = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS).create(SESSION); - TransactionLogTail transactionLogTail = TransactionLogTail.loadNewTail(fileSystem, tableLocation, Optional.of(10L), Optional.empty()); - return transactionLogTail.getFileEntries(); + TransactionLogTail transactionLogTail = TransactionLogTail.loadNewTail(fileSystem, tableLocation, Optional.of(10L), Optional.empty(), DEFAULT_TRANSACTION_LOG_MAX_CACHED_SIZE); + return transactionLogTail.getFileEntries(fileSystem); } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeMinioReads.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeMinioReads.java index 351ea5a17a1e..88df1a82af34 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeMinioReads.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/BaseTestDeltaLakeMinioReads.java @@ -79,7 +79,7 @@ public void testReadRegionTable() format("SELECT count(name) FROM delta.default.\"%s\"", tableName))) .containsOnly(row(5L)); - assertNotificationsCount(NOTIFICATIONS_TABLE, OBJECT_ACCESSED_HEAD, tableName + "/_delta_log/00000000000000000000.json", 0); + assertNotificationsCount(NOTIFICATIONS_TABLE, OBJECT_ACCESSED_HEAD, tableName + "/_delta_log/00000000000000000000.json", 1); assertNotificationsCount(NOTIFICATIONS_TABLE, OBJECT_ACCESSED_GET, tableName + "/_delta_log/00000000000000000000.json", 1); onTrino().executeQuery(format("DROP TABLE delta.default.\"%s\"", tableName)); } From 590bbb3ea6818abe02610fbbbd0a13966832d8db Mon Sep 17 00:00:00 2001 From: Marius Grama Date: Wed, 10 May 2023 09:01:52 +0200 Subject: [PATCH 065/158] Move `databricksTestJdbcUrl()` method after the constructor --- .../environment/AbstractSinglenodeDeltaLakeDatabricks.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java index 651a9a3edc6a..4f9ce1af383a 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java @@ -40,14 +40,14 @@ public abstract class AbstractSinglenodeDeltaLakeDatabricks private final DockerFiles dockerFiles; - abstract String databricksTestJdbcUrl(); - public AbstractSinglenodeDeltaLakeDatabricks(Standard standard, DockerFiles dockerFiles) { super(standard); this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); } + abstract String databricksTestJdbcUrl(); + @Override public void extendEnvironment(Environment.Builder builder) { From 3257342dd9cbd9e591ed8bc5e573691f648403ae Mon Sep 17 00:00:00 2001 From: Marius Grama Date: Wed, 10 May 2023 09:03:43 +0200 Subject: [PATCH 066/158] Use `ZSTD` Parquet compression codec for Delta Lake by default --- docs/src/main/sphinx/connector/delta-lake.md | 2 +- .../plugin/deltalake/DeltaLakeConfig.java | 2 +- ...stDeltaLakeAlluxioCacheFileOperations.java | 48 +++++++++---------- ...LakeAlluxioCacheMutableTransactionLog.java | 16 +++---- .../plugin/deltalake/TestDeltaLakeConfig.java | 2 +- .../TestDeltaLakeInsertCompatibility.java | 4 +- 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/src/main/sphinx/connector/delta-lake.md b/docs/src/main/sphinx/connector/delta-lake.md index 2e6785c3a40a..0069b9fe65af 100644 --- a/docs/src/main/sphinx/connector/delta-lake.md +++ b/docs/src/main/sphinx/connector/delta-lake.md @@ -108,7 +108,7 @@ values. Typical usage does not require you to configure them. * `GZIP` The equivalent catalog session property is `compression_codec`. - - `SNAPPY` + - `ZSTD` * - `delta.max-partitions-per-writer` - Maximum number of partitions per writer. - `100` diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java index e02a6f5d8cce..1d553252dfe8 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java @@ -80,7 +80,7 @@ public class DeltaLakeConfig private boolean tableStatisticsEnabled = true; private boolean extendedStatisticsEnabled = true; private boolean collectExtendedStatisticsOnWrite = true; - private HiveCompressionCodec compressionCodec = HiveCompressionCodec.SNAPPY; + private HiveCompressionCodec compressionCodec = HiveCompressionCodec.ZSTD; private long perTransactionMetastoreCacheMaximumSize = 1000; private boolean storeTableMetadataEnabled; private int storeTableMetadataThreads = 5; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java index 32bdc7c2cb7c..d7e92d3a063e 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java @@ -95,12 +95,12 @@ public void testCacheFileOperations() .add(new CacheOperation("Alluxio.writeCache", "00000000000000000002.json", 0, 658)) .add(new CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 220)) - .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 220)) - .add(new CacheOperation("Input.readFully", "key=p1/", 0, 220)) - .add(new CacheOperation("Input.readFully", "key=p2/", 0, 220)) - .add(new CacheOperation("Alluxio.writeCache", "key=p1/", 0, 220)) - .add(new CacheOperation("Alluxio.writeCache", "key=p2/", 0, 220)) + .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) + .add(new CacheOperation("Input.readFully", "key=p1/", 0, 227)) + .add(new CacheOperation("Input.readFully", "key=p2/", 0, 227)) + .add(new CacheOperation("Alluxio.writeCache", "key=p1/", 0, 227)) + .add(new CacheOperation("Alluxio.writeCache", "key=p2/", 0, 227)) .build()); assertFileSystemAccesses( "SELECT * FROM test_cache_file_operations", @@ -113,8 +113,8 @@ public void testCacheFileOperations() .add(new CacheOperation("InputFile.length", "00000000000000000002.json")) .add(new CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 220)) - .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 220)) + .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) .build()); assertUpdate("INSERT INTO test_cache_file_operations VALUES ('p3', '3-xyz')", 1); assertUpdate("INSERT INTO test_cache_file_operations VALUES ('p4', '4-xyz')", 1); @@ -139,17 +139,17 @@ public void testCacheFileOperations() .add(new CacheOperation("InputFile.length", "00000000000000000005.json")) .add(new CacheOperation("InputFile.length", "00000000000000000006.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 220)) - .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 220)) - .add(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 220)) - .add(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 220)) - .add(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 220)) - .add(new CacheOperation("Input.readFully", "key=p3/", 0, 220)) - .add(new CacheOperation("Input.readFully", "key=p4/", 0, 220)) - .add(new CacheOperation("Input.readFully", "key=p5/", 0, 220)) - .add(new CacheOperation("Alluxio.writeCache", "key=p3/", 0, 220)) - .add(new CacheOperation("Alluxio.writeCache", "key=p4/", 0, 220)) - .add(new CacheOperation("Alluxio.writeCache", "key=p5/", 0, 220)) + .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 227)) + .add(new CacheOperation("Input.readFully", "key=p3/", 0, 227)) + .add(new CacheOperation("Input.readFully", "key=p4/", 0, 227)) + .add(new CacheOperation("Input.readFully", "key=p5/", 0, 227)) + .add(new CacheOperation("Alluxio.writeCache", "key=p3/", 0, 227)) + .add(new CacheOperation("Alluxio.writeCache", "key=p4/", 0, 227)) + .add(new CacheOperation("Alluxio.writeCache", "key=p5/", 0, 227)) .build()); assertFileSystemAccesses( "SELECT * FROM test_cache_file_operations", @@ -168,11 +168,11 @@ public void testCacheFileOperations() .add(new CacheOperation("InputFile.length", "00000000000000000005.json")) .add(new CacheOperation("InputFile.length", "00000000000000000006.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 220), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 220), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 220), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 220), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 220), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 227), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 227), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 227), 1) .build()); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java index 910c3657acfd..04b3cbfba622 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java @@ -79,12 +79,12 @@ public void testTableDataCachedWhileTransactionLogNotCached() .addCopies(new CacheFileSystemTraceUtils.CacheOperation("Input.readTail", "00000000000000000002.checkpoint.parquet"), 2) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 220)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 220)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p1/", 0, 220)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p2/", 0, 220)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p1/", 0, 220)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p2/", 0, 220)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p1/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p2/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p1/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p2/", 0, 227)) .build()); assertFileSystemAccesses( "SELECT * FROM test_transaction_log_not_cached", @@ -93,8 +93,8 @@ public void testTableDataCachedWhileTransactionLogNotCached() .addCopies(new CacheFileSystemTraceUtils.CacheOperation("Input.readTail", "00000000000000000002.checkpoint.parquet"), 2) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 220)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 220)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) .build()); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java index 78750f664d20..fb4555c6fb1d 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java @@ -61,7 +61,7 @@ public void testDefaults() .setTableStatisticsEnabled(true) .setExtendedStatisticsEnabled(true) .setCollectExtendedStatisticsOnWrite(true) - .setCompressionCodec(HiveCompressionCodec.SNAPPY) + .setCompressionCodec(HiveCompressionCodec.ZSTD) .setDeleteSchemaLocationsFallback(false) .setParquetTimeZone(TimeZone.getDefault().getID()) .setPerTransactionMetastoreCacheMaximumSize(1000) diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java index 25bb045d3c61..b7290e0839b0 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java @@ -435,8 +435,8 @@ public void verifyCompressionCodecsDataProvider() assertThat(onTrino().executeQuery("SHOW SESSION LIKE 'delta.compression_codec'")) .containsOnly(row( "delta.compression_codec", - "SNAPPY", - "SNAPPY", + "ZSTD", + "ZSTD", "varchar", "Compression codec to use when writing new data files. Possible values: " + Stream.of(compressionCodecs()) From c6c77cf91586aecb5dd23a7a3f7285edbdc3693f Mon Sep 17 00:00:00 2001 From: Raunaq Morarka Date: Tue, 24 Dec 2024 17:20:10 +0530 Subject: [PATCH 067/158] Add query execution metrics to JDBC QueryStats Adds planningTimeMillis, analysisTimeMillis, finishingTimeMillis, physicalInputBytes, physicalWrittenBytes and internalNetworkInputBytes to allow JDBC clients to get some important metrics about query execution --- .../java/io/trino/client/StatementStats.java | 72 +++++++++++++++++++ .../test/java/io/trino/client/TestRetry.java | 2 +- .../main/java/io/trino/jdbc/QueryStats.java | 54 ++++++++++++++ .../trino/jdbc/TestAsyncResultIterator.java | 4 ++ .../io/trino/jdbc/TestProgressMonitor.java | 2 +- .../io/trino/execution/QueryStateMachine.java | 4 ++ .../java/io/trino/server/BasicQueryStats.java | 44 ++++++++++++ .../trino/server/protocol/ProtocolUtil.java | 4 ++ .../execution/MockManagedQueryExecution.java | 4 ++ .../protocol/TestQueryDataSerialization.java | 4 ++ .../TestQueryResultsSerialization.java | 8 +++ .../memory/TestClusterMemoryLeakDetector.java | 4 ++ 12 files changed, 204 insertions(+), 2 deletions(-) diff --git a/client/trino-client/src/main/java/io/trino/client/StatementStats.java b/client/trino-client/src/main/java/io/trino/client/StatementStats.java index b0d3e94ef929..415b07d2c03a 100644 --- a/client/trino-client/src/main/java/io/trino/client/StatementStats.java +++ b/client/trino-client/src/main/java/io/trino/client/StatementStats.java @@ -36,14 +36,18 @@ public class StatementStats private final int queuedSplits; private final int runningSplits; private final int completedSplits; + private final long planningTimeMillis; + private final long analysisTimeMillis; private final long cpuTimeMillis; private final long wallTimeMillis; private final long queuedTimeMillis; private final long elapsedTimeMillis; + private final long finishingTimeMillis; private final long processedRows; private final long processedBytes; private final long physicalInputBytes; private final long physicalWrittenBytes; + private final long internalNetworkInputBytes; private final long peakMemoryBytes; private final long spilledBytes; private final StageStats rootStage; @@ -60,14 +64,18 @@ public StatementStats( @JsonProperty("queuedSplits") int queuedSplits, @JsonProperty("runningSplits") int runningSplits, @JsonProperty("completedSplits") int completedSplits, + @JsonProperty("planningTimeMillis") long planningTimeMillis, + @JsonProperty("analysisTimeMillis") long analysisTimeMillis, @JsonProperty("cpuTimeMillis") long cpuTimeMillis, @JsonProperty("wallTimeMillis") long wallTimeMillis, @JsonProperty("queuedTimeMillis") long queuedTimeMillis, @JsonProperty("elapsedTimeMillis") long elapsedTimeMillis, + @JsonProperty("finishingTimeMillis") long finishingTimeMillis, @JsonProperty("processedRows") long processedRows, @JsonProperty("processedBytes") long processedBytes, @JsonProperty("physicalInputBytes") long physicalInputBytes, @JsonProperty("physicalWrittenBytes") long physicalWrittenBytes, + @JsonProperty("internalNetworkInputBytes") long internalNetworkInputBytes, @JsonProperty("peakMemoryBytes") long peakMemoryBytes, @JsonProperty("spilledBytes") long spilledBytes, @JsonProperty("rootStage") StageStats rootStage) @@ -82,14 +90,18 @@ public StatementStats( this.queuedSplits = queuedSplits; this.runningSplits = runningSplits; this.completedSplits = completedSplits; + this.planningTimeMillis = planningTimeMillis; + this.analysisTimeMillis = analysisTimeMillis; this.cpuTimeMillis = cpuTimeMillis; this.wallTimeMillis = wallTimeMillis; this.queuedTimeMillis = queuedTimeMillis; this.elapsedTimeMillis = elapsedTimeMillis; + this.finishingTimeMillis = finishingTimeMillis; this.processedRows = processedRows; this.processedBytes = processedBytes; this.physicalInputBytes = physicalInputBytes; this.physicalWrittenBytes = physicalWrittenBytes; + this.internalNetworkInputBytes = internalNetworkInputBytes; this.peakMemoryBytes = peakMemoryBytes; this.spilledBytes = spilledBytes; this.rootStage = rootStage; @@ -155,6 +167,18 @@ public int getCompletedSplits() return completedSplits; } + @JsonProperty + public long getPlanningTimeMillis() + { + return planningTimeMillis; + } + + @JsonProperty + public long getAnalysisTimeMillis() + { + return analysisTimeMillis; + } + @JsonProperty public long getCpuTimeMillis() { @@ -179,6 +203,12 @@ public long getElapsedTimeMillis() return elapsedTimeMillis; } + @JsonProperty + public long getFinishingTimeMillis() + { + return finishingTimeMillis; + } + @JsonProperty public long getProcessedRows() { @@ -203,6 +233,12 @@ public long getPhysicalWrittenBytes() return physicalWrittenBytes; } + @JsonProperty + public long getInternalNetworkInputBytes() + { + return internalNetworkInputBytes; + } + @JsonProperty public long getPeakMemoryBytes() { @@ -236,14 +272,18 @@ public String toString() .add("queuedSplits", queuedSplits) .add("runningSplits", runningSplits) .add("completedSplits", completedSplits) + .add("planningTimeMillis", planningTimeMillis) + .add("analysisTimeMillis", analysisTimeMillis) .add("cpuTimeMillis", cpuTimeMillis) .add("wallTimeMillis", wallTimeMillis) .add("queuedTimeMillis", queuedTimeMillis) .add("elapsedTimeMillis", elapsedTimeMillis) + .add("finishingTimeMillis", finishingTimeMillis) .add("processedRows", processedRows) .add("processedBytes", processedBytes) .add("physicalInputBytes", physicalInputBytes) .add("physicalWrittenBytes", physicalWrittenBytes) + .add("internalNetworkInputBytes", internalNetworkInputBytes) .add("peakMemoryBytes", peakMemoryBytes) .add("spilledBytes", spilledBytes) .add("rootStage", rootStage) @@ -267,14 +307,18 @@ public static class Builder private int queuedSplits; private int runningSplits; private int completedSplits; + private long planningTimeMillis; + private long analysisTimeMillis; private long cpuTimeMillis; private long wallTimeMillis; private long queuedTimeMillis; private long elapsedTimeMillis; + private long finishingTimeMillis; private long processedRows; private long processedBytes; private long physicalInputBytes; private long physicalWrittenBytes; + private long internalNetworkInputBytes; private long peakMemoryBytes; private long spilledBytes; private StageStats rootStage; @@ -341,6 +385,18 @@ public Builder setCompletedSplits(int completedSplits) return this; } + public Builder setPlanningTimeMillis(long planningTimeMillis) + { + this.planningTimeMillis = planningTimeMillis; + return this; + } + + public Builder setAnalysisTimeMillis(long analysisTimeMillis) + { + this.analysisTimeMillis = analysisTimeMillis; + return this; + } + public Builder setCpuTimeMillis(long cpuTimeMillis) { this.cpuTimeMillis = cpuTimeMillis; @@ -365,6 +421,12 @@ public Builder setElapsedTimeMillis(long elapsedTimeMillis) return this; } + public Builder setFinishingTimeMillis(long finishingTimeMillis) + { + this.finishingTimeMillis = finishingTimeMillis; + return this; + } + public Builder setProcessedRows(long processedRows) { this.processedRows = processedRows; @@ -389,6 +451,12 @@ public Builder setPhysicalWrittenBytes(long physicalWrittenBytes) return this; } + public Builder setInternalNetworkInputBytes(long internalNetworkInputBytes) + { + this.internalNetworkInputBytes = internalNetworkInputBytes; + return this; + } + public Builder setPeakMemoryBytes(long peakMemoryBytes) { this.peakMemoryBytes = peakMemoryBytes; @@ -420,14 +488,18 @@ public StatementStats build() queuedSplits, runningSplits, completedSplits, + planningTimeMillis, + analysisTimeMillis, cpuTimeMillis, wallTimeMillis, queuedTimeMillis, elapsedTimeMillis, + finishingTimeMillis, processedRows, processedBytes, physicalInputBytes, physicalWrittenBytes, + internalNetworkInputBytes, peakMemoryBytes, spilledBytes, rootStage); diff --git a/client/trino-client/src/test/java/io/trino/client/TestRetry.java b/client/trino-client/src/test/java/io/trino/client/TestRetry.java index ab72f2db0278..d000d2da97b5 100644 --- a/client/trino-client/src/test/java/io/trino/client/TestRetry.java +++ b/client/trino-client/src/test/java/io/trino/client/TestRetry.java @@ -143,7 +143,7 @@ private String newQueryResults(String state) TypedQueryData.of(IntStream.range(0, numRecords) .mapToObj(index -> Stream.of((Object) index, "a").collect(toList())) .collect(toList())), - new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), + new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), null, ImmutableList.of(), null, diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java index d09f3297de66..cac77f441ca0 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java @@ -32,13 +32,19 @@ public final class QueryStats private final int queuedSplits; private final int runningSplits; private final int completedSplits; + private final long planningTimeMillis; + private final long analysisTimeMillis; private final long cpuTimeMillis; private final long wallTimeMillis; private final long queuedTimeMillis; private final long elapsedTimeMillis; + private final long finishingTimeMillis; private final long processedRows; private final long processedBytes; private final long peakMemoryBytes; + private final long physicalInputBytes; + private final long physicalWrittenBytes; + private final long internalNetworkInputBytes; private final Optional rootStage; public QueryStats( @@ -52,13 +58,19 @@ public QueryStats( int queuedSplits, int runningSplits, int completedSplits, + long planningTimeMillis, + long analysisTimeMillis, long cpuTimeMillis, long wallTimeMillis, long queuedTimeMillis, long elapsedTimeMillis, + long finishingTimeMillis, long processedRows, long processedBytes, long peakMemoryBytes, + long physicalInputBytes, + long physicalWrittenBytes, + long internalNetworkInputBytes, Optional rootStage) { this.queryId = requireNonNull(queryId, "queryId is null"); @@ -71,13 +83,19 @@ public QueryStats( this.queuedSplits = queuedSplits; this.runningSplits = runningSplits; this.completedSplits = completedSplits; + this.planningTimeMillis = planningTimeMillis; + this.analysisTimeMillis = analysisTimeMillis; this.cpuTimeMillis = cpuTimeMillis; this.wallTimeMillis = wallTimeMillis; this.queuedTimeMillis = queuedTimeMillis; this.elapsedTimeMillis = elapsedTimeMillis; + this.finishingTimeMillis = finishingTimeMillis; this.processedRows = processedRows; this.processedBytes = processedBytes; this.peakMemoryBytes = peakMemoryBytes; + this.physicalInputBytes = physicalInputBytes; + this.physicalWrittenBytes = physicalWrittenBytes; + this.internalNetworkInputBytes = internalNetworkInputBytes; this.rootStage = requireNonNull(rootStage, "rootStage is null"); } @@ -94,13 +112,19 @@ static QueryStats create(String queryId, StatementStats stats) stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits(), + stats.getPlanningTimeMillis(), + stats.getAnalysisTimeMillis(), stats.getCpuTimeMillis(), stats.getWallTimeMillis(), stats.getQueuedTimeMillis(), stats.getElapsedTimeMillis(), + stats.getFinishingTimeMillis(), stats.getProcessedRows(), stats.getProcessedBytes(), stats.getPeakMemoryBytes(), + stats.getPhysicalInputBytes(), + stats.getPhysicalWrittenBytes(), + stats.getInternalNetworkInputBytes(), Optional.ofNullable(stats.getRootStage()).map(StageStats::create)); } @@ -154,6 +178,16 @@ public int getCompletedSplits() return completedSplits; } + public long getPlanningTimeMillis() + { + return planningTimeMillis; + } + + public long getAnalysisTimeMillis() + { + return analysisTimeMillis; + } + public long getCpuTimeMillis() { return cpuTimeMillis; @@ -174,6 +208,11 @@ public long getElapsedTimeMillis() return elapsedTimeMillis; } + public long getFinishingTimeMillis() + { + return finishingTimeMillis; + } + public long getProcessedRows() { return processedRows; @@ -189,6 +228,21 @@ public long getPeakMemoryBytes() return peakMemoryBytes; } + public long getPhysicalInputBytes() + { + return physicalInputBytes; + } + + public long getPhysicalWrittenBytes() + { + return physicalWrittenBytes; + } + + public long getInternalNetworkInputBytes() + { + return internalNetworkInputBytes; + } + public Optional getRootStage() { return rootStage; diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java index dd41950adfd6..7aeaddedebd4 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java @@ -333,6 +333,10 @@ public StatementStats getStats() 100, 100, 100, + 100, + 100, + 100, + 100, StageStats.builder() .setStageId("id") .setDone(false) diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java index 000b47195e8c..09d92c170579 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java @@ -99,7 +99,7 @@ private String newQueryResults(Integer partialCancelId, Integer nextUriId, List< nextUriId == null ? null : server.url(format("/v1/statement/%s/%s", queryId, nextUriId)).uri(), responseColumns, TypedQueryData.of(data), - new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), + new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), null, ImmutableList.of(), null, diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java index a46a02b28f2c..bd2aabb64d62 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java @@ -547,6 +547,7 @@ private BasicQueryStats createBasicQueryStats(BasicStageStats stageStats) stageStats.getSpilledDataSize(), stageStats.getPhysicalInputDataSize(), stageStats.getPhysicalWrittenDataSize(), + stageStats.getInternalNetworkInputDataSize(), stageStats.getCumulativeUserMemory(), stageStats.getFailedCumulativeUserMemory(), @@ -555,10 +556,13 @@ private BasicQueryStats createBasicQueryStats(BasicStageStats stageStats) succinctBytes(getPeakUserMemoryInBytes()), succinctBytes(getPeakTotalMemoryInBytes()), + queryStateTimer.getPlanningTime(), + queryStateTimer.getAnalysisTime(), stageStats.getTotalCpuTime(), stageStats.getFailedCpuTime(), stageStats.getTotalScheduledTime(), stageStats.getFailedScheduledTime(), + queryStateTimer.getFinishingTime(), stageStats.isFullyBlocked(), stageStats.getBlockedReasons(), diff --git a/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java b/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java index 76d7b10400ae..cfd3378db7b3 100644 --- a/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java +++ b/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java @@ -57,16 +57,20 @@ public class BasicQueryStats private final DataSize spilledDataSize; private final DataSize physicalInputDataSize; private final DataSize physicalWrittenDataSize; + private final DataSize internalNetworkInputDataSize; private final double cumulativeUserMemory; private final double failedCumulativeUserMemory; private final DataSize userMemoryReservation; private final DataSize totalMemoryReservation; private final DataSize peakUserMemoryReservation; private final DataSize peakTotalMemoryReservation; + private final Duration planningTime; + private final Duration analysisTime; private final Duration totalCpuTime; private final Duration failedCpuTime; private final Duration totalScheduledTime; private final Duration failedScheduledTime; + private final Duration finishingTime; private final boolean fullyBlocked; private final Set blockedReasons; @@ -92,16 +96,20 @@ public BasicQueryStats( @JsonProperty("spilledDataSize") DataSize spilledDataSize, @JsonProperty("physicalInputDataSize") DataSize physicalInputDataSize, @JsonProperty("physicalWrittenDataSize") DataSize physicalWrittenDataSize, + @JsonProperty("internalNetworkInputDataSize") DataSize internalNetworkInputDataSize, @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, @JsonProperty("failedCumulativeUserMemory") double failedCumulativeUserMemory, @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, @JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation, + @JsonProperty("planningTime") Duration planningTime, + @JsonProperty("analysisTime") Duration analysisTime, @JsonProperty("totalCpuTime") Duration totalCpuTime, @JsonProperty("failedCpuTime") Duration failedCpuTime, @JsonProperty("totalScheduledTime") Duration totalScheduledTime, @JsonProperty("failedScheduledTime") Duration failedScheduledTime, + @JsonProperty("finishingTime") Duration finishingTime, @JsonProperty("fullyBlocked") boolean fullyBlocked, @JsonProperty("blockedReasons") Set blockedReasons, @JsonProperty("progressPercentage") OptionalDouble progressPercentage, @@ -133,6 +141,7 @@ public BasicQueryStats( this.spilledDataSize = spilledDataSize; this.physicalInputDataSize = physicalInputDataSize; this.physicalWrittenDataSize = physicalWrittenDataSize; + this.internalNetworkInputDataSize = internalNetworkInputDataSize; this.cumulativeUserMemory = cumulativeUserMemory; this.failedCumulativeUserMemory = failedCumulativeUserMemory; @@ -140,10 +149,13 @@ public BasicQueryStats( this.totalMemoryReservation = totalMemoryReservation; this.peakUserMemoryReservation = peakUserMemoryReservation; this.peakTotalMemoryReservation = peakTotalMemoryReservation; + this.planningTime = planningTime; + this.analysisTime = analysisTime; this.totalCpuTime = totalCpuTime; this.failedCpuTime = failedCpuTime; this.totalScheduledTime = totalScheduledTime; this.failedScheduledTime = failedScheduledTime; + this.finishingTime = finishingTime; this.fullyBlocked = fullyBlocked; this.blockedReasons = ImmutableSet.copyOf(requireNonNull(blockedReasons, "blockedReasons is null")); @@ -170,16 +182,20 @@ public BasicQueryStats(QueryStats queryStats) queryStats.getSpilledDataSize(), queryStats.getPhysicalInputDataSize(), queryStats.getPhysicalWrittenDataSize(), + queryStats.getInternalNetworkInputDataSize(), queryStats.getCumulativeUserMemory(), queryStats.getFailedCumulativeUserMemory(), queryStats.getUserMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), queryStats.getPeakTotalMemoryReservation(), + queryStats.getPlanningTime(), + queryStats.getAnalysisTime(), queryStats.getTotalCpuTime(), queryStats.getFailedCpuTime(), queryStats.getTotalScheduledTime(), queryStats.getFailedScheduledTime(), + queryStats.getFinishingTime(), queryStats.isFullyBlocked(), queryStats.getBlockedReasons(), queryStats.getProgressPercentage(), @@ -206,6 +222,7 @@ public static BasicQueryStats immediateFailureQueryStats() DataSize.ofBytes(0), DataSize.ofBytes(0), DataSize.ofBytes(0), + DataSize.ofBytes(0), 0, 0, DataSize.ofBytes(0), @@ -216,6 +233,9 @@ public static BasicQueryStats immediateFailureQueryStats() new Duration(0, MILLISECONDS), new Duration(0, MILLISECONDS), new Duration(0, MILLISECONDS), + new Duration(0, MILLISECONDS), + new Duration(0, MILLISECONDS), + new Duration(0, MILLISECONDS), false, ImmutableSet.of(), OptionalDouble.empty(), @@ -318,6 +338,12 @@ public DataSize getPhysicalWrittenDataSize() return physicalWrittenDataSize; } + @JsonProperty + public DataSize getInternalNetworkInputDataSize() + { + return internalNetworkInputDataSize; + } + @JsonProperty public double getCumulativeUserMemory() { @@ -354,6 +380,18 @@ public DataSize getPeakTotalMemoryReservation() return peakTotalMemoryReservation; } + @JsonProperty + public Duration getPlanningTime() + { + return planningTime; + } + + @JsonProperty + public Duration getAnalysisTime() + { + return analysisTime; + } + @JsonProperty public Duration getTotalCpuTime() { @@ -378,6 +416,12 @@ public Duration getFailedScheduledTime() return failedScheduledTime; } + @JsonProperty + public Duration getFinishingTime() + { + return finishingTime; + } + @JsonProperty public boolean isFullyBlocked() { diff --git a/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java b/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java index 61adcb946f45..c8997ebe0125 100644 --- a/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java +++ b/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java @@ -182,14 +182,18 @@ public static StatementStats toStatementStats(ResultQueryInfo queryInfo) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) + .setPlanningTimeMillis(queryStats.getPlanningTime().toMillis()) + .setAnalysisTimeMillis(queryStats.getAnalysisTime().toMillis()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) + .setFinishingTimeMillis(queryStats.getFinishingTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPhysicalInputBytes(queryStats.getPhysicalInputDataSize().toBytes()) .setPhysicalWrittenBytes(queryStats.getPhysicalWrittenDataSize().toBytes()) + .setInternalNetworkInputBytes(queryStats.getInternalNetworkInputDataSize().toBytes()) .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) .setRootStage(rootStageStats) diff --git a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java index 9b3d998c0540..787e3e00dab7 100644 --- a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java +++ b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java @@ -136,6 +136,7 @@ public BasicQueryInfo getBasicQueryInfo() DataSize.ofBytes(13), DataSize.ofBytes(13), DataSize.ofBytes(13), + DataSize.ofBytes(13), 16.0, 17.0, memoryUsage, @@ -146,6 +147,9 @@ public BasicQueryInfo getBasicQueryInfo() new Duration(21, NANOSECONDS), new Duration(22, NANOSECONDS), new Duration(23, NANOSECONDS), + new Duration(24, NANOSECONDS), + new Duration(25, NANOSECONDS), + new Duration(26, NANOSECONDS), false, ImmutableSet.of(), OptionalDouble.empty(), diff --git a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java index d1beddc5ca75..40b8f7797b41 100644 --- a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java +++ b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java @@ -245,14 +245,18 @@ private String queryResultsJson(String expectedDataField) "queuedSplits": 0, "runningSplits": 0, "completedSplits": 0, + "planningTimeMillis": 0, + "analysisTimeMillis": 0, "cpuTimeMillis": 0, "wallTimeMillis": 0, "queuedTimeMillis": 0, "elapsedTimeMillis": 0, + "finishingTimeMillis": 0, "processedRows": 0, "processedBytes": 0, "physicalInputBytes": 0, "physicalWrittenBytes": 0, + "internalNetworkInputBytes": 0, "peakMemoryBytes": 0, "spilledBytes": 0 }, diff --git a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java index 56fc7a5cad13..d1ae9c5cabfa 100644 --- a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java +++ b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java @@ -77,14 +77,18 @@ public void testNullDataSerialization() "queuedSplits" : 0, "runningSplits" : 0, "completedSplits" : 0, + "planningTimeMillis": 0, + "analysisTimeMillis": 0, "cpuTimeMillis" : 0, "wallTimeMillis" : 0, "queuedTimeMillis" : 0, "elapsedTimeMillis" : 0, + "finishingTimeMillis": 0, "processedRows" : 0, "processedBytes" : 0, "physicalInputBytes" : 0, "physicalWrittenBytes" : 0, + "internalNetworkInputBytes": 0, "peakMemoryBytes" : 0, "spilledBytes" : 0 }, @@ -153,14 +157,18 @@ private String queryResultsJson(String expectedDataField) "queuedSplits" : 0, "runningSplits" : 0, "completedSplits" : 0, + "planningTimeMillis": 0, + "analysisTimeMillis": 0, "cpuTimeMillis" : 0, "wallTimeMillis" : 0, "queuedTimeMillis" : 0, "elapsedTimeMillis" : 0, + "finishingTimeMillis": 0, "processedRows" : 0, "processedBytes" : 0, "physicalInputBytes" : 0, "physicalWrittenBytes" : 0, + "internalNetworkInputBytes": 0, "peakMemoryBytes" : 0, "spilledBytes" : 0 }, diff --git a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java index f6af3da9e751..6c10463a742d 100644 --- a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java +++ b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java @@ -95,6 +95,7 @@ private static BasicQueryInfo createQueryInfo(String queryId, QueryState state) DataSize.valueOf("23GB"), DataSize.valueOf("23GB"), DataSize.valueOf("23GB"), + DataSize.valueOf("23GB"), 24, 25, DataSize.valueOf("26GB"), @@ -105,6 +106,9 @@ private static BasicQueryInfo createQueryInfo(String queryId, QueryState state) new Duration(31, MINUTES), new Duration(32, MINUTES), new Duration(33, MINUTES), + new Duration(34, MINUTES), + new Duration(35, MINUTES), + new Duration(36, MINUTES), true, ImmutableSet.of(WAITING_FOR_MEMORY), OptionalDouble.of(20), From bddec4947c77a37f15ee5940ae02957cf354b823 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 25 Dec 2024 08:29:14 +0900 Subject: [PATCH 068/158] Allow configuring parquet_bloom_filter_columns in Iceberg --- .../trino/plugin/iceberg/IcebergMetadata.java | 20 +++++++++++++++++++ .../io/trino/plugin/iceberg/IcebergUtil.java | 2 +- .../TestIcebergParquetWithBloomFilters.java | 19 ++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 73afe67c3a2c..1a2f1f01080b 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -133,6 +133,7 @@ import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeleteFiles; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.IsolationLevel; @@ -281,12 +282,14 @@ import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.OBJECT_STORE_LAYOUT_ENABLED_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.SORTED_BY_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.getPartitioning; import static io.trino.plugin.iceberg.IcebergTableProperties.getTableLocation; import static io.trino.plugin.iceberg.IcebergUtil.buildPath; import static io.trino.plugin.iceberg.IcebergUtil.canEnforceColumnConstraintInSpecs; +import static io.trino.plugin.iceberg.IcebergUtil.checkFormatForProperty; import static io.trino.plugin.iceberg.IcebergUtil.commit; import static io.trino.plugin.iceberg.IcebergUtil.createColumnHandle; import static io.trino.plugin.iceberg.IcebergUtil.deserializePartitionValue; @@ -370,6 +373,7 @@ import static org.apache.iceberg.TableProperties.DELETE_ISOLATION_LEVEL_DEFAULT; import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.OBJECT_STORE_ENABLED; +import static org.apache.iceberg.TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX; import static org.apache.iceberg.TableProperties.WRITE_DATA_LOCATION; import static org.apache.iceberg.TableProperties.WRITE_LOCATION_PROVIDER_IMPL; import static org.apache.iceberg.expressions.Expressions.alwaysTrue; @@ -392,6 +396,7 @@ public class IcebergMetadata .add(FORMAT_VERSION_PROPERTY) .add(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY) .add(DATA_LOCATION_PROPERTY) + .add(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY) .add(PARTITIONING_PROPERTY) .add(SORTED_BY_PROPERTY) .build(); @@ -2273,6 +2278,21 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta extraProperties.forEach(updateProperties::set); } + if (properties.containsKey(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY)) { + checkFormatForProperty(getFileFormat(icebergTable).toIceberg(), FileFormat.PARQUET, PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY); + //noinspection unchecked + List parquetBloomFilterColumns = (List) properties.get(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY) + .orElseThrow(() -> new IllegalArgumentException("The parquet_bloom_filter_columns property cannot be empty")); + if (parquetBloomFilterColumns.isEmpty()) { + icebergTable.properties().keySet().stream() + .filter(key -> key.startsWith(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX)) + .forEach(updateProperties::remove); + } + else { + parquetBloomFilterColumns.forEach(column -> updateProperties.set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column, "true")); + } + } + if (properties.containsKey(FILE_FORMAT_PROPERTY)) { IcebergFileFormat fileFormat = (IcebergFileFormat) properties.get(FILE_FORMAT_PROPERTY) .orElseThrow(() -> new IllegalArgumentException("The format property cannot be empty")); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java index ecd82d188260..4fcef5f2da55 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java @@ -986,7 +986,7 @@ public static long getSnapshotIdAsOfTime(Table table, long epochMillis) .snapshotId(); } - private static void checkFormatForProperty(FileFormat actualStorageFormat, FileFormat expectedStorageFormat, String propertyName) + public static void checkFormatForProperty(FileFormat actualStorageFormat, FileFormat expectedStorageFormat, String propertyName) { if (actualStorageFormat != expectedStorageFormat) { throw new TrinoException(INVALID_TABLE_PROPERTY, format("Cannot specify %s table property for storage format: %s", propertyName, actualStorageFormat)); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java index e29607eeaf12..f8d5e3591ed0 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java @@ -58,6 +58,25 @@ public void testBloomFilterPropertiesArePersistedDuringCreate() "format = 'parquet'," + "parquet_bloom_filter_columns = array['a','B'])"); + verifyTableProperties(tableName); + } + + @Test + void testBloomFilterPropertiesArePersistedDuringSetProperties() + { + String tableName = "test_metadata_write_properties_" + randomNameSuffix(); + assertQuerySucceeds("CREATE TABLE " + tableName + "(A bigint, b bigint, c bigint)"); + + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY['a','B']"); + verifyTableProperties(tableName); + + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY[]"); + assertThat((String) computeScalar("SHOW CREATE TABLE " + tableName)) + .doesNotContain("parquet_bloom_filter_columns"); + } + + private void verifyTableProperties(String tableName) + { MaterializedResult actualProperties = computeActual("SELECT * FROM \"" + tableName + "$properties\""); assertThat(actualProperties).isNotNull(); MaterializedResult expectedProperties = resultBuilder(getSession()) From 93f6a49a934cfc49eb9922cb45a41d1cc17a145f Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 25 Dec 2024 12:52:46 +0900 Subject: [PATCH 069/158] Verify invalid bloom filter properties in Iceberg --- .../trino/plugin/iceberg/IcebergMetadata.java | 2 ++ .../io/trino/plugin/iceberg/IcebergUtil.java | 6 +++--- .../TestIcebergParquetWithBloomFilters.java | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 1a2f1f01080b..e1dcdde7a750 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -307,6 +307,7 @@ import static io.trino.plugin.iceberg.IcebergUtil.getTopLevelColumns; import static io.trino.plugin.iceberg.IcebergUtil.newCreateTableTransaction; import static io.trino.plugin.iceberg.IcebergUtil.schemaFromMetadata; +import static io.trino.plugin.iceberg.IcebergUtil.validateParquetBloomFilterColumns; import static io.trino.plugin.iceberg.IcebergUtil.verifyExtraProperties; import static io.trino.plugin.iceberg.PartitionFields.parsePartitionFields; import static io.trino.plugin.iceberg.SortFieldUtils.parseSortFields; @@ -2283,6 +2284,7 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta //noinspection unchecked List parquetBloomFilterColumns = (List) properties.get(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY) .orElseThrow(() -> new IllegalArgumentException("The parquet_bloom_filter_columns property cannot be empty")); + validateParquetBloomFilterColumns(getColumnMetadatas(SchemaParser.fromJson(table.getTableSchemaJson()), typeManager), parquetBloomFilterColumns); if (parquetBloomFilterColumns.isEmpty()) { icebergTable.properties().keySet().stream() .filter(key -> key.startsWith(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX)) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java index 4fcef5f2da55..ee3a2160eb63 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java @@ -876,7 +876,7 @@ public static Map createTableProperties(ConnectorTableMetadata t List parquetBloomFilterColumns = IcebergTableProperties.getParquetBloomFilterColumns(tableMetadata.getProperties()); if (!parquetBloomFilterColumns.isEmpty()) { checkFormatForProperty(fileFormat.toIceberg(), FileFormat.PARQUET, PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY); - validateParquetBloomFilterColumns(tableMetadata, parquetBloomFilterColumns); + validateParquetBloomFilterColumns(tableMetadata.getColumns(), parquetBloomFilterColumns); for (String column : parquetBloomFilterColumns) { propertiesBuilder.put(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column, "true"); } @@ -1003,9 +1003,9 @@ private static void validateOrcBloomFilterColumns(ConnectorTableMetadata tableMe } } - private static void validateParquetBloomFilterColumns(ConnectorTableMetadata tableMetadata, List parquetBloomFilterColumns) + public static void validateParquetBloomFilterColumns(List columns, List parquetBloomFilterColumns) { - Map columnTypes = tableMetadata.getColumns().stream() + Map columnTypes = columns.stream() .collect(toImmutableMap(ColumnMetadata::getName, ColumnMetadata::getType)); for (String column : parquetBloomFilterColumns) { Type type = columnTypes.get(column); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java index f8d5e3591ed0..bd85fcaa82f9 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java @@ -75,6 +75,26 @@ void testBloomFilterPropertiesArePersistedDuringSetProperties() .doesNotContain("parquet_bloom_filter_columns"); } + @Test + void testInvalidBloomFilterProperties() + { + String tableName = "test_invalid_bloom_filter_properties_" + randomNameSuffix(); + assertQueryFails( + "CREATE TABLE " + tableName + "(x int) WITH (parquet_bloom_filter_columns = ARRAY['missing_column'])", + "Parquet Bloom filter column missing_column not present in schema"); + assertQueryFails( + "CREATE TABLE " + tableName + "(x array(int)) WITH (parquet_bloom_filter_columns = ARRAY['x'])", + "\\QParquet Bloom filter column x has unsupported type array(integer)"); + + assertQuerySucceeds("CREATE TABLE " + tableName + "(x array(integer))"); + assertQueryFails( + "ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY['missing_column']", + "Parquet Bloom filter column missing_column not present in schema"); + assertQueryFails( + "ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY['x']", + "\\QParquet Bloom filter column x has unsupported type array(integer)"); + } + private void verifyTableProperties(String tableName) { MaterializedResult actualProperties = computeActual("SELECT * FROM \"" + tableName + "$properties\""); From cb8df9d26433920eecd0963fbb3c25b6433f7784 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 25 Dec 2024 13:42:28 +0900 Subject: [PATCH 070/158] Remove unspecified bloom filter when setting properties in Iceberg --- .../io/trino/plugin/iceberg/IcebergMetadata.java | 16 ++++++++-------- .../TestIcebergParquetWithBloomFilters.java | 4 ++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index e1dcdde7a750..0e3843c56f39 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -2285,14 +2285,14 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta List parquetBloomFilterColumns = (List) properties.get(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY) .orElseThrow(() -> new IllegalArgumentException("The parquet_bloom_filter_columns property cannot be empty")); validateParquetBloomFilterColumns(getColumnMetadatas(SchemaParser.fromJson(table.getTableSchemaJson()), typeManager), parquetBloomFilterColumns); - if (parquetBloomFilterColumns.isEmpty()) { - icebergTable.properties().keySet().stream() - .filter(key -> key.startsWith(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX)) - .forEach(updateProperties::remove); - } - else { - parquetBloomFilterColumns.forEach(column -> updateProperties.set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column, "true")); - } + + Set existingParquetBloomFilterColumns = icebergTable.properties().keySet().stream() + .filter(key -> key.startsWith(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX)) + .map(key -> key.substring(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX.length())) + .collect(toImmutableSet()); + Set removeParquetBloomFilterColumns = Sets.difference(existingParquetBloomFilterColumns, Set.copyOf(parquetBloomFilterColumns)); + removeParquetBloomFilterColumns.forEach(column -> updateProperties.remove(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column)); + parquetBloomFilterColumns.forEach(column -> updateProperties.set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column, "true")); } if (properties.containsKey(FILE_FORMAT_PROPERTY)) { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java index bd85fcaa82f9..a3f4648c3a7b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetWithBloomFilters.java @@ -70,6 +70,10 @@ void testBloomFilterPropertiesArePersistedDuringSetProperties() assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY['a','B']"); verifyTableProperties(tableName); + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY['a']"); + assertThat((String) computeScalar("SHOW CREATE TABLE " + tableName)) + .contains("parquet_bloom_filter_columns = ARRAY['a']"); + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY[]"); assertThat((String) computeScalar("SHOW CREATE TABLE " + tableName)) .doesNotContain("parquet_bloom_filter_columns"); From 6ff4cada54081995ccbf1f9aaf88261b70be63f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 10 Nov 2024 11:05:27 +0100 Subject: [PATCH 071/158] Support managing views in the Faker connector --- plugin/trino-faker/pom.xml | 6 + .../io/trino/plugin/faker/ColumnInfo.java | 6 + .../io/trino/plugin/faker/FakerMetadata.java | 169 +++++++++- .../trino/plugin/faker/FakerQueryRunner.java | 4 + .../io/trino/plugin/faker/TestFakerViews.java | 291 ++++++++++++++++++ 5 files changed, 460 insertions(+), 16 deletions(-) create mode 100644 plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerViews.java diff --git a/plugin/trino-faker/pom.xml b/plugin/trino-faker/pom.xml index b9f8f35e05b0..060abf3c9b14 100644 --- a/plugin/trino-faker/pom.xml +++ b/plugin/trino-faker/pom.xml @@ -157,6 +157,12 @@ test + + io.trino + trino-testing-services + test + + io.trino trino-tpch diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java index e2c66de8d1dc..55f84eb9df03 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java @@ -14,6 +14,7 @@ package io.trino.plugin.faker; import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.type.Type; import java.util.Optional; @@ -35,6 +36,11 @@ public String name() return metadata.getName(); } + public Type type() + { + return metadata.getType(); + } + @Override public String toString() { diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java index 0e3be1ef747f..9312f33e1f41 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java @@ -15,6 +15,7 @@ package io.trino.plugin.faker; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.concurrent.GuardedBy; import io.airlift.slice.Slice; import io.trino.spi.TrinoException; @@ -28,15 +29,16 @@ import io.trino.spi.connector.ConnectorTableLayout; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTableVersion; +import io.trino.spi.connector.ConnectorViewDefinition; import io.trino.spi.connector.Constraint; import io.trino.spi.connector.ConstraintApplicationResult; import io.trino.spi.connector.LimitApplicationResult; +import io.trino.spi.connector.RelationColumnsMetadata; import io.trino.spi.connector.RetryMode; import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaNotFoundException; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.spi.connector.TableColumnsMetadata; +import io.trino.spi.connector.ViewNotFoundException; import io.trino.spi.function.BoundSignature; import io.trino.spi.function.FunctionDependencyDeclaration; import io.trino.spi.function.FunctionId; @@ -59,14 +61,20 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.Maps.filterKeys; +import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_REFERENCE; +import static io.trino.spi.StandardErrorCode.NOT_FOUND; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.spi.StandardErrorCode.SCHEMA_ALREADY_EXISTS; import static io.trino.spi.StandardErrorCode.SCHEMA_NOT_FOUND; @@ -85,11 +93,12 @@ public class FakerMetadata private final List schemas = new ArrayList<>(); private final double nullProbability; private final long defaultLimit; + private final FakerFunctionProvider functionsProvider; @GuardedBy("this") private final Map tables = new HashMap<>(); - - private final FakerFunctionProvider functionsProvider; + @GuardedBy("this") + private final Map views = new HashMap<>(); @Inject public FakerMetadata(FakerConfig config, FakerFunctionProvider functionProvider) @@ -168,7 +177,7 @@ public synchronized ConnectorTableMetadata getTableMetadata( @Override public synchronized List listTables(ConnectorSession session, Optional schemaName) { - return tables.keySet().stream() + return Stream.concat(tables.keySet().stream(), views.keySet().stream()) .filter(table -> schemaName.map(table.getSchemaName()::contentEquals).orElse(true)) .collect(toImmutableList()); } @@ -197,13 +206,33 @@ public synchronized ColumnMetadata getColumnMetadata( } @Override - public synchronized Iterator streamTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + public synchronized Iterator streamRelationColumns( + ConnectorSession session, + Optional schemaName, + UnaryOperator> relationFilter) { - return tables.entrySet().stream() + Map relationColumns = new HashMap<>(); + + SchemaTablePrefix prefix = schemaName.map(SchemaTablePrefix::new) + .orElseGet(SchemaTablePrefix::new); + tables.entrySet().stream() .filter(entry -> prefix.matches(entry.getKey())) - .map(entry -> TableColumnsMetadata.forTable( - entry.getKey(), - entry.getValue().columns().stream().map(ColumnInfo::metadata).collect(toImmutableList()))) + .forEach(entry -> { + SchemaTableName name = entry.getKey(); + RelationColumnsMetadata columns = RelationColumnsMetadata.forTable( + name, + entry.getValue().columns().stream() + .map(ColumnInfo::metadata) + .collect(toImmutableList())); + relationColumns.put(name, columns); + }); + + for (Map.Entry entry : getViews(session, schemaName).entrySet()) { + relationColumns.put(entry.getKey(), RelationColumnsMetadata.forView(entry.getKey(), entry.getValue().getColumns())); + } + + return relationFilter.apply(relationColumns.keySet()).stream() + .map(relationColumns::get) .iterator(); } @@ -219,12 +248,12 @@ public synchronized void renameTable(ConnectorSession session, ConnectorTableHan { checkSchemaExists(newTableName.getSchemaName()); checkTableNotExists(newTableName); + checkViewNotExists(newTableName); FakerTableHandle handle = (FakerTableHandle) tableHandle; SchemaTableName oldTableName = handle.schemaTableName(); - tables.remove(oldTableName); - tables.put(newTableName, tables.get(oldTableName)); + tables.put(newTableName, tables.remove(oldTableName)); } @Override @@ -293,6 +322,7 @@ public synchronized FakerOutputTableHandle beginCreateTable(ConnectorSession ses SchemaTableName tableName = tableMetadata.getTable(); SchemaInfo schema = getSchema(tableName.getSchemaName()); checkTableNotExists(tableMetadata.getTable()); + checkViewNotExists(tableMetadata.getTable()); double schemaNullProbability = (double) schema.properties().getOrDefault(SchemaInfo.NULL_PROBABILITY_PROPERTY, nullProbability); double tableNullProbability = (double) tableMetadata.getProperties().getOrDefault(TableInfo.NULL_PROBABILITY_PROPERTY, schemaNullProbability); @@ -349,7 +379,7 @@ private boolean isCharacterColumn(ColumnMetadata column) private synchronized void checkSchemaExists(String schemaName) { if (schemas.stream().noneMatch(schema -> schema.name().equals(schemaName))) { - throw new SchemaNotFoundException(schemaName); + throw new TrinoException(SCHEMA_NOT_FOUND, format("Schema '%s' does not exist", schemaName)); } } @@ -360,8 +390,19 @@ private synchronized void checkTableNotExists(SchemaTableName tableName) } } + private synchronized void checkViewNotExists(SchemaTableName viewName) + { + if (views.containsKey(viewName)) { + throw new TrinoException(ALREADY_EXISTS, format("View '%s' already exists", viewName)); + } + } + @Override - public synchronized Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) + public synchronized Optional finishCreateTable( + ConnectorSession session, + ConnectorOutputTableHandle tableHandle, + Collection fragments, + Collection computedStatistics) { requireNonNull(tableHandle, "tableHandle is null"); FakerOutputTableHandle fakerOutputHandle = (FakerOutputTableHandle) tableHandle; @@ -371,12 +412,108 @@ public synchronized Optional finishCreateTable(Connecto TableInfo info = tables.get(tableName); requireNonNull(info, "info is null"); - // TODO ensure fragments is empty? - tables.put(tableName, new TableInfo(info.columns(), info.properties(), info.comment())); + return Optional.empty(); } + @Override + public synchronized List listViews(ConnectorSession session, Optional schemaName) + { + return views.keySet().stream() + .filter(table -> schemaName.map(table.getSchemaName()::equals).orElse(true)) + .collect(toImmutableList()); + } + + @Override + public synchronized Map getViews(ConnectorSession session, Optional schemaName) + { + SchemaTablePrefix prefix = schemaName.map(SchemaTablePrefix::new).orElseGet(SchemaTablePrefix::new); + return ImmutableMap.copyOf(filterKeys(views, prefix::matches)); + } + + @Override + public synchronized Optional getView(ConnectorSession session, SchemaTableName viewName) + { + return Optional.ofNullable(views.get(viewName)); + } + + @Override + public synchronized void createView( + ConnectorSession session, + SchemaTableName viewName, + ConnectorViewDefinition definition, + Map viewProperties, + boolean replace) + { + checkArgument(viewProperties.isEmpty(), "This connector does not support creating views with properties"); + checkSchemaExists(viewName.getSchemaName()); + checkTableNotExists(viewName); + + if (replace) { + views.put(viewName, definition); + } + else if (views.putIfAbsent(viewName, definition) != null) { + throw new TrinoException(ALREADY_EXISTS, "View '%s' already exists".formatted(viewName)); + } + } + + @Override + public synchronized void setViewComment(ConnectorSession session, SchemaTableName viewName, Optional comment) + { + ConnectorViewDefinition view = getView(session, viewName).orElseThrow(() -> new ViewNotFoundException(viewName)); + views.put(viewName, new ConnectorViewDefinition( + view.getOriginalSql(), + view.getCatalog(), + view.getSchema(), + view.getColumns(), + comment, + view.getOwner(), + view.isRunAsInvoker(), + view.getPath())); + } + + @Override + public synchronized void setViewColumnComment(ConnectorSession session, SchemaTableName viewName, String columnName, Optional comment) + { + ConnectorViewDefinition view = getView(session, viewName).orElseThrow(() -> new ViewNotFoundException(viewName)); + views.put(viewName, new ConnectorViewDefinition( + view.getOriginalSql(), + view.getCatalog(), + view.getSchema(), + view.getColumns().stream() + .map(currentViewColumn -> columnName.equals(currentViewColumn.getName()) ? + new ConnectorViewDefinition.ViewColumn(currentViewColumn.getName(), currentViewColumn.getType(), comment) + : currentViewColumn) + .collect(toImmutableList()), + view.getComment(), + view.getOwner(), + view.isRunAsInvoker(), + view.getPath())); + } + + @Override + public synchronized void renameView(ConnectorSession session, SchemaTableName source, SchemaTableName target) + { + checkSchemaExists(target.getSchemaName()); + if (!views.containsKey(source)) { + throw new TrinoException(NOT_FOUND, "View not found: " + source); + } + checkTableNotExists(target); + + if (views.putIfAbsent(target, views.remove(source)) != null) { + throw new TrinoException(ALREADY_EXISTS, "View '%s' already exists".formatted(target)); + } + } + + @Override + public synchronized void dropView(ConnectorSession session, SchemaTableName viewName) + { + if (views.remove(viewName) == null) { + throw new ViewNotFoundException(viewName); + } + } + @Override public synchronized Map getSchemaProperties(ConnectorSession session, String schemaName) { diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java index b64e2303ede4..f591c4ddff7c 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java @@ -17,6 +17,7 @@ import io.airlift.log.Level; import io.airlift.log.Logger; import io.airlift.log.Logging; +import io.trino.plugin.tpch.TpchPlugin; import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; @@ -67,6 +68,9 @@ public DistributedQueryRunner build() queryRunner.installPlugin(new FakerPlugin()); queryRunner.createCatalog(CATALOG, "faker", properties); + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch", ImmutableMap.of()); + return queryRunner; } catch (Exception e) { diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerViews.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerViews.java new file mode 100644 index 000000000000..809bf8de3302 --- /dev/null +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerViews.java @@ -0,0 +1,291 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static io.trino.testing.TestingNames.randomNameSuffix; +import static org.assertj.core.api.Assertions.assertThat; + +final class TestFakerViews + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return FakerQueryRunner.builder().build(); + } + + @Test + void testView() + { + @Language("SQL") String query = "SELECT orderkey, orderstatus, (totalprice / 2) half FROM tpch.tiny.orders"; + @Language("SQL") String expectedQuery = "SELECT orderkey, orderstatus, (totalprice / 2) half FROM orders"; + + String catalogName = getSession().getCatalog().orElseThrow(); + String schemaName = getSession().getSchema().orElseThrow(); + String testView = "test_view_" + randomNameSuffix(); + String testViewWithComment = "test_view_with_comment_" + randomNameSuffix(); + assertThat(computeActual("SHOW TABLES").getOnlyColumnAsSet()) // prime the cache, if any + .doesNotContain(testView); + assertUpdate("CREATE VIEW " + testView + " AS SELECT 123 x"); + assertThat(computeActual("SHOW TABLES").getOnlyColumnAsSet()) + .contains(testView); + assertUpdate("CREATE OR REPLACE VIEW " + testView + " AS " + query); + + assertUpdate("CREATE VIEW " + testViewWithComment + " COMMENT 'view comment' AS SELECT 123 x"); + assertUpdate("CREATE OR REPLACE VIEW " + testViewWithComment + " COMMENT 'view comment updated' AS " + query); + + String testViewOriginal = "test_view_original_" + randomNameSuffix(); + String testViewRenamed = "test_view_renamed_" + randomNameSuffix(); + assertUpdate("CREATE VIEW " + testViewOriginal + " AS " + query); + assertUpdate("ALTER VIEW " + testViewOriginal + " RENAME TO " + testViewRenamed); + + // verify comment + assertThat((String) computeScalar("SHOW CREATE VIEW " + testViewWithComment)).contains("COMMENT 'view comment updated'"); + assertThat(query( + """ + SELECT table_name, comment + FROM system.metadata.table_comments + WHERE catalog_name = '%s' AND schema_name = '%s' + """.formatted(catalogName, schemaName))) + .skippingTypesCheck() + .containsAll("VALUES ('" + testView + "', null), ('" + testViewWithComment + "', 'view comment updated')"); + + assertUpdate("COMMENT ON VIEW " + testViewWithComment + " IS 'view comment updated twice'"); + assertThat((String) computeScalar("SHOW CREATE VIEW " + testViewWithComment)).contains("COMMENT 'view comment updated twice'"); + + // reading + assertQuery("SELECT * FROM " + testView, expectedQuery); + assertQuery("SELECT * FROM " + testViewRenamed, expectedQuery); + assertQuery("SELECT * FROM " + testViewWithComment, expectedQuery); + + assertQuery( + """ + SELECT * + FROM %1$s a + JOIN %1$s b on a.orderkey = b.orderkey + """.formatted(testView), + """ + SELECT * + FROM (%1$s) a + JOIN (%1$s) b ON a.orderkey = b.orderkey + """.formatted(expectedQuery)); + + assertQuery( + """ + WITH orders AS ( + SELECT * + FROM tpch.tiny.orders + LIMIT 0 + ) + SELECT * + FROM %s + """.formatted(testView), + expectedQuery); + + assertQuery("SELECT * FROM %s.%s.%s".formatted(catalogName, schemaName, testView), expectedQuery); + + assertUpdate("DROP VIEW " + testViewWithComment); + + // information_schema.views without table_name filter + assertThat(query( + """ + SELECT table_name, regexp_replace(view_definition, '\\s', '') + FROM information_schema.views + WHERE table_schema = '%s' + """.formatted(schemaName))) + .skippingTypesCheck() + .containsAll("VALUES ('%1$s', '%3$s'), ('%2$s', '%3$s')".formatted(testView, testViewRenamed, query.replaceAll("\\s", ""))); + // information_schema.views with table_name filter + assertQuery( + """ + SELECT table_name, regexp_replace(view_definition, '\\s', '') + FROM information_schema.views + WHERE table_schema = '%s' and table_name IN ('%s', '%s') + """.formatted(schemaName, testView, testViewRenamed), + "VALUES ('%1$s', '%3$s'), ('%2$s', '%3$s')".formatted(testView, testViewRenamed, query.replaceAll("\\s", ""))); + + // table listing + assertThat(query("SHOW TABLES")) + .skippingTypesCheck() + .containsAll("VALUES '%s', '%s'".formatted(testView, testViewRenamed)); + // information_schema.tables without table_name filter + assertThat(query( + """ + SELECT table_name, table_type + FROM information_schema.tables + WHERE table_schema = '%s' + """.formatted(schemaName))) + .skippingTypesCheck() + .containsAll("VALUES ('%s', 'VIEW'), ('%s', 'VIEW')".formatted(testView, testViewRenamed)); + // information_schema.tables with table_name filter + assertQuery( + """ + SELECT table_name, table_type + FROM information_schema.tables + WHERE table_schema = '%s' and table_name IN ('%s', '%s') + """.formatted(schemaName, testView, testViewRenamed), + "VALUES ('%s', 'VIEW'), ('%s', 'VIEW')".formatted(testView, testViewRenamed)); + + // system.jdbc.tables without filter + assertThat(query( + """ + SELECT table_schem, table_name, table_type + FROM system.jdbc.tables + """)) + .skippingTypesCheck() + .containsAll("VALUES ('%1$s', '%2$s', 'VIEW'), ('%1$s', '%3$s', 'VIEW')".formatted(schemaName, testView, testViewRenamed)); + + // system.jdbc.tables with table prefix filter + assertQuery( + """ + SELECT table_schem, table_name, table_type + FROM system.jdbc.tables + WHERE table_cat = '%s' AND table_schem = '%s' AND table_name IN ('%s', '%s') + """.formatted(catalogName, schemaName, testView, testViewRenamed), + "VALUES ('%1$s', '%2$s', 'VIEW'), ('%1$s', '%3$s', 'VIEW')".formatted(schemaName, testView, testViewRenamed)); + + // column listing + assertThat(query("SHOW COLUMNS FROM " + testView)) + .result() + .projected("Column") // column types can very between connectors + .skippingTypesCheck() + .matches("VALUES 'orderkey', 'orderstatus', 'half'"); + + assertThat(query("DESCRIBE " + testView)) + .result() + .projected("Column") // column types can very between connectors + .skippingTypesCheck() + .matches("VALUES 'orderkey', 'orderstatus', 'half'"); + + // information_schema.columns without table_name filter + assertThat(query( + """ + SELECT table_name, column_name + FROM information_schema.columns + WHERE table_schema = '%s' + """.formatted(schemaName))) + .skippingTypesCheck() + .containsAll( + """ + SELECT * + FROM (VALUES '%s', '%s') + CROSS JOIN UNNEST(ARRAY['orderkey', 'orderstatus', 'half']) + """.formatted(testView, testViewRenamed)); + + // information_schema.columns with table_name filter + assertThat(query( + """ + SELECT table_name, column_name + FROM information_schema.columns + WHERE table_schema = '%s' and table_name IN ('%s', '%s') + """.formatted(schemaName, testView, testViewRenamed))) + .skippingTypesCheck() + .containsAll( + """ + SELECT * + FROM (VALUES '%s', '%s') + CROSS JOIN UNNEST(ARRAY['orderkey', 'orderstatus', 'half']) + """.formatted(testView, testViewRenamed)); + + // view-specific listings + assertThat(query( + """ + SELECT table_name + FROM information_schema.views + WHERE table_schema = '%s' + """.formatted(schemaName))) + .skippingTypesCheck() + .containsAll("VALUES '%s', '%s'".formatted(testView, testViewRenamed)); + + // system.jdbc.columns without filter + assertThat(query( + """ + SELECT table_schem, table_name, column_name + FROM system.jdbc.columns + """)) + .skippingTypesCheck() + .containsAll( + """ + SELECT * + FROM (VALUES ('%1$s', '%2$s'), ('%1$s', '%3$s')) + CROSS JOIN UNNEST(ARRAY['orderkey', 'orderstatus', 'half']) + """.formatted(schemaName, testView, testViewRenamed)); + + // system.jdbc.columns with schema filter + assertThat(query( + """ + SELECT table_schem, table_name, column_name + FROM system.jdbc.columns + WHERE table_schem LIKE '%%%s%%' + """.formatted(schemaName))) + .skippingTypesCheck() + .containsAll( + """ + SELECT * + FROM (VALUES ('%1$s', '%2$s'), ('%1$s', '%3$s')) + CROSS JOIN UNNEST(ARRAY['orderkey', 'orderstatus', 'half']) + """.formatted(schemaName, testView, testViewRenamed)); + + // system.jdbc.columns with table filter + assertThat(query( + """ + SELECT table_schem, table_name, column_name + FROM system.jdbc.columns + WHERE table_name LIKE '%%%s%%' OR table_name LIKE '%%%s%%' + """.formatted(testView, testViewRenamed))) + .skippingTypesCheck() + .containsAll( + """ + SELECT * + FROM (VALUES ('%1$s', '%2$s'), ('%1$s', '%3$s')) + CROSS JOIN UNNEST(ARRAY['orderkey', 'orderstatus', 'half']) + """.formatted(schemaName, testView, testViewRenamed)); + + assertUpdate("DROP VIEW " + testView); + assertUpdate("DROP VIEW " + testViewRenamed); + assertThat(computeActual("SHOW TABLES").getOnlyColumnAsSet()) + .doesNotContainAnyElementsOf(List.of(testView, testViewRenamed, testViewWithComment)); + } + + @Test + void testViewConflicts() + { + String catalogName = getSession().getCatalog().orElseThrow(); + String schemaName = getSession().getSchema().orElseThrow(); + + String testTable = "test_table_" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + testTable + " (orderkey INT, orderstatus VARCHAR(255), half VARCHAR(255))"); + + assertQueryFails("CREATE VIEW " + testTable + " AS SELECT 123 x", "line 1:1: Table already exists: '%s.%s.%s'".formatted(catalogName, schemaName, testTable)); + + String testView = "test_view_" + randomNameSuffix(); + assertUpdate("CREATE VIEW " + testView + " AS SELECT 123 x"); + + assertQueryFails("CREATE VIEW " + testView + " AS SELECT 123 x", "line 1:1: View already exists: '%s.%s.%s'".formatted(catalogName, schemaName, testView)); + assertQueryFails("CREATE TABLE " + testView + " (orderkey INT, orderstatus VARCHAR(255), half VARCHAR(255))", "View '%s.%s' already exists".formatted(schemaName, testView)); + assertQueryFails("ALTER VIEW " + testView + " RENAME TO " + testTable, "line 1:1: Target view '%s.%s.%s' does not exist, but a table with that name exists.".formatted(catalogName, schemaName, testTable)); + assertQueryFails("ALTER TABLE " + testTable + " RENAME TO " + testView, "line 1:1: Target table '%s.%s.%s' does not exist, but a view with that name exists.".formatted(catalogName, schemaName, testView)); + + assertUpdate("DROP VIEW " + testView); + assertUpdate("DROP TABLE " + testTable); + } +} From 8957c9226c93fec9c9bb530bf4d1ad68dd3d73e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 26 Dec 2024 10:14:20 +0100 Subject: [PATCH 072/158] Add views support to the Faker connector docs --- docs/src/main/sphinx/connector/faker.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/main/sphinx/connector/faker.md b/docs/src/main/sphinx/connector/faker.md index 793064b7178e..e721b60645c5 100644 --- a/docs/src/main/sphinx/connector/faker.md +++ b/docs/src/main/sphinx/connector/faker.md @@ -238,6 +238,7 @@ To define the schema for generating data, it supports the following features: - [](/sql/drop-table) - [](/sql/create-schema) - [](/sql/drop-schema) +- [](sql-view-management) (faker-usage)= ## Usage From bc7bc7839bf858b6d1a47664e7fa41c81db281fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 24 Nov 2024 23:48:42 +0100 Subject: [PATCH 073/158] Add min, max, and allowed_values column properties in Faker connector Allow constraining generated values by setting the min, max, or allowed_values column properties. --- docs/src/main/sphinx/connector/faker.md | 58 ++- plugin/trino-faker/pom.xml | 5 + .../io/trino/plugin/faker/ColumnInfo.java | 3 + .../trino/plugin/faker/FakerColumnHandle.java | 107 +++- .../io/trino/plugin/faker/FakerConnector.java | 29 +- .../io/trino/plugin/faker/FakerMetadata.java | 27 +- .../trino/plugin/faker/FakerPageSource.java | 14 +- .../java/io/trino/plugin/faker/Literal.java | 250 +++++++++ .../trino/plugin/faker/TestFakerQueries.java | 487 ++++++++++++++++++ 9 files changed, 931 insertions(+), 49 deletions(-) create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/Literal.java diff --git a/docs/src/main/sphinx/connector/faker.md b/docs/src/main/sphinx/connector/faker.md index e721b60645c5..ed21d2b551f8 100644 --- a/docs/src/main/sphinx/connector/faker.md +++ b/docs/src/main/sphinx/connector/faker.md @@ -102,6 +102,15 @@ The following table details all supported column properties. sentence from the [Lorem](https://javadoc.io/doc/net.datafaker/datafaker/latest/net/datafaker/providers/base/Lorem.html) provider. +* - `min` + - Minimum generated value (inclusive). Cannot be set for character-based type + columns. +* - `max` + - Maximum generated value (inclusive). Cannot be set for character-based type + columns. +* - `allowed_values` + - List of allowed values. Cannot be set together with the `min`, or `max` + properties. ::: ### Character types @@ -166,7 +175,7 @@ Faker supports the following non-character types: - `UUID` You can not use generator expressions for non-character-based columns. To limit -their data range, specify constraints in the `WHERE` clause - see +their data range, set the `min` and/or `max` column properties - see [](faker-usage). ### Unsupported types @@ -184,12 +193,11 @@ can be combined, like in the following example: ```sql CREATE TABLE faker.default.prices ( currency VARCHAR NOT NULL WITH (generator = '#{Currency.code}'), - price DECIMAL(8,2) NOT NULL + price DECIMAL(8,2) NOT NULL WITH (min = '0') ); SELECT JSON_OBJECT(KEY currency VALUE price) AS complex FROM faker.default.prices -WHERE price > 0 LIMIT 3; ``` @@ -261,37 +269,49 @@ CREATE TABLE generator.default.customer (LIKE production.public.customer EXCLUDI Insert random data into the original table, by selecting it from the `generator` catalog. Data generated by the Faker connector for columns of -non-character types cover the whole range of that data type. Add constraints to -adjust the data as desired. The following example ensures that date of birth -and age in years are related and realistic values. +non-character types cover the whole range of that data type. Set the `min` and +`max` column properties, to adjust the generated data as desired. The following +example ensures that date of birth and age in years are related and realistic +values. + +Start with getting the complete definition of +a table: + +```sql +SHOW CREATE TABLE production.public.customers; +``` + +Modify the output of the previous query and add some column properties. + +```sql +CREATE TABLE generator.default.customer ( + id UUID NOT NULL, + name VARCHAR NOT NULL, + address VARCHAR NOT NULL, + born_at DATE WITH (min = '1900-01-01', max = '2025-01-01'), + age_years INTEGER WITH (min = '0', max = '150'), + group_id INTEGER WITH (allowed_values = ARRAY['10', '32', '81']) +); +``` ```sql INSERT INTO production.public.customers SELECT * FROM generator.default.customers -WHERE - born_at BETWEEN CURRENT_DATE - INTERVAL '150' YEAR AND CURRENT_DATE - AND age_years BETWEEN 0 AND 150 LIMIT 100; ``` To generate even more realistic data, choose specific generators by setting the -`generator` property on columns. Start with getting the complete definition of -a table: - -```sql -SHOW CREATE TABLE production.public.customers; -``` - -Modify the output of the previous query and add some column properties. +`generator` property on columns. ```sql CREATE TABLE generator.default.customer ( id UUID NOT NULL, name VARCHAR NOT NULL WITH (generator = '#{Name.first_name} #{Name.last_name}'), address VARCHAR NOT NULL WITH (generator = '#{Address.fullAddress}'), - born_at DATE, - age_years INTEGER + born_at DATE WITH (min = '1900-01-01', max = '2025-01-01'), + age_years INTEGER WITH (min = '0', max = '150'), + group_id INTEGER WITH (allowed_values = ARRAY['10', '32', '81']) ); ``` diff --git a/plugin/trino-faker/pom.xml b/plugin/trino-faker/pom.xml index 060abf3c9b14..8a87b91f122e 100644 --- a/plugin/trino-faker/pom.xml +++ b/plugin/trino-faker/pom.xml @@ -45,6 +45,11 @@ trino-main + + io.trino + trino-parser + + io.trino trino-plugin-toolkit diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java index 55f84eb9df03..bd52ae4fa776 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java @@ -24,6 +24,9 @@ public record ColumnInfo(FakerColumnHandle handle, ColumnMetadata metadata) { public static final String NULL_PROBABILITY_PROPERTY = "null_probability"; public static final String GENERATOR_PROPERTY = "generator"; + public static final String MIN_PROPERTY = "min"; + public static final String MAX_PROPERTY = "max"; + public static final String ALLOWED_VALUES_PROPERTY = "allowed_values"; public ColumnInfo { diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java index e6ae47dec501..941103460d22 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java @@ -14,9 +14,28 @@ package io.trino.plugin.faker; +import com.google.common.collect.ImmutableList; +import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.Range; +import io.trino.spi.predicate.ValueSet; +import io.trino.spi.type.CharType; import io.trino.spi.type.Type; +import io.trino.spi.type.VarbinaryType; +import io.trino.spi.type.VarcharType; +import java.util.Collection; +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.plugin.faker.ColumnInfo.ALLOWED_VALUES_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.GENERATOR_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.MAX_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.MIN_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.NULL_PROBABILITY_PROPERTY; +import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; import static java.util.Objects.requireNonNull; public record FakerColumnHandle( @@ -24,7 +43,8 @@ public record FakerColumnHandle( String name, Type type, double nullProbability, - String generator) + String generator, + Domain domain) implements ColumnHandle { public FakerColumnHandle @@ -32,4 +52,89 @@ public record FakerColumnHandle( requireNonNull(name, "name is null"); requireNonNull(type, "type is null"); } + + public static FakerColumnHandle of(int columnId, ColumnMetadata column, double defaultNullProbability) + { + double nullProbability = 0; + if (column.isNullable()) { + nullProbability = (double) column.getProperties().getOrDefault(NULL_PROBABILITY_PROPERTY, defaultNullProbability); + } + String generator = (String) column.getProperties().get(GENERATOR_PROPERTY); + if (generator != null && !isCharacterColumn(column)) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property can only be set for CHAR, VARCHAR or VARBINARY columns".formatted(GENERATOR_PROPERTY)); + } + // only parse min, max, and options to validate literals - FakerColumnHandle needs to be serializable, + // and some internal Trino types are not (Int128, LongTimestamp, LongTimestampWithTimeZone), so they cannot be stored in the handle as native types + String min = (String) column.getProperties().get(MIN_PROPERTY); + try { + Literal.parse(min, column.getType()); + } + catch (IllegalArgumentException e) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MIN_PROPERTY, column.getType().getDisplayName()), e); + } + String max = (String) column.getProperties().get(MAX_PROPERTY); + try { + Literal.parse(max, column.getType()); + } + catch (IllegalArgumentException e) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MAX_PROPERTY, column.getType().getDisplayName()), e); + } + Domain domain = Domain.all(column.getType()); + if (min != null || max != null) { + if (isCharacterColumn(column)) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` and `%s` properties cannot be set for CHAR, VARCHAR or VARBINARY columns".formatted(MIN_PROPERTY, MAX_PROPERTY)); + } + domain = Domain.create(ValueSet.ofRanges(range(column.getType(), min, max)), false); + } + if (column.getProperties().containsKey(ALLOWED_VALUES_PROPERTY)) { + if (min != null || max != null || generator != null) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property cannot be set together with `%s`, `%s`, and `%s` properties".formatted(ALLOWED_VALUES_PROPERTY, MIN_PROPERTY, MAX_PROPERTY, GENERATOR_PROPERTY)); + } + ImmutableList.Builder builder = ImmutableList.builder(); + for (String value : strings((List) column.getProperties().get(ALLOWED_VALUES_PROPERTY))) { + try { + builder.add(Literal.parse(value, column.getType())); + } + catch (IllegalArgumentException | ClassCastException e) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must only contain valid %s literals, failed to parse `%s`".formatted(ALLOWED_VALUES_PROPERTY, column.getType().getDisplayName(), value), e); + } + } + domain = Domain.create(ValueSet.copyOf(column.getType(), builder.build()), false); + } + + return new FakerColumnHandle( + columnId, + column.getName(), + column.getType(), + nullProbability, + generator, + domain); + } + + private static boolean isCharacterColumn(ColumnMetadata column) + { + return column.getType() instanceof CharType || column.getType() instanceof VarcharType || column.getType() instanceof VarbinaryType; + } + + private static Range range(Type type, String min, String max) + { + requireNonNull(type, "type is null"); + if (min == null && max == null) { + return Range.all(type); + } + if (max == null) { + return Range.greaterThanOrEqual(type, Literal.parse(min, type)); + } + if (min == null) { + return Range.lessThanOrEqual(type, Literal.parse(max, type)); + } + return Range.range(type, Literal.parse(min, type), true, Literal.parse(max, type), true); + } + + private static List strings(Collection values) + { + return values.stream() + .map(String.class::cast) + .collect(toImmutableList()); + } } diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java index 0d29c986f965..2a5819c04b47 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java @@ -28,12 +28,17 @@ import io.trino.spi.function.FunctionProvider; import io.trino.spi.session.PropertyMetadata; import io.trino.spi.transaction.IsolationLevel; +import io.trino.spi.type.ArrayType; import jakarta.inject.Inject; import java.util.List; import java.util.Optional; import java.util.Set; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.plugin.faker.ColumnInfo.ALLOWED_VALUES_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.MAX_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.MIN_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; @@ -41,6 +46,7 @@ import static io.trino.spi.session.PropertyMetadata.doubleProperty; import static io.trino.spi.session.PropertyMetadata.longProperty; import static io.trino.spi.session.PropertyMetadata.stringProperty; +import static io.trino.spi.type.VarcharType.VARCHAR; import static java.util.Objects.requireNonNull; public class FakerConnector @@ -161,7 +167,28 @@ public List> getColumnProperties() throw new TrinoException(INVALID_COLUMN_PROPERTY, "generator must be a valid Faker expression", e); } }, - false)); + false), + stringProperty( + MIN_PROPERTY, + "Minimum generated value (inclusive)", + null, + false), + stringProperty( + MAX_PROPERTY, + "Maximum generated value (inclusive)", + null, + false), + new PropertyMetadata<>( + ALLOWED_VALUES_PROPERTY, + "List of allowed values", + new ArrayType(VARCHAR), + List.class, + null, + false, + value -> ((List) value).stream() + .map(String.class::cast) + .collect(toImmutableList()), + value -> value)); } private static void checkProperty(boolean expression, ErrorCodeSupplier errorCode, String errorMessage) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java index 9312f33e1f41..1ab3c20d129e 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java @@ -49,9 +49,6 @@ import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.statistics.ComputedStatistics; import io.trino.spi.type.BigintType; -import io.trino.spi.type.CharType; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; import jakarta.inject.Inject; import java.util.ArrayList; @@ -72,7 +69,6 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Maps.filterKeys; import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; -import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_REFERENCE; import static io.trino.spi.StandardErrorCode.NOT_FOUND; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; @@ -331,21 +327,8 @@ public synchronized FakerOutputTableHandle beginCreateTable(ConnectorSession ses int columnId = 0; for (; columnId < tableMetadata.getColumns().size(); columnId++) { ColumnMetadata column = tableMetadata.getColumns().get(columnId); - double nullProbability = 0; - if (column.isNullable()) { - nullProbability = (double) column.getProperties().getOrDefault(ColumnInfo.NULL_PROBABILITY_PROPERTY, tableNullProbability); - } - String generator = (String) column.getProperties().get(ColumnInfo.GENERATOR_PROPERTY); - if (generator != null && !isCharacterColumn(column)) { - throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `generator` property can only be set for CHAR, VARCHAR or VARBINARY columns"); - } columns.add(new ColumnInfo( - new FakerColumnHandle( - columnId, - column.getName(), - column.getType(), - nullProbability, - generator), + FakerColumnHandle.of(columnId, column, tableNullProbability), column)); } @@ -355,7 +338,8 @@ public synchronized FakerOutputTableHandle beginCreateTable(ConnectorSession ses ROW_ID_COLUMN_NAME, BigintType.BIGINT, 0, - ""), + "", + Domain.all(BigintType.BIGINT)), ColumnMetadata.builder() .setName(ROW_ID_COLUMN_NAME) .setType(BigintType.BIGINT) @@ -371,11 +355,6 @@ public synchronized FakerOutputTableHandle beginCreateTable(ConnectorSession ses return new FakerOutputTableHandle(tableName); } - private boolean isCharacterColumn(ColumnMetadata column) - { - return column.getType() instanceof CharType || column.getType() instanceof VarcharType || column.getType() instanceof VarbinaryType; - } - private synchronized void checkSchemaExists(String schemaName) { if (schemas.stream().noneMatch(schema -> schema.name().equals(schemaName))) { diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java index cf8ac1310c1f..ed676cf0cc49 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -152,8 +152,10 @@ private Generator getGenerator( long offset) { if (ROW_ID_COLUMN_NAME.equals(column.name())) { - return new Generator() { + return new Generator() + { long currentRowId = offset; + @Override public void accept(BlockBuilder blockBuilder) { @@ -234,8 +236,10 @@ private Generator constraintedValueGenerator(FakerColumnHandle handle, Domain do ObjectWriter singleValueWriter = objectWriter(handle.type()); return (blockBuilder) -> singleValueWriter.accept(blockBuilder, domain.getSingleValue()); } - if (domain.getValues().isDiscreteSet()) { - List values = domain.getValues().getDiscreteSet(); + if (domain.getValues().isDiscreteSet() || handle.domain().getValues().isDiscreteSet()) { + List values = domain.getValues().isDiscreteSet() + ? domain.getValues().getDiscreteSet() + : handle.domain().getValues().getDiscreteSet(); ObjectWriter singleValueWriter = objectWriter(handle.type()); return (blockBuilder) -> singleValueWriter.accept(blockBuilder, values.get(random.nextInt(values.size()))); } @@ -257,8 +261,10 @@ private Generator constraintedValueGenerator(FakerColumnHandle handle, Domain do }; } - private Generator randomValueGenerator(FakerColumnHandle handle, Range range) + private Generator randomValueGenerator(FakerColumnHandle handle, Range predicateRange) { + Range range = predicateRange.intersect(handle.domain().getValues().getRanges().getSpan()) + .orElseThrow(() -> new TrinoException(INVALID_ROW_FILTER, "Predicates do not overlap with column min and max properties")); if (handle.generator() != null) { if (!range.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for columns with a generator expression are not supported"); diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/Literal.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/Literal.java new file mode 100644 index 000000000000..410f259bcb0c --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/Literal.java @@ -0,0 +1,250 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import com.google.common.base.CharMatcher; +import com.google.common.io.BaseEncoding; +import com.google.common.net.InetAddresses; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.trino.spi.type.CharType; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.Int128; +import io.trino.spi.type.TimeType; +import io.trino.spi.type.TimeWithTimeZoneType; +import io.trino.spi.type.TimestampType; +import io.trino.spi.type.TimestampWithTimeZoneType; +import io.trino.spi.type.Type; +import io.trino.spi.type.UuidType; +import io.trino.spi.type.VarcharType; +import io.trino.sql.tree.IntervalLiteral; +import io.trino.type.IpAddressType; + +import java.math.BigInteger; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.UuidType.javaUuidToTrinoUuid; +import static io.trino.spi.type.VarbinaryType.VARBINARY; +import static io.trino.type.DateTimes.parseTime; +import static io.trino.type.DateTimes.parseTimeWithTimeZone; +import static io.trino.type.DateTimes.parseTimestamp; +import static io.trino.type.DateTimes.parseTimestampWithTimeZone; +import static io.trino.type.IntervalDayTimeType.INTERVAL_DAY_TIME; +import static io.trino.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; +import static io.trino.util.DateTimeUtils.parseDate; +import static io.trino.util.DateTimeUtils.parseDayTimeInterval; +import static io.trino.util.DateTimeUtils.parseYearMonthInterval; +import static java.lang.Float.floatToRawIntBits; +import static java.lang.System.arraycopy; +import static java.util.Locale.ENGLISH; + +public class Literal +{ + private static final CharMatcher WHITESPACE_MATCHER = CharMatcher.whitespace(); + private static final CharMatcher HEX_DIGIT_MATCHER = CharMatcher.inRange('A', 'F') + .or(CharMatcher.inRange('0', '9')) + .precomputed(); + private static final Pattern DECIMAL_PATTERN = Pattern.compile("([+-]?)(\\d(?:_?\\d)*)?(?:\\.(\\d(?:_?\\d)*)?)?"); + + private Literal() {} + + public static Object parse(String value, Type type) + { + if (value == null) { + return null; + } + if (BIGINT.equals(type) || INTEGER.equals(type) || SMALLINT.equals(type) || TINYINT.equals(type)) { + return parseLong(value); + } + if (BOOLEAN.equals(type)) { + return parseBoolean(value); + } + if (DATE.equals(type)) { + return (long) parseDate(value); + } + if (type instanceof DecimalType decimalType) { + return parseDecimal(value, decimalType); + } + if (REAL.equals(type)) { + return (long) floatToRawIntBits(Float.parseFloat(value)); + } + if (DOUBLE.equals(type)) { + return Double.parseDouble(value); + } + // not supported: HYPER_LOG_LOG, QDIGEST, TDIGEST, P4_HYPER_LOG_LOG + if (INTERVAL_DAY_TIME.equals(type)) { + return parseDayTimeInterval(value, IntervalLiteral.IntervalField.SECOND, Optional.empty()); + } + if (INTERVAL_YEAR_MONTH.equals(type)) { + return parseYearMonthInterval(value, IntervalLiteral.IntervalField.MONTH, Optional.empty()); + } + if (type instanceof TimestampType timestampType) { + return parseTimestamp(timestampType.getPrecision(), value); + } + if (type instanceof TimestampWithTimeZoneType timestampWithTimeZoneType) { + return parseTimestampWithTimeZone(timestampWithTimeZoneType.getPrecision(), value); + } + if (type instanceof TimeType) { + return parseTime(value); + } + if (type instanceof TimeWithTimeZoneType timeWithTimeZoneType) { + return parseTimeWithTimeZone(timeWithTimeZoneType.getPrecision(), value); + } + if (VARBINARY.equals(type)) { + return parseBinary(value); + } + if (type instanceof VarcharType || type instanceof CharType) { + return utf8Slice(value); + } + // not supported: ROW, ARRAY, MAP, JSON + if (type instanceof IpAddressType) { + return parseIpAddress(value); + } + // not supported: GEOMETRY + if (type instanceof UuidType) { + return javaUuidToTrinoUuid(UUID.fromString(value)); + } + throw new IllegalArgumentException("Unsupported literal type: " + type); + } + + private static long parseLong(String value) + { + value = value.replace("_", ""); + + if (value.startsWith("0x") || value.startsWith("0X")) { + return Long.parseLong(value.substring(2), 16); + } + else if (value.startsWith("0b") || value.startsWith("0B")) { + return Long.parseLong(value.substring(2), 2); + } + else if (value.startsWith("0o") || value.startsWith("0O")) { + return Long.parseLong(value.substring(2), 8); + } + else { + return Long.parseLong(value); + } + } + + private static Boolean parseBoolean(String value) + { + value = value.toLowerCase(ENGLISH); + checkArgument(value.equals("true") || value.equals("false")); + return value.equals("true"); + } + + /** + * Based on {@link io.trino.spi.type.Decimals#parse}, but doesn't infer the type from the value. + */ + private static Object parseDecimal(String value, DecimalType decimalType) + { + Matcher matcher = DECIMAL_PATTERN.matcher(value); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid DECIMAL value '" + value + "'"); + } + + String sign = getMatcherGroup(matcher, 1); + if (sign.isEmpty()) { + sign = "+"; + } + String integralPart = getMatcherGroup(matcher, 2); + String fractionalPart = getMatcherGroup(matcher, 3); + + if (integralPart.isEmpty() && fractionalPart.isEmpty()) { + throw new IllegalArgumentException("Invalid DECIMAL value '" + value + "'"); + } + + integralPart = stripLeadingZeros(integralPart.replace("_", "")); + fractionalPart = fractionalPart.replace("_", ""); + + String unscaledValue = sign + (integralPart.isEmpty() ? "0" : "") + integralPart + fractionalPart; + if (decimalType.isShort()) { + return Long.parseLong(unscaledValue); + } + return Int128.valueOf(new BigInteger(unscaledValue)); + } + + private static String getMatcherGroup(MatchResult matcher, int group) + { + String groupValue = matcher.group(group); + if (groupValue == null) { + groupValue = ""; + } + return groupValue; + } + + private static String stripLeadingZeros(String number) + { + for (int i = 0; i < number.length(); i++) { + if (number.charAt(i) != '0') { + return number.substring(i); + } + } + + return ""; + } + + private static Slice parseBinary(String value) + { + String hexString = WHITESPACE_MATCHER.removeFrom(value).toUpperCase(ENGLISH); + if (!HEX_DIGIT_MATCHER.matchesAllOf(hexString)) { + throw new IllegalArgumentException("Binary literal can only contain hexadecimal digits"); + } + if (hexString.length() % 2 != 0) { + throw new IllegalArgumentException("Binary literal must contain an even number of digits"); + } + return Slices.wrappedBuffer(BaseEncoding.base16().decode(hexString)); + } + + private static Slice parseIpAddress(String value) + { + byte[] address; + try { + address = InetAddresses.forString(value).getAddress(); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Cannot cast value to IPADDRESS: " + value); + } + + byte[] bytes; + if (address.length == 4) { + bytes = new byte[16]; + bytes[10] = (byte) 0xff; + bytes[11] = (byte) 0xff; + arraycopy(address, 0, bytes, 12, 4); + } + else if (address.length == 16) { + bytes = address; + } + else { + throw new IllegalArgumentException("Invalid InetAddress length: " + address.length); + } + + return wrappedBuffer(bytes); + } +} diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java index cb75ebaeb777..f651da4ccdd0 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java @@ -850,4 +850,491 @@ AND rnd_uuid IN (UUID '1fc74d96-0216-449b-a145-455578a9eaa5', UUID '3ee49ede-002 assertUpdate("DROP TABLE faker.default.all_types_in"); } + + @Test + void testSelectRangeProperties() + { + @Language("SQL") + String tableQuery; + @Language("SQL") + String testQuery; + + // inclusive ranges that produce only 2 values + // high boundary float value obtained using `Math.nextUp((float) 0.0)` + tableQuery = + """ + CREATE TABLE faker.default.all_types_range_prop ( + rnd_bigint bigint NOT NULL WITH (min = '0', max = '1'), + rnd_integer integer NOT NULL WITH (min = '0', max = '1'), + rnd_smallint smallint NOT NULL WITH (min = '0', max = '1'), + rnd_tinyint tinyint NOT NULL WITH (min = '0', max = '1'), + rnd_boolean boolean NOT NULL, + rnd_date date NOT NULL WITH (min = '2022-03-01', max = '2022-03-02'), + rnd_decimal1 decimal NOT NULL WITH (min = '0', max = '1'), + rnd_decimal2 decimal(18,5) NOT NULL WITH (min = '0.00000', max = '0.00001'), + rnd_decimal3 decimal(38,0) NOT NULL WITH (min = '0', max = '1'), + rnd_decimal4 decimal(38,38) NOT NULL WITH (min = '0.00000000000000000000000000000000000000', max = '0.00000000000000000000000000000000000001'), + rnd_decimal5 decimal(5,2) NOT NULL WITH (min = '0.00', max = '0.01'), + rnd_real real NOT NULL WITH (min = '0.0', max = '1.4E-45'), + rnd_double double NOT NULL WITH (min = '0.0', max = '4.9E-324'), + rnd_interval_day_time interval day to second NOT NULL WITH (min = '0.000', max = '0.001'), + rnd_interval_year interval year to month NOT NULL WITH (min = '0', max = '1'), + rnd_timestamp timestamp NOT NULL WITH (min = '2022-03-21 00:00:00.000', max = '2022-03-21 00:00:00.001'), + rnd_timestamp0 timestamp(0) NOT NULL WITH (min = '2022-03-21 00:00:00', max = '2022-03-21 00:00:01'), + rnd_timestamp6 timestamp(6) NOT NULL WITH (min = '2022-03-21 00:00:00.000000', max = '2022-03-21 00:00:00.000001'), + rnd_timestamp9 timestamp(9) NOT NULL WITH (min = '2022-03-21 00:00:00.000000000', max = '2022-03-21 00:00:00.000000001'), + rnd_timestamptz timestamp with time zone NOT NULL WITH (min = '2022-03-21 00:00:00.000 +01:00', max = '2022-03-21 00:00:00.001 +01:00'), + rnd_timestamptz0 timestamp(0) with time zone NOT NULL WITH (min = '2022-03-21 00:00:00 +01:00', max = '2022-03-21 00:00:01 +01:00'), + rnd_timestamptz6 timestamp(6) with time zone NOT NULL WITH (min = '2022-03-21 00:00:00.000000 +01:00', max = '2022-03-21 00:00:00.000001 +01:00'), + rnd_timestamptz9 timestamp(9) with time zone NOT NULL WITH (min = '2022-03-21 00:00:00.000000000 +01:00', max = '2022-03-21 00:00:00.000000001 +01:00'), + rnd_time time NOT NULL WITH (min = '01:02:03.456', max = '01:02:03.457'), + rnd_time0 time(0) NOT NULL WITH (min = '01:02:03', max = '01:02:04'), + rnd_time6 time(6) NOT NULL WITH (min = '01:02:03.000456', max = '01:02:03.000457'), + rnd_time9 time(9) NOT NULL WITH (min = '01:02:03.000000456', max = '01:02:03.000000457'), + rnd_timetz time with time zone NOT NULL WITH (min = '01:02:03.456 +01:00', max = '01:02:03.457 +01:00'), + rnd_timetz0 time(0) with time zone NOT NULL WITH (min = '01:02:03 +01:00', max = '01:02:04 +01:00'), + rnd_timetz6 time(6) with time zone NOT NULL WITH (min = '01:02:03.000456 +01:00', max = '01:02:03.000457 +01:00'), + rnd_timetz9 time(9) with time zone NOT NULL WITH (min = '01:02:03.000000456 +01:00', max = '01:02:03.000000457 +01:00'), + rnd_timetz12 time(12) with time zone NOT NULL, + rnd_varbinary varbinary NOT NULL, + rnd_varchar varchar NOT NULL, + rnd_nvarchar varchar(1000) NOT NULL, + rnd_char char NOT NULL, + rnd_nchar char(1000) NOT NULL, + rnd_ipaddress ipaddress NOT NULL, + rnd_uuid uuid NOT NULL)"""; + assertUpdate(tableQuery); + + testQuery = + """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(distinct rnd_smallint), + count(distinct rnd_tinyint), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(distinct rnd_decimal5), + count(distinct rnd_real), + count(distinct rnd_double), + count(distinct rnd_interval_day_time), + count(distinct rnd_interval_year), + count(distinct rnd_timestamp), + count(distinct rnd_timestamp0), + count(distinct rnd_timestamp6), + count(distinct rnd_timestamp9), + count(distinct rnd_timestamptz), + count(distinct rnd_timestamptz0), + count(distinct rnd_timestamptz6), + count(distinct rnd_timestamptz9), + count(distinct rnd_time), + count(distinct rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(distinct rnd_timetz), + count(distinct rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9) + FROM all_types_range_prop + """; + assertQuery(testQuery, + """ + VALUES (2, + 2, + 2, + 2, + -- date + 2, + -- decimal + 2, + 2, + 2, + 2, + 2, + -- real, double + 2, + 2, + -- intervals + 2, + 2, + -- timestamps + 2, + 2, + 2, + 2, + -- timestamps with time zone + 2, + 2, + 2, + 2, + -- time + 2, + 2, + 2, + 2, + -- time with time zone + 2, + 2, + 2, + 2) + """); + + // inclusive range to get the min low bound + tableQuery = + """ + CREATE TABLE faker.default.all_types_max_prop ( + rnd_bigint bigint NOT NULL WITH (max = '-9223372036854775808'), + rnd_integer integer NOT NULL WITH (max = '-2147483648'), + rnd_smallint smallint NOT NULL WITH (max = '-32768'), + rnd_tinyint tinyint NOT NULL WITH (max = '-128'), + rnd_boolean boolean NOT NULL, + rnd_date date NOT NULL WITH (max = '-5877641-06-23'), + rnd_decimal1 decimal NOT NULL WITH (max = '-99999999999999999999999999999999999999'), + rnd_decimal2 decimal(18,5) NOT NULL WITH (max = '-9999999999999.99999'), + rnd_decimal3 decimal(38,0) NOT NULL WITH (max = '-99999999999999999999999999999999999999'), + rnd_decimal4 decimal(38,38) NOT NULL WITH (max = '-0.99999999999999999999999999999999999999'), + -- TODO it actually retdurns '-999.98' + rnd_decimal5 decimal(5,2) NOT NULL WITH (max = '-999.99'), + rnd_real real NOT NULL WITH (max = '1.4E-45'), + rnd_double double NOT NULL WITH (max = '4.9E-324'), + -- interval literals can't represent smallest possible values allowed by the engine + rnd_interval_day_time interval day to second NOT NULL, + rnd_interval_year interval year to month NOT NULL, + -- can't test timestamps because their extreme values cannot be expressed as literals + rnd_timestamp timestamp NOT NULL, + rnd_timestamp0 timestamp(0) NOT NULL, + rnd_timestamp6 timestamp(6) NOT NULL, + rnd_timestamp9 timestamp(9) NOT NULL, + rnd_timestamptz timestamp with time zone NOT NULL, + rnd_timestamptz0 timestamp(0) with time zone NOT NULL, + rnd_timestamptz6 timestamp(6) with time zone NOT NULL, + rnd_timestamptz9 timestamp(9) with time zone NOT NULL, + rnd_time time NOT NULL WITH (max = '00:00:00.000'), + rnd_time0 time(0) NOT NULL WITH (max = '00:00:00'), + rnd_time6 time(6) NOT NULL WITH (max = '00:00:00.000000'), + rnd_time9 time(9) NOT NULL WITH (max = '00:00:00.000000000'), + rnd_timetz time with time zone NOT NULL WITH (max = '00:00:00.000 +01:00'), + rnd_timetz0 time(0) with time zone NOT NULL WITH (max = '00:00:00 +01:00'), + rnd_timetz6 time(6) with time zone NOT NULL WITH (max = '00:00:00.000000 +01:00'), + rnd_timetz9 time(9) with time zone NOT NULL WITH (max = '00:00:00.000000000 +01:00'), + rnd_timetz12 time(12) with time zone NOT NULL, + rnd_varbinary varbinary NOT NULL, + rnd_varchar varchar NOT NULL, + rnd_nvarchar varchar(1000) NOT NULL, + rnd_char char NOT NULL, + rnd_nchar char(1000) NOT NULL, + rnd_ipaddress ipaddress NOT NULL, + rnd_uuid uuid NOT NULL)"""; + assertUpdate(tableQuery); + + testQuery = + """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(distinct rnd_smallint), + count(distinct rnd_tinyint), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(distinct rnd_decimal5), + count(distinct rnd_real), + count(distinct rnd_double), + -- interval literals can't represent smallest possible values allowed by the engine + -- can't count timestamps because their extreme values cannot be expressed as literals + count(distinct rnd_time), + count(distinct rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(distinct rnd_timetz), + count(distinct rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9) + FROM all_types_max_prop + """; + assertQuery(testQuery, + """ + VALUES (1, + 1, + 1, + 1, + -- date + 1, + -- decimal + 1, + 1, + 1, + 1, + 1, + -- real, double + 1, + 1, + -- time + 1, + 1, + 1, + 1, + -- time with time zone + 1, + 1, + 1, + 1) + """); + + // exclusive range to get the max high bound + tableQuery = + """ + CREATE TABLE faker.default.all_types_min_prop ( + rnd_bigint bigint NOT NULL WITH (min = '9223372036854775807'), + rnd_integer integer NOT NULL WITH (min = '2147483647'), + rnd_smallint smallint NOT NULL WITH (min = '32767'), + rnd_tinyint tinyint NOT NULL WITH (min = '127'), + rnd_boolean boolean NOT NULL, + rnd_date date NOT NULL WITH (min = '5881580-07-11'), + rnd_decimal1 decimal NOT NULL WITH (min = '99999999999999999999999999999999999999'), + rnd_decimal2 decimal(18,5) NOT NULL WITH (min = '9999999999999.99999'), + rnd_decimal3 decimal(38,0) NOT NULL WITH (min = '99999999999999999999999999999999999999'), + rnd_decimal4 decimal(38,38) NOT NULL WITH (min = '0.99999999999999999999999999999999999999'), + rnd_decimal5 decimal(5,2) NOT NULL WITH (min = '999.99'), + rnd_real real NOT NULL WITH (min = '1.4E45'), + rnd_double double NOT NULL WITH (min = '4.9E324'), + -- interval literals can't represent smallest possible values allowed by the engine + rnd_interval_day_time interval day to second NOT NULL, + rnd_interval_year interval year to month NOT NULL, + -- can't test timestamps because their extreme values cannot be expressed as literals + rnd_timestamp timestamp NOT NULL, + rnd_timestamp0 timestamp(0) NOT NULL, + rnd_timestamp6 timestamp(6) NOT NULL, + rnd_timestamp9 timestamp(9) NOT NULL, + rnd_timestamptz timestamp with time zone NOT NULL, + rnd_timestamptz0 timestamp(0) with time zone NOT NULL, + rnd_timestamptz6 timestamp(6) with time zone NOT NULL, + rnd_timestamptz9 timestamp(9) with time zone NOT NULL, + rnd_time time NOT NULL WITH (min = '23:59:59.999'), + rnd_time0 time(0) NOT NULL WITH (min = '23:59:59'), + rnd_time6 time(6) NOT NULL WITH (min = '23:59:59.999999'), + rnd_time9 time(9) NOT NULL WITH (min = '23:59:59.999999999'), + rnd_timetz time with time zone NOT NULL WITH (min = '23:59:59.999 +01:00'), + rnd_timetz0 time(0) with time zone NOT NULL WITH (min = '23:59:59 +01:00'), + rnd_timetz6 time(6) with time zone NOT NULL WITH (min = '23:59:59.999999 +01:00'), + rnd_timetz9 time(9) with time zone NOT NULL WITH (min = '23:59:59.999999999 +01:00'), + rnd_timetz12 time(12) with time zone NOT NULL, + rnd_varbinary varbinary NOT NULL, + rnd_varchar varchar NOT NULL, + rnd_nvarchar varchar(1000) NOT NULL, + rnd_char char NOT NULL, + rnd_nchar char(1000) NOT NULL, + rnd_ipaddress ipaddress NOT NULL, + rnd_uuid uuid NOT NULL)"""; + assertUpdate(tableQuery); + + testQuery = + """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(distinct rnd_smallint), + count(distinct rnd_tinyint), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(distinct rnd_decimal5), + count(distinct rnd_real), + count(distinct rnd_double), + -- interval literals can't represent smallest possible values allowed by the engine + -- can't count timestamps because their extreme values cannot be expressed as literals + count(distinct rnd_time), + count(distinct rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(distinct rnd_timetz), + count(distinct rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9) + FROM all_types_min_prop + """; + assertQuery(testQuery, + """ + VALUES (1, + 1, + 1, + 1, + -- date + 1, + -- decimal + 1, + 1, + 1, + 1, + 1, + -- real, double + 1, + 1, + -- time + 1, + 1, + 1, + 1, + -- time with time zone + 1, + 1, + 1, + 1) + """); + + assertUpdate("DROP TABLE faker.default.all_types_range_prop"); + assertUpdate("DROP TABLE faker.default.all_types_max_prop"); + assertUpdate("DROP TABLE faker.default.all_types_min_prop"); + } + + @Test + void testSelectValuesProperty() + { + @Language("SQL") + String tableQuery = + """ + CREATE TABLE faker.default.all_types_values ( + rnd_bigint bigint NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), + rnd_integer integer NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), + rnd_smallint smallint NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), + rnd_tinyint tinyint NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), + rnd_boolean boolean NOT NULL WITH ("allowed_values" = ARRAY['true', 'false']), + rnd_date date NOT NULL WITH ("allowed_values" = ARRAY['2022-03-01', '2022-03-02']), + rnd_decimal1 decimal NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), + rnd_decimal2 decimal(18,5) NOT NULL WITH ("allowed_values" = ARRAY['0.00000', '0.00001']), + rnd_decimal3 decimal(38,0) NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), + rnd_decimal4 decimal(38,38) NOT NULL WITH ("allowed_values" = ARRAY['0.00000000000000000000000000000000000000', '0.00000000000000000000000000000000000001']), + rnd_decimal5 decimal(5,2) NOT NULL WITH ("allowed_values" = ARRAY['0.00', '0.01']), + rnd_real real NOT NULL WITH ("allowed_values" = ARRAY['0.0', '1.4E-45']), + rnd_double double NOT NULL WITH ("allowed_values" = ARRAY['0.0', '4.9E-324']), + rnd_interval_day_time interval day to second NOT NULL WITH ("allowed_values" = ARRAY['0.000', '0.001']), + rnd_interval_year interval year to month NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), + rnd_timestamp timestamp NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000', '2022-03-21 00:00:00.001']), + rnd_timestamp0 timestamp(0) NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00', '2022-03-21 00:00:01']), + rnd_timestamp6 timestamp(6) NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000', '2022-03-21 00:00:00.000001']), + rnd_timestamp9 timestamp(9) NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000000', '2022-03-21 00:00:00.000000001']), + rnd_timestamptz timestamp with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000 +01:00', '2022-03-21 00:00:00.001 +01:00']), + rnd_timestamptz0 timestamp(0) with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00 +01:00', '2022-03-21 00:00:01 +01:00']), + rnd_timestamptz6 timestamp(6) with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000 +01:00', '2022-03-21 00:00:00.000001 +01:00']), + rnd_timestamptz9 timestamp(9) with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000000 +01:00', '2022-03-21 00:00:00.000000001 +01:00']), + rnd_time time NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.456', '01:02:03.457']), + rnd_time0 time(0) NOT NULL WITH ("allowed_values" = ARRAY['01:02:03', '01:02:04']), + rnd_time6 time(6) NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000456', '01:02:03.000457']), + rnd_time9 time(9) NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000000456', '01:02:03.000000457']), + rnd_timetz time with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.456 +01:00', '01:02:03.457 +01:00']), + rnd_timetz0 time(0) with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03 +01:00', '01:02:04 +01:00']), + rnd_timetz6 time(6) with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000456 +01:00', '01:02:03.000457 +01:00']), + rnd_timetz9 time(9) with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000000456 +01:00', '01:02:03.000000457 +01:00']), + rnd_timetz12 time(12) with time zone NOT NULL, + rnd_varbinary varbinary NOT NULL WITH ("allowed_values" = ARRAY['ff', '00']), + rnd_varchar varchar NOT NULL WITH ("allowed_values" = ARRAY['aa', 'bb']), + rnd_nvarchar varchar(1000) NOT NULL WITH ("allowed_values" = ARRAY['aa', 'bb']), + rnd_ipaddress ipaddress NOT NULL WITH ("allowed_values" = ARRAY['0.0.0.0', '1.2.3.4']), + rnd_uuid uuid NOT NULL WITH ("allowed_values" = ARRAY['1fc74d96-0216-449b-a145-455578a9eaa5', '3ee49ede-0026-45e4-ba06-08404f794557'])) + """; + assertUpdate(tableQuery); + + @Language("SQL") + String testQuery; + + // inclusive ranges (BETWEEN) that produce only 2 values + // obtained using `Math.nextUp((float) 0.0)` + testQuery = + """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(distinct rnd_smallint), + count(distinct rnd_tinyint), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(distinct rnd_decimal5), + count(distinct rnd_real), + count(distinct rnd_double), + count(distinct rnd_interval_day_time), + count(distinct rnd_interval_year), + count(distinct rnd_timestamp), + count(distinct rnd_timestamp0), + count(distinct rnd_timestamp6), + count(distinct rnd_timestamp9), + count(distinct rnd_timestamptz), + count(distinct rnd_timestamptz0), + count(distinct rnd_timestamptz6), + count(distinct rnd_timestamptz9), + count(distinct rnd_time), + count(distinct rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(distinct rnd_timetz), + count(distinct rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9), + count(distinct rnd_varbinary), + count(distinct rnd_varchar), + count(distinct rnd_nvarchar), + count(distinct rnd_ipaddress), + count(distinct rnd_uuid) + FROM all_types_values + """; + assertQuery(testQuery, + """ + VALUES (2, + 2, + 2, + 2, + -- date + 2, + -- decimal + 2, + 2, + 2, + 2, + 2, + -- real, double + 2, + 2, + -- intervals + 2, + 2, + -- timestamps + 2, + 2, + 2, + 2, + -- timestamps with time zone + 2, + 2, + 2, + 2, + -- time + 2, + 2, + 2, + 2, + -- time with time zone + 2, + 2, + 2, + 2, + -- character types + 2, + 2, + 2, + -- ip, uuid + 2, + 2) + """); + + assertUpdate("DROP TABLE faker.default.all_types_values"); + } } From 7fb3df846fec25024072eb5f974f0c28d1e94d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 1 Dec 2024 17:21:04 +0100 Subject: [PATCH 074/158] Remove predicate pushdowns in the Faker connector Predicate pushdown in the Faker connector violates the SQL semantics, because when applied to separate columns, correlation between columns is not preserved, and returned results are not deterministic. The `min`, `max`, and `options` column properties should be used instead. --- .../io/trino/plugin/faker/FakerMetadata.java | 61 +- .../trino/plugin/faker/FakerPageSource.java | 35 +- .../plugin/faker/FakerPageSourceProvider.java | 3 +- .../trino/plugin/faker/FakerTableHandle.java | 12 +- .../trino/plugin/faker/TestFakerQueries.java | 541 ------------------ .../plugin/faker/TestFakerSplitManager.java | 3 +- 6 files changed, 13 insertions(+), 642 deletions(-) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java index 1ab3c20d129e..275d04d62004 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java @@ -30,8 +30,6 @@ import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTableVersion; import io.trino.spi.connector.ConnectorViewDefinition; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.ConstraintApplicationResult; import io.trino.spi.connector.LimitApplicationResult; import io.trino.spi.connector.RelationColumnsMetadata; import io.trino.spi.connector.RetryMode; @@ -45,7 +43,6 @@ import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.SchemaFunctionName; import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.statistics.ComputedStatistics; import io.trino.spi.type.BigintType; @@ -152,7 +149,7 @@ public synchronized ConnectorTableHandle getTableHandle(ConnectorSession session } long schemaLimit = (long) schema.properties().getOrDefault(SchemaInfo.DEFAULT_LIMIT_PROPERTY, defaultLimit); long tableLimit = (long) tables.get(tableName).properties().getOrDefault(TableInfo.DEFAULT_LIMIT_PROPERTY, schemaLimit); - return new FakerTableHandle(tableName, TupleDomain.all(), tableLimit); + return new FakerTableHandle(tableName, tableLimit); } @Override @@ -522,62 +519,6 @@ public Optional> applyLimit( true)); } - @Override - public Optional> applyFilter( - ConnectorSession session, - ConnectorTableHandle table, - Constraint constraint) - { - FakerTableHandle fakerTable = (FakerTableHandle) table; - - TupleDomain summary = constraint.getSummary(); - if (summary.isAll()) { - return Optional.empty(); - } - // the only reason not to use isNone is so the linter doesn't complain about not checking an Optional - if (summary.getDomains().isEmpty()) { - throw new IllegalArgumentException("summary cannot be none"); - } - - TupleDomain currentConstraint = fakerTable.constraint(); - if (currentConstraint.getDomains().isEmpty()) { - throw new IllegalArgumentException("currentConstraint is none but should be all!"); - } - - // push down everything, unsupported constraints will throw an exception during data generation - boolean anyUpdated = false; - for (Map.Entry entry : summary.getDomains().get().entrySet()) { - FakerColumnHandle column = (FakerColumnHandle) entry.getKey(); - Domain domain = entry.getValue(); - - if (currentConstraint.getDomains().get().containsKey(column)) { - Domain currentDomain = currentConstraint.getDomains().get().get(column); - // it is important to avoid processing same constraint multiple times - // so that planner doesn't get stuck in a loop - if (currentDomain.equals(domain)) { - continue; - } - // TODO write test cases for this, it doesn't seem to work with IS NULL - currentDomain.union(domain); - } - else { - Map domains = new HashMap<>(currentConstraint.getDomains().get()); - domains.put(column, domain); - currentConstraint = TupleDomain.withColumnDomains(domains); - } - anyUpdated = true; - } - if (!anyUpdated) { - return Optional.empty(); - } - - return Optional.of(new ConstraintApplicationResult<>( - fakerTable.withConstraint(currentConstraint), - TupleDomain.all(), - constraint.getExpression(), - true)); - } - @Override public Collection listFunctions(ConnectorSession session, String schemaName) { diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java index ed676cf0cc49..d911ebc10db5 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -19,11 +19,8 @@ import io.trino.spi.PageBuilder; import io.trino.spi.TrinoException; import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.TupleDomain; import io.trino.spi.type.CharType; import io.trino.spi.type.DecimalType; import io.trino.spi.type.Decimals; @@ -126,7 +123,6 @@ class FakerPageSource Faker faker, Random random, List columns, - TupleDomain constraint, long offset, long limit) { @@ -136,19 +132,17 @@ class FakerPageSource .stream() .map(FakerColumnHandle::type) .collect(toImmutableList()); - requireNonNull(constraint, "constraint is null"); this.limit = limit; this.generators = columns .stream() - .map(column -> getGenerator(column, constraint, offset)) + .map(column -> getGenerator(column, offset)) .collect(toImmutableList()); this.pageBuilder = new PageBuilder(types); } private Generator getGenerator( FakerColumnHandle column, - TupleDomain constraint, long offset) { if (ROW_ID_COLUMN_NAME.equals(column.name())) { @@ -164,9 +158,7 @@ public void accept(BlockBuilder blockBuilder) }; } - return constraintedValueGenerator( - column, - constraint.getDomains().get().getOrDefault(column, Domain.all(column.type()))); + return constraintedValueGenerator(column); } @Override @@ -230,24 +222,14 @@ public void close() closed = true; } - private Generator constraintedValueGenerator(FakerColumnHandle handle, Domain domain) + private Generator constraintedValueGenerator(FakerColumnHandle handle) { - if (domain.isSingleValue()) { - ObjectWriter singleValueWriter = objectWriter(handle.type()); - return (blockBuilder) -> singleValueWriter.accept(blockBuilder, domain.getSingleValue()); - } - if (domain.getValues().isDiscreteSet() || handle.domain().getValues().isDiscreteSet()) { - List values = domain.getValues().isDiscreteSet() - ? domain.getValues().getDiscreteSet() - : handle.domain().getValues().getDiscreteSet(); + if (handle.domain().getValues().isDiscreteSet()) { + List values = handle.domain().getValues().getDiscreteSet(); ObjectWriter singleValueWriter = objectWriter(handle.type()); return (blockBuilder) -> singleValueWriter.accept(blockBuilder, values.get(random.nextInt(values.size()))); } - if (domain.getValues().getRanges().getRangeCount() > 1) { - // this would require calculating weights for each range to retain uniform distribution - throw new TrinoException(INVALID_ROW_FILTER, "Generating random values from more than one range is not supported"); - } - Generator generator = randomValueGenerator(handle, domain.getValues().getRanges().getSpan()); + Generator generator = randomValueGenerator(handle); if (handle.nullProbability() == 0) { return generator; } @@ -261,10 +243,9 @@ private Generator constraintedValueGenerator(FakerColumnHandle handle, Domain do }; } - private Generator randomValueGenerator(FakerColumnHandle handle, Range predicateRange) + private Generator randomValueGenerator(FakerColumnHandle handle) { - Range range = predicateRange.intersect(handle.domain().getValues().getRanges().getSpan()) - .orElseThrow(() -> new TrinoException(INVALID_ROW_FILTER, "Predicates do not overlap with column min and max properties")); + Range range = handle.domain().getValues().getRanges().getSpan(); if (handle.generator() != null) { if (!range.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for columns with a generator expression are not supported"); diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java index a13c89a0d25b..d9eebc39146b 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java @@ -66,10 +66,9 @@ public ConnectorPageSource createPageSource( .map(FakerColumnHandle.class::cast) .collect(toImmutableList()); - FakerTableHandle fakerTable = (FakerTableHandle) table; FakerSplit fakerSplit = (FakerSplit) split; Random random = random(fakerSplit.splitNumber()); - return new FakerPageSource(new Faker(locale, random), random, handles, fakerTable.constraint(), fakerSplit.rowsOffset(), fakerSplit.rowsCount()); + return new FakerPageSource(new Faker(locale, random), random, handles, fakerSplit.rowsOffset(), fakerSplit.rowsCount()); } private Random random(long index) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java index e9535e7126ce..8f821dcbbdc1 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java @@ -14,29 +14,21 @@ package io.trino.plugin.faker; -import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.TupleDomain; import static java.util.Objects.requireNonNull; -public record FakerTableHandle(SchemaTableName schemaTableName, TupleDomain constraint, long limit) +public record FakerTableHandle(SchemaTableName schemaTableName, long limit) implements ConnectorTableHandle { public FakerTableHandle { requireNonNull(schemaTableName, "schemaTableName is null"); - requireNonNull(constraint, "constraint is null"); - } - - public FakerTableHandle withConstraint(TupleDomain constraint) - { - return new FakerTableHandle(schemaTableName, constraint, limit); } public FakerTableHandle withLimit(long limit) { - return new FakerTableHandle(schemaTableName, constraint, limit); + return new FakerTableHandle(schemaTableName, limit); } } diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java index f651da4ccdd0..df09ebcfda4f 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java @@ -310,547 +310,6 @@ void testSelectFunctions() assertQuery(testQuery, "VALUES (true)"); } - @Test - void testSelectRange() - { - @Language("SQL") - String tableQuery = - """ - CREATE TABLE faker.default.all_types_range ( - rnd_bigint bigint NOT NULL, - rnd_integer integer NOT NULL, - rnd_smallint smallint NOT NULL, - rnd_tinyint tinyint NOT NULL, - rnd_boolean boolean NOT NULL, - rnd_date date NOT NULL, - rnd_decimal1 decimal NOT NULL, - rnd_decimal2 decimal(18,5) NOT NULL, - rnd_decimal3 decimal(38,0) NOT NULL, - rnd_decimal4 decimal(38,38) NOT NULL, - rnd_decimal5 decimal(5,2) NOT NULL, - rnd_real real NOT NULL, - rnd_double double NOT NULL, - rnd_interval_day_time interval day to second NOT NULL, - rnd_interval_year interval year to month NOT NULL, - rnd_timestamp timestamp NOT NULL, - rnd_timestamp0 timestamp(0) NOT NULL, - rnd_timestamp6 timestamp(6) NOT NULL, - rnd_timestamp9 timestamp(9) NOT NULL, - rnd_timestamptz timestamp with time zone NOT NULL, - rnd_timestamptz0 timestamp(0) with time zone NOT NULL, - rnd_timestamptz6 timestamp(6) with time zone NOT NULL, - rnd_timestamptz9 timestamp(9) with time zone NOT NULL, - rnd_time time NOT NULL, - rnd_time0 time(0) NOT NULL, - rnd_time6 time(6) NOT NULL, - rnd_time9 time(9) NOT NULL, - rnd_timetz time with time zone NOT NULL, - rnd_timetz0 time(0) with time zone NOT NULL, - rnd_timetz6 time(6) with time zone NOT NULL, - rnd_timetz9 time(9) with time zone NOT NULL, - rnd_timetz12 time(12) with time zone NOT NULL, - rnd_varbinary varbinary NOT NULL, - rnd_varchar varchar NOT NULL, - rnd_nvarchar varchar(1000) NOT NULL, - rnd_char char NOT NULL, - rnd_nchar char(1000) NOT NULL, - rnd_ipaddress ipaddress NOT NULL, - rnd_uuid uuid NOT NULL)"""; - assertUpdate(tableQuery); - - @Language("SQL") - String testQuery; - - // inclusive ranges (BETWEEN) that produce only 2 values - // obtained using `Math.nextUp((float) 0.0)` - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - count(distinct rnd_interval_day_time), - count(distinct rnd_interval_year), - count(distinct rnd_timestamp), - count(distinct rnd_timestamp0), - count(distinct rnd_timestamp6), - count(distinct rnd_timestamp9), - count(distinct rnd_timestamptz), - count(distinct rnd_timestamptz0), - count(distinct rnd_timestamptz6), - count(distinct rnd_timestamptz9), - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9) - FROM all_types_range - WHERE 1=1 - AND rnd_bigint BETWEEN 0 AND 1 - AND rnd_integer BETWEEN 0 AND 1 - AND rnd_smallint BETWEEN 0 AND 1 - AND rnd_tinyint BETWEEN 0 AND 1 - AND rnd_date BETWEEN DATE '2022-03-01' AND DATE '2022-03-02' - AND rnd_decimal1 BETWEEN 0 AND 1 - AND rnd_decimal2 BETWEEN 0.00000 AND 0.00001 - AND rnd_decimal3 BETWEEN 0 AND 1 - AND rnd_decimal4 BETWEEN DECIMAL '0.00000000000000000000000000000000000000' AND DECIMAL '0.00000000000000000000000000000000000001' - AND rnd_decimal5 BETWEEN 0.00 AND 0.01 - AND rnd_real BETWEEN REAL '0.0' AND REAL '1.4E-45' - AND rnd_double BETWEEN DOUBLE '0.0' AND DOUBLE '4.9E-324' - AND rnd_interval_day_time BETWEEN INTERVAL '0.000' SECOND AND INTERVAL '0.001' SECOND - AND rnd_interval_year BETWEEN INTERVAL '0' MONTH AND INTERVAL '1' MONTH - AND rnd_timestamp BETWEEN TIMESTAMP '2022-03-21 00:00:00.000' AND TIMESTAMP '2022-03-21 00:00:00.001' - AND rnd_timestamp0 BETWEEN TIMESTAMP '2022-03-21 00:00:00' AND TIMESTAMP '2022-03-21 00:00:01' - AND rnd_timestamp6 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000' AND TIMESTAMP '2022-03-21 00:00:00.000001' - AND rnd_timestamp9 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000000' AND TIMESTAMP '2022-03-21 00:00:00.000000001' - AND rnd_timestamptz BETWEEN TIMESTAMP '2022-03-21 00:00:00.000 +01:00' AND TIMESTAMP '2022-03-21 00:00:00.001 +01:00' - AND rnd_timestamptz0 BETWEEN TIMESTAMP '2022-03-21 00:00:00 +01:00' AND TIMESTAMP '2022-03-21 00:00:01 +01:00' - AND rnd_timestamptz6 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000 +01:00' AND TIMESTAMP '2022-03-21 00:00:00.000001 +01:00' - AND rnd_timestamptz9 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000000 +01:00' AND TIMESTAMP '2022-03-21 00:00:00.000000001 +01:00' - AND rnd_time BETWEEN TIME '01:02:03.456' AND TIME '01:02:03.457' - AND rnd_time0 BETWEEN TIME '01:02:03' AND TIME '01:02:04' - AND rnd_time6 BETWEEN TIME '01:02:03.000456' AND TIME '01:02:03.000457' - AND rnd_time9 BETWEEN TIME '01:02:03.000000456' AND TIME '01:02:03.000000457' - AND rnd_timetz BETWEEN TIME '01:02:03.456 +01:00' AND TIME '01:02:03.457 +01:00' - AND rnd_timetz0 BETWEEN TIME '01:02:03 +01:00' AND TIME '01:02:04 +01:00' - AND rnd_timetz6 BETWEEN TIME '01:02:03.000456 +01:00' AND TIME '01:02:03.000457 +01:00' - AND rnd_timetz9 BETWEEN TIME '01:02:03.000000456 +01:00' AND TIME '01:02:03.000000457 +01:00'\s"""; - assertQuery(testQuery, - """ - VALUES (2, - 2, - 2, - 2, - -- date - 2, - -- decimal - 2, - 2, - 2, - 2, - 2, - -- real, double - 2, - 2, - -- intervals - 2, - 2, - -- timestamps - 2, - 2, - 2, - 2, - -- timestamps with time zone - 2, - 2, - 2, - 2, - -- time - 2, - 2, - 2, - 2, - -- time with time zone - 2, - 2, - 2, - 2) - """); - - // exclusive ranges that produce only 1 value - // obtained using `Math.nextUp((float) 0.0)` - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - count(distinct rnd_interval_day_time), - count(distinct rnd_interval_year), - count(distinct rnd_timestamp), - count(distinct rnd_timestamp0), - count(distinct rnd_timestamp6), - count(distinct rnd_timestamp9), - count(distinct rnd_timestamptz), - count(distinct rnd_timestamptz0), - count(distinct rnd_timestamptz6), - count(distinct rnd_timestamptz9), - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9) - FROM all_types_range - WHERE 1=1 - AND rnd_bigint > 0 AND rnd_bigint < 2 - AND rnd_integer > 0 AND rnd_integer < 2 - AND rnd_smallint > 0 AND rnd_smallint < 2 - AND rnd_tinyint > 0 AND rnd_tinyint < 2 - AND rnd_date > DATE '2022-03-01' AND rnd_date < DATE '2022-03-03' - AND rnd_decimal1 > 0 AND rnd_decimal1 < 2 - AND rnd_decimal2 > 0.00000 AND rnd_decimal2 < 0.00002 - AND rnd_decimal3 > 0 AND rnd_decimal3 < 2 - AND rnd_decimal4 > DECIMAL '0.00000000000000000000000000000000000000' AND rnd_decimal4 < DECIMAL '0.00000000000000000000000000000000000002' - AND rnd_decimal5 > 0.00 AND rnd_decimal5 < 0.02 - AND rnd_real > REAL '0.0' AND rnd_real < REAL '2.8E-45' - AND rnd_double > DOUBLE '0.0' AND rnd_double < DOUBLE '1.0E-323' - AND rnd_interval_day_time > INTERVAL '0.000' SECOND AND rnd_interval_day_time < INTERVAL '0.002' SECOND - AND rnd_interval_year > INTERVAL '0' MONTH AND rnd_interval_year < INTERVAL '2' MONTH - AND rnd_timestamp > TIMESTAMP '2022-03-21 00:00:00.000' AND rnd_timestamp < TIMESTAMP '2022-03-21 00:00:00.002' - AND rnd_timestamp0 > TIMESTAMP '2022-03-21 00:00:00' AND rnd_timestamp0 < TIMESTAMP '2022-03-21 00:00:02' - AND rnd_timestamp6 > TIMESTAMP '2022-03-21 00:00:00.000000' AND rnd_timestamp6 < TIMESTAMP '2022-03-21 00:00:00.000002' - AND rnd_timestamp9 > TIMESTAMP '2022-03-21 00:00:00.000000000' AND rnd_timestamp9 < TIMESTAMP '2022-03-21 00:00:00.000000002' - AND rnd_timestamptz > TIMESTAMP '2022-03-21 00:00:00.000 +01:00' AND rnd_timestamptz < TIMESTAMP '2022-03-21 00:00:00.002 +01:00' - AND rnd_timestamptz0 > TIMESTAMP '2022-03-21 00:00:00 +01:00' AND rnd_timestamptz0 < TIMESTAMP '2022-03-21 00:00:02 +01:00' - AND rnd_timestamptz6 > TIMESTAMP '2022-03-21 00:00:00.000000 +01:00' AND rnd_timestamptz6 < TIMESTAMP '2022-03-21 00:00:00.000002 +01:00' - AND rnd_timestamptz9 > TIMESTAMP '2022-03-21 00:00:00.000000000 +01:00' AND rnd_timestamptz9 < TIMESTAMP '2022-03-21 00:00:00.000000002 +01:00' - AND rnd_time > TIME '01:02:03.456' AND rnd_time < TIME '01:02:03.458' - AND rnd_time0 > TIME '01:02:03' AND rnd_time0 < TIME '01:02:05' - AND rnd_time6 > TIME '01:02:03.000456' AND rnd_time6 < TIME '01:02:03.000458' - AND rnd_time9 > TIME '01:02:03.000000456' AND rnd_time9 < TIME '01:02:03.000000458' - AND rnd_timetz > TIME '01:02:03.456 +01:00' AND rnd_timetz < TIME '01:02:03.458 +01:00' - AND rnd_timetz0 > TIME '01:02:03 +01:00' AND rnd_timetz0 < TIME '01:02:05 +01:00' - AND rnd_timetz6 > TIME '01:02:03.000456 +01:00' AND rnd_timetz6 < TIME '01:02:03.000458 +01:00' - AND rnd_timetz9 > TIME '01:02:03.000000456 +01:00' AND rnd_timetz9 < TIME '01:02:03.000000458 +01:00'\s"""; - assertQuery(testQuery, - """ - VALUES (1, - 1, - 1, - 1, - -- date - 1, - -- decimal - 1, - 1, - 1, - 1, - 1, - -- real, double - 1, - 1, - -- intervals - 1, - 1, - -- timestamps - 1, - 1, - 1, - 1, - -- timestamps with time zone - 1, - 1, - 1, - 1, - -- time - 1, - 1, - 1, - 1, - -- time with time zone - 1, - 1, - 1, - 1) - """); - - // inclusive range to get the min low bound - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - -- interval literals can't represent smallest possible values allowed by the engine - --count(distinct rnd_interval_day_time), - --count(distinct rnd_interval_year), - -- can't count timestamps because their extreme values cannot be expressed as literals - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9) - FROM all_types_range - WHERE 1=1 - AND rnd_bigint <= -9223372036854775808 - AND rnd_integer <= -2147483648 - AND rnd_smallint <= -32768 - AND rnd_tinyint <= -128 - -- TODO it actually returns -5877641-06-23 - there's definitely some overflow happening in the engine - AND rnd_date <= DATE '-5877641-06-23' - AND rnd_decimal1 <= DECIMAL '-99999999999999999999999999999999999999' - AND rnd_decimal2 <= DECIMAL '-9999999999999.99999' - AND rnd_decimal3 <= DECIMAL '-99999999999999999999999999999999999999' - AND rnd_decimal4 <= DECIMAL '-0.99999999999999999999999999999999999999' - -- TODO it actually retdurns '-999.98' - AND rnd_decimal5 <= DECIMAL '-999.99' - AND rnd_real <= REAL '1.4E-45' - AND rnd_double <= DOUBLE '4.9E-324' - -- interval literals can't represent smallest possible values allowed by the engine - --AND rnd_interval_day_time <= INTERVAL '-2147483647' SECOND - --AND rnd_interval_year <= INTERVAL '-2147483647' MONTH - AND rnd_time <= TIME '00:00:00.000' - AND rnd_time0 <= TIME '00:00:00' - AND rnd_time6 <= TIME '00:00:00.000000' - AND rnd_time9 <= TIME '00:00:00.000000000' - AND rnd_timetz <= TIME '00:00:00.000 +01:00' - AND rnd_timetz0 <= TIME '00:00:00 +01:00' - AND rnd_timetz6 <= TIME '00:00:00.000000 +01:00' - AND rnd_timetz9 <= TIME '00:00:00.000000000 +01:00' - """; - assertQuery(testQuery, - """ - VALUES (1, - 1, - 1, - 1, - -- date - 1, - -- decimal - 1, - 1, - 1, - 1, - 1, - -- real, double - 1, - 1, - -- intervals - --1, - --1, - -- time - 1, - 1, - 1, - 1, - -- time with time zone - 1, - 1, - 1, - 1) - """); - - // exclusive range to get the max high bound - - assertUpdate("DROP TABLE faker.default.all_types_range"); - } - - @Test - void testSelectIn() - { - @Language("SQL") - String tableQuery = - """ - CREATE TABLE faker.default.all_types_in ( - rnd_bigint bigint NOT NULL, - rnd_integer integer NOT NULL, - rnd_smallint smallint NOT NULL, - rnd_tinyint tinyint NOT NULL, - rnd_boolean boolean NOT NULL, - rnd_date date NOT NULL, - rnd_decimal1 decimal NOT NULL, - rnd_decimal2 decimal(18,5) NOT NULL, - rnd_decimal3 decimal(38,0) NOT NULL, - rnd_decimal4 decimal(38,38) NOT NULL, - rnd_decimal5 decimal(5,2) NOT NULL, - rnd_real real NOT NULL, - rnd_double double NOT NULL, - rnd_interval_day_time interval day to second NOT NULL, - rnd_interval_year interval year to month NOT NULL, - rnd_timestamp timestamp NOT NULL, - rnd_timestamp0 timestamp(0) NOT NULL, - rnd_timestamp6 timestamp(6) NOT NULL, - rnd_timestamp9 timestamp(9) NOT NULL, - rnd_timestamptz timestamp with time zone NOT NULL, - rnd_timestamptz0 timestamp(0) with time zone NOT NULL, - rnd_timestamptz6 timestamp(6) with time zone NOT NULL, - rnd_timestamptz9 timestamp(9) with time zone NOT NULL, - rnd_time time NOT NULL, - rnd_time0 time(0) NOT NULL, - rnd_time6 time(6) NOT NULL, - rnd_time9 time(9) NOT NULL, - rnd_timetz time with time zone NOT NULL, - rnd_timetz0 time(0) with time zone NOT NULL, - rnd_timetz6 time(6) with time zone NOT NULL, - rnd_timetz9 time(9) with time zone NOT NULL, - rnd_timetz12 time(12) with time zone NOT NULL, - rnd_varbinary varbinary NOT NULL, - rnd_varchar varchar NOT NULL, - rnd_nvarchar varchar(1000) NOT NULL, - rnd_ipaddress ipaddress NOT NULL, - rnd_uuid uuid NOT NULL)"""; - assertUpdate(tableQuery); - - @Language("SQL") - String testQuery; - - // inclusive ranges (BETWEEN) that produce only 2 values - // obtained using `Math.nextUp((float) 0.0)` - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - count(distinct rnd_interval_day_time), - count(distinct rnd_interval_year), - count(distinct rnd_timestamp), - count(distinct rnd_timestamp0), - count(distinct rnd_timestamp6), - count(distinct rnd_timestamp9), - count(distinct rnd_timestamptz), - count(distinct rnd_timestamptz0), - count(distinct rnd_timestamptz6), - count(distinct rnd_timestamptz9), - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9), - count(distinct rnd_varbinary), - count(distinct rnd_varchar), - count(distinct rnd_nvarchar), - count(distinct rnd_ipaddress), - count(distinct rnd_uuid) - FROM all_types_in - WHERE 1=1 - AND rnd_bigint IN (0, 1) - AND rnd_integer IN (0, 1) - AND rnd_smallint IN (0, 1) - AND rnd_tinyint IN (0, 1) - AND rnd_date IN (DATE '2022-03-01', DATE '2022-03-02') - AND rnd_decimal1 IN (0, 1) - AND rnd_decimal2 IN (0.00000, 0.00001) - AND rnd_decimal3 IN (0, 1) - AND rnd_decimal4 IN (DECIMAL '0.00000000000000000000000000000000000000', DECIMAL '0.00000000000000000000000000000000000001') - AND rnd_decimal5 IN (0.00, 0.01) - AND rnd_real IN (REAL '0.0', REAL '1.4E-45') - AND rnd_double IN (DOUBLE '0.0', DOUBLE '4.9E-324') - AND rnd_interval_day_time IN (INTERVAL '0.000' SECOND, INTERVAL '0.001' SECOND) - AND rnd_interval_year IN (INTERVAL '0' MONTH, INTERVAL '1' MONTH) - AND rnd_timestamp IN (TIMESTAMP '2022-03-21 00:00:00.000', TIMESTAMP '2022-03-21 00:00:00.001') - AND rnd_timestamp0 IN (TIMESTAMP '2022-03-21 00:00:00', TIMESTAMP '2022-03-21 00:00:01') - AND rnd_timestamp6 IN (TIMESTAMP '2022-03-21 00:00:00.000000', TIMESTAMP '2022-03-21 00:00:00.000001') - AND rnd_timestamp9 IN (TIMESTAMP '2022-03-21 00:00:00.000000000', TIMESTAMP '2022-03-21 00:00:00.000000001') - AND rnd_timestamptz IN (TIMESTAMP '2022-03-21 00:00:00.000 +01:00', TIMESTAMP '2022-03-21 00:00:00.001 +01:00') - AND rnd_timestamptz0 IN (TIMESTAMP '2022-03-21 00:00:00 +01:00', TIMESTAMP '2022-03-21 00:00:01 +01:00') - AND rnd_timestamptz6 IN (TIMESTAMP '2022-03-21 00:00:00.000000 +01:00', TIMESTAMP '2022-03-21 00:00:00.000001 +01:00') - AND rnd_timestamptz9 IN (TIMESTAMP '2022-03-21 00:00:00.000000000 +01:00', TIMESTAMP '2022-03-21 00:00:00.000000001 +01:00') - AND rnd_time IN (TIME '01:02:03.456', TIME '01:02:03.457') - AND rnd_time0 IN (TIME '01:02:03', TIME '01:02:04') - AND rnd_time6 IN (TIME '01:02:03.000456', TIME '01:02:03.000457') - AND rnd_time9 IN (TIME '01:02:03.000000456', TIME '01:02:03.000000457') - AND rnd_timetz IN (TIME '01:02:03.456 +01:00', TIME '01:02:03.457 +01:00') - AND rnd_timetz0 IN (TIME '01:02:03 +01:00', TIME '01:02:04 +01:00') - AND rnd_timetz6 IN (TIME '01:02:03.000456 +01:00', TIME '01:02:03.000457 +01:00') - AND rnd_timetz9 IN (TIME '01:02:03.000000456 +01:00', TIME '01:02:03.000000457 +01:00') - AND rnd_varbinary IN (x'ff', x'00') - AND rnd_varchar IN ('aa', 'bb') - AND rnd_nvarchar IN ('aa', 'bb') - AND rnd_ipaddress IN (IPADDRESS '0.0.0.0', IPADDRESS '1.2.3.4') - AND rnd_uuid IN (UUID '1fc74d96-0216-449b-a145-455578a9eaa5', UUID '3ee49ede-0026-45e4-ba06-08404f794557') - """; - assertQuery(testQuery, - """ - VALUES (2, - 2, - 2, - 2, - -- date - 2, - -- decimal - 2, - 2, - 2, - 2, - 2, - -- real, double - 2, - 2, - -- intervals - 2, - 2, - -- timestamps - 2, - 2, - 2, - 2, - -- timestamps with time zone - 2, - 2, - 2, - 2, - -- time - 2, - 2, - 2, - 2, - -- time with time zone - 2, - 2, - 2, - 2, - -- character types - 2, - 2, - 2, - -- ip, uuid - 2, - 2) - """); - - assertUpdate("DROP TABLE faker.default.all_types_in"); - } - @Test void testSelectRangeProperties() { diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerSplitManager.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerSplitManager.java index 898d16cf25fd..c5a4be4a5e08 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerSplitManager.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerSplitManager.java @@ -18,7 +18,6 @@ import io.trino.spi.connector.Constraint; import io.trino.spi.connector.DynamicFilter; import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.TupleDomain; import io.trino.testing.TestingConnectorSession; import io.trino.testing.TestingTransactionHandle; import org.junit.jupiter.api.Test; @@ -39,7 +38,7 @@ void testSplits() ConnectorSplitSource splitSource = new FakerSplitManager().getSplits( TestingTransactionHandle.create(), TestingConnectorSession.SESSION, - new FakerTableHandle(new SchemaTableName("schema", "table"), TupleDomain.all(), expectedRows), + new FakerTableHandle(new SchemaTableName("schema", "table"), expectedRows), DynamicFilter.EMPTY, Constraint.alwaysTrue()); List splits = splitSource.getNextBatch(1_000_000).get().getSplits(); From 587ca9bbaa83383a82aa2d1d63f0b688c1600c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 26 Dec 2024 14:12:44 +0100 Subject: [PATCH 075/158] Refactor Faker tests to be more readable --- .../trino/plugin/faker/TestFakerQueries.java | 824 +++++------------- 1 file changed, 215 insertions(+), 609 deletions(-) diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java index df09ebcfda4f..f5b537bcb36b 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java @@ -13,6 +13,7 @@ */ package io.trino.plugin.faker; +import com.google.common.collect.ImmutableList; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.H2QueryRunner; import io.trino.testing.QueryAssertions; @@ -21,10 +22,13 @@ import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.Map; +import static io.trino.plugin.faker.ColumnInfo.ALLOWED_VALUES_PROPERTY; import static io.trino.plugin.faker.FakerSplitManager.MAX_ROWS_PER_SPLIT; import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_REFERENCE; +import static java.util.stream.Collectors.joining; import static org.assertj.core.api.Assertions.assertThat; final class TestFakerQueries @@ -68,146 +72,53 @@ void testCannotCommentRowId() @Test void testSelectFromTable() { - @Language("SQL") - String tableQuery = - """ - CREATE TABLE faker.default.all_types ( - rnd_bigint bigint NOT NULL, - rnd_integer integer NOT NULL, - rnd_smallint smallint NOT NULL, - rnd_tinyint tinyint NOT NULL, - rnd_boolean boolean NOT NULL, - rnd_date date NOT NULL, - rnd_decimal1 decimal NOT NULL, - rnd_decimal2 decimal(18,5) NOT NULL, - rnd_decimal3 decimal(38,0) NOT NULL, - rnd_decimal4 decimal(38,38) NOT NULL, - rnd_decimal5 decimal(5,2) NOT NULL, - rnd_real real NOT NULL, - rnd_double double NOT NULL, - rnd_interval_day_time interval day to second NOT NULL, - rnd_interval_year interval year to month NOT NULL, - rnd_timestamp timestamp NOT NULL, - rnd_timestamp0 timestamp(0) NOT NULL, - rnd_timestamp6 timestamp(6) NOT NULL, - rnd_timestamp9 timestamp(9) NOT NULL, - rnd_timestamptz timestamp with time zone NOT NULL, - rnd_timestamptz0 timestamp(0) with time zone NOT NULL, - rnd_timestamptz6 timestamp(6) with time zone NOT NULL, - rnd_timestamptz9 timestamp(9) with time zone NOT NULL, - rnd_time time NOT NULL, - rnd_time0 time(0) NOT NULL, - rnd_time6 time(6) NOT NULL, - rnd_time9 time(9) NOT NULL, - rnd_timetz time with time zone NOT NULL, - rnd_timetz0 time(0) with time zone NOT NULL, - rnd_timetz6 time(6) with time zone NOT NULL, - rnd_timetz9 time(9) with time zone NOT NULL, - rnd_timetz12 time(12) with time zone NOT NULL, - rnd_varbinary varbinary NOT NULL, - rnd_varchar varchar NOT NULL, - rnd_nvarchar varchar(1000) NOT NULL, - rnd_char char NOT NULL, - rnd_nchar char(1000) NOT NULL, - rnd_ipaddress ipaddress NOT NULL, - rnd_uuid uuid NOT NULL)"""; - assertUpdate(tableQuery); - - @Language("SQL") - String testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(rnd_smallint), - count(rnd_tinyint), - count(distinct rnd_boolean), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(rnd_decimal5), - count(rnd_real), - count(distinct rnd_double), - count(distinct rnd_interval_day_time), - count(distinct rnd_interval_year), - count(distinct rnd_timestamp), - count(distinct rnd_timestamp0), - count(distinct rnd_timestamp6), - count(distinct rnd_timestamp9), - count(distinct rnd_timestamptz), - count(distinct rnd_timestamptz0), - count(distinct rnd_timestamptz6), - count(distinct rnd_timestamptz9), - count(rnd_time), - count(rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(rnd_timetz), - count(rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9), - count(distinct rnd_varbinary), - count(rnd_varchar), - count(rnd_nvarchar), - count(distinct rnd_char), - count(distinct rnd_nchar), - count(distinct rnd_ipaddress), - count(distinct rnd_uuid) - FROM all_types"""; - assertQuery(testQuery, - """ - VALUES ( - 1000, - 1000, - 1000, - 1000, - -- boolean, date - 2, - 1000, - -- decimal - 1000, - 1000, - 1000, - 1000, - 1000, - -- real, double - 1000, - 1000, - -- intervals - 1000, - 1000, - -- timestamps - 1000, - 1000, - 1000, - 1000, - -- timestamps with time zone - 1000, - 1000, - 1000, - 1000, - -- time - 1000, - 1000, - 1000, - 1000, - -- time with time zone - 1000, - 1000, - 1000, - 1000, - -- varbinary, varchar, nvarchar, char, nchar - 1000, - 1000, - 1000, - 19, - 1000, - -- ip address, uuid - 1000, - 1000)"""); - assertUpdate("DROP TABLE faker.default.all_types"); + List testCases = ImmutableList.builder() + .add(new TestDataType("rnd_bigint", "bigint", "count(distinct rnd_bigint)", "1000")) + .add(new TestDataType("rnd_integer", "integer", "count(distinct rnd_integer)", "1000")) + .add(new TestDataType("rnd_smallint", "smallint", "count(rnd_smallint)", "1000")) + .add(new TestDataType("rnd_tinyint", "tinyint", "count(rnd_tinyint)", "1000")) + .add(new TestDataType("rnd_boolean", "boolean", "count(distinct rnd_boolean)", "2")) + .add(new TestDataType("rnd_date", "date", "count(distinct rnd_date)", "1000")) + .add(new TestDataType("rnd_decimal1", "decimal", "count(distinct rnd_decimal1)", "1000")) + .add(new TestDataType("rnd_decimal2", "decimal(18,5)", "count(distinct rnd_decimal2)", "1000")) + .add(new TestDataType("rnd_decimal3", "decimal(38,0)", "count(distinct rnd_decimal3)", "1000")) + .add(new TestDataType("rnd_decimal4", "decimal(38,38)", "count(distinct rnd_decimal4)", "1000")) + .add(new TestDataType("rnd_decimal5", "decimal(5,2)", "count(rnd_decimal5)", "1000")) + .add(new TestDataType("rnd_real", "real", "count(rnd_real)", "1000")) + .add(new TestDataType("rnd_double", "double", "count(distinct rnd_double)", "1000")) + .add(new TestDataType("rnd_interval1", "interval day to second", "count(distinct rnd_interval1)", "1000")) + .add(new TestDataType("rnd_interval2", "interval year to month", "count(distinct rnd_interval2)", "1000")) + .add(new TestDataType("rnd_timestamp", "timestamp", "count(distinct rnd_timestamp)", "1000")) + .add(new TestDataType("rnd_timestamp0", "timestamp(0)", "count(distinct rnd_timestamp0)", "1000")) + .add(new TestDataType("rnd_timestamp6", "timestamp(6)", "count(distinct rnd_timestamp6)", "1000")) + .add(new TestDataType("rnd_timestamp9", "timestamp(9)", "count(distinct rnd_timestamp9)", "1000")) + .add(new TestDataType("rnd_timestamptz", "timestamp with time zone", "count(distinct rnd_timestamptz)", "1000")) + .add(new TestDataType("rnd_timestamptz0", "timestamp(0) with time zone", "count(distinct rnd_timestamptz0)", "1000")) + .add(new TestDataType("rnd_timestamptz6", "timestamp(6) with time zone", "count(distinct rnd_timestamptz6)", "1000")) + .add(new TestDataType("rnd_timestamptz9", "timestamp(9) with time zone", "count(distinct rnd_timestamptz9)", "1000")) + .add(new TestDataType("rnd_time", "time", "count(rnd_time)", "1000")) + .add(new TestDataType("rnd_time0", "time(0)", "count(rnd_time0)", "1000")) + .add(new TestDataType("rnd_time6", "time(6)", "count(distinct rnd_time6)", "1000")) + .add(new TestDataType("rnd_time9", "time(9)", "count(distinct rnd_time9)", "1000")) + .add(new TestDataType("rnd_timetz", "time with time zone", "count(rnd_timetz)", "1000")) + .add(new TestDataType("rnd_timetz0", "time(0) with time zone", "count(rnd_timetz0)", "1000")) + .add(new TestDataType("rnd_timetz6", "time(6) with time zone", "count(distinct rnd_timetz6)", "1000")) + .add(new TestDataType("rnd_timetz9", "time(9) with time zone", "count(distinct rnd_timetz9)", "1000")) + .add(new TestDataType("rnd_timetz12", "time(12) with time zone", "count(distinct rnd_timetz12)", "1000")) + .add(new TestDataType("rnd_varbinary", "varbinary", "count(distinct rnd_varbinary)", "1000")) + .add(new TestDataType("rnd_varchar", "varchar", "count(rnd_varchar)", "1000")) + .add(new TestDataType("rnd_nvarchar", "varchar(1000)", "count(rnd_nvarchar)", "1000")) + .add(new TestDataType("rnd_char", "char", "count(distinct rnd_char)", "19")) + .add(new TestDataType("rnd_nchar", "char(1000)", "count(distinct rnd_nchar)", "1000")) + .add(new TestDataType("rnd_ipaddress", "ipaddress", "count(distinct rnd_ipaddress)", "1000")) + .add(new TestDataType("rnd_uuid", "uuid", "count(distinct rnd_uuid)", "1000")) + .build(); + + for (TestDataType testCase : testCases) { + try (TestTable table = new TestTable(getQueryRunner()::execute, "types_" + testCase.name(), "(%s)".formatted(testCase.columnSchema()))) { + assertQuery("SELECT %s FROM %s".formatted(testCase.queryExpression(), table.getName()), "VALUES (%s)".formatted(testCase.expectedValue())); + } + } } @Test @@ -313,487 +224,182 @@ void testSelectFunctions() @Test void testSelectRangeProperties() { - @Language("SQL") - String tableQuery; - @Language("SQL") - String testQuery; - // inclusive ranges that produce only 2 values // high boundary float value obtained using `Math.nextUp((float) 0.0)` - tableQuery = - """ - CREATE TABLE faker.default.all_types_range_prop ( - rnd_bigint bigint NOT NULL WITH (min = '0', max = '1'), - rnd_integer integer NOT NULL WITH (min = '0', max = '1'), - rnd_smallint smallint NOT NULL WITH (min = '0', max = '1'), - rnd_tinyint tinyint NOT NULL WITH (min = '0', max = '1'), - rnd_boolean boolean NOT NULL, - rnd_date date NOT NULL WITH (min = '2022-03-01', max = '2022-03-02'), - rnd_decimal1 decimal NOT NULL WITH (min = '0', max = '1'), - rnd_decimal2 decimal(18,5) NOT NULL WITH (min = '0.00000', max = '0.00001'), - rnd_decimal3 decimal(38,0) NOT NULL WITH (min = '0', max = '1'), - rnd_decimal4 decimal(38,38) NOT NULL WITH (min = '0.00000000000000000000000000000000000000', max = '0.00000000000000000000000000000000000001'), - rnd_decimal5 decimal(5,2) NOT NULL WITH (min = '0.00', max = '0.01'), - rnd_real real NOT NULL WITH (min = '0.0', max = '1.4E-45'), - rnd_double double NOT NULL WITH (min = '0.0', max = '4.9E-324'), - rnd_interval_day_time interval day to second NOT NULL WITH (min = '0.000', max = '0.001'), - rnd_interval_year interval year to month NOT NULL WITH (min = '0', max = '1'), - rnd_timestamp timestamp NOT NULL WITH (min = '2022-03-21 00:00:00.000', max = '2022-03-21 00:00:00.001'), - rnd_timestamp0 timestamp(0) NOT NULL WITH (min = '2022-03-21 00:00:00', max = '2022-03-21 00:00:01'), - rnd_timestamp6 timestamp(6) NOT NULL WITH (min = '2022-03-21 00:00:00.000000', max = '2022-03-21 00:00:00.000001'), - rnd_timestamp9 timestamp(9) NOT NULL WITH (min = '2022-03-21 00:00:00.000000000', max = '2022-03-21 00:00:00.000000001'), - rnd_timestamptz timestamp with time zone NOT NULL WITH (min = '2022-03-21 00:00:00.000 +01:00', max = '2022-03-21 00:00:00.001 +01:00'), - rnd_timestamptz0 timestamp(0) with time zone NOT NULL WITH (min = '2022-03-21 00:00:00 +01:00', max = '2022-03-21 00:00:01 +01:00'), - rnd_timestamptz6 timestamp(6) with time zone NOT NULL WITH (min = '2022-03-21 00:00:00.000000 +01:00', max = '2022-03-21 00:00:00.000001 +01:00'), - rnd_timestamptz9 timestamp(9) with time zone NOT NULL WITH (min = '2022-03-21 00:00:00.000000000 +01:00', max = '2022-03-21 00:00:00.000000001 +01:00'), - rnd_time time NOT NULL WITH (min = '01:02:03.456', max = '01:02:03.457'), - rnd_time0 time(0) NOT NULL WITH (min = '01:02:03', max = '01:02:04'), - rnd_time6 time(6) NOT NULL WITH (min = '01:02:03.000456', max = '01:02:03.000457'), - rnd_time9 time(9) NOT NULL WITH (min = '01:02:03.000000456', max = '01:02:03.000000457'), - rnd_timetz time with time zone NOT NULL WITH (min = '01:02:03.456 +01:00', max = '01:02:03.457 +01:00'), - rnd_timetz0 time(0) with time zone NOT NULL WITH (min = '01:02:03 +01:00', max = '01:02:04 +01:00'), - rnd_timetz6 time(6) with time zone NOT NULL WITH (min = '01:02:03.000456 +01:00', max = '01:02:03.000457 +01:00'), - rnd_timetz9 time(9) with time zone NOT NULL WITH (min = '01:02:03.000000456 +01:00', max = '01:02:03.000000457 +01:00'), - rnd_timetz12 time(12) with time zone NOT NULL, - rnd_varbinary varbinary NOT NULL, - rnd_varchar varchar NOT NULL, - rnd_nvarchar varchar(1000) NOT NULL, - rnd_char char NOT NULL, - rnd_nchar char(1000) NOT NULL, - rnd_ipaddress ipaddress NOT NULL, - rnd_uuid uuid NOT NULL)"""; - assertUpdate(tableQuery); - - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - count(distinct rnd_interval_day_time), - count(distinct rnd_interval_year), - count(distinct rnd_timestamp), - count(distinct rnd_timestamp0), - count(distinct rnd_timestamp6), - count(distinct rnd_timestamp9), - count(distinct rnd_timestamptz), - count(distinct rnd_timestamptz0), - count(distinct rnd_timestamptz6), - count(distinct rnd_timestamptz9), - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9) - FROM all_types_range_prop - """; - assertQuery(testQuery, - """ - VALUES (2, - 2, - 2, - 2, - -- date - 2, - -- decimal - 2, - 2, - 2, - 2, - 2, - -- real, double - 2, - 2, - -- intervals - 2, - 2, - -- timestamps - 2, - 2, - 2, - 2, - -- timestamps with time zone - 2, - 2, - 2, - 2, - -- time - 2, - 2, - 2, - 2, - -- time with time zone - 2, - 2, - 2, - 2) - """); + List testCases = ImmutableList.builder() + .add(new TestDataType("rnd_bigint", "bigint", Map.of("min", "0", "max", "1"), "count(distinct rnd_bigint)", "2")) + .add(new TestDataType("rnd_integer", "integer", Map.of("min", "0", "max", "1"), "count(distinct rnd_integer)", "2")) + .add(new TestDataType("rnd_smallint", "smallint", Map.of("min", "0", "max", "1"), "count(distinct rnd_smallint)", "2")) + .add(new TestDataType("rnd_tinyint", "tinyint", Map.of("min", "0", "max", "1"), "count(distinct rnd_tinyint)", "2")) + .add(new TestDataType("rnd_date", "date", Map.of("min", "2022-03-01", "max", "2022-03-02"), "count(distinct rnd_date)", "2")) + .add(new TestDataType("rnd_decimal1", "decimal", Map.of("min", "0", "max", "1"), "count(distinct rnd_decimal1)", "2")) + .add(new TestDataType("rnd_decimal2", "decimal(18,5)", Map.of("min", "0.00000", "max", "0.00001"), "count(distinct rnd_decimal2)", "2")) + .add(new TestDataType("rnd_decimal3", "decimal(38,0)", Map.of("min", "0", "max", "1"), "count(distinct rnd_decimal3)", "2")) + .add(new TestDataType("rnd_decimal4", "decimal(38,38)", Map.of("min", "0.00000000000000000000000000000000000000", "max", "0.00000000000000000000000000000000000001"), "count(distinct rnd_decimal4)", "2")) + .add(new TestDataType("rnd_decimal5", "decimal(5,2)", Map.of("min", "0.00", "max", "0.01"), "count(distinct rnd_decimal5)", "2")) + .add(new TestDataType("rnd_real", "real", Map.of("min", "0.0", "max", "1.4E-45"), "count(distinct rnd_real)", "2")) + .add(new TestDataType("rnd_double", "double", Map.of("min", "0.0", "max", "4.9E-324"), "count(distinct rnd_double)", "2")) + .add(new TestDataType("rnd_interval1", "interval day to second", Map.of("min", "0.000", "max", "0.001"), "count(distinct rnd_interval1)", "2")) + .add(new TestDataType("rnd_interval2", "interval year to month", Map.of("min", "0", "max", "1"), "count(distinct rnd_interval2)", "2")) + .add(new TestDataType("rnd_timestamp", "timestamp", Map.of("min", "2022-03-21 00:00:00.000", "max", "2022-03-21 00:00:00.001"), "count(distinct rnd_timestamp)", "2")) + .add(new TestDataType("rnd_timestamp0", "timestamp(0)", Map.of("min", "2022-03-21 00:00:00", "max", "2022-03-21 00:00:01"), "count(distinct rnd_timestamp0)", "2")) + .add(new TestDataType("rnd_timestamp6", "timestamp(6)", Map.of("min", "2022-03-21 00:00:00.000000", "max", "2022-03-21 00:00:00.000001"), "count(distinct rnd_timestamp6)", "2")) + .add(new TestDataType("rnd_timestamp9", "timestamp(9)", Map.of("min", "2022-03-21 00:00:00.000000000", "max", "2022-03-21 00:00:00.000000001"), "count(distinct rnd_timestamp9)", "2")) + .add(new TestDataType("rnd_timestamptz", "timestamp with time zone", Map.of("min", "2022-03-21 00:00:00.000 +01:00", "max", "2022-03-21 00:00:00.001 +01:00"), "count(distinct rnd_timestamptz)", "2")) + .add(new TestDataType("rnd_timestamptz0", "timestamp(0) with time zone", Map.of("min", "2022-03-21 00:00:00 +01:00", "max", "2022-03-21 00:00:01 +01:00"), "count(distinct rnd_timestamptz0)", "2")) + .add(new TestDataType("rnd_timestamptz6", "timestamp(6) with time zone", Map.of("min", "2022-03-21 00:00:00.000000 +01:00", "max", "2022-03-21 00:00:00.000001 +01:00"), "count(distinct rnd_timestamptz6)", "2")) + .add(new TestDataType("rnd_timestamptz9", "timestamp(9) with time zone", Map.of("min", "2022-03-21 00:00:00.000000000 +01:00", "max", "2022-03-21 00:00:00.000000001 +01:00"), "count(distinct rnd_timestamptz9)", "2")) + .add(new TestDataType("rnd_time", "time", Map.of("min", "01:02:03.456", "max", "01:02:03.457"), "count(distinct rnd_time)", "2")) + .add(new TestDataType("rnd_time0", "time(0)", Map.of("min", "01:02:03", "max", "01:02:04"), "count(distinct rnd_time0)", "2")) + .add(new TestDataType("rnd_time6", "time(6)", Map.of("min", "01:02:03.000456", "max", "01:02:03.000457"), "count(distinct rnd_time6)", "2")) + .add(new TestDataType("rnd_time9", "time(9)", Map.of("min", "01:02:03.000000456", "max", "01:02:03.000000457"), "count(distinct rnd_time9)", "2")) + .add(new TestDataType("rnd_timetz", "time with time zone", Map.of("min", "01:02:03.456 +01:00", "max", "01:02:03.457 +01:00"), "count(distinct rnd_timetz)", "2")) + .add(new TestDataType("rnd_timetz0", "time(0) with time zone", Map.of("min", "01:02:03 +01:00", "max", "01:02:04 +01:00"), "count(distinct rnd_timetz0)", "2")) + .add(new TestDataType("rnd_timetz6", "time(6) with time zone", Map.of("min", "01:02:03.000456 +01:00", "max", "01:02:03.000457 +01:00"), "count(distinct rnd_timetz6)", "2")) + .add(new TestDataType("rnd_timetz9", "time(9) with time zone", Map.of("min", "01:02:03.000000456 +01:00", "max", "01:02:03.000000457 +01:00"), "count(distinct rnd_timetz9)", "2")) + .add(new TestDataType("rnd_timetz12", "time(12) with time zone", Map.of("min", "01:02:03.000000000456 +01:00", "max", "01:02:03.000000000457 +01:00"), "count(distinct rnd_timetz12)", "2")) + .build(); + + for (TestDataType testCase : testCases) { + try (TestTable table = new TestTable(getQueryRunner()::execute, "range_small_" + testCase.name(), "(%s)".formatted(testCase.columnSchema()))) { + assertQuery("SELECT %s FROM %s".formatted(testCase.queryExpression(), table.getName()), "VALUES (%s)".formatted(testCase.expectedValue())); + } + } // inclusive range to get the min low bound - tableQuery = - """ - CREATE TABLE faker.default.all_types_max_prop ( - rnd_bigint bigint NOT NULL WITH (max = '-9223372036854775808'), - rnd_integer integer NOT NULL WITH (max = '-2147483648'), - rnd_smallint smallint NOT NULL WITH (max = '-32768'), - rnd_tinyint tinyint NOT NULL WITH (max = '-128'), - rnd_boolean boolean NOT NULL, - rnd_date date NOT NULL WITH (max = '-5877641-06-23'), - rnd_decimal1 decimal NOT NULL WITH (max = '-99999999999999999999999999999999999999'), - rnd_decimal2 decimal(18,5) NOT NULL WITH (max = '-9999999999999.99999'), - rnd_decimal3 decimal(38,0) NOT NULL WITH (max = '-99999999999999999999999999999999999999'), - rnd_decimal4 decimal(38,38) NOT NULL WITH (max = '-0.99999999999999999999999999999999999999'), - -- TODO it actually retdurns '-999.98' - rnd_decimal5 decimal(5,2) NOT NULL WITH (max = '-999.99'), - rnd_real real NOT NULL WITH (max = '1.4E-45'), - rnd_double double NOT NULL WITH (max = '4.9E-324'), - -- interval literals can't represent smallest possible values allowed by the engine - rnd_interval_day_time interval day to second NOT NULL, - rnd_interval_year interval year to month NOT NULL, - -- can't test timestamps because their extreme values cannot be expressed as literals - rnd_timestamp timestamp NOT NULL, - rnd_timestamp0 timestamp(0) NOT NULL, - rnd_timestamp6 timestamp(6) NOT NULL, - rnd_timestamp9 timestamp(9) NOT NULL, - rnd_timestamptz timestamp with time zone NOT NULL, - rnd_timestamptz0 timestamp(0) with time zone NOT NULL, - rnd_timestamptz6 timestamp(6) with time zone NOT NULL, - rnd_timestamptz9 timestamp(9) with time zone NOT NULL, - rnd_time time NOT NULL WITH (max = '00:00:00.000'), - rnd_time0 time(0) NOT NULL WITH (max = '00:00:00'), - rnd_time6 time(6) NOT NULL WITH (max = '00:00:00.000000'), - rnd_time9 time(9) NOT NULL WITH (max = '00:00:00.000000000'), - rnd_timetz time with time zone NOT NULL WITH (max = '00:00:00.000 +01:00'), - rnd_timetz0 time(0) with time zone NOT NULL WITH (max = '00:00:00 +01:00'), - rnd_timetz6 time(6) with time zone NOT NULL WITH (max = '00:00:00.000000 +01:00'), - rnd_timetz9 time(9) with time zone NOT NULL WITH (max = '00:00:00.000000000 +01:00'), - rnd_timetz12 time(12) with time zone NOT NULL, - rnd_varbinary varbinary NOT NULL, - rnd_varchar varchar NOT NULL, - rnd_nvarchar varchar(1000) NOT NULL, - rnd_char char NOT NULL, - rnd_nchar char(1000) NOT NULL, - rnd_ipaddress ipaddress NOT NULL, - rnd_uuid uuid NOT NULL)"""; - assertUpdate(tableQuery); - - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - -- interval literals can't represent smallest possible values allowed by the engine - -- can't count timestamps because their extreme values cannot be expressed as literals - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9) - FROM all_types_max_prop - """; - assertQuery(testQuery, - """ - VALUES (1, - 1, - 1, - 1, - -- date - 1, - -- decimal - 1, - 1, - 1, - 1, - 1, - -- real, double - 1, - 1, - -- time - 1, - 1, - 1, - 1, - -- time with time zone - 1, - 1, - 1, - 1) - """); + testCases = ImmutableList.builder() + .add(new TestDataType("rnd_bigint", "bigint", Map.of("max", "-9223372036854775808"), "count(distinct rnd_bigint)", "1")) + .add(new TestDataType("rnd_integer", "integer", Map.of("max", "-2147483648"), "count(distinct rnd_integer)", "1")) + .add(new TestDataType("rnd_smallint", "smallint", Map.of("max", "-32768"), "count(distinct rnd_smallint)", "1")) + .add(new TestDataType("rnd_tinyint", "tinyint", Map.of("max", "-128"), "count(distinct rnd_tinyint)", "1")) + .add(new TestDataType("rnd_date", "date", Map.of("max", "-5877641-06-23"), "count(distinct rnd_date)", "1")) + .add(new TestDataType("rnd_decimal1", "decimal", Map.of("max", "-99999999999999999999999999999999999999"), "count(distinct rnd_decimal1)", "1")) + .add(new TestDataType("rnd_decimal2", "decimal(18,5)", Map.of("max", "-9999999999999.99999"), "count(distinct rnd_decimal2)", "1")) + .add(new TestDataType("rnd_decimal3", "decimal(38,0)", Map.of("max", "-99999999999999999999999999999999999999"), "count(distinct rnd_decimal3)", "1")) + .add(new TestDataType("rnd_decimal4", "decimal(38,38)", Map.of("max", "-0.99999999999999999999999999999999999999"), "count(distinct rnd_decimal4)", "1")) + .add(new TestDataType("rnd_decimal5", "decimal(5,2)", Map.of("max", "-999.99"), "count(distinct rnd_decimal5)", "1")) + .add(new TestDataType("rnd_real", "real", Map.of("max", "1.4E-45"), "count(distinct rnd_real)", "1")) + .add(new TestDataType("rnd_double", "double", Map.of("max", "4.9E-324"), "count(distinct rnd_double)", "1")) + // interval literals can't represent smallest possible values allowed by the engine, so they're not included here + // can't test timestamps because their extreme values cannot be expressed as literals + .add(new TestDataType("rnd_time", "time", Map.of("max", "00:00:00.000"), "count(distinct rnd_time)", "1")) + .add(new TestDataType("rnd_time0", "time(0)", Map.of("max", "00:00:00"), "count(distinct rnd_time0)", "1")) + .add(new TestDataType("rnd_time6", "time(6)", Map.of("max", "00:00:00.000000"), "count(distinct rnd_time6)", "1")) + .add(new TestDataType("rnd_time9", "time(9)", Map.of("max", "00:00:00.000000000"), "count(distinct rnd_time9)", "1")) + .add(new TestDataType("rnd_timetz", "time with time zone", Map.of("max", "00:00:00.000 +01:00"), "count(distinct rnd_timetz)", "1")) + .add(new TestDataType("rnd_timetz0", "time(0) with time zone", Map.of("max", "00:00:00 +01:00"), "count(distinct rnd_timetz0)", "1")) + .add(new TestDataType("rnd_timetz6", "time(6) with time zone", Map.of("max", "00:00:00.000000 +01:00"), "count(distinct rnd_timetz6)", "1")) + .add(new TestDataType("rnd_timetz9", "time(9) with time zone", Map.of("max", "00:00:00.000000000 +01:00"), "count(distinct rnd_timetz9)", "1")) + .add(new TestDataType("rnd_timetz12", "time(12) with time zone", Map.of("max", "00:00:00.000000000000 +01:00"), "count(distinct rnd_timetz12)", "1")) + .build(); + + for (TestDataType testCase : testCases) { + try (TestTable table = new TestTable(getQueryRunner()::execute, "range_max_" + testCase.name(), "(%s)".formatted(testCase.columnSchema()))) { + assertQuery("SELECT %s FROM %s".formatted(testCase.queryExpression(), table.getName()), "VALUES (%s)".formatted(testCase.expectedValue())); + } + } // exclusive range to get the max high bound - tableQuery = - """ - CREATE TABLE faker.default.all_types_min_prop ( - rnd_bigint bigint NOT NULL WITH (min = '9223372036854775807'), - rnd_integer integer NOT NULL WITH (min = '2147483647'), - rnd_smallint smallint NOT NULL WITH (min = '32767'), - rnd_tinyint tinyint NOT NULL WITH (min = '127'), - rnd_boolean boolean NOT NULL, - rnd_date date NOT NULL WITH (min = '5881580-07-11'), - rnd_decimal1 decimal NOT NULL WITH (min = '99999999999999999999999999999999999999'), - rnd_decimal2 decimal(18,5) NOT NULL WITH (min = '9999999999999.99999'), - rnd_decimal3 decimal(38,0) NOT NULL WITH (min = '99999999999999999999999999999999999999'), - rnd_decimal4 decimal(38,38) NOT NULL WITH (min = '0.99999999999999999999999999999999999999'), - rnd_decimal5 decimal(5,2) NOT NULL WITH (min = '999.99'), - rnd_real real NOT NULL WITH (min = '1.4E45'), - rnd_double double NOT NULL WITH (min = '4.9E324'), - -- interval literals can't represent smallest possible values allowed by the engine - rnd_interval_day_time interval day to second NOT NULL, - rnd_interval_year interval year to month NOT NULL, - -- can't test timestamps because their extreme values cannot be expressed as literals - rnd_timestamp timestamp NOT NULL, - rnd_timestamp0 timestamp(0) NOT NULL, - rnd_timestamp6 timestamp(6) NOT NULL, - rnd_timestamp9 timestamp(9) NOT NULL, - rnd_timestamptz timestamp with time zone NOT NULL, - rnd_timestamptz0 timestamp(0) with time zone NOT NULL, - rnd_timestamptz6 timestamp(6) with time zone NOT NULL, - rnd_timestamptz9 timestamp(9) with time zone NOT NULL, - rnd_time time NOT NULL WITH (min = '23:59:59.999'), - rnd_time0 time(0) NOT NULL WITH (min = '23:59:59'), - rnd_time6 time(6) NOT NULL WITH (min = '23:59:59.999999'), - rnd_time9 time(9) NOT NULL WITH (min = '23:59:59.999999999'), - rnd_timetz time with time zone NOT NULL WITH (min = '23:59:59.999 +01:00'), - rnd_timetz0 time(0) with time zone NOT NULL WITH (min = '23:59:59 +01:00'), - rnd_timetz6 time(6) with time zone NOT NULL WITH (min = '23:59:59.999999 +01:00'), - rnd_timetz9 time(9) with time zone NOT NULL WITH (min = '23:59:59.999999999 +01:00'), - rnd_timetz12 time(12) with time zone NOT NULL, - rnd_varbinary varbinary NOT NULL, - rnd_varchar varchar NOT NULL, - rnd_nvarchar varchar(1000) NOT NULL, - rnd_char char NOT NULL, - rnd_nchar char(1000) NOT NULL, - rnd_ipaddress ipaddress NOT NULL, - rnd_uuid uuid NOT NULL)"""; - assertUpdate(tableQuery); - - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - -- interval literals can't represent smallest possible values allowed by the engine - -- can't count timestamps because their extreme values cannot be expressed as literals - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9) - FROM all_types_min_prop - """; - assertQuery(testQuery, - """ - VALUES (1, - 1, - 1, - 1, - -- date - 1, - -- decimal - 1, - 1, - 1, - 1, - 1, - -- real, double - 1, - 1, - -- time - 1, - 1, - 1, - 1, - -- time with time zone - 1, - 1, - 1, - 1) - """); - - assertUpdate("DROP TABLE faker.default.all_types_range_prop"); - assertUpdate("DROP TABLE faker.default.all_types_max_prop"); - assertUpdate("DROP TABLE faker.default.all_types_min_prop"); + testCases = ImmutableList.builder() + .add(new TestDataType("rnd_bigint", "bigint", Map.of("min", "9223372036854775807"), "count(distinct rnd_bigint)", "1")) + .add(new TestDataType("rnd_integer", "integer", Map.of("min", "2147483647"), "count(distinct rnd_integer)", "1")) + .add(new TestDataType("rnd_smallint", "smallint", Map.of("min", "32767"), "count(distinct rnd_smallint)", "1")) + .add(new TestDataType("rnd_tinyint", "tinyint", Map.of("min", "127"), "count(distinct rnd_tinyint)", "1")) + .add(new TestDataType("rnd_date", "date", Map.of("min", "5881580-07-11"), "count(distinct rnd_date)", "1")) + .add(new TestDataType("rnd_decimal1", "decimal", Map.of("min", "99999999999999999999999999999999999999"), "count(distinct rnd_decimal1)", "1")) + .add(new TestDataType("rnd_decimal2", "decimal(18,5)", Map.of("min", "9999999999999.99999"), "count(distinct rnd_decimal2)", "1")) + .add(new TestDataType("rnd_decimal3", "decimal(38,0)", Map.of("min", "99999999999999999999999999999999999999"), "count(distinct rnd_decimal3)", "1")) + .add(new TestDataType("rnd_decimal4", "decimal(38,38)", Map.of("min", "0.99999999999999999999999999999999999999"), "count(distinct rnd_decimal4)", "1")) + .add(new TestDataType("rnd_decimal5", "decimal(5,2)", Map.of("min", "999.99"), "count(distinct rnd_decimal5)", "1")) + .add(new TestDataType("rnd_real", "real", Map.of("min", "1.4E45"), "count(distinct rnd_real)", "1")) + .add(new TestDataType("rnd_double", "double", Map.of("min", "4.9E324"), "count(distinct rnd_double)", "1")) + // interval literals can't represent smallest possible values allowed by the engine, so they're not included here + // can't test timestamps because their extreme values cannot be expressed as literals + .add(new TestDataType("rnd_time", "time", Map.of("min", "23:59:59.999"), "count(distinct rnd_time)", "1")) + .add(new TestDataType("rnd_time0", "time(0)", Map.of("min", "23:59:59"), "count(distinct rnd_time0)", "1")) + .add(new TestDataType("rnd_time6", "time(6)", Map.of("min", "23:59:59.999999"), "count(distinct rnd_time6)", "1")) + .add(new TestDataType("rnd_time9", "time(9)", Map.of("min", "23:59:59.999999999"), "count(distinct rnd_time9)", "1")) + .add(new TestDataType("rnd_timetz", "time with time zone", Map.of("min", "23:59:59.999 +01:00"), "count(distinct rnd_timetz)", "1")) + .add(new TestDataType("rnd_timetz0", "time(0) with time zone", Map.of("min", "23:59:59 +01:00"), "count(distinct rnd_timetz0)", "1")) + .add(new TestDataType("rnd_timetz6", "time(6) with time zone", Map.of("min", "23:59:59.999999 +01:00"), "count(distinct rnd_timetz6)", "1")) + .add(new TestDataType("rnd_timetz9", "time(9) with time zone", Map.of("min", "23:59:59.999999999 +01:00"), "count(distinct rnd_timetz9)", "1")) + .add(new TestDataType("rnd_timetz12", "time(12) with time zone", Map.of("min", "23:59:59.999999999999 +01:00"), "count(distinct rnd_timetz12)", "1")) + .build(); + + for (TestDataType testCase : testCases) { + try (TestTable table = new TestTable(getQueryRunner()::execute, "range_min_" + testCase.name(), "(%s)".formatted(testCase.columnSchema()))) { + assertQuery("SELECT %s FROM %s".formatted(testCase.queryExpression(), table.getName()), "VALUES (%s)".formatted(testCase.expectedValue())); + } + } } @Test void testSelectValuesProperty() { - @Language("SQL") - String tableQuery = - """ - CREATE TABLE faker.default.all_types_values ( - rnd_bigint bigint NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), - rnd_integer integer NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), - rnd_smallint smallint NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), - rnd_tinyint tinyint NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), - rnd_boolean boolean NOT NULL WITH ("allowed_values" = ARRAY['true', 'false']), - rnd_date date NOT NULL WITH ("allowed_values" = ARRAY['2022-03-01', '2022-03-02']), - rnd_decimal1 decimal NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), - rnd_decimal2 decimal(18,5) NOT NULL WITH ("allowed_values" = ARRAY['0.00000', '0.00001']), - rnd_decimal3 decimal(38,0) NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), - rnd_decimal4 decimal(38,38) NOT NULL WITH ("allowed_values" = ARRAY['0.00000000000000000000000000000000000000', '0.00000000000000000000000000000000000001']), - rnd_decimal5 decimal(5,2) NOT NULL WITH ("allowed_values" = ARRAY['0.00', '0.01']), - rnd_real real NOT NULL WITH ("allowed_values" = ARRAY['0.0', '1.4E-45']), - rnd_double double NOT NULL WITH ("allowed_values" = ARRAY['0.0', '4.9E-324']), - rnd_interval_day_time interval day to second NOT NULL WITH ("allowed_values" = ARRAY['0.000', '0.001']), - rnd_interval_year interval year to month NOT NULL WITH ("allowed_values" = ARRAY['0', '1']), - rnd_timestamp timestamp NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000', '2022-03-21 00:00:00.001']), - rnd_timestamp0 timestamp(0) NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00', '2022-03-21 00:00:01']), - rnd_timestamp6 timestamp(6) NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000', '2022-03-21 00:00:00.000001']), - rnd_timestamp9 timestamp(9) NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000000', '2022-03-21 00:00:00.000000001']), - rnd_timestamptz timestamp with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000 +01:00', '2022-03-21 00:00:00.001 +01:00']), - rnd_timestamptz0 timestamp(0) with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00 +01:00', '2022-03-21 00:00:01 +01:00']), - rnd_timestamptz6 timestamp(6) with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000 +01:00', '2022-03-21 00:00:00.000001 +01:00']), - rnd_timestamptz9 timestamp(9) with time zone NOT NULL WITH ("allowed_values" = ARRAY['2022-03-21 00:00:00.000000000 +01:00', '2022-03-21 00:00:00.000000001 +01:00']), - rnd_time time NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.456', '01:02:03.457']), - rnd_time0 time(0) NOT NULL WITH ("allowed_values" = ARRAY['01:02:03', '01:02:04']), - rnd_time6 time(6) NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000456', '01:02:03.000457']), - rnd_time9 time(9) NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000000456', '01:02:03.000000457']), - rnd_timetz time with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.456 +01:00', '01:02:03.457 +01:00']), - rnd_timetz0 time(0) with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03 +01:00', '01:02:04 +01:00']), - rnd_timetz6 time(6) with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000456 +01:00', '01:02:03.000457 +01:00']), - rnd_timetz9 time(9) with time zone NOT NULL WITH ("allowed_values" = ARRAY['01:02:03.000000456 +01:00', '01:02:03.000000457 +01:00']), - rnd_timetz12 time(12) with time zone NOT NULL, - rnd_varbinary varbinary NOT NULL WITH ("allowed_values" = ARRAY['ff', '00']), - rnd_varchar varchar NOT NULL WITH ("allowed_values" = ARRAY['aa', 'bb']), - rnd_nvarchar varchar(1000) NOT NULL WITH ("allowed_values" = ARRAY['aa', 'bb']), - rnd_ipaddress ipaddress NOT NULL WITH ("allowed_values" = ARRAY['0.0.0.0', '1.2.3.4']), - rnd_uuid uuid NOT NULL WITH ("allowed_values" = ARRAY['1fc74d96-0216-449b-a145-455578a9eaa5', '3ee49ede-0026-45e4-ba06-08404f794557'])) - """; - assertUpdate(tableQuery); - - @Language("SQL") - String testQuery; - - // inclusive ranges (BETWEEN) that produce only 2 values + // inclusive ranges that produce only 2 values // obtained using `Math.nextUp((float) 0.0)` - testQuery = - """ - SELECT - count(distinct rnd_bigint), - count(distinct rnd_integer), - count(distinct rnd_smallint), - count(distinct rnd_tinyint), - count(distinct rnd_date), - count(distinct rnd_decimal1), - count(distinct rnd_decimal2), - count(distinct rnd_decimal3), - count(distinct rnd_decimal4), - count(distinct rnd_decimal5), - count(distinct rnd_real), - count(distinct rnd_double), - count(distinct rnd_interval_day_time), - count(distinct rnd_interval_year), - count(distinct rnd_timestamp), - count(distinct rnd_timestamp0), - count(distinct rnd_timestamp6), - count(distinct rnd_timestamp9), - count(distinct rnd_timestamptz), - count(distinct rnd_timestamptz0), - count(distinct rnd_timestamptz6), - count(distinct rnd_timestamptz9), - count(distinct rnd_time), - count(distinct rnd_time0), - count(distinct rnd_time6), - count(distinct rnd_time9), - count(distinct rnd_timetz), - count(distinct rnd_timetz0), - count(distinct rnd_timetz6), - count(distinct rnd_timetz9), - count(distinct rnd_varbinary), - count(distinct rnd_varchar), - count(distinct rnd_nvarchar), - count(distinct rnd_ipaddress), - count(distinct rnd_uuid) - FROM all_types_values - """; - assertQuery(testQuery, - """ - VALUES (2, - 2, - 2, - 2, - -- date - 2, - -- decimal - 2, - 2, - 2, - 2, - 2, - -- real, double - 2, - 2, - -- intervals - 2, - 2, - -- timestamps - 2, - 2, - 2, - 2, - -- timestamps with time zone - 2, - 2, - 2, - 2, - -- time - 2, - 2, - 2, - 2, - -- time with time zone - 2, - 2, - 2, - 2, - -- character types - 2, - 2, - 2, - -- ip, uuid - 2, - 2) - """); + List testCases = ImmutableList.builder() + .add(new TestDataType("rnd_bigint", "bigint", Map.of("allowed_values", "ARRAY['0', '1']"), "count(distinct rnd_bigint)", "2")) + .add(new TestDataType("rnd_integer", "integer", Map.of("allowed_values", "ARRAY['0', '1']"), "count(distinct rnd_integer)", "2")) + .add(new TestDataType("rnd_smallint", "smallint", Map.of("allowed_values", "ARRAY['0', '1']"), "count(distinct rnd_smallint)", "2")) + .add(new TestDataType("rnd_tinyint", "tinyint", Map.of("allowed_values", "ARRAY['0', '1']"), "count(distinct rnd_tinyint)", "2")) + .add(new TestDataType("rnd_boolean", "boolean", Map.of("allowed_values", "ARRAY['true', 'false']"), "count(distinct rnd_boolean)", "2")) + .add(new TestDataType("rnd_date", "date", Map.of("allowed_values", "ARRAY['2022-03-01', '2022-03-02']"), "count(distinct rnd_date)", "2")) + .add(new TestDataType("rnd_decimal1", "decimal", Map.of("allowed_values", "ARRAY['0', '1']"), "count(distinct rnd_decimal1)", "2")) + .add(new TestDataType("rnd_decimal2", "decimal(18,5)", Map.of("allowed_values", "ARRAY['0.00000', '0.00001']"), "count(distinct rnd_decimal2)", "2")) + .add(new TestDataType("rnd_decimal3", "decimal(38,0)", Map.of("allowed_values", "ARRAY['0', '1']"), "count(distinct rnd_decimal3)", "2")) + .add(new TestDataType("rnd_decimal4", "decimal(38,38)", Map.of("allowed_values", "ARRAY['0.00000000000000000000000000000000000000', '0.00000000000000000000000000000000000001']"), "count(distinct rnd_decimal4)", "2")) + .add(new TestDataType("rnd_decimal5", "decimal(5,2)", Map.of("allowed_values", "ARRAY['0.00', '0.01']"), "count(distinct rnd_decimal5)", "2")) + .add(new TestDataType("rnd_real", "real", Map.of("allowed_values", "ARRAY['0.0', '1.4E-45']"), "count(distinct rnd_real)", "2")) + .add(new TestDataType("rnd_double", "double", Map.of("allowed_values", "ARRAY['0.0', '4.9E-324']"), "count(distinct rnd_double)", "2")) + .add(new TestDataType("rnd_interval1", "interval day to second", Map.of("allowed_values", "ARRAY['0.000', '0.001']"), "count(distinct rnd_interval1)", "2")) + .add(new TestDataType("rnd_interval2", "interval year to month", Map.of("allowed_values", "ARRAY['0', '1']"), "count(distinct rnd_interval2)", "2")) + .add(new TestDataType("rnd_timestamp", "timestamp", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00.000', '2022-03-21 00:00:00.001']"), "count(distinct rnd_timestamp)", "2")) + .add(new TestDataType("rnd_timestamp0", "timestamp(0)", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00', '2022-03-21 00:00:01']"), "count(distinct rnd_timestamp0)", "2")) + .add(new TestDataType("rnd_timestamp6", "timestamp(6)", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00.000000', '2022-03-21 00:00:00.000001']"), "count(distinct rnd_timestamp6)", "2")) + .add(new TestDataType("rnd_timestamp9", "timestamp(9)", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00.000000000', '2022-03-21 00:00:00.000000001']"), "count(distinct rnd_timestamp9)", "2")) + .add(new TestDataType("rnd_timestamptz", "timestamp with time zone", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00.000 +01:00', '2022-03-21 00:00:00.001 +01:00']"), "count(distinct rnd_timestamptz)", "2")) + .add(new TestDataType("rnd_timestamptz0", "timestamp(0) with time zone", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00 +01:00', '2022-03-21 00:00:01 +01:00']"), "count(distinct rnd_timestamptz0)", "2")) + .add(new TestDataType("rnd_timestamptz6", "timestamp(6) with time zone", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00.000000 +01:00', '2022-03-21 00:00:00.000001 +01:00']"), "count(distinct rnd_timestamptz6)", "2")) + .add(new TestDataType("rnd_timestamptz9", "timestamp(9) with time zone", Map.of("allowed_values", "ARRAY['2022-03-21 00:00:00.000000000 +01:00', '2022-03-21 00:00:00.000000001 +01:00']"), "count(distinct rnd_timestamptz9)", "2")) + .add(new TestDataType("rnd_time", "time", Map.of("allowed_values", "ARRAY['01:02:03.456', '01:02:03.457']"), "count(distinct rnd_time)", "2")) + .add(new TestDataType("rnd_time0", "time(0)", Map.of("allowed_values", "ARRAY['01:02:03', '01:02:04']"), "count(distinct rnd_time0)", "2")) + .add(new TestDataType("rnd_time6", "time(6)", Map.of("allowed_values", "ARRAY['01:02:03.000456', '01:02:03.000457']"), "count(distinct rnd_time6)", "2")) + .add(new TestDataType("rnd_time9", "time(9)", Map.of("allowed_values", "ARRAY['01:02:03.000000456', '01:02:03.000000457']"), "count(distinct rnd_time9)", "2")) + .add(new TestDataType("rnd_timetz", "time with time zone", Map.of("allowed_values", "ARRAY['01:02:03.456 +01:00', '01:02:03.457 +01:00']"), "count(distinct rnd_timetz)", "2")) + .add(new TestDataType("rnd_timetz0", "time(0) with time zone", Map.of("allowed_values", "ARRAY['01:02:03 +01:00', '01:02:04 +01:00']"), "count(distinct rnd_timetz0)", "2")) + .add(new TestDataType("rnd_timetz6", "time(6) with time zone", Map.of("allowed_values", "ARRAY['01:02:03.000456 +01:00', '01:02:03.000457 +01:00']"), "count(distinct rnd_timetz6)", "2")) + .add(new TestDataType("rnd_timetz9", "time(9) with time zone", Map.of("allowed_values", "ARRAY['01:02:03.000000456 +01:00', '01:02:03.000000457 +01:00']"), "count(distinct rnd_timetz9)", "2")) + .add(new TestDataType("rnd_timetz12", "time(12) with time zone", Map.of("allowed_values", "ARRAY['01:02:03.000000000456 +01:00', '01:02:03.000000000457 +01:00']"), "count(distinct rnd_timetz12)", "2")) + .add(new TestDataType("rnd_varbinary", "varbinary", Map.of("allowed_values", "ARRAY['ff', '00']"), "count(distinct rnd_varbinary)", "2")) + .add(new TestDataType("rnd_varchar", "varchar", Map.of("allowed_values", "ARRAY['aa', 'bb']"), "count(distinct rnd_varchar)", "2")) + .add(new TestDataType("rnd_nvarchar", "varchar(1000)", Map.of("allowed_values", "ARRAY['aa', 'bb']"), "count(distinct rnd_nvarchar)", "2")) + .add(new TestDataType("rnd_ipaddress", "ipaddress", Map.of("allowed_values", "ARRAY['0.0.0.0', '1.2.3.4']"), "count(distinct rnd_ipaddress)", "2")) + .add(new TestDataType("rnd_uuid", "uuid", Map.of("allowed_values", "ARRAY['1fc74d96-0216-449b-a145-455578a9eaa5', '3ee49ede-0026-45e4-ba06-08404f794557']"), "count(distinct rnd_uuid)", "2")) + .build(); + + for (TestDataType testCase : testCases) { + try (TestTable table = new TestTable(getQueryRunner()::execute, "values_" + testCase.name(), "(%s)".formatted(testCase.columnSchema()))) { + assertQuery("SELECT %s FROM %s".formatted(testCase.queryExpression(), table.getName()), "VALUES (%s)".formatted(testCase.expectedValue())); + } + } + } - assertUpdate("DROP TABLE faker.default.all_types_values"); + private record TestDataType(String name, String type, Map properties, String queryExpression, String expectedValue) + { + public TestDataType(String name, String type, String queryExpression, String expectedValue) + { + this(name, type, Map.of(), queryExpression, expectedValue); + } + + String columnSchema() + { + String propertiesSchema = properties.entrySet().stream() + .map(entry -> "\"%s\" = %s".formatted( + entry.getKey(), + entry.getKey().equals(ALLOWED_VALUES_PROPERTY) ? entry.getValue() : "'%s'".formatted(entry.getValue()))) + .collect(joining(", ")); + return "%s %s NOT NULL%s".formatted(name, type, propertiesSchema.isEmpty() ? "" : " WITH (%s)".formatted(propertiesSchema)); + } } } From 43e5eadd76cec56f9845b9f2458b4a274ac79c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 26 Dec 2024 15:41:22 +0100 Subject: [PATCH 076/158] Remove outdated limitations in Faker's docs --- docs/src/main/sphinx/connector/faker.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/src/main/sphinx/connector/faker.md b/docs/src/main/sphinx/connector/faker.md index ed21d2b551f8..904307253323 100644 --- a/docs/src/main/sphinx/connector/faker.md +++ b/docs/src/main/sphinx/connector/faker.md @@ -314,7 +314,3 @@ CREATE TABLE generator.default.customer ( group_id INTEGER WITH (allowed_values = ARRAY['10', '32', '81']) ); ``` - -## Limitations - -* It is not possible to choose the locale used by the Datafaker's generators. From 832398a17ef6d2fd42f3f5018e91c31c93a3a683 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Thu, 26 Dec 2024 22:12:38 +0900 Subject: [PATCH 077/158] Rename to Trino in product tests --- .../trino/tests/product/cli/TestTrinoCli.java | 4 +- .../tests/product/cli/TestTrinoLdapCli.java | 8 +- ...aLakeCreateTableAsSelectCompatibility.java | 12 +- .../TestDeltaLakeUpdateCompatibility.java | 16 +- .../product/hive/AbstractTestHiveViews.java | 4 +- .../product/hive/BaseTestHiveCoercion.java | 12 +- .../tests/product/hive/TestAvroSchemaUrl.java | 16 +- .../tests/product/hive/TestGrantRevoke.java | 2 +- .../hive/TestHiveBasicTableStatistics.java | 16 +- .../product/hive/TestHiveTableStatistics.java | 4 +- .../product/hive/TestHiveViewsLegacy.java | 2 +- .../trino/tests/product/hive/TestRoles.java | 240 +++++++++--------- .../tests/product/jdbc/BaseLdapJdbcTest.java | 10 +- .../tests/product/jdbc/TestLdapTrinoJdbc.java | 4 +- 14 files changed, 175 insertions(+), 175 deletions(-) diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoCli.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoCli.java index 836aba8a92da..715573f254b3 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoCli.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoCli.java @@ -113,10 +113,10 @@ public void shouldDisplayVersion() throws IOException { launchTrinoCli("--version"); - assertThat(trino.readRemainingOutputLines()).containsExactly("Trino CLI " + readPrestoCliVersion()); + assertThat(trino.readRemainingOutputLines()).containsExactly("Trino CLI " + readTrinoCliVersion()); } - private static String readPrestoCliVersion() + private static String readTrinoCliVersion() { try { String version = Resources.toString(Resources.getResource("trino-cli-version.txt"), UTF_8).trim(); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoLdapCli.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoLdapCli.java index e30ea63db6d4..a647e87e2a3f 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoLdapCli.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/cli/TestTrinoLdapCli.java @@ -313,16 +313,16 @@ private void launchTrinoCliWithServerArgument(String... arguments) requireNonNull(ldapServerAddress, "ldapServerAddress is null"); requireNonNull(ldapUserPassword, "ldapUserPassword is null"); - ImmutableList.Builder prestoClientOptions = ImmutableList.builder(); - prestoClientOptions.add( + ImmutableList.Builder trinoClientOptions = ImmutableList.builder(); + trinoClientOptions.add( "--server", ldapServerAddress, "--truststore-path", ldapTruststorePath, "--truststore-password", ldapTruststorePassword, "--user", ldapUserName, "--password"); - prestoClientOptions.add(arguments); - ProcessBuilder processBuilder = getProcessBuilder(prestoClientOptions.build()); + trinoClientOptions.add(arguments); + ProcessBuilder processBuilder = getProcessBuilder(trinoClientOptions.build()); processBuilder.environment().put("TRINO_PASSWORD", ldapUserPassword); trino = new TrinoCliProcess(processBuilder.start()); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java index 6dec47308100..a6845c3492f5 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java @@ -45,7 +45,7 @@ public class TestDeltaLakeCreateTableAsSelectCompatibility { @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) - public void testPrestoTypesWithDatabricks() + public void testTrinoTypesWithDatabricks() { String tableName = "test_dl_ctas_" + randomNameSuffix(); @@ -63,8 +63,8 @@ public void testPrestoTypesWithDatabricks() .containsOnly(row(7)); QueryResult databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s", tableName)); - QueryResult prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\"", tableName)); - assertThat(databricksResult).containsOnly(prestoResult.rows().stream() + QueryResult trinoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\"", tableName)); + assertThat(databricksResult).containsOnly(trinoResult.rows().stream() .map(QueryAssert.Row::new) .collect(toImmutableList())); } @@ -75,7 +75,7 @@ public void testPrestoTypesWithDatabricks() @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) - public void testPrestoTimestampsWithDatabricks() + public void testTrinoTimestampsWithDatabricks() { String tableName = "test_dl_ctas_timestamps_" + randomNameSuffix(); @@ -90,8 +90,8 @@ public void testPrestoTimestampsWithDatabricks() .containsOnly(row(4)); QueryResult databricksResult = onDelta().executeQuery("SELECT id, date_format(timestamp_in_utc, \"yyyy-MM-dd HH:mm:ss.SSS\"), date_format(timestamp_in_new_york, \"yyyy-MM-dd HH:mm:ss.SSS\"), date_format(timestamp_in_warsaw, \"yyyy-MM-dd HH:mm:ss.SSS\") FROM default." + tableName); - QueryResult prestoResult = onTrino().executeQuery("SELECT id, format('%1$tF %1$tT.%1$tL', timestamp_in_utc), format('%1$tF %1$tT.%1$tL', timestamp_in_new_york), format('%1$tF %1$tT.%1$tL', timestamp_in_warsaw) FROM delta.default.\"" + tableName + "\""); - assertThat(databricksResult).containsOnly(prestoResult.rows().stream() + QueryResult trinoResult = onTrino().executeQuery("SELECT id, format('%1$tF %1$tT.%1$tL', timestamp_in_utc), format('%1$tF %1$tT.%1$tL', timestamp_in_new_york), format('%1$tF %1$tT.%1$tL', timestamp_in_warsaw) FROM delta.default.\"" + tableName + "\""); + assertThat(databricksResult).containsOnly(trinoResult.rows().stream() .map(QueryAssert.Row::new) .collect(toImmutableList())); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java index 37ebaaa6e6bd..918507de550e 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java @@ -54,23 +54,23 @@ public void testUpdatesFromDatabricks() try { QueryResult databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s ORDER BY id", tableName)); - QueryResult prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); - assertThat(databricksResult).containsExactlyInOrder(toRows(prestoResult)); + QueryResult trinoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); + assertThat(databricksResult).containsExactlyInOrder(toRows(trinoResult)); onDelta().executeQuery(format("UPDATE default.%s SET value = 'France' WHERE id = 2", tableName)); databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s ORDER BY id", tableName)); - prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); - assertThat(databricksResult).containsExactlyInOrder(toRows(prestoResult)); + trinoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); + assertThat(databricksResult).containsExactlyInOrder(toRows(trinoResult)); onDelta().executeQuery(format("UPDATE default.%s SET value = 'Spain' WHERE id = 2", tableName)); databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s ORDER BY id", tableName)); - prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); - assertThat(databricksResult).containsExactlyInOrder(toRows(prestoResult)); + trinoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); + assertThat(databricksResult).containsExactlyInOrder(toRows(trinoResult)); onDelta().executeQuery(format("UPDATE default.%s SET value = 'Portugal' WHERE id = 2", tableName)); databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s ORDER BY id", tableName)); - prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); - assertThat(databricksResult).containsExactlyInOrder(toRows(prestoResult)); + trinoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); + assertThat(databricksResult).containsExactlyInOrder(toRows(trinoResult)); } finally { dropDeltaTableWithRetry("default." + tableName); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/AbstractTestHiveViews.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/AbstractTestHiveViews.java index 88a8c224f943..73173dc63053 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/AbstractTestHiveViews.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/AbstractTestHiveViews.java @@ -81,7 +81,7 @@ public void testArrayIndexingInView() { onHive().executeQuery("DROP TABLE IF EXISTS test_hive_view_array_index_table"); onHive().executeQuery("CREATE TABLE test_hive_view_array_index_table(an_index int, an_array array)"); - onHive().executeQuery("INSERT INTO TABLE test_hive_view_array_index_table SELECT 1, array('presto','hive') FROM nation WHERE n_nationkey = 1"); + onHive().executeQuery("INSERT INTO TABLE test_hive_view_array_index_table SELECT 1, array('trino','hive') FROM nation WHERE n_nationkey = 1"); // literal array index onHive().executeQuery("DROP VIEW IF EXISTS test_hive_view_array_index_view"); @@ -761,7 +761,7 @@ public void testRunAsInvoker() protected static void assertViewQuery(String query, Consumer assertion) { - // Ensure Hive and Presto view compatibility by comparing the results + // Ensure Hive and Trino view compatibility by comparing the results assertion.accept(assertThat(onHive().executeQuery(query))); assertion.accept(assertThat(onTrino().executeQuery(query))); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java index 9745a3dc051d..7f6d3cc1b18d 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java @@ -203,17 +203,17 @@ protected void doTestHiveCoercion(HiveTableDefinition tableDefinition) Function>> expected = engine -> expectedValuesForEngineProvider(engine, tableName, booleanToVarcharVal); // For Trino, remove unsupported columns - List prestoReadColumns = removeUnsupportedColumnsForTrino(allColumns, tableName); - Map> expectedPrestoResults = expected.apply(Engine.TRINO); + List trinoReadColumns = removeUnsupportedColumnsForTrino(allColumns, tableName); + Map> expectedTrinoResults = expected.apply(Engine.TRINO); // In case of unpartitioned tables we don't support all the column coercion thereby making this assertion conditional if (expectedExceptionsWithTrinoContext().isEmpty()) { - assertThat(ImmutableSet.copyOf(prestoReadColumns)).isEqualTo(expectedPrestoResults.keySet()); + assertThat(ImmutableSet.copyOf(trinoReadColumns)).isEqualTo(expectedTrinoResults.keySet()); } - String prestoSelectQuery = format("SELECT %s FROM %s", String.join(", ", prestoReadColumns), tableName); - assertQueryResults(Engine.TRINO, prestoSelectQuery, expectedPrestoResults, prestoReadColumns, 2); + String trinoSelectQuery = format("SELECT %s FROM %s", String.join(", ", trinoReadColumns), tableName); + assertQueryResults(Engine.TRINO, trinoSelectQuery, expectedTrinoResults, trinoReadColumns, 2); // Additional assertions for VARBINARY coercion - if (prestoReadColumns.contains("binary_to_string")) { + if (trinoReadColumns.contains("binary_to_string")) { List hexRepresentedValue = ImmutableList.of("58EFBFBDEFBFBDEFBFBDEFBFBD", "58EFBFBDEFBFBDEFBFBDEFBFBD58"); if (tableName.toLowerCase(ENGLISH).contains("orc")) { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaUrl.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaUrl.java index 94368abc6094..e5aafaccc4be 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaUrl.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaUrl.java @@ -76,7 +76,7 @@ private void saveResourceOnHdfs(String resource, String location) public Object[][] avroSchemaLocations() { return new Object[][] { - {"file:///docker/trino-product-tests/avro/original_schema.avsc"}, // mounted in hadoop and presto containers + {"file:///docker/trino-product-tests/avro/original_schema.avsc"}, // mounted in hadoop and trino containers {"hdfs://hadoop-master:9000/user/hive/warehouse/TestAvroSchemaUrl/schemas/original_schema.avsc"}, {"hdfs:///user/hive/warehouse/TestAvroSchemaUrl/schemas/original_schema.avsc"}, {"/user/hive/warehouse/TestAvroSchemaUrl/schemas/original_schema.avsc"}, // `avro.schema.url` can actually be path on HDFS (not URL) @@ -149,16 +149,16 @@ public void testAvroSchemaUrlInSerdeProperties() @Test(dataProvider = "avroSchemaLocations", groups = STORAGE_FORMATS) @Flaky(issue = RETRYABLE_FAILURES_ISSUES, match = RETRYABLE_FAILURES_MATCH) - public void testPrestoCreatedTable(String schemaLocation) + public void testTrinoCreatedTable(String schemaLocation) { - onTrino().executeQuery("DROP TABLE IF EXISTS test_avro_schema_url_presto"); - onTrino().executeQuery(format("CREATE TABLE test_avro_schema_url_presto (dummy_col VARCHAR) WITH (format='AVRO', avro_schema_url='%s')", schemaLocation)); - onTrino().executeQuery("INSERT INTO test_avro_schema_url_presto VALUES ('some text', 123042)"); + onTrino().executeQuery("DROP TABLE IF EXISTS test_avro_schema_url_trino"); + onTrino().executeQuery(format("CREATE TABLE test_avro_schema_url_trino (dummy_col VARCHAR) WITH (format='AVRO', avro_schema_url='%s')", schemaLocation)); + onTrino().executeQuery("INSERT INTO test_avro_schema_url_trino VALUES ('some text', 123042)"); - assertThat(onHive().executeQuery("SELECT * FROM test_avro_schema_url_presto")).containsExactlyInOrder(row("some text", 123042)); - assertThat(onTrino().executeQuery("SELECT * FROM test_avro_schema_url_presto")).containsExactlyInOrder(row("some text", 123042)); + assertThat(onHive().executeQuery("SELECT * FROM test_avro_schema_url_trino")).containsExactlyInOrder(row("some text", 123042)); + assertThat(onTrino().executeQuery("SELECT * FROM test_avro_schema_url_trino")).containsExactlyInOrder(row("some text", 123042)); - onTrino().executeQuery("DROP TABLE test_avro_schema_url_presto"); + onTrino().executeQuery("DROP TABLE test_avro_schema_url_trino"); } @Test(groups = STORAGE_FORMATS) diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestGrantRevoke.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestGrantRevoke.java index 10d875656afb..098f74a7d2ac 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestGrantRevoke.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestGrantRevoke.java @@ -58,7 +58,7 @@ public class TestGrantRevoke * Pre-requisites for the tests in this class: * * (1) hive.properties file should have this property set: hive.security=sql-standard - * (2) tempto-configuration.yaml file should have definitions for the following connections to Presto server: + * (2) tempto-configuration.yaml file should have definitions for the following connections to Trino server: * - "alice@trino" that has "jdbc_user: alice" * - "bob@trino" that has "jdbc_user: bob" * - "charlie@trino" that has "jdbc_user: charlie" diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveBasicTableStatistics.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveBasicTableStatistics.java index 9cad3fe41ec4..f8adc77c7480 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveBasicTableStatistics.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveBasicTableStatistics.java @@ -43,7 +43,7 @@ public class TestHiveBasicTableStatistics @Test public void testCreateUnpartitioned() { - String tableName = "test_basic_statistics_unpartitioned_ctas_presto"; + String tableName = "test_basic_statistics_unpartitioned_ctas_trino"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onTrino().executeQuery(format("CREATE TABLE %s AS SELECT * FROM nation", tableName)); @@ -61,7 +61,7 @@ public void testCreateUnpartitioned() @Test public void testCreateExternalUnpartitioned() { - String tableName = "test_basic_statistics_external_unpartitioned_presto"; + String tableName = "test_basic_statistics_external_unpartitioned_trino"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); @@ -87,7 +87,7 @@ public void testCreateExternalUnpartitioned() @Test public void testCreateTableWithNoData() { - String tableName = "test_basic_statistics_unpartitioned_ctas_presto_with_no_data"; + String tableName = "test_basic_statistics_unpartitioned_ctas_trino_with_no_data"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onTrino().executeQuery(format("CREATE TABLE %s AS SELECT * FROM nation WITH NO DATA", tableName)); @@ -104,7 +104,7 @@ public void testCreateTableWithNoData() @Test public void testInsertUnpartitioned() { - String tableName = "test_basic_statistics_unpartitioned_insert_presto"; + String tableName = "test_basic_statistics_unpartitioned_insert_trino"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onTrino().executeQuery(format("" + @@ -140,7 +140,7 @@ public void testInsertUnpartitioned() @Test public void testCreatePartitioned() { - String tableName = "test_basic_statistics_partitioned_ctas_presto"; + String tableName = "test_basic_statistics_partitioned_ctas_trino"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onTrino().executeQuery(format("" + @@ -252,7 +252,7 @@ public void testAnalyzeUnpartitioned() @Test public void testInsertPartitioned() { - String tableName = "test_basic_statistics_partitioned_insert_presto"; + String tableName = "test_basic_statistics_partitioned_insert_trino"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onTrino().executeQuery(format("" + @@ -294,7 +294,7 @@ public void testInsertPartitioned() @Flaky(issue = RETRYABLE_FAILURES_ISSUES, match = RETRYABLE_FAILURES_MATCH) public void testInsertBucketed() { - String tableName = "test_basic_statistics_bucketed_insert_presto"; + String tableName = "test_basic_statistics_bucketed_insert_trino"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onTrino().executeQuery(format("" + @@ -333,7 +333,7 @@ public void testInsertBucketed() @Test public void testInsertBucketedPartitioned() { - String tableName = "test_basic_statistics_bucketed_partitioned_insert_presto"; + String tableName = "test_basic_statistics_bucketed_partitioned_insert_trino"; onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onTrino().executeQuery(format("" + diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java index bdf1c9a202ce..aff4c7072059 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java @@ -1403,9 +1403,9 @@ public void testComputeStatisticsForTableWithOnlyDateColumns() @Test @Flaky(issue = RETRYABLE_FAILURES_ISSUES, match = RETRYABLE_FAILURES_MATCH) - public void testMixedHiveAndPrestoStatistics() + public void testMixedHiveAndTrinoStatistics() { - String tableName = "test_mixed_hive_and_presto_statistics"; + String tableName = "test_mixed_hive_and_trino_statistics"; onHive().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); onHive().executeQuery(format("CREATE TABLE %s (a INT) PARTITIONED BY (p INT) STORED AS ORC TBLPROPERTIES ('transactional' = 'false')", tableName)); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveViewsLegacy.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveViewsLegacy.java index 5cff6090ac5c..d49318f8f349 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveViewsLegacy.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveViewsLegacy.java @@ -114,7 +114,7 @@ public void testArrayIndexingInView() "[hive]\n" + "\n" + "actual rows:\n" + - "[presto]"); + "[trino]"); } @Override diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestRoles.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestRoles.java index 3b517e44c0c8..d214c71c6906 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestRoles.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestRoles.java @@ -128,20 +128,20 @@ public void testCreateDropRoleAccessControl() { // Only users that are granted with "admin" role can create, drop and list roles // Alice is not granted with "admin" role - assertQueryFailure(() -> onPrestoAlice().executeQuery(format("CREATE ROLE %s IN hive", ROLE3))) + assertQueryFailure(() -> onTrinoAlice().executeQuery(format("CREATE ROLE %s IN hive", ROLE3))) .hasMessageContaining("Cannot create role %s", ROLE3); - assertQueryFailure(() -> onPrestoAlice().executeQuery(format("DROP ROLE %s IN hive", ROLE3))) + assertQueryFailure(() -> onTrinoAlice().executeQuery(format("DROP ROLE %s IN hive", ROLE3))) .hasMessageContaining("Cannot drop role %s", ROLE3); - assertQueryFailure(() -> onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.roles")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.roles")) .hasMessageContaining("Cannot select from table information_schema.roles"); } @Test(groups = {AUTHORIZATION, PROFILE_SPECIFIC_TESTS}) public void testPublicRoleIsGrantedToEveryone() { - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .contains(row("alice", "USER", "public", "NO")); - assertThat(onPrestoBob().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoBob().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .contains(row("bob", "USER", "public", "NO")); } @@ -157,7 +157,7 @@ public void testGrantRoleToUser() { onTrino().executeQuery("CREATE ROLE role1 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO")); @@ -170,7 +170,7 @@ public void testGrantRoleToRole() onTrino().executeQuery("CREATE ROLE role2 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO"), @@ -184,7 +184,7 @@ public void testGrantRoleWithAdminOption() onTrino().executeQuery("CREATE ROLE role2 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice WITH ADMIN OPTION IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 WITH ADMIN OPTION IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "YES"), @@ -204,7 +204,7 @@ public void testGrantRoleMultipleTimes() onTrino().executeQuery("GRANT role2 TO ROLE role1 WITH ADMIN OPTION IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice WITH ADMIN OPTION IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 WITH ADMIN OPTION IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "YES"), @@ -216,13 +216,13 @@ public void testRevokeRoleFromUser() { onTrino().executeQuery("CREATE ROLE role1 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO")); onTrino().executeQuery("REVOKE role1 FROM USER alice IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO")); } @@ -234,14 +234,14 @@ public void testRevokeRoleFromRole() onTrino().executeQuery("CREATE ROLE role2 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO"), row("role1", "ROLE", "role2", "NO")); onTrino().executeQuery("REVOKE role2 FROM ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO")); @@ -251,8 +251,8 @@ public void testRevokeRoleFromRole() public void testRevokeRoleFromOwner() { try { - onPrestoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); - assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table")) + onTrinoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); + assertThat(onTrinoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table")) .containsOnly(ImmutableList.of( row("alice", "USER", "alice", "USER", "hive", "default", "test_table", "SELECT", "YES", null), row("alice", "USER", "alice", "USER", "hive", "default", "test_table", "DELETE", "YES", null), @@ -262,14 +262,14 @@ public void testRevokeRoleFromOwner() onTrino().executeQuery("REVOKE SELECT ON hive.default.test_table FROM USER alice"); // now there should be no SELECT privileges shown even though alice has OWNERSHIP - assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table")) + assertThat(onTrinoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table")) .containsOnly(ImmutableList.of( row("alice", "USER", "alice", "USER", "hive", "default", "test_table", "DELETE", "YES", null), row("alice", "USER", "alice", "USER", "hive", "default", "test_table", "UPDATE", "YES", null), row("alice", "USER", "alice", "USER", "hive", "default", "test_table", "INSERT", "YES", null))); } finally { - onPrestoAlice().executeQuery("DROP TABLE IF EXISTS hive.default.test_table"); + onTrinoAlice().executeQuery("DROP TABLE IF EXISTS hive.default.test_table"); } } @@ -278,13 +278,13 @@ public void testDropGrantedRole() { onTrino().executeQuery("CREATE ROLE role1 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO")); onTrino().executeQuery("DROP ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO")); } @@ -296,14 +296,14 @@ public void testRevokeTransitiveRoleFromUser() onTrino().executeQuery("CREATE ROLE role2 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO"), row("role1", "ROLE", "role2", "NO")); onTrino().executeQuery("REVOKE role1 FROM USER alice IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO")); } @@ -317,7 +317,7 @@ public void testRevokeTransitiveRoleFromRole() onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 IN hive"); onTrino().executeQuery("GRANT role3 TO ROLE role2 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO"), @@ -325,7 +325,7 @@ public void testRevokeTransitiveRoleFromRole() row("role2", "ROLE", "role3", "NO")); onTrino().executeQuery("REVOKE role2 FROM ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO")); @@ -340,7 +340,7 @@ public void testDropTransitiveRole() onTrino().executeQuery("GRANT role1 TO USER alice IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 IN hive"); onTrino().executeQuery("GRANT role3 TO ROLE role2 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO"), @@ -348,7 +348,7 @@ public void testDropTransitiveRole() row("role2", "ROLE", "role3", "NO")); onTrino().executeQuery("DROP ROLE role2 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO")); @@ -361,7 +361,7 @@ public void testRevokeAdminOption() onTrino().executeQuery("CREATE ROLE role2 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice WITH ADMIN OPTION IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 WITH ADMIN OPTION IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "YES"), @@ -370,7 +370,7 @@ public void testRevokeAdminOption() onTrino().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER alice IN hive"); onTrino().executeQuery("REVOKE ADMIN OPTION FOR role2 FROM ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO"), @@ -384,7 +384,7 @@ public void testRevokeMultipleTimes() onTrino().executeQuery("CREATE ROLE role2 IN hive"); onTrino().executeQuery("GRANT role1 TO USER alice WITH ADMIN OPTION IN hive"); onTrino().executeQuery("GRANT role2 TO ROLE role1 WITH ADMIN OPTION IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "YES"), @@ -395,7 +395,7 @@ public void testRevokeMultipleTimes() onTrino().executeQuery("REVOKE ADMIN OPTION FOR role2 FROM ROLE role1 IN hive"); onTrino().executeQuery("REVOKE ADMIN OPTION FOR role2 FROM ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO"), row("alice", "USER", "role1", "NO"), @@ -406,7 +406,7 @@ public void testRevokeMultipleTimes() onTrino().executeQuery("REVOKE role2 FROM ROLE role1 IN hive"); onTrino().executeQuery("REVOKE role2 FROM ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.applicable_roles")) .containsOnly( row("alice", "USER", "public", "NO")); } @@ -417,50 +417,50 @@ public void testGrantRevokeRoleAccessControl() onTrino().executeQuery("CREATE ROLE role1 IN hive"); onTrino().executeQuery("CREATE ROLE role2 IN hive"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("GRANT role1 TO USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("GRANT role1 TO USER bob IN hive")) .hasMessageContaining("Cannot grant roles [role1] to [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive")) .hasMessageContaining("Cannot grant roles [role1] to [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive")) .hasMessageContaining("Cannot revoke roles [role1] from [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive")) .hasMessageContaining("Cannot revoke roles [role1] from [USER bob]"); onTrino().executeQuery("GRANT role1 TO USER alice WITH ADMIN OPTION IN hive"); - onPrestoAlice().executeQuery("GRANT role1 TO USER bob IN hive"); - onPrestoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive"); - onPrestoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive"); - onPrestoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive"); + onTrinoAlice().executeQuery("GRANT role1 TO USER bob IN hive"); + onTrinoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive"); + onTrinoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive"); + onTrinoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive"); onTrino().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER alice IN hive"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("GRANT role1 TO USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("GRANT role1 TO USER bob IN hive")) .hasMessageContaining("Cannot grant roles [role1] to [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive")) .hasMessageContaining("Cannot grant roles [role1] to [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive")) .hasMessageContaining("Cannot revoke roles [role1] from [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive")) .hasMessageContaining("Cannot revoke roles [role1] from [USER bob]"); onTrino().executeQuery("GRANT role2 TO USER alice IN hive"); onTrino().executeQuery("GRANT role1 TO ROLE role2 WITH ADMIN OPTION IN hive"); - onPrestoAlice().executeQuery("GRANT role1 TO USER bob IN hive"); - onPrestoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive"); - onPrestoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive"); - onPrestoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive"); + onTrinoAlice().executeQuery("GRANT role1 TO USER bob IN hive"); + onTrinoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive"); + onTrinoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive"); + onTrinoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive"); - onPrestoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM ROLE role2 IN hive"); + onTrinoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM ROLE role2 IN hive"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("GRANT role1 TO USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("GRANT role1 TO USER bob IN hive")) .hasMessageContaining("Cannot grant roles [role1] to [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("GRANT role1 TO USER bob WITH ADMIN OPTION IN hive")) .hasMessageContaining("Cannot grant roles [role1] to [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("REVOKE role1 FROM USER bob IN hive")) .hasMessageContaining("Cannot revoke roles [role1] from [USER bob]"); - assertQueryFailure(() -> onPrestoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("REVOKE ADMIN OPTION FOR role1 FROM USER bob IN hive")) .hasMessageContaining("Cannot revoke roles [role1] from [USER bob]"); } @@ -474,36 +474,36 @@ public void testSetRole() onTrino().executeQuery("GRANT role2 TO ROLE role1 IN hive"); onTrino().executeQuery("GRANT role3 TO ROLE role2 IN hive"); - onPrestoAlice().executeQuery("SET ROLE ALL IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) + onTrinoAlice().executeQuery("SET ROLE ALL IN hive"); + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) .containsOnly( row("public"), row("role1"), row("role2"), row("role3")); - onPrestoAlice().executeQuery("SET ROLE NONE IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) + onTrinoAlice().executeQuery("SET ROLE NONE IN hive"); + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) .containsOnly( row("public")); - onPrestoAlice().executeQuery("SET ROLE role1 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) + onTrinoAlice().executeQuery("SET ROLE role1 IN hive"); + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) .containsOnly( row("public"), row("role1"), row("role2"), row("role3")); - onPrestoAlice().executeQuery("SET ROLE role2 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) + onTrinoAlice().executeQuery("SET ROLE role2 IN hive"); + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) .containsOnly( row("public"), row("role2"), row("role3")); - onPrestoAlice().executeQuery("SET ROLE role3 IN hive"); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) + onTrinoAlice().executeQuery("SET ROLE role3 IN hive"); + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.enabled_roles")) .containsOnly( row("public"), row("role3")); @@ -536,11 +536,11 @@ public void testShowRoles() row("public"), row("admin"), row("role1")); - assertQueryFailure(() -> onPrestoAlice().executeQuery("SHOW ROLES FROM hive")) + assertQueryFailure(() -> onTrinoAlice().executeQuery("SHOW ROLES FROM hive")) .hasMessageContaining("Cannot show roles"); onTrino().executeQuery("GRANT admin TO alice IN hive"); - onPrestoAlice().executeQuery("SET ROLE admin IN hive"); - assertThat(onPrestoAlice().executeQuery("SHOW ROLES FROM hive")) + onTrinoAlice().executeQuery("SET ROLE admin IN hive"); + assertThat(onTrinoAlice().executeQuery("SHOW ROLES FROM hive")) .containsOnly( row("public"), row("admin"), @@ -550,20 +550,20 @@ public void testShowRoles() @Test(groups = {AUTHORIZATION, PROFILE_SPECIFIC_TESTS}) public void testShowCurrentRoles() { - assertThat(onPrestoAlice().executeQuery("SHOW CURRENT ROLES FROM hive")) + assertThat(onTrinoAlice().executeQuery("SHOW CURRENT ROLES FROM hive")) .containsOnly( row("public")); onTrino().executeQuery("CREATE ROLE role1 IN hive"); onTrino().executeQuery("CREATE ROLE role2 IN hive"); onTrino().executeQuery("GRANT role1 TO alice IN hive"); onTrino().executeQuery("GRANT role2 TO alice IN hive"); - assertThat(onPrestoAlice().executeQuery("SHOW CURRENT ROLES FROM hive")) + assertThat(onTrinoAlice().executeQuery("SHOW CURRENT ROLES FROM hive")) .containsOnly( row("public"), row("role1"), row("role2")); - onPrestoAlice().executeQuery("SET ROLE role2 IN hive"); - assertThat(onPrestoAlice().executeQuery("SHOW CURRENT ROLES FROM hive")) + onTrinoAlice().executeQuery("SET ROLE role2 IN hive"); + assertThat(onTrinoAlice().executeQuery("SHOW CURRENT ROLES FROM hive")) .containsOnly( row("public"), row("role2")); @@ -576,7 +576,7 @@ public void testShowRoleGrants() .containsOnly( row("public"), row("admin")); - assertThat(onPrestoAlice().executeQuery("SHOW ROLE GRANTS FROM hive")) + assertThat(onTrinoAlice().executeQuery("SHOW ROLE GRANTS FROM hive")) .containsOnly( row("public")); onTrino().executeQuery("CREATE ROLE role1 IN hive"); @@ -587,7 +587,7 @@ public void testShowRoleGrants() .containsOnly( row("public"), row("admin")); - assertThat(onPrestoAlice().executeQuery("SHOW ROLE GRANTS FROM hive")) + assertThat(onTrinoAlice().executeQuery("SHOW ROLE GRANTS FROM hive")) .containsOnly( row("public"), row("role1")); @@ -603,61 +603,61 @@ public void testSetRoleCreateDropSchema() @Test(groups = {AUTHORIZATION, PROFILE_SPECIFIC_TESTS}) public void testAdminCanDropAnyTable() { - onPrestoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); + onTrinoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); assertAdminExecute("DROP TABLE hive.default.test_table"); } @Test(groups = {AUTHORIZATION, PROFILE_SPECIFIC_TESTS}) public void testAdminCanRenameAnyTable() { - onPrestoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); + onTrinoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); assertAdminExecute("ALTER TABLE hive.default.test_table RENAME TO hive.default.test_table_1"); - onPrestoAlice().executeQuery("DROP TABLE hive.default.test_table_1"); + onTrinoAlice().executeQuery("DROP TABLE hive.default.test_table_1"); } @Test(groups = {AUTHORIZATION, PROFILE_SPECIFIC_TESTS}) public void testAdminCanAddColumnToAnyTable() { - onPrestoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); + onTrinoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); assertAdminExecute("ALTER TABLE hive.default.test_table ADD COLUMN bar DATE"); - onPrestoAlice().executeQuery("DROP TABLE hive.default.test_table"); + onTrinoAlice().executeQuery("DROP TABLE hive.default.test_table"); } @Test(groups = {AUTHORIZATION, PROFILE_SPECIFIC_TESTS}) public void testAdminCanRenameColumnInAnyTable() { - onPrestoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); + onTrinoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); assertAdminExecute("ALTER TABLE hive.default.test_table RENAME COLUMN foo TO bar"); - onPrestoAlice().executeQuery("DROP TABLE hive.default.test_table"); + onTrinoAlice().executeQuery("DROP TABLE hive.default.test_table"); } @Test(groups = {AUTHORIZATION, PROFILE_SPECIFIC_TESTS}) public void testAdminCanShowAllGrants() { try { - onPrestoBob().executeQuery("CREATE TABLE hive.default.test_table_bob (foo BIGINT)"); - onPrestoAlice().executeQuery("CREATE TABLE hive.default.test_table_alice (foo BIGINT)"); + onTrinoBob().executeQuery("CREATE TABLE hive.default.test_table_bob (foo BIGINT)"); + onTrinoAlice().executeQuery("CREATE TABLE hive.default.test_table_alice (foo BIGINT)"); onTrino().executeQuery("GRANT admin TO alice IN hive"); - onPrestoAlice().executeQuery("SET ROLE ADMIN IN hive"); + onTrinoAlice().executeQuery("SET ROLE ADMIN IN hive"); - assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_alice")) + assertThat(onTrinoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_alice")) .containsOnly(ImmutableList.of( row("alice", "USER", "alice", "USER", "hive", "default", "test_table_alice", "SELECT", "YES", null), row("alice", "USER", "alice", "USER", "hive", "default", "test_table_alice", "DELETE", "YES", null), row("alice", "USER", "alice", "USER", "hive", "default", "test_table_alice", "UPDATE", "YES", null), row("alice", "USER", "alice", "USER", "hive", "default", "test_table_alice", "INSERT", "YES", null))); - assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_bob")) + assertThat(onTrinoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_bob")) .containsOnly(ImmutableList.of( row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "SELECT", "YES", null), row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "DELETE", "YES", null), row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "UPDATE", "YES", null), row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "INSERT", "YES", null))); - onPrestoAlice().executeQuery("GRANT SELECT ON hive.default.test_table_alice TO bob WITH GRANT OPTION"); - onPrestoAlice().executeQuery("GRANT INSERT ON hive.default.test_table_alice TO bob"); + onTrinoAlice().executeQuery("GRANT SELECT ON hive.default.test_table_alice TO bob WITH GRANT OPTION"); + onTrinoAlice().executeQuery("GRANT INSERT ON hive.default.test_table_alice TO bob"); - assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_alice")) + assertThat(onTrinoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_alice")) .containsOnly(ImmutableList.of( row("alice", "USER", "alice", "USER", "hive", "default", "test_table_alice", "SELECT", "YES", null), row("alice", "USER", "alice", "USER", "hive", "default", "test_table_alice", "DELETE", "YES", null), @@ -667,8 +667,8 @@ public void testAdminCanShowAllGrants() row("alice", "USER", "bob", "USER", "hive", "default", "test_table_alice", "INSERT", "NO", null))); } finally { - onPrestoAlice().executeQuery("DROP TABLE IF EXISTS hive.default.test_table_alice"); - onPrestoBob().executeQuery("DROP TABLE IF EXISTS hive.default.test_table_bob"); + onTrinoAlice().executeQuery("DROP TABLE IF EXISTS hive.default.test_table_alice"); + onTrinoBob().executeQuery("DROP TABLE IF EXISTS hive.default.test_table_bob"); onTrino().executeQuery("REVOKE admin FROM alice IN hive"); } } @@ -677,26 +677,26 @@ public void testAdminCanShowAllGrants() public void testAdminCanShowGrantsOnlyFromCurrentSchema() { try { - onPrestoBob().executeQuery("CREATE TABLE hive.default.test_table_bob (foo BIGINT)"); + onTrinoBob().executeQuery("CREATE TABLE hive.default.test_table_bob (foo BIGINT)"); onTrino().executeQuery("CREATE SCHEMA hive.test"); onTrino().executeQuery("GRANT admin TO alice IN hive"); - onPrestoAlice().executeQuery("SET ROLE ADMIN IN hive"); - onPrestoAlice().executeQuery("CREATE TABLE hive.test.test_table_bob (foo BIGINT) with (external_location='/tmp')"); + onTrinoAlice().executeQuery("SET ROLE ADMIN IN hive"); + onTrinoAlice().executeQuery("CREATE TABLE hive.test.test_table_bob (foo BIGINT) with (external_location='/tmp')"); - assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_bob")) + assertThat(onTrinoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_bob")) .containsOnly(ImmutableList.of( row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "SELECT", "YES", null), row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "DELETE", "YES", null), row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "UPDATE", "YES", null), row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "INSERT", "YES", null))); - assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.test.test_table_bob")) + assertThat(onTrinoAlice().executeQuery("SHOW GRANTS ON hive.test.test_table_bob")) .containsOnly(ImmutableList.of( row("alice", "USER", "alice", "USER", "hive", "test", "test_table_bob", "SELECT", "YES", null), row("alice", "USER", "alice", "USER", "hive", "test", "test_table_bob", "DELETE", "YES", null), row("alice", "USER", "alice", "USER", "hive", "test", "test_table_bob", "UPDATE", "YES", null), row("alice", "USER", "alice", "USER", "hive", "test", "test_table_bob", "INSERT", "YES", null))); - assertThat(onPrestoAlice().executeQuery("SELECT * FROM hive.information_schema.table_privileges where table_name = 'test_table_bob'")) + assertThat(onTrinoAlice().executeQuery("SELECT * FROM hive.information_schema.table_privileges where table_name = 'test_table_bob'")) .containsOnly(ImmutableList.of( row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "SELECT", "YES", null), row("bob", "USER", "bob", "USER", "hive", "default", "test_table_bob", "DELETE", "YES", null), @@ -708,8 +708,8 @@ public void testAdminCanShowGrantsOnlyFromCurrentSchema() row("alice", "USER", "alice", "USER", "hive", "test", "test_table_bob", "INSERT", "YES", null))); } finally { - onPrestoBob().executeQuery("DROP TABLE IF EXISTS hive.default.test_table_bob"); - onPrestoAlice().executeQuery("DROP TABLE IF EXISTS hive.test.test_table_bob"); + onTrinoBob().executeQuery("DROP TABLE IF EXISTS hive.default.test_table_bob"); + onTrinoAlice().executeQuery("DROP TABLE IF EXISTS hive.test.test_table_bob"); onTrino().executeQuery("DROP SCHEMA IF EXISTS hive.test"); onTrino().executeQuery("REVOKE admin FROM alice IN hive"); } @@ -724,9 +724,9 @@ public void testSetRoleTablePermissions() onTrino().executeQuery("GRANT role1 TO USER bob IN hive"); onTrino().executeQuery("GRANT role2 TO USER bob IN hive"); - onPrestoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); - onPrestoAlice().executeQuery("GRANT SELECT ON hive.default.test_table TO ROLE role1"); - onPrestoAlice().executeQuery("GRANT INSERT ON hive.default.test_table TO ROLE role2"); + onTrinoAlice().executeQuery("CREATE TABLE hive.default.test_table (foo BIGINT)"); + onTrinoAlice().executeQuery("GRANT SELECT ON hive.default.test_table TO ROLE role1"); + onTrinoAlice().executeQuery("GRANT INSERT ON hive.default.test_table TO ROLE role2"); String select = "SELECT * FROM hive.default.test_table"; String insert = "INSERT INTO hive.default.test_table (foo) VALUES (1)"; @@ -734,46 +734,46 @@ public void testSetRoleTablePermissions() assertAdminExecute(select); assertAdminExecute(insert); - onPrestoBob().executeQuery(select); - onPrestoBob().executeQuery(insert); - assertThat(onPrestoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) + onTrinoBob().executeQuery(select); + onTrinoBob().executeQuery(insert); + assertThat(onTrinoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) .containsOnly(ImmutableList.of( row("alice", "USER", "role1", "ROLE", "hive", "default", "test_table", "SELECT", "NO", null), row("alice", "USER", "role2", "ROLE", "hive", "default", "test_table", "INSERT", "NO", null))); - onPrestoBob().executeQuery("SET ROLE ALL IN hive"); - onPrestoBob().executeQuery(select); - onPrestoBob().executeQuery(insert); - assertThat(onPrestoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) + onTrinoBob().executeQuery("SET ROLE ALL IN hive"); + onTrinoBob().executeQuery(select); + onTrinoBob().executeQuery(insert); + assertThat(onTrinoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) .containsOnly(ImmutableList.of( row("alice", "USER", "role1", "ROLE", "hive", "default", "test_table", "SELECT", "NO", null), row("alice", "USER", "role2", "ROLE", "hive", "default", "test_table", "INSERT", "NO", null))); - onPrestoBob().executeQuery("SET ROLE NONE IN hive"); - assertQueryFailure(() -> onPrestoBob().executeQuery(select)) + onTrinoBob().executeQuery("SET ROLE NONE IN hive"); + assertQueryFailure(() -> onTrinoBob().executeQuery(select)) .hasMessageContaining("Access Denied"); - assertQueryFailure(() -> onPrestoBob().executeQuery(insert)) + assertQueryFailure(() -> onTrinoBob().executeQuery(insert)) .hasMessageContaining("Access Denied"); - assertThat(onPrestoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) + assertThat(onTrinoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) .containsOnly(ImmutableList.of()); - onPrestoBob().executeQuery("SET ROLE role1 IN hive"); - onPrestoBob().executeQuery(select); - assertQueryFailure(() -> onPrestoBob().executeQuery(insert)) + onTrinoBob().executeQuery("SET ROLE role1 IN hive"); + onTrinoBob().executeQuery(select); + assertQueryFailure(() -> onTrinoBob().executeQuery(insert)) .hasMessageContaining("Access Denied"); - assertThat(onPrestoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) + assertThat(onTrinoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) .containsOnly(ImmutableList.of( row("alice", "USER", "role1", "ROLE", "hive", "default", "test_table", "SELECT", "NO", null))); - onPrestoBob().executeQuery("SET ROLE role2 IN hive"); - assertQueryFailure(() -> onPrestoBob().executeQuery(select)) + onTrinoBob().executeQuery("SET ROLE role2 IN hive"); + assertQueryFailure(() -> onTrinoBob().executeQuery(select)) .hasMessageContaining("Access Denied"); - onPrestoBob().executeQuery(insert); - assertThat(onPrestoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) + onTrinoBob().executeQuery(insert); + assertThat(onTrinoBob().executeQuery("SHOW GRANTS ON hive.default.test_table")) .containsOnly(ImmutableList.of( row("alice", "USER", "role2", "ROLE", "hive", "default", "test_table", "INSERT", "NO", null))); - onPrestoAlice().executeQuery("DROP TABLE hive.default.test_table"); + onTrinoAlice().executeQuery("DROP TABLE hive.default.test_table"); } private static void assertAdminExecute(String query) @@ -790,12 +790,12 @@ private static void assertAdminExecute(String query) onTrino().executeQuery(query); } - private static QueryExecutor onPrestoAlice() + private static QueryExecutor onTrinoAlice() { return connectToTrino("alice@trino"); } - private static QueryExecutor onPrestoBob() + private static QueryExecutor onTrinoBob() { return connectToTrino("bob@trino"); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/BaseLdapJdbcTest.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/BaseLdapJdbcTest.java index 869c08584868..2932f1ec0d73 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/BaseLdapJdbcTest.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/BaseLdapJdbcTest.java @@ -58,7 +58,7 @@ public abstract class BaseLdapJdbcTest @Inject @Named("databases.trino.cli_ldap_server_address") - private String prestoServer; + private String trinoServer; @Override public Requirement getRequirements(Configuration configuration) @@ -89,16 +89,16 @@ private Connection getLdapConnection(String name, String password) return DriverManager.getConnection(getLdapUrl(), name, password); } - protected String prestoServer() + protected String trinoServer() { String prefix = "https://"; - checkState(prestoServer.startsWith(prefix), "invalid server address: %s", prestoServer); - return prestoServer.substring(prefix.length()); + checkState(trinoServer.startsWith(prefix), "invalid server address: %s", trinoServer); + return trinoServer.substring(prefix.length()); } protected String getLdapUrl() { - return format(getLdapUrlFormat(), prestoServer(), ldapTruststorePath, ldapTruststorePassword); + return format(getLdapUrlFormat(), trinoServer(), ldapTruststorePath, ldapTruststorePassword); } protected abstract String getLdapUrlFormat(); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/TestLdapTrinoJdbc.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/TestLdapTrinoJdbc.java index 551be50ab413..f98d2fb8fe34 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/TestLdapTrinoJdbc.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/jdbc/TestLdapTrinoJdbc.java @@ -128,7 +128,7 @@ public void shouldFailQueryForLdapWithoutPassword() @Test(groups = {LDAP, TRINO_JDBC, PROFILE_SPECIFIC_TESTS}, timeOut = TIMEOUT) public void shouldFailQueryForLdapWithoutSsl() { - assertThatThrownBy(() -> DriverManager.getConnection("jdbc:trino://" + prestoServer(), ldapUserName, ldapUserPassword)) + assertThatThrownBy(() -> DriverManager.getConnection("jdbc:trino://" + trinoServer(), ldapUserName, ldapUserPassword)) .isInstanceOf(SQLException.class) .hasMessageContaining("TLS/SSL is required for authentication with username and password"); } @@ -136,7 +136,7 @@ public void shouldFailQueryForLdapWithoutSsl() @Test(groups = {LDAP, TRINO_JDBC, PROFILE_SPECIFIC_TESTS}, timeOut = TIMEOUT) public void shouldFailForIncorrectTrustStore() { - String url = format("jdbc:trino://%s?SSL=true&SSLTrustStorePath=%s&SSLTrustStorePassword=%s", prestoServer(), ldapTruststorePath, "wrong_password"); + String url = format("jdbc:trino://%s?SSL=true&SSLTrustStorePath=%s&SSLTrustStorePassword=%s", trinoServer(), ldapTruststorePath, "wrong_password"); assertThatThrownBy(() -> DriverManager.getConnection(url, ldapUserName, ldapUserPassword)) .isInstanceOf(SQLException.class) .hasMessageContaining("Error setting up SSL: keystore password was incorrect"); From 5706953ac9ede004c9746db8073681f2fd8a3e64 Mon Sep 17 00:00:00 2001 From: Krzysztof Sobolewski Date: Fri, 27 Dec 2024 13:45:33 +0100 Subject: [PATCH 078/158] Extract dep.gib.version property --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b17d8681c88..492ba2be73c8 100644 --- a/pom.xml +++ b/pom.xml @@ -197,6 +197,7 @@ 1.15.1 v22.11.0 10.9.0 + 4.5.4 1.45.3 5.3.1 1.7.1 @@ -2989,7 +2990,7 @@ io.github.gitflow-incremental-builder gitflow-incremental-builder - 4.5.4 + ${dep.gib.version} true true From 0c3330a5c5f9946cd46fdbf29f39af97db2277f3 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 25 Dec 2024 22:00:37 +0900 Subject: [PATCH 079/158] Convert testRollbackToSnapshotWithNullArgument to integration test --- .../plugin/iceberg/BaseIcebergConnectorTest.java | 8 ++++++++ .../product/iceberg/TestIcebergProcedureCalls.java | 12 ------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java index 670200247be5..2de9c8dfa04e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java @@ -1709,6 +1709,14 @@ public void testRollbackSnapshot() assertUpdate("DROP TABLE test_rollback"); } + @Test + void testRollbackToSnapshotWithNullArgument() + { + assertQueryFails("CALL system.rollback_to_snapshot(NULL, 'customer_orders', 8954597067493422955)", ".*schema cannot be null.*"); + assertQueryFails("CALL system.rollback_to_snapshot('testdb', NULL, 8954597067493422955)", ".*table cannot be null.*"); + assertQueryFails("CALL system.rollback_to_snapshot('testdb', 'customer_orders', NULL)", ".*snapshot_id cannot be null.*"); + } + @Override protected String errorMessageForInsertIntoNotNullColumn(String columnName) { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergProcedureCalls.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergProcedureCalls.java index 61deb2f3d90e..ca90567730d4 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergProcedureCalls.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergProcedureCalls.java @@ -366,18 +366,6 @@ public void testRollbackToSnapshot() onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", tableName)); } - @Test(groups = {ICEBERG, PROFILE_SPECIFIC_TESTS}) - public void testRollbackToSnapshotWithNullArgument() - { - onTrino().executeQuery("USE iceberg.default"); - assertQueryFailure(() -> onTrino().executeQuery("CALL system.rollback_to_snapshot(NULL, 'customer_orders', 8954597067493422955)")) - .hasMessageMatching(".*schema cannot be null.*"); - assertQueryFailure(() -> onTrino().executeQuery("CALL system.rollback_to_snapshot('testdb', NULL, 8954597067493422955)")) - .hasMessageMatching(".*table cannot be null.*"); - assertQueryFailure(() -> onTrino().executeQuery("CALL system.rollback_to_snapshot('testdb', 'customer_orders', NULL)")) - .hasMessageMatching(".*snapshot_id cannot be null.*"); - } - private long getSecondOldestTableSnapshot(String tableName) { return (Long) onTrino().executeQuery( From 4d76f58fba3a31ed4a2c53a2a889dd48b62ff358 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Tue, 3 Dec 2024 07:26:43 +0900 Subject: [PATCH 080/158] Add newTrinoTable method to AbstractTestQueryFramework --- .../plugin/jdbc/BaseJdbcConnectorTest.java | 52 ++-- .../jdbc/BaseJdbcTableStatisticsTest.java | 3 +- .../bigquery/BaseBigQueryConnectorTest.java | 10 +- .../bigquery/BaseBigQueryTypeMapping.java | 4 +- .../bigquery/TestBigQueryWithProxyTest.java | 2 +- .../BaseCassandraConnectorSmokeTest.java | 2 +- ...raProtocolVersionV3ConnectorSmokeTest.java | 2 +- .../clickhouse/BaseClickHouseTypeMapping.java | 4 +- .../TestClickHouseConnectorTest.java | 30 +-- .../BaseDeltaLakeConnectorSmokeTest.java | 16 +- .../deltalake/TestDeltaLakeAnalyze.java | 12 +- .../plugin/deltalake/TestDeltaLakeBasic.java | 7 +- .../deltalake/TestDeltaLakeConnectorTest.java | 246 +++++++----------- ...estDeltaLakeMetastoreAccessOperations.java | 18 +- .../glue/TestDeltaLakeViewsGlueMetastore.java | 2 +- .../trino/plugin/faker/TestFakerQueries.java | 12 +- .../plugin/hive/BaseHiveConnectorTest.java | 25 +- .../TestGlueHiveMetastoreSkipArchive.java | 2 +- .../hive/orc/TestHiveOrcWithShortZoneId.java | 6 +- .../BaseIcebergConnectorSmokeTest.java | 26 +- .../iceberg/BaseIcebergConnectorTest.java | 92 +++---- .../iceberg/BaseIcebergSystemTables.java | 9 +- .../TestIcebergConnectorSmokeTest.java | 4 +- .../TestIcebergLocalConcurrentWrites.java | 15 +- .../TestIcebergMinioOrcConnectorTest.java | 4 +- .../TestIcebergParquetConnectorTest.java | 9 +- .../trino/plugin/iceberg/TestIcebergV2.java | 16 +- ...tIcebergGlueCatalogConnectorSmokeTest.java | 2 +- .../TestIcebergGlueCatalogSkipArchive.java | 4 +- ...estIcebergNessieCatalogWithBearerAuth.java | 2 +- .../ignite/TestIgniteConnectorTest.java | 17 +- .../plugin/kudu/TestKuduConnectorTest.java | 19 +- .../mariadb/BaseMariaDbConnectorTest.java | 6 +- .../mariadb/TestMariaDbTypeMapping.java | 4 +- .../memory/TestMemoryConnectorTest.java | 2 +- .../mongodb/BaseMongoConnectorSmokeTest.java | 12 +- .../mongodb/TestMongoConnectorTest.java | 16 +- .../plugin/mongodb/TestMongoTypeMapping.java | 2 +- .../plugin/mysql/BaseMySqlConnectorTest.java | 2 +- .../oracle/AbstractTestOracleTypeMapping.java | 8 +- .../oracle/BaseOracleConnectorTest.java | 10 +- .../phoenix5/TestPhoenixConnectorTest.java | 8 +- .../phoenix5/TestPhoenixTypeMapping.java | 2 +- .../TestPostgreSqlConnectorTest.java | 36 +-- .../redshift/TestRedshiftCastPushdown.java | 3 +- .../redshift/TestRedshiftConnectorTest.java | 16 +- .../TestSingleStoreConnectorTest.java | 6 +- .../TestSingleStoreLatestTypeMapping.java | 8 +- .../sqlserver/BaseSqlServerConnectorTest.java | 4 +- .../vertica/TestVerticaConnectorTest.java | 3 +- ...erantExecutionCoordinatorExcludedTest.java | 2 +- .../testing/AbstractTestQueryFramework.java | 12 + .../trino/testing/BaseConnectorSmokeTest.java | 22 +- .../io/trino/testing/BaseConnectorTest.java | 240 ++++++++--------- 54 files changed, 470 insertions(+), 628 deletions(-) diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java index 59e9c9d75f5e..1f0726e30d0d 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java @@ -337,8 +337,7 @@ public void testCaseSensitiveAggregationPushdown() PlanMatchPattern aggregationOverTableScan = node(AggregationNode.class, node(TableScanNode.class)); PlanMatchPattern groupingAggregationOverTableScan = node(AggregationNode.class, node(TableScanNode.class)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_cs_agg_pushdown", "(a_string varchar(1), a_char char(1), a_bigint bigint)", ImmutableList.of( @@ -666,7 +665,7 @@ public void testCountDistinctWithStringTypes() .setSystemProperty(DISTINCT_AGGREGATIONS_STRATEGY, "pre_aggregate") .build(); - try (TestTable testTable = new TestTable(getQueryRunner()::execute, "distinct_strings", "(t_char CHAR(5), t_varchar VARCHAR(5))", rows)) { + try (TestTable testTable = newTrinoTable("distinct_strings", "(t_char CHAR(5), t_varchar VARCHAR(5))", rows)) { if (!(hasBehavior(SUPPORTS_AGGREGATION_PUSHDOWN) && hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY))) { // disabling hash generation to prevent extra projections in the plan which make it hard to write matchers for isNotFullyPushedDown Session optimizeHashGenerationDisabled = Session.builder(getSession()) @@ -1121,8 +1120,7 @@ public void testNullSensitiveTopNPushdown() return; } - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_null_sensitive_topn_pushdown", "(name varchar(10), a bigint)", List.of( @@ -1186,8 +1184,7 @@ public void testCaseSensitiveTopNPushdown() boolean expectTopNPushdown = hasBehavior(SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR); PlanMatchPattern topNOverTableScan = project(node(TopNNode.class, anyTree(node(TableScanNode.class)))); - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_case_sensitive_topn_pushdown", "(a_string varchar(10), a_char char(10), a_bigint bigint)", List.of( @@ -1278,9 +1275,8 @@ public void testJoinPushdown() return; } - try (TestTable nationLowercaseTable = new TestTable( + try (TestTable nationLowercaseTable = newTrinoTable( // If a connector supports Join pushdown, but does not allow CTAS, we need to make the table creation here overridable. - getQueryRunner()::execute, "nation_lowercase", "AS SELECT nationkey, lower(name) name, regionkey FROM nation")) { for (JoinOperator joinOperator : JoinOperator.values()) { @@ -1547,12 +1543,10 @@ public void testBulkColumnListingOptions() String schemaName = "test_columns_listing_" + randomNameSuffix(); assertUpdate("CREATE SCHEMA " + schemaName); try { - try (TestTable newNation = new TestTable( - getQueryRunner()::execute, + try (TestTable newNation = newTrinoTable( schemaName + ".nation", "(name varchar(25), nationkey bigint)"); - TestTable newRegion = new TestTable( - getQueryRunner()::execute, + TestTable newRegion = newTrinoTable( schemaName + ".region", "(name varchar(25), regionkey bigint)")) { if (hasBehavior(SUPPORTS_COMMENT_ON_TABLE)) { @@ -1755,7 +1749,7 @@ public void testUpdateNotNullColumn() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "update_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { + try (TestTable table = newTrinoTable("update_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { assertUpdate(format("INSERT INTO %s (nullable_col, not_null_col) VALUES (1, 10)", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (1, 10)"); assertQueryFails("UPDATE " + table.getName() + " SET not_null_col = NULL WHERE nullable_col = 1", MODIFYING_ROWS_MESSAGE); @@ -1774,7 +1768,7 @@ public void testUpdateRowType() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_UPDATE) && hasBehavior(SUPPORTS_ROW_TYPE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_update_with_predicates_on_row_types", "(int_t INT, row_t ROW(f1 INT, f2 INT))")) { + try (TestTable table = newTrinoTable("test_update_with_predicates_on_row_types", "(int_t INT, row_t ROW(f1 INT, f2 INT))")) { String tableName = table.getName(); assertUpdate("INSERT INTO " + tableName + " VALUES (1, ROW(2, 3)), (11, ROW(12, 13)), (21, ROW(22, 23))", 3); assertQueryFails("UPDATE " + tableName + " SET int_t = int_t - 1 WHERE row_t.f2 = 3", MODIFYING_ROWS_MESSAGE); @@ -1793,7 +1787,7 @@ public void testUpdateRowConcurrently() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_UPDATE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_update_row", "(a INT, b INT, c INT)", ImmutableList.of("1, 2, 3"))) { + try (TestTable table = newTrinoTable("test_update_row", "(a INT, b INT, c INT)", ImmutableList.of("1, 2, 3"))) { assertQueryFails("UPDATE " + table.getName() + " SET a = a + 1", MODIFYING_ROWS_MESSAGE); } } @@ -1809,7 +1803,7 @@ public void testUpdateAllValues() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_UPDATE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_update_all", "(a INT, b INT, c INT)", ImmutableList.of("1, 2, 3"))) { + try (TestTable table = newTrinoTable("test_update_all", "(a INT, b INT, c INT)", ImmutableList.of("1, 2, 3"))) { assertUpdate("UPDATE " + table.getName() + " SET a = 1, b = 1, c = 2", 1); } } @@ -1826,7 +1820,7 @@ public void testUpdateWithPredicates() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_UPDATE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_predicates", "(a INT, b INT, c INT)")) { + try (TestTable table = newTrinoTable("test_row_predicates", "(a INT, b INT, c INT)")) { String tableName = table.getName(); assertUpdate("INSERT INTO " + tableName + " VALUES (1, 2, 3), (11, 12, 13), (21, 22, 23)", 3); assertUpdate("UPDATE " + tableName + " SET a = 5 WHERE c = 3", 1); @@ -1861,7 +1855,7 @@ public void testUpdateWithPredicates() public void testConstantUpdateWithVarcharEqualityPredicates() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_UPDATE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_update_varchar", "(col1 INT, col2 varchar(1))", ImmutableList.of("1, 'a'", "2, 'A'"))) { + try (TestTable table = newTrinoTable("test_update_varchar", "(col1 INT, col2 varchar(1))", ImmutableList.of("1, 'a'", "2, 'A'"))) { if (!hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_EQUALITY)) { assertQueryFails("UPDATE " + table.getName() + " SET col1 = 20 WHERE col2 = 'A'", MODIFYING_ROWS_MESSAGE); return; @@ -1910,7 +1904,7 @@ public void testDeleteWithBigintEqualityPredicate() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_bigint", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_delete_bigint", "AS SELECT * FROM region")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE regionkey = 1", 1); assertQuery( "SELECT regionkey, name FROM " + table.getName(), @@ -1927,7 +1921,7 @@ public void testDeleteWithVarcharEqualityPredicate() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_varchar", "(col varchar(1))", ImmutableList.of("'a'", "'A'", "null"))) { + try (TestTable table = newTrinoTable("test_delete_varchar", "(col varchar(1))", ImmutableList.of("'a'", "'A'", "null"))) { if (!hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_EQUALITY)) { assertQueryFails("DELETE FROM " + table.getName() + " WHERE col = 'A'", MODIFYING_ROWS_MESSAGE); return; @@ -2042,8 +2036,7 @@ public void testInsertWithoutTemporaryTable() .setCatalogSessionProperty(getSession().getCatalog().orElseThrow(), "non_transactional_insert", "false") .build(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_bypass_temp", "(a varchar(36), b bigint)")) { int numberOfRows = 50; @@ -2072,8 +2065,7 @@ private void testWriteBatchSizeSessionProperty(int batchSize, int numberOfRows) .setCatalogSessionProperty(getSession().getCatalog().orElseThrow(), "write_batch_size", Integer.toString(batchSize)) .build(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "write_batch_size", "(a varchar(36), b bigint)")) { String values = String.join(",", buildRowsForInsert(numberOfRows)); @@ -2103,8 +2095,7 @@ private void testWriteTaskParallelismSessionProperty(int parallelism, int number .build(); QueryRunner queryRunner = getQueryRunner(); - try (TestTable table = new TestTable( - queryRunner::execute, + try (TestTable table = newTrinoTable( "write_parallelism", "(a varchar(128), b bigint)")) { Plan plan = newTransaction() @@ -2274,8 +2265,8 @@ public void testJoinPushdownWithLongIdentifiers() .orElse(65536 + 5); String validColumnName = baseColumnName + "z".repeat(maxLength - baseColumnName.length()); - try (TestTable left = new TestTable(getQueryRunner()::execute, "test_long_id_l", format("(%s BIGINT)", validColumnName)); - TestTable right = new TestTable(getQueryRunner()::execute, "test_long_id_r", format("(%s BIGINT)", validColumnName))) { + try (TestTable left = newTrinoTable("test_long_id_l", format("(%s BIGINT)", validColumnName)); + TestTable right = newTrinoTable("test_long_id_r", format("(%s BIGINT)", validColumnName))) { assertThat(query(joinPushdownEnabled(getSession()), """ SELECT l.%1$s, r.%1$s @@ -2403,8 +2394,7 @@ public void testDynamicFilteringCaseInsensitiveDomainCompaction() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); skipTestUnless(hasBehavior(SUPPORTS_DYNAMIC_FILTER_PUSHDOWN)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_caseinsensitive", "(id varchar(1))", ImmutableList.of("'0'", "'a'", "'B'"))) { diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcTableStatisticsTest.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcTableStatisticsTest.java index 243038d66d3e..8cfd98769680 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcTableStatisticsTest.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcTableStatisticsTest.java @@ -171,8 +171,7 @@ public void testStatsWithVarcharPredicatePushdown() "('comment', 1e0, 0e0, null)," + "(null, null, null, 1e0)"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "varchar_duplicates", // each letter A-E repeated 5 times " AS SELECT nationkey, chr(codepoint('A') + nationkey / 5) fl FROM tpch.tiny.nation")) { diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java index 48646ef1c81f..f81bbac4a50a 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java @@ -205,7 +205,7 @@ public void testCreateTableWithRowTypeWithoutField() @Test public void testCreateTableAlreadyExists() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_table_already_exists", "(col1 int)")) { + try (TestTable table = newTrinoTable("test_create_table_already_exists", "(col1 int)")) { assertQueryFails( "CREATE TABLE " + table.getName() + "(col1 int)", "\\Qline 1:1: Table 'bigquery.tpch." + table.getName() + "' already exists\\E"); @@ -387,7 +387,7 @@ public void testStreamCommentTableSpecialCharacter() @Override // Override because the base test exceeds rate limits per a table public void testCommentColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_comment_column_", "(a integer)")) { + try (TestTable table = newTrinoTable("test_comment_column_", "(a integer)")) { // comment set assertUpdate("COMMENT ON COLUMN " + table.getName() + ".a IS 'new comment'"); assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())).contains("COMMENT 'new comment'"); @@ -398,7 +398,7 @@ public void testCommentColumn() assertThat(getColumnComment(table.getName(), "a")).isEqualTo(null); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_comment_column_", "(a integer COMMENT 'test comment')")) { + try (TestTable table = newTrinoTable("test_comment_column_", "(a integer COMMENT 'test comment')")) { assertThat(getColumnComment(table.getName(), "a")).isEqualTo("test comment"); // comment set new value assertUpdate("COMMENT ON COLUMN " + table.getName() + ".a IS 'updated comment'"); @@ -550,7 +550,7 @@ public void testPredicatePushdownPrunnedColumns() @Test public void testColumnPositionMismatch() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test.test_column_position_mismatch", "(c_varchar VARCHAR, c_int INT, c_date DATE)")) { + try (TestTable table = newTrinoTable("test.test_column_position_mismatch", "(c_varchar VARCHAR, c_int INT, c_date DATE)")) { onBigQuery("INSERT INTO " + table.getName() + " VALUES ('a', 1, '2021-01-01')"); // Adding a CAST makes BigQuery return columns in a different order assertQuery("SELECT c_varchar, CAST(c_int AS SMALLINT), c_date FROM " + table.getName(), "VALUES ('a', 1, '2021-01-01')"); @@ -1458,7 +1458,7 @@ private void assertLimitPushdownReadsLessData(Session session, String tableName) public void testInsertArray() { // Override because the connector disallows writing a NULL in ARRAY - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_array_", "(a ARRAY, b ARRAY)")) { + try (TestTable table = newTrinoTable("test_insert_array_", "(a ARRAY, b ARRAY)")) { assertUpdate("INSERT INTO " + table.getName() + " (a, b) VALUES (ARRAY[1.23E1], ARRAY[1.23E1])", 1); assertQuery("SELECT a[1], b[1] FROM " + table.getName(), "VALUES (12.3, 12)"); } diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryTypeMapping.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryTypeMapping.java index 9724da1b9b55..d23d22a4d099 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryTypeMapping.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryTypeMapping.java @@ -738,7 +738,7 @@ public void testArray() @Test public void testArrayType() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_array_", "(a BIGINT, b ARRAY, c ARRAY)")) { + try (TestTable table = newTrinoTable("test_array_", "(a BIGINT, b ARRAY, c ARRAY)")) { assertUpdate("INSERT INTO " + table.getName() + " (a, b, c) VALUES (5, ARRAY[1.23E1], ARRAY[15]), (6, ARRAY[1.24E1, 1.27E1, 2.23E1], ARRAY[25, 26, 36])", 2); assertThat(query("SELECT * FROM " + table.getName())) .matches("VALUES " + @@ -752,7 +752,7 @@ public void testUnsupportedNullArray() { // BigQuery translates a NULL ARRAY into an empty ARRAY in the query result // This test ensures that the connector disallows writing a NULL ARRAY - try (TestTable table = new TestTable(getQueryRunner()::execute, "test.test_null_array", "(col ARRAY(INT))")) { + try (TestTable table = newTrinoTable("test.test_null_array", "(col ARRAY(INT))")) { assertQueryFails("INSERT INTO " + table.getName() + " VALUES (NULL)", "NULL value not allowed for NOT NULL column: col"); } } diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryWithProxyTest.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryWithProxyTest.java index a875ff62f53d..cd98f539c8c2 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryWithProxyTest.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryWithProxyTest.java @@ -48,7 +48,7 @@ protected QueryRunner createQueryRunner() void testCreateTableAsSelect() { // This test covers all client (BigQuery, BigQueryReadClient and BigQueryWriteClient) - try (TestTable table = new TestTable(getQueryRunner()::execute, "test.test_ctas", "AS SELECT 42 x")) { + try (TestTable table = newTrinoTable("test.test_ctas", "AS SELECT 42 x")) { assertQuery("SELECT * FROM " + table.getName(), "VALUES 42"); } } diff --git a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/BaseCassandraConnectorSmokeTest.java b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/BaseCassandraConnectorSmokeTest.java index e4dbdc5fc0b4..a9d4ce254ee7 100644 --- a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/BaseCassandraConnectorSmokeTest.java +++ b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/BaseCassandraConnectorSmokeTest.java @@ -76,7 +76,7 @@ public void testRowLevelDelete() @Test public void testInsertDate() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_", "(a_date date)")) { + try (TestTable table = newTrinoTable("test_insert_", "(a_date date)")) { assertUpdate("INSERT INTO " + table.getName() + " (a_date) VALUES ( DATE '2020-05-11')", 1); assertThat(query("SELECT a_date FROM " + table.getName())).matches("VALUES (DATE '2020-05-11')"); } diff --git a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraProtocolVersionV3ConnectorSmokeTest.java b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraProtocolVersionV3ConnectorSmokeTest.java index 428291628ea1..7ba3a1e68795 100644 --- a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraProtocolVersionV3ConnectorSmokeTest.java +++ b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraProtocolVersionV3ConnectorSmokeTest.java @@ -43,7 +43,7 @@ protected QueryRunner createQueryRunner() @Override public void testInsertDate() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_", "(a_date date)")) { + try (TestTable table = newTrinoTable("test_insert_", "(a_date date)")) { assertUpdate("INSERT INTO " + table.getName() + " (a_date) VALUES ('2020-05-11')", 1); assertThat(query("SELECT a_date FROM " + table.getName())).matches("VALUES (CAST('2020-05-11' AS varchar))"); } diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java index 2444ea4bf6c6..90f9744c288d 100644 --- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java +++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseTypeMapping.java @@ -889,7 +889,7 @@ private void testUnsupportedDate(String unsupportedDate) String minSupportedDate = "1970-01-01"; String maxSupportedDate = "2149-06-06"; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_unsupported_date", "(dt date)")) { + try (TestTable table = newTrinoTable("test_unsupported_date", "(dt date)")) { assertQueryFails( format("INSERT INTO %s VALUES (DATE '%s')", table.getName(), unsupportedDate), format("Date must be between %s and %s in ClickHouse: %s", minSupportedDate, maxSupportedDate, unsupportedDate)); @@ -984,7 +984,7 @@ public void testUnsupportedTimestamp(String unsupportedTimestamp) String minSupportedTimestamp = "1970-01-01 00:00:00"; String maxSupportedTimestamp = "2106-02-07 06:28:15"; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_unsupported_timestamp", "(dt timestamp(0))")) { + try (TestTable table = newTrinoTable("test_unsupported_timestamp", "(dt timestamp(0))")) { assertQueryFails( format("INSERT INTO %s VALUES (TIMESTAMP '%s')", table.getName(), unsupportedTimestamp), format("Timestamp must be between %s and %s in ClickHouse: %s", minSupportedTimestamp, maxSupportedTimestamp, unsupportedTimestamp)); diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java index 6f5e0093c65b..4069feced7f8 100644 --- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java +++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java @@ -126,8 +126,7 @@ public void testRenameColumn() @Override public void testRenameColumnWithComment() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_rename_column_", "(id INT NOT NULL, col INT COMMENT 'test column comment') WITH (engine = 'MergeTree', order_by = ARRAY['id'])")) { assertThat(getColumnComment(table.getName(), "col")).isEqualTo("test column comment"); @@ -141,7 +140,7 @@ public void testRenameColumnWithComment() public void testAddColumnWithCommentSpecialCharacter(String comment) { // Override because default storage engine doesn't support renaming columns - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_column_", "(a_varchar varchar NOT NULL) WITH (engine = 'mergetree', order_by = ARRAY['a_varchar'])")) { + try (TestTable table = newTrinoTable("test_add_column_", "(a_varchar varchar NOT NULL) WITH (engine = 'mergetree', order_by = ARRAY['a_varchar'])")) { assertUpdate("ALTER TABLE " + table.getName() + " ADD COLUMN b_varchar varchar COMMENT " + varcharLiteral(comment)); assertThat(getColumnComment(table.getName(), "b_varchar")).isEqualTo(comment); } @@ -151,7 +150,7 @@ public void testAddColumnWithCommentSpecialCharacter(String comment) @Override public void testDropAndAddColumnWithSameName() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_add_column", "(x int NOT NULL, y int, z int) WITH (engine = 'MergeTree', order_by = ARRAY['x'])", ImmutableList.of("1,2,3"))) { + try (TestTable table = newTrinoTable("test_drop_add_column", "(x int NOT NULL, y int, z int) WITH (engine = 'MergeTree', order_by = ARRAY['x'])", ImmutableList.of("1,2,3"))) { assertUpdate("ALTER TABLE " + table.getName() + " DROP COLUMN y"); assertQuery("SELECT * FROM " + table.getName(), "VALUES (1, 3)"); @@ -210,7 +209,7 @@ public void testDropColumn() @Override protected TestTable createTableWithOneIntegerColumn(String namePrefix) { - return new TestTable(getQueryRunner()::execute, namePrefix, "(col integer NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['col'])"); + return newTrinoTable(namePrefix, "(col integer NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['col'])"); } @Override @@ -223,7 +222,7 @@ protected String tableDefinitionForAddColumn() @Override // Overridden because the default storage type doesn't support adding columns public void testAddNotNullColumnToEmptyTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_notnull_col_to_empty", "(a_varchar varchar NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['a_varchar'])")) { + try (TestTable table = newTrinoTable("test_add_notnull_col_to_empty", "(a_varchar varchar NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['a_varchar'])")) { String tableName = table.getName(); assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN b_varchar varchar NOT NULL"); @@ -239,7 +238,7 @@ public void testAddNotNullColumnToEmptyTable() @Override // Overridden because (a) the default storage type doesn't support adding columns and (b) ClickHouse has implicit default value for new NON NULL column public void testAddNotNullColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_notnull_col", "(a_varchar varchar NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['a_varchar'])")) { + try (TestTable table = newTrinoTable("test_add_notnull_col", "(a_varchar varchar NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['a_varchar'])")) { String tableName = table.getName(); assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN b_varchar varchar NOT NULL"); @@ -259,7 +258,7 @@ public void testAddNotNullColumn() public void testAddColumnWithComment() { // Override because the default storage type doesn't support adding columns - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_col_desc_", "(a_varchar varchar NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['a_varchar'])")) { + try (TestTable table = newTrinoTable("test_add_col_desc_", "(a_varchar varchar NOT NULL) WITH (engine = 'MergeTree', order_by = ARRAY['a_varchar'])")) { String tableName = table.getName(); assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN b_varchar varchar COMMENT 'test new column comment'"); @@ -532,8 +531,7 @@ public void testTableProperty() public void testSetTableProperties() throws Exception { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_alter_table_properties", "(p1 int NOT NULL, p2 boolean NOT NULL, x VARCHAR) WITH (engine = 'MergeTree', order_by = ARRAY['p1', 'p2'], primary_key = ARRAY['p1', 'p2'])")) { assertThat(getTableProperties("tpch", table.getName())) @@ -558,8 +556,7 @@ public void testSetTableProperties() @Test public void testAlterInvalidTableProperties() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_alter_table_properties", "(p1 int NOT NULL, p2 int NOT NULL, x VARCHAR) WITH (engine = 'MergeTree', order_by = ARRAY['p1', 'p2'], primary_key = ARRAY['p1', 'p2'])")) { assertQueryFails( @@ -644,7 +641,7 @@ protected TestTable createTableWithDoubleAndRealColumns(String name, List> futures = new ArrayList<>(); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace", "(col integer)")) { + try (TestTable table = newTrinoTable("test_create_or_replace", "(col integer)")) { String tableName = table.getName(); getQueryRunner().execute("CREATE OR REPLACE TABLE " + tableName + " AS SELECT 1 a"); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java index ac3f29b7413e..b17d8344812b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAnalyze.java @@ -451,8 +451,7 @@ public void testAnalyzeSomeColumns() @Test public void testDropExtendedStats() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_extended_stats", "AS SELECT * FROM tpch.sf1.nation")) { String query = "SHOW STATS FOR " + table.getName(); @@ -485,8 +484,7 @@ public void testDropExtendedStats() @Test public void testDropMissingStats() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_missing_stats", "AS SELECT * FROM tpch.sf1.nation")) { // When there are no extended stats, the procedure should have no effect @@ -506,8 +504,7 @@ public void testDropMissingStats() @Test public void testDropStatsAccessControl() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_deny_drop_stats", "AS SELECT * FROM tpch.sf1.nation")) { assertAccessDenied( @@ -524,8 +521,7 @@ public void testDropStatsAccessControl() @Test public void testStatsOnTpcDsData() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_old_date_stats", "AS SELECT d_date FROM tpcds.tiny.date_dim")) { assertUpdate("ANALYZE " + table.getName()); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java index f02b6c6e10e2..617514e09f41 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java @@ -347,8 +347,7 @@ private void testPartitionValuesParsedCheckpoint(ColumnMappingMode columnMapping checkArgument(inputValues.size() == 2, "inputValues size must be 2"); checkArgument(expectedPartitionValuesParsed.size() == 2, "expectedPartitionValuesParsed size must be 2"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_values_parsed_checkpoint", "(x int, part " + inputType + ") WITH (checkpoint_interval = 2, column_mapping_mode = '" + columnMappingMode + "', partitioned_by = ARRAY['part'])")) { for (String inputValue : inputValues) { @@ -1199,7 +1198,7 @@ public void testDeletionVectorsEnabledCreateTable() private void testDeletionVectorsEnabledCreateTable(String tableDefinition) throws Exception { - try (TestTable table = new TestTable(getQueryRunner()::execute, "deletion_vectors", tableDefinition)) { + try (TestTable table = newTrinoTable("deletion_vectors", tableDefinition)) { assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())) .contains("deletion_vectors_enabled = true"); @@ -1232,7 +1231,7 @@ public void testDeletionVectorsDisabledCreateTable() private void testDeletionVectorsDisabledCreateTable(String tableDefinition) throws Exception { - try (TestTable table = new TestTable(getQueryRunner()::execute, "deletion_vectors", tableDefinition)) { + try (TestTable table = newTrinoTable("deletion_vectors", tableDefinition)) { assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())) .doesNotContain("deletion_vectors_enabled"); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java index 943caec8b254..11f1544e92bc 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java @@ -344,8 +344,7 @@ public void testPartialFilterWhenPartitionColumnOrderIsDifferentFromTableDefinit private void testPartialFilterWhenPartitionColumnOrderIsDifferentFromTableDefinition(ColumnMappingMode columnMappingMode) { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_delete_with_partial_filter_composed_partition", "(_bigint BIGINT, _date DATE, _varchar VARCHAR) WITH (column_mapping_mode='" + columnMappingMode + "', partitioned_by = ARRAY['_varchar', '_date'])")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1, CAST('2019-09-10' AS DATE), 'a'), (2, CAST('2019-09-10' AS DATE), 'a')", 2); @@ -403,8 +402,7 @@ public void testInsertIntoUnsupportedVarbinaryPartitionType() { // TODO https://github.com/trinodb/trino/issues/24155 Cannot insert varbinary values into partitioned columns // Update TestDeltaLakeBasic.testPartitionValuesParsedCheckpoint() when fixing this issue - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_varbinary_partition", "(x int, part varbinary) WITH (partitioned_by = ARRAY['part'])")) { assertQueryFails("INSERT INTO " + table.getName() + " VALUES (1, X'01')", "Unsupported type for partition: varbinary"); @@ -588,8 +586,7 @@ public void testRenameColumnName() public void testCharVarcharComparison() { // with char->varchar coercion on table creation, this is essentially varchar/varchar comparison - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_char_varchar", "(k, v) AS VALUES" + " (-1, CAST(NULL AS CHAR(3))), " + @@ -818,7 +815,7 @@ public void testTimestampWithTimeZoneOptimization() @Test public void testShowStatsForTimestampWithTimeZone() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_stats_timestamptz_", "(x TIMESTAMP(3) WITH TIME ZONE) WITH (checkpoint_interval = 2)")) { + try (TestTable table = newTrinoTable("test_stats_timestamptz_", "(x TIMESTAMP(3) WITH TIME ZONE) WITH (checkpoint_interval = 2)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (TIMESTAMP '+10000-01-02 13:34:56.123 +01:00')", 1); assertThat(query("SHOW STATS FOR " + table.getName())) .result() @@ -843,7 +840,7 @@ public void testShowStatsForTimestampWithTimeZone() @Test public void testAddColumnToPartitionedTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_column_partitioned_table_", "(x VARCHAR, part VARCHAR) WITH (partitioned_by = ARRAY['part'])")) { + try (TestTable table = newTrinoTable("test_add_column_partitioned_table_", "(x VARCHAR, part VARCHAR) WITH (partitioned_by = ARRAY['part'])")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT 'first', 'part-0001'", 1); assertQueryFails("ALTER TABLE " + table.getName() + " ADD COLUMN x bigint", ".* Column 'x' already exists"); assertQueryFails("ALTER TABLE " + table.getName() + " ADD COLUMN part bigint", ".* Column 'part' already exists"); @@ -877,7 +874,7 @@ private QueryInfo getQueryInfo(QueryRunner queryRunner, MaterializedResultWithPl @Test public void testAddColumnAndOptimize() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_column_and_optimize", "(x VARCHAR)")) { + try (TestTable table = newTrinoTable("test_add_column_and_optimize", "(x VARCHAR)")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT 'first'", 1); assertUpdate("ALTER TABLE " + table.getName() + " ADD COLUMN a varchar(50)"); @@ -905,7 +902,7 @@ public void testAddColumnAndVacuum() .setCatalogSessionProperty(getSession().getCatalog().orElseThrow(), "vacuum_min_retention", "0s") .build(); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_column_and_optimize", "(x VARCHAR)")) { + try (TestTable table = newTrinoTable("test_add_column_and_optimize", "(x VARCHAR)")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT 'first'", 1); assertUpdate("INSERT INTO " + table.getName() + " SELECT 'second'", 1); @@ -1009,7 +1006,7 @@ public void testTargetMaxFileSize() @Test public void testPathColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_path_column", "(x VARCHAR)")) { + try (TestTable table = newTrinoTable("test_path_column", "(x VARCHAR)")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT 'first'", 1); String firstFilePath = (String) computeScalar("SELECT \"$path\" FROM " + table.getName()); assertUpdate("INSERT INTO " + table.getName() + " SELECT 'second'", 1); @@ -1331,11 +1328,11 @@ private void testTableWithNonNullableColumns(ColumnMappingMode mode) public void testCreateTableWithChangeDataFeedColumnName() { for (String columnName : CHANGE_DATA_FEED_COLUMN_NAMES) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_table_cdf", "(" + columnName + " int)")) { + try (TestTable table = newTrinoTable("test_create_table_cdf", "(" + columnName + " int)")) { assertTableColumnNames(table.getName(), columnName); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_table_cdf", "AS SELECT 1 AS " + columnName)) { + try (TestTable table = newTrinoTable("test_create_table_cdf", "AS SELECT 1 AS " + columnName)) { assertTableColumnNames(table.getName(), columnName); } } @@ -1344,7 +1341,7 @@ public void testCreateTableWithChangeDataFeedColumnName() @Test public void testCreateTableWithChangeDataFeed() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cdf", "(x int) WITH (change_data_feed_enabled = true)")) { + try (TestTable table = newTrinoTable("test_cdf", "(x int) WITH (change_data_feed_enabled = true)")) { assertThat(query("SELECT * FROM \"" + table.getName() + "$properties\"")) .skippingTypesCheck() .matches("VALUES " + @@ -1355,7 +1352,7 @@ public void testCreateTableWithChangeDataFeed() } // timestamp type requires reader version 3 and writer version 7 - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cdf", "(x timestamp) WITH (change_data_feed_enabled = true)")) { + try (TestTable table = newTrinoTable("test_cdf", "(x timestamp) WITH (change_data_feed_enabled = true)")) { assertThat(query("SELECT * FROM \"" + table.getName() + "$properties\"")) .skippingTypesCheck() .matches("VALUES " + @@ -1371,8 +1368,7 @@ public void testCreateTableWithChangeDataFeed() @Test public void testChangeDataFeedWithDeletionVectors() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_cdf", "(x VARCHAR, y INT) WITH (change_data_feed_enabled = true, deletion_vectors_enabled = true)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES('test1', 1)", 1); @@ -1415,7 +1411,7 @@ public void testUnsupportedCreateTableWithChangeDataFeed() public void testUnsupportedAddColumnWithChangeDataFeed() { for (String columnName : CHANGE_DATA_FEED_COLUMN_NAMES) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_column", "(col int) WITH (change_data_feed_enabled = true)")) { + try (TestTable table = newTrinoTable("test_add_column", "(col int) WITH (change_data_feed_enabled = true)")) { assertQueryFails( "ALTER TABLE " + table.getName() + " ADD COLUMN " + columnName + " int", "\\QColumn name %s is forbidden when change data feed is enabled\\E".formatted(columnName)); @@ -1432,7 +1428,7 @@ public void testUnsupportedAddColumnWithChangeDataFeed() public void testUnsupportedRenameColumnWithChangeDataFeed() { for (String columnName : CHANGE_DATA_FEED_COLUMN_NAMES) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_rename_column", "(col int) WITH (change_data_feed_enabled = true)")) { + try (TestTable table = newTrinoTable("test_rename_column", "(col int) WITH (change_data_feed_enabled = true)")) { assertQueryFails( "ALTER TABLE " + table.getName() + " RENAME COLUMN col TO " + columnName, "Cannot rename column when change data feed is enabled"); @@ -1445,7 +1441,7 @@ public void testUnsupportedRenameColumnWithChangeDataFeed() public void testUnsupportedSetTablePropertyWithChangeDataFeed() { for (String columnName : CHANGE_DATA_FEED_COLUMN_NAMES) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_properties", "(" + columnName + " int)")) { + try (TestTable table = newTrinoTable("test_set_properties", "(" + columnName + " int)")) { assertQueryFails( "ALTER TABLE " + table.getName() + " SET PROPERTIES change_data_feed_enabled = true", "\\QUnable to enable change data feed because table contains [%s] columns\\E".formatted(columnName)); @@ -1534,7 +1530,7 @@ private void testCreateTableColumnMappingMode(ColumnMappingMode mode, Consumer catalogProperties = getSession().getCatalogProperties(getSession().getCatalog().orElseThrow()); assertThat(catalogProperties).doesNotContainKey("query_partition_filter_required"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_not_demanded", "(x varchar, part varchar) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("'a', 'part_a'", "'b', 'part_b'"))) { @@ -3717,8 +3697,7 @@ public void testPartitionFilterQueryNotDemanded() public void testQueryWithoutPartitionOnNonPartitionedTableNotDemanded() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_no_partition_table_", "(x varchar, part varchar)", ImmutableList.of("('a', 'part_a')", "('b', 'part_b')"))) { @@ -3731,8 +3710,7 @@ public void testQueryWithoutPartitionOnNonPartitionedTableNotDemanded() public void testQueryWithoutPartitionFilterNotAllowed() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_no_partition_filter_", "(x varchar, part varchar) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("('a', 'part_a')", "('b', 'part_b')"))) { @@ -3747,8 +3725,7 @@ public void testQueryWithoutPartitionFilterNotAllowed() public void testPartitionFilterRemovedByPlanner() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_removed_", "(x varchar, part varchar) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("('a', 'part_a')", "('b', 'part_b')"))) { @@ -3763,8 +3740,7 @@ public void testPartitionFilterRemovedByPlanner() public void testPartitionFilterIncluded() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_included", "(x varchar, part integer) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("('a', 1)", "('a', 2)", "('a', 3)", "('a', 4)", "('b', 1)", "('b', 2)", "('b', 3)", "('b', 4)"))) { @@ -3792,8 +3768,7 @@ public void testRequiredPartitionFilterOnJoin() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable leftTable = new TestTable( - getQueryRunner()::execute, + try (TestTable leftTable = newTrinoTable( "test_partition_left_", "(x varchar, part varchar)", ImmutableList.of("('a', 'part_a')")); @@ -3818,8 +3793,7 @@ public void testRequiredPartitionFilterOnJoinBothTablePartitioned() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable leftTable = new TestTable( - getQueryRunner()::execute, + try (TestTable leftTable = newTrinoTable( "test_partition_inferred_left_", "(x varchar, part varchar) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("('a', 'part_a')")); @@ -3843,8 +3817,7 @@ public void testRequiredPartitionFilterOnJoinBothTablePartitioned() public void testComplexPartitionPredicateWithCasting() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_predicate", "(x varchar, part varchar) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("('a', '1')", "('b', '2')"))) { @@ -3856,8 +3829,7 @@ public void testComplexPartitionPredicateWithCasting() public void testPartitionPredicateInOuterQuery() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_predicate", "(x integer, part integer) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("(1, 11)", "(2, 22)"))) { @@ -3869,8 +3841,7 @@ public void testPartitionPredicateInOuterQuery() public void testPartitionPredicateInInnerQuery() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_predicate", "(x integer, part integer) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("(1, 11)", "(2, 22)"))) { @@ -3882,8 +3853,7 @@ public void testPartitionPredicateInInnerQuery() public void testPartitionPredicateFilterAndAnalyzeOnPartitionedTable() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_predicate_analyze_", "(x integer, part integer) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("(1, 11)", "(2, 22)"))) { @@ -3898,8 +3868,7 @@ public void testPartitionPredicateFilterAndAnalyzeOnPartitionedTable() public void testPartitionPredicateFilterAndAnalyzeOnNonPartitionedTable() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable nonPartitioned = new TestTable( - getQueryRunner()::execute, + try (TestTable nonPartitioned = newTrinoTable( "test_partition_predicate_analyze_nonpartitioned", "(a integer, b integer) ", ImmutableList.of("(1, 11)", "(2, 22)"))) { @@ -3912,8 +3881,7 @@ public void testPartitionPredicateFilterAndAnalyzeOnNonPartitionedTable() public void testPartitionFilterMultiplePartition() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_multiple_partition_", "(x varchar, part1 integer, part2 integer) WITH (partitioned_by = ARRAY['part1', 'part2'])", ImmutableList.of("('a', 1, 1)", "('a', 1, 2)", "('a', 2, 1)", "('a', 2, 2)", "('b', 1, 1)", "('b', 1, 2)", "('b', 2, 1)", "('b', 2, 2)"))) { @@ -3942,8 +3910,7 @@ public void testPartitionFilterMultiplePartition() public void testPartitionFilterRequiredAndOptimize() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_optimize", "(part integer, name varchar(50)) WITH (partitioned_by = ARRAY['part'])", ImmutableList.of("(1, 'Bob')", "(2, 'Alice')"))) { @@ -3976,8 +3943,7 @@ public void testPartitionFilterRequiredAndOptimize() public void testPartitionFilterEnabledAndOptimizeForNonPartitionedTable() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_nonpartitioned_optimize", "(part integer, name varchar(50))", ImmutableList.of("(1, 'Bob')", "(2, 'Alice')"))) { @@ -4002,8 +3968,7 @@ public void testPartitionFilterEnabledAndOptimizeForNonPartitionedTable() public void testPartitionFilterRequiredAndWriteOperation() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_table_changes", "(x integer, part integer) WITH (partitioned_by = ARRAY['part'], change_data_feed_enabled = true)", ImmutableList.of("(1, 11)", "(2, 22)", "(3, 33)"))) { @@ -4041,8 +4006,7 @@ public void testPartitionFilterRequiredAndWriteOperation() public void testPartitionFilterRequiredAndTableChanges() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_table_changes", "(x integer, part integer) WITH (partitioned_by = ARRAY['part'], change_data_feed_enabled = true)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1, 11)", 1); @@ -4093,8 +4057,7 @@ public void testPartitionFilterRequiredAndTableChanges() public void testPartitionFilterRequiredAndHistoryTable() { Session session = sessionWithPartitionFilterRequirement(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partition_filter_table_changes", "(x integer, part integer) WITH (partitioned_by = ARRAY['part'], change_data_feed_enabled = true)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1, 11)", 1); @@ -4274,8 +4237,7 @@ public void testTypeCoercionOnCreateTable() private void testTimestampCoercionOnCreateTable(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_timestamp_coercion_on_create_table", "(ts TIMESTAMP)")) { assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (" + actualValue + ")", 1); @@ -4287,8 +4249,7 @@ private void testTimestampCoercionOnCreateTable(@Language("SQL") String actualVa private void testCharCoercionOnCreateTable(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_char_coercion_on_create_table", "(vch VARCHAR)")) { assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (" + actualValue + ")", 1); @@ -4336,8 +4297,7 @@ public void testTypeCoercionOnCreateTableAsSelect() private void testTimestampCoercionOnCreateTableAsSelect(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_timestamp_coercion_on_create_table_as_select", "AS SELECT %s ts".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "ts")).isEqualTo("timestamp(6)"); @@ -4348,8 +4308,7 @@ private void testTimestampCoercionOnCreateTableAsSelect(@Language("SQL") String private void testCharCoercionOnCreateTableAsSelect(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_char_coercion_on_create_table_as_select", "AS SELECT %s col".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("varchar"); @@ -4396,8 +4355,7 @@ public void testTypeCoercionOnCreateTableAsSelectWithNoData() private void testTimestampCoercionOnCreateTableAsSelectWithNoData(@Language("SQL") String actualValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_timestamp_coercion_on_create_table_as_select_with_no_data", "AS SELECT %s ts WITH NO DATA".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "ts")).isEqualTo("timestamp(6)"); @@ -4407,8 +4365,7 @@ private void testTimestampCoercionOnCreateTableAsSelectWithNoData(@Language("SQL private void testCharCoercionOnCreateTableAsSelectWithNoData(@Language("SQL") String actualValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_char_coercion_on_create_table_as_select_with_no_data", "AS SELECT %s col WITH NO DATA".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("varchar"); @@ -4455,8 +4412,7 @@ public void testTypeCoercionOnCreateTableAsWithRowType() private void testTimestampCoercionOnCreateTableAsWithRowType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_timestamp_coercion_on_create_table_as_with_row_type", "AS SELECT CAST(row(%s) AS row(value timestamp(6))) ts".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "ts")).isEqualTo("row(value timestamp(6))"); @@ -4469,8 +4425,7 @@ private void testTimestampCoercionOnCreateTableAsWithRowType(@Language("SQL") St private void testCharCoercionOnCreateTableAsWithRowType(@Language("SQL") String actualValue, @Language("SQL") String actualTypeLiteral, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_char_coercion_on_create_table_as_with_row_type", "AS SELECT CAST(row(%s) AS row(value %s)) col".formatted(actualValue, actualTypeLiteral))) { assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("row(value varchar)"); @@ -4519,8 +4474,7 @@ public void testTypeCoercionOnCreateTableAsWithArrayType() private void testTimestampCoercionOnCreateTableAsWithArrayType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_timestamp_coercion_on_create_table_as_with_array_type", "AS SELECT array[%s] ts".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "ts")).isEqualTo("array(timestamp(6))"); @@ -4533,8 +4487,7 @@ private void testTimestampCoercionOnCreateTableAsWithArrayType(@Language("SQL") private void testCharCoercionOnCreateTableAsWithArrayType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_char_coercion_on_create_table_as_with_array_type", "AS SELECT array[%s] col".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("array(varchar)"); @@ -4583,8 +4536,7 @@ public void testTypeCoercionOnCreateTableAsWithMapType() private void testTimestampCoercionOnCreateTableAsWithMapType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_timestamp_coercion_on_create_table_as_with_map_type", "AS SELECT map(array[%1$s], array[%1$s]) ts".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "ts")).isEqualTo("map(timestamp(6), timestamp(6))"); @@ -4597,8 +4549,7 @@ private void testTimestampCoercionOnCreateTableAsWithMapType(@Language("SQL") St private void testCharCoercionOnCreateTableAsWithMapType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_char_coercion_on_create_table_as_with_map_type", "AS SELECT map(array[%1$s], array[%1$s]) col".formatted(actualValue))) { assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("map(varchar, varchar)"); @@ -4635,7 +4586,7 @@ public void testAddColumnWithTypeCoercion() private void testAddColumnWithTypeCoercion(String columnType, String expectedColumnType) { - try (TestTable testTable = new TestTable(getQueryRunner()::execute, "test_coercion_add_column", "(a varchar, b row(x integer))")) { + try (TestTable testTable = newTrinoTable("test_coercion_add_column", "(a varchar, b row(x integer))")) { // TODO: Update this test once the connector supports adding a new field to a row type assertQueryFails("ALTER TABLE " + testTable.getName() + " ADD COLUMN b.y " + columnType, "This connector does not support adding fields"); @@ -4654,8 +4605,7 @@ private void assertTimestampNtzFeature(String tableName) @Test public void testSelectTableUsingVersion() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_select_table_using_version", "(id INT, country VARCHAR)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1, 'India')", 1); @@ -4703,7 +4653,7 @@ public void testSelectTableUsingVersion() @Test public void testReadMultipleVersions() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_read_multiple_versions", "AS SELECT 1 id")) { + try (TestTable table = newTrinoTable("test_read_multiple_versions", "AS SELECT 1 id")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 2", 1); assertQuery( "SELECT * FROM " + table.getName() + " FOR VERSION AS OF 0 " + @@ -4716,7 +4666,7 @@ public void testReadMultipleVersions() @Test public void testReadVersionedTableWithOptimize() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_read_versioned_optimize", "AS SELECT 1 id")) { + try (TestTable table = newTrinoTable("test_read_versioned_optimize", "AS SELECT 1 id")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 2", 1); Set beforeActiveFiles = getActiveFiles(table.getName()); @@ -4743,7 +4693,7 @@ public void testReadVersionedTableWithVacuum() .setCatalogSessionProperty(getSession().getCatalog().orElseThrow(), "vacuum_min_retention", "0s") .build(); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_column_and_vacuum", "(x VARCHAR)")) { + try (TestTable table = newTrinoTable("test_add_column_and_vacuum", "(x VARCHAR)")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT 'first'", 1); assertUpdate("INSERT INTO " + table.getName() + " SELECT 'second'", 1); @@ -4784,8 +4734,8 @@ public void testReadVersionedTableWithVacuum() @Test public void testInsertFromVersionedTable() { - try (TestTable targetTable = new TestTable(getQueryRunner()::execute, "test_read_versioned_insert", "(col int)"); - TestTable sourceTable = new TestTable(getQueryRunner()::execute, "test_read_versioned_insert", "AS SELECT 1 col")) { + try (TestTable targetTable = newTrinoTable("test_read_versioned_insert", "(col int)"); + TestTable sourceTable = newTrinoTable("test_read_versioned_insert", "AS SELECT 1 col")) { assertUpdate("INSERT INTO " + sourceTable.getName() + " VALUES 2", 1); assertUpdate("INSERT INTO " + sourceTable.getName() + " VALUES 3", 1); @@ -4800,7 +4750,7 @@ public void testInsertFromVersionedTable() @Test public void testInsertFromVersionedSameTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_read_versioned_insert", "AS SELECT 1 id")) { + try (TestTable table = newTrinoTable("test_read_versioned_insert", "AS SELECT 1 id")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 2", 1); assertUpdate("INSERT INTO " + table.getName() + " SELECT * FROM " + table.getName() + " FOR VERSION AS OF 0", 1); @@ -4824,7 +4774,7 @@ public void testInsertFromVersionedSameTable() @Test public void testInsertFromMultipleVersionedSameTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_read_versioned_insert", "AS SELECT 1 id")) { + try (TestTable table = newTrinoTable("test_read_versioned_insert", "AS SELECT 1 id")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 2", 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES 1, 2"); @@ -4841,7 +4791,7 @@ public void testInsertFromMultipleVersionedSameTable() @Test public void testReadVersionedTableWithChangeDataFeed() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_read_versioned_cdf", "WITH (change_data_feed_enabled=true) AS SELECT 1 id")) { + try (TestTable table = newTrinoTable("test_read_versioned_cdf", "WITH (change_data_feed_enabled=true) AS SELECT 1 id")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 2", 1); assertUpdate("UPDATE " + table.getName() + " SET id = -2 WHERE id = 2", 1); assertUpdate("DELETE FROM " + table.getName() + " WHERE id = 1", 1); @@ -4857,7 +4807,7 @@ public void testReadVersionedTableWithChangeDataFeed() public void testSelectTableUsingVersionSchemaEvolution() { // Delta Lake respects the old schema unlike Iceberg connector - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_select_table_using_version", "AS SELECT 1 id")) { + try (TestTable table = newTrinoTable("test_select_table_using_version", "AS SELECT 1 id")) { assertUpdate("ALTER TABLE " + table.getName() + " ADD COLUMN new_col VARCHAR"); assertQuery("SELECT * FROM " + table.getName() + " FOR VERSION AS OF 0", "VALUES 1"); assertQuery("SELECT * FROM " + table.getName() + " FOR VERSION AS OF 1", "VALUES (1, NULL)"); @@ -4897,7 +4847,7 @@ public void testSelectTableUsingVersionDeletedCheckpoints() public void testSelectAfterReadVersionedTable() { // Run normal SELECT after reading from versioned table - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_select_after_version", "AS SELECT 1 id")) { + try (TestTable table = newTrinoTable("test_select_after_version", "AS SELECT 1 id")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 2", 1); assertQuery("SELECT * FROM " + table.getName() + " FOR VERSION AS OF 0", "VALUES 1"); assertQuery("SELECT * FROM " + table.getName(), "VALUES 1, 2"); @@ -4965,7 +4915,7 @@ public void testDuplicatedFieldNames() assertQueryFails("CREATE TABLE " + tableName + "(col row(a row(x int, \"X\" int)))", "Field name 'x' specified more than once"); assertQueryFails("CREATE TABLE " + tableName + " AS SELECT cast(NULL AS row(a row(x int, \"X\" int))) col", "Field name 'x' specified more than once"); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_duplicated_field_names_", "(id int)")) { + try (TestTable table = newTrinoTable("test_duplicated_field_names_", "(id int)")) { assertQueryFails("ALTER TABLE " + table.getName() + " ADD COLUMN col row(x int, \"X\" int)", ".* Field name 'x' specified more than once"); assertUpdate("ALTER TABLE " + table.getName() + " ADD COLUMN col row(\"X\" int)"); @@ -4992,7 +4942,7 @@ void testRegisterTableAccessControl() @Test public void testMetastoreAfterCreateTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int) COMMENT 'test comment'")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int) COMMENT 'test comment'")) { assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains( entry("comment", "test comment"), @@ -5004,7 +4954,7 @@ public void testMetastoreAfterCreateTable() @Test public void testMetastoreAfterCreateOrReplaceTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int) COMMENT 'test comment'")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int) COMMENT 'test comment'")) { assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + "(new_col varchar) COMMENT 'new comment'"); assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains( @@ -5017,7 +4967,7 @@ public void testMetastoreAfterCreateOrReplaceTable() @Test public void testMetastoreAfterCreateTableAsSelect() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "COMMENT 'test comment' AS SELECT 1 col")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "COMMENT 'test comment' AS SELECT 1 col")) { assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains( entry("comment", "test comment"), @@ -5029,7 +4979,7 @@ public void testMetastoreAfterCreateTableAsSelect() @Test public void testMetastoreAfterCreateOrReplaceTableAsSelect() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "COMMENT 'test comment' AS SELECT 1 col")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "COMMENT 'test comment' AS SELECT 1 col")) { assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " COMMENT 'new comment' AS SELECT 'test' new_col", 1); assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains( @@ -5042,7 +4992,7 @@ public void testMetastoreAfterCreateOrReplaceTableAsSelect() @Test public void testMetastoreAfterCommentTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .doesNotContainKey("comment") .contains( @@ -5061,7 +5011,7 @@ public void testMetastoreAfterCommentTable() @Test public void testMetastoreAfterCommentColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int COMMENT 'test comment')")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int COMMENT 'test comment')")) { assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .doesNotContainKey("comment") .contains( @@ -5081,7 +5031,7 @@ public void testMetastoreAfterCommentColumn() public void testMetastoreAfterAlterColumn() { // Use 'name' column mapping mode to allow renaming columns - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int NOT NULL) WITH (column_mapping_mode = 'name')")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int NOT NULL) WITH (column_mapping_mode = 'name')")) { Map initialParameters = metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters(); assertThat(initialParameters) .doesNotContainKey("comment") @@ -5142,7 +5092,7 @@ public void testMetastoreAfterAlterColumn() @Test public void testMetastoreAfterSetTableProperties() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertUpdate("ALTER TABLE " + table.getName() + " SET PROPERTIES change_data_feed_enabled = true"); assertEventually(() -> assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains( @@ -5154,7 +5104,7 @@ public void testMetastoreAfterSetTableProperties() @Test public void testMetastoreAfterOptimize() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertUpdate("ALTER TABLE " + table.getName() + " EXECUTE optimize"); assertEventually(() -> assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains( @@ -5166,7 +5116,7 @@ public void testMetastoreAfterOptimize() @Test public void testMetastoreAfterRegisterTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int) COMMENT 'test comment'")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int) COMMENT 'test comment'")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 1", 1); String tableLocation = metastore.getTable(SCHEMA, table.getName()).orElseThrow().getStorage().getLocation(); metastore.dropTable(SCHEMA, table.getName(), false); @@ -5183,7 +5133,7 @@ public void testMetastoreAfterRegisterTable() @Test public void testMetastoreAfterCreateTableRemotely() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int) COMMENT 'test comment'")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int) COMMENT 'test comment'")) { Table metastoreTable = metastore.getTable(SCHEMA, table.getName()).orElseThrow(); metastore.dropTable(SCHEMA, table.getName(), false); @@ -5211,7 +5161,7 @@ public void testMetastoreAfterDataManipulation() { String schemaString = "{\"type\":\"struct\",\"fields\":[{\"name\":\"col\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains(entry("trino_last_transaction_version", "0"), entry("trino_metadata_schema_string", schemaString)); @@ -5245,7 +5195,7 @@ public void testMetastoreAfterTruncateTable() { String schemaString = "{\"type\":\"struct\",\"fields\":[{\"name\":\"col\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "AS SELECT 1 col")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "AS SELECT 1 col")) { assertThat(metastore.getTable(SCHEMA, table.getName()).orElseThrow().getParameters()) .contains(entry("trino_last_transaction_version", "0"), entry("trino_metadata_schema_string", schemaString)); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java index 9e0c17aa8692..e2f7348e2477 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java @@ -408,7 +408,7 @@ public void testStoreMetastoreCommentTable() private void testStoreMetastoreCommentTable(boolean storeTableMetadata) { Session session = sessionWithStoreTableMetadata(storeTableMetadata); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertMetastoreInvocations(session, "COMMENT ON TABLE " + table.getName() + " IS 'test comment'", ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); } } @@ -423,7 +423,7 @@ public void testStoreMetastoreCommentColumn() private void testStoreMetastoreCommentColumn(boolean storeTableMetadata) { Session session = sessionWithStoreTableMetadata(storeTableMetadata); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int COMMENT 'test comment')")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int COMMENT 'test comment')")) { assertMetastoreInvocations(session, "COMMENT ON COLUMN " + table.getName() + ".col IS 'new test comment'", ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); } } @@ -440,7 +440,7 @@ private void testStoreMetastoreAlterColumn(boolean storeTableMetadata) Session session = sessionWithStoreTableMetadata(storeTableMetadata); // Use 'name' column mapping mode to allow renaming columns - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int NOT NULL) WITH (column_mapping_mode = 'name')")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int NOT NULL) WITH (column_mapping_mode = 'name')")) { assertMetastoreInvocations(session, "ALTER TABLE " + table.getName() + " ALTER COLUMN col DROP NOT NULL", ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); assertMetastoreInvocations(session, "ALTER TABLE " + table.getName() + " ADD COLUMN new_col int COMMENT 'test comment'", ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); assertMetastoreInvocations(session, "ALTER TABLE " + table.getName() + " RENAME COLUMN new_col TO renamed_col", ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); @@ -460,7 +460,7 @@ public void testStoreMetastoreSetTableProperties() private void testStoreMetastoreSetTableProperties(boolean storeTableMetadata) { Session session = sessionWithStoreTableMetadata(storeTableMetadata); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertMetastoreInvocations(session, "ALTER TABLE " + table.getName() + " SET PROPERTIES change_data_feed_enabled = true", ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); } } @@ -475,7 +475,7 @@ public void testStoreMetastoreOptimize() private void testStoreMetastoreOptimize(boolean storeTableMetadata) { Session session = sessionWithStoreTableMetadata(storeTableMetadata); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertMetastoreInvocations(session, "ALTER TABLE " + table.getName() + " EXECUTE optimize", ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); } } @@ -494,7 +494,7 @@ private void testStoreMetastoreVacuum(boolean storeTableMetadata) .setCatalogSessionProperty(getSession().getCatalog().orElseThrow(), "vacuum_min_retention", "0s") .build(); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "AS SELECT 1 a")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "AS SELECT 1 a")) { assertUpdate("UPDATE " + table.getName() + " SET a = 2", 1); assertMetastoreInvocations( session, @@ -513,7 +513,7 @@ public void testStoreMetastoreRegisterTable() private void testStoreMetastoreRegisterTable(boolean storeTableMetadata) { Session session = sessionWithStoreTableMetadata(storeTableMetadata); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int) COMMENT 'test comment'")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int) COMMENT 'test comment'")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES 1", 1); String tableLocation = metastore.getTable(TPCH_SCHEMA, table.getName()).orElseThrow().getStorage().getLocation(); metastore.dropTable(TPCH_SCHEMA, table.getName(), false); @@ -537,7 +537,7 @@ private void testStoreMetastoreDataManipulation(boolean storeTableMetadata) Session session = sessionWithStoreTableMetadata(storeTableMetadata); String schemaString = "{\"type\":\"struct\",\"fields\":[{\"name\":\"col\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "(col int)")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "(col int)")) { assertThat(metastore.getTable(TPCH_SCHEMA, table.getName()).orElseThrow().getParameters()) .contains(entry("trino_last_transaction_version", "0"), entry("trino_metadata_schema_string", schemaString)); @@ -563,7 +563,7 @@ public void testStoreMetastoreTruncateTable() private void testStoreMetastoreTruncateTable(boolean storeTableMetadata) { Session session = sessionWithStoreTableMetadata(storeTableMetadata); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_cache_metastore", "AS SELECT 1 col")) { + try (TestTable table = newTrinoTable("test_cache_metastore", "AS SELECT 1 col")) { assertMetastoreInvocations(session, "TRUNCATE TABLE " + table.getName(), ImmutableMultiset.of(GET_TABLE), asyncInvocations(storeTableMetadata)); } } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java index beeebb49ec41..a87d8e960d1e 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java @@ -70,7 +70,7 @@ public void testCreateView() { String tableName = "test_glue_table_" + randomNameSuffix(); String viewName = "test_glue_view_" + randomNameSuffix(); - try (TestTable table = new TestTable(getQueryRunner()::execute, tableName, "AS SELECT 'test' x"); + try (TestTable table = newTrinoTable(tableName, "AS SELECT 'test' x"); TestView view = new TestView(getQueryRunner()::execute, viewName, "SELECT * FROM " + table.getName())) { assertQuery(format("SELECT * FROM %s", view.getName()), "VALUES 'test'"); assertQuery(format("SELECT table_type FROM information_schema.tables WHERE table_name = '%s' AND table_schema='%s'", view.getName(), schema), "VALUES 'VIEW'"); diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java index f5b537bcb36b..bf48ea914bd1 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java @@ -52,7 +52,7 @@ void testShowTables() @Test void testColumnComment() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "comment", "(id INTEGER, name VARCHAR)")) { + try (TestTable table = newTrinoTable("comment", "(id INTEGER, name VARCHAR)")) { assertUpdate("COMMENT ON COLUMN %s.name IS 'comment text'".formatted(table.getName())); assertQuery("SHOW COLUMNS FROM " + table.getName(), "VALUES ('id', 'integer', '', ''), ('name', 'varchar', '', 'comment text')"); } @@ -61,7 +61,7 @@ void testColumnComment() @Test void testCannotCommentRowId() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "cannot_comment", "(id INTEGER, name VARCHAR)")) { + try (TestTable table = newTrinoTable("cannot_comment", "(id INTEGER, name VARCHAR)")) { assertThat(query("COMMENT ON COLUMN \"%s\".\"$row_id\" IS 'comment text'".formatted(table.getName()))) .failure() .hasErrorCode(INVALID_COLUMN_REFERENCE) @@ -124,7 +124,7 @@ void testSelectFromTable() @Test void testSelectLimit() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "single_column", "(rnd_bigint bigint NOT NULL)")) { + try (TestTable table = newTrinoTable("single_column", "(rnd_bigint bigint NOT NULL)")) { assertQuery("SELECT count(rnd_bigint) FROM (SELECT rnd_bigint FROM %s LIMIT 5) a".formatted(table.getName()), "VALUES (5)"); @@ -161,7 +161,7 @@ SELECT count(1) @Test void testSelectDefaultTableLimit() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "default_table_limit", "(rnd_bigint bigint NOT NULL) WITH (default_limit = 100)")) { + try (TestTable table = newTrinoTable("default_table_limit", "(rnd_bigint bigint NOT NULL) WITH (default_limit = 100)")) { assertQuery("SELECT count(distinct rnd_bigint) FROM " + table.getName(), "VALUES (100)"); } } @@ -169,7 +169,7 @@ void testSelectDefaultTableLimit() @Test public void selectOnlyNulls() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "only_nulls", "(rnd_bigint bigint) WITH (null_probability = 1.0)")) { + try (TestTable table = newTrinoTable("only_nulls", "(rnd_bigint bigint) WITH (null_probability = 1.0)")) { assertQuery("SELECT count(distinct rnd_bigint) FROM " + table.getName(), "VALUES (0)"); } } @@ -177,7 +177,7 @@ public void selectOnlyNulls() @Test void testSelectGenerator() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "generators", + try (TestTable table = newTrinoTable("generators", """ ( name VARCHAR NOT NULL WITH (generator = '#{Name.first_name} #{Name.last_name}'), diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java index f182f68d5a36..2f2e3b53a63b 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java @@ -268,7 +268,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) @Override public void verifySupportsUpdateDeclaration() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_row_update", "AS SELECT * FROM nation")) { assertQueryFails("UPDATE " + table.getName() + " SET nationkey = 100 WHERE regionkey = 2", MODIFYING_NON_TRANSACTIONAL_TABLE_MESSAGE); } } @@ -277,7 +277,7 @@ public void verifySupportsUpdateDeclaration() @Override public void verifySupportsRowLevelUpdateDeclaration() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_supports_update", "AS SELECT * FROM nation")) { assertQueryFails("UPDATE " + table.getName() + " SET nationkey = nationkey * 100 WHERE regionkey = 2", MODIFYING_NON_TRANSACTIONAL_TABLE_MESSAGE); } } @@ -4403,8 +4403,7 @@ public void testShowCreateTable() @Test public void testShowCreateTableWithColumnProperties() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_show_create_table_with_column_properties", "(a INT, b INT WITH (partition_projection_type = 'INTEGER', partition_projection_range = ARRAY['0', '10'])) " + "WITH (" + @@ -8439,8 +8438,7 @@ public void testWriteInvalidPrecisionTimestamp() @Test public void testCoercingVarchar0ToVarchar1() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_create_table_varchar", "(var_column_0 varchar(0), var_column_1 varchar(1), var_column_10 varchar(10))")) { assertThat(getColumnType(testTable.getName(), "var_column_0")).isEqualTo("varchar(1)"); @@ -8452,8 +8450,7 @@ public void testCoercingVarchar0ToVarchar1() @Test public void testCoercingVarchar0ToVarchar1WithCTAS() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_ctas_varchar", "AS SELECT '' AS var_column")) { assertThat(getColumnType(testTable.getName(), "var_column")).isEqualTo("varchar(1)"); @@ -8463,8 +8460,7 @@ public void testCoercingVarchar0ToVarchar1WithCTAS() @Test public void testCoercingVarchar0ToVarchar1WithCTASNoData() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_ctas_nd_varchar", "AS SELECT '' AS var_column WITH NO DATA")) { assertThat(getColumnType(testTable.getName(), "var_column")).isEqualTo("varchar(1)"); @@ -8474,8 +8470,7 @@ public void testCoercingVarchar0ToVarchar1WithCTASNoData() @Test public void testCoercingVarchar0ToVarchar1WithAddColumn() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_add_column_varchar", "(col integer)")) { assertUpdate("ALTER TABLE " + testTable.getName() + " ADD COLUMN var_column varchar(0)"); @@ -8974,8 +8969,7 @@ public void testHiddenColumnNameConflict() private void testHiddenColumnNameConflict(String columnName) { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_hidden_column_name_conflict", format("(\"%s\" int, _bucket int, _partition int) WITH (partitioned_by = ARRAY['_partition'], bucketed_by = ARRAY['_bucket'], bucket_count = 10)", columnName))) { assertThat(query("SELECT * FROM " + table.getName())) @@ -9383,8 +9377,7 @@ public void testSelectWithShortZoneId() Resources.copy(resourceLocation, out); } - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_select_with_short_zone_id_", "(id INT, firstName VARCHAR, lastName VARCHAR) WITH (external_location = '%s')".formatted(tempDir))) { assertThat(query("SELECT * FROM %s".formatted(testTable.getName()))) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestGlueHiveMetastoreSkipArchive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestGlueHiveMetastoreSkipArchive.java index 19a0c276d336..338091df23ce 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestGlueHiveMetastoreSkipArchive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestGlueHiveMetastoreSkipArchive.java @@ -63,7 +63,7 @@ void cleanUpSchema() @Test void testSkipArchive() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_skip_archive", "(col int)")) { + try (TestTable table = newTrinoTable("test_skip_archive", "(col int)")) { List tableVersionsBeforeInsert = getTableVersions(testSchema, table.getName()); assertThat(tableVersionsBeforeInsert).hasSize(1); String versionIdBeforeInsert = getOnlyElement(tableVersionsBeforeInsert).versionId(); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestHiveOrcWithShortZoneId.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestHiveOrcWithShortZoneId.java index 8ab4dc0c8ba5..b450e3950324 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestHiveOrcWithShortZoneId.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestHiveOrcWithShortZoneId.java @@ -63,8 +63,7 @@ protected QueryRunner createQueryRunner() public void testSelectWithShortZoneId() { // When table is created using ORC file that contains short zone id in stripe footer - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_select_with_short_zone_id_", "(id INT, firstName VARCHAR, lastName VARCHAR) WITH (external_location = '%s')".formatted(dataFile.parentDirectory()))) { assertQuery("SELECT * FROM " + testTable.getName(), "VALUES (1, 'John', 'Doe')"); @@ -75,8 +74,7 @@ public void testSelectWithShortZoneId() public void testSelectWithoutShortZoneId() { // When table is created by trino - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_select_without_short_zone_id_", "(id INT, firstName VARCHAR, lastName VARCHAR)", ImmutableList.of("2, 'Alice', 'Doe'"))) { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java index 91f31976d3ce..2d2c37bcdd30 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorSmokeTest.java @@ -124,7 +124,7 @@ public void testShowCreateTable() @Test public void testHiddenPathColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "hidden_file_path", "(a int, b VARCHAR)", ImmutableList.of("(1, 'a')"))) { + try (TestTable table = newTrinoTable("hidden_file_path", "(a int, b VARCHAR)", ImmutableList.of("(1, 'a')"))) { String filePath = (String) computeScalar(format("SELECT file_path FROM \"%s$files\"", table.getName())); assertQuery("SELECT DISTINCT \"$path\" FROM " + table.getName(), "VALUES " + "'" + filePath + "'"); @@ -149,8 +149,7 @@ public void testDeleteRowsConcurrently() String[] expectedErrors = new String[] {"Failed to commit the transaction during write:", "Failed to replace table due to concurrent updates:", "Failed to commit during write:"}; - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_concurrent_delete", "(col0 INTEGER, col1 INTEGER, col2 INTEGER, col3 INTEGER)")) { String tableName = table.getName(); @@ -190,8 +189,7 @@ public void testDeleteRowsConcurrently() @Test public void testCreateOrReplaceTable() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_create_or_replace", " AS SELECT BIGINT '42' a, DOUBLE '-38.5' b")) { assertThat(query("SELECT a, b FROM " + table.getName())) @@ -535,8 +533,7 @@ public void testCreateTableWithNonExistingSchemaVerifyLocation() public void testSortedNationTable() { Session withSmallRowGroups = withSmallRowGroups(getSession()); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_sorted_nation_table", "WITH (sorted_by = ARRAY['comment'], format = '" + format.name() + "') AS SELECT * FROM nation WITH NO DATA")) { assertUpdate(withSmallRowGroups, "INSERT INTO " + table.getName() + " SELECT * FROM nation", 25); @@ -556,8 +553,7 @@ public void testFileSortingWithLargerTable() .setCatalogSessionProperty("iceberg", "parquet_writer_block_size", "20kB") .setCatalogSessionProperty("iceberg", "parquet_writer_batch_size", "200") .build(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_sorted_lineitem_table", "WITH (sorted_by = ARRAY['comment'], format = '" + format.name() + "') AS TABLE tpch.tiny.lineitem WITH NO DATA")) { assertUpdate( @@ -701,8 +697,7 @@ public void testDropTableWithNonExistentTableLocation() @Test public void testMetadataTables() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_metadata_tables", "(id int, part varchar) WITH (partitioning = ARRAY['part'])")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1, 'p1')", 1); @@ -752,8 +747,7 @@ public void testTableChangesFunction() { DateTimeFormatter instantMillisFormatter = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSVV").withZone(UTC); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_table_changes_function_", "AS SELECT nationkey, name FROM tpch.tiny.nation WITH NO DATA")) { long initialSnapshot = getMostRecentSnapshotId(table.getName()); @@ -793,8 +787,7 @@ public void testTableChangesFunction() @Test public void testRowLevelDeletesWithTableChangesFunction() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_row_level_deletes_with_table_changes_function_", "AS SELECT nationkey, regionkey, name FROM tpch.tiny.nation WITH NO DATA")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT nationkey, regionkey, name FROM nation", 25); @@ -814,8 +807,7 @@ public void testCreateOrReplaceWithTableChangesFunction() { DateTimeFormatter instantMillisFormatter = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSVV").withZone(UTC); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_table_changes_function_", "AS SELECT nationkey, name FROM tpch.tiny.nation WITH NO DATA")) { long initialSnapshot = getMostRecentSnapshotId(table.getName()); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java index 2de9c8dfa04e..8f56f7fede55 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java @@ -215,7 +215,7 @@ public void initFileSystem() @BeforeAll public void initStorageTimePrecision() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "inspect_storage_precision", "(i int)")) { + try (TestTable table = newTrinoTable("inspect_storage_precision", "(i int)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1)", 1); assertUpdate("INSERT INTO " + table.getName() + " VALUES (2)", 1); assertUpdate("INSERT INTO " + table.getName() + " VALUES (3)", 1); @@ -243,7 +243,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) @Test public void testAddRowFieldCaseInsensitivity() { - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_add_row_field_case_insensitivity_", "AS SELECT CAST(row(row(2)) AS row(\"CHILD\" row(grandchild_1 integer))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(CHILD row(grandchild_1 integer))"); @@ -301,7 +301,7 @@ protected void verifyConcurrentAddColumnFailurePermissible(Exception e) @Test public void testDeleteOnV1Table() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_", "WITH (format_version = 1) AS SELECT * FROM orders")) { + try (TestTable table = newTrinoTable("test_delete_", "WITH (format_version = 1) AS SELECT * FROM orders")) { assertQueryFails("DELETE FROM " + table.getName() + " WHERE custkey <= 100", "Iceberg table updates require at least format version 2"); } } @@ -311,8 +311,7 @@ public void testDeleteOnV1Table() public void testCharVarcharComparison() { // with char->varchar coercion on table creation, this is essentially varchar/varchar comparison - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_char_varchar", "(k, v) AS VALUES" + " (-1, CAST(NULL AS CHAR(3))), " + @@ -1329,7 +1328,7 @@ private void testCreatePartitionedTableWithQuotedIdentifierCasing(String columnN @Test public void testPartitionColumnNameConflict() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_conflict_partition", "(ts timestamp, ts_day int) WITH (partitioning = ARRAY['day(ts)'])")) { + try (TestTable table = newTrinoTable("test_conflict_partition", "(ts timestamp, ts_day int) WITH (partitioning = ARRAY['day(ts)'])")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (TIMESTAMP '2021-07-24 03:43:57.987654', 1)", 1); assertThat(query("SELECT * FROM " + table.getName())) @@ -1338,7 +1337,7 @@ public void testPartitionColumnNameConflict() .matches("VALUES DATE '2021-07-24'"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_conflict_partition", "(ts timestamp, ts_day int)")) { + try (TestTable table = newTrinoTable("test_conflict_partition", "(ts timestamp, ts_day int)")) { assertUpdate("ALTER TABLE " + table.getName() + " SET PROPERTIES partitioning = ARRAY['day(ts)']"); assertUpdate("INSERT INTO " + table.getName() + " VALUES (TIMESTAMP '2021-07-24 03:43:57.987654', 1)", 1); @@ -1515,8 +1514,7 @@ private void testCreateSortedTableWithSortTransform(String columnName, String so public void testSortOrderChange() { Session withSmallRowGroups = withSmallRowGroups(getSession()); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_sort_order_change", "WITH (sorted_by = ARRAY['comment']) AS SELECT * FROM nation WITH NO DATA")) { assertUpdate(withSmallRowGroups, "INSERT INTO " + table.getName() + " SELECT * FROM nation", 25); @@ -1546,8 +1544,7 @@ public void testSortingDisabled() Session withSortingDisabled = Session.builder(withSmallRowGroups(getSession())) .setCatalogSessionProperty(ICEBERG_CATALOG, "sorted_writing_enabled", "false") .build(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_sorting_disabled", "WITH (sorted_by = ARRAY['comment']) AS SELECT * FROM nation WITH NO DATA")) { assertUpdate(withSortingDisabled, "INSERT INTO " + table.getName() + " SELECT * FROM nation", 25); @@ -1562,8 +1559,7 @@ public void testSortingDisabled() public void testOptimizeWithSortOrder() { Session withSmallRowGroups = withSmallRowGroups(getSession()); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_optimize_with_sort_order", "WITH (sorted_by = ARRAY['comment']) AS SELECT * FROM nation WITH NO DATA")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT * FROM nation WHERE nationkey < 10", 10); @@ -1584,8 +1580,7 @@ public void testOptimizeWithSortOrder() public void testUpdateWithSortOrder() { Session withSmallRowGroups = withSmallRowGroups(getSession()); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_sorted_update", "WITH (sorted_by = ARRAY['comment']) AS TABLE tpch.tiny.customer WITH NO DATA")) { assertUpdate( @@ -1623,8 +1618,7 @@ public void testSortingOnNestedField() public void testDroppingSortColumn() { Session withSmallRowGroups = withSmallRowGroups(getSession()); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_dropping_sort_column", "WITH (sorted_by = ARRAY['comment']) AS SELECT * FROM nation WITH NO DATA")) { assertUpdate(withSmallRowGroups, "INSERT INTO " + table.getName() + " SELECT * FROM nation", 25); @@ -1782,7 +1776,7 @@ public void testDuplicatedFieldNames() assertQueryFails("CREATE TABLE " + tableName + "(col row(a row(x int, \"X\" int)))", "Field name 'x' specified more than once"); assertQueryFails("CREATE TABLE " + tableName + " AS SELECT cast(NULL AS row(a row(x int, \"X\" int))) col", "Field name 'x' specified more than once"); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_duplicated_field_names_", "(id int)")) { + try (TestTable table = newTrinoTable("test_duplicated_field_names_", "(id int)")) { assertQueryFails("ALTER TABLE " + table.getName() + " ADD COLUMN col row(x int, \"X\" int)", ".* Field name 'x' specified more than once"); assertUpdate("ALTER TABLE " + table.getName() + " ADD COLUMN col row(\"X\" int)"); @@ -4114,8 +4108,7 @@ public void testPredicatePushdown() @Test public void testPredicateOnDataColumnIsNotPushedDown() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_predicate_on_data_column_is_not_pushed_down", "(a integer)")) { assertThat(query("SELECT * FROM " + testTable.getName() + " WHERE a = 10")) @@ -5029,7 +5022,7 @@ public void testSplitPruningForFilterOnNonPartitionColumn() if (testSetup.isUnsupportedType()) { return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_split_pruning_non_partitioned", "(row_id int, col " + testSetup.getTrinoTypeName() + ")")) { + try (TestTable table = newTrinoTable("test_split_pruning_non_partitioned", "(row_id int, col " + testSetup.getTrinoTypeName() + ")")) { String tableName = table.getName(); String sampleValue = testSetup.getSampleValueLiteral(); String highValue = testSetup.getHighValueLiteral(); @@ -5108,8 +5101,7 @@ public void testSplitPruningFromDataFileStatistics() if (testSetup.isUnsupportedType()) { return; } - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_split_pruning_data_file_statistics", // Random double is needed to make sure rows are different. Otherwise compression may deduplicate rows, resulting in only one row group "(col " + testSetup.getTrinoTypeName() + ", r double)")) { @@ -6080,7 +6072,7 @@ public void testCollectingStatisticsWithFileModifiedTimeColumnPredicate() @Test public void testDeleteWithPathColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_with_path_", "(key int)")) { + try (TestTable table = newTrinoTable("test_delete_with_path_", "(key int)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1)", 1); sleepUninterruptibly(1, MILLISECONDS); assertUpdate("INSERT INTO " + table.getName() + " VALUES (2)", 1); @@ -6099,7 +6091,7 @@ public void testFileModifiedTimeHiddenColumn() if (storageTimePrecision.toMillis(1) > 1) { storageTimePrecision.sleep(1); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_file_modified_time_", "(col) AS VALUES (1)")) { + try (TestTable table = newTrinoTable("test_file_modified_time_", "(col) AS VALUES (1)")) { // Describe output should not have the $file_modified_time hidden column assertThat(query("DESCRIBE " + table.getName())) .skippingTypesCheck() @@ -6185,7 +6177,7 @@ public void testOptimizeWithFileModifiedTimeColumn() public void testDeleteWithFileModifiedTimeColumn() throws Exception { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_with_file_modified_time_", "(key int)")) { + try (TestTable table = newTrinoTable("test_delete_with_file_modified_time_", "(key int)")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (1)", 1); storageTimePrecision.sleep(1); assertUpdate("INSERT INTO " + table.getName() + " VALUES (2)", 1); @@ -6583,7 +6575,7 @@ public void testEmptyDelete() @Test public void testEmptyFilesTruncate() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_empty_files_truncate_", "AS SELECT 1 AS id")) { + try (TestTable table = newTrinoTable("test_empty_files_truncate_", "AS SELECT 1 AS id")) { assertUpdate("TRUNCATE TABLE " + table.getName()); assertQueryReturnsEmptyResult("SELECT * FROM \"" + table.getName() + "$files\""); } @@ -6954,7 +6946,7 @@ public void testDeleteRetainsMetadataFile() @Test public void testCreateOrReplaceTableSnapshots() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " AS SELECT BIGINT '42' a, DOUBLE '-38.5' b")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " AS SELECT BIGINT '42' a, DOUBLE '-38.5' b")) { long v1SnapshotId = getCurrentSnapshotId(table.getName()); assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " AS SELECT BIGINT '-42' a, DOUBLE '38.5' b", 1); @@ -6969,7 +6961,7 @@ public void testCreateOrReplaceTableSnapshots() @Test public void testCreateOrReplaceTableChangeColumnNamesAndTypes() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " AS SELECT BIGINT '42' a, DOUBLE '-38.5' b")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " AS SELECT BIGINT '42' a, DOUBLE '-38.5' b")) { long v1SnapshotId = getCurrentSnapshotId(table.getName()); assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " AS SELECT CAST(ARRAY[ROW('test')] AS ARRAY(ROW(field VARCHAR))) a, VARCHAR 'test2' b", 1); @@ -6984,7 +6976,7 @@ public void testCreateOrReplaceTableChangeColumnNamesAndTypes() @Test public void testCreateOrReplaceTableChangePartitionedTableIntoUnpartitioned() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " WITH (partitioning=ARRAY['a']) AS SELECT BIGINT '42' a, 'some data' b UNION ALL SELECT BIGINT '43' a, 'another data' b")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " WITH (partitioning=ARRAY['a']) AS SELECT BIGINT '42' a, 'some data' b UNION ALL SELECT BIGINT '43' a, 'another data' b")) { long v1SnapshotId = getCurrentSnapshotId(table.getName()); assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " WITH (sorted_by=ARRAY['a']) AS SELECT BIGINT '22' a, 'new data' b", 1); @@ -7007,7 +6999,7 @@ public void testCreateOrReplaceTableChangePartitionedTableIntoUnpartitioned() @Test public void testCreateOrReplaceTableChangeUnpartitionedTableIntoPartitioned() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " WITH (sorted_by=ARRAY['a']) AS SELECT BIGINT '22' a, CAST('some data' AS VARCHAR) b")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " WITH (sorted_by=ARRAY['a']) AS SELECT BIGINT '22' a, CAST('some data' AS VARCHAR) b")) { long v1SnapshotId = getCurrentSnapshotId(table.getName()); assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " WITH (partitioning=ARRAY['a']) AS SELECT BIGINT '42' a, 'some data' b UNION ALL SELECT BIGINT '43' a, 'another data' b", 2); @@ -7030,7 +7022,7 @@ public void testCreateOrReplaceTableChangeUnpartitionedTableIntoPartitioned() @Test public void testCreateOrReplaceTableWithComments() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " (a BIGINT COMMENT 'This is a column') COMMENT 'This is a table'")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " (a BIGINT COMMENT 'This is a column') COMMENT 'This is a table'")) { long v1SnapshotId = getCurrentSnapshotId(table.getName()); assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " AS SELECT 1 a", 1); @@ -7057,8 +7049,7 @@ public void testCreateOrReplaceTableWithComments() @Test public void testCreateOrReplaceTableWithSameLocation() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_create_or_replace_with_same_location_", "(a integer)")) { String initialTableLocation = getTableLocation(table.getName()); @@ -7089,7 +7080,7 @@ public void testCreateOrReplaceTableWithSameLocation() @Test public void testCreateOrReplaceTableWithChangeInLocation() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_change_location_", "(a integer) ")) { + try (TestTable table = newTrinoTable("test_create_or_replace_change_location_", "(a integer) ")) { String initialTableLocation = getTableLocation(table.getName()) + randomNameSuffix(); long v1SnapshotId = getCurrentSnapshotId(table.getName()); assertQueryFails( @@ -7452,7 +7443,7 @@ protected OptionalInt maxTableRenameLength() @Test public void testSetPartitionedColumnType() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_partitioned_column_type_", "WITH (partitioning = ARRAY['part']) AS SELECT 1 AS id, CAST(123 AS integer) AS part")) { + try (TestTable table = newTrinoTable("test_set_partitioned_column_type_", "WITH (partitioning = ARRAY['part']) AS SELECT 1 AS id, CAST(123 AS integer) AS part")) { assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN part SET DATA TYPE bigint"); assertThat(query("SELECT part FROM " + table.getName())) @@ -7467,7 +7458,7 @@ public void testSetPartitionedColumnType() @Test public void testSetTransformPartitionedColumnType() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_partitioned_column_type_", "WITH (partitioning = ARRAY['bucket(part, 10)']) AS SELECT CAST(123 AS integer) AS part")) { + try (TestTable table = newTrinoTable("test_set_partitioned_column_type_", "WITH (partitioning = ARRAY['bucket(part, 10)']) AS SELECT CAST(123 AS integer) AS part")) { assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN part SET DATA TYPE bigint"); assertThat(query("SELECT * FROM " + table.getName())) @@ -7775,8 +7766,7 @@ public void testNoRetryWhenMetadataFileInvalid() @Test public void testTableChangesFunctionAfterSchemaChange() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_table_changes_function_", "AS SELECT nationkey, name FROM tpch.tiny.nation WITH NO DATA")) { long initialSnapshot = getCurrentSnapshotId(table.getName()); @@ -8152,8 +8142,8 @@ private static Session withPartitionFilterRequired(Session session) public void testUuidDynamicFilter() { String catalog = getSession().getCatalog().orElseThrow(); - try (TestTable dataTable = new TestTable(getQueryRunner()::execute, "data_table", "(value uuid)"); - TestTable filteringTable = new TestTable(getQueryRunner()::execute, "filtering_table", "(filtering_value uuid)")) { + try (TestTable dataTable = newTrinoTable("data_table", "(value uuid)"); + TestTable filteringTable = newTrinoTable("filtering_table", "(filtering_value uuid)")) { assertUpdate("INSERT INTO " + dataTable.getName() + " VALUES UUID 'f73894f0-5447-41c5-a727-436d04c7f8ab', UUID '4f676658-67c9-4e80-83be-ec75f0b9d0c9'", 2); assertUpdate("INSERT INTO " + filteringTable.getName() + " VALUES UUID 'f73894f0-5447-41c5-a727-436d04c7f8ab'", 1); @@ -8170,8 +8160,8 @@ public void testUuidDynamicFilter() public void testDynamicFilterWithExplicitPartitionFilter() { String catalog = getSession().getCatalog().orElseThrow(); - try (TestTable salesTable = new TestTable(getQueryRunner()::execute, "sales_table", "(date date, receipt_id varchar, amount decimal(10,2)) with (partitioning=array['date'])"); - TestTable dimensionTable = new TestTable(getQueryRunner()::execute, "dimension_table", "(date date, following_holiday boolean, year int)")) { + try (TestTable salesTable = newTrinoTable("sales_table", "(date date, receipt_id varchar, amount decimal(10,2)) with (partitioning=array['date'])"); + TestTable dimensionTable = newTrinoTable("dimension_table", "(date date, following_holiday boolean, year int)")) { assertUpdate( """ INSERT INTO %s @@ -8265,8 +8255,7 @@ private void testCreateTableWithCompressionCodec(HiveCompressionCodec compressio public void testTypeCoercionOnCreateTableAsSelect() { for (TypeCoercionTestSetup setup : typeCoercionOnCreateTableAsSelectProvider()) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_show_create_table", format("AS SELECT %s a", setup.sourceValueLiteral))) { assertThat(getColumnType(testTable.getName(), "a")).isEqualTo(setup.newColumnType); @@ -8282,8 +8271,7 @@ public void testTypeCoercionOnCreateTableAsSelect() public void testTypeCoercionOnCreateTableAsSelectWithNoData() { for (TypeCoercionTestSetup setup : typeCoercionOnCreateTableAsSelectProvider()) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_show_create_table", format("AS SELECT %s a WITH NO DATA", setup.sourceValueLiteral))) { assertThat(getColumnType(testTable.getName(), "a")).isEqualTo(setup.newColumnType); @@ -8470,7 +8458,7 @@ public void testAddColumnWithTypeCoercion() private void testAddColumnWithTypeCoercion(String columnType, String expectedColumnType) { - try (TestTable testTable = new TestTable(getQueryRunner()::execute, "test_coercion_add_column", "(a varchar, b row(x integer))")) { + try (TestTable testTable = newTrinoTable("test_coercion_add_column", "(a varchar, b row(x integer))")) { assertUpdate("ALTER TABLE " + testTable.getName() + " ADD COLUMN b.y " + columnType); assertThat(getColumnType(testTable.getName(), "b")).isEqualTo("row(x integer, y %s)".formatted(expectedColumnType)); @@ -8595,7 +8583,7 @@ public void testIllegalExtraPropertyKey() @Test public void testSetIllegalExtraPropertyKey() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_illegal_table_properties", "(x int)")) { + try (TestTable table = newTrinoTable("test_set_illegal_table_properties", "(x int)")) { assertQueryFails( "ALTER TABLE " + table.getName() + " SET PROPERTIES extra_properties = MAP(ARRAY['sorted_by'], ARRAY['id'])", "\\QIllegal keys in extra_properties: [sorted_by]"); @@ -8611,8 +8599,7 @@ public void testSetIllegalExtraPropertyKey() @Test // regression test for https://github.com/trinodb/trino/issues/22922 void testArrayElementChange() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_array_schema_change", "(col array(row(a varchar, b varchar)))", List.of("CAST(array[row('a', 'b')] AS array(row(a varchar, b varchar)))"))) { @@ -8631,8 +8618,7 @@ void testArrayElementChange() @Test void testRowFieldChange() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_row_schema_change", "(col row(a varchar, b varchar))")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT CAST(row('a', 'b') AS row(a varchar, b varchar))", 1); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java index ff5b78fc48f7..be499a524122 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergSystemTables.java @@ -393,7 +393,7 @@ public void testManifestsTable() @Test public void testFilesTable() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_files_table", "AS SELECT 1 x")) { + try (TestTable table = newTrinoTable("test_files_table", "AS SELECT 1 x")) { MaterializedResult result = computeActual("DESCRIBE " + table.getName()); assertThat(result.getMaterializedRows().stream().map(row -> (String) row.getField(0))) .doesNotContain("partition"); @@ -509,7 +509,7 @@ void testFilesTableReadableMetrics() private void testFilesTableReadableMetrics(@Language("SQL") String type, @Language("SQL") String values, @Language("JSON") String... readableMetrics) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_files_table", "(x " + type + ")")) { + try (TestTable table = newTrinoTable("test_files_table", "(x " + type + ")")) { getQueryRunner().execute("INSERT INTO " + table.getName() + " " + values); assertThat(computeActual("SELECT readable_metrics FROM \"" + table.getName() + "$files\"").getOnlyColumnAsSet()) .containsExactlyInAnyOrder(readableMetrics); @@ -519,7 +519,7 @@ private void testFilesTableReadableMetrics(@Language("SQL") String type, @Langua @Test public void testFilesSchemaEvolution() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_files_table", "WITH (partitioning = ARRAY['part']) AS SELECT 1 x, 2 part")) { + try (TestTable table = newTrinoTable("test_files_table", "WITH (partitioning = ARRAY['part']) AS SELECT 1 x, 2 part")) { assertThat(query("SELECT partition FROM \"" + table.getName() + "$files\"")) .matches("SELECT CAST(ROW(2) AS ROW(part int))"); @@ -537,8 +537,7 @@ public void testFilesSchemaEvolution() @Test public void testFilesNestedPartition() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_files_table", "WITH (partitioning = ARRAY['\"part.nested\"']) AS SELECT 1 x, CAST(ROW(2) AS ROW(nested int)) part")) { assertThat(query("SELECT partition.\"part.nested\" FROM \"" + table.getName() + "$files\"")) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java index bcc365bff98b..c698ba1434a4 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConnectorSmokeTest.java @@ -138,9 +138,9 @@ public void testRowConstructorColumnLimitForMergeQuery() columns += "orderkey, custkey, orderstatus, totalprice, orderpriority) "; notMatchedClause += "s.orderkey, s.custkey, s.orderstatus, s.totalprice, s.orderpriority "; matchedClause += "orderkey = s.orderkey, custkey = s.custkey, orderstatus = s.orderstatus, totalprice = t.totalprice, orderpriority = s.orderpriority "; - TestTable table = new TestTable(getQueryRunner()::execute, "test_merge_", tableDefinition); + TestTable table = newTrinoTable("test_merge_", tableDefinition); assertUpdate("INSERT INTO " + table.getName() + " " + columns + " " + selectQuery, 1); - TestTable mergeTable = new TestTable(getQueryRunner()::execute, "test_table_", tableDefinition); + TestTable mergeTable = newTrinoTable("test_table_", tableDefinition); assertUpdate("INSERT INTO " + mergeTable.getName() + " " + columns + " " + selectQuery, 1); assertUpdate( """ diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java index 951e8b7c1533..ed3cc5313aa7 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergLocalConcurrentWrites.java @@ -806,8 +806,7 @@ void testConcurrentUpdateWithPartitionTransformation() List rows = ImmutableList.of("('A', DATE '2024-01-01')", "('B', DATE '2024-02-02')", "('C', DATE '2024-03-03')", "('D', DATE '2024-04-04')"); List partitions = ImmutableList.of("DATE '2024-01-01'", "DATE '2024-02-02'", "DATE '2024-03-03'", "DATE '2024-04-04'"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_concurrent_update_partition_transform_table_", "(data varchar, part date) with (partitioning = array['month(part)'])")) { String tableName = table.getName(); @@ -850,8 +849,7 @@ void testConcurrentUpdateWithNestedPartitionTransformation() List rows = ImmutableList.of("('A', ROW(DATE '2024-01-01'))", "('B', ROW(DATE '2024-02-02'))", "('C', ROW(DATE '2024-03-03'))", "('D', ROW(DATE '2024-04-04'))"); List partitions = ImmutableList.of("DATE '2024-01-01'", "DATE '2024-02-02'", "DATE '2024-03-03'", "DATE '2024-04-04'"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_concurrent_update_partition_transform_table_", "(data varchar, parent ROW (part date)) with (partitioning = array['month(\"parent.part\")'])")) { String tableName = table.getName(); @@ -900,8 +898,7 @@ void testConcurrentUpdateWithMultiplePartitionTransformation() List partitions2 = ImmutableList.of("1", "1", "1", "1"); List partitions3 = ImmutableList.of("'aaa'", "'aab'", "'aac'", "'aad'"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_concurrent_update_multiple_partition_transform_table_", "(data varchar, part1 timestamp, part2 int, part3 varchar) with (partitioning = array['hour(part1)', 'bucket(part2, 10)', 'truncate(part3, 2)'])")) { String tableName = table.getName(); @@ -952,8 +949,7 @@ void testConcurrentUpdateWithOverlappingPartitionTransformation() List rows = ImmutableList.of("('A', DATE '2024-01-01')", "('B', DATE '2024-01-02')", "('C', DATE '2024-03-03')", "('D', DATE '2024-04-04')"); List partitions = ImmutableList.of("DATE '2024-01-01'", "DATE '2024-01-02'", "DATE '2024-03-03'", "DATE '2024-04-04'"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_concurrent_update_overlapping_partition_transform_table_", "(data varchar, part date) with (partitioning = array['month(part)'])")) { String tableName = table.getName(); @@ -1015,8 +1011,7 @@ void testConcurrentUpdateWithEnforcedAndUnenforcedPartitions() List partitions1 = ImmutableList.of("'a'", "'b'", "'c'", "'d'"); List partitions2 = ImmutableList.of("DATE '2024-01-01'", "DATE '2024-02-02'", "DATE '2024-03-03'", "DATE '2024-04-04'"); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_concurrent_update_enforced_unenforced_partition_transform_table_", // part1 is enforced and part2 is unenforced as it has transformation "(data varchar, part1 varchar, part2 date) with (partitioning = array['part1', 'month(part2)'])")) { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMinioOrcConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMinioOrcConnectorTest.java index 114728084a47..23e3e3d68b3b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMinioOrcConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMinioOrcConnectorTest.java @@ -131,7 +131,7 @@ private void testReadSingleIntegerColumnOrcFile(String orcFileResourceName, int throws Exception { checkArgument(expectedValue != 0); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_read_as_integer", "(\"_col0\") AS VALUES 0, NULL")) { + try (TestTable table = newTrinoTable("test_read_as_integer", "(\"_col0\") AS VALUES 0, NULL")) { String orcFilePath = (String) computeScalar(format("SELECT DISTINCT file_path FROM \"%s$files\"", table.getName())); byte[] orcFileData = Resources.toByteArray(getResource(orcFileResourceName)); fileSystem.newOutputFile(Location.of(orcFilePath)).createOrOverwrite(orcFileData); @@ -150,7 +150,7 @@ private void testReadSingleIntegerColumnOrcFile(String orcFileResourceName, int public void testTimeType() { // Regression test for https://github.com/trinodb/trino/issues/15603 - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_time", "(col time(6))")) { + try (TestTable table = newTrinoTable("test_time", "(col time(6))")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (TIME '13:30:00'), (TIME '14:30:00'), (NULL)", 3); assertQuery("SELECT * FROM " + table.getName(), "VALUES '13:30:00', '14:30:00', NULL"); assertQuery( diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetConnectorTest.java index 99a2b2220dee..6c6a9023e59e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetConnectorTest.java @@ -60,8 +60,7 @@ protected boolean supportsRowGroupStatistics(String typeName) @Test public void testRowGroupResetDictionary() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_row_group_reset_dictionary", "(plain_col varchar, dict_col int)")) { String tableName = table.getName(); @@ -89,8 +88,7 @@ protected Optional filterSetColumnTypesDataProvider(SetColum @Test public void testIgnoreParquetStatistics() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_ignore_parquet_statistics", "WITH (sorted_by = ARRAY['custkey']) AS TABLE tpch.tiny.customer WITH NO DATA")) { assertUpdate( @@ -122,8 +120,7 @@ public void testIgnoreParquetStatistics() @Test public void testPushdownPredicateToParquetAfterColumnRename() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_pushdown_predicate_statistics", "WITH (sorted_by = ARRAY['custkey']) AS TABLE tpch.tiny.customer WITH NO DATA")) { assertUpdate( diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index a5e59ec83610..f150da7b8394 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -188,7 +188,7 @@ public void testDefaultFormatVersion() @Test public void testSetPropertiesObjectStoreLayoutEnabled() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_object_store", "(x int) WITH (object_store_layout_enabled = false)")) { + try (TestTable table = newTrinoTable("test_object_store", "(x int) WITH (object_store_layout_enabled = false)")) { assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())) .doesNotContain("object_store_layout_enabled"); assertThat(loadTable(table.getName()).properties()) @@ -205,7 +205,7 @@ public void testSetPropertiesObjectStoreLayoutEnabled() @Test public void testSetPropertiesDataLocation() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_data_location", "(x int)")) { + try (TestTable table = newTrinoTable("test_data_location", "(x int)")) { assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())) .doesNotContain("data_location ="); assertThat(loadTable(table.getName()).properties()) @@ -415,7 +415,7 @@ public void testOptimizePopulateSplitOffsets() .setSystemProperty("task_min_writer_count", "1") .build(); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_optimize_split_offsets", "AS SELECT * FROM tpch.tiny.nation")) { + try (TestTable table = newTrinoTable("test_optimize_split_offsets", "AS SELECT * FROM tpch.tiny.nation")) { assertUpdate(session, "ALTER TABLE " + table.getName() + " EXECUTE optimize"); assertThat(computeActual("SELECT split_offsets FROM \"" + table.getName() + "$files\"")) .isEqualTo(resultBuilder(getSession(), ImmutableList.of(new ArrayType(BIGINT))) @@ -689,8 +689,7 @@ private void runOptimizeDuringWriteOperations(boolean useSmallFiles) String blackholeTable = "blackhole_table_" + randomNameSuffix(); assertUpdate("CREATE TABLE blackhole.default.%s (a INT, b INT) WITH (split_count = 1, pages_per_split = 1, rows_per_page = 1, page_processing_delay = '3s')".formatted(blackholeTable)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_optimize_during_write_operations", "(int_col INT)")) { String tableName = table.getName(); @@ -1022,7 +1021,7 @@ public void testFilesTable() @Test public void testStatsFilePruning() { - try (TestTable testTable = new TestTable(getQueryRunner()::execute, "test_stats_file_pruning_", "(a INT, b INT) WITH (partitioning = ARRAY['b'])")) { + try (TestTable testTable = newTrinoTable("test_stats_file_pruning_", "(a INT, b INT) WITH (partitioning = ARRAY['b'])")) { assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (1, 10), (10, 10)", 2); assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (200, 10), (300, 20)", 2); @@ -1072,7 +1071,7 @@ public void testStatsFilePruning() @Test public void testColumnStatsPruning() { - try (TestTable testTable = new TestTable(getQueryRunner()::execute, "test_column_stats_pruning_", "(a INT, b INT) WITH (partitioning = ARRAY['b'])")) { + try (TestTable testTable = newTrinoTable("test_column_stats_pruning_", "(a INT, b INT) WITH (partitioning = ARRAY['b'])")) { assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (1, 10), (10, 10)", 2); assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (200, 10), (300, 20)", 2); @@ -1479,8 +1478,7 @@ void testMapValueSchemaChange() private void testMapValueSchemaChange(String format, String expectedValue) { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_map_value_schema_change", "WITH (format = '" + format + "') AS SELECT CAST(map(array[1], array[row(2)]) AS map(integer, row(field integer))) col")) { Table icebergTable = loadTable(table.getName()); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java index 8b0c017155c2..d88407b09309 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java @@ -154,7 +154,7 @@ public void testRenameSchema() @Test void testGlueTableLocation() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_table_location", "AS SELECT 1 x")) { + try (TestTable table = newTrinoTable("test_table_location", "AS SELECT 1 x")) { String initialLocation = getStorageDescriptor(getGlueTable(table.getName())).orElseThrow().getLocation(); assertThat(getStorageDescriptor(getGlueTable(table.getName())).orElseThrow().getLocation()) // Using startsWith because the location has UUID suffix diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java index 97529860099b..69be06111502 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java @@ -100,7 +100,7 @@ public void cleanup() @Test public void testSkipArchive() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_skip_archive", "(col int)")) { + try (TestTable table = newTrinoTable("test_skip_archive", "(col int)")) { List tableVersionsBeforeInsert = getTableVersions(schemaName, table.getName()); assertThat(tableVersionsBeforeInsert).hasSize(1); String versionIdBeforeInsert = getOnlyElement(tableVersionsBeforeInsert).getVersionId(); @@ -118,7 +118,7 @@ public void testSkipArchive() @Test public void testNotRemoveExistingArchive() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_remove_archive", "(col int)")) { + try (TestTable table = newTrinoTable("test_remove_archive", "(col int)")) { List tableVersionsBeforeInsert = getTableVersions(schemaName, table.getName()); assertThat(tableVersionsBeforeInsert).hasSize(1); TableVersion initialVersion = getOnlyElement(tableVersionsBeforeInsert); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogWithBearerAuth.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogWithBearerAuth.java index 85ddbcbaa524..d47384cc0595 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogWithBearerAuth.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestIcebergNessieCatalogWithBearerAuth.java @@ -74,7 +74,7 @@ protected QueryRunner createQueryRunner() @Test public void testWithValidAccessToken() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_valid_access_token", "(a INT, b VARCHAR)", ImmutableList.of("(1, 'a')"))) { + try (TestTable table = newTrinoTable("test_valid_access_token", "(a INT, b VARCHAR)", ImmutableList.of("(1, 'a')"))) { assertQuery("SELECT * FROM " + table.getName(), "VALUES(1, 'a')"); } } diff --git a/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java b/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java index c4ee09cdd966..142eaef4a076 100644 --- a/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java +++ b/plugin/trino-ignite/src/test/java/io/trino/plugin/ignite/TestIgniteConnectorTest.java @@ -109,8 +109,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) @Test public void testLikeWithEscape() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_like_with_escape", "(id int, a varchar(4))", List.of( @@ -140,8 +139,7 @@ public void testIsNullPredicatePushdown() assertThat(query("SELECT nationkey FROM nation WHERE name IS NULL")).isFullyPushedDown(); assertThat(query("SELECT nationkey FROM nation WHERE name IS NULL OR name = 'a' OR regionkey = 4")).isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_is_null_predicate_pushdown", "(a_int integer, a_varchar varchar(1))", List.of( @@ -158,8 +156,7 @@ public void testIsNotNullPredicatePushdown() { assertThat(query("SELECT nationkey FROM nation WHERE name IS NOT NULL OR regionkey = 4")).isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_is_not_null_predicate_pushdown", "(a_int integer, a_varchar varchar(1))", List.of( @@ -176,8 +173,7 @@ public void testNotExpressionPushdown() { assertThat(query("SELECT nationkey FROM nation WHERE NOT(name LIKE '%A%')")).isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_is_not_predicate_pushdown", "(a_int integer, a_varchar varchar(2))", List.of( @@ -332,8 +328,7 @@ public void testShowCreateTable() @Test public void testAvgDecimalExceedingSupportedPrecision() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_avg_decimal_exceeding_supported_precision", "(a decimal(38, 38), b bigint)", List.of( @@ -405,7 +400,7 @@ public void testDropAndAddColumnWithSameName() { // Override because Ignite can access old data after dropping and adding a column with same name executeExclusively(() -> { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_add_column", "AS SELECT 1 x, 2 y, 3 z")) { + try (TestTable table = newTrinoTable("test_drop_add_column", "AS SELECT 1 x, 2 y, 3 z")) { assertUpdate("ALTER TABLE " + table.getName() + " DROP COLUMN y"); assertQuery("SELECT * FROM " + table.getName(), "VALUES (1, 3)"); diff --git a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java index 6150f160a19d..ef506197672b 100644 --- a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java +++ b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestKuduConnectorTest.java @@ -523,8 +523,7 @@ public void testAddColumnWithComment() public void testAddColumnWithCommentSpecialCharacter(String comment) { // Override because Kudu connector doesn't support creating a new table without partition columns - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_add_col_", "(id INT WITH (primary_key=true), a_varchar varchar) WITH (partition_by_hash_columns = ARRAY['id'], partition_by_hash_buckets = 2)")) { assertUpdate("ALTER TABLE " + table.getName() + " ADD COLUMN b_varchar varchar COMMENT " + varcharLiteral(comment)); @@ -557,7 +556,7 @@ public void testAddColumnWithDecimal() @Test public void testInsertIntoTableHavingRowUuid() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_", " AS SELECT * FROM region WITH NO DATA")) { + try (TestTable table = newTrinoTable("test_insert_", " AS SELECT * FROM region WITH NO DATA")) { assertUpdate("INSERT INTO " + table.getName() + " SELECT * FROM region", 5); assertThat(query("SELECT * FROM " + table.getName())) @@ -570,7 +569,7 @@ public void testInsertIntoTableHavingRowUuid() public void testInsertUnicode() { // TODO Remove this overriding test once kudu connector can create tables with default partitions - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50) WITH (primary_key=true)) " + "WITH (partition_by_hash_columns = ARRAY['test'], partition_by_hash_buckets = 2)")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'Hello', U&'hello\\6d4B\\8Bd5world\\7F16\\7801' ", 2); @@ -578,7 +577,7 @@ public void testInsertUnicode() .containsExactlyInAnyOrder("Hello", "hello测试world编码"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50) WITH (primary_key=true)) " + "WITH (partition_by_hash_columns = ARRAY['test'], partition_by_hash_buckets = 2)")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'aa', 'bé'", 2); @@ -589,7 +588,7 @@ public void testInsertUnicode() assertQueryReturnsEmptyResult("SELECT test FROM " + table.getName() + " WHERE test = 'ba'"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50) WITH (primary_key=true)) " + "WITH (partition_by_hash_columns = ARRAY['test'], partition_by_hash_buckets = 2)")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'a', 'é'", 2); @@ -606,7 +605,7 @@ public void testInsertUnicode() public void testInsertHighestUnicodeCharacter() { // TODO Remove this overriding test once kudu connector can create tables with default partitions - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50) WITH (primary_key=true)) " + "WITH (partition_by_hash_columns = ARRAY['test'], partition_by_hash_buckets = 2)")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'Hello', U&'hello\\6d4B\\8Bd5\\+10FFFFworld\\7F16\\7801' ", 2); @@ -620,7 +619,7 @@ public void testInsertHighestUnicodeCharacter() public void testInsertNegativeDate() { // TODO Remove this overriding test once kudu connector can create tables with default partitions - try (TestTable table = new TestTable(getQueryRunner()::execute, "insert_date", + try (TestTable table = newTrinoTable("insert_date", "(dt DATE WITH (primary_key=true)) " + "WITH (partition_by_hash_columns = ARRAY['dt'], partition_by_hash_buckets = 2)")) { assertQueryFails(format("INSERT INTO %s VALUES (DATE '-0001-01-01')", table.getName()), errorMessageForInsertNegativeDate("-0001-01-01")); @@ -695,7 +694,7 @@ public void testDeleteWithLike() protected TestTable createTableWithOneIntegerColumn(String namePrefix) { // TODO Remove this overriding method once kudu connector can create tables with default partitions - return new TestTable(getQueryRunner()::execute, namePrefix, + return newTrinoTable(namePrefix, "(col integer WITH (primary_key=true)) " + "WITH (partition_by_hash_columns = ARRAY['col'], partition_by_hash_buckets = 2)"); } @@ -1027,7 +1026,7 @@ public void testCreateTableWithTableComment() protected void testCreateTableWithTableCommentSpecialCharacter(String comment) { // TODO Remove this overriding test once kudu connector can create tables with default partitions - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_create_", "(a bigint WITH (primary_key=true)) COMMENT " + varcharLiteral(comment) + "WITH (partition_by_hash_columns = ARRAY['a'], partition_by_hash_buckets = 2)")) { diff --git a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbConnectorTest.java b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbConnectorTest.java index 8c1bac7c0115..c299e9c9a510 100644 --- a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbConnectorTest.java +++ b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/BaseMariaDbConnectorTest.java @@ -190,7 +190,7 @@ public void testAddNotNullColumn() .isInstanceOf(AssertionError.class) .hasMessage("Should fail to add not null column without a default value to a non-empty table"); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_nn_col", "(a_varchar varchar)")) { + try (TestTable table = newTrinoTable("test_add_nn_col", "(a_varchar varchar)")) { String tableName = table.getName(); assertUpdate("INSERT INTO " + tableName + " VALUES ('a')", 1); @@ -269,7 +269,7 @@ public void testDeleteWithLike() @Override public void testInsertIntoNotNullColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "insert_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { + try (TestTable table = newTrinoTable("insert_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { assertUpdate(format("INSERT INTO %s (not_null_col) VALUES (2)", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (NULL, 2)"); assertQueryFails(format("INSERT INTO %s (nullable_col) VALUES (1)", table.getName()), errorMessageForInsertIntoNotNullColumn("not_null_col")); @@ -282,7 +282,7 @@ public void testInsertIntoNotNullColumn() assertQueryFails(format("INSERT INTO %s (nullable_col) SELECT nationkey FROM nation WHERE regionkey < 0", table.getName()), ".*Field 'not_null_col' doesn't have a default value.*"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "commuted_not_null", "(nullable_col BIGINT, not_null_col BIGINT NOT NULL)")) { + try (TestTable table = newTrinoTable("commuted_not_null", "(nullable_col BIGINT, not_null_col BIGINT NOT NULL)")) { assertUpdate(format("INSERT INTO %s (not_null_col) VALUES (2)", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (NULL, 2)"); // This is enforced by the engine and not the connector diff --git a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestMariaDbTypeMapping.java b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestMariaDbTypeMapping.java index 8ec8b95227c7..bdb1dd2e737a 100644 --- a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestMariaDbTypeMapping.java +++ b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestMariaDbTypeMapping.java @@ -627,7 +627,7 @@ private void testDate(ZoneId sessionZone) @Test public void testUnsupportedDate() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_negative_date", "(dt DATE)")) { + try (TestTable table = newTrinoTable("test_negative_date", "(dt DATE)")) { assertQueryFails(format("INSERT INTO %s VALUES (DATE '-0001-01-01')", table.getName()), ".*Failed to insert data.*"); assertQueryFails(format("INSERT INTO %s VALUES (DATE '10000-01-01')", table.getName()), ".*Failed to insert data.*"); } @@ -777,7 +777,7 @@ private void testTimestamp(ZoneId sessionZone) @Test public void testIncorrectTimestamp() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_incorrect_timestamp", "(dt TIMESTAMP)")) { + try (TestTable table = newTrinoTable("test_incorrect_timestamp", "(dt TIMESTAMP)")) { assertQueryFails(format("INSERT INTO %s VALUES (TIMESTAMP '1970-01-01 00:00:00.000')", table.getName()), ".*Failed to insert data.*"); assertQueryFails(format("INSERT INTO %s VALUES (TIMESTAMP '2038-01-19 03:14:08.000')", table.getName()), ".*Failed to insert data.*"); } diff --git a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java index abc54b106411..f1d71b812e1e 100644 --- a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java +++ b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java @@ -607,7 +607,7 @@ public void testRenameView() @Test void testInsertAfterTruncate() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_truncate", "AS SELECT 1 x")) { + try (TestTable table = newTrinoTable("test_truncate", "AS SELECT 1 x")) { assertUpdate("TRUNCATE TABLE " + table.getName()); assertQueryReturnsEmptyResult("SELECT * FROM " + table.getName()); diff --git a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/BaseMongoConnectorSmokeTest.java b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/BaseMongoConnectorSmokeTest.java index f39404a42ea0..b5de69af1d6d 100644 --- a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/BaseMongoConnectorSmokeTest.java +++ b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/BaseMongoConnectorSmokeTest.java @@ -42,8 +42,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) @Test public void testProjectionPushdown() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_projection_pushdown_multiple_rows_", "(id INT, nested1 ROW(child1 INT, child2 VARCHAR))", ImmutableList.of( @@ -60,8 +59,7 @@ public void testProjectionPushdown() @Test public void testReadDottedField() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_read_dotted_field_", "(root ROW(\"dotted.field\" VARCHAR, field VARCHAR))", ImmutableList.of("ROW(ROW('foo', 'bar'))"))) { @@ -76,8 +74,7 @@ public void testReadDottedField() @Test public void testReadDollarPrefixedField() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_read_dotted_field_", "(root ROW(\"$field1\" VARCHAR, field2 VARCHAR))", ImmutableList.of("ROW(ROW('foo', 'bar'))"))) { @@ -92,8 +89,7 @@ public void testReadDollarPrefixedField() @Test public void testProjectionPushdownWithHighlyNestedData() { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_projection_pushdown_highly_nested_data_", "(id INT, row1_t ROW(f1 INT, f2 INT, row2_t ROW (f1 INT, f2 INT, row3_t ROW(f1 INT, f2 INT))))", ImmutableList.of("(1, ROW(2, 3, ROW(4, 5, ROW(6, 7))))", diff --git a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java index fd85c4abb8a6..512da391977c 100644 --- a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java +++ b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java @@ -356,7 +356,7 @@ public void testPredicatePushdown() private void testPredicatePushdown(String value) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_predicate_pushdown", "AS SELECT %s col".formatted(value))) { + try (TestTable table = newTrinoTable("test_predicate_pushdown", "AS SELECT %s col".formatted(value))) { testPredicatePushdown(table.getName(), "col = " + value); testPredicatePushdown(table.getName(), "col != " + value); testPredicatePushdown(table.getName(), "col < " + value); @@ -380,7 +380,7 @@ public void testPredicatePushdownDoubleType() private void testPredicatePushdownFloatingPoint(String value) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_floating_point_pushdown", "AS SELECT %s col".formatted(value))) { + try (TestTable table = newTrinoTable("test_floating_point_pushdown", "AS SELECT %s col".formatted(value))) { assertThat(query("SELECT * FROM " + table.getName() + " WHERE col = " + value)) .isFullyPushedDown(); assertThat(query("SELECT * FROM " + table.getName() + " WHERE col <= " + value)) @@ -402,8 +402,7 @@ private void testPredicatePushdownFloatingPoint(String value) @Test public void testPredicatePushdownCharWithPaddedSpace() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_predicate_pushdown_char_with_padded_space", "(k, v) AS VALUES" + " (-1, CAST(NULL AS char(3))), " + @@ -436,8 +435,7 @@ public void testPredicatePushdownCharWithPaddedSpace() public void testPredicatePushdownMultipleNotEquals() { // Regression test for https://github.com/trinodb/trino/issues/19404 - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_predicate_pushdown_with_multiple_not_equals", "(id, value) AS VALUES (1, 10), (2, 20), (3, 30)")) { assertThat(query("SELECT * FROM " + table.getName() + " WHERE id != 1 AND value != 20")) @@ -449,8 +447,7 @@ public void testPredicatePushdownMultipleNotEquals() @Test public void testHighPrecisionDecimalPredicate() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_high_precision_decimal_predicate", "(col DECIMAL(34, 0))", Arrays.asList("decimal '3141592653589793238462643383279502'", null))) { @@ -1354,8 +1351,7 @@ private void testFiltersOnDereferenceColumnReadsLessData(String expectedValue, S .setCatalogSessionProperty(getSession().getCatalog().orElseThrow(), "projection_pushdown_enabled", "false") .build(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "filter_on_projection_columns", format("(col_0 ROW(col_1 %1$s, col_2 ROW(col_3 %1$s, col_4 ROW(col_5 %1$s))))", expectedType))) { assertUpdate(format("INSERT INTO %s VALUES NULL", table.getName()), 1); diff --git a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoTypeMapping.java b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoTypeMapping.java index 5d98edd7db21..b3346797f3fb 100644 --- a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoTypeMapping.java +++ b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoTypeMapping.java @@ -398,7 +398,7 @@ public void testArray() public void testArrayNulls() { // Verify only SELECT instead of using SqlDataTypeTest because array comparison not supported for arrays with null elements - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_array_nulls", "(c1 ARRAY(boolean), c2 ARRAY(varchar), c3 ARRAY(varchar))", ImmutableList.of("(NULL, ARRAY[NULL], ARRAY['foo', NULL, 'bar', NULL])"))) { + try (TestTable table = newTrinoTable("test_array_nulls", "(c1 ARRAY(boolean), c2 ARRAY(varchar), c3 ARRAY(varchar))", ImmutableList.of("(NULL, ARRAY[NULL], ARRAY['foo', NULL, 'bar', NULL])"))) { assertThat(query("SELECT c1 FROM " + table.getName())).matches("VALUES CAST(NULL AS ARRAY(boolean))"); assertThat(query("SELECT c2 FROM " + table.getName())).matches("VALUES CAST(ARRAY[NULL] AS ARRAY(varchar))"); assertThat(query("SELECT c3 FROM " + table.getName())).matches("VALUES CAST(ARRAY['foo', NULL, 'bar', NULL] AS ARRAY(varchar))"); diff --git a/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlConnectorTest.java b/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlConnectorTest.java index 31e3801153fb..d030e98a11fa 100644 --- a/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlConnectorTest.java +++ b/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/BaseMySqlConnectorTest.java @@ -271,7 +271,7 @@ public void testAddNotNullColumn() .isInstanceOf(AssertionError.class) .hasMessage("Should fail to add not null column without a default value to a non-empty table"); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_nn_col", "(a_varchar varchar)")) { + try (TestTable table = newTrinoTable("test_add_nn_col", "(a_varchar varchar)")) { String tableName = table.getName(); assertUpdate("INSERT INTO " + tableName + " VALUES ('a')", 1); diff --git a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/AbstractTestOracleTypeMapping.java b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/AbstractTestOracleTypeMapping.java index ba076101d7b4..c70e80df5d53 100644 --- a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/AbstractTestOracleTypeMapping.java +++ b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/AbstractTestOracleTypeMapping.java @@ -692,7 +692,7 @@ private void testDate(ZoneId sessionZone) public void testJulianGregorianDate() { // Oracle TO_DATE function returns +10 days during julian and gregorian calendar switch - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_julian_dt", "(ts date)")) { + try (TestTable table = newTrinoTable("test_julian_dt", "(ts date)")) { assertUpdate(format("INSERT INTO %s VALUES (DATE '1582-10-05')", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES TIMESTAMP '1582-10-15 00:00:00'"); } @@ -701,7 +701,7 @@ public void testJulianGregorianDate() @Test public void testUnsupportedDate() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_unsupported_dt", "(ts date)")) { + try (TestTable table = newTrinoTable("test_unsupported_dt", "(ts date)")) { assertQueryFails( format("INSERT INTO %s VALUES (DATE '-4713-12-31')", table.getName()), """ @@ -960,7 +960,7 @@ public void testTimestampAllPrecisionsOnOracle() public void testJulianGregorianTimestamp() { // Oracle TO_DATE function returns +10 days during julian and gregorian calendar switch - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_julian_ts", "(ts date)")) { + try (TestTable table = newTrinoTable("test_julian_ts", "(ts date)")) { assertUpdate(format("INSERT INTO %s VALUES (timestamp '1582-10-05')", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES TIMESTAMP '1582-10-15 00:00:00'"); } @@ -969,7 +969,7 @@ public void testJulianGregorianTimestamp() @Test public void testUnsupportedTimestamp() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_unsupported_ts", "(ts timestamp)")) { + try (TestTable table = newTrinoTable("test_unsupported_ts", "(ts timestamp)")) { assertQueryFails( format("INSERT INTO %s VALUES (TIMESTAMP '-4713-12-31 00:00:00.000')", table.getName()), """ diff --git a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java index b85473807bf6..435a0fc2f07f 100644 --- a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java +++ b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java @@ -198,8 +198,7 @@ public void testCharVarcharComparison() { // test overridden because super uses all-space char values (' ') that are null-out by Oracle - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_char_varchar", "(k, v) AS VALUES" + " (-1, CAST(NULL AS char(3))), " + @@ -222,8 +221,7 @@ public void testVarcharCharComparison() { // test overridden because Oracle nulls-out '' varchar value, impacting results - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_varchar_char", "(k, v) AS VALUES" + " (-1, CAST(NULL AS varchar(3))), " + @@ -500,8 +498,8 @@ private void predicatePushdownTest(String oracleType, String oracleLiteral, Stri @Test public void testJoinPushdownWithImplicitCast() { - try (TestTable leftTable = new TestTable(getQueryRunner()::execute, "left_table", "(id int, varchar_50 varchar(50))", ImmutableList.of("(1, 'India')", "(2, 'Poland')")); - TestTable rightTable = new TestTable(getQueryRunner()::execute, "right_table_", "(varchar_100 varchar(100), varchar_unbounded varchar)", ImmutableList.of("('India', 'Japan')", "('France', 'Poland')"))) { + try (TestTable leftTable = newTrinoTable("left_table", "(id int, varchar_50 varchar(50))", ImmutableList.of("(1, 'India')", "(2, 'Poland')")); + TestTable rightTable = newTrinoTable("right_table_", "(varchar_100 varchar(100), varchar_unbounded varchar)", ImmutableList.of("('India', 'Japan')", "('France', 'Poland')"))) { String leftTableName = leftTable.getName(); String rightTableName = rightTable.getName(); Session session = joinPushdownEnabled(getSession()); diff --git a/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixConnectorTest.java b/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixConnectorTest.java index 4797960ab08d..b34a36913e9f 100644 --- a/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixConnectorTest.java +++ b/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixConnectorTest.java @@ -335,8 +335,7 @@ public void testCharVarcharComparison() { // test overridden because super uses all-space char values (' ') that are null-out by Phoenix - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_char_varchar", "(k, v) AS VALUES" + " (-1, CAST(NULL AS char(3))), " + @@ -359,8 +358,7 @@ public void testVarcharCharComparison() { // test overridden because Phoenix nulls-out '' varchar value, impacting results - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_varchar_char", "(k, v) AS VALUES" + " (-1, CAST(NULL AS varchar(3))), " + @@ -409,7 +407,7 @@ public void testCountDistinctWithStringTypes() .collect(toImmutableList()); String tableName = "count_distinct_strings" + randomNameSuffix(); - try (TestTable testTable = new TestTable(getQueryRunner()::execute, tableName, "(id int, t_char CHAR(5), t_varchar VARCHAR(5)) WITH (ROWKEYS='id')", rows)) { + try (TestTable testTable = newTrinoTable(tableName, "(id int, t_char CHAR(5), t_varchar VARCHAR(5)) WITH (ROWKEYS='id')", rows)) { assertQuery("SELECT count(DISTINCT t_varchar) FROM " + testTable.getName(), "VALUES 6"); assertQuery("SELECT count(DISTINCT t_char) FROM " + testTable.getName(), "VALUES 6"); assertQuery("SELECT count(DISTINCT t_char), count(DISTINCT t_varchar) FROM " + testTable.getName(), "VALUES (6, 6)"); diff --git a/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixTypeMapping.java b/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixTypeMapping.java index 19b9a0879327..fc59390ded20 100644 --- a/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixTypeMapping.java +++ b/plugin/trino-phoenix5/src/test/java/io/trino/plugin/phoenix5/TestPhoenixTypeMapping.java @@ -715,7 +715,7 @@ public void testArray() public void testArrayNulls() { // Verify only SELECT instead of using SqlDataTypeTest because array comparison not supported for arrays with null elements - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_array_nulls", "(c1 ARRAY(boolean), c2 ARRAY(varchar), c3 ARRAY(varchar))", ImmutableList.of("(NULL, ARRAY[NULL], ARRAY['foo', NULL, 'bar', NULL])"))) { + try (TestTable table = newTrinoTable("test_array_nulls", "(c1 ARRAY(boolean), c2 ARRAY(varchar), c3 ARRAY(varchar))", ImmutableList.of("(NULL, ARRAY[NULL], ARRAY['foo', NULL, 'bar', NULL])"))) { assertThat(query("SELECT c1 FROM " + table.getName())).matches("VALUES CAST(NULL AS ARRAY(boolean))"); assertThat(query("SELECT c2 FROM " + table.getName())).matches("VALUES CAST(ARRAY[NULL] AS ARRAY(varchar))"); assertThat(query("SELECT c3 FROM " + table.getName())).matches("VALUES CAST(ARRAY['foo', NULL, 'bar', NULL] AS ARRAY(varchar))"); diff --git a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java index 4e39f890ab86..03e3389798eb 100644 --- a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java +++ b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java @@ -194,8 +194,7 @@ public void testTimestampPrecisionOnCreateTable() private void testTimestampPrecisionOnCreateTable(String inputType, String expectedType) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_show_create_table", format("(a %s)", inputType))) { assertThat(getColumnType(testTable.getName(), "a")).isEqualTo(expectedType); @@ -234,8 +233,7 @@ public void testTimestampPrecisionOnCreateTableAsSelect() private void testTimestampPrecisionOnCreateTableAsSelect(String inputType, String tableType, String tableValue) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_show_create_table", format("AS SELECT %s a", inputType))) { assertThat(getColumnType(testTable.getName(), "a")).isEqualTo(tableType); @@ -277,8 +275,7 @@ public void testTimestampPrecisionOnCreateTableAsSelectWithNoData() private void testTimestampPrecisionOnCreateTableAsSelectWithNoData(String inputType, String tableType) { - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_coercion_show_create_table", format("AS SELECT %s a WITH NO DATA", inputType))) { assertThat(getColumnType(testTable.getName(), "a")).isEqualTo(tableType); @@ -833,8 +830,7 @@ public void testLikePredicatePushdown() assertThat(query("SELECT nationkey FROM nation WHERE name LIKE '%A%'")) .isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_like_predicate_pushdown", "(id integer, a_varchar varchar(1))", List.of( @@ -856,8 +852,7 @@ public void testLikeWithEscapePredicatePushdown() assertThat(query("SELECT nationkey FROM nation WHERE name LIKE '%A%' ESCAPE '\\'")) .isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_like_with_escape_predicate_pushdown", "(id integer, a_varchar varchar(4))", List.of( @@ -878,8 +873,7 @@ public void testIsNullPredicatePushdown() assertThat(query("SELECT nationkey FROM nation WHERE name IS NULL")).isFullyPushedDown(); assertThat(query("SELECT nationkey FROM nation WHERE name IS NULL OR regionkey = 4")).isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_is_null_predicate_pushdown", "(a_int integer, a_varchar varchar(1))", List.of( @@ -896,8 +890,7 @@ public void testIsNotNullPredicatePushdown() { assertThat(query("SELECT nationkey FROM nation WHERE name IS NOT NULL OR regionkey = 4")).isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_is_not_null_predicate_pushdown", "(a_int integer, a_varchar varchar(1))", List.of( @@ -935,8 +928,7 @@ public void testNotExpressionPushdown() { assertThat(query("SELECT nationkey FROM nation WHERE NOT(name LIKE '%A%' ESCAPE '\\')")).isFullyPushedDown(); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_is_not_predicate_pushdown", "(a_int integer, a_varchar varchar(2))", List.of( @@ -952,8 +944,7 @@ public void testNotExpressionPushdown() @Test public void testInPredicatePushdown() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_in_predicate_pushdown", "(id varchar(1), id2 varchar(1))", List.of( @@ -1095,8 +1086,7 @@ public void testTimestampColumnAndTimestampWithTimeZoneConstant() @Test public void testReverseFunctionProjectionPushDown() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_reverse_pushdown_for_project", "(id BIGINT, varchar_col VARCHAR)", ImmutableList.of("1, 'abc'", "2, null"))) { @@ -1151,8 +1141,7 @@ public void testReverseFunctionProjectionPushDown() @Test public void testPartialProjectionPushDown() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_partial_projection_pushdown", "(id BIGINT, cola VARCHAR, colb VARCHAR)", ImmutableList.of("1, 'abc', 'def'"))) { @@ -1202,8 +1191,7 @@ public void testPartialProjectionPushDown() @Test public void testProjectionsNotPushDownWhenFilterAppliedOnProjectedColumn() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_projection_push_down_with_filter", "(id BIGINT, cola VARCHAR, colb VARCHAR)", ImmutableList.of("1, 'abc', 'def'"))) { diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftCastPushdown.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftCastPushdown.java index 7d86e743e085..5c1dabe79353 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftCastPushdown.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftCastPushdown.java @@ -579,8 +579,7 @@ protected List invalidCast() @Test void testCastPushdownWithCharConvertedToVarchar() { - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( TEST_SCHEMA + "." + "char_converted_to_varchar_", "(a char(4097))", // char(REDSHIFT_MAX_CHAR` + 1) in Trino is mapped to varchar(REDSHIFT_MAX_CHAR` + 1) in Redshift ImmutableList.of("'hello'"))) { diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConnectorTest.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConnectorTest.java index 29f53a68514a..df7eb44513ff 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConnectorTest.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConnectorTest.java @@ -273,7 +273,7 @@ private void testReadNullFromView(String redshiftType, String trinoType, boolean @Test public void testRedshiftAddNotNullColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, TEST_SCHEMA + ".test_add_column_", "(col int)")) { + try (TestTable table = newTrinoTable(TEST_SCHEMA + ".test_add_column_", "(col int)")) { assertThatThrownBy(() -> onRemoteDatabase().execute("ALTER TABLE " + table.getName() + " ADD COLUMN new_col int NOT NULL")) .hasMessageContaining("ERROR: ALTER TABLE ADD COLUMN defined as NOT NULL must have a non-null default expression"); } @@ -316,7 +316,7 @@ public void testRangeQueryConvertedToInClauseQuery() public void testDelete() { // The base tests is very slow because Redshift CTAS is really slow, so use a smaller test - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_delete_", "AS SELECT * FROM nation")) { // delete without matching any rows assertUpdate("DELETE FROM " + table.getName() + " WHERE nationkey < 0", 0); @@ -449,7 +449,7 @@ public void testCountDistinctWithStringTypes() .collect(toImmutableList()); String tableName = "distinct_strings" + randomNameSuffix(); - try (TestTable testTable = new TestTable(getQueryRunner()::execute, tableName, "(t_char CHAR(5), t_varchar VARCHAR(5))", rows)) { + try (TestTable testTable = newTrinoTable(tableName, "(t_char CHAR(5), t_varchar VARCHAR(5))", rows)) { // Single count(DISTINCT ...) can be pushed even down even if SUPPORTS_AGGREGATION_PUSHDOWN_COUNT_DISTINCT == false as GROUP BY assertThat(query("SELECT count(DISTINCT t_varchar) FROM " + testTable.getName())) .matches("VALUES BIGINT '6'") @@ -661,7 +661,7 @@ public void testDecimalAvgPushdownForMaximumDecimalScale() "12345789.9876543210", format("%s.%s", "1".repeat(28), "9".repeat(10))); - try (TestTable testTable = new TestTable(getQueryRunner()::execute, TEST_SCHEMA + ".test_agg_pushdown_avg_max_decimal", + try (TestTable testTable = newTrinoTable(TEST_SCHEMA + ".test_agg_pushdown_avg_max_decimal", "(t_decimal DECIMAL(38, 10))", rows)) { // Redshift avg rounds down decimal result which doesn't match Presto semantics assertThatThrownBy(() -> assertThat(query("SELECT avg(t_decimal) FROM " + testTable.getName())).isFullyPushedDown()) @@ -683,7 +683,7 @@ public void testDecimalAvgPushdownFoShortDecimalScale() "0.987654321234567890", format("0.%s", "1".repeat(18))); - try (TestTable testTable = new TestTable(getQueryRunner()::execute, TEST_SCHEMA + ".test_agg_pushdown_avg_max_decimal", + try (TestTable testTable = newTrinoTable(TEST_SCHEMA + ".test_agg_pushdown_avg_max_decimal", "(t_decimal DECIMAL(18, 18))", rows)) { assertThat(query("SELECT avg(t_decimal) FROM " + testTable.getName())).isFullyPushedDown(); } @@ -699,15 +699,13 @@ public void testInsertRowConcurrently() @Test public void testJoinPushdownWithImplicitCast() { - try (TestTable leftTable = new TestTable( - getQueryRunner()::execute, + try (TestTable leftTable = newTrinoTable( "left_table_", "(id int, c_boolean boolean, c_tinyint tinyint, c_smallint smallint, c_integer integer, c_bigint bigint, c_real real, c_double_precision double precision, c_decimal_10_2 decimal(10, 2), c_varchar_50 varchar(50))", ImmutableList.of( "(11, true, 12, 12, 12, 12, 12.34, 12.34, 12.34, 'India')", "(12, false, 123, 123, 123, 123, 123.67, 123.67, 123.67, 'Poland')")); - TestTable rightTable = new TestTable( - getQueryRunner()::execute, + TestTable rightTable = newTrinoTable( "right_table_", "(id int, c_boolean boolean, c_tinyint tinyint, c_smallint smallint, c_integer integer, c_bigint bigint, c_real real, c_double_precision double precision, c_decimal_10_2 decimal(10, 2), c_varchar_100 varchar(100), c_varchar varchar)", ImmutableList.of( diff --git a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreConnectorTest.java b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreConnectorTest.java index 6ba379deed51..dede72d518ee 100644 --- a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreConnectorTest.java +++ b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreConnectorTest.java @@ -207,7 +207,7 @@ public void testSingleStoreTinyint() @Override public void testInsertIntoNotNullColumn() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "insert_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { + try (TestTable table = newTrinoTable("insert_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { assertUpdate(format("INSERT INTO %s (not_null_col) VALUES (2)", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (NULL, 2)"); assertQueryFails(format("INSERT INTO %s (nullable_col) VALUES (1)", table.getName()), errorMessageForInsertIntoNotNullColumn("not_null_col")); @@ -220,7 +220,7 @@ public void testInsertIntoNotNullColumn() assertQueryFails(format("INSERT INTO %s (nullable_col) SELECT nationkey FROM nation WHERE regionkey < 0", table.getName()), ".*Field 'not_null_col' doesn't have a default value.*"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "commuted_not_null", "(nullable_col BIGINT, not_null_col BIGINT NOT NULL)")) { + try (TestTable table = newTrinoTable("commuted_not_null", "(nullable_col BIGINT, not_null_col BIGINT NOT NULL)")) { assertUpdate(format("INSERT INTO %s (not_null_col) VALUES (2)", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (NULL, 2)"); // This is enforced by the engine and not the connector @@ -256,7 +256,7 @@ public void testAddNotNullColumn() .isInstanceOf(AssertionError.class) .hasMessage("Should fail to add not null column without a default value to a non-empty table"); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_nn_col", "(a_varchar varchar)")) { + try (TestTable table = newTrinoTable("test_add_nn_col", "(a_varchar varchar)")) { String tableName = table.getName(); assertUpdate("INSERT INTO " + tableName + " VALUES ('a')", 1); diff --git a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreLatestTypeMapping.java b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreLatestTypeMapping.java index db43725c72ea..447ff14babe2 100644 --- a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreLatestTypeMapping.java +++ b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestSingleStoreLatestTypeMapping.java @@ -37,7 +37,7 @@ protected QueryRunner createQueryRunner() @Test void testUnsupportedTinyint() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "tpch.test_unsupported_tinyint", "(value tinyint)")) { + try (TestTable table = newTrinoTable("tpch.test_unsupported_tinyint", "(value tinyint)")) { assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (-129)", table.getName()))) .hasMessageContaining("Out of range value"); assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (128)", table.getName()))) @@ -48,7 +48,7 @@ void testUnsupportedTinyint() @Test void testUnsupportedSmallint() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "tpch.test_unsupported_smallint", "(value smallint)")) { + try (TestTable table = newTrinoTable("tpch.test_unsupported_smallint", "(value smallint)")) { assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (-32769)", table.getName()))) .hasMessageContaining("Out of range value"); assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (32768)", table.getName()))) @@ -59,7 +59,7 @@ void testUnsupportedSmallint() @Test void testUnsupportedInteger() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "tpch.test_unsupported_integer", "(value integer)")) { + try (TestTable table = newTrinoTable("tpch.test_unsupported_integer", "(value integer)")) { assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (-2147483649)", table.getName()))) .hasMessageContaining("Out of range value"); assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (2147483648)", table.getName()))) @@ -70,7 +70,7 @@ void testUnsupportedInteger() @Test void testUnsupportedBigint() { - try (TestTable table = new TestTable(getQueryRunner()::execute, "tpch.test_unsupported_bigint", "(value bigint)")) { + try (TestTable table = newTrinoTable("tpch.test_unsupported_bigint", "(value bigint)")) { assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (-9223372036854775809)", table.getName()))) .hasMessageContaining("Out of range value"); assertThatThrownBy(() -> singleStoreServer.execute(format("INSERT INTO %s VALUES (9223372036854775808)", table.getName()))) diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java index 903c8b9c523e..ec67c6c1798d 100644 --- a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java @@ -363,7 +363,7 @@ public void testDeleteWithVarcharInequalityPredicate() // Override this because by enabling this flag SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY, // we assume that we also support range pushdowns, but for now we only support 'not equal' pushdown, // so cannot enable this flag for now - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_varchar", "(col varchar(1))", ImmutableList.of("'a'", "'A'", "null"))) { + try (TestTable table = newTrinoTable("test_delete_varchar", "(col varchar(1))", ImmutableList.of("'a'", "'A'", "null"))) { assertUpdate("DELETE FROM " + table.getName() + " WHERE col != 'A'", 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES 'A', null"); } @@ -834,7 +834,7 @@ THEN INSERT(id) VALUES(SOURCE.id) public void testConstantUpdateWithVarcharInequalityPredicates() { // Sql Server supports push down predicate for not equal operator - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_update_varchar", "(col1 INT, col2 varchar(1))", ImmutableList.of("1, 'a'", "2, 'A'"))) { + try (TestTable table = newTrinoTable("test_update_varchar", "(col1 INT, col2 varchar(1))", ImmutableList.of("1, 'a'", "2, 'A'"))) { assertUpdate("UPDATE " + table.getName() + " SET col1 = 20 WHERE col2 != 'A'", 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (20, 'a'), (2, 'A')"); } diff --git a/plugin/trino-vertica/src/test/java/io/trino/plugin/vertica/TestVerticaConnectorTest.java b/plugin/trino-vertica/src/test/java/io/trino/plugin/vertica/TestVerticaConnectorTest.java index d65709629852..fb38c381de52 100644 --- a/plugin/trino-vertica/src/test/java/io/trino/plugin/vertica/TestVerticaConnectorTest.java +++ b/plugin/trino-vertica/src/test/java/io/trino/plugin/vertica/TestVerticaConnectorTest.java @@ -125,9 +125,8 @@ private void testJoinPushdown(JoinOperator joinOperator) Stream.of(notDistinctOperator)) .collect(toImmutableList()); - try (TestTable nationLowercaseTable = new TestTable( + try (TestTable nationLowercaseTable = newTrinoTable( // If a connector supports Join pushdown, but does not allow CTAS, we need to make the table creation here overridable. - getQueryRunner()::execute, "nation_lowercase", "AS SELECT nationkey, lower(name) name, regionkey FROM nation")) { // basic case diff --git a/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/hive/TestHiveFaultTolerantExecutionCoordinatorExcludedTest.java b/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/hive/TestHiveFaultTolerantExecutionCoordinatorExcludedTest.java index 39cfa116bc6d..34448e4ef58a 100644 --- a/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/hive/TestHiveFaultTolerantExecutionCoordinatorExcludedTest.java +++ b/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/hive/TestHiveFaultTolerantExecutionCoordinatorExcludedTest.java @@ -67,7 +67,7 @@ public void testInsert() { String query = "SELECT name, nationkey, regionkey FROM nation"; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_", "AS " + query + " WITH NO DATA")) { + try (TestTable table = newTrinoTable("test_insert_", "AS " + query + " WITH NO DATA")) { assertQuery("SELECT count(*) FROM " + table.getName(), "SELECT 0"); assertUpdate("INSERT INTO " + table.getName() + " " + query, 25); diff --git a/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java b/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java index 2517e923d4d1..ad314ce27967 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java @@ -13,6 +13,7 @@ */ package io.trino.testing; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.MoreCollectors; import com.google.errorprone.annotations.CanIgnoreReturnValue; @@ -58,6 +59,7 @@ import io.trino.sql.tree.ExplainType; import io.trino.testing.QueryRunner.MaterializedResultWithPlan; import io.trino.testing.TestingAccessControlManager.TestingPrivilege; +import io.trino.testing.sql.TestTable; import org.assertj.core.api.AssertProvider; import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.AfterAll; @@ -667,6 +669,16 @@ private Optional tryGetDistributedQueryRunner() return Optional.empty(); } + protected TestTable newTrinoTable(String namePrefix, @Language("SQL") String tableDefinition) + { + return newTrinoTable(namePrefix, tableDefinition, ImmutableList.of()); + } + + protected TestTable newTrinoTable(String namePrefix, @Language("SQL") String tableDefinition, List rowsToInsert) + { + return new TestTable(getQueryRunner()::execute, namePrefix, tableDefinition, rowsToInsert); + } + protected Session noJoinReordering() { return noJoinReordering(JoinDistributionType.PARTITIONED); diff --git a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java index b491e6ab375c..dfc875d2694a 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java @@ -189,7 +189,7 @@ public void testInsert() throw new AssertionError("Cannot test INSERT without CREATE TABLE, the test needs to be implemented in a connector-specific way"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_", getCreateTableDefaultDefinition())) { + try (TestTable table = newTrinoTable("test_insert_", getCreateTableDefaultDefinition())) { assertUpdate("INSERT INTO " + table.getName() + " (a, b) VALUES (42, -38.5), (13, 99.9)", 2); assertThat(query("SELECT CAST(a AS bigint), b FROM " + table.getName())) .matches(expectedValues("(42, -38.5), (13, 99.9)")); @@ -205,7 +205,7 @@ public void verifySupportsDeleteDeclaration() } assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_delete", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_supports_delete", "AS SELECT * FROM region")) { assertQueryFails("DELETE FROM " + table.getName(), MODIFYING_ROWS_MESSAGE); } } @@ -219,7 +219,7 @@ public void verifySupportsRowLevelDeleteDeclaration() } assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_row_level_delete", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_supports_row_level_delete", "AS SELECT * FROM region")) { assertQueryFails("DELETE FROM " + table.getName() + " WHERE regionkey = 2", MODIFYING_ROWS_MESSAGE); } } @@ -233,7 +233,7 @@ public void verifySupportsUpdateDeclaration() } assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_supports_update", "AS SELECT * FROM nation")) { assertQueryFails("UPDATE " + table.getName() + " SET nationkey = 100 WHERE regionkey = 2", MODIFYING_ROWS_MESSAGE); } } @@ -247,7 +247,7 @@ public void verifySupportsRowLevelUpdateDeclaration() } assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_row_update", "AS SELECT * FROM nation")) { assertQueryFails("UPDATE " + table.getName() + " SET nationkey = nationkey * 100 WHERE regionkey = 2", MODIFYING_ROWS_MESSAGE); } } @@ -256,7 +256,7 @@ public void verifySupportsRowLevelUpdateDeclaration() public void testUpdate() { assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_UPDATE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_row_update", "AS SELECT * FROM nation")) { assertUpdate("UPDATE " + table.getName() + " SET nationkey = 100 WHERE regionkey = 2", 5); assertQuery("SELECT count(*) FROM " + table.getName() + " WHERE nationkey = 100", "VALUES 5"); } @@ -266,7 +266,7 @@ public void testUpdate() public void testDeleteAllDataFromTable() { assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_DELETE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_all_data", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_delete_all_data", "AS SELECT * FROM region")) { // not using assertUpdate as some connectors provide update count and some do not getQueryRunner().execute("DELETE FROM " + table.getName()); assertQuery("SELECT count(*) FROM " + table.getName(), "VALUES 0"); @@ -278,7 +278,7 @@ public void testRowLevelDelete() { assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_delete", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_row_delete", "AS SELECT * FROM region")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE regionkey = 2", 1); assertThat(query("SELECT * FROM " + table.getName() + " WHERE regionkey = 2")) .returnsEmptyResult(); @@ -298,7 +298,7 @@ public void testTruncateTable() assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_truncate", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_truncate", "AS SELECT * FROM region")) { assertUpdate("TRUNCATE TABLE " + table.getName()); assertThat(query("TABLE " + table.getName())) .returnsEmptyResult(); @@ -318,7 +318,7 @@ public void testRowLevelUpdate() throw new AssertionError("Cannot test UPDATE without INSERT"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_update_", getCreateTableDefaultDefinition())) { + try (TestTable table = newTrinoTable("test_update_", getCreateTableDefaultDefinition())) { assertUpdate("INSERT INTO " + table.getName() + " (a, b) SELECT regionkey, regionkey * 2.5 FROM region", "SELECT count(*) FROM region"); assertThat(query("SELECT a, b FROM " + table.getName())) .matches(expectedValues("(0, 0.0), (1, 2.5), (2, 5.0), (3, 7.5), (4, 10.0)")); @@ -344,7 +344,7 @@ public void testMerge() throw new AssertionError("Cannot test MERGE without INSERT"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_merge_", getCreateTableDefaultDefinition())) { + try (TestTable table = newTrinoTable("test_merge_", getCreateTableDefaultDefinition())) { assertUpdate("INSERT INTO " + table.getName() + " (a, b) SELECT regionkey, regionkey * 2.5 FROM region", "SELECT count(*) FROM region"); assertThat(query("SELECT a, b FROM " + table.getName())) .matches(expectedValues("(0, 0.0), (1, 2.5), (2, 5.0), (3, 7.5), (4, 10.0)")); diff --git a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java index c9fbbc3bffa3..1cba708dc25f 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java @@ -355,8 +355,7 @@ public void testCharVarcharComparison() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_char_varchar", "(k, v) AS VALUES" + " (-1, CAST(NULL AS char(3))), " + @@ -389,8 +388,7 @@ public void testVarcharCharComparison() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_varchar_char", "(k, v) AS VALUES" + " (-1, CAST(NULL AS varchar(3))), " + @@ -589,8 +587,7 @@ public void testVarcharCastToDateInPredicate() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "varchar_as_date_pred", "(a varchar)", List.of( @@ -611,8 +608,7 @@ public void testVarcharCastToDateInPredicate() } } - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "varchar_as_date_pred", "(a varchar)", List.of("'2005-06-bad-date'", "'2005-09-10'"))) { @@ -642,8 +638,7 @@ public void testVarcharCastToDateInPredicate() failureAssert -> failureAssert .hasMessage("Value cannot be cast to date: 2005-06-bad-date")); } - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "varchar_as_date_pred", "(a varchar)", List.of("'2005-09-10'"))) { @@ -1300,8 +1295,7 @@ public void testMaterializedViewGracePeriod() return; } - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_base_table", "AS TABLE region")) { Session defaultSession = getSession(); @@ -2277,7 +2271,7 @@ public void testAddColumn() } String tableName; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_column_", tableDefinitionForAddColumn())) { + try (TestTable table = newTrinoTable("test_add_column_", tableDefinitionForAddColumn())) { tableName = table.getName(); assertUpdate("INSERT INTO " + table.getName() + " SELECT 'first'", 1); assertQueryFails("ALTER TABLE " + table.getName() + " ADD COLUMN x bigint", ".* Column 'x' already exists"); @@ -2337,7 +2331,7 @@ public void testAddColumnWithComment() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_col_desc_", "(a_varchar varchar)")) { + try (TestTable table = newTrinoTable("test_add_col_desc_", "(a_varchar varchar)")) { String tableName = table.getName(); assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN b_varchar varchar COMMENT 'test new column comment'"); @@ -2353,7 +2347,7 @@ public void testAddNotNullColumnToEmptyTable() { skipTestUnless(hasBehavior(SUPPORTS_ADD_COLUMN)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_nn_to_empty", "(a_varchar varchar)")) { + try (TestTable table = newTrinoTable("test_add_nn_to_empty", "(a_varchar varchar)")) { String tableName = table.getName(); String addNonNullColumn = "ALTER TABLE " + tableName + " ADD COLUMN b_varchar varchar NOT NULL"; @@ -2380,7 +2374,7 @@ public void testAddNotNullColumn() { skipTestUnless(hasBehavior(SUPPORTS_ADD_COLUMN_NOT_NULL_CONSTRAINT)); // covered by testAddNotNullColumnToEmptyTable - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_nn_col", "(a_varchar varchar)")) { + try (TestTable table = newTrinoTable("test_add_nn_col", "(a_varchar varchar)")) { String tableName = table.getName(); assertUpdate("INSERT INTO " + tableName + " VALUES ('a')", 1); @@ -2424,7 +2418,7 @@ public void testAddRowField() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ROW_TYPE)); if (!hasBehavior(SUPPORTS_ADD_FIELD)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_field_", "AS SELECT CAST(row(1) AS row(x integer)) AS col")) { + try (TestTable table = newTrinoTable("test_add_field_", "AS SELECT CAST(row(1) AS row(x integer)) AS col")) { assertQueryFails( "ALTER TABLE " + table.getName() + " ADD COLUMN col.y integer", "This connector does not support adding fields"); @@ -2432,7 +2426,7 @@ public void testAddRowField() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_add_field_", "AS SELECT CAST(row(1, row(10)) AS row(a integer, b row(x integer))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(a integer, b row(x integer))"); @@ -2462,7 +2456,7 @@ public void testAddRowFieldInArray() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ROW_TYPE)); if (!hasBehavior(SUPPORTS_ADD_FIELD_IN_ARRAY)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_field_in_array_", "AS SELECT CAST(array[row(1)] AS array(row(x integer))) AS col")) { + try (TestTable table = newTrinoTable("test_add_field_in_array_", "AS SELECT CAST(array[row(1)] AS array(row(x integer))) AS col")) { assertQueryFails( "ALTER TABLE " + table.getName() + " ADD COLUMN col.element.y integer", ".*does not support.*"); @@ -2470,7 +2464,7 @@ public void testAddRowFieldInArray() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_add_field_in_array_", "AS SELECT CAST(array[row(1, row(10), array[row(11)])] AS array(row(a integer, b row(x integer), c array(row(v integer))))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("array(row(a integer, b row(x integer), c array(row(v integer))))"); @@ -2506,7 +2500,7 @@ public void testAddRowFieldInArray() } // test row in array of arrays - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_add_field_in_array_nested_", "AS SELECT CAST(array[array[row(1, row(10), array[row(11)])]] AS array(array(row(a integer, b row(x integer), c array(row(v integer)))))) AS col")) { @@ -2537,7 +2531,7 @@ public void testDropColumn() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); String tableName; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_column_", "AS SELECT 123 x, 456 y, 111 a")) { + try (TestTable table = newTrinoTable("test_drop_column_", "AS SELECT 123 x, 456 y, 111 a")) { tableName = table.getName(); assertUpdate("ALTER TABLE " + tableName + " DROP COLUMN x"); assertUpdate("ALTER TABLE " + tableName + " DROP COLUMN IF EXISTS y"); @@ -2561,7 +2555,7 @@ public void testDropRowField() if (!hasBehavior(SUPPORTS_DROP_COLUMN) || !hasBehavior(SUPPORTS_ROW_TYPE)) { return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_field_", "AS SELECT CAST(row(1, 2) AS row(x integer, y integer)) AS col")) { + try (TestTable table = newTrinoTable("test_drop_field_", "AS SELECT CAST(row(1, 2) AS row(x integer, y integer)) AS col")) { assertQueryFails( "ALTER TABLE " + table.getName() + " DROP COLUMN col.x", "This connector does not support dropping fields"); @@ -2569,7 +2563,7 @@ public void testDropRowField() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_field_", "AS SELECT CAST(row(1, 2, row(10, 20)) AS row(a integer, b integer, c row(x integer, y integer))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(a integer, b integer, c row(x integer, y integer))"); @@ -2610,7 +2604,7 @@ public void testDropRowFieldInArray() if (!hasBehavior(SUPPORTS_DROP_COLUMN) || !hasBehavior(SUPPORTS_ROW_TYPE)) { return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_field_in_array_", "AS SELECT CAST(array[row(1, 2)] AS array(row(x integer, y integer))) AS col")) { + try (TestTable table = newTrinoTable("test_drop_field_in_array_", "AS SELECT CAST(array[row(1, 2)] AS array(row(x integer, y integer))) AS col")) { assertQueryFails( "ALTER TABLE " + table.getName() + " DROP COLUMN col.element.x", ".*does not support.*"); @@ -2618,7 +2612,7 @@ public void testDropRowFieldInArray() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_field_in_array_", "AS SELECT CAST(array[row(1, 2, row(10, 20), array[row(30, 40)])] AS array(row(a integer, b integer, c row(x integer, y integer), d array(row(v integer, w integer))))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("array(row(a integer, b integer, c row(x integer, y integer), d array(row(v integer, w integer))))"); @@ -2672,7 +2666,7 @@ public void testDropRowFieldInArray() assertThat(getColumnType(table.getName(), "col")).isEqualTo("array(row(a integer))"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_field_in_array_nested_", "AS SELECT CAST(array[array[row(1, 2, row(10, 20), array[row(30, 40)])]] AS array(array(row(a integer, b integer, c row(x integer, y integer), d array(row(v integer, w integer)))))) AS col")) { @@ -2705,7 +2699,7 @@ public void testDropRowFieldWhenDuplicates() { skipTestUnless(hasBehavior(SUPPORTS_DROP_FIELD)); - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_duplicated_field_", "AS SELECT CAST(row(1, 2, 3) AS row(a integer, a integer, b integer)) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(a integer, a integer, b integer)"); @@ -2722,7 +2716,7 @@ public void testDropRowFieldCaseSensitivity() { skipTestUnless(hasBehavior(SUPPORTS_DROP_FIELD)); - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_row_field_case_sensitivity_", "AS SELECT CAST(row(1, 2) AS row(lower integer, \"UPPER\" integer)) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(lower integer, UPPER integer)"); @@ -2745,7 +2739,7 @@ public void testDropAmbiguousRowFieldCaseSensitivity() { skipTestUnless(hasBehavior(SUPPORTS_DROP_FIELD)); - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_drop_row_field_case_sensitivity_", """ AS SELECT CAST(row(1, 2, 3, 4, 5) AS @@ -2771,7 +2765,7 @@ public void testDropAndAddColumnWithSameName() { skipTestUnless(hasBehavior(SUPPORTS_DROP_COLUMN) && hasBehavior(SUPPORTS_ADD_COLUMN)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_add_column", "AS SELECT 1 x, 2 y, 3 z")) { + try (TestTable table = newTrinoTable("test_drop_add_column", "AS SELECT 1 x, 2 y, 3 z")) { assertUpdate("ALTER TABLE " + table.getName() + " DROP COLUMN y"); assertQuery("SELECT * FROM " + table.getName(), "VALUES (1, 3)"); @@ -2791,7 +2785,7 @@ public void testRenameColumn() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); String tableName; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_rename_column_", "AS SELECT 'some value' x")) { + try (TestTable table = newTrinoTable("test_rename_column_", "AS SELECT 'some value' x")) { tableName = table.getName(); assertUpdate("ALTER TABLE " + tableName + " RENAME COLUMN x TO before_y"); assertUpdate("ALTER TABLE " + tableName + " RENAME COLUMN IF EXISTS before_y TO y"); @@ -2823,7 +2817,7 @@ public void testRenameColumnWithComment() { skipTestUnless(hasBehavior(SUPPORTS_RENAME_COLUMN) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_rename_column_", "(col INT COMMENT 'test column comment')")) { + try (TestTable table = newTrinoTable("test_rename_column_", "(col INT COMMENT 'test column comment')")) { assertThat(getColumnComment(table.getName(), "col")).isEqualTo("test column comment"); assertUpdate("ALTER TABLE " + table.getName() + " RENAME COLUMN col TO renamed_col"); @@ -2837,7 +2831,7 @@ public void testRenameRowField() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ROW_TYPE)); if (!hasBehavior(SUPPORTS_RENAME_FIELD)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_rename_field_", "AS SELECT CAST(row(1) AS row(x integer)) AS col")) { + try (TestTable table = newTrinoTable("test_rename_field_", "AS SELECT CAST(row(1) AS row(x integer)) AS col")) { assertQueryFails( "ALTER TABLE " + table.getName() + " RENAME COLUMN col.x TO x_renamed", "This connector does not support renaming fields"); @@ -2845,7 +2839,7 @@ public void testRenameRowField() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_add_field_", "AS SELECT CAST(row(1, row(10)) AS row(a integer, b row(x integer))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(a integer, b row(x integer))"); @@ -2874,7 +2868,7 @@ public void testRenameRowFieldCaseSensitivity() { skipTestUnless(hasBehavior(SUPPORTS_RENAME_FIELD)); - try (TestTable table = new TestTable(getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_add_row_field_case_sensitivity_", "AS SELECT CAST(row(1, 2) AS row(lower integer, \"UPPER\" integer)) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(lower integer, UPPER integer)"); @@ -2903,7 +2897,7 @@ public void testSetColumnType() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_column_type_", "AS SELECT CAST(123 AS integer) AS col")) { + try (TestTable table = newTrinoTable("test_set_column_type_", "AS SELECT CAST(123 AS integer) AS col")) { assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col SET DATA TYPE bigint"); assertThat(getColumnType(table.getName(), "col")).isEqualTo("bigint"); @@ -2921,7 +2915,7 @@ public void testSetColumnTypes() for (SetColumnTypeSetup setup : setColumnTypesDataProvider()) { TestTable table; try { - table = new TestTable(getQueryRunner()::execute, "test_set_column_type_", " AS SELECT CAST(" + setup.sourceValueLiteral + " AS " + setup.sourceColumnType + ") AS col"); + table = newTrinoTable("test_set_column_type_", " AS SELECT CAST(" + setup.sourceValueLiteral + " AS " + setup.sourceColumnType + ") AS col"); } catch (Exception e) { verifyUnsupportedTypeException(e, setup.sourceColumnType); @@ -3043,7 +3037,7 @@ public void testSetColumnTypeWithNotNull() { skipTestUnless(hasBehavior(SUPPORTS_SET_COLUMN_TYPE) && hasBehavior(SUPPORTS_NOT_NULL_CONSTRAINT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_column_type_null_", "(col int NOT NULL)")) { + try (TestTable table = newTrinoTable("test_set_column_type_null_", "(col int NOT NULL)")) { assertThat(columnIsNullable(table.getName(), "col")).isFalse(); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col SET DATA TYPE bigint"); @@ -3056,7 +3050,7 @@ public void testSetColumnTypeWithComment() { skipTestUnless(hasBehavior(SUPPORTS_SET_COLUMN_TYPE) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_column_type_comment_", "(col int COMMENT 'test comment')")) { + try (TestTable table = newTrinoTable("test_set_column_type_comment_", "(col int COMMENT 'test comment')")) { assertThat(getColumnComment(table.getName(), "col")).isEqualTo("test comment"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col SET DATA TYPE bigint"); @@ -3082,7 +3076,7 @@ public void testSetColumnIncompatibleType() { skipTestUnless(hasBehavior(SUPPORTS_SET_COLUMN_TYPE) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_invalid_column_type_", "AS SELECT 'test' AS col")) { + try (TestTable table = newTrinoTable("test_set_invalid_column_type_", "AS SELECT 'test' AS col")) { assertThatThrownBy(() -> assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col SET DATA TYPE integer")) .satisfies(this::verifySetColumnTypeFailurePermissible); } @@ -3093,7 +3087,7 @@ public void testSetColumnOutOfRangeType() { skipTestUnless(hasBehavior(SUPPORTS_SET_COLUMN_TYPE) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_column_type_invalid_range_", "AS SELECT CAST(9223372036854775807 AS bigint) AS col")) { + try (TestTable table = newTrinoTable("test_set_column_type_invalid_range_", "AS SELECT CAST(9223372036854775807 AS bigint) AS col")) { assertThatThrownBy(() -> assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col SET DATA TYPE integer")) .satisfies(this::verifySetColumnTypeFailurePermissible); } @@ -3110,7 +3104,7 @@ public void testSetFieldType() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ROW_TYPE)); if (!hasBehavior(SUPPORTS_SET_FIELD_TYPE)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_", "(col row(field int))")) { + try (TestTable table = newTrinoTable("test_set_field_type_", "(col row(field int))")) { assertQueryFails( "ALTER TABLE " + table.getName() + " ALTER COLUMN col.field SET DATA TYPE bigint", "This connector does not support setting field types"); @@ -3118,7 +3112,7 @@ public void testSetFieldType() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_", "AS SELECT CAST(row(123) AS row(field integer)) AS col")) { + try (TestTable table = newTrinoTable("test_set_field_type_", "AS SELECT CAST(row(123) AS row(field integer)) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(field integer)"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.field SET DATA TYPE bigint"); @@ -3138,8 +3132,7 @@ public void testSetFieldTypes() for (SetColumnTypeSetup setup : setFieldTypesDataProvider()) { TestTable table; try { - table = new TestTable( - getQueryRunner()::execute, + table = newTrinoTable( "test_set_field_type_", " AS SELECT CAST(row(" + setup.sourceValueLiteral + ") AS row(field " + setup.sourceColumnType + ")) AS col"); } @@ -3182,7 +3175,7 @@ public void testSetFieldTypeCaseSensitivity() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE) && hasBehavior(SUPPORTS_NOT_NULL_CONSTRAINT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_case_", " AS SELECT CAST(row(1) AS row(\"UPPER\" integer)) col")) { + try (TestTable table = newTrinoTable("test_set_field_type_case_", " AS SELECT CAST(row(1) AS row(\"UPPER\" integer)) col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("row(UPPER integer)"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.upper SET DATA TYPE bigint"); @@ -3197,7 +3190,7 @@ public void testSetFieldTypeWithNotNull() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE) && hasBehavior(SUPPORTS_NOT_NULL_CONSTRAINT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_null_", "(col row(field int) NOT NULL)")) { + try (TestTable table = newTrinoTable("test_set_field_type_null_", "(col row(field int) NOT NULL)")) { assertThat(columnIsNullable(table.getName(), "col")).isFalse(); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.field SET DATA TYPE bigint"); @@ -3210,7 +3203,7 @@ public void testSetFieldTypeWithComment() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_comment_", "(col row(field int) COMMENT 'test comment')")) { + try (TestTable table = newTrinoTable("test_set_field_type_comment_", "(col row(field int) COMMENT 'test comment')")) { assertThat(getColumnComment(table.getName(), "col")).isEqualTo("test comment"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.field SET DATA TYPE bigint"); @@ -3223,8 +3216,7 @@ public void testSetFieldIncompatibleType() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_set_invalid_field_type_", "(row_col row(field varchar), nested_col row(field row(nested int)))")) { assertThatThrownBy(() -> assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN row_col.field SET DATA TYPE row(nested integer)")) @@ -3241,8 +3233,7 @@ public void testSetFieldOutOfRangeType() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_set_field_type_invalid_range_", "AS SELECT CAST(row(9223372036854775807) AS row(field bigint)) AS col")) { assertThatThrownBy(() -> assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.field SET DATA TYPE integer")) @@ -3256,7 +3247,7 @@ public void testSetFieldTypeInArray() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ARRAY) && hasBehavior(SUPPORTS_ROW_TYPE)); if (!hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_ARRAY)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_in_array_", "(col array(row(field int)))")) { + try (TestTable table = newTrinoTable("test_set_field_type_in_array_", "(col array(row(field int)))")) { assertQueryFails( "ALTER TABLE " + table.getName() + " ALTER COLUMN col.element.field SET DATA TYPE bigint", ".*does not support.*"); @@ -3264,7 +3255,7 @@ public void testSetFieldTypeInArray() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_in_array_", "AS SELECT CAST(array[row(123)] AS array(row(field integer))) AS col")) { + try (TestTable table = newTrinoTable("test_set_field_type_in_array_", "AS SELECT CAST(array[row(123)] AS array(row(field integer))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("array(row(field integer))"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.element.field SET DATA TYPE bigint"); @@ -3281,7 +3272,7 @@ public void testSetFieldTypeInNestedArray() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_ARRAY) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ARRAY) && hasBehavior(SUPPORTS_ROW_TYPE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_in_nested_array_", "AS SELECT CAST(array[array[row(array[row(123)])]] AS array(array(row(field array(row(a integer)))))) AS col")) { + try (TestTable table = newTrinoTable("test_set_field_type_in_nested_array_", "AS SELECT CAST(array[array[row(array[row(123)])]] AS array(array(row(field array(row(a integer)))))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("array(array(row(field array(row(a integer)))))"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.element.element.field.element.a SET DATA TYPE bigint"); @@ -3304,13 +3295,13 @@ public void testSetFieldMapKeyType() String tableDefinition = "AS SELECT CAST(map(array[row(1)], array[2]) AS map(row(field integer), integer)) AS col"; if (!hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_MAP)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_in_map", tableDefinition)) { + try (TestTable table = newTrinoTable("test_set_field_type_in_map", tableDefinition)) { assertQueryFails("ALTER TABLE " + table.getName() + " ALTER COLUMN col.key.field SET DATA TYPE bigint", ".*does not support.*"); } return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_in_map", tableDefinition)) { + try (TestTable table = newTrinoTable("test_set_field_type_in_map", tableDefinition)) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("map(row(field integer), integer)"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.key.field SET DATA TYPE bigint"); @@ -3328,13 +3319,13 @@ public void testSetFieldMapValueType() String tableDefinition = "AS SELECT CAST(map(array[1], array[row(2)]) AS map(integer, row(field integer))) AS col"; if (!hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_MAP)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_in_map", tableDefinition)) { + try (TestTable table = newTrinoTable("test_set_field_type_in_map", tableDefinition)) { assertQueryFails("ALTER TABLE " + table.getName() + " ALTER COLUMN col.value.field SET DATA TYPE bigint", ".*does not support.*"); } return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_set_field_type_in_map", tableDefinition)) { + try (TestTable table = newTrinoTable("test_set_field_type_in_map", tableDefinition)) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("map(integer, row(field integer))"); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col.value.field SET DATA TYPE bigint"); @@ -3350,8 +3341,7 @@ public void testSetNestedFieldMapKeyType() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_ARRAY) && hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_MAP) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ARRAY) && hasBehavior(SUPPORTS_MAP_TYPE) && hasBehavior(SUPPORTS_ROW_TYPE)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_set_nested_field_type_in_map", "AS SELECT CAST(array[map(array[row(1)], array[2])] AS array(map(row(field integer), integer))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("array(map(row(field integer), integer))"); @@ -3369,8 +3359,7 @@ public void testSetNestedFieldMapValueType() { skipTestUnless(hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_ARRAY) && hasBehavior(SUPPORTS_SET_FIELD_TYPE_IN_MAP) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ARRAY) && hasBehavior(SUPPORTS_MAP_TYPE) && hasBehavior(SUPPORTS_ROW_TYPE)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_set_nested_field_type_in_map", "AS SELECT CAST(array[map(array[1], array[row(2)])] AS array(map(integer, row(field integer)))) AS col")) { assertThat(getColumnType(table.getName(), "col")).isEqualTo("array(map(integer, row(field integer)))"); @@ -3401,7 +3390,7 @@ public void testDropNotNullConstraint() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_NOT_NULL_CONSTRAINT)); if (!hasBehavior(SUPPORTS_DROP_NOT_NULL_CONSTRAINT)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_not_null_", "(col integer NOT NULL)")) { + try (TestTable table = newTrinoTable("test_drop_not_null_", "(col integer NOT NULL)")) { assertQueryFails( "ALTER TABLE " + table.getName() + " ALTER COLUMN col DROP NOT NULL", "This connector does not support dropping a not null constraint"); @@ -3409,7 +3398,7 @@ public void testDropNotNullConstraint() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_not_null_", "(col integer NOT NULL)")) { + try (TestTable table = newTrinoTable("test_drop_not_null_", "(col integer NOT NULL)")) { assertThat(columnIsNullable(table.getName(), "col")).isFalse(); assertUpdate("ALTER TABLE " + table.getName() + " ALTER COLUMN col DROP NOT NULL"); @@ -3426,7 +3415,7 @@ public void testDropNotNullConstraintWithColumnComment() skipTestUnless(hasBehavior(SUPPORTS_DROP_NOT_NULL_CONSTRAINT) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT)); // Verify DROP NOT NULL preserves the existing column comment - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_drop_not_null_", "(col integer NOT NULL COMMENT 'test comment')")) { + try (TestTable table = newTrinoTable("test_drop_not_null_", "(col integer NOT NULL COMMENT 'test comment')")) { assertThat(getColumnComment(table.getName(), "col")).isEqualTo("test comment"); assertThat(columnIsNullable(table.getName(), "col")).isFalse(); @@ -3557,7 +3546,7 @@ public void testCreateOrReplaceTableWhenTableAlreadyExistsSameSchema() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", "AS SELECT CAST(1 AS BIGINT) AS nationkey, 'test' AS name, CAST(2 AS BIGINT) AS regionkey FROM nation LIMIT 1")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", "AS SELECT CAST(1 AS BIGINT) AS nationkey, 'test' AS name, CAST(2 AS BIGINT) AS regionkey FROM nation LIMIT 1")) { @Language("SQL") String query = "SELECT nationkey, name, regionkey FROM nation"; @Language("SQL") String rowCountQuery = "SELECT count(*) FROM nation"; assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " AS " + query, rowCountQuery); @@ -3574,7 +3563,7 @@ public void testCreateOrReplaceTableWhenTableAlreadyExistsSameSchemaNoData() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " AS SELECT nationkey, name, regionkey FROM nation")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " AS SELECT nationkey, name, regionkey FROM nation")) { assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " AS SELECT nationkey, name, regionkey FROM nation WITH NO DATA", 0L); assertQueryReturnsEmptyResult("SELECT * FROM " + table.getName()); } @@ -3589,7 +3578,7 @@ public void testCreateOrReplaceTableWithNewColumnNames() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " AS SELECT nationkey, name, regionkey FROM nation")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " AS SELECT nationkey, name, regionkey FROM nation")) { assertTableColumnNames(table.getName(), "nationkey", "name", "regionkey"); @Language("SQL") String query = "SELECT nationkey AS nationkey_new, name AS name_new_2, regionkey AS region_key_new FROM nation"; @Language("SQL") String rowCountQuery = "SELECT count(*) FROM nation"; @@ -3608,7 +3597,7 @@ public void testCreateOrReplaceTableWithDifferentDataType() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_or_replace_", " AS SELECT nationkey, name FROM nation")) { + try (TestTable table = newTrinoTable("test_create_or_replace_", " AS SELECT nationkey, name FROM nation")) { @Language("SQL") String query = "SELECT name AS nationkey, nationkey AS name FROM nation"; @Language("SQL") String rowCountQuery = "SELECT count(*) FROM nation"; assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " AS " + query, rowCountQuery); @@ -4218,7 +4207,7 @@ public void testCommentTable() String catalogName = getSession().getCatalog().orElseThrow(); String schemaName = getSession().getSchema().orElseThrow(); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_comment_", "(a integer)")) { + try (TestTable table = newTrinoTable("test_comment_", "(a integer)")) { // comment initially not set assertThat(getTableComment(catalogName, schemaName, table.getName())).isEqualTo(null); @@ -4311,7 +4300,7 @@ public void testCommentColumn() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_comment_column_", "(a integer)")) { + try (TestTable table = newTrinoTable("test_comment_column_", "(a integer)")) { // comment set assertUpdate("COMMENT ON COLUMN " + table.getName() + ".a IS 'new comment'"); assertThat((String) computeScalar("SHOW CREATE TABLE " + table.getName())).contains("COMMENT 'new comment'"); @@ -4346,7 +4335,7 @@ protected void testCommentColumnName(String columnName, boolean delimited) String nameInSql = toColumnNameInSql(columnName, delimited); // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_comment_column", "(" + nameInSql + " integer)")) { + try (TestTable table = newTrinoTable("test_comment_column", "(" + nameInSql + " integer)")) { assertUpdate("COMMENT ON COLUMN " + table.getName() + "." + nameInSql + " IS 'test comment'"); assertThat(getColumnComment(table.getName(), columnName.replace("'", "''").toLowerCase(ENGLISH))).isEqualTo("test comment"); } @@ -4416,7 +4405,7 @@ public void testInsert() String query = "SELECT name, nationkey, regionkey FROM nation"; - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_", "AS " + query + " WITH NO DATA")) { + try (TestTable table = newTrinoTable("test_insert_", "AS " + query + " WITH NO DATA")) { assertQuery("SELECT count(*) FROM " + table.getName() + "", "SELECT 0"); assertUpdate("INSERT INTO " + table.getName() + " " + query, 25); @@ -4477,13 +4466,13 @@ public void testInsertUnicode() throw new AssertionError("Cannot test INSERT without CREATE TABLE, the test needs to be implemented in a connector-specific way"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", "(test varchar(50))")) { + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50))")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'Hello', U&'hello\\6d4B\\8Bd5world\\7F16\\7801' ", 2); assertThat(computeActual("SELECT test FROM " + table.getName()).getOnlyColumnAsSet()) .containsExactlyInAnyOrder("Hello", "hello测试world编码"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", "(test varchar(50))")) { + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50))")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'aa', 'bé'", 2); assertQuery("SELECT test FROM " + table.getName(), "VALUES 'aa', 'bé'"); assertQuery("SELECT test FROM " + table.getName() + " WHERE test = 'aa'", "VALUES 'aa'"); @@ -4492,7 +4481,7 @@ public void testInsertUnicode() assertQueryReturnsEmptyResult("SELECT test FROM " + table.getName() + " WHERE test = 'ba'"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", "(test varchar(50))")) { + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50))")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'a', 'é'", 2); assertQuery("SELECT test FROM " + table.getName(), "VALUES 'a', 'é'"); assertQuery("SELECT test FROM " + table.getName() + " WHERE test = 'a'", "VALUES 'a'"); @@ -4510,7 +4499,7 @@ public void testInsertHighestUnicodeCharacter() throw new AssertionError("Cannot test INSERT without CREATE TABLE, the test needs to be implemented in a connector-specific way"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_unicode_", "(test varchar(50))")) { + try (TestTable table = newTrinoTable("test_insert_unicode_", "(test varchar(50))")) { assertUpdate("INSERT INTO " + table.getName() + "(test) VALUES 'Hello', U&'hello\\6d4B\\8Bd5\\+10FFFFworld\\7F16\\7801' ", 2); assertThat(computeActual("SELECT test FROM " + table.getName()).getOnlyColumnAsSet()) .containsExactlyInAnyOrder("Hello", "hello测试􏿿world编码"); @@ -4533,7 +4522,7 @@ public void testInsertArray() abort("not supported"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_array_", "(a ARRAY, b ARRAY)")) { + try (TestTable table = newTrinoTable("test_insert_array_", "(a ARRAY, b ARRAY)")) { assertUpdate("INSERT INTO " + table.getName() + " (a) VALUES (ARRAY[null])", 1); assertUpdate("INSERT INTO " + table.getName() + " (a, b) VALUES (ARRAY[1.23E1], ARRAY[1.23E1])", 1); assertQuery("SELECT a[1], b[1] FROM " + table.getName(), "VALUES (null, null), (12.3, 12)"); @@ -4553,7 +4542,7 @@ public void testInsertMap() abort("not supported"); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_insert_map_", "(col map(integer, integer))")) { + try (TestTable table = newTrinoTable("test_insert_map_", "(col map(integer, integer))")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES map(ARRAY[1], ARRAY[2])", 1); assertThat(query("SELECT * FROM " + table.getName())) .matches("VALUES map(ARRAY[1], ARRAY[2])"); @@ -4566,8 +4555,7 @@ public void testInsertSameValues() skipTestUnless(hasBehavior(SUPPORTS_INSERT)); skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "insert_same_values", "AS " + join(" UNION ALL ", nCopies(2, "SELECT * FROM region")))) { assertQuery("SELECT count(*) FROM " + table.getName(), "VALUES 10"); @@ -4585,13 +4573,13 @@ public void testInsertNegativeDate() throw new AssertionError("Cannot test INSERT negative dates without CREATE TABLE, the test needs to be implemented in a connector-specific way"); } if (!hasBehavior(SUPPORTS_NEGATIVE_DATE)) { - try (TestTable table = new TestTable(getQueryRunner()::execute, "insert_date", "(dt DATE)")) { + try (TestTable table = newTrinoTable("insert_date", "(dt DATE)")) { assertQueryFails(format("INSERT INTO %s VALUES (DATE '-0001-01-01')", table.getName()), errorMessageForInsertNegativeDate("-0001-01-01")); } return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "insert_date", "(dt DATE)")) { + try (TestTable table = newTrinoTable("insert_date", "(dt DATE)")) { assertUpdate(format("INSERT INTO %s VALUES (DATE '-0001-01-01')", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES DATE '-0001-01-01'"); assertQuery(format("SELECT * FROM %s WHERE dt = DATE '-0001-01-01'", table.getName()), "VALUES DATE '-0001-01-01'"); @@ -4616,7 +4604,7 @@ public void testInsertIntoNotNullColumn() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "insert_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { + try (TestTable table = newTrinoTable("insert_not_null", "(nullable_col INTEGER, not_null_col INTEGER NOT NULL)")) { assertUpdate(format("INSERT INTO %s (not_null_col) VALUES (2)", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (NULL, 2)"); // The error message comes from remote databases when ConnectorMetadata.supportsMissingColumnsOnInsert is true @@ -4628,7 +4616,7 @@ public void testInsertIntoNotNullColumn() assertUpdate(format("INSERT INTO %s (nullable_col) SELECT nationkey FROM nation WHERE regionkey < 0", table.getName()), 0); } - try (TestTable table = new TestTable(getQueryRunner()::execute, "commuted_not_null", "(nullable_col BIGINT, not_null_col BIGINT NOT NULL)")) { + try (TestTable table = newTrinoTable("commuted_not_null", "(nullable_col BIGINT, not_null_col BIGINT NOT NULL)")) { assertUpdate(format("INSERT INTO %s (not_null_col) VALUES (2)", table.getName()), 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (NULL, 2)"); // This is enforced by the engine and not the connector @@ -4669,8 +4657,7 @@ public void testInsertInTransaction() skipTestUnless(hasBehavior(SUPPORTS_INSERT)); skipTestUnless(hasBehavior(SUPPORTS_MULTI_STATEMENT_WRITES)); // covered by testWriteNotAllowedInTransaction - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_tx_insert", "(a bigint)")) { String tableName = table.getName(); @@ -4688,8 +4675,7 @@ public void testSelectAfterInsertInTransaction() return; } - try (TestTable table = new TestTable( - getQueryRunner()::execute, + try (TestTable table = newTrinoTable( "test_insert_select_", "AS SELECT nationkey, name, regionkey FROM nation WHERE nationkey = 1")) { String tableName = table.getName(); @@ -4733,7 +4719,7 @@ public void testDelete() skipTestUnless(hasBehavior(SUPPORTS_DELETE)); // delete successive parts of the table - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_", "AS SELECT * FROM orders")) { + try (TestTable table = newTrinoTable("test_delete_", "AS SELECT * FROM orders")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE custkey <= 100", "SELECT count(*) FROM orders WHERE custkey <= 100"); assertQuery("SELECT * FROM " + table.getName(), "SELECT * FROM orders WHERE custkey > 100"); @@ -4745,12 +4731,12 @@ public void testDelete() } // delete without matching any rows - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_", "AS SELECT * FROM orders")) { + try (TestTable table = newTrinoTable("test_delete_", "AS SELECT * FROM orders")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE orderkey < 0", 0); } // delete with a predicate that optimizes to false - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_", "AS SELECT * FROM orders")) { + try (TestTable table = newTrinoTable("test_delete_", "AS SELECT * FROM orders")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE orderkey > 5 AND orderkey < 4", 0); } @@ -4776,7 +4762,7 @@ public void testDeleteWithLike() { skipTestUnless(hasBehavior(SUPPORTS_DELETE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_with_like_", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_with_like_", "AS SELECT * FROM nation")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE name LIKE '%a%'", "VALUES 0"); assertUpdate("DELETE FROM " + table.getName() + " WHERE name LIKE '%A%'", "SELECT count(*) FROM nation WHERE name LIKE '%A%'"); } @@ -4904,7 +4890,7 @@ public void testDeleteWithVarcharPredicate() { skipTestUnless(hasBehavior(SUPPORTS_DELETE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_with_varchar_predicate_", "AS SELECT * FROM orders")) { + try (TestTable table = newTrinoTable("test_delete_with_varchar_predicate_", "AS SELECT * FROM orders")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE orderstatus = 'O'", "SELECT count(*) FROM orders WHERE orderstatus = 'O'"); assertQuery("SELECT * FROM " + table.getName(), "SELECT * FROM orders WHERE orderstatus <> 'O'"); } @@ -4919,7 +4905,7 @@ public void verifySupportsDeleteDeclaration() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_delete", "(regionkey int)")) { + try (TestTable table = newTrinoTable("test_supports_delete", "(regionkey int)")) { assertQueryFails("DELETE FROM " + table.getName(), MODIFYING_ROWS_MESSAGE); } } @@ -4933,7 +4919,7 @@ public void verifySupportsRowLevelDeleteDeclaration() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_row_level_delete", "(regionkey int)")) { + try (TestTable table = newTrinoTable("test_supports_row_level_delete", "(regionkey int)")) { assertQueryFails("DELETE FROM " + table.getName() + " WHERE regionkey = 2", MODIFYING_ROWS_MESSAGE); } } @@ -4942,7 +4928,7 @@ public void verifySupportsRowLevelDeleteDeclaration() public void testDeleteAllDataFromTable() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_DELETE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_delete_all_data", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_delete_all_data", "AS SELECT * FROM region")) { // not using assertUpdate as some connectors provide update count and some not getQueryRunner().execute("DELETE FROM " + table.getName()); assertQuery("SELECT count(*) FROM " + table.getName(), "VALUES 0"); @@ -4954,7 +4940,7 @@ public void testRowLevelDelete() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_delete", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_row_delete", "AS SELECT * FROM region")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE regionkey = 2", 1); assertQuery("SELECT count(*) FROM " + table.getName(), "VALUES 4"); } @@ -4969,7 +4955,7 @@ public void verifySupportsUpdateDeclaration() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_supports_update", "AS SELECT * FROM nation")) { assertQueryFails("UPDATE " + table.getName() + " SET nationkey = 100 WHERE regionkey = 2", MODIFYING_ROWS_MESSAGE); } } @@ -4983,7 +4969,7 @@ public void verifySupportsRowLevelUpdateDeclaration() } skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_supports_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_supports_update", "AS SELECT * FROM nation")) { assertQueryFails("UPDATE " + table.getName() + " SET nationkey = nationkey * 100 WHERE regionkey = 2", MODIFYING_ROWS_MESSAGE); } } @@ -4996,7 +4982,7 @@ public void testUpdate() assertQueryFails("UPDATE nation SET nationkey = nationkey + regionkey WHERE regionkey < 1", MODIFYING_ROWS_MESSAGE); return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_row_update", "AS SELECT * FROM nation")) { assertUpdate("UPDATE " + table.getName() + " SET nationkey = 100 WHERE regionkey = 2", 5); assertQuery("SELECT count(*) FROM " + table.getName() + " WHERE nationkey = 100", "VALUES 5"); } @@ -5007,7 +4993,7 @@ public void testUpdateMultipleCondition() { skipTestUnless(hasBehavior(SUPPORTS_UPDATE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_update", "AS SELECT * FROM (VALUES (1, 10), (1, 20), (2, 10)) AS t(a, b)")) { + try (TestTable table = newTrinoTable("test_row_update", "AS SELECT * FROM (VALUES (1, 10), (1, 20), (2, 10)) AS t(a, b)")) { assertUpdate("UPDATE " + table.getName() + " SET b = 100 WHERE a = 1 AND b = 10", 1); assertQuery("SELECT * FROM " + table.getName(), "VALUES (1, 100), (1, 20), (2, 10)"); } @@ -5043,7 +5029,7 @@ public void testUpdateCaseSensitivity() { skipTestUnless(hasBehavior(SUPPORTS_UPDATE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_row_update", "AS SELECT * FROM nation")) { + try (TestTable table = newTrinoTable("test_row_update", "AS SELECT * FROM nation")) { assertUpdate("UPDATE " + table.getName() + " SET NATIONKEY = 100 WHERE REGIONKEY = 2", 5); assertQuery("SELECT count(*) FROM " + table.getName() + " WHERE nationkey = 100", "VALUES 5"); } @@ -5336,7 +5322,7 @@ public void testCreateOrReplaceTableConcurrently() protected TestTable createTableWithOneIntegerColumn(String namePrefix) { - return new TestTable(getQueryRunner()::execute, namePrefix, "(col integer)"); + return newTrinoTable(namePrefix, "(col integer)"); } @Test @@ -5375,7 +5361,7 @@ public void testUpdateRowType() return; } - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_update_with_predicates_on_row_types", "(int_t INT, row_t ROW(f1 INT, f2 INT))")) { + try (TestTable table = newTrinoTable("test_update_with_predicates_on_row_types", "(int_t INT, row_t ROW(f1 INT, f2 INT))")) { String tableName = table.getName(); assertUpdate("INSERT INTO " + tableName + " VALUES (1, ROW(2, 3)), (11, ROW(12, 13)), (21, ROW(22, 23))", 3); assertUpdate("UPDATE " + tableName + " SET int_t = int_t - 1 WHERE row_t.f2 = 3", 1); @@ -5394,7 +5380,7 @@ public void testPredicateOnRowTypeField() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_INSERT) && hasBehavior(SUPPORTS_ROW_TYPE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_predicate_on_row_type_field", "(int_t INT, row_t row(varchar_t VARCHAR, int_t INT))")) { + try (TestTable table = newTrinoTable("test_predicate_on_row_type_field", "(int_t INT, row_t row(varchar_t VARCHAR, int_t INT))")) { assertUpdate("INSERT INTO " + table.getName() + " VALUES (2, row('first', 1)), (20, row('second', 10)), (200, row('third', 100))", 3); assertQuery("SELECT int_t FROM " + table.getName() + " WHERE row_t.int_t = 1", "VALUES 2"); assertQuery("SELECT int_t FROM " + table.getName() + " WHERE row_t.int_t > 1", "VALUES 20, 200"); @@ -5449,7 +5435,7 @@ public void testTruncateTable() skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_truncate", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_truncate", "AS SELECT * FROM region")) { assertUpdate("TRUNCATE TABLE " + table.getName()); assertQuery("SELECT count(*) FROM " + table.getName(), "VALUES 0"); } @@ -5798,7 +5784,7 @@ protected void testCreateTableWithTableCommentSpecialCharacter(String comment) { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_", "(a bigint) COMMENT " + varcharLiteral(comment))) { + try (TestTable table = newTrinoTable("test_create_", "(a bigint) COMMENT " + varcharLiteral(comment))) { assertThat(getTableComment(getSession().getCatalog().orElseThrow(), getSession().getSchema().orElseThrow(), table.getName())).isEqualTo(comment); } } @@ -5821,7 +5807,7 @@ private void testCreateTableAsSelectWithTableCommentSpecialCharacter(String comm { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_", " COMMENT " + varcharLiteral(comment) + " AS SELECT 1 a")) { + try (TestTable table = newTrinoTable("test_create_", " COMMENT " + varcharLiteral(comment) + " AS SELECT 1 a")) { assertThat(getTableComment(getSession().getCatalog().orElseThrow(), getSession().getSchema().orElseThrow(), table.getName())).isEqualTo(comment); } } @@ -5844,7 +5830,7 @@ private void testCreateTableWithColumnCommentSpecialCharacter(String comment) { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_create_", " (a bigint COMMENT " + varcharLiteral(comment) + ")")) { + try (TestTable table = newTrinoTable("test_create_", " (a bigint COMMENT " + varcharLiteral(comment) + ")")) { assertThat(getColumnComment(table.getName(), "a")).isEqualTo(comment); } } @@ -5867,7 +5853,7 @@ protected void testAddColumnWithCommentSpecialCharacter(String comment) { skipTestUnless(hasBehavior(SUPPORTS_ADD_COLUMN_WITH_COMMENT)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_add_col_", "(a_varchar varchar)")) { + try (TestTable table = newTrinoTable("test_add_col_", "(a_varchar varchar)")) { assertUpdate("ALTER TABLE " + table.getName() + " ADD COLUMN b_varchar varchar COMMENT " + varcharLiteral(comment)); assertThat(getColumnComment(table.getName(), "b_varchar")).isEqualTo(comment); } @@ -5891,7 +5877,7 @@ private void testCommentTableSpecialCharacter(String comment) { skipTestUnless(hasBehavior(SUPPORTS_COMMENT_ON_TABLE)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_comment_table_", "(a integer)")) { + try (TestTable table = newTrinoTable("test_comment_table_", "(a integer)")) { assertUpdate("COMMENT ON TABLE " + table.getName() + " IS " + varcharLiteral(comment)); assertThat(getTableComment(getSession().getCatalog().orElseThrow(), getSession().getSchema().orElseThrow(), table.getName())).isEqualTo(comment); } @@ -5915,7 +5901,7 @@ private void testCommentColumnSpecialCharacter(String comment) { skipTestUnless(hasBehavior(SUPPORTS_COMMENT_ON_COLUMN)); - try (TestTable table = new TestTable(getQueryRunner()::execute, "test_comment_column_", "(a integer)")) { + try (TestTable table = newTrinoTable("test_comment_column_", "(a integer)")) { assertUpdate("COMMENT ON COLUMN " + table.getName() + ".a IS " + varcharLiteral(comment)); assertThat(getColumnComment(table.getName(), "a")).isEqualTo(comment); } @@ -6081,8 +6067,7 @@ public void testTimestampWithTimeZoneCastToDatePredicate() TestTable table; try { - table = new TestTable( - getQueryRunner()::execute, + table = newTrinoTable( "timestamptz_to_date", // These to timestamps are same local time, but different point in times and also different date at UTC time zone """ @@ -6112,8 +6097,7 @@ public void testTimestampWithTimeZoneCastToTimestampPredicate() TestTable table; try { - table = new TestTable( - getQueryRunner()::execute, + table = newTrinoTable( "timestamptz_to_ts", // These to timestamps are same local time, but different point in times """ @@ -7038,8 +7022,7 @@ public void testProjectionPushdown() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ROW_TYPE)); - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_projection_pushdown_", "(id BIGINT, root ROW(f1 BIGINT, f2 BIGINT))", ImmutableList.of("(1, ROW(1, 2))", "(2, NULl)", "(3, ROW(NULL, 4))"))) { @@ -7071,8 +7054,7 @@ public void testProjectionWithCaseSensitiveField() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_DEREFERENCE_PUSHDOWN)); - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_projection_with_case_sensitive_field_", "(id BIGINT, a ROW(\"UPPER_CASE\" BIGINT, \"lower_case\" BIGINT, \"MiXeD_cAsE\" BIGINT))", ImmutableList.of("(1, ROW(2, 3, 4))", "(5, ROW(6, 7, 8))"))) { @@ -7094,8 +7076,7 @@ public void testProjectionPushdownMultipleRows() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_DEREFERENCE_PUSHDOWN)); - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_projection_pushdown_multiple_rows_", "(id BIGINT, nested1 ROW(child1 BIGINT, child2 VARCHAR, child3 BIGINT), nested2 ROW(child1 DOUBLE, child2 BOOLEAN, child3 DATE))", ImmutableList.of( @@ -7143,8 +7124,7 @@ public void testProjectionPushdownWithHighlyNestedData() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_DEREFERENCE_PUSHDOWN)); - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_highly_nested_data_", "(id INT, row1_t ROW(f1 INT, f2 INT, row2_t ROW (f1 INT, f2 INT, row3_t ROW(f1 INT, f2 INT))))", ImmutableList.of("(1, ROW(2, 3, ROW(4, 5, ROW(6, 7))))", @@ -7176,8 +7156,7 @@ public void testProjectionPushdownReadsLessData() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_DEREFERENCE_PUSHDOWN)); - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_projection_pushdown_reads_less_data_", "AS SELECT val AS id, CAST(ROW(val + 1, val + 2) AS ROW(leaf1 BIGINT, leaf2 BIGINT)) AS root FROM UNNEST(SEQUENCE(1, 10)) AS t(val)")) { MaterializedResult expectedResult = computeActual("SELECT val + 2 FROM UNNEST(SEQUENCE(1, 10)) AS t(val)"); @@ -7215,8 +7194,7 @@ public void testProjectionPushdownPhysicalInputSize() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_DEREFERENCE_PUSHDOWN)); - try (TestTable testTable = new TestTable( - getQueryRunner()::execute, + try (TestTable testTable = newTrinoTable( "test_projection_pushdown_physical_input_size_", "AS SELECT val AS id, CAST(ROW(val + 1, val + 2) AS ROW(leaf1 BIGINT, leaf2 BIGINT)) AS root FROM UNNEST(SEQUENCE(1, 10)) AS t(val)")) { // Verify that the physical input size is smaller when reading the root.leaf1 field compared to reading the root field From 0b9e662c80617cc8a091bd27e760d2def6191359 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Wed, 18 Dec 2024 21:19:59 +0100 Subject: [PATCH 081/158] Allow running product tests on IPv6 stack --- .../product/launcher/cli/EnvironmentUp.java | 5 ++- .../tests/product/launcher/cli/TestRun.java | 5 ++- .../product/launcher/env/Environment.java | 18 +++++++-- .../launcher/env/EnvironmentModule.java | 8 ++++ .../launcher/env/EnvironmentOptions.java | 3 ++ .../tests/product/launcher/env/Ipv6.java | 31 ++++++++++++++ .../product/launcher/env/common/Standard.java | 40 +++++++++++++++++-- .../env/common/StandardMultinode.java | 8 +++- .../env/environment/EnvMultinodeTls.java | 8 +++- .../environment/EnvMultinodeTlsKerberos.java | 8 +++- .../EnvMultinodeTlsKerberosDelegation.java | 8 +++- 11 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Ipv6.java diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/EnvironmentUp.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/EnvironmentUp.java index c8d4d016f4aa..1ffd6f84ac17 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/EnvironmentUp.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/EnvironmentUp.java @@ -113,6 +113,7 @@ public static class Execution private final Map extraOptions; private final PrintStream printStream; private final boolean debug; + private final boolean ipv6; @Inject public Execution(EnvironmentFactory environmentFactory, EnvironmentConfig environmentConfig, EnvironmentOptions options, EnvironmentUpOptions environmentUpOptions, PrintStream printStream) @@ -127,6 +128,7 @@ public Execution(EnvironmentFactory environmentFactory, EnvironmentConfig enviro this.extraOptions = ImmutableMap.copyOf(requireNonNull(environmentUpOptions.extraOptions, "environmentUpOptions.extraOptions is null")); this.printStream = requireNonNull(printStream, "printStream is null"); this.debug = options.debug; + this.ipv6 = options.ipv6; } @Override @@ -136,7 +138,8 @@ public Integer call() Environment.Builder builder = environmentFactory.get(environment, printStream, environmentConfig, extraOptions) .setContainerOutputMode(outputMode) .setLogsBaseDir(environmentLogPath) - .removeContainer(TESTS); + .removeContainer(TESTS) + .setIpv6(ipv6); if (withoutCoordinator) { builder.removeContainers(container -> isTrinoContainer(container.getLogicalName())); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java index 7d938caa9dc4..0bf5f559eb0b 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/TestRun.java @@ -156,6 +156,7 @@ public static class Execution private final EnvironmentFactory environmentFactory; private final boolean debug; private final boolean debugSuspend; + private final boolean ipv6; private final JdkProvider jdkProvider; private final File testJar; private final File cliJar; @@ -186,6 +187,7 @@ public Execution( this.environmentFactory = requireNonNull(environmentFactory, "environmentFactory is null"); requireNonNull(environmentOptions, "environmentOptions is null"); this.debug = environmentOptions.debug; + this.ipv6 = environmentOptions.ipv6; this.debugSuspend = testRunOptions.debugSuspend; this.jdkProvider = requireNonNull(jdkProvider, "jdkProvider is null"); this.testJar = requireNonNull(testRunOptions.testJar, "testRunOptions.testJar is null"); @@ -323,7 +325,8 @@ private Environment getEnvironment() Environment.Builder builder = environmentFactory.get(environment, printStream, environmentConfig, extraOptions) .setContainerOutputMode(outputMode) .setStartupRetries(startupRetries) - .setLogsBaseDir(logsDirBase); + .setLogsBaseDir(logsDirBase) + .setIpv6(ipv6); builder.configureContainer(TESTS, this::mountReportsDir); builder.configureContainer(TESTS, container -> { diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Environment.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Environment.java index 5d2253875c8d..e9dff54f4049 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Environment.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Environment.java @@ -103,6 +103,7 @@ public final class Environment private final Map containers; private final EnvironmentListener listener; private final boolean attached; + private final boolean ipv6; private final Map> configuredFeatures; private Environment( @@ -111,12 +112,14 @@ private Environment( Map containers, EnvironmentListener listener, boolean attached, + boolean ipv6, Map> configuredFeatures, List> startupLogs) { this.name = requireNonNull(name, "name is null"); this.startupRetries = startupRetries; this.containers = requireNonNull(containers, "containers is null"); + this.ipv6 = ipv6; this.listener = compose(requireNonNull(listener, "listener is null"), printStartupLogs(startupLogs)); this.attached = attached; this.configuredFeatures = requireNonNull(configuredFeatures, "configuredFeatures is null"); @@ -141,7 +144,7 @@ public Environment start() return Failsafe .with(retryPolicy) .with(executorService) - .get(this::tryStart); + .get(() -> tryStart()); } private Environment tryStart() @@ -159,7 +162,7 @@ private Environment tryStart() } // Create new network when environment tries to start - try (Network network = createNetwork(name)) { + try (Network network = createNetwork(name, ipv6)) { attachNetwork(containers, network); Startables.deepStart(containers).get(); @@ -372,12 +375,13 @@ private static void attachNetwork(Collection values, Network ne values.forEach(container -> container.withNetwork(network)); } - private static Network createNetwork(String environmentName) + private static Network createNetwork(String environmentName, boolean ipv6) { Network network = Network.builder() .createNetworkCmdModifier(createNetworkCmd -> createNetworkCmd .withName(PRODUCT_TEST_LAUNCHER_NETWORK) + .withEnableIpv6(ipv6) .withLabels(ImmutableMap.of( PRODUCT_TEST_LAUNCHER_STARTED_LABEL_NAME, PRODUCT_TEST_LAUNCHER_STARTED_LABEL_VALUE, PRODUCT_TEST_LAUNCHER_ENVIRONMENT_LABEL_NAME, environmentName))) @@ -399,6 +403,7 @@ public static class Builder private int startupRetries = 1; private Optional logsBaseDir = Optional.empty(); private boolean attached; + private boolean ipv6; public Builder(String name, PrintStream printStream) { @@ -649,6 +654,7 @@ public Environment build(EnvironmentListener listener) containers, listener, attached, + ipv6, configuredFeatures, startupLogs.build()); } @@ -738,5 +744,11 @@ public Builder setAttached(boolean attached) this.attached = attached; return this; } + + public Builder setIpv6(boolean ipv6) + { + this.ipv6 = ipv6; + return this; + } } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java index 9597704239b8..606d06491fb6 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java @@ -173,4 +173,12 @@ public boolean provideTracing(EnvironmentOptions options) { return options.tracing; } + + @Provides + @Singleton + @Ipv6 + public boolean provideIpv6(EnvironmentOptions options) + { + return options.ipv6; + } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java index c1c989d66baa..b37dec6773c1 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java @@ -70,6 +70,9 @@ public final class EnvironmentOptions @Nullable public Path jdkDownloadPath; + @Option(names = "--ipv6", paramLabel = "", description = "Enable IPv6 networking") + public boolean ipv6; + @Option(names = "--bind", description = "Bind exposed container ports to host ports, possible values: " + BIND_ON_HOST + ", " + DO_NOT_BIND + ", [port base number] " + DEFAULT_VALUE, defaultValue = BIND_ON_HOST, arity = "0..1", fallbackValue = BIND_ON_HOST) public void setBindOnHost(String value) { diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Ipv6.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Ipv6.java new file mode 100644 index 000000000000..ff9bec867a82 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/Ipv6.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@BindingAnnotation +public @interface Ipv6 +{ +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java index 0af233ec6868..619a0b77873b 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java @@ -22,6 +22,7 @@ import io.trino.tests.product.launcher.env.Environment; import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentContainers; +import io.trino.tests.product.launcher.env.Ipv6; import io.trino.tests.product.launcher.env.ServerPackage; import io.trino.tests.product.launcher.env.Tracing; import io.trino.tests.product.launcher.env.jdk.JdkProvider; @@ -90,6 +91,7 @@ public final class Standard private final File serverPackage; private final boolean debug; private final boolean tracing; + private final boolean ipv6; @Inject public Standard( @@ -99,7 +101,8 @@ public Standard( @ServerPackage File serverPackage, JdkProvider jdkProvider, @Debug boolean debug, - @Tracing boolean tracing) + @Tracing boolean tracing, + @Ipv6 boolean ipv6) { this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); this.portBinder = requireNonNull(portBinder, "portBinder is null"); @@ -107,6 +110,7 @@ public Standard( this.jdkProvider = requireNonNull(jdkProvider, "jdkProvider is null"); this.serverPackage = requireNonNull(serverPackage, "serverPackage is null"); this.debug = debug; + this.ipv6 = ipv6; this.tracing = tracing; checkArgument(serverPackage.getName().endsWith(".tar.gz"), "Currently only server .tar.gz package is supported"); } @@ -169,7 +173,7 @@ private DockerContainer createTracingCollector() private DockerContainer createTrinoCoordinator() { DockerContainer container = - createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, "ghcr.io/trinodb/testing/almalinux9-oj17:" + imagesVersion, COORDINATOR) + createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, ipv6, "ghcr.io/trinodb/testing/almalinux9-oj17:" + imagesVersion, COORDINATOR) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("common/standard/access-control.properties")), CONTAINER_TRINO_ACCESS_CONTROL_PROPERTIES) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("common/standard/config.properties")), CONTAINER_TRINO_CONFIG_PROPERTIES); @@ -188,7 +192,7 @@ private DockerContainer createTestsContainer() } @SuppressWarnings("resource") - public static DockerContainer createTrinoContainer(DockerFiles dockerFiles, File serverPackage, JdkProvider jdkProvider, boolean debug, boolean tracing, String dockerImageName, String logicalName) + public static DockerContainer createTrinoContainer(DockerFiles dockerFiles, File serverPackage, JdkProvider jdkProvider, boolean debug, boolean tracing, boolean ipv6, String dockerImageName, String logicalName) { DockerContainer container = new DockerContainer(dockerImageName, logicalName) .withNetworkAliases(logicalName + ".docker.cluster") @@ -215,6 +219,10 @@ public static DockerContainer createTrinoContainer(DockerFiles dockerFiles, File enableTrinoTracing(container); } + if (ipv6) { + enableTrinoIpv6(container); + } + return jdkProvider.applyTo(container); } @@ -289,6 +297,32 @@ private static void enableTrinoTracing(DockerContainer container) } } + private static void enableTrinoIpv6(DockerContainer container) + { + log.info("Setting IPv6 as preferred networking stack for container: '%s'", container.getLogicalName()); + + try { + FileAttribute> rwx = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwxrwx")); + Path script = Files.createTempFile("enable-ipv6-stack", ".sh", rwx); + script.toFile().deleteOnExit(); + Files.writeString( + script, + format( + "#!/bin/bash\n" + + "ipv6=$(hostname -I | awk '{print $2}')\n" + + "IPv6 address of the node: ${ipv6}\n" + + "echo '-Djava.net.preferIPv6Addresses=true' >> '%s'\n" + + "echo \"node.internal-address=${ipv6}\" >> '%s'\n", + CONTAINER_TRINO_JVM_CONFIG, + CONTAINER_TRINO_CONFIG_PROPERTIES), + UTF_8); + container.withCopyFileToContainer(forHostPath(script), "/docker/presto-init.d/enable-ipv6-stack.sh"); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private static void enableTrinoJmxRmi(DockerContainer dockerContainer) { String logicalName = dockerContainer.getLogicalName(); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/StandardMultinode.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/StandardMultinode.java index a6cc8b395afa..647c9f577783 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/StandardMultinode.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/StandardMultinode.java @@ -20,6 +20,7 @@ import io.trino.tests.product.launcher.env.DockerContainer; import io.trino.tests.product.launcher.env.Environment; import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.env.Ipv6; import io.trino.tests.product.launcher.env.ServerPackage; import io.trino.tests.product.launcher.env.Tracing; import io.trino.tests.product.launcher.env.jdk.JdkProvider; @@ -46,6 +47,7 @@ public class StandardMultinode private final JdkProvider jdkProvider; private final boolean debug; private final boolean tracing; + private final boolean ipv6; @Inject public StandardMultinode( @@ -55,7 +57,8 @@ public StandardMultinode( @ServerPackage File serverPackage, JdkProvider jdkProvider, @Debug boolean debug, - @Tracing boolean tracing) + @Tracing boolean tracing, + @Ipv6 boolean ipv6) { this.standard = requireNonNull(standard, "standard is null"); this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); @@ -65,6 +68,7 @@ public StandardMultinode( this.serverPackage = requireNonNull(serverPackage, "serverPackage is null"); this.debug = debug; this.tracing = tracing; + this.ipv6 = ipv6; checkArgument(serverPackage.getName().endsWith(".tar.gz"), "Currently only server .tar.gz package is supported"); } @@ -85,7 +89,7 @@ public void extendEnvironment(Environment.Builder builder) @SuppressWarnings("resource") private DockerContainer createTrinoWorker() { - return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, "ghcr.io/trinodb/testing/almalinux9-oj17:" + imagesVersion, WORKER) + return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, ipv6, "ghcr.io/trinodb/testing/almalinux9-oj17:" + imagesVersion, WORKER) .withCopyFileToContainer(forHostPath(configDir.getPath("multinode-worker-config.properties")), CONTAINER_TRINO_CONFIG_PROPERTIES); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTls.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTls.java index fc87550104be..71e996073176 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTls.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTls.java @@ -21,6 +21,7 @@ import io.trino.tests.product.launcher.env.Environment; import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.Ipv6; import io.trino.tests.product.launcher.env.ServerPackage; import io.trino.tests.product.launcher.env.Tracing; import io.trino.tests.product.launcher.env.common.Hadoop; @@ -53,6 +54,7 @@ public final class EnvMultinodeTls private final File serverPackage; private final boolean debug; private final boolean tracing; + private final boolean ipv6; private final JdkProvider jdkProvider; @Inject @@ -65,7 +67,8 @@ public EnvMultinodeTls( @ServerPackage File serverPackage, JdkProvider jdkProvider, @Debug boolean debug, - @Tracing boolean tracing) + @Tracing boolean tracing, + @Ipv6 boolean ipv6) { super(ImmutableList.of(standard, hadoop)); this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); @@ -75,6 +78,7 @@ public EnvMultinodeTls( this.serverPackage = requireNonNull(serverPackage, "serverPackage is null"); this.debug = debug; this.tracing = tracing; + this.ipv6 = ipv6; } @Override @@ -97,7 +101,7 @@ public void extendEnvironment(Environment.Builder builder) private DockerContainer createTrinoWorker(String workerName) { - return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, "ghcr.io/trinodb/testing/almalinux9-oj17:" + imagesVersion, workerName) + return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, ipv6, "ghcr.io/trinodb/testing/almalinux9-oj17:" + imagesVersion, workerName) .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withDomainName("docker.cluster")) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls/config-worker.properties")), CONTAINER_TRINO_CONFIG_PROPERTIES) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("common/hadoop/hive.properties")), CONTAINER_TRINO_HIVE_PROPERTIES) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberos.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberos.java index 9d09ea8a5192..55583a81326a 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberos.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberos.java @@ -21,6 +21,7 @@ import io.trino.tests.product.launcher.env.Environment; import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.Ipv6; import io.trino.tests.product.launcher.env.ServerPackage; import io.trino.tests.product.launcher.env.Tracing; import io.trino.tests.product.launcher.env.common.HadoopKerberos; @@ -52,6 +53,7 @@ public final class EnvMultinodeTlsKerberos private final File serverPackage; private final boolean debug; private final boolean tracing; + private final boolean ipv6; @Inject public EnvMultinodeTlsKerberos( @@ -62,10 +64,12 @@ public EnvMultinodeTlsKerberos( @ServerPackage File serverPackage, JdkProvider jdkProvider, @Debug boolean debug, - @Tracing boolean tracing) + @Tracing boolean tracing, + @Ipv6 boolean ipv6) { super(ImmutableList.of(standard, hadoopKerberos)); this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); + this.ipv6 = ipv6; String hadoopBaseImage = config.getHadoopBaseImage(); String hadoopImagesVersion = config.getHadoopImagesVersion(); this.trinoDockerImageName = hadoopBaseImage + "-kerberized:" + hadoopImagesVersion; @@ -93,7 +97,7 @@ public void extendEnvironment(Environment.Builder builder) @SuppressWarnings("resource") private DockerContainer createTrinoWorker(String workerName) { - return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, trinoDockerImageName, workerName) + return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, ipv6, trinoDockerImageName, workerName) .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withDomainName("docker.cluster")) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/config-worker.properties")), CONTAINER_TRINO_CONFIG_PROPERTIES) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/hive.properties")), CONTAINER_TRINO_HIVE_PROPERTIES) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberosDelegation.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberosDelegation.java index 5a9fb0cd1578..9dc8cd3e5428 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberosDelegation.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeTlsKerberosDelegation.java @@ -21,6 +21,7 @@ import io.trino.tests.product.launcher.env.Environment; import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.Ipv6; import io.trino.tests.product.launcher.env.ServerPackage; import io.trino.tests.product.launcher.env.Tracing; import io.trino.tests.product.launcher.env.common.HadoopKerberos; @@ -55,6 +56,7 @@ public final class EnvMultinodeTlsKerberosDelegation private final File serverPackage; private final boolean debug; private final boolean tracing; + private final boolean ipv6; @Inject public EnvMultinodeTlsKerberosDelegation( @@ -65,11 +67,13 @@ public EnvMultinodeTlsKerberosDelegation( @ServerPackage File serverPackage, JdkProvider jdkProvider, @Debug boolean debug, - @Tracing boolean tracing) + @Tracing boolean tracing, + @Ipv6 boolean ipv6) { super(ImmutableList.of(standard, hadoopKerberos)); this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/multinode-tls-kerberos-delegation"); this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); + this.ipv6 = ipv6; String hadoopBaseImage = config.getHadoopBaseImage(); String hadoopImagesVersion = config.getHadoopImagesVersion(); this.trinoDockerImageName = hadoopBaseImage + "-kerberized:" + hadoopImagesVersion; @@ -102,7 +106,7 @@ public void extendEnvironment(Environment.Builder builder) @SuppressWarnings("resource") private DockerContainer createTrinoWorker(String workerName) { - return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, trinoDockerImageName, workerName) + return createTrinoContainer(dockerFiles, serverPackage, jdkProvider, debug, tracing, ipv6, trinoDockerImageName, workerName) .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withDomainName("docker.cluster")) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/config-worker.properties")), CONTAINER_TRINO_CONFIG_PROPERTIES) .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/hive.properties")), CONTAINER_TRINO_HIVE_PROPERTIES) From 67e9cceff1686c610a74a4c88ea39dee621a25e3 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 15:25:25 +0100 Subject: [PATCH 082/158] Update nimbus-jose-jwt to 9.48 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 492ba2be73c8..cf34692c898f 100644 --- a/pom.xml +++ b/pom.xml @@ -604,7 +604,7 @@ com.nimbusds nimbus-jose-jwt - 9.47 + 9.48 From dcdb3c60fcc28739f5d835a9dcd718491843fbe5 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 15:26:09 +0100 Subject: [PATCH 083/158] Update jna to 5.16.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cf34692c898f..393e81b4b13e 100644 --- a/pom.xml +++ b/pom.xml @@ -201,7 +201,7 @@ 1.45.3 5.3.1 1.7.1 - 5.15.0 + 5.16.0 2.12.7 0.12.6 1.20.0 From 0ff4964876ca3b8c40944764f42044756ef0fe9c Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 15:26:38 +0100 Subject: [PATCH 084/158] Update AWS SDK v2 to 2.29.43 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 393e81b4b13e..ba15555437a4 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ software.amazon.awssdk bom - 2.29.37 + 2.29.43 pom import From 0d11519d7a1affeefaa61a160ca7e2cb62968219 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 15:27:46 +0100 Subject: [PATCH 085/158] Update openlineage-java to 1.26.0 --- plugin/trino-openlineage/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-openlineage/pom.xml b/plugin/trino-openlineage/pom.xml index dfb0963dc7fa..3d54ff07a318 100644 --- a/plugin/trino-openlineage/pom.xml +++ b/plugin/trino-openlineage/pom.xml @@ -61,7 +61,7 @@ io.openlineage openlineage-java - 1.25.0 + 1.26.0 From 6519eda62184d1a2fe698b8251e12381d4745b66 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 15:28:31 +0100 Subject: [PATCH 086/158] Update airbase to 209 --- pom.xml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ba15555437a4..01afad752c30 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.airlift airbase - 206 + 209 io.trino @@ -2522,6 +2522,15 @@ com/google/inject/Provider + + + java/io/File."<init>":(Ljava/lang/String;)V + java/io/FileInputStream."<init>":(Ljava/io/File;)V + java/io/FileInputStream."<init>":(Ljava/lang/String;)V + java/io/FileOutputStream."<init>":(Ljava/io/File;)V + java/io/FileOutputStream."<init>":(Ljava/lang/String;)V + java/io/File."<init>":(Ljava/net/URI;)V + java/io/File."<init>":(Ljava/io/File;Ljava/lang/String;)V From 52d15e072c800b1997f6e2dbaac472eeb5c5a1e9 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 15:34:55 +0100 Subject: [PATCH 087/158] Update airlift to 294 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 01afad752c30..5c79c3ca8c63 100644 --- a/pom.xml +++ b/pom.xml @@ -184,7 +184,7 @@ ${air.test.jvm.additional-arguments.default} - 293 + 294 2.9.6 4.13.2 1.12.0 From e8cc8e3f71aa2d150bbb5ad4d553bfa6f9f5d401 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 15:42:13 +0100 Subject: [PATCH 088/158] Update freemarker 2.3.34 --- plugin/trino-druid/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-druid/pom.xml b/plugin/trino-druid/pom.xml index 4a2f5a765657..dc3bb142434b 100644 --- a/plugin/trino-druid/pom.xml +++ b/plugin/trino-druid/pom.xml @@ -198,7 +198,7 @@ org.freemarker freemarker - 2.3.33 + 2.3.34 test From effac4fd0dcb66ee261be103130507296830b43f Mon Sep 17 00:00:00 2001 From: takezoe Date: Sun, 8 Dec 2024 11:16:18 +0900 Subject: [PATCH 089/158] Remove unreachable code in OrderedPeriodParser --- .../java/io/trino/util/DateTimeUtils.java | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/util/DateTimeUtils.java b/core/trino-main/src/main/java/io/trino/util/DateTimeUtils.java index b364630f4a4e..e1ac2dd1e266 100644 --- a/core/trino-main/src/main/java/io/trino/util/DateTimeUtils.java +++ b/core/trino-main/src/main/java/io/trino/util/DateTimeUtils.java @@ -408,8 +408,6 @@ public int parseInto(ReadWritablePeriod period, String text, int position, Local int bestValidPos = position; ReadWritablePeriod bestValidPeriod = null; - int bestInvalidPos = position; - for (PeriodParser parser : parsers) { ReadWritablePeriod parsedPeriod = new MutablePeriod(); int parsePos = parser.parseInto(parsedPeriod, text, position, locale); @@ -422,23 +420,13 @@ public int parseInto(ReadWritablePeriod period, String text, int position, Local } } } - else if (parsePos < 0) { - parsePos = ~parsePos; - if (parsePos > bestInvalidPos) { - bestInvalidPos = parsePos; - } - } } - if (bestValidPos > position || (bestValidPos == position)) { - // Restore the state to the best valid parse. - if (bestValidPeriod != null) { - period.setPeriod(bestValidPeriod); - } - return bestValidPos; + // Restore the state to the best valid parse. + if (bestValidPeriod != null) { + period.setPeriod(bestValidPeriod); } - - return ~bestInvalidPos; + return bestValidPos; } } } From 067b0eab4790982d7b1a5b84f20d442391108747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sat, 28 Dec 2024 11:52:46 +0100 Subject: [PATCH 090/158] Avoid parsing min and max twice --- .../trino/plugin/faker/FakerColumnHandle.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java index 941103460d22..96d870eafb8e 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java @@ -63,18 +63,16 @@ public static FakerColumnHandle of(int columnId, ColumnMetadata column, double d if (generator != null && !isCharacterColumn(column)) { throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property can only be set for CHAR, VARCHAR or VARBINARY columns".formatted(GENERATOR_PROPERTY)); } - // only parse min, max, and options to validate literals - FakerColumnHandle needs to be serializable, - // and some internal Trino types are not (Int128, LongTimestamp, LongTimestampWithTimeZone), so they cannot be stored in the handle as native types - String min = (String) column.getProperties().get(MIN_PROPERTY); + Object min; try { - Literal.parse(min, column.getType()); + min = Literal.parse((String) column.getProperties().get(MIN_PROPERTY), column.getType()); } catch (IllegalArgumentException e) { throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MIN_PROPERTY, column.getType().getDisplayName()), e); } - String max = (String) column.getProperties().get(MAX_PROPERTY); + Object max; try { - Literal.parse(max, column.getType()); + max = Literal.parse((String) column.getProperties().get(MAX_PROPERTY), column.getType()); } catch (IllegalArgumentException e) { throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MAX_PROPERTY, column.getType().getDisplayName()), e); @@ -116,19 +114,19 @@ private static boolean isCharacterColumn(ColumnMetadata column) return column.getType() instanceof CharType || column.getType() instanceof VarcharType || column.getType() instanceof VarbinaryType; } - private static Range range(Type type, String min, String max) + private static Range range(Type type, Object min, Object max) { requireNonNull(type, "type is null"); if (min == null && max == null) { return Range.all(type); } if (max == null) { - return Range.greaterThanOrEqual(type, Literal.parse(min, type)); + return Range.greaterThanOrEqual(type, min); } if (min == null) { - return Range.lessThanOrEqual(type, Literal.parse(max, type)); + return Range.lessThanOrEqual(type, max); } - return Range.range(type, Literal.parse(min, type), true, Literal.parse(max, type), true); + return Range.range(type, min, true, max, true); } private static List strings(Collection values) From adf747440a345d0a5eb5ab549180b4e0e8256107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sat, 28 Dec 2024 11:54:38 +0100 Subject: [PATCH 091/158] Avoid and/or in Faker docs --- docs/src/main/sphinx/connector/faker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/main/sphinx/connector/faker.md b/docs/src/main/sphinx/connector/faker.md index 904307253323..d5fb56791659 100644 --- a/docs/src/main/sphinx/connector/faker.md +++ b/docs/src/main/sphinx/connector/faker.md @@ -175,7 +175,7 @@ Faker supports the following non-character types: - `UUID` You can not use generator expressions for non-character-based columns. To limit -their data range, set the `min` and/or `max` column properties - see +their data range, set the `min` and `max` column properties - see [](faker-usage). ### Unsupported types From adf74c10baa6c93f99a877c804e2edbbd78f0875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sat, 28 Dec 2024 13:39:09 +0100 Subject: [PATCH 092/158] Refactor FakerPageSource to have fewer faker references --- .../trino/plugin/faker/FakerPageSource.java | 89 +++++++++++++++---- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java index d911ebc10db5..41f1c638dc40 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -41,6 +41,7 @@ import io.trino.type.IpAddressType; import net.datafaker.Faker; +import java.math.BigDecimal; import java.math.BigInteger; import java.net.Inet4Address; import java.net.UnknownHostException; @@ -111,6 +112,8 @@ class FakerPageSource private final Random random; private final Faker faker; + private final SentenceGenerator sentenceGenerator; + private final BoundedSentenceGenerator boundedSentenceGenerator; private final long limit; private final List generators; private long completedRows; @@ -128,6 +131,8 @@ class FakerPageSource { this.faker = requireNonNull(faker, "faker is null"); this.random = requireNonNull(random, "random is null"); + this.sentenceGenerator = () -> Slices.utf8Slice(faker.lorem().sentence(3 + random.nextInt(38))); + this.boundedSentenceGenerator = (maxLength) -> Slices.utf8Slice(faker.lorem().maxLengthSentence(maxLength)); List types = requireNonNull(columns, "columns is null") .stream() .map(FakerColumnHandle::type) @@ -308,7 +313,7 @@ private Generator randomValueGenerator(FakerColumnHandle handle) if (!range.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for varbinary columns are not supported"); } - return (blockBuilder) -> varType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().sentence(3 + random.nextInt(38)))); + return (blockBuilder) -> varType.writeSlice(blockBuilder, sentenceGenerator.get()); } if (type instanceof VarcharType varcharType) { if (!range.isAll()) { @@ -316,15 +321,15 @@ private Generator randomValueGenerator(FakerColumnHandle handle) } if (varcharType.getLength().isPresent()) { int length = varcharType.getLength().get(); - return (blockBuilder) -> varcharType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().maxLengthSentence(random.nextInt(length)))); + return (blockBuilder) -> varcharType.writeSlice(blockBuilder, boundedSentenceGenerator.get(random.nextInt(length))); } - return (blockBuilder) -> varcharType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().sentence(3 + random.nextInt(38)))); + return (blockBuilder) -> varcharType.writeSlice(blockBuilder, sentenceGenerator.get()); } if (type instanceof CharType charType) { if (!range.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for char columns are not supported"); } - return (blockBuilder) -> charType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().maxLengthSentence(charType.getLength()))); + return (blockBuilder) -> charType.writeSlice(blockBuilder, boundedSentenceGenerator.get(charType.getLength())); } // not supported: ROW, ARRAY, MAP, JSON if (type instanceof IpAddressType) { @@ -435,7 +440,7 @@ private long generateLong(Range range, long factor) private long generateLongDefaults(Range range, long factor, long min, long max) { - return faker.number().numberBetween( + return numberBetween( roundDiv((long) range.getLowValue().orElse(min), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), // TODO does the inclusion only apply to positive numbers? roundDiv((long) range.getHighValue().orElse(max), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)) * factor; @@ -443,21 +448,21 @@ private long generateLongDefaults(Range range, long factor, long min, long max) private int generateInt(Range range) { - return (int) faker.number().numberBetween( + return (int) numberBetween( (long) range.getLowValue().orElse((long) Integer.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), (long) range.getHighValue().orElse((long) Integer.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); } private short generateShort(Range range) { - return (short) faker.number().numberBetween( + return (short) numberBetween( (long) range.getLowValue().orElse((long) Short.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), (long) range.getHighValue().orElse((long) Short.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); } private byte generateTiny(Range range) { - return (byte) faker.number().numberBetween( + return (byte) numberBetween( (long) range.getLowValue().orElse((long) Byte.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), (long) range.getHighValue().orElse((long) Byte.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); } @@ -550,7 +555,7 @@ private Generator timestampGenerator(Range range, TimestampType tzType) LongTimestamp finalLow = low; LongTimestamp finalHigh = high; return (blockBuilder) -> { - long epochMicros = faker.number().numberBetween(finalLow.getEpochMicros(), finalHigh.getEpochMicros()); + long epochMicros = numberBetween(finalLow.getEpochMicros(), finalHigh.getEpochMicros()); if (tzType.getPrecision() <= 6) { epochMicros *= factor; tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros * factor, 0)); @@ -558,17 +563,17 @@ private Generator timestampGenerator(Range range, TimestampType tzType) } int picosOfMicro; if (epochMicros == finalLow.getEpochMicros()) { - picosOfMicro = faker.number().numberBetween( + picosOfMicro = numberBetween( finalLow.getPicosOfMicro(), finalLow.getEpochMicros() == finalHigh.getEpochMicros() ? finalHigh.getPicosOfMicro() : (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); } else if (epochMicros == finalHigh.getEpochMicros()) { - picosOfMicro = faker.number().numberBetween(0, finalHigh.getPicosOfMicro()); + picosOfMicro = numberBetween(0, finalHigh.getPicosOfMicro()); } else { - picosOfMicro = faker.number().numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); + picosOfMicro = numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); } tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros, picosOfMicro * factor)); }; @@ -584,7 +589,7 @@ private Generator timestampWithTimeZoneGenerator(Range range, TimestampWithTimeZ .orElse(TimeZoneKey.UTC_KEY)); long factor = POWERS_OF_TEN[3 - tzType.getPrecision()]; return (blockBuilder) -> { - long millis = faker.number().numberBetween( + long millis = numberBetween( roundDiv(unpackMillisUtc((long) range.getLowValue().orElse(Long.MIN_VALUE)), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), roundDiv(unpackMillisUtc((long) range.getHighValue().orElse(Long.MAX_VALUE)), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)) * factor; tzType.writeLong(blockBuilder, packDateTimeWithZone(millis, defaultTZ)); @@ -616,20 +621,20 @@ private Generator timestampWithTimeZoneGenerator(Range range, TimestampWithTimeZ LongTimestampWithTimeZone finalLow = low; LongTimestampWithTimeZone finalHigh = high; return (blockBuilder) -> { - long millis = faker.number().numberBetween(finalLow.getEpochMillis(), finalHigh.getEpochMillis()); + long millis = numberBetween(finalLow.getEpochMillis(), finalHigh.getEpochMillis()); int picosOfMilli; if (millis == finalLow.getEpochMillis()) { - picosOfMilli = faker.number().numberBetween( + picosOfMilli = numberBetween( finalLow.getPicosOfMilli(), finalLow.getEpochMillis() == finalHigh.getEpochMillis() ? finalHigh.getPicosOfMilli() : (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1); } else if (millis == finalHigh.getEpochMillis()) { - picosOfMilli = faker.number().numberBetween(0, finalHigh.getPicosOfMilli()); + picosOfMilli = numberBetween(0, finalHigh.getPicosOfMilli()); } else { - picosOfMilli = faker.number().numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1); + picosOfMilli = numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1); } tzType.writeObject(blockBuilder, fromEpochMillisAndFraction(millis, picosOfMilli * factor, defaultTZ)); }; @@ -647,7 +652,7 @@ private Generator timeWithTimeZoneGenerator(Range range, TimeWithTimeZoneType ti long low = roundDiv(range.getLowValue().map(v -> unpackTimeNanos((long) v)).orElse(0L), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); long high = roundDiv(range.getHighValue().map(v -> unpackTimeNanos((long) v)).orElse(NANOSECONDS_PER_DAY), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); return (blockBuilder) -> { - long nanos = faker.number().numberBetween(low, high) * factor; + long nanos = numberBetween(low, high) * factor; timeType.writeLong(blockBuilder, packTimeWithTimeZone(nanos, offsetMinutes)); }; } @@ -667,11 +672,45 @@ private Generator timeWithTimeZoneGenerator(Range range, TimeWithTimeZoneType ti long longLow = roundDiv(low.getPicoseconds(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); long longHigh = roundDiv(high.getPicoseconds(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); return (blockBuilder) -> { - long picoseconds = faker.number().numberBetween(longLow, longHigh) * factor; + long picoseconds = numberBetween(longLow, longHigh) * factor; timeType.writeObject(blockBuilder, new LongTimeWithTimeZone(picoseconds, offsetMinutes)); }; } + private int numberBetween(int min, int max) + { + if (min == max) { + return min; + } + final int realMin = Math.min(min, max); + final int realMax = Math.max(min, max); + final int amplitude = realMax - realMin; + if (amplitude >= 0) { + return random.nextInt(amplitude) + realMin; + } + // handle overflow + return (int) numberBetween(realMin, (long) realMax); + } + + private long numberBetween(long min, long max) + { + if (min == max) { + return min; + } + final long realMin = Math.min(min, max); + final long realMax = Math.max(min, max); + final long amplitude = realMax - realMin; + if (amplitude >= 0) { + return random.nextLong(amplitude) + realMin; + } + // handle overflow + final BigDecimal bigMin = BigDecimal.valueOf(min); + final BigDecimal bigMax = BigDecimal.valueOf(max); + final BigDecimal randomValue = BigDecimal.valueOf(random.nextDouble()); + + return bigMin.add(bigMax.subtract(bigMin).multiply(randomValue)).longValue(); + } + private Generator generateIpV4(Range range) { if (!range.isAll()) { @@ -726,4 +765,16 @@ private interface Generator { void accept(BlockBuilder blockBuilder); } + + @FunctionalInterface + private interface SentenceGenerator + { + Slice get(); + } + + @FunctionalInterface + private interface BoundedSentenceGenerator + { + Slice get(int maxLength); + } } From c68f022a44afa97598a970ee118a8ef843fad2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sat, 28 Dec 2024 18:12:56 +0100 Subject: [PATCH 093/158] Extract typed ranges in Faker's page source --- .../trino/plugin/faker/FakerPageSource.java | 473 ++++++++++-------- 1 file changed, 270 insertions(+), 203 deletions(-) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java index 41f1c638dc40..e2b671f35bd8 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -250,9 +250,9 @@ private Generator constraintedValueGenerator(FakerColumnHandle handle) private Generator randomValueGenerator(FakerColumnHandle handle) { - Range range = handle.domain().getValues().getRanges().getSpan(); + Range genericRange = handle.domain().getValues().getRanges().getSpan(); if (handle.generator() != null) { - if (!range.isAll()) { + if (!genericRange.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for columns with a generator expression are not supported"); } return (blockBuilder) -> VARCHAR.writeSlice(blockBuilder, Slices.utf8Slice(faker.expression(handle.generator()))); @@ -260,63 +260,73 @@ private Generator randomValueGenerator(FakerColumnHandle handle) Type type = handle.type(); // check every type in order defined in StandardTypes if (BIGINT.equals(type)) { - return (blockBuilder) -> BIGINT.writeLong(blockBuilder, generateLong(range, 1)); + LongRange range = LongRange.of(genericRange); + return (blockBuilder) -> BIGINT.writeLong(blockBuilder, numberBetween(range.low, range.high)); } if (INTEGER.equals(type)) { - return (blockBuilder) -> INTEGER.writeLong(blockBuilder, generateInt(range)); + IntRange range = IntRange.of(genericRange); + return (blockBuilder) -> INTEGER.writeLong(blockBuilder, numberBetween(range.low, range.high)); } if (SMALLINT.equals(type)) { - return (blockBuilder) -> SMALLINT.writeLong(blockBuilder, generateShort(range)); + IntRange range = IntRange.of(genericRange, Short.MIN_VALUE, Short.MAX_VALUE); + return (blockBuilder) -> SMALLINT.writeLong(blockBuilder, numberBetween(range.low, range.high)); } if (TINYINT.equals(type)) { - return (blockBuilder) -> TINYINT.writeLong(blockBuilder, generateTiny(range)); + IntRange range = IntRange.of(genericRange, Byte.MIN_VALUE, Byte.MAX_VALUE); + return (blockBuilder) -> TINYINT.writeLong(blockBuilder, numberBetween(range.low, range.high)); } if (BOOLEAN.equals(type)) { - if (!range.isAll()) { + if (!genericRange.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Range or not a single value predicates for boolean columns are not supported"); } return (blockBuilder) -> BOOLEAN.writeBoolean(blockBuilder, random.nextBoolean()); } if (DATE.equals(type)) { - return (blockBuilder) -> DATE.writeLong(blockBuilder, generateInt(range)); + IntRange range = IntRange.of(genericRange); + return (blockBuilder) -> DATE.writeLong(blockBuilder, numberBetween(range.low, range.high)); } if (type instanceof DecimalType decimalType) { - return decimalGenerator(range, decimalType); + return decimalGenerator(genericRange, decimalType); } if (REAL.equals(type)) { - return (blockBuilder) -> REAL.writeLong(blockBuilder, floatToRawIntBits(generateFloat(range))); + FloatRange range = FloatRange.of(genericRange); + return (blockBuilder) -> REAL.writeLong(blockBuilder, floatToRawIntBits(range.low + (range.high - range.low) * random.nextFloat())); } if (DOUBLE.equals(type)) { - return (blockBuilder) -> DOUBLE.writeDouble(blockBuilder, generateDouble(range)); + DoubleRange range = DoubleRange.of(genericRange); + return (blockBuilder) -> DOUBLE.writeDouble(blockBuilder, range.low + (range.high - range.low) * random.nextDouble()); } // not supported: HYPER_LOG_LOG, QDIGEST, TDIGEST, P4_HYPER_LOG_LOG if (INTERVAL_DAY_TIME.equals(type)) { - return (blockBuilder) -> INTERVAL_DAY_TIME.writeLong(blockBuilder, generateLong(range, 1)); + LongRange range = LongRange.of(genericRange); + return (blockBuilder) -> INTERVAL_DAY_TIME.writeLong(blockBuilder, numberBetween(range.low, range.high)); } if (INTERVAL_YEAR_MONTH.equals(type)) { - return (blockBuilder) -> INTERVAL_YEAR_MONTH.writeLong(blockBuilder, generateInt(range)); + IntRange range = IntRange.of(genericRange); + return (blockBuilder) -> INTERVAL_YEAR_MONTH.writeLong(blockBuilder, numberBetween(range.low, range.high)); } if (type instanceof TimestampType) { - return timestampGenerator(range, (TimestampType) type); + return timestampGenerator(genericRange, (TimestampType) type); } if (type instanceof TimestampWithTimeZoneType) { - return timestampWithTimeZoneGenerator(range, (TimestampWithTimeZoneType) type); + return timestampWithTimeZoneGenerator(genericRange, (TimestampWithTimeZoneType) type); } if (type instanceof TimeType timeType) { long factor = POWERS_OF_TEN[12 - timeType.getPrecision()]; - return (blockBuilder) -> timeType.writeLong(blockBuilder, generateLongDefaults(range, factor, 0, PICOSECONDS_PER_DAY)); + LongRange range = LongRange.of(genericRange, factor, 0, PICOSECONDS_PER_DAY); + return (blockBuilder) -> timeType.writeLong(blockBuilder, numberBetween(range.low, range.high) * factor); } if (type instanceof TimeWithTimeZoneType) { - return timeWithTimeZoneGenerator(range, (TimeWithTimeZoneType) type); + return timeWithTimeZoneGenerator(genericRange, (TimeWithTimeZoneType) type); } if (type instanceof VarbinaryType varType) { - if (!range.isAll()) { + if (!genericRange.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for varbinary columns are not supported"); } return (blockBuilder) -> varType.writeSlice(blockBuilder, sentenceGenerator.get()); } if (type instanceof VarcharType varcharType) { - if (!range.isAll()) { + if (!genericRange.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for varchar columns are not supported"); } if (varcharType.getLength().isPresent()) { @@ -326,18 +336,18 @@ private Generator randomValueGenerator(FakerColumnHandle handle) return (blockBuilder) -> varcharType.writeSlice(blockBuilder, sentenceGenerator.get()); } if (type instanceof CharType charType) { - if (!range.isAll()) { + if (!genericRange.isAll()) { throw new TrinoException(INVALID_ROW_FILTER, "Predicates for char columns are not supported"); } return (blockBuilder) -> charType.writeSlice(blockBuilder, boundedSentenceGenerator.get(charType.getLength())); } // not supported: ROW, ARRAY, MAP, JSON if (type instanceof IpAddressType) { - return generateIpV4(range); + return generateIpV4(genericRange); } // not supported: GEOMETRY if (type instanceof UuidType) { - return generateUUID(range); + return generateUUID(genericRange); } throw new IllegalArgumentException("Unsupported type " + type); @@ -433,116 +443,213 @@ private ObjectWriter objectWriter(Type type) throw new IllegalArgumentException("Unsupported type " + type); } - private long generateLong(Range range, long factor) + private Generator decimalGenerator(Range genericRange, DecimalType decimalType) { - return generateLongDefaults(range, factor, Long.MIN_VALUE, Long.MAX_VALUE); + if (decimalType.isShort()) { + ShortDecimalRange range = ShortDecimalRange.of(genericRange, decimalType.getPrecision()); + return (blockBuilder) -> decimalType.writeLong(blockBuilder, numberBetween(range.low, range.high)); + } + Int128Range range = Int128Range.of(genericRange); + BigInteger currentRange = BigInteger.valueOf(Long.MAX_VALUE); + BigInteger desiredRange = range.high.toBigInteger().subtract(range.low.toBigInteger()); + return (blockBuilder) -> decimalType.writeObject(blockBuilder, Int128.valueOf( + new BigInteger(63, random).multiply(desiredRange).divide(currentRange).add(range.low.toBigInteger()))); } - private long generateLongDefaults(Range range, long factor, long min, long max) + private Generator timestampGenerator(Range genericRange, TimestampType tzType) { - return numberBetween( - roundDiv((long) range.getLowValue().orElse(min), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - // TODO does the inclusion only apply to positive numbers? - roundDiv((long) range.getHighValue().orElse(max), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)) * factor; + if (tzType.isShort()) { + long factor = POWERS_OF_TEN[6 - tzType.getPrecision()]; + LongRange range = LongRange.of(genericRange, factor); + return (blockBuilder) -> tzType.writeLong(blockBuilder, numberBetween(range.low, range.high) * factor); + } + LongTimestampRange range = LongTimestampRange.of(genericRange, tzType.getPrecision()); + if (tzType.getPrecision() <= 6) { + return (blockBuilder) -> { + long epochMicros = numberBetween(range.low.getEpochMicros(), range.high.getEpochMicros()); + tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros * range.factor, 0)); + }; + } + return (blockBuilder) -> { + long epochMicros = numberBetween(range.low.getEpochMicros(), range.high.getEpochMicros()); + int picosOfMicro; + if (epochMicros == range.low.getEpochMicros()) { + picosOfMicro = numberBetween( + range.low.getPicosOfMicro(), + range.low.getEpochMicros() == range.high.getEpochMicros() ? + range.high.getPicosOfMicro() + : (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); + } + else if (epochMicros == range.high.getEpochMicros()) { + picosOfMicro = numberBetween(0, range.high.getPicosOfMicro()); + } + else { + picosOfMicro = numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); + } + tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros, picosOfMicro * range.factor)); + }; } - private int generateInt(Range range) + private Generator timestampWithTimeZoneGenerator(Range genericRange, TimestampWithTimeZoneType tzType) { - return (int) numberBetween( - (long) range.getLowValue().orElse((long) Integer.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - (long) range.getHighValue().orElse((long) Integer.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); + if (tzType.isShort()) { + ShortTimestampWithTimeZoneRange range = ShortTimestampWithTimeZoneRange.of(genericRange, tzType.getPrecision()); + return (blockBuilder) -> { + long millis = numberBetween(range.low, range.high) * range.factor; + tzType.writeLong(blockBuilder, packDateTimeWithZone(millis, range.defaultTZ)); + }; + } + LongTimestampWithTimeZoneRange range = LongTimestampWithTimeZoneRange.of(genericRange, tzType.getPrecision()); + int picosOfMilliHigh = (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1; + return (blockBuilder) -> { + long millis = numberBetween(range.low.getEpochMillis(), range.high.getEpochMillis()); + int picosOfMilli; + if (millis == range.low.getEpochMillis()) { + picosOfMilli = numberBetween( + range.low.getPicosOfMilli(), + range.low.getEpochMillis() == range.high.getEpochMillis() ? + range.high.getPicosOfMilli() + : picosOfMilliHigh); + } + else if (millis == range.high.getEpochMillis()) { + picosOfMilli = numberBetween(0, range.high.getPicosOfMilli()); + } + else { + picosOfMilli = numberBetween(0, picosOfMilliHigh); + } + tzType.writeObject(blockBuilder, fromEpochMillisAndFraction(millis, picosOfMilli * range.factor, range.defaultTZ)); + }; } - private short generateShort(Range range) + private Generator timeWithTimeZoneGenerator(Range genericRange, TimeWithTimeZoneType timeType) { - return (short) numberBetween( - (long) range.getLowValue().orElse((long) Short.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - (long) range.getHighValue().orElse((long) Short.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); + if (timeType.isShort()) { + ShortTimeWithTimeZoneRange range = ShortTimeWithTimeZoneRange.of(genericRange, timeType.getPrecision()); + return (blockBuilder) -> { + long nanos = numberBetween(range.low, range.high) * range.factor; + timeType.writeLong(blockBuilder, packTimeWithTimeZone(nanos, range.offsetMinutes)); + }; + } + LongTimeWithTimeZoneRange range = LongTimeWithTimeZoneRange.of(genericRange, timeType.getPrecision()); + return (blockBuilder) -> { + long picoseconds = numberBetween(range.low, range.high) * range.factor; + timeType.writeObject(blockBuilder, new LongTimeWithTimeZone(picoseconds, range.offsetMinutes)); + }; } - private byte generateTiny(Range range) + private record LongRange(long low, long high) { - return (byte) numberBetween( - (long) range.getLowValue().orElse((long) Byte.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - (long) range.getHighValue().orElse((long) Byte.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); - } + static LongRange of(Range range) + { + return of(range, 1, Long.MIN_VALUE, Long.MAX_VALUE); + } - private float generateFloat(Range range) - { - // TODO normalize ranges in applyFilter, so they always have bounds - float minValue = range.getLowValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MIN_VALUE); - if (!range.isLowUnbounded() && !range.isLowInclusive()) { - minValue = Math.nextUp(minValue); + static LongRange of(Range range, long factor) + { + return of(range, factor, Long.MIN_VALUE, Long.MAX_VALUE); } - float maxValue = range.getHighValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MAX_VALUE); - if (!range.isHighUnbounded() && !range.isHighInclusive()) { - maxValue = Math.nextDown(maxValue); + + static LongRange of(Range range, long factor, long defaultMin, long defaultMax) + { + return new LongRange( + roundDiv((long) range.getLowValue().orElse(defaultMin), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + roundDiv((long) range.getHighValue().orElse(defaultMax), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); } - return minValue + (maxValue - minValue) * random.nextFloat(); } - private double generateDouble(Range range) + private record IntRange(int low, int high) { - double minValue = (double) range.getLowValue().orElse(Double.MIN_VALUE); - if (!range.isLowUnbounded() && !range.isLowInclusive()) { - minValue = Math.nextUp(minValue); + static IntRange of(Range range) + { + return of(range, Integer.MIN_VALUE, Integer.MAX_VALUE); } - double maxValue = (double) range.getHighValue().orElse(Double.MAX_VALUE); - if (!range.isHighUnbounded() && !range.isHighInclusive()) { - maxValue = Math.nextDown(maxValue); + + static IntRange of(Range range, long defaultMin, long defaultMax) + { + return new IntRange( + toIntExact((long) range.getLowValue().orElse(defaultMin)) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + toIntExact((long) range.getHighValue().orElse(defaultMax)) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); } - return minValue + (maxValue - minValue) * random.nextDouble(); } - private Generator decimalGenerator(Range range, DecimalType decimalType) + private record FloatRange(float low, float high) { - if (decimalType.isShort()) { - long min = -999999999999999999L / POWERS_OF_TEN[18 - decimalType.getPrecision()]; - long max = 999999999999999999L / POWERS_OF_TEN[18 - decimalType.getPrecision()]; - return (blockBuilder) -> decimalType.writeLong(blockBuilder, generateLongDefaults(range, 1, min, max)); - } - Int128 low = (Int128) range.getLowValue().orElse(Decimals.MIN_UNSCALED_DECIMAL); - Int128 high = (Int128) range.getHighValue().orElse(Decimals.MAX_UNSCALED_DECIMAL); - if (!range.isLowUnbounded() && !range.isLowInclusive()) { - long[] result = new long[2]; - Int128Math.add(low.getHigh(), low.getLow(), 0, 1, result, 0); - low = Int128.valueOf(result); - } - if (!range.isHighUnbounded() && range.isHighInclusive()) { - long[] result = new long[2]; - Int128Math.add(high.getHigh(), high.getLow(), 0, 1, result, 0); - high = Int128.valueOf(result); + static FloatRange of(Range range) + { + float low = range.getLowValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MIN_VALUE); + if (!range.isLowUnbounded() && !range.isLowInclusive()) { + low = Math.nextUp(low); + } + float high = range.getHighValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MAX_VALUE); + if (!range.isHighUnbounded() && !range.isHighInclusive()) { + high = Math.nextDown(high); + } + return new FloatRange(low, high); } + } - BigInteger currentRange = BigInteger.valueOf(Long.MAX_VALUE); - BigInteger desiredRange = high.toBigInteger().subtract(low.toBigInteger()); - Int128 finalLow = low; - return (blockBuilder) -> decimalType.writeObject(blockBuilder, Int128.valueOf( - new BigInteger(63, random).multiply(desiredRange).divide(currentRange).add(finalLow.toBigInteger()))); + private record DoubleRange(double low, double high) + { + static DoubleRange of(Range range) + { + double low = (double) range.getLowValue().orElse(Double.MIN_VALUE); + if (!range.isLowUnbounded() && !range.isLowInclusive()) { + low = Math.nextUp(low); + } + double high = (double) range.getHighValue().orElse(Double.MAX_VALUE); + if (!range.isHighUnbounded() && !range.isHighInclusive()) { + high = Math.nextDown(high); + } + return new DoubleRange(low, high); + } } - private Generator timestampGenerator(Range range, TimestampType tzType) + private record ShortDecimalRange(long low, long high) { - if (tzType.isShort()) { - long factor = POWERS_OF_TEN[6 - tzType.getPrecision()]; - return (blockBuilder) -> tzType.writeLong(blockBuilder, generateLong(range, factor)); + static ShortDecimalRange of(Range range, int precision) + { + long defaultMin = -999999999999999999L / POWERS_OF_TEN[18 - precision]; + long defaultMax = 999999999999999999L / POWERS_OF_TEN[18 - precision]; + long low = (long) range.getLowValue().orElse(defaultMin) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + long high = (long) range.getHighValue().orElse(defaultMax) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + return new ShortDecimalRange(low, high); } - LongTimestamp low = (LongTimestamp) range.getLowValue() - .orElse(new LongTimestamp(Long.MIN_VALUE, 0)); - LongTimestamp high = (LongTimestamp) range.getHighValue() - .orElse(new LongTimestamp(Long.MAX_VALUE, PICOSECONDS_PER_MICROSECOND - 1)); - int factor; - if (tzType.getPrecision() <= 6) { - factor = (int) POWERS_OF_TEN[6 - tzType.getPrecision()]; - low = new LongTimestamp( - roundDiv(low.getEpochMicros(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - 0); - high = new LongTimestamp( - roundDiv(high.getEpochMicros(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0), - 0); + } + + private record Int128Range(Int128 low, Int128 high) + { + static Int128Range of(Range range) + { + Int128 low = (Int128) range.getLowValue().orElse(Decimals.MIN_UNSCALED_DECIMAL); + Int128 high = (Int128) range.getHighValue().orElse(Decimals.MAX_UNSCALED_DECIMAL); + if (!range.isLowUnbounded() && !range.isLowInclusive()) { + long[] result = new long[2]; + Int128Math.add(low.getHigh(), low.getLow(), 0, 1, result, 0); + low = Int128.valueOf(result); + } + if (!range.isHighUnbounded() && range.isHighInclusive()) { + long[] result = new long[2]; + Int128Math.add(high.getHigh(), high.getLow(), 0, 1, result, 0); + high = Int128.valueOf(result); + } + return new Int128Range(low, high); } - else { - factor = (int) POWERS_OF_TEN[12 - tzType.getPrecision()]; + } + + private record LongTimestampRange(LongTimestamp low, LongTimestamp high, int factor) + { + static LongTimestampRange of(Range range, int precision) + { + LongTimestamp low = (LongTimestamp) range.getLowValue().orElse(new LongTimestamp(Long.MIN_VALUE, 0)); + LongTimestamp high = (LongTimestamp) range.getHighValue().orElse(new LongTimestamp(Long.MAX_VALUE, PICOSECONDS_PER_MICROSECOND - 1)); + int factor; + if (precision <= 6) { + factor = (int) POWERS_OF_TEN[6 - precision]; + low = new LongTimestamp(roundDiv(low.getEpochMicros(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), 0); + high = new LongTimestamp(roundDiv(high.getEpochMicros(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0), 0); + return new LongTimestampRange(low, high, factor); + } + factor = (int) POWERS_OF_TEN[12 - precision]; int lowPicosOfMicro = roundDiv(low.getPicosOfMicro(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); low = new LongTimestamp( low.getEpochMicros() - (lowPicosOfMicro < 0 ? 1 : 0), @@ -551,130 +658,90 @@ private Generator timestampGenerator(Range range, TimestampType tzType) high = new LongTimestamp( high.getEpochMicros() + (highPicosOfMicro > factor ? 1 : 0), highPicosOfMicro % factor); + return new LongTimestampRange(low, high, factor); } - LongTimestamp finalLow = low; - LongTimestamp finalHigh = high; - return (blockBuilder) -> { - long epochMicros = numberBetween(finalLow.getEpochMicros(), finalHigh.getEpochMicros()); - if (tzType.getPrecision() <= 6) { - epochMicros *= factor; - tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros * factor, 0)); - return; - } - int picosOfMicro; - if (epochMicros == finalLow.getEpochMicros()) { - picosOfMicro = numberBetween( - finalLow.getPicosOfMicro(), - finalLow.getEpochMicros() == finalHigh.getEpochMicros() ? - finalHigh.getPicosOfMicro() - : (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); - } - else if (epochMicros == finalHigh.getEpochMicros()) { - picosOfMicro = numberBetween(0, finalHigh.getPicosOfMicro()); - } - else { - picosOfMicro = numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); - } - tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros, picosOfMicro * factor)); - }; } - private Generator timestampWithTimeZoneGenerator(Range range, TimestampWithTimeZoneType tzType) + private record ShortTimestampWithTimeZoneRange(long low, long high, long factor, TimeZoneKey defaultTZ) { - if (tzType.isShort()) { + static ShortTimestampWithTimeZoneRange of(Range range, int precision) + { TimeZoneKey defaultTZ = range.getLowValue() .map(v -> unpackZoneKey((long) v)) .orElse(range.getHighValue() .map(v -> unpackZoneKey((long) v)) .orElse(TimeZoneKey.UTC_KEY)); - long factor = POWERS_OF_TEN[3 - tzType.getPrecision()]; - return (blockBuilder) -> { - long millis = numberBetween( - roundDiv(unpackMillisUtc((long) range.getLowValue().orElse(Long.MIN_VALUE)), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - roundDiv(unpackMillisUtc((long) range.getHighValue().orElse(Long.MAX_VALUE)), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)) * factor; - tzType.writeLong(blockBuilder, packDateTimeWithZone(millis, defaultTZ)); - }; + long factor = POWERS_OF_TEN[3 - precision]; + long low = roundDiv(unpackMillisUtc((long) range.getLowValue().orElse(Long.MIN_VALUE)), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + long high = roundDiv(unpackMillisUtc((long) range.getHighValue().orElse(Long.MAX_VALUE)), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + return new ShortTimestampWithTimeZoneRange(low, high, factor, defaultTZ); } - short defaultTZ = range.getLowValue() - .map(v -> ((LongTimestampWithTimeZone) v).getTimeZoneKey()) - .orElse(range.getHighValue() - .map(v -> ((LongTimestampWithTimeZone) v).getTimeZoneKey()) - .orElse(TimeZoneKey.UTC_KEY.getKey())); - LongTimestampWithTimeZone low = (LongTimestampWithTimeZone) range.getLowValue() - .orElse(fromEpochMillisAndFraction(Long.MIN_VALUE >> 12, 0, defaultTZ)); - LongTimestampWithTimeZone high = (LongTimestampWithTimeZone) range.getHighValue() - .orElse(fromEpochMillisAndFraction(Long.MAX_VALUE >> 12, PICOSECONDS_PER_MILLISECOND - 1, defaultTZ)); - if (low.getTimeZoneKey() != high.getTimeZoneKey()) { - throw new TrinoException(INVALID_ROW_FILTER, "Range boundaries for timestamp with time zone columns must have the same time zone"); - } - int factor = (int) POWERS_OF_TEN[12 - tzType.getPrecision()]; - int lowPicosOfMilli = roundDiv(low.getPicosOfMilli(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); - low = fromEpochMillisAndFraction( - low.getEpochMillis() - (lowPicosOfMilli < 0 ? 1 : 0), - (lowPicosOfMilli + factor) % factor, - low.getTimeZoneKey()); - int highPicosOfMilli = roundDiv(high.getPicosOfMilli(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); - high = fromEpochMillisAndFraction( - high.getEpochMillis() + (highPicosOfMilli > factor ? 1 : 0), - highPicosOfMilli % factor, - high.getTimeZoneKey()); - LongTimestampWithTimeZone finalLow = low; - LongTimestampWithTimeZone finalHigh = high; - return (blockBuilder) -> { - long millis = numberBetween(finalLow.getEpochMillis(), finalHigh.getEpochMillis()); - int picosOfMilli; - if (millis == finalLow.getEpochMillis()) { - picosOfMilli = numberBetween( - finalLow.getPicosOfMilli(), - finalLow.getEpochMillis() == finalHigh.getEpochMillis() ? - finalHigh.getPicosOfMilli() - : (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1); - } - else if (millis == finalHigh.getEpochMillis()) { - picosOfMilli = numberBetween(0, finalHigh.getPicosOfMilli()); - } - else { - picosOfMilli = numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1); + } + + private record LongTimestampWithTimeZoneRange(LongTimestampWithTimeZone low, LongTimestampWithTimeZone high, int factor, short defaultTZ) + { + static LongTimestampWithTimeZoneRange of(Range range, int precision) + { + short defaultTZ = range.getLowValue() + .map(v -> ((LongTimestampWithTimeZone) v).getTimeZoneKey()) + .orElse(range.getHighValue() + .map(v -> ((LongTimestampWithTimeZone) v).getTimeZoneKey()) + .orElse(TimeZoneKey.UTC_KEY.getKey())); + LongTimestampWithTimeZone low = (LongTimestampWithTimeZone) range.getLowValue().orElse(fromEpochMillisAndFraction(Long.MIN_VALUE >> 12, 0, defaultTZ)); + LongTimestampWithTimeZone high = (LongTimestampWithTimeZone) range.getHighValue().orElse(fromEpochMillisAndFraction(Long.MAX_VALUE >> 12, PICOSECONDS_PER_MILLISECOND - 1, defaultTZ)); + if (low.getTimeZoneKey() != high.getTimeZoneKey()) { + throw new TrinoException(INVALID_ROW_FILTER, "Range boundaries for timestamp with time zone columns must have the same time zone"); } - tzType.writeObject(blockBuilder, fromEpochMillisAndFraction(millis, picosOfMilli * factor, defaultTZ)); - }; + int factor = (int) POWERS_OF_TEN[12 - precision]; + int lowPicosOfMilli = roundDiv(low.getPicosOfMilli(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + low = fromEpochMillisAndFraction( + low.getEpochMillis() - (lowPicosOfMilli < 0 ? 1 : 0), + (lowPicosOfMilli + factor) % factor, + low.getTimeZoneKey()); + int highPicosOfMilli = roundDiv(high.getPicosOfMilli(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + high = fromEpochMillisAndFraction( + high.getEpochMillis() + (highPicosOfMilli > factor ? 1 : 0), + highPicosOfMilli % factor, + high.getTimeZoneKey()); + return new LongTimestampWithTimeZoneRange(low, high, factor, defaultTZ); + } } - private Generator timeWithTimeZoneGenerator(Range range, TimeWithTimeZoneType timeType) + private record ShortTimeWithTimeZoneRange(long low, long high, long factor, int offsetMinutes) { - if (timeType.isShort()) { + static ShortTimeWithTimeZoneRange of(Range range, int precision) + { int offsetMinutes = range.getLowValue() .map(v -> unpackOffsetMinutes((long) v)) .orElse(range.getHighValue() .map(v -> unpackOffsetMinutes((long) v)) .orElse(0)); - long factor = POWERS_OF_TEN[9 - timeType.getPrecision()]; + long factor = POWERS_OF_TEN[9 - precision]; long low = roundDiv(range.getLowValue().map(v -> unpackTimeNanos((long) v)).orElse(0L), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); long high = roundDiv(range.getHighValue().map(v -> unpackTimeNanos((long) v)).orElse(NANOSECONDS_PER_DAY), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); - return (blockBuilder) -> { - long nanos = numberBetween(low, high) * factor; - timeType.writeLong(blockBuilder, packTimeWithTimeZone(nanos, offsetMinutes)); - }; + return new ShortTimeWithTimeZoneRange(low, high, factor, offsetMinutes); + } + } + + private record LongTimeWithTimeZoneRange(long low, long high, int factor, int offsetMinutes) + { + static LongTimeWithTimeZoneRange of(Range range, int precision) + { + int offsetMinutes = range.getLowValue() + .map(v -> ((LongTimeWithTimeZone) v).getOffsetMinutes()) + .orElse(range.getHighValue() + .map(v -> ((LongTimeWithTimeZone) v).getOffsetMinutes()) + .orElse(0)); + LongTimeWithTimeZone low = (LongTimeWithTimeZone) range.getLowValue().orElse(new LongTimeWithTimeZone(0, offsetMinutes)); + LongTimeWithTimeZone high = (LongTimeWithTimeZone) range.getHighValue().orElse(new LongTimeWithTimeZone(PICOSECONDS_PER_DAY, offsetMinutes)); + if (low.getOffsetMinutes() != high.getOffsetMinutes()) { + throw new TrinoException(INVALID_ROW_FILTER, "Range boundaries for time with time zone columns must have the same time zone"); + } + int factor = (int) POWERS_OF_TEN[12 - precision]; + long longLow = roundDiv(low.getPicoseconds(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + long longHigh = roundDiv(high.getPicoseconds(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + return new LongTimeWithTimeZoneRange(longLow, longHigh, factor, offsetMinutes); } - int offsetMinutes = range.getLowValue() - .map(v -> ((LongTimeWithTimeZone) v).getOffsetMinutes()) - .orElse(range.getHighValue() - .map(v -> ((LongTimeWithTimeZone) v).getOffsetMinutes()) - .orElse(0)); - LongTimeWithTimeZone low = (LongTimeWithTimeZone) range.getLowValue() - .orElse(new LongTimeWithTimeZone(0, offsetMinutes)); - LongTimeWithTimeZone high = (LongTimeWithTimeZone) range.getHighValue() - .orElse(new LongTimeWithTimeZone(PICOSECONDS_PER_DAY, offsetMinutes)); - if (low.getOffsetMinutes() != high.getOffsetMinutes()) { - throw new TrinoException(INVALID_ROW_FILTER, "Range boundaries for time with time zone columns must have the same time zone"); - } - int factor = (int) POWERS_OF_TEN[12 - timeType.getPrecision()]; - long longLow = roundDiv(low.getPicoseconds(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); - long longHigh = roundDiv(high.getPicoseconds(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); - return (blockBuilder) -> { - long picoseconds = numberBetween(longLow, longHigh) * factor; - timeType.writeObject(blockBuilder, new LongTimeWithTimeZone(picoseconds, offsetMinutes)); - }; } private int numberBetween(int min, int max) From ba46ff0f53a88ad5b02f6dda71b72dc19ef51325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 29 Dec 2024 09:41:30 +0100 Subject: [PATCH 094/158] Fix handling upper bounds in FakerPageSource Fix handling upper bounds for floating point types. The implementation did not account for rounding issue near the bound, and the test was using values outside of the allowed range. --- .../java/io/trino/plugin/faker/FakerPageSource.java | 12 ++++++------ .../java/io/trino/plugin/faker/TestFakerQueries.java | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java index e2b671f35bd8..b15aacb47be4 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -290,11 +290,11 @@ private Generator randomValueGenerator(FakerColumnHandle handle) } if (REAL.equals(type)) { FloatRange range = FloatRange.of(genericRange); - return (blockBuilder) -> REAL.writeLong(blockBuilder, floatToRawIntBits(range.low + (range.high - range.low) * random.nextFloat())); + return (blockBuilder) -> REAL.writeLong(blockBuilder, floatToRawIntBits(range.low == range.high ? range.low : random.nextFloat(range.low, range.high))); } if (DOUBLE.equals(type)) { DoubleRange range = DoubleRange.of(genericRange); - return (blockBuilder) -> DOUBLE.writeDouble(blockBuilder, range.low + (range.high - range.low) * random.nextDouble()); + return (blockBuilder) -> DOUBLE.writeDouble(blockBuilder, range.low == range.high ? range.low : random.nextDouble(range.low, range.high)); } // not supported: HYPER_LOG_LOG, QDIGEST, TDIGEST, P4_HYPER_LOG_LOG if (INTERVAL_DAY_TIME.equals(type)) { @@ -581,8 +581,8 @@ static FloatRange of(Range range) low = Math.nextUp(low); } float high = range.getHighValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MAX_VALUE); - if (!range.isHighUnbounded() && !range.isHighInclusive()) { - high = Math.nextDown(high); + if (!range.isHighUnbounded() && range.isHighInclusive()) { + high = Math.nextUp(high); } return new FloatRange(low, high); } @@ -597,8 +597,8 @@ static DoubleRange of(Range range) low = Math.nextUp(low); } double high = (double) range.getHighValue().orElse(Double.MAX_VALUE); - if (!range.isHighUnbounded() && !range.isHighInclusive()) { - high = Math.nextDown(high); + if (!range.isHighUnbounded() && range.isHighInclusive()) { + high = Math.nextUp(high); } return new DoubleRange(low, high); } diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java index bf48ea914bd1..503cfbed8319 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java @@ -311,8 +311,8 @@ void testSelectRangeProperties() .add(new TestDataType("rnd_decimal3", "decimal(38,0)", Map.of("min", "99999999999999999999999999999999999999"), "count(distinct rnd_decimal3)", "1")) .add(new TestDataType("rnd_decimal4", "decimal(38,38)", Map.of("min", "0.99999999999999999999999999999999999999"), "count(distinct rnd_decimal4)", "1")) .add(new TestDataType("rnd_decimal5", "decimal(5,2)", Map.of("min", "999.99"), "count(distinct rnd_decimal5)", "1")) - .add(new TestDataType("rnd_real", "real", Map.of("min", "1.4E45"), "count(distinct rnd_real)", "1")) - .add(new TestDataType("rnd_double", "double", Map.of("min", "4.9E324"), "count(distinct rnd_double)", "1")) + .add(new TestDataType("rnd_real", "real", Map.of("min", "3.4028235E38"), "count(distinct rnd_real)", "1")) + .add(new TestDataType("rnd_double", "double", Map.of("min", "1.7976931348623157E308"), "count(distinct rnd_double)", "1")) // interval literals can't represent smallest possible values allowed by the engine, so they're not included here // can't test timestamps because their extreme values cannot be expressed as literals .add(new TestDataType("rnd_time", "time", Map.of("min", "23:59:59.999"), "count(distinct rnd_time)", "1")) From c2fbb012d68e4c5b663d307fb6d92ae33689807a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 29 Dec 2024 09:45:42 +0100 Subject: [PATCH 095/158] Refactor FakerPageSource Refactor to make subsequent commit's diff smaller --- .../trino/plugin/faker/FakerPageSource.java | 59 +++++++++---------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java index b15aacb47be4..32ed38b935e6 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -126,7 +126,7 @@ class FakerPageSource Faker faker, Random random, List columns, - long offset, + long rowOffset, long limit) { this.faker = requireNonNull(faker, "faker is null"); @@ -141,19 +141,19 @@ class FakerPageSource this.generators = columns .stream() - .map(column -> getGenerator(column, offset)) + .map(column -> getGenerator(column, rowOffset)) .collect(toImmutableList()); this.pageBuilder = new PageBuilder(types); } private Generator getGenerator( FakerColumnHandle column, - long offset) + long rowOffset) { if (ROW_ID_COLUMN_NAME.equals(column.name())) { return new Generator() { - long currentRowId = offset; + long currentRowId = rowOffset; @Override public void accept(BlockBuilder blockBuilder) @@ -163,7 +163,23 @@ public void accept(BlockBuilder blockBuilder) }; } - return constraintedValueGenerator(column); + if (column.domain().getValues().isDiscreteSet()) { + List values = column.domain().getValues().getDiscreteSet(); + ObjectWriter singleValueWriter = objectWriter(column.type()); + return (blockBuilder) -> singleValueWriter.accept(blockBuilder, values.get(random.nextInt(values.size()))); + } + Generator generator = randomValueGenerator(column); + if (column.nullProbability() == 0) { + return generator; + } + return (blockBuilder) -> { + if (random.nextDouble() <= column.nullProbability()) { + blockBuilder.appendNull(); + } + else { + generator.accept(blockBuilder); + } + }; } @Override @@ -227,27 +243,6 @@ public void close() closed = true; } - private Generator constraintedValueGenerator(FakerColumnHandle handle) - { - if (handle.domain().getValues().isDiscreteSet()) { - List values = handle.domain().getValues().getDiscreteSet(); - ObjectWriter singleValueWriter = objectWriter(handle.type()); - return (blockBuilder) -> singleValueWriter.accept(blockBuilder, values.get(random.nextInt(values.size()))); - } - Generator generator = randomValueGenerator(handle); - if (handle.nullProbability() == 0) { - return generator; - } - return (blockBuilder) -> { - if (random.nextDouble() <= handle.nullProbability()) { - blockBuilder.appendNull(); - } - else { - generator.accept(blockBuilder); - } - }; - } - private Generator randomValueGenerator(FakerColumnHandle handle) { Range genericRange = handle.domain().getValues().getRanges().getSpan(); @@ -305,19 +300,19 @@ private Generator randomValueGenerator(FakerColumnHandle handle) IntRange range = IntRange.of(genericRange); return (blockBuilder) -> INTERVAL_YEAR_MONTH.writeLong(blockBuilder, numberBetween(range.low, range.high)); } - if (type instanceof TimestampType) { - return timestampGenerator(genericRange, (TimestampType) type); + if (type instanceof TimestampType timestampType) { + return timestampGenerator(genericRange, timestampType); } - if (type instanceof TimestampWithTimeZoneType) { - return timestampWithTimeZoneGenerator(genericRange, (TimestampWithTimeZoneType) type); + if (type instanceof TimestampWithTimeZoneType timestampWithTimeZoneType) { + return timestampWithTimeZoneGenerator(genericRange, timestampWithTimeZoneType); } if (type instanceof TimeType timeType) { long factor = POWERS_OF_TEN[12 - timeType.getPrecision()]; LongRange range = LongRange.of(genericRange, factor, 0, PICOSECONDS_PER_DAY); return (blockBuilder) -> timeType.writeLong(blockBuilder, numberBetween(range.low, range.high) * factor); } - if (type instanceof TimeWithTimeZoneType) { - return timeWithTimeZoneGenerator(genericRange, (TimeWithTimeZoneType) type); + if (type instanceof TimeWithTimeZoneType timeWithTimeZoneType) { + return timeWithTimeZoneGenerator(genericRange, timeWithTimeZoneType); } if (type instanceof VarbinaryType varType) { if (!genericRange.isAll()) { From 764bc427d7744b6a6708c86915249c3cf1eb8772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 29 Dec 2024 09:57:03 +0100 Subject: [PATCH 096/158] Extract a method in FakerColumnHandle --- .../trino/plugin/faker/FakerColumnHandle.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java index 96d870eafb8e..c1895eb2d2e5 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java @@ -51,6 +51,7 @@ public record FakerColumnHandle( { requireNonNull(name, "name is null"); requireNonNull(type, "type is null"); + requireNonNull(domain, "domain is null"); } public static FakerColumnHandle of(int columnId, ColumnMetadata column, double defaultNullProbability) @@ -63,20 +64,8 @@ public static FakerColumnHandle of(int columnId, ColumnMetadata column, double d if (generator != null && !isCharacterColumn(column)) { throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property can only be set for CHAR, VARCHAR or VARBINARY columns".formatted(GENERATOR_PROPERTY)); } - Object min; - try { - min = Literal.parse((String) column.getProperties().get(MIN_PROPERTY), column.getType()); - } - catch (IllegalArgumentException e) { - throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MIN_PROPERTY, column.getType().getDisplayName()), e); - } - Object max; - try { - max = Literal.parse((String) column.getProperties().get(MAX_PROPERTY), column.getType()); - } - catch (IllegalArgumentException e) { - throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MAX_PROPERTY, column.getType().getDisplayName()), e); - } + Object min = propertyValue(column, MIN_PROPERTY); + Object max = propertyValue(column, MAX_PROPERTY); Domain domain = Domain.all(column.getType()); if (min != null || max != null) { if (isCharacterColumn(column)) { @@ -114,6 +103,16 @@ private static boolean isCharacterColumn(ColumnMetadata column) return column.getType() instanceof CharType || column.getType() instanceof VarcharType || column.getType() instanceof VarbinaryType; } + private static Object propertyValue(ColumnMetadata column, String property) + { + try { + return Literal.parse((String) column.getProperties().get(property), column.getType()); + } + catch (IllegalArgumentException e) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(property, column.getType().getDisplayName()), e); + } + } + private static Range range(Type type, Object min, Object max) { requireNonNull(type, "type is null"); From fdf49648329fa2e9efc781dd83c841721f2efd72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sat, 28 Dec 2024 23:04:32 +0100 Subject: [PATCH 097/158] Support generating sequences in the Faker connector --- docs/src/main/sphinx/connector/faker.md | 3 + plugin/trino-faker/pom.xml | 11 +- .../io/trino/plugin/faker/ColumnInfo.java | 1 + .../trino/plugin/faker/FakerColumnHandle.java | 44 ++- .../io/trino/plugin/faker/FakerConnector.java | 8 +- .../io/trino/plugin/faker/FakerMetadata.java | 5 +- .../trino/plugin/faker/FakerPageSource.java | 339 +++++++++++++++--- .../trino/plugin/faker/TestFakerQueries.java | 45 +++ 8 files changed, 399 insertions(+), 57 deletions(-) diff --git a/docs/src/main/sphinx/connector/faker.md b/docs/src/main/sphinx/connector/faker.md index d5fb56791659..21260339de23 100644 --- a/docs/src/main/sphinx/connector/faker.md +++ b/docs/src/main/sphinx/connector/faker.md @@ -111,6 +111,9 @@ The following table details all supported column properties. * - `allowed_values` - List of allowed values. Cannot be set together with the `min`, or `max` properties. +* - `step` + - If set, generate sequential values with this step. For date and time columns + set this to a duration. Cannot be set for character-based type columns. ::: ### Character types diff --git a/plugin/trino-faker/pom.xml b/plugin/trino-faker/pom.xml index 8a87b91f122e..c0866b9f3445 100644 --- a/plugin/trino-faker/pom.xml +++ b/plugin/trino-faker/pom.xml @@ -40,6 +40,11 @@ configuration + + io.airlift + units + + io.trino trino-main @@ -119,12 +124,6 @@ runtime - - io.airlift - units - runtime - - io.trino trino-client diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java index bd52ae4fa776..9bd28af238b4 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java @@ -27,6 +27,7 @@ public record ColumnInfo(FakerColumnHandle handle, ColumnMetadata metadata) public static final String MIN_PROPERTY = "min"; public static final String MAX_PROPERTY = "max"; public static final String ALLOWED_VALUES_PROPERTY = "allowed_values"; + public static final String STEP_PROPERTY = "step"; public ColumnInfo { diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java index c1895eb2d2e5..c7af196596ca 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java @@ -15,6 +15,7 @@ package io.trino.plugin.faker; import com.google.common.collect.ImmutableList; +import io.airlift.units.Duration; import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; @@ -22,20 +23,29 @@ import io.trino.spi.predicate.Range; import io.trino.spi.predicate.ValueSet; import io.trino.spi.type.CharType; +import io.trino.spi.type.TimeType; +import io.trino.spi.type.TimeWithTimeZoneType; +import io.trino.spi.type.TimestampType; +import io.trino.spi.type.TimestampWithTimeZoneType; import io.trino.spi.type.Type; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; import java.util.Collection; import java.util.List; +import java.util.concurrent.TimeUnit; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.plugin.faker.ColumnInfo.ALLOWED_VALUES_PROPERTY; import static io.trino.plugin.faker.ColumnInfo.GENERATOR_PROPERTY; import static io.trino.plugin.faker.ColumnInfo.MAX_PROPERTY; import static io.trino.plugin.faker.ColumnInfo.MIN_PROPERTY; import static io.trino.plugin.faker.ColumnInfo.NULL_PROBABILITY_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.STEP_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.DateType.DATE; import static java.util.Objects.requireNonNull; public record FakerColumnHandle( @@ -44,7 +54,8 @@ public record FakerColumnHandle( Type type, double nullProbability, String generator, - Domain domain) + Domain domain, + ValueSet step) implements ColumnHandle { public FakerColumnHandle @@ -52,6 +63,8 @@ public record FakerColumnHandle( requireNonNull(name, "name is null"); requireNonNull(type, "type is null"); requireNonNull(domain, "domain is null"); + requireNonNull(step, "step is null"); + checkState(step.isNone() || step.isSingleValue(), "step must be a single value"); } public static FakerColumnHandle of(int columnId, ColumnMetadata column, double defaultNullProbability) @@ -95,7 +108,8 @@ public static FakerColumnHandle of(int columnId, ColumnMetadata column, double d column.getType(), nullProbability, generator, - domain); + domain, + stepValue(column)); } private static boolean isCharacterColumn(ColumnMetadata column) @@ -113,6 +127,32 @@ private static Object propertyValue(ColumnMetadata column, String property) } } + private static ValueSet stepValue(ColumnMetadata column) + { + Type type = column.getType(); + String rawStep = (String) column.getProperties().get(STEP_PROPERTY); + if (rawStep == null) { + return ValueSet.none(type); + } + if (isCharacterColumn(column)) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property cannot be set for CHAR, VARCHAR or VARBINARY columns".formatted(STEP_PROPERTY)); + } + if (DATE.equals(column.getType()) || type instanceof TimestampType || type instanceof TimestampWithTimeZoneType || type instanceof TimeType || type instanceof TimeWithTimeZoneType) { + try { + return ValueSet.of(BIGINT, Duration.valueOf(rawStep).roundTo(TimeUnit.NANOSECONDS)); + } + catch (IllegalArgumentException e) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property for a %s column must be a valid duration literal".formatted(STEP_PROPERTY, column.getType().getDisplayName()), e); + } + } + try { + return ValueSet.of(type, Literal.parse(rawStep, type)); + } + catch (IllegalArgumentException e) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property for a %s column must be a valid %s literal".formatted(STEP_PROPERTY, column.getType().getDisplayName(), type.getDisplayName()), e); + } + } + private static Range range(Type type, Object min, Object max) { requireNonNull(type, "type is null"); diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java index 2a5819c04b47..734fa513b355 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java @@ -39,6 +39,7 @@ import static io.trino.plugin.faker.ColumnInfo.ALLOWED_VALUES_PROPERTY; import static io.trino.plugin.faker.ColumnInfo.MAX_PROPERTY; import static io.trino.plugin.faker.ColumnInfo.MIN_PROPERTY; +import static io.trino.plugin.faker.ColumnInfo.STEP_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY; import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; @@ -188,7 +189,12 @@ public List> getColumnProperties() value -> ((List) value).stream() .map(String.class::cast) .collect(toImmutableList()), - value -> value)); + value -> value), + stringProperty( + STEP_PROPERTY, + "If set, generate sequential values with this step. For date and time columns set this to a duration", + null, + false)); } private static void checkProperty(boolean expression, ErrorCodeSupplier errorCode, String errorMessage) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java index 275d04d62004..46d246e9d8d0 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java @@ -43,6 +43,8 @@ import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.SchemaFunctionName; import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.Range; +import io.trino.spi.predicate.ValueSet; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.statistics.ComputedStatistics; import io.trino.spi.type.BigintType; @@ -336,7 +338,8 @@ public synchronized FakerOutputTableHandle beginCreateTable(ConnectorSession ses BigintType.BIGINT, 0, "", - Domain.all(BigintType.BIGINT)), + Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(BigintType.BIGINT, 0L)), false), + ValueSet.of(BigintType.BIGINT, 1L)), ColumnMetadata.builder() .setName(ROW_ID_COLUMN_NAME) .setType(BigintType.BIGINT) diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java index 32ed38b935e6..e1df2900bccb 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -50,7 +50,6 @@ import java.util.Random; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.faker.FakerMetadata.ROW_ID_COLUMN_NAME; import static io.trino.spi.StandardErrorCode.INVALID_ROW_FILTER; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.BooleanType.BOOLEAN; @@ -67,9 +66,12 @@ import static io.trino.spi.type.RealType.REAL; import static io.trino.spi.type.SmallintType.SMALLINT; import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_DAY; +import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND; +import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MILLISECOND; import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_DAY; import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_MICROSECOND; import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_MILLISECOND; +import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND; import static io.trino.spi.type.Timestamps.roundDiv; import static io.trino.spi.type.TinyintType.TINYINT; import static io.trino.spi.type.UuidType.UUID; @@ -150,25 +152,18 @@ private Generator getGenerator( FakerColumnHandle column, long rowOffset) { - if (ROW_ID_COLUMN_NAME.equals(column.name())) { - return new Generator() - { - long currentRowId = rowOffset; - - @Override - public void accept(BlockBuilder blockBuilder) - { - BIGINT.writeLong(blockBuilder, currentRowId++); - } - }; - } - if (column.domain().getValues().isDiscreteSet()) { List values = column.domain().getValues().getDiscreteSet(); ObjectWriter singleValueWriter = objectWriter(column.type()); return (blockBuilder) -> singleValueWriter.accept(blockBuilder, values.get(random.nextInt(values.size()))); } - Generator generator = randomValueGenerator(column); + Generator generator; + if (!column.step().isNone()) { + generator = sequenceGenerator(column, rowOffset); + } + else { + generator = randomValueGenerator(column); + } if (column.nullProbability() == 0) { return generator; } @@ -243,6 +238,103 @@ public void close() closed = true; } + private Generator sequenceGenerator(FakerColumnHandle handle, long rowOffset) + { + SequenceWriter writer = sequenceWriter(handle); + + return new Generator() + { + long currentRowId = rowOffset; + + @Override + public void accept(BlockBuilder blockBuilder) + { + writer.accept(blockBuilder, currentRowId++); + } + }; + } + + private SequenceWriter sequenceWriter(FakerColumnHandle handle) + { + Range genericRange = handle.domain().getValues().getRanges().getSpan(); + Type type = handle.type(); + // check every type in order defined in StandardTypes + // not supported: BOOLEAN, HYPER_LOG_LOG, QDIGEST, TDIGEST, P4_HYPER_LOG_LOG, VARBINARY, VARCHAR, CHAR, ROW, ARRAY, MAP, JSON, IPADDRESS, GEOMETRY, UUID + if (BIGINT.equals(type)) { + LongRange range = LongRange.of(genericRange, 1, (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> BIGINT.writeLong(blockBuilder, range.at(rowId)); + } + if (INTEGER.equals(type)) { + IntRange range = IntRange.of(genericRange, (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> INTEGER.writeLong(blockBuilder, range.at(rowId)); + } + if (SMALLINT.equals(type)) { + IntRange range = IntRange.of(genericRange, Short.MIN_VALUE, Short.MAX_VALUE, (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> SMALLINT.writeLong(blockBuilder, range.at(rowId)); + } + if (TINYINT.equals(type)) { + IntRange range = IntRange.of(genericRange, Byte.MIN_VALUE, Byte.MAX_VALUE, (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> TINYINT.writeLong(blockBuilder, range.at(rowId)); + } + if (DATE.equals(type)) { + IntRange range = IntRange.of(genericRange, (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> DATE.writeLong(blockBuilder, range.at(rowId, NANOSECONDS_PER_DAY)); + } + if (type instanceof DecimalType decimalType) { + if (decimalType.isShort()) { + ShortDecimalRange range = ShortDecimalRange.of(genericRange, decimalType.getPrecision(), (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> decimalType.writeLong(blockBuilder, range.at(rowId)); + } + Int128Range range = Int128Range.of(genericRange, (Int128) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> decimalType.writeObject(blockBuilder, range.at(rowId)); + } + if (REAL.equals(type)) { + FloatRange range = FloatRange.of(genericRange, intBitsToFloat(toIntExact((long) handle.step().getSingleValue()))); + return (blockBuilder, rowId) -> REAL.writeLong(blockBuilder, floatToRawIntBits(range.at(rowId))); + } + if (DOUBLE.equals(type)) { + DoubleRange range = DoubleRange.of(genericRange, (double) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> DOUBLE.writeDouble(blockBuilder, range.at(rowId)); + } + if (INTERVAL_DAY_TIME.equals(type) || INTERVAL_YEAR_MONTH.equals(type)) { + // step is seconds or months + IntRange range = IntRange.of(genericRange, (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> type.writeLong(blockBuilder, range.at(rowId)); + } + if (type instanceof TimestampType timestampType) { + if (timestampType.isShort()) { + long factor = POWERS_OF_TEN[6 - timestampType.getPrecision()]; + LongRange range = LongRange.of(genericRange, factor, (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> timestampType.writeLong(blockBuilder, range.at(rowId, factor * NANOSECONDS_PER_MICROSECOND) * factor); + } + LongTimestampRange range = LongTimestampRange.of(genericRange, timestampType.getPrecision(), (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> timestampType.writeObject(blockBuilder, range.at(rowId, NANOSECONDS_PER_MICROSECOND)); + } + if (type instanceof TimestampWithTimeZoneType tzType) { + if (tzType.isShort()) { + ShortTimestampWithTimeZoneRange range = ShortTimestampWithTimeZoneRange.of(genericRange, tzType.getPrecision(), (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> tzType.writeLong(blockBuilder, range.at(rowId, NANOSECONDS_PER_MILLISECOND)); + } + LongTimestampWithTimeZoneRange range = LongTimestampWithTimeZoneRange.of(genericRange, tzType.getPrecision(), (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> tzType.writeObject(blockBuilder, range.at(rowId, NANOSECONDS_PER_MILLISECOND)); + } + if (type instanceof TimeType timeType) { + long factor = POWERS_OF_TEN[12 - timeType.getPrecision()]; + LongRange range = LongRange.of(genericRange, factor, 0, PICOSECONDS_PER_DAY, (long) handle.step().getSingleValue() * PICOSECONDS_PER_NANOSECOND); + return (blockBuilder, rowId) -> timeType.writeLong(blockBuilder, range.at(rowId, factor) * factor); + } + if (type instanceof TimeWithTimeZoneType timeType) { + if (timeType.isShort()) { + ShortTimeWithTimeZoneRange range = ShortTimeWithTimeZoneRange.of(genericRange, timeType.getPrecision(), (long) handle.step().getSingleValue()); + return (blockBuilder, rowId) -> timeType.writeLong(blockBuilder, range.at(rowId)); + } + LongTimeWithTimeZoneRange range = LongTimeWithTimeZoneRange.of(genericRange, timeType.getPrecision(), (long) handle.step().getSingleValue() * PICOSECONDS_PER_NANOSECOND); + return (blockBuilder, rowId) -> timeType.writeObject(blockBuilder, range.at(rowId)); + } + + throw new IllegalArgumentException("Unsupported type " + type); + } + private Generator randomValueGenerator(FakerColumnHandle handle) { Range genericRange = handle.domain().getValues().getRanges().getSpan(); @@ -532,44 +624,91 @@ private Generator timeWithTimeZoneGenerator(Range genericRange, TimeWithTimeZone }; } - private record LongRange(long low, long high) + private record LongRange(long low, long high, long step) { static LongRange of(Range range) { - return of(range, 1, Long.MIN_VALUE, Long.MAX_VALUE); + return of(range, 1, Long.MIN_VALUE, Long.MAX_VALUE, 1); } static LongRange of(Range range, long factor) { - return of(range, factor, Long.MIN_VALUE, Long.MAX_VALUE); + return of(range, factor, Long.MIN_VALUE, Long.MAX_VALUE, 1); + } + + static LongRange of(Range range, long factor, long step) + { + return of(range, factor, Long.MIN_VALUE, Long.MAX_VALUE, step); } static LongRange of(Range range, long factor, long defaultMin, long defaultMax) + { + return of(range, factor, defaultMin, defaultMax, 1); + } + + static LongRange of(Range range, long factor, long defaultMin, long defaultMax, long step) { return new LongRange( roundDiv((long) range.getLowValue().orElse(defaultMin), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - roundDiv((long) range.getHighValue().orElse(defaultMax), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); + roundDiv((long) range.getHighValue().orElse(defaultMax), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0), + step); + } + + long at(long index) + { + return Math.min(low + index * step, high - 1); + } + + long at(long index, long factor) + { + return Math.min(low + roundDiv(index * step, factor), high - 1); } } - private record IntRange(int low, int high) + private record IntRange(int low, int high, long step) { static IntRange of(Range range) { - return of(range, Integer.MIN_VALUE, Integer.MAX_VALUE); + return of(range, Integer.MIN_VALUE, Integer.MAX_VALUE, 1); + } + + static IntRange of(Range range, long step) + { + return of(range, Integer.MIN_VALUE, Integer.MAX_VALUE, step); } static IntRange of(Range range, long defaultMin, long defaultMax) + { + return of(range, defaultMin, defaultMax, 1); + } + + static IntRange of(Range range, long defaultMin, long defaultMax, long step) { return new IntRange( toIntExact((long) range.getLowValue().orElse(defaultMin)) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), - toIntExact((long) range.getHighValue().orElse(defaultMax)) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); + toIntExact((long) range.getHighValue().orElse(defaultMax)) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0), + step); + } + + long at(long index) + { + return Math.min(low + index * step, high - 1); + } + + long at(long index, long factor) + { + return Math.min(low + roundDiv(index * step, factor), high - 1); } } - private record FloatRange(float low, float high) + private record FloatRange(float low, float high, float step) { static FloatRange of(Range range) + { + return of(range, 1); + } + + static FloatRange of(Range range, float step) { float low = range.getLowValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MIN_VALUE); if (!range.isLowUnbounded() && !range.isLowInclusive()) { @@ -579,13 +718,23 @@ static FloatRange of(Range range) if (!range.isHighUnbounded() && range.isHighInclusive()) { high = Math.nextUp(high); } - return new FloatRange(low, high); + return new FloatRange(low, high, step); + } + + float at(long index) + { + return Math.min(low + index * step, Math.nextDown(high)); } } - private record DoubleRange(double low, double high) + private record DoubleRange(double low, double high, double step) { static DoubleRange of(Range range) + { + return of(range, 1); + } + + static DoubleRange of(Range range, double step) { double low = (double) range.getLowValue().orElse(Double.MIN_VALUE); if (!range.isLowUnbounded() && !range.isLowInclusive()) { @@ -595,45 +744,80 @@ static DoubleRange of(Range range) if (!range.isHighUnbounded() && range.isHighInclusive()) { high = Math.nextUp(high); } - return new DoubleRange(low, high); + return new DoubleRange(low, high, step); + } + + double at(long index) + { + return Math.min(low + index * step, Math.nextDown(high)); } } - private record ShortDecimalRange(long low, long high) + private record ShortDecimalRange(long low, long high, long step) { static ShortDecimalRange of(Range range, int precision) + { + return of(range, precision, 1); + } + + static ShortDecimalRange of(Range range, int precision, long step) { long defaultMin = -999999999999999999L / POWERS_OF_TEN[18 - precision]; long defaultMax = 999999999999999999L / POWERS_OF_TEN[18 - precision]; long low = (long) range.getLowValue().orElse(defaultMin) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); long high = (long) range.getHighValue().orElse(defaultMax) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); - return new ShortDecimalRange(low, high); + return new ShortDecimalRange(low, high, step); + } + + long at(long index) + { + return Math.min(low + index * step, high - 1); } } - private record Int128Range(Int128 low, Int128 high) + private record Int128Range(Int128 low, Int128 high, Int128 step) { static Int128Range of(Range range) + { + return of(range, Int128.ONE); + } + + static Int128Range of(Range range, Int128 step) { Int128 low = (Int128) range.getLowValue().orElse(Decimals.MIN_UNSCALED_DECIMAL); Int128 high = (Int128) range.getHighValue().orElse(Decimals.MAX_UNSCALED_DECIMAL); if (!range.isLowUnbounded() && !range.isLowInclusive()) { - long[] result = new long[2]; - Int128Math.add(low.getHigh(), low.getLow(), 0, 1, result, 0); - low = Int128.valueOf(result); + low = add(low, Int128.ONE); } if (!range.isHighUnbounded() && range.isHighInclusive()) { - long[] result = new long[2]; - Int128Math.add(high.getHigh(), high.getLow(), 0, 1, result, 0); - high = Int128.valueOf(result); + high = add(high, Int128.ONE); } - return new Int128Range(low, high); + return new Int128Range(low, high, step); + } + + Int128 at(long index) + { + Int128 nextValue = add(low, Int128Math.multiply(Int128.valueOf(index), step)); + Int128 highInclusive = Int128Math.subtract(high, Int128.ONE); + return highInclusive.compareTo(nextValue) < 0 ? highInclusive : nextValue; } } - private record LongTimestampRange(LongTimestamp low, LongTimestamp high, int factor) + private static Int128 add(Int128 left, Int128 right) + { + long[] result = new long[2]; + Int128Math.add(left.getHigh(), left.getLow(), right.getHigh(), right.getLow(), result, 0); + return Int128.valueOf(result); + } + + private record LongTimestampRange(LongTimestamp low, LongTimestamp high, int factor, long step) { static LongTimestampRange of(Range range, int precision) + { + return of(range, precision, 1); + } + + static LongTimestampRange of(Range range, int precision, long step) { LongTimestamp low = (LongTimestamp) range.getLowValue().orElse(new LongTimestamp(Long.MIN_VALUE, 0)); LongTimestamp high = (LongTimestamp) range.getHighValue().orElse(new LongTimestamp(Long.MAX_VALUE, PICOSECONDS_PER_MICROSECOND - 1)); @@ -642,7 +826,7 @@ static LongTimestampRange of(Range range, int precision) factor = (int) POWERS_OF_TEN[6 - precision]; low = new LongTimestamp(roundDiv(low.getEpochMicros(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), 0); high = new LongTimestamp(roundDiv(high.getEpochMicros(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0), 0); - return new LongTimestampRange(low, high, factor); + return new LongTimestampRange(low, high, factor, step); } factor = (int) POWERS_OF_TEN[12 - precision]; int lowPicosOfMicro = roundDiv(low.getPicosOfMicro(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); @@ -653,13 +837,26 @@ static LongTimestampRange of(Range range, int precision) high = new LongTimestamp( high.getEpochMicros() + (highPicosOfMicro > factor ? 1 : 0), highPicosOfMicro % factor); - return new LongTimestampRange(low, high, factor); + return new LongTimestampRange(low, high, factor, step); + } + + LongTimestamp at(long index, long stepFactor) + { + // TODO support nanosecond increments + // TODO handle exclusive high + long epochMicros = low.getEpochMicros() + roundDiv(index * step, stepFactor); + return new LongTimestamp(step > 0 ? Math.min(epochMicros, high.getEpochMicros()) : Math.max(epochMicros, high.getEpochMicros()), 0); } } - private record ShortTimestampWithTimeZoneRange(long low, long high, long factor, TimeZoneKey defaultTZ) + private record ShortTimestampWithTimeZoneRange(long low, long high, long factor, TimeZoneKey defaultTZ, long step) { static ShortTimestampWithTimeZoneRange of(Range range, int precision) + { + return of(range, precision, 1); + } + + static ShortTimestampWithTimeZoneRange of(Range range, int precision, long step) { TimeZoneKey defaultTZ = range.getLowValue() .map(v -> unpackZoneKey((long) v)) @@ -669,13 +866,25 @@ static ShortTimestampWithTimeZoneRange of(Range range, int precision) long factor = POWERS_OF_TEN[3 - precision]; long low = roundDiv(unpackMillisUtc((long) range.getLowValue().orElse(Long.MIN_VALUE)), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); long high = roundDiv(unpackMillisUtc((long) range.getHighValue().orElse(Long.MAX_VALUE)), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); - return new ShortTimestampWithTimeZoneRange(low, high, factor, defaultTZ); + return new ShortTimestampWithTimeZoneRange(low, high, factor, defaultTZ, step); + } + + long at(long index, long stepFactor) + { + // TODO support nanosecond increments + long millis = low + roundDiv(index * step, factor * stepFactor); + return packDateTimeWithZone((step > 0 ? Math.min(millis, high - 1) : Math.max(millis, high - 1)) * factor, defaultTZ); } } - private record LongTimestampWithTimeZoneRange(LongTimestampWithTimeZone low, LongTimestampWithTimeZone high, int factor, short defaultTZ) + private record LongTimestampWithTimeZoneRange(LongTimestampWithTimeZone low, LongTimestampWithTimeZone high, int factor, short defaultTZ, long step) { static LongTimestampWithTimeZoneRange of(Range range, int precision) + { + return of(range, precision, 1); + } + + static LongTimestampWithTimeZoneRange of(Range range, int precision, long step) { short defaultTZ = range.getLowValue() .map(v -> ((LongTimestampWithTimeZone) v).getTimeZoneKey()) @@ -698,13 +907,26 @@ static LongTimestampWithTimeZoneRange of(Range range, int precision) high.getEpochMillis() + (highPicosOfMilli > factor ? 1 : 0), highPicosOfMilli % factor, high.getTimeZoneKey()); - return new LongTimestampWithTimeZoneRange(low, high, factor, defaultTZ); + return new LongTimestampWithTimeZoneRange(low, high, factor, defaultTZ, step); + } + + LongTimestampWithTimeZone at(long index, long stepFactor) + { + // TODO support nanosecond increments + // TODO handle exclusive high + long millis = low.getEpochMillis() + roundDiv(index * step, stepFactor); + return fromEpochMillisAndFraction(step > 0 ? Math.min(millis, high.getEpochMillis()) : Math.max(millis, high.getEpochMillis()), 0, defaultTZ); } } - private record ShortTimeWithTimeZoneRange(long low, long high, long factor, int offsetMinutes) + private record ShortTimeWithTimeZoneRange(long low, long high, long factor, int offsetMinutes, long step) { static ShortTimeWithTimeZoneRange of(Range range, int precision) + { + return of(range, precision, 1); + } + + static ShortTimeWithTimeZoneRange of(Range range, int precision, long step) { int offsetMinutes = range.getLowValue() .map(v -> unpackOffsetMinutes((long) v)) @@ -714,13 +936,24 @@ static ShortTimeWithTimeZoneRange of(Range range, int precision) long factor = POWERS_OF_TEN[9 - precision]; long low = roundDiv(range.getLowValue().map(v -> unpackTimeNanos((long) v)).orElse(0L), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); long high = roundDiv(range.getHighValue().map(v -> unpackTimeNanos((long) v)).orElse(NANOSECONDS_PER_DAY), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); - return new ShortTimeWithTimeZoneRange(low, high, factor, offsetMinutes); + return new ShortTimeWithTimeZoneRange(low, high, factor, offsetMinutes, step); + } + + long at(long index) + { + long nanos = low + roundDiv(index * step, factor); + return packTimeWithTimeZone((step > 0 ? Math.min(nanos, high - 1) : Math.max(nanos, high - 1)) * factor, offsetMinutes); } } - private record LongTimeWithTimeZoneRange(long low, long high, int factor, int offsetMinutes) + private record LongTimeWithTimeZoneRange(long low, long high, int factor, int offsetMinutes, long step) { static LongTimeWithTimeZoneRange of(Range range, int precision) + { + return of(range, precision, 1); + } + + static LongTimeWithTimeZoneRange of(Range range, int precision, long step) { int offsetMinutes = range.getLowValue() .map(v -> ((LongTimeWithTimeZone) v).getOffsetMinutes()) @@ -735,7 +968,13 @@ static LongTimeWithTimeZoneRange of(Range range, int precision) int factor = (int) POWERS_OF_TEN[12 - precision]; long longLow = roundDiv(low.getPicoseconds(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); long longHigh = roundDiv(high.getPicoseconds(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); - return new LongTimeWithTimeZoneRange(longLow, longHigh, factor, offsetMinutes); + return new LongTimeWithTimeZoneRange(longLow, longHigh, factor, offsetMinutes, step); + } + + LongTimeWithTimeZone at(long index) + { + long picoseconds = low + roundDiv(index * step, factor); + return new LongTimeWithTimeZone((step > 0 ? Math.min(picoseconds, high - 1) : Math.max(picoseconds, high - 1)) * factor, offsetMinutes); } } @@ -816,6 +1055,12 @@ private Generator generateUUID(Range range) }; } + @FunctionalInterface + private interface SequenceWriter + { + void accept(BlockBuilder blockBuilder, long rowId); + } + @FunctionalInterface private interface ObjectWriter { diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java index 503cfbed8319..4861a9ece0c8 100644 --- a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java @@ -385,6 +385,51 @@ void testSelectValuesProperty() } } + @Test + void testSelectStepProperties() + { + // small step in small ranges that produce only 10 unique values for 1000 rows + List testCases = ImmutableList.builder() + .add(new TestDataType("rnd_bigint", "bigint", Map.of("min", "0", "max", "9", "step", "1"), "count(distinct rnd_bigint)", "10")) + .add(new TestDataType("rnd_integer", "integer", Map.of("min", "0", "max", "9", "step", "1"), "count(distinct rnd_integer)", "10")) + .add(new TestDataType("rnd_smallint", "smallint", Map.of("min", "0", "max", "9", "step", "1"), "count(distinct rnd_smallint)", "10")) + .add(new TestDataType("rnd_tinyint", "tinyint", Map.of("min", "0", "max", "9", "step", "1"), "count(distinct rnd_tinyint)", "10")) + .add(new TestDataType("rnd_date", "date", Map.of("min", "2022-03-01", "max", "2022-03-10", "step", "1d"), "count(distinct rnd_date)", "10")) + .add(new TestDataType("rnd_decimal1", "decimal", Map.of("min", "0", "max", "9", "step", "1"), "count(distinct rnd_decimal1)", "10")) + .add(new TestDataType("rnd_decimal2", "decimal(18,5)", Map.of("min", "0.00000", "max", "0.00009", "step", "0.00001"), "count(distinct rnd_decimal2)", "10")) + .add(new TestDataType("rnd_decimal3", "decimal(38,0)", Map.of("min", "0", "max", "9", "step", "1"), "count(distinct rnd_decimal3)", "10")) + .add(new TestDataType("rnd_decimal4", "decimal(38,38)", Map.of("min", "0.00000000000000000000000000000000000000", "max", "0.00000000000000000000000000000000000009", "step", "0.00000000000000000000000000000000000001"), "count(distinct rnd_decimal4)", "10")) + .add(new TestDataType("rnd_decimal5", "decimal(5,2)", Map.of("min", "0.00", "max", "1.09", "step", "0.01"), "count(distinct rnd_decimal5)", "110")) + .add(new TestDataType("rnd_real", "real", Map.of("min", "0.0", "max", "1.3E-44", "step", "1.4E-45"), "count(distinct rnd_real)", "10")) + .add(new TestDataType("rnd_double", "double", Map.of("min", "0.0", "max", "4.4E-323", "step", "4.9E-324"), "count(distinct rnd_double)", "10")) + .add(new TestDataType("rnd_interval1", "interval day to second", Map.of("min", "0.000", "max", "0.009", "step", "0.001"), "count(distinct rnd_interval1)", "10")) + .add(new TestDataType("rnd_interval2", "interval year to month", Map.of("min", "0", "max", "9", "step", "1"), "count(distinct rnd_interval2)", "10")) + .add(new TestDataType("rnd_timestamp", "timestamp", Map.of("min", "2022-03-21 00:00:00.000", "max", "2022-03-21 00:00:00.009", "step", "1ms"), "count(distinct rnd_timestamp)", "10")) + .add(new TestDataType("rnd_timestamp0", "timestamp(0)", Map.of("min", "2022-03-21 00:00:00", "max", "2022-03-21 00:00:09", "step", "1s"), "count(distinct rnd_timestamp0)", "10")) + .add(new TestDataType("rnd_timestamp6", "timestamp(6)", Map.of("min", "2022-03-21 00:00:00.000000", "max", "2022-03-21 00:00:00.000009", "step", "1us"), "count(distinct rnd_timestamp6)", "10")) + .add(new TestDataType("rnd_timestamp9", "timestamp(9)", Map.of("min", "2022-03-21 00:00:00.000000000", "max", "2022-03-21 00:00:00.000009000", "step", "1us"), "count(distinct rnd_timestamp9)", "10")) + .add(new TestDataType("rnd_timestamptz", "timestamp with time zone", Map.of("min", "2022-03-21 00:00:00.000 +01:00", "max", "2022-03-21 00:00:00.009 +01:00", "step", "1ms"), "count(distinct rnd_timestamptz)", "10")) + .add(new TestDataType("rnd_timestamptz0", "timestamp(0) with time zone", Map.of("min", "2022-03-21 00:00:00 +01:00", "max", "2022-03-21 00:00:09 +01:00", "step", "1s"), "count(distinct rnd_timestamptz0)", "10")) + .add(new TestDataType("rnd_timestamptz6", "timestamp(6) with time zone", Map.of("min", "2022-03-21 00:00:00.000000 +01:00", "max", "2022-03-21 00:00:00.009000 +01:00", "step", "1ms"), "count(distinct rnd_timestamptz6)", "10")) + .add(new TestDataType("rnd_timestamptz9", "timestamp(9) with time zone", Map.of("min", "2022-03-21 00:00:00.000000000 +01:00", "max", "2022-03-21 00:00:00.009000000 +01:00", "step", "1ms"), "count(distinct rnd_timestamptz9)", "10")) + .add(new TestDataType("rnd_time", "time", Map.of("min", "01:02:03.456", "max", "01:02:03.465", "step", "1ms"), "count(distinct rnd_time)", "10")) + .add(new TestDataType("rnd_time0", "time(0)", Map.of("min", "01:02:03", "max", "01:02:12", "step", "1s"), "count(distinct rnd_time0)", "10")) + .add(new TestDataType("rnd_time6", "time(6)", Map.of("min", "01:02:03.000456", "max", "01:02:03.000465", "step", "1us"), "count(distinct rnd_time6)", "10")) + .add(new TestDataType("rnd_time9", "time(9)", Map.of("min", "01:02:03.000000456", "max", "01:02:03.000000465", "step", "1ns"), "count(distinct rnd_time9)", "10")) + .add(new TestDataType("rnd_timetz", "time with time zone", Map.of("min", "01:02:03.456 +01:00", "max", "01:02:03.465 +01:00", "step", "1ms"), "count(distinct rnd_timetz)", "10")) + .add(new TestDataType("rnd_timetz0", "time(0) with time zone", Map.of("min", "01:02:03 +01:00", "max", "01:02:12 +01:00", "step", "1s"), "count(distinct rnd_timetz0)", "10")) + .add(new TestDataType("rnd_timetz6", "time(6) with time zone", Map.of("min", "01:02:03.000456 +01:00", "max", "01:02:03.000465 +01:00", "step", "1us"), "count(distinct rnd_timetz6)", "10")) + .add(new TestDataType("rnd_timetz9", "time(9) with time zone", Map.of("min", "01:02:03.000000456 +01:00", "max", "01:02:03.000000465 +01:00", "step", "1ns"), "count(distinct rnd_timetz9)", "10")) + .add(new TestDataType("rnd_timetz12", "time(12) with time zone", Map.of("min", "01:02:03.000000000456 +01:00", "max", "01:02:03.000000009456 +01:00", "step", "1ns"), "count(distinct rnd_timetz12)", "10")) + .build(); + + for (TestDataType testCase : testCases) { + try (TestTable table = new TestTable(getQueryRunner()::execute, "step_small_" + testCase.name(), "(%s)".formatted(testCase.columnSchema()))) { + assertQuery("SELECT %s FROM %s".formatted(testCase.queryExpression(), table.getName()), "VALUES (%s)".formatted(testCase.expectedValue())); + } + } + } + private record TestDataType(String name, String type, Map properties, String queryExpression, String expectedValue) { public TestDataType(String name, String type, String queryExpression, String expectedValue) From f9a553a25a5f432dc3f074c55e68f610b028aa08 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 16:26:01 +0100 Subject: [PATCH 098/158] Configure SSL for unauthenticated client Unauthenticated client can connect to the Trino cluster when loading segment data. If cluster has its' own certificate chain - client needs to accept it according to the configuration. --- .../trino/client/uri/HttpClientFactory.java | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java b/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java index 1266f5a2307c..570ee5ec3324 100644 --- a/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java +++ b/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java @@ -54,10 +54,6 @@ public static OkHttpClient.Builder toHttpClientBuilder(TrinoUri uri, String user OkHttpClient.Builder builder = unauthenticatedClientBuilder(uri, userAgent); setupCookieJar(builder); - if (!uri.isUseSecureConnection()) { - setupInsecureSsl(builder); - } - if (uri.hasPassword()) { if (!uri.isUseSecureConnection()) { throw new RuntimeException("TLS/SSL is required for authentication with username and password"); @@ -65,34 +61,6 @@ public static OkHttpClient.Builder toHttpClientBuilder(TrinoUri uri, String user builder.addNetworkInterceptor(basicAuth(uri.getRequiredUser(), uri.getPassword().orElseThrow(() -> new RuntimeException("Password expected")))); } - if (uri.isUseSecureConnection()) { - ConnectionProperties.SslVerificationMode sslVerificationMode = uri.getSslVerification(); - if (sslVerificationMode.equals(FULL) || sslVerificationMode.equals(CA)) { - setupSsl( - builder, - uri.getSslKeyStorePath(), - uri.getSslKeyStorePassword(), - uri.getSslKeyStoreType(), - uri.getSslUseSystemKeyStore(), - uri.getSslTrustStorePath(), - uri.getSslTrustStorePassword(), - uri.getSslTrustStoreType(), - uri.getSslUseSystemTrustStore()); - } - if (sslVerificationMode.equals(FULL)) { - uri.getHostnameInCertificate().ifPresent(certHostname -> - setupAlternateHostnameVerification(builder, certHostname)); - } - - if (sslVerificationMode.equals(CA)) { - builder.hostnameVerifier((hostname, session) -> true); - } - - if (sslVerificationMode.equals(NONE)) { - setupInsecureSsl(builder); - } - } - if (uri.getKerberosRemoteServiceName().isPresent()) { if (!uri.isUseSecureConnection()) { throw new RuntimeException("TLS/SSL is required for Kerberos authentication"); @@ -145,7 +113,6 @@ public static OkHttpClient.Builder toHttpClientBuilder(TrinoUri uri, String user builder.addNetworkInterceptor(authenticator); } - uri.getDnsResolver().ifPresent(resolverClass -> builder.dns(instantiateDnsResolver(resolverClass, uri.getDnsResolverContext())::lookup)); return builder; } @@ -157,6 +124,41 @@ public static OkHttpClient.Builder unauthenticatedClientBuilder(TrinoUri uri, St setupHttpProxy(builder, uri.getHttpProxy()); setupTimeouts(builder, toIntExact(uri.getTimeout().toMillis()), TimeUnit.MILLISECONDS); setupHttpLogging(builder, uri.getHttpLoggingLevel()); + + if (!uri.isUseSecureConnection()) { + setupInsecureSsl(builder); + } + + if (uri.isUseSecureConnection()) { + ConnectionProperties.SslVerificationMode sslVerificationMode = uri.getSslVerification(); + if (sslVerificationMode.equals(FULL) || sslVerificationMode.equals(CA)) { + setupSsl( + builder, + uri.getSslKeyStorePath(), + uri.getSslKeyStorePassword(), + uri.getSslKeyStoreType(), + uri.getSslUseSystemKeyStore(), + uri.getSslTrustStorePath(), + uri.getSslTrustStorePassword(), + uri.getSslTrustStoreType(), + uri.getSslUseSystemTrustStore()); + } + if (sslVerificationMode.equals(FULL)) { + uri.getHostnameInCertificate().ifPresent(certHostname -> + setupAlternateHostnameVerification(builder, certHostname)); + } + + if (sslVerificationMode.equals(CA)) { + builder.hostnameVerifier((hostname, session) -> true); + } + + if (sslVerificationMode.equals(NONE)) { + setupInsecureSsl(builder); + } + } + + uri.getDnsResolver().ifPresent(resolverClass -> builder.dns(instantiateDnsResolver(resolverClass, uri.getDnsResolverContext())::lookup)); + return builder; } From e7c3c1df54eca0e25b7dd33734c4140536512c40 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Mon, 30 Dec 2024 17:01:54 +0100 Subject: [PATCH 099/158] Simplify conditions --- .../main/java/io/trino/client/uri/HttpClientFactory.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java b/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java index 570ee5ec3324..796a42b40605 100644 --- a/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java +++ b/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java @@ -125,10 +125,6 @@ public static OkHttpClient.Builder unauthenticatedClientBuilder(TrinoUri uri, St setupTimeouts(builder, toIntExact(uri.getTimeout().toMillis()), TimeUnit.MILLISECONDS); setupHttpLogging(builder, uri.getHttpLoggingLevel()); - if (!uri.isUseSecureConnection()) { - setupInsecureSsl(builder); - } - if (uri.isUseSecureConnection()) { ConnectionProperties.SslVerificationMode sslVerificationMode = uri.getSslVerification(); if (sslVerificationMode.equals(FULL) || sslVerificationMode.equals(CA)) { @@ -156,6 +152,9 @@ public static OkHttpClient.Builder unauthenticatedClientBuilder(TrinoUri uri, St setupInsecureSsl(builder); } } + else { + setupInsecureSsl(builder); + } uri.getDnsResolver().ifPresent(resolverClass -> builder.dns(instantiateDnsResolver(resolverClass, uri.getDnsResolverContext())::lookup)); From 482fa957c0d036c1d205623eca60a984aeff4605 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Thu, 19 Dec 2024 12:51:32 +0100 Subject: [PATCH 100/158] Allow resource access type on class-level For io.trino resources it's now impossible to use class-level @ResourceType annotations due to the invalid condition. --- .../main/java/io/trino/server/security/ResourceAccessType.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java b/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java index 9fe79418b358..b71a2d0b9e03 100644 --- a/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java +++ b/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java @@ -49,7 +49,6 @@ public AccessType getAccessType(ResourceInfo resourceInfo) // check if the resource class has an access type declared for all methods accessType = resourceAccessTypeLoader.getAccessType(resourceInfo.getResourceClass()); if (accessType.isPresent()) { - verifyNotTrinoResource(resourceInfo); return accessType.get(); } // in some cases there the resource is a nested class, so check the parent class @@ -57,7 +56,6 @@ public AccessType getAccessType(ResourceInfo resourceInfo) if (resourceInfo.getResourceClass().getDeclaringClass() != null) { accessType = resourceAccessTypeLoader.getAccessType(resourceInfo.getResourceClass().getDeclaringClass()); if (accessType.isPresent()) { - verifyNotTrinoResource(resourceInfo); return accessType.get(); } } From 8b0bd63d83b995f682e3d36b56c66628a1d5890a Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Thu, 19 Dec 2024 12:54:10 +0100 Subject: [PATCH 101/158] Check whole class hierarchy for @ResourceSecurity --- .../trino/server/security/ResourceAccessType.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java b/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java index b71a2d0b9e03..4dba39d87d5d 100644 --- a/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java +++ b/core/trino-main/src/main/java/io/trino/server/security/ResourceAccessType.java @@ -46,18 +46,14 @@ public AccessType getAccessType(ResourceInfo resourceInfo) if (accessType.isPresent()) { return accessType.get(); } - // check if the resource class has an access type declared for all methods - accessType = resourceAccessTypeLoader.getAccessType(resourceInfo.getResourceClass()); - if (accessType.isPresent()) { - return accessType.get(); - } - // in some cases there the resource is a nested class, so check the parent class - // we currently only check one level, but we could handle multiple nesting levels if necessary - if (resourceInfo.getResourceClass().getDeclaringClass() != null) { - accessType = resourceAccessTypeLoader.getAccessType(resourceInfo.getResourceClass().getDeclaringClass()); + // check if the resource class or enclosing classes have an access type declared for all methods + Class current = resourceInfo.getResourceClass(); + while (current != null) { + accessType = resourceAccessTypeLoader.getAccessType(current); if (accessType.isPresent()) { return accessType.get(); } + current = current.getDeclaringClass(); } } // Trino resources are required to have a declared access control From bb80ff8be6073bc970501825e2e7f304b3b9d643 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Thu, 19 Dec 2024 13:13:51 +0100 Subject: [PATCH 102/158] Use class-level @ResourceSecurity annotations --- .../src/main/java/io/trino/server/NodeResource.java | 3 +-- .../src/main/java/io/trino/server/QueryResource.java | 6 +----- .../java/io/trino/server/QueryStateInfoResource.java | 3 +-- .../trino/server/ResourceGroupStateInfoResource.java | 2 +- .../main/java/io/trino/server/StatusResource.java | 3 +-- .../java/io/trino/server/TaskExecutorResource.java | 2 +- .../src/main/java/io/trino/server/TaskResource.java | 12 +----------- .../main/java/io/trino/server/ThreadResource.java | 2 +- .../server/protocol/ExecutingStatementResource.java | 4 +--- .../spooling/CoordinatorSegmentResource.java | 2 -- .../protocol/spooling/WorkerSegmentResource.java | 1 - .../security/oauth2/OAuth2CallbackResource.java | 2 +- .../security/oauth2/OAuth2TokenExchangeResource.java | 4 +--- .../java/io/trino/server/ui/ClusterResource.java | 2 +- .../io/trino/server/ui/ClusterStatsResource.java | 2 +- .../main/java/io/trino/server/ui/LoginResource.java | 4 +--- .../java/io/trino/server/ui/UiQueryResource.java | 5 +---- .../trino/server/ui/WebUiPreviewStaticResource.java | 3 +-- .../java/io/trino/server/ui/WebUiStaticResource.java | 2 +- .../main/java/io/trino/server/ui/WorkerResource.java | 5 +---- 20 files changed, 18 insertions(+), 51 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/server/NodeResource.java b/core/trino-main/src/main/java/io/trino/server/NodeResource.java index 06244e74b208..b92b127573b6 100644 --- a/core/trino-main/src/main/java/io/trino/server/NodeResource.java +++ b/core/trino-main/src/main/java/io/trino/server/NodeResource.java @@ -26,6 +26,7 @@ import static io.trino.server.security.ResourceSecurity.AccessType.MANAGEMENT_READ; @Path("/v1/node") +@ResourceSecurity(MANAGEMENT_READ) public class NodeResource { private final HeartbeatFailureDetector failureDetector; @@ -36,14 +37,12 @@ public NodeResource(HeartbeatFailureDetector failureDetector) this.failureDetector = failureDetector; } - @ResourceSecurity(MANAGEMENT_READ) @GET public Collection getNodeStats() { return failureDetector.getStats().values(); } - @ResourceSecurity(MANAGEMENT_READ) @GET @Path("failed") public Collection getFailed() diff --git a/core/trino-main/src/main/java/io/trino/server/QueryResource.java b/core/trino-main/src/main/java/io/trino/server/QueryResource.java index 1aef011787d4..fa9e4f9c4987 100644 --- a/core/trino-main/src/main/java/io/trino/server/QueryResource.java +++ b/core/trino-main/src/main/java/io/trino/server/QueryResource.java @@ -55,6 +55,7 @@ * Manage queries scheduled on this node */ @Path("/v1/query") +@ResourceSecurity(AUTHENTICATED_USER) public class QueryResource { private final DispatchManager dispatchManager; @@ -69,7 +70,6 @@ public QueryResource(DispatchManager dispatchManager, AccessControl accessContro this.sessionContextFactory = requireNonNull(sessionContextFactory, "sessionContextFactory is null"); } - @ResourceSecurity(AUTHENTICATED_USER) @GET public List getAllQueryInfo(@QueryParam("state") String stateFilter, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) { @@ -87,7 +87,6 @@ public List getAllQueryInfo(@QueryParam("state") String stateFil return builder.build(); } - @ResourceSecurity(AUTHENTICATED_USER) @GET @Path("{queryId}") public Response getQueryInfo(@PathParam("queryId") QueryId queryId, @QueryParam("pruned") @DefaultValue("false") boolean pruned, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) @@ -108,7 +107,6 @@ public Response getQueryInfo(@PathParam("queryId") QueryId queryId, @QueryParam( } } - @ResourceSecurity(AUTHENTICATED_USER) @DELETE @Path("{queryId}") public void cancelQuery(@PathParam("queryId") QueryId queryId, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) @@ -127,7 +125,6 @@ public void cancelQuery(@PathParam("queryId") QueryId queryId, @Context HttpServ } } - @ResourceSecurity(AUTHENTICATED_USER) @PUT @Path("{queryId}/killed") public Response killQuery(@PathParam("queryId") QueryId queryId, String message, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) @@ -135,7 +132,6 @@ public Response killQuery(@PathParam("queryId") QueryId queryId, String message, return failQuery(queryId, createKillQueryException(message), servletRequest, httpHeaders); } - @ResourceSecurity(AUTHENTICATED_USER) @PUT @Path("{queryId}/preempted") public Response preemptQuery(@PathParam("queryId") QueryId queryId, String message, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) diff --git a/core/trino-main/src/main/java/io/trino/server/QueryStateInfoResource.java b/core/trino-main/src/main/java/io/trino/server/QueryStateInfoResource.java index bbf16bcede64..dfc00faba788 100644 --- a/core/trino-main/src/main/java/io/trino/server/QueryStateInfoResource.java +++ b/core/trino-main/src/main/java/io/trino/server/QueryStateInfoResource.java @@ -49,6 +49,7 @@ import static java.util.Objects.requireNonNull; @Path("/v1/queryState") +@ResourceSecurity(AUTHENTICATED_USER) public class QueryStateInfoResource { private final DispatchManager dispatchManager; @@ -69,7 +70,6 @@ public QueryStateInfoResource( this.sessionContextFactory = requireNonNull(sessionContextFactory, "sessionContextFactory is null"); } - @ResourceSecurity(AUTHENTICATED_USER) @GET @Produces(MediaType.APPLICATION_JSON) public List getQueryStateInfos(@QueryParam("user") String user, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) @@ -102,7 +102,6 @@ private QueryStateInfo getQueryStateInfo(BasicQueryInfo queryInfo) return createQueryStateInfo(queryInfo, groupId); } - @ResourceSecurity(AUTHENTICATED_USER) @GET @Path("{queryId}") @Produces(MediaType.APPLICATION_JSON) diff --git a/core/trino-main/src/main/java/io/trino/server/ResourceGroupStateInfoResource.java b/core/trino-main/src/main/java/io/trino/server/ResourceGroupStateInfoResource.java index b3f743283467..33f2c722b72b 100644 --- a/core/trino-main/src/main/java/io/trino/server/ResourceGroupStateInfoResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ResourceGroupStateInfoResource.java @@ -35,6 +35,7 @@ import static java.util.Objects.requireNonNull; @Path("/v1/resourceGroupState") +@ResourceSecurity(MANAGEMENT_READ) public class ResourceGroupStateInfoResource { private final ResourceGroupInfoProvider resourceGroupInfoProvider; @@ -45,7 +46,6 @@ public ResourceGroupStateInfoResource(ResourceGroupInfoProvider resourceGroupInf this.resourceGroupInfoProvider = requireNonNull(resourceGroupInfoProvider, "resourceGroupInfoProvider is null"); } - @ResourceSecurity(MANAGEMENT_READ) @GET @Produces(MediaType.APPLICATION_JSON) @Encoded diff --git a/core/trino-main/src/main/java/io/trino/server/StatusResource.java b/core/trino-main/src/main/java/io/trino/server/StatusResource.java index 25a640f8b4ae..f9f227887fb4 100644 --- a/core/trino-main/src/main/java/io/trino/server/StatusResource.java +++ b/core/trino-main/src/main/java/io/trino/server/StatusResource.java @@ -34,6 +34,7 @@ import static java.util.Objects.requireNonNull; @Path("/v1/status") +@ResourceSecurity(PUBLIC) public class StatusResource { private final NodeInfo nodeInfo; @@ -64,7 +65,6 @@ public StatusResource(NodeVersion nodeVersion, NodeInfo nodeInfo, ServerConfig s } } - @ResourceSecurity(PUBLIC) @HEAD @Produces(APPLICATION_JSON) // to match the GET route public Response statusPing() @@ -72,7 +72,6 @@ public Response statusPing() return Response.ok().build(); } - @ResourceSecurity(PUBLIC) @GET @Produces(APPLICATION_JSON) public NodeStatus getStatus() diff --git a/core/trino-main/src/main/java/io/trino/server/TaskExecutorResource.java b/core/trino-main/src/main/java/io/trino/server/TaskExecutorResource.java index 0f8d19d52fba..e8d0abc6a06e 100644 --- a/core/trino-main/src/main/java/io/trino/server/TaskExecutorResource.java +++ b/core/trino-main/src/main/java/io/trino/server/TaskExecutorResource.java @@ -25,6 +25,7 @@ import static java.util.Objects.requireNonNull; @Path("/v1/maxActiveSplits") +@ResourceSecurity(MANAGEMENT_READ) public class TaskExecutorResource { private final TimeSharingTaskExecutor taskExecutor; @@ -36,7 +37,6 @@ public TaskExecutorResource( this.taskExecutor = requireNonNull(taskExecutor, "taskExecutor is null"); } - @ResourceSecurity(MANAGEMENT_READ) @GET @Produces(MediaType.TEXT_PLAIN) public String getMaxActiveSplit() diff --git a/core/trino-main/src/main/java/io/trino/server/TaskResource.java b/core/trino-main/src/main/java/io/trino/server/TaskResource.java index ed2057e93b1a..97ac4d980aa8 100644 --- a/core/trino-main/src/main/java/io/trino/server/TaskResource.java +++ b/core/trino-main/src/main/java/io/trino/server/TaskResource.java @@ -91,6 +91,7 @@ * Manages tasks on this worker node */ @Path("/v1/task") +@ResourceSecurity(INTERNAL_ONLY) public class TaskResource { private static final Logger log = Logger.get(TaskResource.class); @@ -127,7 +128,6 @@ public TaskResource( this.failureInjector = requireNonNull(failureInjector, "failureInjector is null"); } - @ResourceSecurity(INTERNAL_ONLY) @GET @Produces(MediaType.APPLICATION_JSON) public List getAllTaskInfo(@Context UriInfo uriInfo) @@ -139,7 +139,6 @@ public List getAllTaskInfo(@Context UriInfo uriInfo) return allTaskInfo; } - @ResourceSecurity(INTERNAL_ONLY) @POST @Path("{taskId}") @Consumes(MediaType.APPLICATION_JSON) @@ -178,7 +177,6 @@ public void createOrUpdateTask( asyncResponse.resume(Response.ok().entity(taskInfo).build()); } - @ResourceSecurity(INTERNAL_ONLY) @GET @Path("{taskId}") @Produces(MediaType.APPLICATION_JSON) @@ -228,7 +226,6 @@ public void getTaskInfo( bindAsyncResponse(asyncResponse, withFallbackAfterTimeout(response, timeout, () -> serviceUnavailable(timeout), timeoutExecutor), responseExecutor); } - @ResourceSecurity(INTERNAL_ONLY) @GET @Path("{taskId}/status") @Produces(MediaType.APPLICATION_JSON) @@ -273,7 +270,6 @@ public void getTaskStatus( bindAsyncResponse(asyncResponse, withFallbackAfterTimeout(response, timeout, () -> serviceUnavailable(timeout), timeoutExecutor), responseExecutor); } - @ResourceSecurity(INTERNAL_ONLY) @GET @Path("{taskId}/dynamicfilters") @Produces(MediaType.APPLICATION_JSON) @@ -295,7 +291,6 @@ public void acknowledgeAndGetNewDynamicFilterDomains( asyncResponse.resume(taskManager.acknowledgeAndGetNewDynamicFilterDomains(taskId, currentDynamicFiltersVersion)); } - @ResourceSecurity(INTERNAL_ONLY) @DELETE @Path("{taskId}") @Produces(MediaType.APPLICATION_JSON) @@ -320,7 +315,6 @@ public TaskInfo deleteTask( return taskInfo; } - @ResourceSecurity(INTERNAL_ONLY) @POST @Path("{taskId}/fail") @Consumes(MediaType.APPLICATION_JSON) @@ -334,7 +328,6 @@ public TaskInfo failTask( return taskManager.failTask(taskId, failTaskRequest.getFailureInfo().toException()); } - @ResourceSecurity(INTERNAL_ONLY) @GET @Path("{taskId}/results/{bufferId}/{token}") @Produces(TRINO_PAGES) @@ -375,7 +368,6 @@ public void getResults( responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor()); } - @ResourceSecurity(INTERNAL_ONLY) @GET @Path("{taskId}/results/{bufferId}/{token}/acknowledge") public Response acknowledgeResults( @@ -390,7 +382,6 @@ public Response acknowledgeResults( return Response.ok().build(); } - @ResourceSecurity(INTERNAL_ONLY) @DELETE @Path("{taskId}/results/{bufferId}") public void destroyTaskResults( @@ -409,7 +400,6 @@ public void destroyTaskResults( asyncResponse.resume(Response.noContent().build()); } - @ResourceSecurity(INTERNAL_ONLY) @POST @Path("pruneCatalogs") @Consumes(MediaType.APPLICATION_JSON) diff --git a/core/trino-main/src/main/java/io/trino/server/ThreadResource.java b/core/trino-main/src/main/java/io/trino/server/ThreadResource.java index 9a540e9f2bef..47a5ad2b74b4 100644 --- a/core/trino-main/src/main/java/io/trino/server/ThreadResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ThreadResource.java @@ -37,9 +37,9 @@ import static java.util.Comparator.comparing; @Path("/v1/thread") +@ResourceSecurity(MANAGEMENT_READ) public class ThreadResource { - @ResourceSecurity(MANAGEMENT_READ) @GET @Produces(MediaType.APPLICATION_JSON) public List getThreadInfo() diff --git a/core/trino-main/src/main/java/io/trino/server/protocol/ExecutingStatementResource.java b/core/trino-main/src/main/java/io/trino/server/protocol/ExecutingStatementResource.java index eef5b7185c8e..41bb19f64fa1 100644 --- a/core/trino-main/src/main/java/io/trino/server/protocol/ExecutingStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/server/protocol/ExecutingStatementResource.java @@ -72,6 +72,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; @Path("/v1/statement/executing") +@ResourceSecurity(PUBLIC) public class ExecutingStatementResource { private static final Logger log = Logger.get(ExecutingStatementResource.class); @@ -156,7 +157,6 @@ public void stop() queryPurger.shutdownNow(); } - @ResourceSecurity(PUBLIC) @GET @Path("{queryId}/{slug}/{token}") @Produces(MediaType.APPLICATION_JSON) @@ -295,7 +295,6 @@ private Response toResponse(QueryResultsResponse resultsResponse, Optionalfalse "; @@ -61,7 +62,6 @@ public LoginResource(FormWebUiAuthenticationFilter formWebUiAuthenticationManage verify(loginHtml.contains(REPLACEMENT_TEXT), "login.html does not contain the replacement text"); } - @ResourceSecurity(WEB_UI) @GET @Path(LOGIN_FORM) public Response getFile(@Context SecurityContext securityContext) @@ -72,7 +72,6 @@ public Response getFile(@Context SecurityContext securityContext) .build(); } - @ResourceSecurity(WEB_UI) @POST @Path(UI_LOGIN) public Response login( @@ -101,7 +100,6 @@ public Response login( .build(); } - @ResourceSecurity(WEB_UI) @GET @Path(UI_LOGOUT) public Response logout(@Context HttpHeaders httpHeaders, @Context SecurityContext securityContext, @BeanParam ExternalUriInfo externalUriInfo) diff --git a/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java b/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java index f0205bb0d0d1..c600efb4dc1f 100644 --- a/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java @@ -53,6 +53,7 @@ import static java.util.Objects.requireNonNull; @Path("/ui/api/query") +@ResourceSecurity(WEB_UI) @DisableHttpCache public class UiQueryResource { @@ -68,7 +69,6 @@ public UiQueryResource(DispatchManager dispatchManager, AccessControl accessCont this.sessionContextFactory = requireNonNull(sessionContextFactory, "sessionContextFactory is null"); } - @ResourceSecurity(WEB_UI) @GET public List getAllQueryInfo(@QueryParam("state") String stateFilter, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) { @@ -86,7 +86,6 @@ public List getAllQueryInfo(@QueryParam("state") String s return builder.build(); } - @ResourceSecurity(WEB_UI) @GET @Path("{queryId}") public Response getQueryInfo(@PathParam("queryId") QueryId queryId, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) @@ -106,7 +105,6 @@ public Response getQueryInfo(@PathParam("queryId") QueryId queryId, @Context Htt throw new GoneException(); } - @ResourceSecurity(WEB_UI) @PUT @Path("{queryId}/killed") public Response killQuery(@PathParam("queryId") QueryId queryId, String message, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) @@ -114,7 +112,6 @@ public Response killQuery(@PathParam("queryId") QueryId queryId, String message, return failQuery(queryId, createKillQueryException(message), servletRequest, httpHeaders); } - @ResourceSecurity(WEB_UI) @PUT @Path("{queryId}/preempted") public Response preemptQuery(@PathParam("queryId") QueryId queryId, String message, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) diff --git a/core/trino-main/src/main/java/io/trino/server/ui/WebUiPreviewStaticResource.java b/core/trino-main/src/main/java/io/trino/server/ui/WebUiPreviewStaticResource.java index a8487c3738c5..c1b5217cee51 100644 --- a/core/trino-main/src/main/java/io/trino/server/ui/WebUiPreviewStaticResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ui/WebUiPreviewStaticResource.java @@ -27,16 +27,15 @@ import static io.trino.web.ui.WebUiResources.webUiResource; @Path("/ui/preview") +@ResourceSecurity(WEB_UI) public class WebUiPreviewStaticResource { - @ResourceSecurity(WEB_UI) @GET public Response getUiPreview(@BeanParam ExternalUriInfo externalUriInfo) { return Response.seeOther(externalUriInfo.absolutePath("/ui/preview/index.html")).build(); } - @ResourceSecurity(WEB_UI) @GET @Path("{path: .*}") public Response getFile(@PathParam("path") String path) diff --git a/core/trino-main/src/main/java/io/trino/server/ui/WebUiStaticResource.java b/core/trino-main/src/main/java/io/trino/server/ui/WebUiStaticResource.java index db286b6745cc..801bef0bdffb 100644 --- a/core/trino-main/src/main/java/io/trino/server/ui/WebUiStaticResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ui/WebUiStaticResource.java @@ -30,6 +30,7 @@ import static io.trino.web.ui.WebUiResources.webUiResource; @Path("") +@ResourceSecurity(PUBLIC) public class WebUiStaticResource { @ResourceSecurity(PUBLIC) @@ -39,7 +40,6 @@ public Response getRoot(@BeanParam ExternalUriInfo externalUriInfo) return Response.seeOther(externalUriInfo.absolutePath("/ui/")).build(); } - @ResourceSecurity(PUBLIC) @GET @Path("/ui") public Response getUi(@BeanParam ExternalUriInfo externalUriInfo) diff --git a/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java b/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java index 5fb017148de1..cee38f624f43 100644 --- a/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java @@ -62,6 +62,7 @@ import static java.util.Objects.requireNonNull; @Path("/ui/api/worker") +@ResourceSecurity(WEB_UI) public class WorkerResource { private final DispatchManager dispatchManager; @@ -85,7 +86,6 @@ public WorkerResource( this.sessionContextFactory = requireNonNull(sessionContextFactory, "sessionContextFactory is null"); } - @ResourceSecurity(WEB_UI) @GET @Path("{nodeId}/status") public Response getStatus(@PathParam("nodeId") String nodeId) @@ -93,7 +93,6 @@ public Response getStatus(@PathParam("nodeId") String nodeId) return proxyJsonResponse(nodeId, "v1/status"); } - @ResourceSecurity(WEB_UI) @GET @Path("{nodeId}/thread") public Response getThreads(@PathParam("nodeId") String nodeId) @@ -101,7 +100,6 @@ public Response getThreads(@PathParam("nodeId") String nodeId) return proxyJsonResponse(nodeId, "v1/thread"); } - @ResourceSecurity(WEB_UI) @GET @Path("{nodeId}/task/{taskId}") public Response getThreads( @@ -124,7 +122,6 @@ public Response getThreads( throw new GoneException(); } - @ResourceSecurity(WEB_UI) @GET public Response getWorkerList() { From e7557fe64e3e9ac4fa5a2c9b704734ab84595105 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Tue, 31 Dec 2024 13:42:10 +0100 Subject: [PATCH 103/158] Fix parsing of negative 0x, 0b, 0o long literals --- .../java/io/trino/sql/tree/LongLiteral.java | 9 +++++++ .../io/trino/sql/parser/TestSqlParser.java | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/core/trino-parser/src/main/java/io/trino/sql/tree/LongLiteral.java b/core/trino-parser/src/main/java/io/trino/sql/tree/LongLiteral.java index 0ef8289ccbcd..17591d94ee29 100644 --- a/core/trino-parser/src/main/java/io/trino/sql/tree/LongLiteral.java +++ b/core/trino-parser/src/main/java/io/trino/sql/tree/LongLiteral.java @@ -107,12 +107,21 @@ private static long parse(String value) if (value.startsWith("0x") || value.startsWith("0X")) { return Long.parseLong(value.substring(2), 16); } + else if (value.startsWith("-0x") || value.startsWith("-0X")) { + return Long.parseLong("-" + value.substring(3), 16); + } else if (value.startsWith("0b") || value.startsWith("0B")) { return Long.parseLong(value.substring(2), 2); } + else if (value.startsWith("-0b") || value.startsWith("-0B")) { + return Long.parseLong("-" + value.substring(3), 2); + } else if (value.startsWith("0o") || value.startsWith("0O")) { return Long.parseLong(value.substring(2), 8); } + else if (value.startsWith("-0o") || value.startsWith("-0O")) { + return Long.parseLong("-" + value.substring(3), 8); + } else { return Long.parseLong(value); } diff --git a/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java b/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java index dcc7e3f4b927..9b5196cf1d8a 100644 --- a/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java +++ b/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java @@ -486,6 +486,14 @@ public void testNumbers() .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0X123_ABC_DEF")) .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(4893429231L)); + assertThat(expression("-0x123_abc_def")) + .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "-0x123_abc_def")) + .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(-4893429231L)); + + assertThat(expression("-0X123_ABC_DEF")) + .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "-0X123_ABC_DEF")) + .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(-4893429231L)); + assertThatThrownBy(() -> SQL_PARSER.createExpression("0x123_ABC_DEF_")) .isInstanceOf(ParsingException.class); @@ -497,6 +505,14 @@ public void testNumbers() .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0o012_345")) .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(5349L)); + assertThat(expression("-0O012_345")) + .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "-0O012_345")) + .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(-5349L)); + + assertThat(expression("-0o012_345")) + .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "-0o012_345")) + .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(-5349L)); + assertThatThrownBy(() -> SQL_PARSER.createExpression("0o012_345_")) .isInstanceOf(ParsingException.class); @@ -508,6 +524,14 @@ public void testNumbers() .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0b110_010")) .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(50L)); + assertThat(expression("-0B110_010")) + .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "-0B110_010")) + .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(-50L)); + + assertThat(expression("-0b110_010")) + .isEqualTo(new LongLiteral(new NodeLocation(1, 1), "-0b110_010")) + .satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(-50L)); + assertThatThrownBy(() -> SQL_PARSER.createExpression("0b110_010_")) .isInstanceOf(ParsingException.class); } From 804f7d19030007d3eac937aeae70e09ebb5b8208 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Tue, 31 Dec 2024 14:51:21 +0100 Subject: [PATCH 104/158] Update lucene-analysis-common to 10.1.0 --- core/trino-main/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index 75ae527e0385..2b0fb44cbfd1 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -337,7 +337,7 @@ org.apache.lucene lucene-analysis-common - 10.0.0 + 10.1.0 From 097536b9e3defc6bebd7a08469ad81bf28ca6b5e Mon Sep 17 00:00:00 2001 From: Mayank Vadariya Date: Mon, 23 Dec 2024 01:37:02 -0500 Subject: [PATCH 105/158] Correctly merge multiple splits info Previously SplitOperatorInfo wasn't Mergeable and hence base OperatorInfo(OperatorStats#getMergeableInfoOrNull) was null. --- .../io/trino/operator/SplitOperatorInfo.java | 19 ++++++++++++++++++- .../io/trino/operator/TestOperatorStats.java | 3 ++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/operator/SplitOperatorInfo.java b/core/trino-main/src/main/java/io/trino/operator/SplitOperatorInfo.java index 7a3cac6f6a8d..af8902db194e 100644 --- a/core/trino-main/src/main/java/io/trino/operator/SplitOperatorInfo.java +++ b/core/trino-main/src/main/java/io/trino/operator/SplitOperatorInfo.java @@ -15,14 +15,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.Mergeable; import io.trino.spi.connector.CatalogHandle; +import java.util.List; import java.util.Map; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; public class SplitOperatorInfo - implements OperatorInfo + implements OperatorInfo, Mergeable { private final CatalogHandle catalogHandle; private final Map splitInfo; @@ -53,4 +56,18 @@ public CatalogHandle getCatalogHandle() { return catalogHandle; } + + @Override + public SplitOperatorInfo mergeWith(SplitOperatorInfo other) + { + return mergeWith(List.of(other)); + } + + @Override + public SplitOperatorInfo mergeWith(List others) + { + return new SplitOperatorInfo( + catalogHandle, + splitInfo.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue() + " (" + others.size() + " more)"))); + } } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java b/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java index d16cee4e9c96..0662bc263cb8 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java @@ -241,7 +241,8 @@ public void testAdd() assertThat(actual.getPeakRevocableMemoryReservation()).isEqualTo(DataSize.ofBytes(24)); assertThat(actual.getPeakTotalMemoryReservation()).isEqualTo(DataSize.ofBytes(25)); assertThat(actual.getSpilledDataSize()).isEqualTo(DataSize.ofBytes(3 * 26)); - assertThat(actual.getInfo()).isNull(); + assertThat(actual.getInfo()).isInstanceOf(SplitOperatorInfo.class); + assertThat(((SplitOperatorInfo) actual.getInfo()).getSplitInfo().get("some_info")).isEqualTo("some_value (2 more)"); } @Test From b8785daaf7c099eeb648561cbb69f7c35f94ab20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Fri, 18 Oct 2024 14:44:58 +0200 Subject: [PATCH 106/158] Prepare to implement a page source provider for Redshift --- .../plugin/redshift/RedshiftClientModule.java | 6 + .../plugin/redshift/RedshiftConnector.java | 180 ++++++++++++++++++ .../redshift/RedshiftConnectorFactory.java | 68 +++++++ .../redshift/RedshiftPageSourceProvider.java | 53 ++++++ .../trino/plugin/redshift/RedshiftPlugin.java | 11 +- 5 files changed, 314 insertions(+), 4 deletions(-) create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnector.java create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java index d0e590c9a52d..93962bd5a501 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java @@ -16,6 +16,7 @@ import com.amazon.redshift.Driver; import com.google.inject.Binder; import com.google.inject.Provides; +import com.google.inject.Scopes; import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.opentelemetry.api.OpenTelemetry; @@ -27,10 +28,12 @@ import io.trino.plugin.jdbc.JdbcClient; import io.trino.plugin.jdbc.JdbcJoinPushdownSupportModule; import io.trino.plugin.jdbc.JdbcMetadataConfig; +import io.trino.plugin.jdbc.JdbcRecordSetProvider; import io.trino.plugin.jdbc.JdbcStatisticsConfig; import io.trino.plugin.jdbc.RemoteQueryCancellationModule; import io.trino.plugin.jdbc.credential.CredentialProvider; import io.trino.plugin.jdbc.ptf.Query; +import io.trino.spi.connector.ConnectorRecordSetProvider; import io.trino.spi.function.table.ConnectorTableFunction; import java.util.Properties; @@ -54,6 +57,9 @@ public void setup(Binder binder) install(new DecimalModule()); install(new JdbcJoinPushdownSupportModule()); install(new RemoteQueryCancellationModule()); + binder.bind(ConnectorRecordSetProvider.class).to(JdbcRecordSetProvider.class).in(Scopes.SINGLETON); + + binder.bind(RedshiftConnector.class).in(Scopes.SINGLETON); } @Singleton diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnector.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnector.java new file mode 100644 index 000000000000..e1f98eed5703 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnector.java @@ -0,0 +1,180 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.common.collect.ImmutableSet; +import com.google.inject.Inject; +import io.airlift.bootstrap.LifeCycleManager; +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorMetadata; +import io.trino.plugin.base.session.SessionPropertiesProvider; +import io.trino.plugin.jdbc.JdbcTransactionManager; +import io.trino.plugin.jdbc.TablePropertiesProvider; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorAccessControl; +import io.trino.spi.connector.ConnectorCapabilities; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorPageSinkProvider; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorRecordSetProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.function.table.ConnectorTableFunction; +import io.trino.spi.procedure.Procedure; +import io.trino.spi.session.PropertyMetadata; +import io.trino.spi.transaction.IsolationLevel; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Sets.immutableEnumSet; +import static io.trino.spi.connector.ConnectorCapabilities.NOT_NULL_COLUMN_CONSTRAINT; +import static java.util.Objects.requireNonNull; + +public class RedshiftConnector + implements Connector +{ + private final LifeCycleManager lifeCycleManager; + private final ConnectorSplitManager jdbcSplitManager; + private final ConnectorPageSinkProvider jdbcPageSinkProvider; + private final Optional accessControl; + private final Set procedures; + private final Set connectorTableFunctions; + private final List> sessionProperties; + private final List> tableProperties; + private final JdbcTransactionManager transactionManager; + private final RedshiftPageSourceProvider pageSourceProvider; + + @Inject + public RedshiftConnector( + LifeCycleManager lifeCycleManager, + ConnectorSplitManager jdbcSplitManager, + ConnectorRecordSetProvider jdbcRecordSetProvider, + ConnectorPageSinkProvider jdbcPageSinkProvider, + Optional accessControl, + Set procedures, + Set connectorTableFunctions, + Set sessionProperties, + Set tableProperties, + JdbcTransactionManager transactionManager) + { + this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); + this.jdbcSplitManager = requireNonNull(jdbcSplitManager, "jdbcSplitManager is null"); + this.jdbcPageSinkProvider = requireNonNull(jdbcPageSinkProvider, "jdbcPageSinkProvider is null"); + this.accessControl = requireNonNull(accessControl, "accessControl is null"); + this.procedures = ImmutableSet.copyOf(requireNonNull(procedures, "procedures is null")); + this.connectorTableFunctions = ImmutableSet.copyOf(requireNonNull(connectorTableFunctions, "connectorTableFunctions is null")); + this.sessionProperties = sessionProperties.stream() + .flatMap(sessionPropertiesProvider -> sessionPropertiesProvider.getSessionProperties().stream()) + .collect(toImmutableList()); + this.tableProperties = tableProperties.stream() + .flatMap(tablePropertiesProvider -> tablePropertiesProvider.getTableProperties().stream()) + .collect(toImmutableList()); + this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.pageSourceProvider = new RedshiftPageSourceProvider(jdbcRecordSetProvider); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit) + { + return transactionManager.beginTransaction(isolationLevel, readOnly, autoCommit); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transaction) + { + return new ClassLoaderSafeConnectorMetadata(transactionManager.getMetadata(transaction), getClass().getClassLoader()); + } + + @Override + public void commit(ConnectorTransactionHandle transaction) + { + transactionManager.commit(transaction); + } + + @Override + public void rollback(ConnectorTransactionHandle transaction) + { + transactionManager.rollback(transaction); + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return jdbcSplitManager; + } + + @Override + public ConnectorRecordSetProvider getRecordSetProvider() + { + // throwing this exception will ensure using the RedshiftPageSourceProvider, that supports two modes of operation + throw new UnsupportedOperationException(); + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + return pageSourceProvider; + } + + @Override + public ConnectorPageSinkProvider getPageSinkProvider() + { + return jdbcPageSinkProvider; + } + + @Override + public ConnectorAccessControl getAccessControl() + { + return accessControl.orElseThrow(UnsupportedOperationException::new); + } + + @Override + public Set getProcedures() + { + return procedures; + } + + @Override + public Set getTableFunctions() + { + return connectorTableFunctions; + } + + @Override + public List> getSessionProperties() + { + return sessionProperties; + } + + @Override + public List> getTableProperties() + { + return tableProperties; + } + + @Override + public final void shutdown() + { + lifeCycleManager.stop(); + } + + @Override + public Set getCapabilities() + { + return immutableEnumSet(NOT_NULL_COLUMN_CONSTRAINT); + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java new file mode 100644 index 000000000000..5312ca010f68 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.opentelemetry.api.OpenTelemetry; +import io.trino.plugin.jdbc.ExtraCredentialsBasedIdentityCacheMappingModule; +import io.trino.plugin.jdbc.JdbcModule; +import io.trino.plugin.jdbc.credential.CredentialProviderModule; +import io.trino.spi.NodeManager; +import io.trino.spi.VersionEmbedder; +import io.trino.spi.catalog.CatalogName; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.type.TypeManager; + +import java.util.Map; + +import static io.trino.plugin.base.Versions.checkStrictSpiVersionMatch; +import static java.util.Objects.requireNonNull; + +public class RedshiftConnectorFactory + implements ConnectorFactory +{ + @Override + public String getName() + { + return "redshift"; + } + + @Override + public Connector create(String catalogName, Map requiredConfig, ConnectorContext context) + { + requireNonNull(requiredConfig, "requiredConfig is null"); + checkStrictSpiVersionMatch(context, this); + + Bootstrap app = new Bootstrap( + binder -> binder.bind(TypeManager.class).toInstance(context.getTypeManager()), + binder -> binder.bind(NodeManager.class).toInstance(context.getNodeManager()), + binder -> binder.bind(VersionEmbedder.class).toInstance(context.getVersionEmbedder()), + binder -> binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()), + binder -> binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)), + new JdbcModule(), + new CredentialProviderModule(), + new ExtraCredentialsBasedIdentityCacheMappingModule(), + new RedshiftClientModule()); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(requiredConfig) + .initialize(); + + return injector.getInstance(RedshiftConnector.class); + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java new file mode 100644 index 000000000000..60f22c656d29 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorRecordSetProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.DynamicFilter; +import io.trino.spi.connector.RecordPageSource; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class RedshiftPageSourceProvider + implements ConnectorPageSourceProvider +{ + private final ConnectorRecordSetProvider recordSetProvider; + + public RedshiftPageSourceProvider(ConnectorRecordSetProvider recordSetProvider) + { + this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); + } + + @Override + public ConnectorPageSource createPageSource( + ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorSplit split, + ConnectorTableHandle table, + List columns, + DynamicFilter dynamicFilter) + { + // TODO switch between two modes of operation - JDBC and UNLOAD + return new RecordPageSource(recordSetProvider.getRecordSet(transaction, session, split, table, columns)); + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPlugin.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPlugin.java index 25309d0e6484..5757afe7742a 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPlugin.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPlugin.java @@ -13,13 +13,16 @@ */ package io.trino.plugin.redshift; -import io.trino.plugin.jdbc.JdbcPlugin; +import com.google.common.collect.ImmutableList; +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; public class RedshiftPlugin - extends JdbcPlugin + implements Plugin { - public RedshiftPlugin() + @Override + public Iterable getConnectorFactories() { - super("redshift", RedshiftClientModule::new); + return ImmutableList.of(new RedshiftConnectorFactory()); } } From b382e030959a9d4d2ca7727c8de34ed8ca2d21c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 24 Oct 2024 09:56:42 +0200 Subject: [PATCH 107/158] Fetch Redshift query results unloaded to S3 Co-authored-by: Mayank Vadariya --- .github/workflows/ci.yml | 2 + docs/src/main/sphinx/connector/redshift.md | 37 +++ plugin/trino-redshift/pom.xml | 81 +++++- .../plugin/redshift/RedshiftClientModule.java | 29 ++- .../trino/plugin/redshift/RedshiftConfig.java | 29 +++ .../redshift/RedshiftConnectorFactory.java | 2 +- .../plugin/redshift/RedshiftErrorCode.java | 4 + .../redshift/RedshiftPageSourceProvider.java | 98 +++++++- .../redshift/RedshiftParquetPageSource.java | 127 ++++++++++ .../redshift/RedshiftSessionProperties.java | 62 +++++ .../plugin/redshift/RedshiftSplitManager.java | 157 ++++++++++++ ...ctor.java => RedshiftUnloadConnector.java} | 13 +- .../RedshiftUnloadJdbcQueryEventListener.java | 57 +++++ .../plugin/redshift/RedshiftUnloadSplit.java | 47 ++++ .../redshift/RedshiftUnloadSplitSource.java | 196 +++++++++++++++ .../redshift/TrinoParquetDataSource.java | 68 +++++ .../plugin/redshift/RedshiftQueryRunner.java | 2 +- .../plugin/redshift/TestRedshiftConfig.java | 10 +- .../plugin/redshift/TestRedshiftPlugin.java | 18 ++ .../plugin/redshift/TestRedshiftUnload.java | 232 ++++++++++++++++++ .../TestRedshiftUnloadTypeMapping.java | 54 ++++ 21 files changed, 1308 insertions(+), 17 deletions(-) create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftParquetPageSource.java create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSessionProperties.java create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSplitManager.java rename plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/{RedshiftConnector.java => RedshiftUnloadConnector.java} (93%) create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadJdbcQueryEventListener.java create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplit.java create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplitSource.java create mode 100644 plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/TrinoParquetDataSource.java create mode 100644 plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java create mode 100644 plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnloadTypeMapping.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 086fc01298ca..913ed2e81b63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -740,6 +740,7 @@ jobs: REDSHIFT_IAM_ROLES: ${{ vars.REDSHIFT_IAM_ROLES }} REDSHIFT_VPC_SECURITY_GROUP_IDS: ${{ vars.REDSHIFT_VPC_SECURITY_GROUP_IDS }} REDSHIFT_S3_TPCH_TABLES_ROOT: ${{ vars.REDSHIFT_S3_TPCH_TABLES_ROOT }} + REDSHIFT_S3_UNLOAD_ROOT: ${{ vars.REDSHIFT_S3_UNLOAD_ROOT }} if: >- contains(matrix.modules, 'trino-redshift') && (contains(matrix.profile, 'cloud-tests') || contains(matrix.profile, 'fte-tests')) && @@ -752,6 +753,7 @@ jobs: -Dtest.redshift.jdbc.password="${REDSHIFT_PASSWORD}" \ -Dtest.redshift.jdbc.endpoint="${REDSHIFT_ENDPOINT}:${REDSHIFT_PORT}/" \ -Dtest.redshift.s3.tpch.tables.root="${REDSHIFT_S3_TPCH_TABLES_ROOT}" \ + -Dtest.redshift.s3.unload.root="${REDSHIFT_S3_UNLOAD_ROOT}" \ -Dtest.redshift.iam.role="${REDSHIFT_IAM_ROLES}" \ -Dtest.redshift.aws.region="${AWS_REGION}" \ -Dtest.redshift.aws.access-key="${AWS_ACCESS_KEY_ID}" \ diff --git a/docs/src/main/sphinx/connector/redshift.md b/docs/src/main/sphinx/connector/redshift.md index ba21b13134cd..659905f56ff5 100644 --- a/docs/src/main/sphinx/connector/redshift.md +++ b/docs/src/main/sphinx/connector/redshift.md @@ -64,6 +64,43 @@ documentation](https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-configura ```{include} jdbc-authentication.fragment ``` +### UNLOAD configuration + +This feature enables using Amazon S3 to efficiently transfer data out of Redshift +instead of the default single threaded JDBC based implementation. +The connector automatically triggers the appropriate `UNLOAD` command +on Redshift to extract the output from Redshift to the configured +S3 bucket in the form of Parquet files. These Parquet files are read in parallel +from S3 to improve latency of reading from Redshift tables. The Parquet +files will be removed when Trino finishes executing the query. It is recommended +to define a custom life cycle policy on the S3 bucket used for unloading the +Redshift query results. +This feature is supported only when the Redshift cluster and the configured S3 +bucket are in the same AWS region. + +The following table describes configuration properties for using +`UNLOAD` command in Redshift connector. `redshift.unload-location` must be set +to use `UNLOAD`. + +:::{list-table} UNLOAD configuration properties +:widths: 30, 60 +:header-rows: 1 + +* - Property value + - Description +* - `redshift.unload-location` + - A writeable location in Amazon S3, to be used for temporarily unloading + Redshift query results. +* - `redshift.unload-iam-role` + - Optional. Fully specified ARN of the IAM Role attached to the Redshift cluster. + Provided role will be used in `UNLOAD` command. IAM role must have access to + Redshift cluster and write access to S3 bucket. The default IAM role attached to + Redshift cluster is used when this property is not configured. +::: + +Additionally, define appropriate [S3 configurations](/object-storage/file-system-s3) +except `fs.native-s3.enabled`, required to read Parquet files from S3 bucket. + ### Multiple Redshift databases or clusters The Redshift connector can only access a single database within diff --git a/plugin/trino-redshift/pom.xml b/plugin/trino-redshift/pom.xml index e755eaa2ae92..8ebcc07a11a8 100644 --- a/plugin/trino-redshift/pom.xml +++ b/plugin/trino-redshift/pom.xml @@ -20,6 +20,11 @@ 2.1.0.30 + + com.fasterxml.jackson.core + jackson-databind + + com.google.guava guava @@ -30,21 +35,61 @@ guice + + io.airlift + bootstrap + + io.airlift configuration + + io.airlift + json + + + + io.airlift + log + + + + io.airlift + units + + io.trino trino-base-jdbc + + io.trino + trino-filesystem + + + + io.trino + trino-filesystem-s3 + + io.trino trino-matching + + io.trino + trino-memory-context + + + + io.trino + trino-parquet + + io.trino trino-plugin-toolkit @@ -55,6 +100,16 @@ jakarta.validation-api + + joda-time + joda-time + + + + org.apache.parquet + parquet-column + + org.jdbi jdbi3-core @@ -116,19 +171,31 @@ io.airlift - log + log-manager runtime - io.airlift - log-manager + software.amazon.awssdk + auth runtime - io.airlift - units + software.amazon.awssdk + aws-core + runtime + + + + software.amazon.awssdk + regions + runtime + + + + software.amazon.awssdk + s3 runtime @@ -236,9 +303,11 @@ **/TestRedshiftAutomaticJoinPushdown.java **/TestRedshiftCastPushdown.java **/TestRedshiftConnectorTest.java + **/TestRedshiftUnload.java **/TestRedshiftConnectorSmokeTest.java **/TestRedshiftTableStatisticsReader.java **/TestRedshiftTypeMapping.java + **/TestRedshiftUnloadTypeMapping.java **/Test*FailureRecoveryTest.java **/Test*FailureRecoverySmokeTest.java @@ -265,6 +334,8 @@ **/TestRedshiftCastPushdown.java **/TestRedshiftConnectorSmokeTest.java + **/TestRedshiftUnloadTypeMapping.java + **/TestRedshiftUnload.java diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java index 93962bd5a501..0c6e01999ab1 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClientModule.java @@ -15,32 +15,44 @@ import com.amazon.redshift.Driver; import com.google.inject.Binder; +import com.google.inject.Key; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.opentelemetry.api.OpenTelemetry; +import io.trino.filesystem.s3.S3FileSystemModule; +import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.jdbc.BaseJdbcConfig; import io.trino.plugin.jdbc.ConnectionFactory; import io.trino.plugin.jdbc.DecimalModule; import io.trino.plugin.jdbc.DriverConnectionFactory; import io.trino.plugin.jdbc.ForBaseJdbc; +import io.trino.plugin.jdbc.ForJdbcDynamicFiltering; import io.trino.plugin.jdbc.JdbcClient; +import io.trino.plugin.jdbc.JdbcConnector; import io.trino.plugin.jdbc.JdbcJoinPushdownSupportModule; import io.trino.plugin.jdbc.JdbcMetadataConfig; +import io.trino.plugin.jdbc.JdbcQueryEventListener; import io.trino.plugin.jdbc.JdbcRecordSetProvider; +import io.trino.plugin.jdbc.JdbcSplitManager; import io.trino.plugin.jdbc.JdbcStatisticsConfig; import io.trino.plugin.jdbc.RemoteQueryCancellationModule; import io.trino.plugin.jdbc.credential.CredentialProvider; import io.trino.plugin.jdbc.ptf.Query; +import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorRecordSetProvider; +import io.trino.spi.connector.ConnectorSplitManager; import io.trino.spi.function.table.ConnectorTableFunction; import java.util.Properties; import static com.google.inject.Scopes.SINGLETON; import static com.google.inject.multibindings.Multibinder.newSetBinder; +import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; +import static io.airlift.configuration.ConditionalModule.conditionalModule; import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.trino.plugin.jdbc.JdbcModule.bindSessionPropertiesProvider; public class RedshiftClientModule extends AbstractConfigurationAwareModule @@ -53,13 +65,28 @@ public void setup(Binder binder) configBinder(binder).bindConfigDefaults(JdbcMetadataConfig.class, config -> config.setBulkListColumns(true)); newSetBinder(binder, ConnectorTableFunction.class).addBinding().toProvider(Query.class).in(SINGLETON); configBinder(binder).bindConfig(JdbcStatisticsConfig.class); + bindSessionPropertiesProvider(binder, RedshiftSessionProperties.class); install(new DecimalModule()); install(new JdbcJoinPushdownSupportModule()); install(new RemoteQueryCancellationModule()); binder.bind(ConnectorRecordSetProvider.class).to(JdbcRecordSetProvider.class).in(Scopes.SINGLETON); - binder.bind(RedshiftConnector.class).in(Scopes.SINGLETON); + install(conditionalModule( + RedshiftConfig.class, + config -> config.getUnloadLocation().isPresent(), + unloadBinder -> { + install(new S3FileSystemModule()); + unloadBinder.bind(JdbcSplitManager.class).in(Scopes.SINGLETON); + unloadBinder.bind(Connector.class).to(RedshiftUnloadConnector.class).in(Scopes.SINGLETON); + unloadBinder.bind(FileFormatDataSourceStats.class).in(Scopes.SINGLETON); + + newSetBinder(unloadBinder, JdbcQueryEventListener.class).addBinding().to(RedshiftUnloadJdbcQueryEventListener.class).in(Scopes.SINGLETON); + + newOptionalBinder(unloadBinder, Key.get(ConnectorSplitManager.class, ForJdbcDynamicFiltering.class)) + .setBinding().to(RedshiftSplitManager.class).in(SINGLETON); + }, + jdbcBinder -> jdbcBinder.bind(Connector.class).to(JdbcConnector.class).in(Scopes.SINGLETON))); } @Singleton diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConfig.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConfig.java index 370f31e7dc8b..0d8a193ff5f4 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConfig.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConfig.java @@ -17,6 +17,7 @@ import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.DefunctConfig; import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.Pattern; import java.util.Optional; @@ -27,6 +28,8 @@ public class RedshiftConfig { private Integer fetchSize; + private String unloadLocation; + private String unloadIamRole; public Optional<@Min(0) Integer> getFetchSize() { @@ -40,4 +43,30 @@ public RedshiftConfig setFetchSize(Integer fetchSize) this.fetchSize = fetchSize; return this; } + + public Optional<@Pattern(regexp = "^s3://[^/]+(/[^/]+)?$", message = "Path shouldn't end with trailing slash") String> getUnloadLocation() + { + return Optional.ofNullable(unloadLocation); + } + + @Config("redshift.unload-location") + @ConfigDescription("A writeable location in Amazon S3, to be used for unloading Redshift query results") + public RedshiftConfig setUnloadLocation(String unloadLocation) + { + this.unloadLocation = unloadLocation; + return this; + } + + public Optional getUnloadIamRole() + { + return Optional.ofNullable(unloadIamRole); + } + + @Config("redshift.unload-iam-role") + @ConfigDescription("Fully specified ARN of the IAM Role attached to the Redshift cluster and having access to S3") + public RedshiftConfig setUnloadIamRole(String unloadIamRole) + { + this.unloadIamRole = unloadIamRole; + return this; + } } diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java index 5312ca010f68..6a2a5e2e563c 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java @@ -63,6 +63,6 @@ public Connector create(String catalogName, Map requiredConfig, .setRequiredConfigurationProperties(requiredConfig) .initialize(); - return injector.getInstance(RedshiftConnector.class); + return injector.getInstance(Connector.class); } } diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftErrorCode.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftErrorCode.java index e09279270270..2d9fa999a2ac 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftErrorCode.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftErrorCode.java @@ -23,6 +23,10 @@ public enum RedshiftErrorCode implements ErrorCodeSupplier { REDSHIFT_INVALID_TYPE(0, EXTERNAL), + REDSHIFT_PARQUET_BAD_DATA(1, EXTERNAL), + REDSHIFT_PARQUET_CURSOR_ERROR(2, EXTERNAL), + REDSHIFT_FILESYSTEM_ERROR(3, EXTERNAL), + REDSHIFT_S3_CROSS_REGION_UNSUPPORTED(4, EXTERNAL), /**/; private final ErrorCode errorCode; diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java index 60f22c656d29..ff14082a2878 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java @@ -13,6 +13,22 @@ */ package io.trino.plugin.redshift; +import com.google.common.collect.ImmutableList; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.filesystem.TrinoInputFile; +import io.trino.parquet.Column; +import io.trino.parquet.ParquetReaderOptions; +import io.trino.parquet.metadata.BlockMetadata; +import io.trino.parquet.metadata.ParquetMetadata; +import io.trino.parquet.reader.MetadataReader; +import io.trino.parquet.reader.ParquetReader; +import io.trino.parquet.reader.RowGroupInfo; +import io.trino.plugin.base.metrics.FileFormatDataSourceStats; +import io.trino.plugin.jdbc.JdbcColumnHandle; +import io.trino.plugin.jdbc.JdbcSplit; +import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorPageSource; import io.trino.spi.connector.ConnectorPageSourceProvider; @@ -23,19 +39,38 @@ import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.DynamicFilter; import io.trino.spi.connector.RecordPageSource; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.schema.MessageType; +import org.joda.time.DateTimeZone; +import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Optional; +import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.trino.parquet.ParquetTypeUtils.constructField; +import static io.trino.parquet.ParquetTypeUtils.getColumnIO; +import static io.trino.parquet.ParquetTypeUtils.getDescriptors; +import static io.trino.parquet.ParquetTypeUtils.lookupColumnByName; +import static io.trino.parquet.metadata.PrunedBlockMetadata.createPrunedColumnsMetadata; +import static io.trino.plugin.redshift.RedshiftErrorCode.REDSHIFT_PARQUET_CURSOR_ERROR; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public class RedshiftPageSourceProvider implements ConnectorPageSourceProvider { private final ConnectorRecordSetProvider recordSetProvider; + private final TrinoFileSystemFactory fileSystemFactory; + private final FileFormatDataSourceStats fileFormatDataSourceStats; - public RedshiftPageSourceProvider(ConnectorRecordSetProvider recordSetProvider) + public RedshiftPageSourceProvider(ConnectorRecordSetProvider recordSetProvider, TrinoFileSystemFactory fileSystemFactory, FileFormatDataSourceStats fileFormatDataSourceStats) { this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); + this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); + this.fileFormatDataSourceStats = requireNonNull(fileFormatDataSourceStats, "fileFormatDataSourceStats is null"); } @Override @@ -47,7 +82,64 @@ public ConnectorPageSource createPageSource( List columns, DynamicFilter dynamicFilter) { - // TODO switch between two modes of operation - JDBC and UNLOAD - return new RecordPageSource(recordSetProvider.getRecordSet(transaction, session, split, table, columns)); + if (split instanceof JdbcSplit) { + return new RecordPageSource(recordSetProvider.getRecordSet(transaction, session, split, table, columns)); + } + + RedshiftUnloadSplit redshiftUnloadSplit = ((RedshiftUnloadSplit) split); + String path = redshiftUnloadSplit.path(); + Location location = Location.of(path); + TrinoFileSystem fileSystem = fileSystemFactory.create(session); + TrinoInputFile inputFile = fileSystem.newInputFile(location, redshiftUnloadSplit.length()); + ParquetReader parquetReader; + try { + parquetReader = parquetReader(inputFile, columns); + } + catch (IOException e) { + throw new TrinoException(REDSHIFT_PARQUET_CURSOR_ERROR, format("Failed to open Parquet file: %s", path), e); + } + return new RedshiftParquetPageSource(parquetReader); + } + + private ParquetReader parquetReader(TrinoInputFile inputFile, List columns) + throws IOException + { + ParquetReaderOptions options = new ParquetReaderOptions(); + TrinoParquetDataSource dataSource = new TrinoParquetDataSource(inputFile, options, fileFormatDataSourceStats); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + MessageType fileSchema = parquetMetadata.getFileMetaData().getSchema(); + MessageColumnIO messageColumn = getColumnIO(fileSchema, fileSchema); + Map, ColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); + DateTimeZone timeZone = DateTimeZone.UTC; + List fields = fields(columns, messageColumn); + long nextStart = 0; + ImmutableList.Builder rowGroupInfoBuilder = ImmutableList.builder(); + for (BlockMetadata block : parquetMetadata.getBlocks()) { + rowGroupInfoBuilder.add(new RowGroupInfo(createPrunedColumnsMetadata(block, dataSource.getId(), descriptorsByPath), nextStart, Optional.empty())); + nextStart += block.rowCount(); + } + return new ParquetReader( + Optional.ofNullable(parquetMetadata.getFileMetaData().getCreatedBy()), + fields, + rowGroupInfoBuilder.build(), + dataSource, + timeZone, + newSimpleAggregatedMemoryContext(), + options, + RedshiftParquetPageSource::handleException, + Optional.empty(), + Optional.empty()); + } + + private static List fields(List columns, MessageColumnIO messageColumn) + { + ImmutableList.Builder parquetColumnFieldsBuilder = ImmutableList.builder(); + for (ColumnHandle column : columns) { + JdbcColumnHandle jdbcColumn = (JdbcColumnHandle) column; + constructField(jdbcColumn.getColumnType(), lookupColumnByName(messageColumn, jdbcColumn.getColumnName())) + .ifPresent(field -> parquetColumnFieldsBuilder.add(new Column(jdbcColumn.getColumnName(), field))); + } + + return parquetColumnFieldsBuilder.build(); } } diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftParquetPageSource.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftParquetPageSource.java new file mode 100644 index 000000000000..3371f28c6e98 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftParquetPageSource.java @@ -0,0 +1,127 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import io.trino.parquet.ParquetCorruptionException; +import io.trino.parquet.reader.ParquetReader; +import io.trino.spi.Page; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.metrics.Metrics; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.OptionalLong; + +import static io.trino.plugin.base.util.Closables.closeAllSuppress; +import static io.trino.plugin.redshift.RedshiftErrorCode.REDSHIFT_PARQUET_BAD_DATA; +import static io.trino.plugin.redshift.RedshiftErrorCode.REDSHIFT_PARQUET_CURSOR_ERROR; +import static java.util.Objects.requireNonNull; + +public class RedshiftParquetPageSource + implements ConnectorPageSource +{ + private final ParquetReader parquetReader; + private boolean closed; + private long completedPositions; + + public RedshiftParquetPageSource(ParquetReader parquetReader) + { + this.parquetReader = requireNonNull(parquetReader, "parquetReader is null"); + } + + @Override + public long getCompletedBytes() + { + return parquetReader.getDataSource().getReadBytes(); + } + + @Override + public OptionalLong getCompletedPositions() + { + return OptionalLong.of(completedPositions); + } + + @Override + public long getReadTimeNanos() + { + return parquetReader.getDataSource().getReadTimeNanos(); + } + + @Override + public boolean isFinished() + { + return closed; + } + + @Override + public Page getNextPage() + { + Page page; + try { + page = parquetReader.nextPage(); + } + catch (IOException | RuntimeException e) { + closeAllSuppress(e, this); + throw handleException(e); + } + + if (closed || page == null) { + close(); + return null; + } + + completedPositions += page.getPositionCount(); + return page; + } + + @Override + public long getMemoryUsage() + { + return parquetReader.getMemoryContext().getBytes(); + } + + @Override + public void close() + { + if (closed) { + return; + } + closed = true; + + try { + parquetReader.close(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public Metrics getMetrics() + { + return parquetReader.getMetrics(); + } + + static TrinoException handleException(Exception exception) + { + if (exception instanceof TrinoException) { + return (TrinoException) exception; + } + if (exception instanceof ParquetCorruptionException) { + return new TrinoException(REDSHIFT_PARQUET_BAD_DATA, exception); + } + return new TrinoException(REDSHIFT_PARQUET_CURSOR_ERROR, exception.getMessage(), exception); + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSessionProperties.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSessionProperties.java new file mode 100644 index 000000000000..48c7c75a2290 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSessionProperties.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.plugin.base.session.SessionPropertiesProvider; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.session.PropertyMetadata; + +import java.util.List; + +import static io.trino.spi.StandardErrorCode.INVALID_SESSION_PROPERTY; +import static io.trino.spi.session.PropertyMetadata.booleanProperty; + +public class RedshiftSessionProperties + implements SessionPropertiesProvider +{ + private static final String UNLOAD_ENABLED = "unload_enabled"; + + private final List> sessionProperties; + + @Inject + public RedshiftSessionProperties(RedshiftConfig config) + { + sessionProperties = ImmutableList.>builder() + .add(booleanProperty( + UNLOAD_ENABLED, + "Use UNLOAD for reading query results", + config.getUnloadLocation().isPresent(), + value -> { + if (value && config.getUnloadLocation().isEmpty()) { + throw new TrinoException(INVALID_SESSION_PROPERTY, "Cannot use UNLOAD when unload location is not configured"); + } + }, + false)) + .build(); + } + + @Override + public List> getSessionProperties() + { + return sessionProperties; + } + + public static boolean isUnloadEnabled(ConnectorSession session) + { + return session.getProperty(UNLOAD_ENABLED, Boolean.class); + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSplitManager.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSplitManager.java new file mode 100644 index 000000000000..5851bfb3c151 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftSplitManager.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.inject.Inject; +import io.airlift.log.Logger; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.filesystem.s3.FileSystemS3; +import io.trino.plugin.jdbc.ForRecordCursor; +import io.trino.plugin.jdbc.JdbcClient; +import io.trino.plugin.jdbc.JdbcColumnHandle; +import io.trino.plugin.jdbc.JdbcProcedureHandle; +import io.trino.plugin.jdbc.JdbcSplit; +import io.trino.plugin.jdbc.JdbcSplitManager; +import io.trino.plugin.jdbc.JdbcTableHandle; +import io.trino.plugin.jdbc.QueryBuilder; +import io.trino.plugin.jdbc.logging.RemoteQueryModifier; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorSplitSource; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.Constraint; +import io.trino.spi.connector.DynamicFilter; +import io.trino.spi.connector.FixedSplitSource; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.TimeType; +import io.trino.spi.type.VarbinaryType; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; + +import static io.trino.plugin.jdbc.JdbcDynamicFilteringSessionProperties.dynamicFilteringEnabled; +import static io.trino.plugin.redshift.RedshiftSessionProperties.isUnloadEnabled; +import static java.util.Objects.requireNonNull; + +public class RedshiftSplitManager + implements ConnectorSplitManager +{ + private static final Logger log = Logger.get(RedshiftSplitManager.class); + + private final JdbcClient jdbcClient; + private final QueryBuilder queryBuilder; + private final RemoteQueryModifier queryModifier; + private final JdbcSplitManager jdbcSplitManager; + private final Optional unloadLocation; + private final Optional unloadAuthorization; + private final ExecutorService executor; + private final TrinoFileSystemFactory fileSystemFactory; + + @Inject + public RedshiftSplitManager( + JdbcClient jdbcClient, + QueryBuilder queryBuilder, + RemoteQueryModifier queryModifier, + JdbcSplitManager jdbcSplitManager, + RedshiftConfig redshiftConfig, + @FileSystemS3 TrinoFileSystemFactory fileSystemFactory, + @ForRecordCursor ExecutorService executor) + { + this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null"); + this.queryBuilder = requireNonNull(queryBuilder, "queryBuilder is null"); + this.queryModifier = requireNonNull(queryModifier, "queryModifier is null"); + this.jdbcSplitManager = requireNonNull(jdbcSplitManager, "jdbcSplitManager is null"); + this.unloadLocation = redshiftConfig.getUnloadLocation(); + this.unloadAuthorization = redshiftConfig.getUnloadIamRole(); + this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); + this.executor = requireNonNull(executor, "executor is null"); + } + + @Override + public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableHandle table, DynamicFilter dynamicFilter, Constraint constraint) + { + if (table instanceof JdbcProcedureHandle) { + return jdbcSplitManager.getSplits(transaction, session, table, dynamicFilter, constraint); + } + ConnectorSplitSource fallbackSplitSource = new FixedSplitSource(new JdbcSplit(Optional.empty())); + if (!isUnloadEnabled(session)) { + return fallbackSplitSource; + } + JdbcTableHandle jdbcTable = (JdbcTableHandle) table; + JdbcTableHandle jdbcTableHandle = dynamicFilteringEnabled(session) ? jdbcTable.intersectedWithConstraint(dynamicFilter.getCurrentPredicate()) : jdbcTable; + List columns = jdbcTableHandle.getColumns() + .orElseGet(() -> jdbcClient.getColumns( + session, + jdbcTableHandle.getRequiredNamedRelation().getSchemaTableName(), + jdbcTableHandle.getRequiredNamedRelation().getRemoteTableName())); + + if (!isUnloadSupported(jdbcTable, columns)) { + log.debug("Unsupported query shape detected. Falling back to using JDBC"); + return fallbackSplitSource; + } + return new RedshiftUnloadSplitSource( + executor, + session, + jdbcClient, + jdbcTableHandle, + columns, + queryBuilder, + queryModifier, + unloadLocation.orElseThrow(), + unloadAuthorization, + fileSystemFactory.create(session)); + } + + private static boolean isUnloadSupported(JdbcTableHandle table, List columns) + { + // Nothing to unload as there are no columns to be fetched from Redshift + if (table.getColumns().isPresent() && table.getColumns().get().isEmpty()) { + return false; + } + if (containsUnsupportedType(columns)) { + return false; + } + // Unload command doesn't support limit clause. However, Trino can implement the workaround of wrapping limit query as inner query. See https://github.com/trinodb/trino/issues/24480 + if (table.getLimit().isPresent()) { + return false; + } + if (containsFilterConditionOnDecimalTypeColumn(table)) { + return false; + } + return true; + } + + // Unsupported unload command data types when using Parquet output file format + private static boolean containsUnsupportedType(List columns) + { + // ERROR: UNLOAD varbyte column "col_0" is only supported for TEXT/CSV. + // ERROR: UNLOAD time without time zone column "value" is only supported for TEXT/CSV. + return columns.stream().anyMatch(column -> column.getColumnType() instanceof TimeType || column.getColumnType() instanceof VarbinaryType); + } + + // Redshift driver generates incorrect cast precision in select query for filter condition on decimal columns. See https://github.com/aws/amazon-redshift-jdbc-driver/issues/129 + private static boolean containsFilterConditionOnDecimalTypeColumn(JdbcTableHandle table) + { + if (table.getConstraint().getDomains() + .map(domains -> domains.keySet().stream().anyMatch(column -> ((JdbcColumnHandle) column).getColumnType() instanceof DecimalType)) + .orElse(false)) { + return true; + } + return table.getConstraintExpressions().stream() + .flatMap(expression -> expression.parameters().stream()) + .anyMatch(parameter -> parameter.getType() instanceof DecimalType); + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnector.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadConnector.java similarity index 93% rename from plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnector.java rename to plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadConnector.java index e1f98eed5703..c3a17e9642e5 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnector.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadConnector.java @@ -16,7 +16,10 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import io.airlift.bootstrap.LifeCycleManager; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.filesystem.s3.FileSystemS3; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorMetadata; +import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.jdbc.JdbcTransactionManager; import io.trino.plugin.jdbc.TablePropertiesProvider; @@ -44,7 +47,7 @@ import static io.trino.spi.connector.ConnectorCapabilities.NOT_NULL_COLUMN_CONSTRAINT; import static java.util.Objects.requireNonNull; -public class RedshiftConnector +public class RedshiftUnloadConnector implements Connector { private final LifeCycleManager lifeCycleManager; @@ -59,7 +62,7 @@ public class RedshiftConnector private final RedshiftPageSourceProvider pageSourceProvider; @Inject - public RedshiftConnector( + public RedshiftUnloadConnector( LifeCycleManager lifeCycleManager, ConnectorSplitManager jdbcSplitManager, ConnectorRecordSetProvider jdbcRecordSetProvider, @@ -69,7 +72,9 @@ public RedshiftConnector( Set connectorTableFunctions, Set sessionProperties, Set tableProperties, - JdbcTransactionManager transactionManager) + JdbcTransactionManager transactionManager, + @FileSystemS3 TrinoFileSystemFactory fileSystemFactory, + FileFormatDataSourceStats fileFormatDataSourceStats) { this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); this.jdbcSplitManager = requireNonNull(jdbcSplitManager, "jdbcSplitManager is null"); @@ -84,7 +89,7 @@ public RedshiftConnector( .flatMap(tablePropertiesProvider -> tablePropertiesProvider.getTableProperties().stream()) .collect(toImmutableList()); this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); - this.pageSourceProvider = new RedshiftPageSourceProvider(jdbcRecordSetProvider); + this.pageSourceProvider = new RedshiftPageSourceProvider(jdbcRecordSetProvider, fileSystemFactory, fileFormatDataSourceStats); } @Override diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadJdbcQueryEventListener.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadJdbcQueryEventListener.java new file mode 100644 index 000000000000..5b3055a3db17 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadJdbcQueryEventListener.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.inject.Inject; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.filesystem.s3.FileSystemS3; +import io.trino.plugin.jdbc.JdbcQueryEventListener; +import io.trino.spi.connector.ConnectorSession; + +import java.io.IOException; +import java.io.UncheckedIOException; + +import static io.trino.plugin.redshift.RedshiftSessionProperties.isUnloadEnabled; +import static java.util.Objects.requireNonNull; + +public class RedshiftUnloadJdbcQueryEventListener + implements JdbcQueryEventListener +{ + private final TrinoFileSystemFactory fileSystemFactory; + private final String unloadLocation; + + @Inject + public RedshiftUnloadJdbcQueryEventListener(@FileSystemS3 TrinoFileSystemFactory fileSystemFactory, RedshiftConfig redshiftConfig) + { + this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); + this.unloadLocation = redshiftConfig.getUnloadLocation().orElseThrow(); + } + + @Override + public void beginQuery(ConnectorSession session) {} + + @Override + public void cleanupQuery(ConnectorSession session) + { + if (isUnloadEnabled(session)) { + try { + fileSystemFactory.create(session).deleteDirectory(Location.of(unloadLocation + "/" + session.getQueryId())); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplit.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplit.java new file mode 100644 index 000000000000..049c6fc69b58 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplit.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.common.collect.ImmutableMap; +import io.trino.spi.connector.ConnectorSplit; + +import java.util.Map; + +import static io.airlift.slice.SizeOf.estimatedSizeOf; +import static io.airlift.slice.SizeOf.instanceSize; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; + +public record RedshiftUnloadSplit(String path, long length) + implements ConnectorSplit +{ + private static final int INSTANCE_SIZE = instanceSize(RedshiftUnloadSplit.class); + + public RedshiftUnloadSplit + { + requireNonNull(path, "path is null"); + } + + @Override + public Map getSplitInfo() + { + return ImmutableMap.of("path", path); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + estimatedSizeOf(path) + sizeOf(length); + } +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplitSource.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplitSource.java new file mode 100644 index 000000000000..5acb5dd6bd21 --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftUnloadSplitSource.java @@ -0,0 +1,196 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.amazon.redshift.jdbc.RedshiftPreparedStatement; +import com.amazon.redshift.util.RedshiftException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.ObjectMapperProvider; +import io.airlift.log.Logger; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoInputFile; +import io.trino.filesystem.TrinoInputStream; +import io.trino.plugin.jdbc.JdbcClient; +import io.trino.plugin.jdbc.JdbcColumnHandle; +import io.trino.plugin.jdbc.JdbcTableHandle; +import io.trino.plugin.jdbc.PreparedQuery; +import io.trino.plugin.jdbc.QueryBuilder; +import io.trino.plugin.jdbc.logging.RemoteQueryModifier; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorSplitSource; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.units.Duration.nanosSince; +import static io.trino.plugin.redshift.RedshiftErrorCode.REDSHIFT_FILESYSTEM_ERROR; +import static io.trino.plugin.redshift.RedshiftErrorCode.REDSHIFT_S3_CROSS_REGION_UNSUPPORTED; +import static java.util.Objects.requireNonNull; + +public class RedshiftUnloadSplitSource + implements ConnectorSplitSource +{ + private static final Logger log = Logger.get(RedshiftUnloadSplitSource.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapperProvider().get(); + + private final JdbcClient jdbcClient; + private final QueryBuilder queryBuilder; + private final RemoteQueryModifier queryModifier; + private final Optional unloadAuthorization; + private final String unloadOutputPath; + private final TrinoFileSystem fileSystem; + private final CompletableFuture resultSetFuture; + + private boolean finished; + + public RedshiftUnloadSplitSource( + ExecutorService executor, + ConnectorSession session, + JdbcClient jdbcClient, + JdbcTableHandle jdbcTableHandle, + List columns, + QueryBuilder queryBuilder, + RemoteQueryModifier queryModifier, + String unloadLocation, + Optional unloadAuthorization, + TrinoFileSystem fileSystem) + { + requireNonNull(executor, "executor is null"); + requireNonNull(session, "session is null"); + this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null"); + requireNonNull(jdbcTableHandle, "jdbcTableHandle is null"); + requireNonNull(columns, "columns is null"); + this.queryBuilder = requireNonNull(queryBuilder, "queryBuilder is null"); + this.queryModifier = requireNonNull(queryModifier, "queryModifier is null"); + this.unloadAuthorization = requireNonNull(unloadAuthorization, "unloadAuthorization is null"); + this.fileSystem = requireNonNull(fileSystem, "fileSystem is null"); + + String queryFragmentId = session.getQueryId() + "/" + UUID.randomUUID(); + this.unloadOutputPath = unloadLocation + "/" + queryFragmentId + "/"; + + resultSetFuture = CompletableFuture.runAsync(() -> { + try (Connection connection = jdbcClient.getConnection(session)) { + String redshiftSelectSql = buildRedshiftSelectSql(session, connection, jdbcTableHandle, columns); + try (PreparedStatement statement = buildRedshiftUnloadSql(session, connection, columns, redshiftSelectSql, unloadOutputPath)) { + // Exclusively set readOnly to false to avoid query failing with "ERROR: transaction is read-only". + connection.setReadOnly(false); + log.debug("Executing: %s", statement); + long start = System.nanoTime(); + statement.execute(); // Return value of `statement.execute()` is not useful to determine whether UNLOAD command produced any result as it always return false. + log.info("Redshift UNLOAD command for %s query took %s", queryFragmentId, nanosSince(start)); + } + } + catch (SQLException e) { + if (e instanceof RedshiftException && e.getMessage() != null && e.getMessage().contains("The S3 bucket addressed by the query is in a different region from this cluster")) { + throw new TrinoException(REDSHIFT_S3_CROSS_REGION_UNSUPPORTED, "Redshift cluster and S3 bucket in different regions is not supported", e); + } + throw new RuntimeException(e); + } + }, executor); + } + + @Override + public CompletableFuture getNextBatch(int maxSize) + { + return resultSetFuture + .thenApply(_ -> { + ConnectorSplitBatch connectorSplitBatch = new ConnectorSplitBatch(readUnloadedFilePaths().stream() + .map(fileInfo -> (ConnectorSplit) new RedshiftUnloadSplit(fileInfo.path, fileInfo.size)) + .collect(toImmutableList()), true); + finished = true; + return connectorSplitBatch; + }); + } + + @Override + public void close() + { + resultSetFuture.cancel(true); + } + + @Override + public boolean isFinished() + { + return finished; + } + + private String buildRedshiftSelectSql(ConnectorSession session, Connection connection, JdbcTableHandle table, List columns) + throws SQLException + { + PreparedQuery preparedQuery = jdbcClient.prepareQuery(session, table, Optional.empty(), columns, ImmutableMap.of()); + String selectQuerySql; + try (PreparedStatement openTelemetryPreparedStatement = queryBuilder.prepareStatement(jdbcClient, session, connection, preparedQuery, Optional.of(columns.size()))) { + RedshiftPreparedStatement redshiftPreparedStatement = openTelemetryPreparedStatement.unwrap(RedshiftPreparedStatement.class); + selectQuerySql = redshiftPreparedStatement.toString(); + } + return queryModifier.apply(session, selectQuerySql); + } + + private PreparedStatement buildRedshiftUnloadSql(ConnectorSession session, Connection connection, List columns, String redshiftSelectSql, String unloadOutputPath) + throws SQLException + { + String unloadSql = "UNLOAD ('%s') TO '%s' IAM_ROLE %s FORMAT PARQUET MAXFILESIZE 64MB MANIFEST VERBOSE".formatted( + escapeUnloadIllegalCharacters(redshiftSelectSql), + unloadOutputPath, + unloadAuthorization.map("'%s'"::formatted).orElse("DEFAULT")); + return queryBuilder.prepareStatement(jdbcClient, session, connection, new PreparedQuery(unloadSql, List.of()), Optional.of(columns.size())); + } + + private List readUnloadedFilePaths() + { + Location manifestLocation = Location.of(unloadOutputPath + "manifest"); + TrinoInputFile inputFile = fileSystem.newInputFile(manifestLocation); + JsonNode outputFileEntries; + try (TrinoInputStream inputStream = inputFile.newStream()) { + byte[] manifestContent = inputStream.readAllBytes(); + outputFileEntries = OBJECT_MAPPER.readTree(manifestContent).path("entries"); + } + // manifest is not generated if unload query doesn't produce any results. + // Rely on the catching `FileNotFoundException` as opposed to calling `TrinoInputFile#exists` for determining absence of manifest file as `TrinoInputFile#exists` adds additional call to S3. + catch (FileNotFoundException e) { + return ImmutableList.of(); + } + catch (IOException e) { + throw new TrinoException(REDSHIFT_FILESYSTEM_ERROR, e); + } + ImmutableList.Builder unloadedFilePaths = ImmutableList.builder(); + outputFileEntries.elements() + .forEachRemaining(fileInfo -> unloadedFilePaths.add(new FileInfo(fileInfo.get("url").asText(), fileInfo.get("meta").get("content_length").longValue()))); + return unloadedFilePaths.build(); + } + + private static String escapeUnloadIllegalCharacters(String value) + { + return value + .replace("'", "''") // escape single quotes with single quotes + .replace("\\b", "\\\\b"); // escape backspace with backslash + } + + private record FileInfo(String path, long size) {} +} diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/TrinoParquetDataSource.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/TrinoParquetDataSource.java new file mode 100644 index 000000000000..57d5a1f7fc9d --- /dev/null +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/TrinoParquetDataSource.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import io.airlift.slice.Slice; +import io.trino.filesystem.TrinoInput; +import io.trino.filesystem.TrinoInputFile; +import io.trino.parquet.AbstractParquetDataSource; +import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.ParquetReaderOptions; +import io.trino.plugin.base.metrics.FileFormatDataSourceStats; + +import java.io.IOException; + +import static java.util.Objects.requireNonNull; + +// Copied as-is from io.trino.plugin.hive.parquet.TrinoParquetDataSource +public class TrinoParquetDataSource + extends AbstractParquetDataSource +{ + private final FileFormatDataSourceStats stats; + private final TrinoInput input; + + public TrinoParquetDataSource(TrinoInputFile file, ParquetReaderOptions options, FileFormatDataSourceStats stats) + throws IOException + { + super(new ParquetDataSourceId(file.location().toString()), file.length(), options); + this.stats = requireNonNull(stats, "stats is null"); + this.input = file.newInput(); + } + + @Override + public void close() + throws IOException + { + input.close(); + } + + @Override + protected Slice readTailInternal(int length) + throws IOException + { + long readStart = System.nanoTime(); + Slice tail = input.readTail(length); + stats.readDataBytesPerSecond(tail.length(), System.nanoTime() - readStart); + return tail; + } + + @Override + protected void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength) + throws IOException + { + long readStart = System.nanoTime(); + input.readFully(position, buffer, bufferOffset, bufferLength); + stats.readDataBytesPerSecond(bufferLength, System.nanoTime() - readStart); + } +} diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/RedshiftQueryRunner.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/RedshiftQueryRunner.java index d1d806f0a2eb..c74274090d2d 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/RedshiftQueryRunner.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/RedshiftQueryRunner.java @@ -56,7 +56,7 @@ private RedshiftQueryRunner() {} private static final Logger log = Logger.get(RedshiftQueryRunner.class); private static final String S3_TPCH_TABLES_ROOT = requiredNonEmptySystemProperty("test.redshift.s3.tpch.tables.root"); - private static final String IAM_ROLE = requiredNonEmptySystemProperty("test.redshift.iam.role"); + public static final String IAM_ROLE = requiredNonEmptySystemProperty("test.redshift.iam.role"); private static final String TEST_CATALOG = "redshift"; private static final String CONNECTOR_NAME = "redshift"; diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConfig.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConfig.java index 6a507d29d87b..2c7498cb19d4 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConfig.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConfig.java @@ -28,7 +28,9 @@ public class TestRedshiftConfig public void testDefaults() { assertRecordedDefaults(recordDefaults(RedshiftConfig.class) - .setFetchSize(null)); + .setFetchSize(null) + .setUnloadLocation(null) + .setUnloadIamRole(null)); } @Test @@ -36,10 +38,14 @@ public void testExplicitPropertyMappings() { Map properties = ImmutableMap.builder() .put("redshift.fetch-size", "2000") + .put("redshift.unload-location", "s3://bucket") + .put("redshift.unload-iam-role", "arn:aws:iam::123456789000:role/redshift_iam_role") .buildOrThrow(); RedshiftConfig expected = new RedshiftConfig() - .setFetchSize(2000); + .setFetchSize(2000) + .setUnloadLocation("s3://bucket") + .setUnloadIamRole("arn:aws:iam::123456789000:role/redshift_iam_role"); assertFullMapping(properties, expected); } diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftPlugin.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftPlugin.java index 9d4fdaad3e80..596ec7ffb1a1 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftPlugin.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftPlugin.java @@ -35,4 +35,22 @@ public void testCreateConnector() "bootstrap.quiet", "true"), new TestingConnectorContext()).shutdown(); } + + @Test + public void testCreateUnloadConnector() + { + Plugin plugin = new RedshiftPlugin(); + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + factory.create( + "test", + ImmutableMap.of( + "connection-url", "jdbc:redshift:test", + "redshift.unload-location", "s3://bucket/path", + "redshift.unload-iam-role", "role", + "s3.aws-access-key", "access-key", + "s3.aws-secret-key", "secret-key", + "s3.region", "region", + "bootstrap.quiet", "true"), + new TestingConnectorContext()).shutdown(); + } } diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java new file mode 100644 index 000000000000..d526361e98ce --- /dev/null +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java @@ -0,0 +1,232 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.common.collect.ImmutableList; +import io.trino.Session; +import io.trino.operator.OperatorInfo; +import io.trino.operator.SplitOperatorInfo; +import io.trino.testing.AbstractTestQueries; +import io.trino.testing.QueryRunner; +import io.trino.testing.sql.SqlExecutor; +import io.trino.testing.sql.TestTable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import java.net.URI; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.MoreCollectors.onlyElement; +import static io.trino.plugin.redshift.RedshiftQueryRunner.IAM_ROLE; +import static io.trino.plugin.redshift.TestingRedshiftServer.TEST_SCHEMA; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.testing.TestingProperties.requiredNonEmptySystemProperty; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static io.trino.tpch.TpchTable.NATION; +import static java.util.Locale.ENGLISH; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; + +@TestInstance(PER_CLASS) +@Execution(CONCURRENT) +final class TestRedshiftUnload + extends AbstractTestQueries +{ + private static final String S3_UNLOAD_ROOT = requiredNonEmptySystemProperty("test.redshift.s3.unload.root"); + private static final String AWS_REGION = requiredNonEmptySystemProperty("test.redshift.aws.region"); + private static final String AWS_ACCESS_KEY = requiredNonEmptySystemProperty("test.redshift.aws.access-key"); + private static final String AWS_SECRET_KEY = requiredNonEmptySystemProperty("test.redshift.aws.secret-key"); + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return RedshiftQueryRunner.builder() + .setConnectorProperties( + Map.of( + "redshift.unload-location", S3_UNLOAD_ROOT, + "redshift.unload-iam-role", IAM_ROLE, + "s3.region", AWS_REGION, + "s3.aws-access-key", AWS_ACCESS_KEY, + "s3.aws-secret-key", AWS_SECRET_KEY)) + .setInitialTables(List.of(NATION)) + .build(); + } + + @Test + void testUnloadEnabled() + { + assertQuery( + "SHOW SESSION LIKE 'redshift.unload_enabled'", + "VALUES ('redshift.unload_enabled', 'true', 'true', 'boolean', 'Use UNLOAD for reading query results')"); + } + + @Test + void testUnload() + { + assertQueryStats( + getSession(), + """ + SELECT nationkey, name FROM nation WHERE regionkey = 0 + UNION + SELECT nationkey, name FROM nation WHERE regionkey = 1 + """, + queryStats -> { + List> splitInfos = + queryStats.getOperatorSummaries() + .stream() + .filter(summary -> summary.getOperatorType().startsWith("TableScanOperator")) + .map(operatorStat -> ((SplitOperatorInfo) operatorStat.getInfo()).getSplitInfo()) + .collect(toImmutableList()); + splitInfos.forEach(splitInfo -> assertThat(splitInfo.get("path")).matches("%s/.*/.*/.*.parquet.*".formatted(S3_UNLOAD_ROOT))); + String unloadedFilePath = splitInfos.getFirst().get("path"); + assertThat(unloadedFilePath).matches("%s/.*/.*/.*.parquet.*".formatted(S3_UNLOAD_ROOT)); + try (S3Client s3 = S3Client.builder() + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(AWS_ACCESS_KEY, AWS_SECRET_KEY))) + .region(Region.of(AWS_REGION)) + .build()) { + URI s3Path = URI.create(unloadedFilePath.substring(0, unloadedFilePath.lastIndexOf("/", unloadedFilePath.lastIndexOf("/") - 1))); + assertThat(s3.listObjectsV2(request -> request.bucket(s3Path.getHost()).prefix(s3Path.getPath().substring(1))).contents()).isEmpty(); + } + }, + results -> assertThat(results.getRowCount()).isEqualTo(10)); + } + + @Test + void testUnloadDisabled() + { + Session unloadDisabledSession = testSessionBuilder(getSession()) + .setCatalogSessionProperty("redshift", "unload_enabled", "false") + .build(); + assertQueryStats( + unloadDisabledSession, + "SELECT nationkey, name FROM nation WHERE regionkey = 0", + queryStats -> { + OperatorInfo operatorInfo = queryStats.getOperatorSummaries() + .stream() + .filter(summary -> summary.getOperatorType().startsWith("TableScanOperator")) + .collect(onlyElement()) + .getInfo(); + assertThat(operatorInfo).isNull(); + }, + results -> assertThat(results.getRowCount()).isEqualTo(5)); + } + + @Test + void testUnloadProduceEmptyResults() + { + assertQueryStats( + getSession(), + "SELECT * FROM nation WHERE name = 'INVALID'", + queryStats -> { + OperatorInfo operatorInfo = queryStats.getOperatorSummaries() + .stream() + .filter(summary -> summary.getOperatorType().startsWith("TableScanOperator")) + .collect(onlyElement()) + .getInfo(); + assertThat(operatorInfo).isNull(); + }, + results -> assertThat(results.getRowCount()).isEqualTo(0)); + } + + @Test + void testUnloadFallbackToJdbc() + { + // Fallback to JDBC as limit clause is not supported by UNLOAD + assertQueryStats( + getSession(), + "SELECT nationkey, name FROM nation WHERE regionkey = 0 LIMIT 1", + queryStats -> { + OperatorInfo operatorInfo = queryStats.getOperatorSummaries() + .stream() + .filter(summary -> summary.getOperatorType().startsWith("TableScanOperator")) + .collect(onlyElement()) + .getInfo(); + assertThat(operatorInfo).isNull(); + }, + results -> assertThat(results.getRowCount()).isEqualTo(1)); + } + + @Test + void testColumnName() + { + List columnNames = ImmutableList.builder() + .add("lowercase") + .add("UPPERCASE") + .add("MixedCase") + .add("an_underscore") + .add("a-hyphen-minus") // ASCII '-' is HYPHEN-MINUS in Unicode + .add("a space") + .add("atrailingspace ") + .add(" aleadingspace") + .add("a.dot") + .add("a,comma") + .add("a:colon") + .add("a;semicolon") + .add("an@at") + // .add("a\"quote") // TODO escape "(double quotes) in UNLOAD manifest(manifest json contains unescaped double quotes in field value `"name": "a"quote"`) + .add("an'apostrophe") + .add("a`backtick`") + .add("a/slash`") + .add("a\\backslash`") + .add("adigit0") + .add("0startwithdigit") + .add("カラム") + .build(); + for (String columnName : columnNames) { + testColumnName(columnName, requiresDelimiting(columnName)); + } + } + + private void testColumnName(String columnName, boolean delimited) + { + String nameInSql = toColumnNameInSql(columnName, delimited); + String tableNamePrefix = "tcn_" + nameInSql.toLowerCase(ENGLISH).replaceAll("[^a-z0-9]", "") + randomNameSuffix(); + + try (TestTable table = new TestTable( + onRemoteDatabase(), + TEST_SCHEMA + "." + tableNamePrefix, + "(%s varchar(50))".formatted(nameInSql), + ImmutableList.of("'abc'"))) { + assertQuery("SELECT " + nameInSql + " FROM " + table.getName(), "VALUES ('abc')"); + } + } + + private static String toColumnNameInSql(String columnName, boolean delimited) + { + String nameInSql = columnName; + if (delimited) { + nameInSql = "\"" + columnName.replace("\"", "\"\"") + "\""; + } + return nameInSql; + } + + private static boolean requiresDelimiting(String identifierName) + { + return !identifierName.matches("[a-zA-Z][a-zA-Z0-9_]*"); + } + + private static SqlExecutor onRemoteDatabase() + { + return TestingRedshiftServer::executeInRedshift; + } +} diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnloadTypeMapping.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnloadTypeMapping.java new file mode 100644 index 000000000000..3a9275550a97 --- /dev/null +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnloadTypeMapping.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.redshift; + +import com.google.common.collect.ImmutableMap; +import io.trino.testing.QueryRunner; + +import java.util.Map; + +import static io.trino.plugin.redshift.RedshiftQueryRunner.IAM_ROLE; +import static io.trino.plugin.redshift.TestingRedshiftServer.JDBC_PASSWORD; +import static io.trino.plugin.redshift.TestingRedshiftServer.JDBC_URL; +import static io.trino.plugin.redshift.TestingRedshiftServer.JDBC_USER; +import static io.trino.testing.TestingProperties.requiredNonEmptySystemProperty; + +public class TestRedshiftUnloadTypeMapping + extends TestRedshiftTypeMapping +{ + private static final String S3_UNLOAD_ROOT = requiredNonEmptySystemProperty("test.redshift.s3.unload.root"); + private static final String AWS_REGION = requiredNonEmptySystemProperty("test.redshift.aws.region"); + private static final String AWS_ACCESS_KEY = requiredNonEmptySystemProperty("test.redshift.aws.access-key"); + private static final String AWS_SECRET_KEY = requiredNonEmptySystemProperty("test.redshift.aws.secret-key"); + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + Map properties = ImmutableMap.builder() + .put("redshift.unload-location", S3_UNLOAD_ROOT) + .put("redshift.unload-iam-role", IAM_ROLE) + .put("s3.region", AWS_REGION) + .put("s3.aws-access-key", AWS_ACCESS_KEY) + .put("s3.aws-secret-key", AWS_SECRET_KEY) + .put("connection-url", JDBC_URL) + .put("connection-user", JDBC_USER) + .put("connection-password", JDBC_PASSWORD) + .buildOrThrow(); + + return RedshiftQueryRunner.builder() + .setConnectorProperties(properties) + .build(); + } +} From 41d08f674e870a8c907d2bf29f67abf68c98859f Mon Sep 17 00:00:00 2001 From: Mayank Vadariya Date: Tue, 31 Dec 2024 13:14:18 -0500 Subject: [PATCH 108/158] Copy all TPCH tables during initialization in TestRedshiftUnload --- .../test/java/io/trino/plugin/redshift/TestRedshiftUnload.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java index d526361e98ce..433830c495d0 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java @@ -40,7 +40,6 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.TestingProperties.requiredNonEmptySystemProperty; import static io.trino.testing.TestingSession.testSessionBuilder; -import static io.trino.tpch.TpchTable.NATION; import static java.util.Locale.ENGLISH; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -68,7 +67,7 @@ protected QueryRunner createQueryRunner() "s3.region", AWS_REGION, "s3.aws-access-key", AWS_ACCESS_KEY, "s3.aws-secret-key", AWS_SECRET_KEY)) - .setInitialTables(List.of(NATION)) + .setInitialTables(REQUIRED_TPCH_TABLES) .build(); } From 5033b03c6ffd34bc68474de817b753107f5fe777 Mon Sep 17 00:00:00 2001 From: Raunaq Morarka Date: Tue, 31 Dec 2024 23:31:22 +0530 Subject: [PATCH 109/158] Add physicalInputTimeMillis to io.trino.jdbc.QueryStats --- .../java/io/trino/client/StatementStats.java | 18 ++++++++++++++++++ .../test/java/io/trino/client/TestRetry.java | 2 +- .../main/java/io/trino/jdbc/QueryStats.java | 9 +++++++++ .../io/trino/jdbc/TestAsyncResultIterator.java | 1 + .../io/trino/jdbc/TestProgressMonitor.java | 2 +- .../io/trino/execution/QueryStateMachine.java | 1 + .../java/io/trino/server/BasicQueryStats.java | 11 +++++++++++ .../io/trino/server/protocol/ProtocolUtil.java | 1 + .../execution/MockManagedQueryExecution.java | 1 + .../protocol/TestQueryDataSerialization.java | 1 + .../TestQueryResultsSerialization.java | 2 ++ .../memory/TestClusterMemoryLeakDetector.java | 1 + 12 files changed, 48 insertions(+), 2 deletions(-) diff --git a/client/trino-client/src/main/java/io/trino/client/StatementStats.java b/client/trino-client/src/main/java/io/trino/client/StatementStats.java index 415b07d2c03a..d4a68eb62788 100644 --- a/client/trino-client/src/main/java/io/trino/client/StatementStats.java +++ b/client/trino-client/src/main/java/io/trino/client/StatementStats.java @@ -43,6 +43,7 @@ public class StatementStats private final long queuedTimeMillis; private final long elapsedTimeMillis; private final long finishingTimeMillis; + private final long physicalInputTimeMillis; private final long processedRows; private final long processedBytes; private final long physicalInputBytes; @@ -71,6 +72,7 @@ public StatementStats( @JsonProperty("queuedTimeMillis") long queuedTimeMillis, @JsonProperty("elapsedTimeMillis") long elapsedTimeMillis, @JsonProperty("finishingTimeMillis") long finishingTimeMillis, + @JsonProperty("physicalInputTimeMillis") long physicalInputTimeMillis, @JsonProperty("processedRows") long processedRows, @JsonProperty("processedBytes") long processedBytes, @JsonProperty("physicalInputBytes") long physicalInputBytes, @@ -97,6 +99,7 @@ public StatementStats( this.queuedTimeMillis = queuedTimeMillis; this.elapsedTimeMillis = elapsedTimeMillis; this.finishingTimeMillis = finishingTimeMillis; + this.physicalInputTimeMillis = physicalInputTimeMillis; this.processedRows = processedRows; this.processedBytes = processedBytes; this.physicalInputBytes = physicalInputBytes; @@ -209,6 +212,12 @@ public long getFinishingTimeMillis() return finishingTimeMillis; } + @JsonProperty + public long getPhysicalInputTimeMillis() + { + return physicalInputTimeMillis; + } + @JsonProperty public long getProcessedRows() { @@ -279,6 +288,7 @@ public String toString() .add("queuedTimeMillis", queuedTimeMillis) .add("elapsedTimeMillis", elapsedTimeMillis) .add("finishingTimeMillis", finishingTimeMillis) + .add("physicalInputTimeMillis", physicalInputTimeMillis) .add("processedRows", processedRows) .add("processedBytes", processedBytes) .add("physicalInputBytes", physicalInputBytes) @@ -314,6 +324,7 @@ public static class Builder private long queuedTimeMillis; private long elapsedTimeMillis; private long finishingTimeMillis; + private long physicalInputTimeMillis; private long processedRows; private long processedBytes; private long physicalInputBytes; @@ -427,6 +438,12 @@ public Builder setFinishingTimeMillis(long finishingTimeMillis) return this; } + public Builder setPhysicalInputTimeMillis(long physicalInputTimeMillis) + { + this.physicalInputTimeMillis = physicalInputTimeMillis; + return this; + } + public Builder setProcessedRows(long processedRows) { this.processedRows = processedRows; @@ -495,6 +512,7 @@ public StatementStats build() queuedTimeMillis, elapsedTimeMillis, finishingTimeMillis, + physicalInputTimeMillis, processedRows, processedBytes, physicalInputBytes, diff --git a/client/trino-client/src/test/java/io/trino/client/TestRetry.java b/client/trino-client/src/test/java/io/trino/client/TestRetry.java index d000d2da97b5..3ea7a940e665 100644 --- a/client/trino-client/src/test/java/io/trino/client/TestRetry.java +++ b/client/trino-client/src/test/java/io/trino/client/TestRetry.java @@ -143,7 +143,7 @@ private String newQueryResults(String state) TypedQueryData.of(IntStream.range(0, numRecords) .mapToObj(index -> Stream.of((Object) index, "a").collect(toList())) .collect(toList())), - new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), + new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), null, ImmutableList.of(), null, diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java index cac77f441ca0..9acea4fce352 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java @@ -39,6 +39,7 @@ public final class QueryStats private final long queuedTimeMillis; private final long elapsedTimeMillis; private final long finishingTimeMillis; + private final long physicalInputTimeMillis; private final long processedRows; private final long processedBytes; private final long peakMemoryBytes; @@ -65,6 +66,7 @@ public QueryStats( long queuedTimeMillis, long elapsedTimeMillis, long finishingTimeMillis, + long physicalInputTimeMillis, long processedRows, long processedBytes, long peakMemoryBytes, @@ -90,6 +92,7 @@ public QueryStats( this.queuedTimeMillis = queuedTimeMillis; this.elapsedTimeMillis = elapsedTimeMillis; this.finishingTimeMillis = finishingTimeMillis; + this.physicalInputTimeMillis = physicalInputTimeMillis; this.processedRows = processedRows; this.processedBytes = processedBytes; this.peakMemoryBytes = peakMemoryBytes; @@ -119,6 +122,7 @@ static QueryStats create(String queryId, StatementStats stats) stats.getQueuedTimeMillis(), stats.getElapsedTimeMillis(), stats.getFinishingTimeMillis(), + stats.getPhysicalInputTimeMillis(), stats.getProcessedRows(), stats.getProcessedBytes(), stats.getPeakMemoryBytes(), @@ -213,6 +217,11 @@ public long getFinishingTimeMillis() return finishingTimeMillis; } + public long getPhysicalInputTimeMillis() + { + return physicalInputTimeMillis; + } + public long getProcessedRows() { return processedRows; diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java index 7aeaddedebd4..963c782f934a 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java @@ -337,6 +337,7 @@ public StatementStats getStats() 100, 100, 100, + 100, StageStats.builder() .setStageId("id") .setDone(false) diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java index 09d92c170579..76d7ff7e5955 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java @@ -99,7 +99,7 @@ private String newQueryResults(Integer partialCancelId, Integer nextUriId, List< nextUriId == null ? null : server.url(format("/v1/statement/%s/%s", queryId, nextUriId)).uri(), responseColumns, TypedQueryData.of(data), - new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), + new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null), null, ImmutableList.of(), null, diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java index bd2aabb64d62..4b94bbdd1fa7 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java @@ -563,6 +563,7 @@ private BasicQueryStats createBasicQueryStats(BasicStageStats stageStats) stageStats.getTotalScheduledTime(), stageStats.getFailedScheduledTime(), queryStateTimer.getFinishingTime(), + stageStats.getPhysicalInputReadTime(), stageStats.isFullyBlocked(), stageStats.getBlockedReasons(), diff --git a/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java b/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java index cfd3378db7b3..b4b485ed5126 100644 --- a/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java +++ b/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java @@ -71,6 +71,7 @@ public class BasicQueryStats private final Duration totalScheduledTime; private final Duration failedScheduledTime; private final Duration finishingTime; + private final Duration physicalInputReadTime; private final boolean fullyBlocked; private final Set blockedReasons; @@ -110,6 +111,7 @@ public BasicQueryStats( @JsonProperty("totalScheduledTime") Duration totalScheduledTime, @JsonProperty("failedScheduledTime") Duration failedScheduledTime, @JsonProperty("finishingTime") Duration finishingTime, + @JsonProperty("physicalInputReadTime") Duration physicalInputReadTime, @JsonProperty("fullyBlocked") boolean fullyBlocked, @JsonProperty("blockedReasons") Set blockedReasons, @JsonProperty("progressPercentage") OptionalDouble progressPercentage, @@ -156,6 +158,7 @@ public BasicQueryStats( this.totalScheduledTime = totalScheduledTime; this.failedScheduledTime = failedScheduledTime; this.finishingTime = finishingTime; + this.physicalInputReadTime = physicalInputReadTime; this.fullyBlocked = fullyBlocked; this.blockedReasons = ImmutableSet.copyOf(requireNonNull(blockedReasons, "blockedReasons is null")); @@ -196,6 +199,7 @@ public BasicQueryStats(QueryStats queryStats) queryStats.getTotalScheduledTime(), queryStats.getFailedScheduledTime(), queryStats.getFinishingTime(), + queryStats.getPhysicalInputReadTime(), queryStats.isFullyBlocked(), queryStats.getBlockedReasons(), queryStats.getProgressPercentage(), @@ -236,6 +240,7 @@ public static BasicQueryStats immediateFailureQueryStats() new Duration(0, MILLISECONDS), new Duration(0, MILLISECONDS), new Duration(0, MILLISECONDS), + new Duration(0, MILLISECONDS), false, ImmutableSet.of(), OptionalDouble.empty(), @@ -422,6 +427,12 @@ public Duration getFinishingTime() return finishingTime; } + @JsonProperty + public Duration getPhysicalInputReadTime() + { + return physicalInputReadTime; + } + @JsonProperty public boolean isFullyBlocked() { diff --git a/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java b/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java index c8997ebe0125..7894c6fb68b8 100644 --- a/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java +++ b/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java @@ -189,6 +189,7 @@ public static StatementStats toStatementStats(ResultQueryInfo queryInfo) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) .setFinishingTimeMillis(queryStats.getFinishingTime().toMillis()) + .setPhysicalInputTimeMillis(queryStats.getPhysicalInputReadTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPhysicalInputBytes(queryStats.getPhysicalInputDataSize().toBytes()) diff --git a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java index 787e3e00dab7..4029f63085bc 100644 --- a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java +++ b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java @@ -150,6 +150,7 @@ public BasicQueryInfo getBasicQueryInfo() new Duration(24, NANOSECONDS), new Duration(25, NANOSECONDS), new Duration(26, NANOSECONDS), + new Duration(27, NANOSECONDS), false, ImmutableSet.of(), OptionalDouble.empty(), diff --git a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java index 40b8f7797b41..980c9844652a 100644 --- a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java +++ b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryDataSerialization.java @@ -252,6 +252,7 @@ private String queryResultsJson(String expectedDataField) "queuedTimeMillis": 0, "elapsedTimeMillis": 0, "finishingTimeMillis": 0, + "physicalInputTimeMillis": 0, "processedRows": 0, "processedBytes": 0, "physicalInputBytes": 0, diff --git a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java index d1ae9c5cabfa..badc453a165e 100644 --- a/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java +++ b/core/trino-main/src/test/java/io/trino/server/protocol/TestQueryResultsSerialization.java @@ -84,6 +84,7 @@ public void testNullDataSerialization() "queuedTimeMillis" : 0, "elapsedTimeMillis" : 0, "finishingTimeMillis": 0, + "physicalInputTimeMillis": 0, "processedRows" : 0, "processedBytes" : 0, "physicalInputBytes" : 0, @@ -164,6 +165,7 @@ private String queryResultsJson(String expectedDataField) "queuedTimeMillis" : 0, "elapsedTimeMillis" : 0, "finishingTimeMillis": 0, + "physicalInputTimeMillis": 0, "processedRows" : 0, "processedBytes" : 0, "physicalInputBytes" : 0, diff --git a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java index 6c10463a742d..60d923fb3b34 100644 --- a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java +++ b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java @@ -109,6 +109,7 @@ private static BasicQueryInfo createQueryInfo(String queryId, QueryState state) new Duration(34, MINUTES), new Duration(35, MINUTES), new Duration(36, MINUTES), + new Duration(37, MINUTES), true, ImmutableSet.of(WAITING_FOR_MEMORY), OptionalDouble.of(20), From 5972dc36117addfa8bcd745073c219f2a0be5514 Mon Sep 17 00:00:00 2001 From: tanyajun <2498618501@qq.com> Date: Mon, 4 Nov 2024 09:14:43 +0800 Subject: [PATCH 110/158] Fix listing of files in AlluxioFileSystem Co-authored by: JiamingMai --- .../alluxio/AlluxioFileIterator.java | 10 +-- .../filesystem/alluxio/AlluxioFileSystem.java | 7 ++- .../filesystem/alluxio/AlluxioUtils.java | 10 +++ .../alluxio/TestAlluxioFileIterator.java | 62 +++++++++++++++++++ 4 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileIterator.java diff --git a/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileIterator.java b/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileIterator.java index e67a51f16e90..d20bc4cc2aff 100644 --- a/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileIterator.java +++ b/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileIterator.java @@ -24,19 +24,18 @@ import java.util.List; import java.util.Optional; -import static io.trino.filesystem.alluxio.AlluxioUtils.convertToLocation; import static java.util.Objects.requireNonNull; public class AlluxioFileIterator implements FileIterator { private final Iterator files; - private final String mountRoot; + private final String basePath; - public AlluxioFileIterator(List files, String mountRoot) + public AlluxioFileIterator(List files, String basePath) { this.files = requireNonNull(files.iterator(), "files is null"); - this.mountRoot = requireNonNull(mountRoot, "mountRoot is null"); + this.basePath = requireNonNull(basePath, "basePath is null"); } @Override @@ -54,7 +53,8 @@ public FileEntry next() return null; } URIStatus fileStatus = files.next(); - Location location = convertToLocation(fileStatus.getPath(), mountRoot); + String filePath = fileStatus.getPath(); + Location location = Location.of(basePath + filePath); return new FileEntry( location, fileStatus.getLength(), diff --git a/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystem.java b/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystem.java index b6a949ee44d6..94654da3f1b2 100644 --- a/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystem.java +++ b/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioFileSystem.java @@ -38,6 +38,7 @@ import java.util.stream.Collectors; import static io.trino.filesystem.alluxio.AlluxioUtils.convertToAlluxioURI; +import static io.trino.filesystem.alluxio.AlluxioUtils.getAlluxioBase; import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; @@ -189,20 +190,20 @@ public FileIterator listFiles(Location location) try { URIStatus status = alluxioClient.getStatus(convertToAlluxioURI(location, mountRoot)); if (status == null) { - new AlluxioFileIterator(Collections.emptyList(), mountRoot); + new AlluxioFileIterator(Collections.emptyList(), getAlluxioBase(location.toString())); } if (!status.isFolder()) { throw new IOException("Location is not a directory: %s".formatted(location)); } } catch (NotFoundRuntimeException | AlluxioException e) { - return new AlluxioFileIterator(Collections.emptyList(), mountRoot); + return new AlluxioFileIterator(Collections.emptyList(), getAlluxioBase(location.toString())); } try { List filesStatus = alluxioClient.listStatus(convertToAlluxioURI(location, mountRoot), ListStatusPOptions.newBuilder().setRecursive(true).build()); - return new AlluxioFileIterator(filesStatus.stream().filter(status -> !status.isFolder() & status.isCompleted()).toList(), mountRoot); + return new AlluxioFileIterator(filesStatus.stream().filter(status -> !status.isFolder() & status.isCompleted()).toList(), getAlluxioBase(location.toString())); } catch (AlluxioException e) { throw new IOException("Error listFiles %s".formatted(location), e); diff --git a/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioUtils.java b/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioUtils.java index 49ece36a8b93..ac95c970dd36 100644 --- a/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioUtils.java +++ b/lib/trino-filesystem-alluxio/src/main/java/io/trino/filesystem/alluxio/AlluxioUtils.java @@ -50,6 +50,16 @@ public static Location convertToLocation(String path, String mountRoot) return Location.of(schema + mountRootWithSlash + path); } + public static String getAlluxioBase(String path) + { + requireNonNull(path, "path is null"); + if (!path.startsWith("alluxio://")) { + throw new IllegalArgumentException("path is not an alluxio://"); + } + int index = path.indexOf('/', "alluxio://".length()); + return path.substring(0, index); + } + public static String simplifyPath(String path) { // Use a deque to store the path components diff --git a/lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileIterator.java b/lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileIterator.java new file mode 100644 index 000000000000..6ae971673887 --- /dev/null +++ b/lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileIterator.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.filesystem.alluxio; + +import alluxio.client.file.URIStatus; +import alluxio.wire.FileInfo; +import io.trino.filesystem.FileEntry; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +final class TestAlluxioFileIterator +{ + private final String fileName = "000000_0"; + private final String filePath = "/s3a/tables/sales/000000_0"; + private final String ufsFilePath = "s3a://test-bucket/tables/sales/000000_0"; + + @Test + void testNext() + throws IOException + { + String alluxioBasePath = "alluxio://master:19998"; + FileInfo fileInfo = new FileInfo(); + fileInfo.setName(fileName); + fileInfo.setPath(filePath); + fileInfo.setUfsPath(ufsFilePath); + URIStatus fileStatus = new URIStatus(fileInfo); + AlluxioFileIterator iterator = new AlluxioFileIterator( + List.of(fileStatus), + alluxioBasePath); + FileEntry fileEntry = iterator.next(); + assertThat(fileEntry.location().toString()) + .isEqualTo(alluxioBasePath + filePath); + + alluxioBasePath = "alluxio:/"; + fileInfo = new FileInfo(); + fileInfo.setName(fileName); + fileInfo.setPath(filePath); + fileInfo.setUfsPath(ufsFilePath); + fileStatus = new URIStatus(fileInfo); + iterator = new AlluxioFileIterator( + List.of(fileStatus), + alluxioBasePath); + fileEntry = iterator.next(); + assertThat(fileEntry.location().toString()) + .isEqualTo(alluxioBasePath + filePath); + } +} From 0c056c74530063add8a790919a0bc838396816cf Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Tue, 31 Dec 2024 18:38:01 -0800 Subject: [PATCH 111/158] Add info about views in memory only for Faker connector --- docs/src/main/sphinx/connector/faker.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/main/sphinx/connector/faker.md b/docs/src/main/sphinx/connector/faker.md index 21260339de23..2a3f01cd9c09 100644 --- a/docs/src/main/sphinx/connector/faker.md +++ b/docs/src/main/sphinx/connector/faker.md @@ -30,9 +30,9 @@ reading from tables returns random, but deterministic data. As a result, repeated invocation of a query returns identical data. See [](faker-usage) for more examples. -Schemas and tables in a catalog are not persisted, and are stored in the memory -of the coordinator only. They need to be recreated every time after restarting -the coordinator. +Schemas, tables, and views in a catalog are not persisted, and are stored in the +memory of the coordinator only. They need to be recreated every time after +restarting the coordinator. The following table details all general configuration properties: From c21d3903d35a63d562f9b31922abb87e4f3e0112 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 14:56:38 -0800 Subject: [PATCH 112/158] Minor improvements to Python UDF docs --- docs/src/main/sphinx/udf/python.md | 137 +++++++++++++++-------------- 1 file changed, 69 insertions(+), 68 deletions(-) diff --git a/docs/src/main/sphinx/udf/python.md b/docs/src/main/sphinx/udf/python.md index 32e7dd229e6d..56900550c137 100644 --- a/docs/src/main/sphinx/udf/python.md +++ b/docs/src/main/sphinx/udf/python.md @@ -27,16 +27,16 @@ the following steps: The following snippet shows pseudo-code: ```text - FUNCTION python_udf_name(input_parameter data_type) - RETURNS result_data_type - LANGUAGE PYTHON - WITH (handler = 'python_function') - AS $$ - ... - def python_function(input): - return ... - ... - $$ +FUNCTION python_udf_name(input_parameter data_type) + RETURNS result_data_type + LANGUAGE PYTHON + WITH (handler = 'python_function') + AS $$ + ... + def python_function(input): + return ... + ... + $$ ``` A minimal example declares the UDF `doubleup` that returns the input integer @@ -97,15 +97,15 @@ not available within a Python UDF: * `multiprocessing` * `pdb` * `pydoc` -* `socketserver*` +* `socketserver` * `sqlite3` * `ssl` -* `subprocess*` +* `subprocess` * `tkinter` -* `turtle*` +* `turtle` * `unittest` * `venv` -* `webbrowser*` +* `webbrowser` * `wsgiref` * `xmlrpc` @@ -114,68 +114,69 @@ not available within a Python UDF: The following table shows supported Trino types and their corresponding Python types for input and output values of a Python UDF: -:::{list-table} File system support properties -:widths: 50, 50 +:::{list-table} +:widths: 40, 60 :header-rows: 1 * - Trino type - Python type -* - row - - tuple -* - array - - list -* - map - - dict -* - boolean - - bool -* - tinyint - - int -* - smallint - - int -* - integer - - int -* - bigint - - int -* - real - - float -* - double - - float -* - decimal - - decimal.Decimal -* - varchar - - str -* - varbinary - - bytes -* - date - - datetime.date -* - time - - datetime.time -* - time with time zone - - datetime.time with datetime.tzinfo -* - timestamp - - datetime.datetime -* - timestamp with time zone - - datetime.datetime with datetime.tzinfo 1 -* - interval year to month - - int as the number of months -* - interval day to second - - datetime.timedelta -* - json - - str -* - uuid - - uuid.UUID -* - ipaddress - - ipaddress.IPv4Address or ipaddress.IPv6Address +* - `ROW` + - `tuple` +* - `ARRAY` + - `list` +* - `MAP` + - `dict` +* - `BOOLEAN` + - `bool` +* - `TINYINT` + - `int` +* - `SMALLINT` + - `int` +* - `INTEGER` + - `int` +* - `BIGINT` + - `int` +* - `REAL` + - `float` +* - `DOUBLE` + - `float` +* - `DECIMAL` + - `decimal.Decimal` +* - `VARCHAR` + - `str` +* - `VARBINARY` + - `bytes` +* - `DATE` + - `datetime.date` +* - `TIME` + - `datetime.time` +* - `TIME WITH TIME ZONE` + - `datetime.time` with `datetime.tzinfo` +* - `TIMESTAMP` + - `datetime.datetime` +* - `TIMESTAMP WITH TIME ZONE` + - `datetime.datetime` with `datetime.tzinfo` +* - `INTERVAL YEAR TO MONTH` + - `int` as the number of months +* - `INTERVAL DAY TO SECOND` + - `datetime.timedelta` +* - `JSON` + - `str` +* - `UUID` + - `uuid.UUID` +* - `IPADDRESS` + - `ipaddress.IPv4Address` or `ipaddress.IPv6Address` ::: -### Date and time +### Time and timestamp -Python datetime objects only support microsecond precision. Trino argument -values with greater precision arerounded when converted to Python values, and -Python return values are rounded if the Trino return type has less than -microsecond precision. +Python `datetime` and `time` objects only support microsecond precision. +Trino argument values with greater precision are rounded when converted to +Python values, and Python return values are rounded if the Trino return type +has less than microsecond precision. + +### Timestamp with time zone Only fixed offset time zones are supported. Timestamps with political time zones have the zone converted to the zone's offset for the timestamp's instant. - From 75c15cc3165ac68f2cab8025e87e9045a141e44c Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Wed, 18 Dec 2024 14:35:26 -0800 Subject: [PATCH 113/158] Improve docs for non-transactional merge As applicable for PostgreSQL connector for now. Also extract into a fragment so it can be reused in other connectors. --- .../connector/non-transactional-merge.fragment | 11 +++++++++++ docs/src/main/sphinx/connector/postgresql.md | 13 ++----------- 2 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 docs/src/main/sphinx/connector/non-transactional-merge.fragment diff --git a/docs/src/main/sphinx/connector/non-transactional-merge.fragment b/docs/src/main/sphinx/connector/non-transactional-merge.fragment new file mode 100644 index 000000000000..60e905372a38 --- /dev/null +++ b/docs/src/main/sphinx/connector/non-transactional-merge.fragment @@ -0,0 +1,11 @@ +### Non-transactional MERGE + +The connector supports adding, updating, and deleting rows using [MERGE +statements](/sql/merge), if the `merge.non-transactional-merge.enabled` catalog +property or the corresponding `non_transactional_merge_enabled` catalog session +property is set to `true`. Merge is only supported for directly modifying target +tables. + +In rare cases, expections occur during the merge operation, potentially +resulting in a partial update. + diff --git a/docs/src/main/sphinx/connector/postgresql.md b/docs/src/main/sphinx/connector/postgresql.md index 62d3d7789518..1a1dd092b1ee 100644 --- a/docs/src/main/sphinx/connector/postgresql.md +++ b/docs/src/main/sphinx/connector/postgresql.md @@ -115,17 +115,8 @@ catalog named `sales` using the configured connector. ```{include} non-transactional-insert.fragment ``` -### Non-transactional MERGE - -The connector supports adding rows using {doc}`MERGE statements `. -However, the connector only support merge modifying directly to the target -table at current, to use merge you need to set the `merge.non-transactional-merge.enabled` -catalog property or the corresponding `non_transactional_merge_enabled` catalog session property to -`true`. - -Note that with this property enabled, data can be corrupted in rare cases where -exceptions occur during the merge operation. With transactions disabled, no -rollback can be performed. +```{include} non-transactional-merge.fragment +``` (postgresql-fte-support)= ### Fault-tolerant execution support From 699db9e1a3c4f1860d609e348c03844e245569a0 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Wed, 18 Dec 2024 15:01:17 -0800 Subject: [PATCH 114/158] Improve SQL support section in JDBC connectors - No content changes but... - Consistent wording _ Markdown link syntax - Move related configs to SQL support section - Improve list and rejig as small local ToC, add links --- .../alter-schema-limitation.fragment | 2 +- .../connector/alter-table-limitation.fragment | 2 +- docs/src/main/sphinx/connector/clickhouse.md | 25 +++++++---- docs/src/main/sphinx/connector/druid.md | 12 ++++-- docs/src/main/sphinx/connector/exasol.md | 12 ++++-- docs/src/main/sphinx/connector/ignite.md | 25 ++++++----- docs/src/main/sphinx/connector/mariadb.md | 39 ++++++++++------- docs/src/main/sphinx/connector/mysql.md | 39 ++++++++++------- docs/src/main/sphinx/connector/oracle.md | 40 ++++++++++------- docs/src/main/sphinx/connector/phoenix.md | 38 ++++++++-------- docs/src/main/sphinx/connector/postgresql.md | 43 ++++++++++++------- docs/src/main/sphinx/connector/redshift.md | 27 ++++++++---- docs/src/main/sphinx/connector/singlestore.md | 32 ++++++++------ docs/src/main/sphinx/connector/snowflake.md | 35 ++++++++------- .../connector/sql-delete-limitation.fragment | 2 +- .../connector/sql-update-limitation.fragment | 2 +- docs/src/main/sphinx/connector/sqlserver.md | 32 +++++++++----- docs/src/main/sphinx/connector/vertica.md | 5 ++- 18 files changed, 255 insertions(+), 157 deletions(-) diff --git a/docs/src/main/sphinx/connector/alter-schema-limitation.fragment b/docs/src/main/sphinx/connector/alter-schema-limitation.fragment index 39065f900324..33100d90c262 100644 --- a/docs/src/main/sphinx/connector/alter-schema-limitation.fragment +++ b/docs/src/main/sphinx/connector/alter-schema-limitation.fragment @@ -1,4 +1,4 @@ -### ALTER SCHEMA +### ALTER SCHEMA limitation The connector supports renaming a schema with the `ALTER SCHEMA RENAME` statement. `ALTER SCHEMA SET AUTHORIZATION` is not supported. diff --git a/docs/src/main/sphinx/connector/alter-table-limitation.fragment b/docs/src/main/sphinx/connector/alter-table-limitation.fragment index de64a02a9c3b..d3101270ee69 100644 --- a/docs/src/main/sphinx/connector/alter-table-limitation.fragment +++ b/docs/src/main/sphinx/connector/alter-table-limitation.fragment @@ -1,4 +1,4 @@ -### ALTER TABLE RENAME TO +### ALTER TABLE RENAME TO limitation The connector does not support renaming tables across multiple schemas. For example, the following statement is supported: diff --git a/docs/src/main/sphinx/connector/clickhouse.md b/docs/src/main/sphinx/connector/clickhouse.md index 0da7a863bb28..fc34ac02ee17 100644 --- a/docs/src/main/sphinx/connector/clickhouse.md +++ b/docs/src/main/sphinx/connector/clickhouse.md @@ -93,9 +93,6 @@ configured connector to create a catalog named `sales`. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - ## Querying ClickHouse The ClickHouse connector provides a schema for every ClickHouse *database*. @@ -317,18 +314,27 @@ No other types are supported. (clickhouse-sql-support)= ## SQL support -The connector provides read and write access to data and metadata in -a ClickHouse catalog. In addition to the {ref}`globally available -` and {ref}`read operation ` +The connector provides read and write access to data and metadata in a +ClickHouse catalog. In addition to the [globally +available](sql-globally-available) and [read operation](sql-read-operations) statements, the connector supports the following features: -- {doc}`/sql/insert` -- {doc}`/sql/truncate` -- {ref}`sql-schema-table-management` +- [](/sql/insert), see also [](clickhouse-insert) +- [](/sql/truncate) +- [](sql-schema-table-management), see also: + - [](clickhouse-alter-table) +- [](clickhouse-procedures) +- [](clickhouse-table-functions) + +(clickhouse-insert)= +```{include} non-transactional-insert.fragment +``` +(clickhouse-alter-table)= ```{include} alter-schema-limitation.fragment ``` +(clickhouse-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -336,6 +342,7 @@ statements, the connector supports the following features: ```{include} procedures-execute.fragment ``` +(clickhouse-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/druid.md b/docs/src/main/sphinx/connector/druid.md index ab6f4d269a2b..261047c85dcd 100644 --- a/docs/src/main/sphinx/connector/druid.md +++ b/docs/src/main/sphinx/connector/druid.md @@ -116,10 +116,15 @@ be an empty string `''`, and so forth. (druid-sql-support)= ## SQL support -The connector provides {ref}`globally available ` and -{ref}`read operation ` statements to access data and -metadata in the Druid database. +The connector provides read access to data and metadata in the Druid database. +In addition to the [globally available](sql-globally-available) and [read +operation](sql-read-operations) statements, the connector supports the following +features: +- [](druid-procedures) +- [](druid-table-functions) + +(druid-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -127,6 +132,7 @@ metadata in the Druid database. ```{include} procedures-execute.fragment ``` +(druid-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/exasol.md b/docs/src/main/sphinx/connector/exasol.md index a6b7de26d8c2..23147958a70c 100644 --- a/docs/src/main/sphinx/connector/exasol.md +++ b/docs/src/main/sphinx/connector/exasol.md @@ -133,10 +133,15 @@ Exasol does not support longer values. (exasol-sql-support)= ## SQL support -The connector provides {ref}`globally available ` and -{ref}`read operation ` statements to access data and -metadata in the Exasol database. +The connector provides read access to data and metadata in Exasol. In addition +to the [globally available](sql-globally-available) and [read +operation](sql-read-operations) statements, the connector supports the following +features: +- [](exasol-procedures) +- [](exasol-table-functions) + +(exasol-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -144,6 +149,7 @@ metadata in the Exasol database. ```{include} procedures-execute.fragment ``` +(exasol-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/ignite.md b/docs/src/main/sphinx/connector/ignite.md index abd43a529bd4..8bde66f1de07 100644 --- a/docs/src/main/sphinx/connector/ignite.md +++ b/docs/src/main/sphinx/connector/ignite.md @@ -75,9 +75,6 @@ configured connector to create a catalog named `sales`. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - ## Table properties Table property usage example: @@ -173,20 +170,28 @@ Ignite. In addition to the {ref}`globally available ` and {ref}`read operation ` statements, the connector supports the following features: -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/alter-table` +- [](/sql/insert), see also [](ignite-insert) +- [](/sql/update), see also [](ignite-update) +- [](/sql/delete) +- [](/sql/create-table) +- [](/sql/create-table-as) +- [](/sql/drop-table) +- [](/sql/alter-table), see also [](ignite-alter-table) +- [](ignite-procedures) + +(ignite-insert)= +```{include} non-transactional-insert.fragment +``` +(ignite-update)= ```{include} sql-update-limitation.fragment ``` +(ignite-alter-table)= ```{include} alter-table-limitation.fragment ``` +(ignite-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment diff --git a/docs/src/main/sphinx/connector/mariadb.md b/docs/src/main/sphinx/connector/mariadb.md index e07d37cd585d..37d1762cce74 100644 --- a/docs/src/main/sphinx/connector/mariadb.md +++ b/docs/src/main/sphinx/connector/mariadb.md @@ -52,9 +52,6 @@ properties files. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - (mariadb-fte-support)= ### Fault-tolerant execution support @@ -283,28 +280,37 @@ Complete list of [MariaDB data types](https://mariadb.com/kb/en/data-types/). (mariadb-sql-support)= ## SQL support -The connector provides read access and write access to data and metadata in -a MariaDB database. In addition to the {ref}`globally available -` and {ref}`read operation ` +The connector provides read access and write access to data and metadata in a +MariaDB database. In addition to the [globally +available](sql-globally-available) and [read operation](sql-read-operations) statements, the connector supports the following features: -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/alter-table` -- {doc}`/sql/create-schema` -- {doc}`/sql/drop-schema` +- [](/sql/insert), see also [](mariadb-insert) +- [](/sql/update), see also [](mariadb-update) +- [](/sql/delete), see also [](mariadb-delete) +- [](/sql/truncate) +- [](/sql/create-table) +- [](/sql/create-table-as) +- [](/sql/drop-table) +- [](/sql/alter-table) +- [](/sql/create-schema) +- [](/sql/drop-schema) +- [](mariadb-procedures) +- [](mariadb-table-functions) + +(mariadb-insert)= +```{include} non-transactional-insert.fragment +``` +(mariadb-update)= ```{include} sql-update-limitation.fragment ``` +(mariadb-delete)= ```{include} sql-delete-limitation.fragment ``` +(mariadb-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -312,6 +318,7 @@ statements, the connector supports the following features: ```{include} procedures-execute.fragment ``` +(mariadb-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/mysql.md b/docs/src/main/sphinx/connector/mysql.md index 61d5457d786d..30f5e2bac925 100644 --- a/docs/src/main/sphinx/connector/mysql.md +++ b/docs/src/main/sphinx/connector/mysql.md @@ -99,9 +99,6 @@ creates a catalog named `sales` using the configured connector. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - (mysql-fte-support)= ### Fault-tolerant execution support @@ -334,26 +331,35 @@ that catalog name instead of `example` in the above examples. ## SQL support The connector provides read access and write access to data and metadata in the -MySQL database. In addition to the {ref}`globally available ` and -{ref}`read operation ` statements, the connector supports -the following statements: - -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/create-schema` -- {doc}`/sql/drop-schema` +MySQL database. In addition to the [globally available](sql-globally-available) +and [read operation](sql-read-operations) statements, the connector supports the +following features: + +- [](/sql/insert), see also [](mysql-insert) +- [](/sql/update), see also [](mysql-update) +- [](/sql/delete), see also [](mysql-delete) +- [](/sql/truncate) +- [](/sql/create-table) +- [](/sql/create-table-as) +- [](/sql/drop-table) +- [](/sql/create-schema) +- [](/sql/drop-schema) +- [](mysql-procedures) +- [](mysql-table-functions) + +(mysql-insert)= +```{include} non-transactional-insert.fragment +``` +(mysql-update)= ```{include} sql-update-limitation.fragment ``` +(mysql-delete)= ```{include} sql-delete-limitation.fragment ``` +(mysql-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -361,6 +367,7 @@ the following statements: ```{include} procedures-execute.fragment ``` +(mysql-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/oracle.md b/docs/src/main/sphinx/connector/oracle.md index de4cb33b0b3d..238f046bb60a 100644 --- a/docs/src/main/sphinx/connector/oracle.md +++ b/docs/src/main/sphinx/connector/oracle.md @@ -98,9 +98,6 @@ you name the property file `sales.properties`, Trino creates a catalog named ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - (oracle-fte-support)= ### Fault-tolerant execution support @@ -405,29 +402,39 @@ fails. This is also true for the equivalent `VARCHAR` types. ## SQL support The connector provides read access and write access to data and metadata in -Oracle. In addition to the {ref}`globally available ` -and {ref}`read operation ` statements, the connector -supports the following statements: - -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/alter-table` -- {doc}`/sql/comment` +Oracle. In addition to the [globally available](sql-globally-available) and +[read operation](sql-read-operations) statements, the connector supports the +following features: + +- [](/sql/insert), see also [](oracle-insert) +- [](/sql/update), see also [](oracle-update) +- [](/sql/delete), see also [](oracle-delete) +- [](/sql/truncate) +- [](/sql/create-table) +- [](/sql/create-table-as) +- [](/sql/drop-table) +- [](/sql/alter-table), see also [](oracle-alter-table) +- [](/sql/comment) +- [](oracle-procedures) +- [](oracle-table-functions) + +(oracle-insert)= +```{include} non-transactional-insert.fragment +``` +(oracle-update)= ```{include} sql-update-limitation.fragment ``` +(oracle-delete)= ```{include} sql-delete-limitation.fragment ``` +(oracle-alter-table)= ```{include} alter-table-limitation.fragment ``` +(oracle-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -435,6 +442,7 @@ supports the following statements: ```{include} procedures-execute.fragment ``` +(oracle-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/phoenix.md b/docs/src/main/sphinx/connector/phoenix.md index a1116b13234f..6d7f2ef613ba 100644 --- a/docs/src/main/sphinx/connector/phoenix.md +++ b/docs/src/main/sphinx/connector/phoenix.md @@ -63,9 +63,6 @@ The following Phoenix-specific configuration properties are available: ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - ## Querying Phoenix tables The default empty schema in Phoenix maps to a schema named `default` in Trino. @@ -268,24 +265,31 @@ Use them in the same way as above: in the `WITH` clause of the `CREATE TABLE` st (phoenix-sql-support)= ## SQL support -The connector provides read and write access to data and metadata in -Phoenix. In addition to the {ref}`globally available -` and {ref}`read operation ` -statements, the connector supports the following features: - -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/merge` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/create-schema` -- {doc}`/sql/drop-schema` +The connector provides read and write access to data and metadata in Phoenix. In +addition to the [globally available](sql-globally-available) and [read +operation](sql-read-operations) statements, the connector supports the following +features: + +- [](/sql/insert), see also [](phoenix-insert) +- [](/sql/update) +- [](/sql/delete), see also [](phoenix-delete) +- [](/sql/merge) +- [](/sql/create-table) +- [](/sql/create-table-as) +- [](/sql/drop-table) +- [](/sql/create-schema) +- [](/sql/drop-schema) +- [](phoenix-procedures) + +(phoenix-insert)= +```{include} non-transactional-insert.fragment +``` +(phoenix-delete)= ```{include} sql-delete-limitation.fragment ``` +(phoenix-procedures)= ### Procedures ```{include} procedures-execute.fragment diff --git a/docs/src/main/sphinx/connector/postgresql.md b/docs/src/main/sphinx/connector/postgresql.md index 1a1dd092b1ee..30eb90c0ce0d 100644 --- a/docs/src/main/sphinx/connector/postgresql.md +++ b/docs/src/main/sphinx/connector/postgresql.md @@ -112,12 +112,6 @@ catalog named `sales` using the configured connector. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - -```{include} non-transactional-merge.fragment -``` - (postgresql-fte-support)= ### Fault-tolerant execution support @@ -351,28 +345,46 @@ that catalog name instead of `example` in the above examples. ## SQL support The connector provides read access and write access to data and metadata in -PostgreSQL. In addition to the {ref}`globally available -` and {ref}`read operation ` -statements, the connector supports the following features: - -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {ref}`sql-schema-table-management` +PostgreSQL. In addition to the [globally available](sql-globally-available) and +[read operation](sql-read-operations) statements, the connector supports the +following features: + +- [](/sql/insert), see also [](postgresql-insert) +- [](/sql/update), see also [](postgresql-update) +- [](/sql/delete), see also [](postgresql-delete) +- [](/sql/merge), see also [](postgresql-merge) +- [](/sql/truncate) +- [](sql-schema-table-management), see also: + - [](postgresql-alter-table) + - [](postgresql-alter-schema) +- [](postgresql-procedures) +- [](postgresql-table-functions) + +(postgresql-insert)= +```{include} non-transactional-insert.fragment +``` +(postgresql-update)= ```{include} sql-update-limitation.fragment ``` +(postgresql-delete)= ```{include} sql-delete-limitation.fragment ``` +(postgresql-merge)= +```{include} non-transactional-merge.fragment +``` + +(postgresql-alter-table)= ```{include} alter-table-limitation.fragment ``` +(postgresql-alter-schema)= ```{include} alter-schema-limitation.fragment ``` +(postgresql-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -380,6 +392,7 @@ statements, the connector supports the following features: ```{include} procedures-execute.fragment ``` +(postgresql-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/redshift.md b/docs/src/main/sphinx/connector/redshift.md index 659905f56ff5..1543834e5911 100644 --- a/docs/src/main/sphinx/connector/redshift.md +++ b/docs/src/main/sphinx/connector/redshift.md @@ -125,9 +125,6 @@ catalog named `sales` using the configured connector. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - (redshift-fte-support)= ## Fault-tolerant execution support @@ -181,24 +178,37 @@ Redshift. In addition to the {ref}`globally available ` and {ref}`read operation ` statements, the connector supports the following features: -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {ref}`sql-schema-table-management` +- [](/sql/insert), see also [](redshift-insert) +- [](/sql/update), see also [](redshift-update) +- [](/sql/delete), see also [](redshift-delete) +- [](/sql/truncate) +- [](sql-schema-table-management), see also: + - [](redshift-alter-table) + - [](redshift-alter-schema) +- [](redshift-procedures) +- [](redshift-table-functions) + +(redshift-insert)= +```{include} non-transactional-insert.fragment +``` +(redshift-update)= ```{include} sql-update-limitation.fragment ``` +(redshift-delete)= ```{include} sql-delete-limitation.fragment ``` +(redshift-alter-table)= ```{include} alter-table-limitation.fragment ``` +(redshift-alter-schema)= ```{include} alter-schema-limitation.fragment ``` +(redshift-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -206,6 +216,7 @@ statements, the connector supports the following features: ```{include} procedures-execute.fragment ``` +(redshift-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/singlestore.md b/docs/src/main/sphinx/connector/singlestore.md index e30f770e1798..c11467e1e330 100644 --- a/docs/src/main/sphinx/connector/singlestore.md +++ b/docs/src/main/sphinx/connector/singlestore.md @@ -85,9 +85,6 @@ will create a catalog named `sales` using the configured connector. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - ## Querying SingleStore The SingleStore connector provides a schema for every SingleStore *database*. @@ -315,26 +312,35 @@ a SingleStore database. In addition to the {ref}`globally available ` and {ref}`read operation ` statements, the connector supports the following features: -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/alter-table` -- {doc}`/sql/create-schema` -- {doc}`/sql/drop-schema` +- [](/sql/insert), see also [](singlestore-insert) +- [](/sql/update), see also [](singlestore-update) +- [](/sql/delete), see also [](singlestore-delete) +- [](/sql/truncate) +- [](/sql/create-table) +- [](/sql/create-table-as) +- [](/sql/alter-table), see also [](singlestore-alter-table) +- [](/sql/drop-table) +- [](/sql/create-schema) +- [](/sql/drop-schema) +- [](singlestore-procedures) + +(singlestore-insert)= +```{include} non-transactional-insert.fragment +``` +(singlestore-update)= ```{include} sql-update-limitation.fragment ``` +(singlestore-delete)= ```{include} sql-delete-limitation.fragment ``` +(singlestore-alter-table)= ```{include} alter-table-limitation.fragment ``` +(singlestore-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment diff --git a/docs/src/main/sphinx/connector/snowflake.md b/docs/src/main/sphinx/connector/snowflake.md index e17ca51a8b47..8d82faa722ce 100644 --- a/docs/src/main/sphinx/connector/snowflake.md +++ b/docs/src/main/sphinx/connector/snowflake.md @@ -62,9 +62,6 @@ multiple instances of the Snowflake connector. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - % snowflake-type-mapping: ## Type mapping @@ -233,21 +230,28 @@ No other types are supported. (snowflake-sql-support)= ## SQL support -The connector provides read access and write access to data and metadata in -a Snowflake database. In addition to the {ref}`globally available -` and {ref}`read operation ` +The connector provides read access and write access to data and metadata in a +Snowflake database. In addition to the [globally +available](sql-globally-available) and [read operation](sql-read-operations) statements, the connector supports the following features: -- {doc}`/sql/insert` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/alter-table` -- {doc}`/sql/create-schema` -- {doc}`/sql/drop-schema` +- [](/sql/insert), see also [](snowflake-insert) +- [](/sql/delete) +- [](/sql/truncate) +- [](/sql/create-table) +- [](/sql/create-table-as) +- [](/sql/drop-table) +- [](/sql/alter-table) +- [](/sql/create-schema) +- [](/sql/drop-schema) +- [](snowflake-procedures) +- [](snowflake-table-functions) + +(snowflake-insert)= +```{include} non-transactional-insert.fragment +``` +(snowflake-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -255,6 +259,7 @@ statements, the connector supports the following features: ```{include} procedures-execute.fragment ``` +(snowflake-table-functions)= ### Table functions The connector provides specific [table functions](/functions/table) to diff --git a/docs/src/main/sphinx/connector/sql-delete-limitation.fragment b/docs/src/main/sphinx/connector/sql-delete-limitation.fragment index 5c3a201ca369..aeb1e9bd9e18 100644 --- a/docs/src/main/sphinx/connector/sql-delete-limitation.fragment +++ b/docs/src/main/sphinx/connector/sql-delete-limitation.fragment @@ -1,4 +1,4 @@ -### SQL DELETE +### DELETE limitation If a ``WHERE`` clause is specified, the ``DELETE`` operation only works if the predicate in the clause can be fully pushed down to the data source. diff --git a/docs/src/main/sphinx/connector/sql-update-limitation.fragment b/docs/src/main/sphinx/connector/sql-update-limitation.fragment index 6143a8e0762d..0e3ece216322 100644 --- a/docs/src/main/sphinx/connector/sql-update-limitation.fragment +++ b/docs/src/main/sphinx/connector/sql-update-limitation.fragment @@ -1,4 +1,4 @@ -### UPDATE +### UPDATE limitation Only `UPDATE` statements with constant assignments and predicates are supported. For example, the following statement is supported because the values diff --git a/docs/src/main/sphinx/connector/sqlserver.md b/docs/src/main/sphinx/connector/sqlserver.md index 705083eb3281..469864d9fe9e 100644 --- a/docs/src/main/sphinx/connector/sqlserver.md +++ b/docs/src/main/sphinx/connector/sqlserver.md @@ -111,9 +111,6 @@ behavior of the connector and the issues queries to the database. ```{include} jdbc-case-insensitive-matching.fragment ``` -```{include} non-transactional-insert.fragment -``` - (sqlserver-fte-support)= ### Fault-tolerant execution support @@ -325,25 +322,37 @@ For Trino `VARCHAR(n)`: ## SQL support The connector provides read access and write access to data and metadata in SQL -Server. In addition to the {ref}`globally available ` -and {ref}`read operation ` statements, the connector -supports the following features: +Server. In addition to the [globally available](sql-globally-available) and +[read operation](sql-read-operations) statements, the connector supports the +following features: + +- [](/sql/insert), see also [](sqlserver-insert) +- [](/sql/update), see also [](sqlserver-update) +- [](/sql/delete), see also [](sqlserver-delete) +- [](/sql/truncate) +- [](sql-schema-table-management), see also: + - [](sqlserver-alter-table) +- [](sqlserver-procedures) +- [](sqlserver-table-functions) -- {doc}`/sql/insert` -- {doc}`/sql/update` -- {doc}`/sql/delete` -- {doc}`/sql/truncate` -- {ref}`sql-schema-table-management` +(sqlserver-insert)= +```{include} non-transactional-insert.fragment +``` + +(sqlserver-update)= ```{include} sql-update-limitation.fragment ``` +(sqlserver-delete)= ```{include} sql-delete-limitation.fragment ``` +(sqlserver-alter-table)= ```{include} alter-table-limitation.fragment ``` +(sqlserver-procedures)= ### Procedures ```{include} jdbc-procedures-flush.fragment @@ -351,6 +360,7 @@ supports the following features: ```{include} procedures-execute.fragment ``` +(sqlserver-table-functions)= ### Table functions The connector provides specific {doc}`table functions ` to diff --git a/docs/src/main/sphinx/connector/vertica.md b/docs/src/main/sphinx/connector/vertica.md index 72960dba32fb..79f2ff079b44 100644 --- a/docs/src/main/sphinx/connector/vertica.md +++ b/docs/src/main/sphinx/connector/vertica.md @@ -165,13 +165,16 @@ features: - [](/sql/create-table) - [](/sql/create-table-as) - [](/sql/drop-table) -- [](/sql/alter-table) excluding `DROP COLUMN` +- [](/sql/alter-table) excluding `DROP COLUMN`, see also [](vertica-alter-table) - [](/sql/create-schema) - [](/sql/drop-schema) +- [](vertica-table-functions) +(vertica-alter-table)= ```{include} alter-table-limitation.fragment ``` +(vertica-table-functions)= ## Table functions The connector provides specific [table functions](/functions/table) to From 084c45d27fc6189db645b1eb051b4346d5eac900 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Thu, 26 Dec 2024 21:39:08 -0800 Subject: [PATCH 115/158] Add non-transactional MERGE docs for Phoenix --- docs/src/main/sphinx/connector/phoenix.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/src/main/sphinx/connector/phoenix.md b/docs/src/main/sphinx/connector/phoenix.md index 6d7f2ef613ba..7efdbe2e98c1 100644 --- a/docs/src/main/sphinx/connector/phoenix.md +++ b/docs/src/main/sphinx/connector/phoenix.md @@ -273,7 +273,7 @@ features: - [](/sql/insert), see also [](phoenix-insert) - [](/sql/update) - [](/sql/delete), see also [](phoenix-delete) -- [](/sql/merge) +- [](/sql/merge), see also [](phoenix-merge) - [](/sql/create-table) - [](/sql/create-table-as) - [](/sql/drop-table) @@ -289,6 +289,10 @@ features: ```{include} sql-delete-limitation.fragment ``` +(phoenix-merge)= +```{include} non-transactional-merge.fragment +``` + (phoenix-procedures)= ### Procedures From 265e7dd0b711a90efc64b97157d92d9dac006f6b Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Thu, 26 Dec 2024 21:40:20 -0800 Subject: [PATCH 116/158] Add non-transactional MERGE docs for Ignite --- docs/src/main/sphinx/connector/ignite.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/src/main/sphinx/connector/ignite.md b/docs/src/main/sphinx/connector/ignite.md index 8bde66f1de07..108ebc4a2a7e 100644 --- a/docs/src/main/sphinx/connector/ignite.md +++ b/docs/src/main/sphinx/connector/ignite.md @@ -173,6 +173,7 @@ statements, the connector supports the following features: - [](/sql/insert), see also [](ignite-insert) - [](/sql/update), see also [](ignite-update) - [](/sql/delete) +- [](/sql/merge), see also [](ignite-merge) - [](/sql/create-table) - [](/sql/create-table-as) - [](/sql/drop-table) @@ -187,6 +188,10 @@ statements, the connector supports the following features: ```{include} sql-update-limitation.fragment ``` +(ignite-merge)= +```{include} non-transactional-merge.fragment +``` + (ignite-alter-table)= ```{include} alter-table-limitation.fragment ``` From 7c129fb655bd4b37d4707735a25ddd220c142c89 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 00:10:50 -0700 Subject: [PATCH 117/158] Remove unused metastore classes --- .../hive/metastore/DatabaseFunctionKey.java | 25 ------ .../DatabaseFunctionSignatureKey.java | 29 ------- .../hive/metastore/UserDatabaseKey.java | 78 ------------------- 3 files changed, 132 deletions(-) delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionKey.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionSignatureKey.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/UserDatabaseKey.java diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionKey.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionKey.java deleted file mode 100644 index 1237525abb70..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionKey.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore; - -import static java.util.Objects.requireNonNull; - -public record DatabaseFunctionKey(String databaseName, String functionName) -{ - public DatabaseFunctionKey - { - requireNonNull(databaseName, "databaseName is null"); - requireNonNull(functionName, "functionName is null"); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionSignatureKey.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionSignatureKey.java deleted file mode 100644 index 905d23fcdd50..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/DatabaseFunctionSignatureKey.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore; - -import static java.util.Objects.requireNonNull; - -public record DatabaseFunctionSignatureKey( - String databaseName, - String functionName, - String signatureToken) -{ - public DatabaseFunctionSignatureKey - { - requireNonNull(databaseName, "databaseName is null"); - requireNonNull(functionName, "functionName is null"); - requireNonNull(signatureToken, "signatureToken is null"); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/UserDatabaseKey.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/UserDatabaseKey.java deleted file mode 100644 index d25645aa5e0f..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/UserDatabaseKey.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.errorprone.annotations.Immutable; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -@Immutable -public class UserDatabaseKey -{ - private final String user; - private final String database; - - @JsonCreator - public UserDatabaseKey(@JsonProperty("user") String user, @JsonProperty("database") String database) - { - this.user = requireNonNull(user, "user is null"); - this.database = requireNonNull(database, "database is null"); - } - - @JsonProperty - public String getUser() - { - return user; - } - - @JsonProperty - public String getDatabase() - { - return database; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - UserDatabaseKey that = (UserDatabaseKey) o; - return Objects.equals(user, that.user) && - Objects.equals(database, that.database); - } - - @Override - public int hashCode() - { - return Objects.hash(user, database); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("principalName", user) - .add("database", database) - .toString(); - } -} From 68d5565730158c5f2e2d318ffd39c6f847f5b2c6 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 00:23:19 -0700 Subject: [PATCH 118/158] Remove unused TestingIcebergHiveMetastoreCatalogModule --- ...tingIcebergHiveMetastoreCatalogModule.java | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java deleted file mode 100644 index 9bd5e87b1654..000000000000 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.iceberg.catalog.hms; - -import com.google.inject.Binder; -import com.google.inject.Scopes; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.units.Duration; -import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; -import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreFactory; -import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; -import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; - -import java.util.concurrent.TimeUnit; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.util.Objects.requireNonNull; - -public class TestingIcebergHiveMetastoreCatalogModule - extends AbstractConfigurationAwareModule -{ - private final HiveMetastore hiveMetastore; - private final ThriftMetastoreFactory thriftMetastoreFactory; - - public TestingIcebergHiveMetastoreCatalogModule(HiveMetastore hiveMetastore, ThriftMetastoreFactory thriftMetastoreFactory) - { - this.hiveMetastore = requireNonNull(hiveMetastore, "hiveMetastore is null"); - this.thriftMetastoreFactory = requireNonNull(thriftMetastoreFactory, "thriftMetastoreFactory is null"); - } - - @Override - protected void setup(Binder binder) - { - install(new CachingHiveMetastoreModule(false)); - binder.bind(ThriftMetastoreFactory.class).toInstance(this.thriftMetastoreFactory); - binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(this.hiveMetastore)); - binder.bind(IcebergTableOperationsProvider.class).to(HiveMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); - binder.bind(TrinoCatalogFactory.class).to(TrinoHiveCatalogFactory.class).in(Scopes.SINGLETON); - - configBinder(binder).bindConfigDefaults(CachingHiveMetastoreConfig.class, config -> { - // ensure caching metastore wrapper isn't created, as it's not leveraged by Iceberg - config.setStatsCacheTtl(new Duration(0, TimeUnit.SECONDS)); - }); - } -} From b645f13adeea4dfe3bfcb8b2182c9baf17197ee6 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 10:32:34 -0700 Subject: [PATCH 119/158] Replace usage of RetryDriver in HiveMetadata --- .../main/java/io/trino/plugin/hive/HiveMetadata.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 6b958cd2c694..4d0996f6724b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -24,6 +24,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; import io.airlift.slice.Slice; @@ -319,7 +321,6 @@ import static io.trino.plugin.hive.util.HiveWriteUtils.createPartitionValues; import static io.trino.plugin.hive.util.HiveWriteUtils.isFileCreatedByQuery; import static io.trino.plugin.hive.util.HiveWriteUtils.isWritableType; -import static io.trino.plugin.hive.util.RetryDriver.retry; import static io.trino.plugin.hive.util.Statistics.createComputedStatisticsToPartitionMap; import static io.trino.plugin.hive.util.Statistics.createEmptyPartitionStatistics; import static io.trino.plugin.hive.util.Statistics.fromComputedStatistics; @@ -388,6 +389,12 @@ public class HiveMetadata public static final String MODIFYING_NON_TRANSACTIONAL_TABLE_MESSAGE = "Modifying Hive table rows is only supported for transactional tables"; + private static final RetryPolicy DELETE_RETRY_POLICY = RetryPolicy.builder() + .withDelay(java.time.Duration.ofSeconds(1)) + .withMaxDuration(java.time.Duration.ofSeconds(30)) + .withMaxAttempts(10) + .build(); + private final CatalogName catalogName; private final SemiTransactionalHiveMetastore metastore; private final boolean autoCommit; @@ -2687,14 +2694,13 @@ private void finishOptimize(ConnectorSession session, ConnectorTableExecuteHandl if (firstScannedPath.isEmpty()) { firstScannedPath = Optional.of(scannedPath); } - retry().run("delete " + scannedPath, () -> { + Failsafe.with(DELETE_RETRY_POLICY).run(() -> { try { fileSystem.deleteFile(scannedPath); } catch (FileNotFoundException e) { // ignore missing files } - return null; }); someDeleted = true; remainingFilesToDelete.remove(scannedPath); From 745ae010b341fc71f2a53bfbdb8dd42e79a33dd9 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Tue, 30 Jul 2024 23:40:24 -0700 Subject: [PATCH 120/158] Move partition utility methods to Partitions class --- .../java/io/trino/metastore/Partition.java | 64 -------- .../metastore/PartitionWithStatistics.java | 2 +- .../java/io/trino/metastore/Partitions.java | 154 ++++++++++++++++++ .../io/trino/metastore/TestPartitions.java | 116 +++++++++++++ .../deltalake/AbstractDeltaLakePageSink.java | 10 +- .../deltalake/util/DeltaLakeWriteUtils.java | 2 +- .../plugin/hive/HiveAnalyzeProperties.java | 2 +- .../io/trino/plugin/hive/HiveMetadata.java | 5 +- .../trino/plugin/hive/HivePartitionKey.java | 3 +- .../plugin/hive/HivePartitionManager.java | 2 +- .../trino/plugin/hive/HiveWriterFactory.java | 2 +- .../hive/metastore/HivePartitionName.java | 2 +- .../plugin/hive/metastore/MetastoreUtil.java | 2 +- .../SemiTransactionalHiveMetastore.java | 4 +- .../metastore/cache/CachingHiveMetastore.java | 2 +- .../metastore/file/FileHiveMetastore.java | 6 +- .../DefaultGlueColumnStatisticsProvider.java | 2 +- .../metastore/glue/v1/GlueHiveMetastore.java | 5 +- .../thrift/BridgingHiveMetastore.java | 3 +- .../CreateEmptyPartitionProcedure.java | 2 +- .../hive/procedure/DropStatsProcedure.java | 2 +- .../FlushMetadataCacheProcedure.java | 2 +- .../procedure/RegisterPartitionProcedure.java | 2 +- .../UnregisterPartitionProcedure.java | 2 +- .../hive/projection/PartitionProjection.java | 4 +- .../io/trino/plugin/hive/util/HiveUtil.java | 65 +------- .../plugin/hive/util/HiveWriteUtils.java | 2 +- .../plugin/hive/TestHiveFileFormats.java | 2 +- .../TestMetastoreHiveStatisticsProvider.java | 2 +- .../trino/plugin/hive/util/TestHiveUtil.java | 96 ----------- .../plugin/hudi/HudiPageSourceProvider.java | 2 +- .../hudi/partition/HiveHudiPartitionInfo.java | 5 +- 32 files changed, 314 insertions(+), 262 deletions(-) create mode 100644 lib/trino-metastore/src/main/java/io/trino/metastore/Partitions.java create mode 100644 lib/trino-metastore/src/test/java/io/trino/metastore/TestPartitions.java diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/Partition.java b/lib/trino-metastore/src/main/java/io/trino/metastore/Partition.java index ed3f5dddac88..f24395ee56ff 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/Partition.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/Partition.java @@ -21,7 +21,6 @@ import com.google.errorprone.annotations.Immutable; import io.trino.spi.connector.SchemaTableName; -import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.Objects; @@ -221,67 +220,4 @@ public Partition build() return new Partition(databaseName, tableName, values, storageBuilder.build(), columns, parameters); } } - - public static List toPartitionValues(String partitionName) - { - // mimics Warehouse.makeValsFromName - ImmutableList.Builder resultBuilder = ImmutableList.builder(); - int start = 0; - while (true) { - while (start < partitionName.length() && partitionName.charAt(start) != '=') { - start++; - } - start++; - int end = start; - while (end < partitionName.length() && partitionName.charAt(end) != '/') { - end++; - } - if (start > partitionName.length()) { - break; - } - resultBuilder.add(unescapePathName(partitionName.substring(start, end))); - start = end + 1; - } - return resultBuilder.build(); - } - - // copy of org.apache.hadoop.hive.common.FileUtils#unescapePathName - @SuppressWarnings("NumericCastThatLosesPrecision") - public static String unescapePathName(String path) - { - // fast path, no escaped characters and therefore no copying necessary - int escapedAtIndex = path.indexOf('%'); - if (escapedAtIndex < 0 || escapedAtIndex + 2 >= path.length()) { - return path; - } - - // slow path, unescape into a new string copy - StringBuilder sb = new StringBuilder(); - int fromIndex = 0; - while (escapedAtIndex >= 0 && escapedAtIndex + 2 < path.length()) { - // preceding sequence without escaped characters - if (escapedAtIndex > fromIndex) { - sb.append(path, fromIndex, escapedAtIndex); - } - // try to parse the to digits after the percent sign as hex - try { - int code = HexFormat.fromHexDigits(path, escapedAtIndex + 1, escapedAtIndex + 3); - sb.append((char) code); - // advance past the percent sign and both hex digits - fromIndex = escapedAtIndex + 3; - } - catch (NumberFormatException e) { - // invalid escape sequence, only advance past the percent sign - sb.append('%'); - fromIndex = escapedAtIndex + 1; - } - // find next escaped character - escapedAtIndex = path.indexOf('%', fromIndex); - } - // trailing sequence without escaped characters - if (fromIndex < path.length()) { - sb.append(path, fromIndex, path.length()); - } - return sb.toString(); - } } diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/PartitionWithStatistics.java b/lib/trino-metastore/src/main/java/io/trino/metastore/PartitionWithStatistics.java index 77c136c0e6f1..fa695bb5d410 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/PartitionWithStatistics.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/PartitionWithStatistics.java @@ -14,7 +14,7 @@ package io.trino.metastore; import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.metastore.Partition.toPartitionValues; +import static io.trino.metastore.Partitions.toPartitionValues; import static java.util.Objects.requireNonNull; public class PartitionWithStatistics diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/Partitions.java b/lib/trino-metastore/src/main/java/io/trino/metastore/Partitions.java new file mode 100644 index 000000000000..f6450f32a50a --- /dev/null +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/Partitions.java @@ -0,0 +1,154 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.metastore; + +import com.google.common.base.CharMatcher; +import com.google.common.collect.ImmutableList; + +import java.util.HexFormat; +import java.util.List; + +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Locale.ENGLISH; + +public final class Partitions +{ + public static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; + + private static final HexFormat HEX_UPPER_FORMAT = HexFormat.of().withUpperCase(); + + private static final CharMatcher PATH_CHAR_TO_ESCAPE = CharMatcher.inRange((char) 0, (char) 31) + .or(CharMatcher.anyOf("\"#%'*/:=?\\\u007F{[]^")) + .precomputed(); + + private Partitions() {} + + public static List toPartitionValues(String partitionName) + { + // mimics Warehouse.makeValsFromName + ImmutableList.Builder resultBuilder = ImmutableList.builder(); + int start = 0; + while (true) { + while (start < partitionName.length() && partitionName.charAt(start) != '=') { + start++; + } + start++; + int end = start; + while (end < partitionName.length() && partitionName.charAt(end) != '/') { + end++; + } + if (start > partitionName.length()) { + break; + } + resultBuilder.add(unescapePathName(partitionName.substring(start, end))); + start = end + 1; + } + return resultBuilder.build(); + } + + // copy of org.apache.hadoop.hive.common.FileUtils#unescapePathName + @SuppressWarnings("NumericCastThatLosesPrecision") + public static String unescapePathName(String path) + { + // fast path, no escaped characters and therefore no copying necessary + int escapedAtIndex = path.indexOf('%'); + if (escapedAtIndex < 0 || escapedAtIndex + 2 >= path.length()) { + return path; + } + + // slow path, unescape into a new string copy + StringBuilder sb = new StringBuilder(); + int fromIndex = 0; + while (escapedAtIndex >= 0 && escapedAtIndex + 2 < path.length()) { + // preceding sequence without escaped characters + if (escapedAtIndex > fromIndex) { + sb.append(path, fromIndex, escapedAtIndex); + } + // try to parse the to digits after the percent sign as hex + try { + int code = HexFormat.fromHexDigits(path, escapedAtIndex + 1, escapedAtIndex + 3); + sb.append((char) code); + // advance past the percent sign and both hex digits + fromIndex = escapedAtIndex + 3; + } + catch (NumberFormatException e) { + // invalid escape sequence, only advance past the percent sign + sb.append('%'); + fromIndex = escapedAtIndex + 1; + } + // find next escaped character + escapedAtIndex = path.indexOf('%', fromIndex); + } + // trailing sequence without escaped characters + if (fromIndex < path.length()) { + sb.append(path, fromIndex, path.length()); + } + return sb.toString(); + } + + // copy of org.apache.hadoop.hive.common.FileUtils#escapePathName + public static String escapePathName(String path) + { + if (isNullOrEmpty(path)) { + return HIVE_DEFAULT_DYNAMIC_PARTITION; + } + + // Fast-path detection, no escaping and therefore no copying necessary + int escapeAtIndex = PATH_CHAR_TO_ESCAPE.indexIn(path); + if (escapeAtIndex < 0) { + return path; + } + + // slow path, escape beyond the first required escape character into a new string + StringBuilder sb = new StringBuilder(); + int fromIndex = 0; + while (escapeAtIndex >= 0 && escapeAtIndex < path.length()) { + // preceding characters without escaping needed + if (escapeAtIndex > fromIndex) { + sb.append(path, fromIndex, escapeAtIndex); + } + // escape single character + char c = path.charAt(escapeAtIndex); + sb.append('%').append(HEX_UPPER_FORMAT.toHighHexDigit(c)).append(HEX_UPPER_FORMAT.toLowHexDigit(c)); + // find next character to escape + fromIndex = escapeAtIndex + 1; + if (fromIndex < path.length()) { + escapeAtIndex = PATH_CHAR_TO_ESCAPE.indexIn(path, fromIndex); + } + else { + escapeAtIndex = -1; + } + } + // trailing characters without escaping needed + if (fromIndex < path.length()) { + sb.append(path, fromIndex, path.length()); + } + return sb.toString(); + } + + // copy of org.apache.hadoop.hive.common.FileUtils#makePartName + public static String makePartName(List columns, List values) + { + StringBuilder name = new StringBuilder(); + for (int i = 0; i < columns.size(); i++) { + if (i > 0) { + name.append('/'); + } + name.append(escapePathName(columns.get(i).toLowerCase(ENGLISH))); + name.append('='); + name.append(escapePathName(values.get(i))); + } + return name.toString(); + } +} diff --git a/lib/trino-metastore/src/test/java/io/trino/metastore/TestPartitions.java b/lib/trino-metastore/src/test/java/io/trino/metastore/TestPartitions.java new file mode 100644 index 000000000000..c19a3afc5cd8 --- /dev/null +++ b/lib/trino-metastore/src/test/java/io/trino/metastore/TestPartitions.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.metastore; + +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.junit.jupiter.api.Test; + +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.List; + +import static io.trino.metastore.Partitions.toPartitionValues; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestPartitions +{ + @Test + public void testToPartitionValues() + throws MetaException + { + assertToPartitionValues("ds=2015-12-30/event_type=QueryCompletion"); + assertToPartitionValues("ds=2015-12-30"); + assertToPartitionValues("a=1/b=2/c=3"); + assertToPartitionValues("a=1"); + assertToPartitionValues("pk=!@%23$%25%5E&%2A()%2F%3D"); + assertToPartitionValues("pk=__HIVE_DEFAULT_PARTITION__"); + } + + private static void assertToPartitionValues(String partitionName) + throws MetaException + { + List actual = toPartitionValues(partitionName); + AbstractList expected = new ArrayList<>(); + actual.forEach(s -> expected.add(null)); + Warehouse.makeValsFromName(partitionName, expected); + assertThat(actual).isEqualTo(expected); + } + + @Test + public void testUnescapePathName() + { + assertUnescapePathName("", ""); + assertUnescapePathName("x", "x"); + assertUnescapePathName("abc", "abc"); + assertUnescapePathName("abc%", "abc%"); + assertUnescapePathName("%", "%"); + assertUnescapePathName("%41", "A"); + assertUnescapePathName("%41%x", "A%x"); + assertUnescapePathName("%41%xxZ", "A%xxZ"); + assertUnescapePathName("%41%%Z", "A%%Z"); + assertUnescapePathName("%41%25%25Z", "A%%Z"); + assertUnescapePathName("abc%41%42%43", "abcABC"); + assertUnescapePathName("abc%3Axyz", "abc:xyz"); + assertUnescapePathName("abc%3axyz", "abc:xyz"); + assertUnescapePathName("abc%BBxyz", "abc\u00BBxyz"); + } + + private static void assertUnescapePathName(String value, String expected) + { + assertThat(FileUtils.unescapePathName(value)).isEqualTo(expected); + assertThat(Partitions.unescapePathName(value)).isEqualTo(expected); + } + + @Test + public void testEscapePathName() + { + assertEscapePathName(null, "__HIVE_DEFAULT_PARTITION__"); + assertEscapePathName("", "__HIVE_DEFAULT_PARTITION__"); + assertEscapePathName("x", "x"); + assertEscapePathName("abc", "abc"); + assertEscapePathName("%", "%25"); + assertEscapePathName("A", "A"); + assertEscapePathName("A%x", "A%25x"); + assertEscapePathName("A%xxZ", "A%25xxZ"); + assertEscapePathName("A%%Z", "A%25%25Z"); + assertEscapePathName("abcABC", "abcABC"); + assertEscapePathName("abc:xyz", "abc%3Axyz"); + assertEscapePathName("abc\u00BBxyz", "abc\u00BBxyz"); + assertEscapePathName("\u0000\t\b\r\n\u001F", "%00%09%08%0D%0A%1F"); + assertEscapePathName("#%^&*=[]{\\:'\"/?", "%23%25%5E&%2A%3D%5B%5D%7B%5C%3A%27%22%2F%3F"); + assertEscapePathName("~`!@$()-_+}|;,.<>", "~`!@$()-_+}|;,.<>"); + } + + private static void assertEscapePathName(String value, String expected) + { + assertThat(FileUtils.escapePathName(value)).isEqualTo(expected); + assertThat(Partitions.escapePathName(value)).isEqualTo(expected); + } + + @Test + public void testMakePartName() + { + assertMakePartName(List.of("abc"), List.of("xyz"), "abc=xyz"); + assertMakePartName(List.of("abc:qqq"), List.of("xyz/yyy=zzz"), "abc%3Aqqq=xyz%2Fyyy%3Dzzz"); + assertMakePartName(List.of("abc", "def", "xyz"), List.of("qqq", "rrr", "sss"), "abc=qqq/def=rrr/xyz=sss"); + } + + private static void assertMakePartName(List columns, List values, String expected) + { + assertThat(FileUtils.makePartName(columns, values)).isEqualTo(expected); + assertThat(Partitions.makePartName(columns, values)).isEqualTo(expected); + } +} diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/AbstractDeltaLakePageSink.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/AbstractDeltaLakePageSink.java index 14a0f162b55a..0029fda7970d 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/AbstractDeltaLakePageSink.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/AbstractDeltaLakePageSink.java @@ -24,12 +24,11 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.Partitions; import io.trino.parquet.writer.ParquetWriterOptions; import io.trino.plugin.deltalake.DataFileInfo.DataFileType; import io.trino.plugin.deltalake.util.DeltaLakeWriteUtils; -import io.trino.plugin.hive.HivePartitionKey; import io.trino.plugin.hive.parquet.ParquetFileWriter; -import io.trino.plugin.hive.util.HiveUtil; import io.trino.spi.Page; import io.trino.spi.PageIndexer; import io.trino.spi.PageIndexerFactory; @@ -56,13 +55,14 @@ import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.wrappedBuffer; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static io.trino.metastore.Partitions.escapePathName; import static io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_BAD_WRITE; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getCompressionCodec; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getParquetWriterBlockSize; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getParquetWriterPageSize; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getParquetWriterPageValueCount; import static io.trino.plugin.deltalake.DeltaLakeTypes.toParquetType; -import static io.trino.plugin.hive.util.HiveUtil.escapePathName; import static java.lang.Math.min; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -442,7 +442,7 @@ protected void closeWriter(int writerIndex) } /** - * Copy of {@link HiveUtil#makePartName} modified to preserve case of partition columns. + * Copy of {@link Partitions#makePartName} modified to preserve case of partition columns. */ private static String makePartName(List partitionColumns, List partitionValues) { @@ -464,7 +464,7 @@ private static String makePartName(List partitionColumns, List p public static List createPartitionValues(List partitionColumnTypes, Page partitionColumns, int position) { return DeltaLakeWriteUtils.createPartitionValues(partitionColumnTypes, partitionColumns, position).stream() - .map(value -> value.equals(HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION) ? null : value) + .map(value -> value.equals(HIVE_DEFAULT_DYNAMIC_PARTITION) ? null : value) .collect(toList()); } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/util/DeltaLakeWriteUtils.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/util/DeltaLakeWriteUtils.java index a30846588cf2..33c6f1997d89 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/util/DeltaLakeWriteUtils.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/util/DeltaLakeWriteUtils.java @@ -32,8 +32,8 @@ import java.util.List; import static com.google.common.io.BaseEncoding.base16; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; -import static io.trino.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.BooleanType.BOOLEAN; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveAnalyzeProperties.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveAnalyzeProperties.java index 549fc0c32995..77f25932cbf2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveAnalyzeProperties.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveAnalyzeProperties.java @@ -28,7 +28,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.trino.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY; import static io.trino.spi.type.VarcharType.VARCHAR; import static java.lang.String.format; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 4d0996f6724b..73ff0d872439 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -45,6 +45,7 @@ import io.trino.metastore.HiveType; import io.trino.metastore.Partition; import io.trino.metastore.PartitionStatistics; +import io.trino.metastore.Partitions; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.SortingColumn; import io.trino.metastore.StorageFormat; @@ -173,7 +174,7 @@ import static io.trino.metastore.HiveBasicStatistics.createEmptyStatistics; import static io.trino.metastore.HiveBasicStatistics.createZeroStatistics; import static io.trino.metastore.HiveType.HIVE_STRING; -import static io.trino.metastore.Partition.toPartitionValues; +import static io.trino.metastore.Partitions.toPartitionValues; import static io.trino.metastore.PrincipalPrivileges.NO_PRIVILEGES; import static io.trino.metastore.PrincipalPrivileges.fromHivePrivilegeInfos; import static io.trino.metastore.StatisticsUpdateMode.MERGE_INCREMENTAL; @@ -1694,7 +1695,7 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH .orElseThrow(() -> new TableNotFoundException(tableName)); partitionValuesList = partitionNames .stream() - .map(Partition::toPartitionValues) + .map(Partitions::toPartitionValues) .collect(toImmutableList()); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionKey.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionKey.java index cfc0ee267e2f..a6dd1dcc452a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionKey.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionKey.java @@ -15,14 +15,13 @@ import static io.airlift.slice.SizeOf.estimatedSizeOf; import static io.airlift.slice.SizeOf.instanceSize; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; import static java.util.Objects.requireNonNull; public record HivePartitionKey(String name, String value) { private static final int INSTANCE_SIZE = instanceSize(HivePartitionKey.class); - public static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; - public HivePartitionKey { requireNonNull(name, "name is null"); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionManager.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionManager.java index fd2e7dcb873e..1bfd12de825a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionManager.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePartitionManager.java @@ -38,7 +38,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.metastore.Partition.unescapePathName; +import static io.trino.metastore.Partitions.unescapePathName; import static io.trino.plugin.hive.metastore.MetastoreUtil.computePartitionKeyFilter; import static io.trino.plugin.hive.metastore.MetastoreUtil.toPartitionName; import static io.trino.plugin.hive.util.HiveBucketing.getHiveBucketFilter; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java index 28d60c873c1a..1cdc3935d84d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java @@ -65,6 +65,7 @@ import static com.google.common.collect.MoreCollectors.onlyElement; import static io.trino.hive.formats.HiveClassNames.HIVE_IGNORE_KEY_OUTPUT_FORMAT_CLASS; import static io.trino.metastore.AcidOperation.CREATE_TABLE; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.plugin.hive.HiveCompressionCodecs.selectCompressionCodec; import static io.trino.plugin.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA; @@ -84,7 +85,6 @@ import static io.trino.plugin.hive.util.HiveTypeUtil.getType; import static io.trino.plugin.hive.util.HiveUtil.getColumnNames; import static io.trino.plugin.hive.util.HiveUtil.getColumnTypes; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.plugin.hive.util.HiveWriteUtils.createPartitionValues; import static io.trino.plugin.hive.util.SerdeConstants.LIST_COLUMNS; import static io.trino.plugin.hive.util.SerdeConstants.LIST_COLUMN_TYPES; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HivePartitionName.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HivePartitionName.java index 9c8a9c5ce3cd..9aeba984c4fa 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HivePartitionName.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HivePartitionName.java @@ -23,7 +23,7 @@ import java.util.Optional; import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.metastore.Partition.toPartitionValues; +import static io.trino.metastore.Partitions.toPartitionValues; import static java.util.Objects.requireNonNull; @Immutable diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java index 5280dd5bb578..561a4a336ff7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java @@ -88,12 +88,12 @@ import static io.trino.hive.thrift.metastore.hive_metastoreConstants.META_TABLE_NAME; import static io.trino.hive.thrift.metastore.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS; import static io.trino.hive.thrift.metastore.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.plugin.hive.HiveMetadata.AVRO_SCHEMA_LITERAL_KEY; import static io.trino.plugin.hive.HiveMetadata.AVRO_SCHEMA_URL_KEY; import static io.trino.plugin.hive.HiveSplitManager.PRESTO_OFFLINE; import static io.trino.plugin.hive.HiveStorageFormat.AVRO; import static io.trino.plugin.hive.metastore.SparkMetastoreUtil.getSparkBasicStatistics; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.plugin.hive.util.SerdeConstants.LIST_COLUMN_COMMENTS; import static io.trino.plugin.hive.util.SerdeConstants.SERIALIZATION_LIB; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java index 5f4015494c93..070ea9e0b631 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java @@ -111,7 +111,8 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.trino.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; -import static io.trino.metastore.Partition.toPartitionValues; +import static io.trino.metastore.Partitions.makePartName; +import static io.trino.metastore.Partitions.toPartitionValues; import static io.trino.metastore.PrincipalPrivileges.NO_PRIVILEGES; import static io.trino.metastore.StatisticsUpdateMode.MERGE_INCREMENTAL; import static io.trino.metastore.StatisticsUpdateMode.OVERWRITE_ALL; @@ -135,7 +136,6 @@ import static io.trino.plugin.hive.metastore.SparkMetastoreUtil.getSparkTableStatistics; import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getPartitionProjectionFromTable; import static io.trino.plugin.hive.util.AcidTables.isTransactionalTable; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.plugin.hive.util.HiveWriteUtils.isFileCreatedByQuery; import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java index f3cf6ee5db16..b8c44879c49f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java @@ -85,13 +85,13 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.trino.cache.CacheUtils.invalidateAllIf; import static io.trino.cache.CacheUtils.uncheckedCacheGet; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.plugin.hive.metastore.HivePartitionName.hivePartitionName; import static io.trino.plugin.hive.metastore.HiveTableName.hiveTableName; import static io.trino.plugin.hive.metastore.PartitionFilter.partitionFilter; import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType.OTHER; import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType.PARTITION; import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType.STATS; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java index 7f6efda69f4b..68b3b0edcc2f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java @@ -93,8 +93,9 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.hash.Hashing.sha256; import static io.trino.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; -import static io.trino.metastore.Partition.toPartitionValues; -import static io.trino.metastore.Partition.unescapePathName; +import static io.trino.metastore.Partitions.escapePathName; +import static io.trino.metastore.Partitions.toPartitionValues; +import static io.trino.metastore.Partitions.unescapePathName; import static io.trino.metastore.Table.TABLE_COMMENT; import static io.trino.plugin.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED; import static io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR; @@ -116,7 +117,6 @@ import static io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig.VersionCompatibility.UNSAFE_ASSUME_COMPATIBILITY; import static io.trino.plugin.hive.util.HiveUtil.DELTA_LAKE_PROVIDER; import static io.trino.plugin.hive.util.HiveUtil.SPARK_TABLE_PROVIDER_KEY; -import static io.trino.plugin.hive.util.HiveUtil.escapePathName; import static io.trino.plugin.hive.util.HiveUtil.escapeSchemaName; import static io.trino.plugin.hive.util.HiveUtil.escapeTableName; import static io.trino.plugin.hive.util.HiveUtil.isIcebergTable; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java index 7dc27115fa8f..fb3e912956a2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java @@ -53,7 +53,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.difference; import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.trino.metastore.Partition.toPartitionValues; +import static io.trino.metastore.Partitions.toPartitionValues; import static io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static io.trino.plugin.hive.HiveErrorCode.HIVE_PARTITION_NOT_FOUND; import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueStatConverter.fromGlueColumnStatistics; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java index 5734df27bd79..10ad2d54c40b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java @@ -85,6 +85,7 @@ import io.trino.metastore.Partition; import io.trino.metastore.PartitionStatistics; import io.trino.metastore.PartitionWithStatistics; +import io.trino.metastore.Partitions; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; @@ -136,7 +137,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.trino.metastore.Partition.toPartitionValues; +import static io.trino.metastore.Partitions.toPartitionValues; import static io.trino.metastore.Table.TABLE_COMMENT; import static io.trino.plugin.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA; @@ -857,7 +858,7 @@ private Map> getPartitionsByNamesInternal(Table tabl List partitions = batchGetPartition(table, partitionNames); Map> partitionNameToPartitionValuesMap = partitionNames.stream() - .collect(toMap(identity(), Partition::toPartitionValues)); + .collect(toMap(identity(), Partitions::toPartitionValues)); Map, Partition> partitionValuesToPartitionMap = partitions.stream() .collect(toMap(Partition::getValues, identity())); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java index 7d1efdafe889..ecb79587b170 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java @@ -30,6 +30,7 @@ import io.trino.metastore.Partition; import io.trino.metastore.PartitionStatistics; import io.trino.metastore.PartitionWithStatistics; +import io.trino.metastore.Partitions; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; @@ -388,7 +389,7 @@ public Map> getPartitionsByNames(Table table, List> partitionNameToPartitionValuesMap = partitionNames.stream() - .collect(Collectors.toMap(identity(), Partition::toPartitionValues)); + .collect(Collectors.toMap(identity(), Partitions::toPartitionValues)); Map, Partition> partitionValuesToPartitionMap = delegate.getPartitionsByNames(table.getDatabaseName(), table.getTableName(), partitionNames).stream() .map(partition -> fromMetastoreApiPartition(table, partition)) .collect(Collectors.toMap(Partition::getValues, identity())); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java index a8f20e585d7d..97044f95cdc6 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java @@ -45,8 +45,8 @@ import java.util.Optional; import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.plugin.base.util.Procedures.checkProcedureArgument; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; import static io.trino.spi.StandardErrorCode.INVALID_PROCEDURE_ARGUMENT; import static io.trino.spi.connector.RetryMode.NO_RETRIES; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java index e084fac9a238..add2f10235e8 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java @@ -43,9 +43,9 @@ import java.util.OptionalLong; import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.metastore.StatisticsUpdateMode.CLEAR_ALL; import static io.trino.plugin.base.util.Procedures.checkProcedureArgument; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.spi.StandardErrorCode.INVALID_PROCEDURE_ARGUMENT; import static io.trino.spi.type.VarcharType.VARCHAR; import static java.lang.String.format; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java index 78a7e63931f1..80b9a544dcc2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java @@ -41,7 +41,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.spi.type.VarcharType.VARCHAR; import static java.lang.String.format; import static java.lang.invoke.MethodHandles.lookup; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/RegisterPartitionProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/RegisterPartitionProcedure.java index ff0ecee88e85..dbf699288e1f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/RegisterPartitionProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/RegisterPartitionProcedure.java @@ -42,12 +42,12 @@ import java.util.List; import java.util.Optional; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.plugin.base.util.Procedures.checkProcedureArgument; import static io.trino.plugin.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static io.trino.plugin.hive.HiveMetadata.TRINO_QUERY_ID_NAME; import static io.trino.plugin.hive.procedure.Procedures.checkIsPartitionedTable; import static io.trino.plugin.hive.procedure.Procedures.checkPartitionColumns; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; import static io.trino.spi.StandardErrorCode.INVALID_PROCEDURE_ARGUMENT; import static io.trino.spi.StandardErrorCode.PERMISSION_DENIED; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/UnregisterPartitionProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/UnregisterPartitionProcedure.java index 111c4e57f0f4..8ef9d3a5de69 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/UnregisterPartitionProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/UnregisterPartitionProcedure.java @@ -34,10 +34,10 @@ import java.lang.invoke.MethodHandle; import java.util.List; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.plugin.base.util.Procedures.checkProcedureArgument; import static io.trino.plugin.hive.procedure.Procedures.checkIsPartitionedTable; import static io.trino.plugin.hive.procedure.Procedures.checkPartitionColumns; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.spi.StandardErrorCode.NOT_FOUND; import static io.trino.spi.type.VarcharType.VARCHAR; import static java.lang.String.format; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjection.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjection.java index b6c22d7d8143..4f29ed1d63eb 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjection.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/projection/PartitionProjection.java @@ -33,9 +33,9 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.cartesianProduct; -import static io.trino.metastore.Partition.toPartitionValues; +import static io.trino.metastore.Partitions.escapePathName; +import static io.trino.metastore.Partitions.toPartitionValues; import static io.trino.plugin.hive.projection.InvalidProjectionException.invalidProjectionMessage; -import static io.trino.plugin.hive.util.HiveUtil.escapePathName; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java index a4cf437aa3b4..cf4339f11bd2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveUtil.java @@ -60,7 +60,6 @@ import org.joda.time.format.DateTimePrinter; import java.math.BigDecimal; -import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.Optional; @@ -78,6 +77,8 @@ import static io.trino.hive.formats.HiveClassNames.HUDI_REALTIME_INPUT_FORMAT; import static io.trino.hive.thrift.metastore.hive_metastoreConstants.FILE_INPUT_FORMAT; import static io.trino.metastore.HiveType.toHiveTypes; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static io.trino.metastore.Partitions.escapePathName; import static io.trino.metastore.SortingColumn.Order.ASCENDING; import static io.trino.metastore.SortingColumn.Order.DESCENDING; import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; @@ -101,7 +102,6 @@ import static io.trino.plugin.hive.HiveMetadata.PARQUET_BLOOM_FILTER_COLUMNS_KEY; import static io.trino.plugin.hive.HiveMetadata.SKIP_FOOTER_COUNT_KEY; import static io.trino.plugin.hive.HiveMetadata.SKIP_HEADER_COUNT_KEY; -import static io.trino.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.trino.plugin.hive.HiveTableProperties.ORC_BLOOM_FILTER_FPP; import static io.trino.plugin.hive.projection.PartitionProjectionProperties.getPartitionProjectionTrinoColumnProperties; @@ -149,8 +149,6 @@ public final class HiveUtil public static final String ICEBERG_TABLE_TYPE_NAME = "table_type"; public static final String ICEBERG_TABLE_TYPE_VALUE = "iceberg"; - private static final HexFormat HEX_UPPER_FORMAT = HexFormat.of().withUpperCase(); - private static final LocalDateTime EPOCH_DAY = new LocalDateTime(1970, 1, 1, 0, 0); private static final DateTimeFormatter HIVE_DATE_PARSER; private static final DateTimeFormatter HIVE_TIMESTAMP_PARSER; @@ -159,10 +157,6 @@ public final class HiveUtil private static final Splitter COLUMN_NAMES_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); - private static final CharMatcher PATH_CHAR_TO_ESCAPE = CharMatcher.inRange((char) 0, (char) 31) - .or(CharMatcher.anyOf("\"#%'*/:=?\\\u007F{[]^")) - .precomputed(); - private static final CharMatcher DOT_MATCHER = CharMatcher.is('.'); public static String splitError(Throwable t, Location location, long start, long length) @@ -879,59 +873,4 @@ public static String escapeTableName(String tableName) } return escapePathName(tableName); } - - // copy of org.apache.hadoop.hive.common.FileUtils#escapePathName - public static String escapePathName(String path) - { - if (isNullOrEmpty(path)) { - return HIVE_DEFAULT_DYNAMIC_PARTITION; - } - - // Fast-path detection, no escaping and therefore no copying necessary - int escapeAtIndex = PATH_CHAR_TO_ESCAPE.indexIn(path); - if (escapeAtIndex < 0) { - return path; - } - - // slow path, escape beyond the first required escape character into a new string - StringBuilder sb = new StringBuilder(); - int fromIndex = 0; - while (escapeAtIndex >= 0 && escapeAtIndex < path.length()) { - // preceding characters without escaping needed - if (escapeAtIndex > fromIndex) { - sb.append(path, fromIndex, escapeAtIndex); - } - // escape single character - char c = path.charAt(escapeAtIndex); - sb.append('%').append(HEX_UPPER_FORMAT.toHighHexDigit(c)).append(HEX_UPPER_FORMAT.toLowHexDigit(c)); - // find next character to escape - fromIndex = escapeAtIndex + 1; - if (fromIndex < path.length()) { - escapeAtIndex = PATH_CHAR_TO_ESCAPE.indexIn(path, fromIndex); - } - else { - escapeAtIndex = -1; - } - } - // trailing characters without escaping needed - if (fromIndex < path.length()) { - sb.append(path, fromIndex, path.length()); - } - return sb.toString(); - } - - // copy of org.apache.hadoop.hive.common.FileUtils#makePartName - public static String makePartName(List columns, List values) - { - StringBuilder name = new StringBuilder(); - for (int i = 0; i < columns.size(); i++) { - if (i > 0) { - name.append('/'); - } - name.append(escapePathName(columns.get(i).toLowerCase(ENGLISH))); - name.append('='); - name.append(escapePathName(values.get(i))); - } - return name.toString(); - } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveWriteUtils.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveWriteUtils.java index 8744cc44283c..fc6cf307e54a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveWriteUtils.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveWriteUtils.java @@ -54,10 +54,10 @@ import java.util.Optional; import static com.google.common.io.BaseEncoding.base16; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.hive.HiveErrorCode.HIVE_DATABASE_LOCATION_ERROR; import static io.trino.plugin.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; -import static io.trino.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.hive.TableType.MANAGED_TABLE; import static io.trino.plugin.hive.TableType.MATERIALIZED_VIEW; import static io.trino.plugin.hive.metastore.MetastoreUtil.getProtectMode; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java index 332f2a3ae9bb..d4efe5fe6fc3 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java @@ -133,12 +133,12 @@ import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.base.type.TrinoTimestampEncoderFactory.createTimestampEncoder; import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.REGULAR; import static io.trino.plugin.hive.HiveColumnHandle.createBaseColumn; import static io.trino.plugin.hive.HivePageSourceProvider.ColumnMapping.buildColumnMappings; -import static io.trino.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.hive.HiveStorageFormat.AVRO; import static io.trino.plugin.hive.HiveStorageFormat.CSV; import static io.trino.plugin.hive.HiveStorageFormat.JSON; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/statistics/TestMetastoreHiveStatisticsProvider.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/statistics/TestMetastoreHiveStatisticsProvider.java index 61845ce0bac8..f74ea30fb771 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/statistics/TestMetastoreHiveStatisticsProvider.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/statistics/TestMetastoreHiveStatisticsProvider.java @@ -53,11 +53,11 @@ import static io.trino.metastore.HivePartition.UNPARTITIONED_ID; import static io.trino.metastore.HiveType.HIVE_LONG; import static io.trino.metastore.HiveType.HIVE_STRING; +import static io.trino.metastore.Partitions.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.REGULAR; import static io.trino.plugin.hive.HiveColumnHandle.createBaseColumn; import static io.trino.plugin.hive.HiveErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; -import static io.trino.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static io.trino.plugin.hive.HivePartitionManager.parsePartition; import static io.trino.plugin.hive.HiveTestUtils.SESSION; import static io.trino.plugin.hive.HiveTestUtils.getHiveSession; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveUtil.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveUtil.java index 6630f9b96858..b4b80d360c56 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveUtil.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveUtil.java @@ -13,20 +13,11 @@ */ package io.trino.plugin.hive.util; -import io.trino.metastore.Partition; -import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.junit.jupiter.api.Test; -import java.util.AbstractList; -import java.util.ArrayList; -import java.util.List; - -import static io.trino.metastore.Partition.toPartitionValues; import static io.trino.plugin.hive.util.HiveUtil.escapeSchemaName; import static io.trino.plugin.hive.util.HiveUtil.escapeTableName; import static io.trino.plugin.hive.util.HiveUtil.parseHiveTimestamp; @@ -47,43 +38,6 @@ public void testParseHiveTimestamp() assertThat(parse(time, "yyyy-MM-dd HH:mm:ss.SSSSSSSSS")).isEqualTo(unixTime(time, 7)); } - @Test - public void testToPartitionValues() - throws MetaException - { - assertToPartitionValues("ds=2015-12-30/event_type=QueryCompletion"); - assertToPartitionValues("ds=2015-12-30"); - assertToPartitionValues("a=1/b=2/c=3"); - assertToPartitionValues("a=1"); - assertToPartitionValues("pk=!@%23$%25%5E&%2A()%2F%3D"); - assertToPartitionValues("pk=__HIVE_DEFAULT_PARTITION__"); - } - - @Test - public void testUnescapePathName() - { - assertUnescapePathName("", ""); - assertUnescapePathName("x", "x"); - assertUnescapePathName("abc", "abc"); - assertUnescapePathName("abc%", "abc%"); - assertUnescapePathName("%", "%"); - assertUnescapePathName("%41", "A"); - assertUnescapePathName("%41%x", "A%x"); - assertUnescapePathName("%41%xxZ", "A%xxZ"); - assertUnescapePathName("%41%%Z", "A%%Z"); - assertUnescapePathName("%41%25%25Z", "A%%Z"); - assertUnescapePathName("abc%41%42%43", "abcABC"); - assertUnescapePathName("abc%3Axyz", "abc:xyz"); - assertUnescapePathName("abc%3axyz", "abc:xyz"); - assertUnescapePathName("abc%BBxyz", "abc\u00BBxyz"); - } - - private static void assertUnescapePathName(String value, String expected) - { - assertThat(FileUtils.unescapePathName(value)).isEqualTo(expected); - assertThat(Partition.unescapePathName(value)).isEqualTo(expected); - } - @Test public void testEscapeDatabaseName() { @@ -108,56 +62,6 @@ public void testEscapeTableName() assertThat(escapeTableName("../../table1")).isEqualTo("..%2F..%2Ftable1"); } - @Test - public void testEscapePathName() - { - assertEscapePathName(null, "__HIVE_DEFAULT_PARTITION__"); - assertEscapePathName("", "__HIVE_DEFAULT_PARTITION__"); - assertEscapePathName("x", "x"); - assertEscapePathName("abc", "abc"); - assertEscapePathName("%", "%25"); - assertEscapePathName("A", "A"); - assertEscapePathName("A%x", "A%25x"); - assertEscapePathName("A%xxZ", "A%25xxZ"); - assertEscapePathName("A%%Z", "A%25%25Z"); - assertEscapePathName("abcABC", "abcABC"); - assertEscapePathName("abc:xyz", "abc%3Axyz"); - assertEscapePathName("abc\u00BBxyz", "abc\u00BBxyz"); - assertEscapePathName("\u0000\t\b\r\n\u001F", "%00%09%08%0D%0A%1F"); - assertEscapePathName("#%^&*=[]{\\:'\"/?", "%23%25%5E&%2A%3D%5B%5D%7B%5C%3A%27%22%2F%3F"); - assertEscapePathName("~`!@$()-_+}|;,.<>", "~`!@$()-_+}|;,.<>"); - } - - private static void assertEscapePathName(String value, String expected) - { - assertThat(FileUtils.escapePathName(value)).isEqualTo(expected); - assertThat(HiveUtil.escapePathName(value)).isEqualTo(expected); - } - - @Test - public void testMakePartName() - { - assertMakePartName(List.of("abc"), List.of("xyz"), "abc=xyz"); - assertMakePartName(List.of("abc:qqq"), List.of("xyz/yyy=zzz"), "abc%3Aqqq=xyz%2Fyyy%3Dzzz"); - assertMakePartName(List.of("abc", "def", "xyz"), List.of("qqq", "rrr", "sss"), "abc=qqq/def=rrr/xyz=sss"); - } - - private static void assertMakePartName(List columns, List values, String expected) - { - assertThat(FileUtils.makePartName(columns, values)).isEqualTo(expected); - assertThat(HiveUtil.makePartName(columns, values)).isEqualTo(expected); - } - - private static void assertToPartitionValues(String partitionName) - throws MetaException - { - List actual = toPartitionValues(partitionName); - AbstractList expected = new ArrayList<>(); - actual.forEach(s -> expected.add(null)); - Warehouse.makeValsFromName(partitionName, expected); - assertThat(actual).isEqualTo(expected); - } - private static long parse(DateTime time, String pattern) { return parseHiveTimestamp(DateTimeFormat.forPattern(pattern).print(time)); diff --git a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java index bee386c71553..3bb7b1c42a79 100644 --- a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java +++ b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java @@ -70,6 +70,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.slice.Slices.utf8Slice; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.trino.metastore.Partitions.makePartName; import static io.trino.parquet.ParquetTypeUtils.getColumnIO; import static io.trino.parquet.ParquetTypeUtils.getDescriptors; import static io.trino.parquet.predicate.PredicateUtils.buildPredicate; @@ -80,7 +81,6 @@ import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.createParquetPageSource; import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.getParquetMessageType; import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.getParquetTupleDomain; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; import static io.trino.plugin.hudi.HudiErrorCode.HUDI_BAD_DATA; import static io.trino.plugin.hudi.HudiErrorCode.HUDI_CANNOT_OPEN_SPLIT; import static io.trino.plugin.hudi.HudiErrorCode.HUDI_CURSOR_ERROR; diff --git a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java index 7efa635f4456..906be1436c0f 100644 --- a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java +++ b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java @@ -29,6 +29,7 @@ import java.util.Optional; import static com.google.common.base.MoreObjects.toStringHelper; +import static io.trino.metastore.Partitions.toPartitionValues; import static io.trino.plugin.hudi.HudiErrorCode.HUDI_PARTITION_NOT_FOUND; import static io.trino.plugin.hudi.HudiUtil.buildPartitionKeys; import static io.trino.plugin.hudi.HudiUtil.partitionMatchesPredicates; @@ -72,7 +73,7 @@ public HiveHudiPartitionInfo( public String getRelativePartitionPath() { if (relativePartitionPath == null) { - loadPartitionInfo(hiveMetastore.getPartition(table, Partition.toPartitionValues(hivePartitionName))); + loadPartitionInfo(hiveMetastore.getPartition(table, toPartitionValues(hivePartitionName))); } return relativePartitionPath; } @@ -81,7 +82,7 @@ public String getRelativePartitionPath() public List getHivePartitionKeys() { if (hivePartitionKeys == null) { - loadPartitionInfo(hiveMetastore.getPartition(table, Partition.toPartitionValues(hivePartitionName))); + loadPartitionInfo(hiveMetastore.getPartition(table, toPartitionValues(hivePartitionName))); } return hivePartitionKeys; } From 7e3aadaf10830b3fea947aa48377ad76f39931c9 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Tue, 30 Jul 2024 23:49:14 -0700 Subject: [PATCH 121/158] Move HiveMetastoreFactory to metastore module --- .../main/java/io/trino}/metastore/HiveMetastoreFactory.java | 3 +-- .../io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java | 2 +- .../file/DeltaLakeFileMetastoreTableOperationsProvider.java | 2 +- .../DeltaLakeThriftMetastoreTableOperationsProvider.java | 2 +- .../deltalake/BaseDeltaLakeRegisterTableProcedureTest.java | 2 +- .../plugin/deltalake/BaseDeltaLakeTableWithCustomLocation.java | 2 +- .../io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java | 2 +- .../io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java | 2 +- .../java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java | 2 +- .../TestDeltaLakeMinioAndLockBasedSynchronizerSmokeTest.java | 2 +- .../plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java | 2 +- .../io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java | 2 +- .../metastore/TestDeltaLakeMetastoreAccessOperations.java | 2 +- .../deltalake/metastore/TestingDeltaLakeMetastoreModule.java | 2 +- .../deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java | 2 +- .../main/java/io/trino/plugin/hive/HiveMetadataFactory.java | 2 +- .../main/java/io/trino/plugin/hive/HivePageSinkProvider.java | 2 +- .../plugin/hive/metastore/CachingHiveMetastoreModule.java | 1 + .../io/trino/plugin/hive/metastore/HiveMetastoreModule.java | 1 + .../plugin/hive/metastore/cache/SharedHiveMetastoreCache.java | 2 +- .../plugin/hive/metastore/file/FileHiveMetastoreFactory.java | 2 +- .../trino/plugin/hive/metastore/file/FileMetastoreModule.java | 2 +- .../plugin/hive/metastore/glue/GlueHiveMetastoreFactory.java | 2 +- .../trino/plugin/hive/metastore/glue/GlueMetastoreModule.java | 2 +- .../hive/metastore/glue/v1/GlueHiveMetastoreFactory.java | 2 +- .../plugin/hive/metastore/glue/v1/GlueMetastoreModule.java | 2 +- .../hive/metastore/thrift/BridgingHiveMetastoreFactory.java | 2 +- .../plugin/hive/metastore/thrift/ThriftMetastoreModule.java | 2 +- .../plugin/hive/procedure/FlushMetadataCacheProcedure.java | 2 +- .../test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java | 2 +- .../src/test/java/io/trino/plugin/hive/HiveQueryRunner.java | 2 +- .../plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java | 2 +- .../src/test/java/io/trino/plugin/hive/TestHivePageSink.java | 2 +- .../trino/plugin/hive/fs/BaseCachingDirectoryListerTest.java | 2 +- .../cache/TestCachingHiveMetastoreWithQueryRunner.java | 2 +- .../hive/metastore/thrift/TestHiveMetastoreCatalogs.java | 2 +- .../TestHiveMetastoreMetadataQueriesAccessOperations.java | 2 +- .../hive/optimizer/TestConnectorPushdownRulesWithHive.java | 2 +- .../java/io/trino/plugin/hive/optimizer/TestHivePlans.java | 2 +- .../optimizer/TestHiveProjectionPushdownIntoTableScan.java | 2 +- .../main/java/io/trino/plugin/hudi/HudiMetadataFactory.java | 2 +- .../src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java | 2 +- .../plugin/hudi/testing/ResourceHudiTablesInitializer.java | 2 +- .../trino/plugin/hudi/testing/TpchHudiTablesInitializer.java | 2 +- .../src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java | 2 +- .../java/io/trino/plugin/iceberg/IcebergMetadataFactory.java | 2 +- .../src/main/java/io/trino/plugin/iceberg/IcebergModule.java | 2 +- .../plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java | 2 +- .../io/trino/plugin/iceberg/procedure/MigrateProcedure.java | 2 +- .../java/io/trino/plugin/iceberg/procedure/MigrationUtils.java | 2 +- .../test/java/io/trino/plugin/iceberg/IcebergTestUtils.java | 2 +- .../io/trino/plugin/iceberg/TestIcebergMetadataListing.java | 2 +- .../plugin/iceberg/TestIcebergProjectionPushdownPlans.java | 2 +- .../io/trino/plugin/iceberg/TestMetadataQueryOptimization.java | 2 +- .../catalog/file/TestingIcebergFileMetastoreCatalogModule.java | 2 +- .../optimizer/TestConnectorPushdownRulesWithIceberg.java | 2 +- .../io/trino/sql/planner/IcebergCostBasedPlanTestSetup.java | 2 +- 57 files changed, 57 insertions(+), 56 deletions(-) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino}/metastore/HiveMetastoreFactory.java (95%) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreFactory.java b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastoreFactory.java similarity index 95% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreFactory.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastoreFactory.java index cab15145c2df..5b0564c8ffe0 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreFactory.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastoreFactory.java @@ -11,9 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore; +package io.trino.metastore; -import io.trino.metastore.HiveMetastore; import io.trino.spi.security.ConnectorIdentity; import java.util.Optional; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java index 49cfc79e0c52..dc1241ed504d 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java @@ -17,6 +17,7 @@ import io.airlift.concurrent.BoundedExecutor; import io.airlift.json.JsonCodec; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.deltalake.metastore.DeltaLakeTableMetadataScheduler; import io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore; import io.trino.plugin.deltalake.statistics.CachingExtendedStatisticsAccess; @@ -26,7 +27,6 @@ import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.security.AccessControlMetadata; import io.trino.spi.NodeManager; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreTableOperationsProvider.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreTableOperationsProvider.java index 898679cb6497..5a7ecbdee9ac 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreTableOperationsProvider.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreTableOperationsProvider.java @@ -14,9 +14,9 @@ package io.trino.plugin.deltalake.metastore.file; import com.google.inject.Inject; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.deltalake.metastore.DeltaLakeTableOperations; import io.trino.plugin.deltalake.metastore.DeltaLakeTableOperationsProvider; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.connector.ConnectorSession; import java.util.Optional; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreTableOperationsProvider.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreTableOperationsProvider.java index 9ff99a10cc55..d170376601ed 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreTableOperationsProvider.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreTableOperationsProvider.java @@ -14,9 +14,9 @@ package io.trino.plugin.deltalake.metastore.thrift; import com.google.inject.Inject; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.deltalake.metastore.DeltaLakeTableOperations; import io.trino.plugin.deltalake.metastore.DeltaLakeTableOperationsProvider; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreFactory; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.security.ConnectorIdentity; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java index 471b6c212d62..843d708028e1 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeRegisterTableProcedureTest.java @@ -17,7 +17,7 @@ import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.spi.security.ConnectorIdentity; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeTableWithCustomLocation.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeTableWithCustomLocation.java index ddbdfbac1148..46fa3e58002a 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeTableWithCustomLocation.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeTableWithCustomLocation.java @@ -16,8 +16,8 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.MaterializedRow; import org.junit.jupiter.api.Test; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java index 11f1544e92bc..c16c2c1c641c 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java @@ -24,10 +24,10 @@ import io.trino.Session; import io.trino.execution.QueryInfo; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; import io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.ColumnMappingMode; import io.trino.plugin.hive.HiveCompressionCodec; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.tpch.TpchPlugin; import io.trino.spi.connector.ColumnMetadata; import io.trino.sql.planner.plan.FilterNode; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java index 38e348644f4e..98230e78aa5d 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java @@ -21,9 +21,9 @@ import io.trino.Session; import io.trino.SystemSessionProperties; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; import io.trino.plugin.deltalake.metastore.DeltaLakeTableMetadataScheduler; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java index fbecc7fa09be..8ffa699674f9 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java @@ -30,13 +30,13 @@ import io.trino.hdfs.HdfsEnvironment; import io.trino.hdfs.TrinoHdfsFileSystemStats; import io.trino.metastore.Database; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastore; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule; import io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore; import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; import io.trino.plugin.hive.NodeVersion; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.spi.NodeManager; import io.trino.spi.PageIndexerFactory; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMinioAndLockBasedSynchronizerSmokeTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMinioAndLockBasedSynchronizerSmokeTest.java index a3e4e6842e16..9bb7240b94a1 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMinioAndLockBasedSynchronizerSmokeTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMinioAndLockBasedSynchronizerSmokeTest.java @@ -19,8 +19,8 @@ import io.airlift.json.ObjectMapperProvider; import io.airlift.units.Duration; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.deltalake.transactionlog.writer.S3LockBasedTransactionLogSynchronizer; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.tpch.TpchPlugin; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java index 93f0741f7ae0..755a065daea2 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeProjectionPushdownPlans.java @@ -23,7 +23,7 @@ import io.trino.metadata.TestingFunctionResolution; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.function.OperatorType; import io.trino.spi.predicate.Domain; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java index 779318497889..15ca606fb0e6 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java @@ -22,6 +22,7 @@ import io.trino.filesystem.cache.DefaultCachingHostAddressProvider; import io.trino.filesystem.hdfs.HdfsFileSystemFactory; import io.trino.filesystem.memory.MemoryFileSystemFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.deltalake.metastore.DeltaLakeTableMetadataScheduler; import io.trino.plugin.deltalake.metastore.file.DeltaLakeFileMetastoreTableOperationsProvider; @@ -41,7 +42,6 @@ import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory; import io.trino.plugin.hive.HiveTransactionHandle; import io.trino.plugin.hive.NodeVersion; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hive.parquet.ParquetWriterConfig; import io.trino.spi.SplitWeight; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java index e2f7348e2477..5211e96c5490 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java @@ -20,10 +20,10 @@ import io.opentelemetry.sdk.trace.data.SpanData; import io.trino.Session; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; import io.trino.plugin.deltalake.DeltaLakeQueryRunner; import io.trino.plugin.deltalake.TestingDeltaLakeUtils; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.MetastoreMethod; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java index adea03218e84..98488d0330e4 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java @@ -18,12 +18,12 @@ import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.deltalake.AllowDeltaLakeManagedTableRename; import io.trino.plugin.deltalake.MaxTableParameterLength; import io.trino.plugin.deltalake.metastore.file.DeltaLakeFileMetastoreTableOperationsProvider; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import static java.util.Objects.requireNonNull; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java index 9ffae4586f88..3922dc250f73 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java @@ -28,6 +28,7 @@ import io.trino.metastore.Column; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; import io.trino.plugin.base.session.SessionPropertiesProvider; @@ -36,7 +37,6 @@ import io.trino.plugin.deltalake.DeltaLakeModule; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule; import io.trino.plugin.hive.NodeVersion; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.NodeManager; import io.trino.spi.PageIndexerFactory; import io.trino.spi.TrinoException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java index 136b9ab09267..f63c5dad2b42 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java @@ -20,10 +20,10 @@ import io.airlift.units.Duration; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.fs.TransactionScopeCachingDirectoryListerFactory; import io.trino.plugin.hive.metastore.HiveMetastoreConfig; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.SemiTransactionalHiveMetastore; import io.trino.plugin.hive.security.AccessControlMetadataFactory; import io.trino.plugin.hive.statistics.MetastoreHiveStatisticsProvider; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java index e8e987b1c728..d1030f73de35 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java @@ -21,8 +21,8 @@ import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.SortingColumn; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.HivePageSinkMetadataProvider; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.spi.PageIndexerFactory; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java index d9294b4eb916..3d6182349366 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java @@ -18,6 +18,7 @@ import com.google.inject.Scopes; import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java index d3e8b21e4182..9703885f428b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java @@ -20,6 +20,7 @@ import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.AllowHiveTableRename; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.file.FileMetastoreModule; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java index 364d25a02047..8e2c731b8d92 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java @@ -24,7 +24,7 @@ import com.google.inject.Inject; import io.airlift.units.Duration; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType; import io.trino.spi.NodeManager; import io.trino.spi.TrinoException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java index 28141fb1c668..6d1a9101c7a4 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java @@ -17,10 +17,10 @@ import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.tracing.TracingHiveMetastore; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.NodeVersion; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.ConnectorIdentity; import java.util.Optional; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java index 71b0a7450679..e990434bea34 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java @@ -17,8 +17,8 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Scopes; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.AllowHiveTableRename; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import static io.airlift.configuration.ConfigBinder.configBinder; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastoreFactory.java index 83ca8b6090c7..8b5075e606e6 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastoreFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastoreFactory.java @@ -16,8 +16,8 @@ import com.google.inject.Inject; import io.opentelemetry.api.trace.Tracer; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.tracing.TracingHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.ConnectorIdentity; import java.util.Optional; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java index 584ad46efe92..d1431bb30de7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java @@ -27,9 +27,9 @@ import io.airlift.units.Duration; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.awssdk.v2_2.AwsSdkTelemetry; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.AllowHiveTableRename; import io.trino.plugin.hive.HideDeltaLakeTables; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.spi.NodeManager; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreFactory.java index e52fdcf75c56..8d6412b08f5f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreFactory.java @@ -16,8 +16,8 @@ import com.google.inject.Inject; import io.opentelemetry.api.trace.Tracer; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.tracing.TracingHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.ConnectorIdentity; import java.util.Optional; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java index 5bdd79f6ace3..eddec0839eb6 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java @@ -31,8 +31,8 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.awssdk.v1_11.AwsSdkTelemetry; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.AllowHiveTableRename; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastoreFactory.java index b4875165b79f..632045081656 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastoreFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastoreFactory.java @@ -16,8 +16,8 @@ import com.google.inject.Inject; import io.opentelemetry.api.trace.Tracer; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.tracing.TracingHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.ConnectorIdentity; import java.util.Optional; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java index f3b3c488260d..8c5f0b71b37c 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java @@ -21,10 +21,10 @@ import com.google.inject.TypeLiteral; import com.google.inject.multibindings.OptionalBinder; import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.base.security.UserNameProvider; import io.trino.plugin.hive.AllowHiveTableRename; import io.trino.plugin.hive.ForHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import java.util.concurrent.ExecutorService; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java index 80b9a544dcc2..8cf5ae65c4c7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java @@ -18,10 +18,10 @@ import com.google.inject.Provider; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; import io.trino.plugin.hive.HiveErrorCode; import io.trino.plugin.hive.fs.DirectoryLister; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.glue.GlueCache; import io.trino.plugin.hive.metastore.glue.PartitionName; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java index 2f2e3b53a63b..765e7625827c 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java @@ -32,11 +32,11 @@ import io.trino.metadata.TableMetadata; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.HiveType; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Storage; import io.trino.metastore.Table; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.connector.CatalogSchemaTableName; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.Constraint; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java index 786108fc342a..a84652e6320a 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java @@ -22,7 +22,7 @@ import io.trino.metadata.QualifiedObjectName; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.tpcds.TpcdsPlugin; import io.trino.plugin.tpch.ColumnNaming; import io.trino.plugin.tpch.DecimalTypeMapping; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java index 6d55f9cd915d..ced91cf9a85d 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCustomCatalogConnectorSmokeTest.java @@ -15,9 +15,9 @@ import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.PrincipalType; import io.trino.testing.BaseConnectorSmokeTest; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java index b0bb0e9854ff..af40a28e0d83 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java @@ -25,9 +25,9 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.memory.MemoryFileSystemFactory; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.operator.FlatHashStrategyCompiler; import io.trino.operator.GroupByHashPageIndexerFactory; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.HivePageSinkMetadata; import io.trino.spi.Page; import io.trino.spi.PageBuilder; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/fs/BaseCachingDirectoryListerTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/fs/BaseCachingDirectoryListerTest.java index b37bdddc2825..53324069d5f0 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/fs/BaseCachingDirectoryListerTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/fs/BaseCachingDirectoryListerTest.java @@ -16,10 +16,10 @@ import com.google.common.collect.ImmutableList; import io.trino.filesystem.Location; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; import io.trino.plugin.hive.HiveQueryRunner; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.MaterializedRow; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java index 2bdfd788ff01..6acda90db491 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java @@ -19,8 +19,8 @@ import com.google.inject.Key; import io.trino.Session; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.HiveQueryRunner; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.spi.security.Identity; import io.trino.spi.security.SelectedRole; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java index 23a18de4e7e5..11ab15937fa5 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreCatalogs.java @@ -17,10 +17,10 @@ import io.trino.Session; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.HiveQueryRunner; import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.PrincipalType; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java index 0c6bbcf0d0d9..aa8fcba701c5 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java @@ -21,11 +21,11 @@ import io.trino.metastore.Column; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.HiveType; import io.trino.metastore.Table; import io.trino.plugin.hive.HiveQueryRunner; import io.trino.plugin.hive.containers.HiveHadoop; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.MetastoreMethod; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestConnectorPushdownRulesWithHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestConnectorPushdownRulesWithHive.java index 844787214493..25b6b48853c3 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestConnectorPushdownRulesWithHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestConnectorPushdownRulesWithHive.java @@ -23,12 +23,12 @@ import io.trino.metadata.TestingFunctionResolution; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.HiveColumnProjectionInfo; import io.trino.plugin.hive.HiveTableHandle; import io.trino.plugin.hive.HiveTransactionHandle; import io.trino.plugin.hive.TestingHiveConnectorFactory; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.connector.CatalogHandle; import io.trino.spi.function.OperatorType; import io.trino.spi.predicate.Domain; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHivePlans.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHivePlans.java index 61af8171ee8a..e177d5415f39 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHivePlans.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHivePlans.java @@ -20,8 +20,8 @@ import io.trino.metadata.TestingFunctionResolution; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.TestingHiveConnectorFactory; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.function.OperatorType; import io.trino.spi.security.PrincipalType; import io.trino.sql.ir.Between; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHiveProjectionPushdownIntoTableScan.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHiveProjectionPushdownIntoTableScan.java index 478cdc257609..fa6499bce512 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHiveProjectionPushdownIntoTableScan.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHiveProjectionPushdownIntoTableScan.java @@ -23,10 +23,10 @@ import io.trino.metadata.TestingFunctionResolution; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.HiveTableHandle; import io.trino.plugin.hive.TestingHiveConnectorFactory; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.function.OperatorType; import io.trino.spi.predicate.Domain; diff --git a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java index 4a5fe257a598..1489d191d0a0 100644 --- a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java +++ b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java @@ -15,7 +15,7 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; diff --git a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java index 915ca2b9ead2..e7ebda90a29e 100644 --- a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java +++ b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/HudiQueryRunner.java @@ -19,9 +19,9 @@ import io.airlift.log.Logging; import io.trino.filesystem.Location; import io.trino.metastore.Database; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.base.util.Closables; import io.trino.plugin.hive.containers.Hive3MinioDataLake; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hudi.testing.HudiTablesInitializer; import io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer; import io.trino.plugin.hudi.testing.TpchHudiTablesInitializer; diff --git a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java index 47c7b195abeb..55d7c78b22fb 100644 --- a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java +++ b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java @@ -20,6 +20,7 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.HiveType; import io.trino.metastore.Partition; import io.trino.metastore.PartitionStatistics; @@ -27,7 +28,6 @@ import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.StorageFormat; import io.trino.metastore.Table; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hudi.HudiConnector; import io.trino.spi.security.ConnectorIdentity; import io.trino.testing.QueryRunner; diff --git a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java index fc9d04ced4a6..9987fdba387e 100644 --- a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java +++ b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/testing/TpchHudiTablesInitializer.java @@ -23,11 +23,11 @@ import io.trino.hdfs.HdfsEnvironment; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.HiveType; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.StorageFormat; import io.trino.metastore.Table; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hudi.HudiConnector; import io.trino.plugin.tpch.TpchPlugin; import io.trino.spi.connector.CatalogSchemaName; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 0e3843c56f39..b34240379f8a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -36,6 +36,7 @@ import io.trino.filesystem.TrinoFileSystem; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.TableInfo; import io.trino.plugin.base.classloader.ClassLoaderSafeSystemTable; import io.trino.plugin.base.filter.UtcConstraintExtractor; @@ -43,7 +44,6 @@ import io.trino.plugin.base.projection.ApplyProjectionUtil.ProjectedColumnRepresentation; import io.trino.plugin.hive.HiveStorageFormat; import io.trino.plugin.hive.HiveWrittenPartitions; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.iceberg.aggregation.DataSketchStateSerializer; import io.trino.plugin.iceberg.aggregation.IcebergThetaSketchForStats; import io.trino.plugin.iceberg.catalog.TrinoCatalog; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java index 6b451c241b31..e7fbb29f4a55 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java @@ -18,7 +18,7 @@ import com.google.inject.Inject; import io.airlift.concurrent.BoundedExecutor; import io.airlift.json.JsonCodec; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.spi.connector.CatalogHandle; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index 386f7dd221b9..254a994cb673 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -22,6 +22,7 @@ import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; import io.trino.filesystem.cache.CacheKeyProvider; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSinkProvider; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProviderFactory; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; @@ -30,7 +31,6 @@ import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.hive.SortingFileWriterConfig; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.hive.orc.OrcReaderConfig; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java index 98476c7ac942..d18785562efa 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java @@ -16,9 +16,9 @@ import com.google.inject.Inject; import io.airlift.concurrent.BoundedExecutor; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.ForIcebergMetadata; import io.trino.plugin.iceberg.IcebergConfig; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java index c9718c3ceb92..b56f2b8289f7 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java @@ -23,11 +23,11 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.Column; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Partition; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Storage; import io.trino.plugin.hive.HiveStorageFormat; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergFileFormat; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java index f869928de41d..ab33cb3330ec 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java @@ -23,6 +23,7 @@ import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoInputFile; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Partition; import io.trino.metastore.Storage; import io.trino.parquet.ParquetDataSource; @@ -31,7 +32,6 @@ import io.trino.parquet.reader.MetadataReader; import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.HiveStorageFormat; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.parquet.TrinoParquetDataSource; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.fileio.ForwardingInputFile; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java index 702cd94ff027..c1d4f0a0b485 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java @@ -22,6 +22,7 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.TrinoInputFile; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.orc.OrcDataSource; import io.trino.orc.OrcReader; import io.trino.orc.OrcReaderOptions; @@ -35,7 +36,6 @@ import io.trino.parquet.reader.MetadataReader; import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.parquet.TrinoParquetDataSource; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java index d72e9454fb8e..b1d8ae8afce6 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataListing.java @@ -16,8 +16,8 @@ import com.google.common.collect.ImmutableMap; import io.trino.Session; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.TestingHivePlugin; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.spi.security.Identity; import io.trino.spi.security.SelectedRole; import io.trino.testing.AbstractTestQueryFramework; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergProjectionPushdownPlans.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergProjectionPushdownPlans.java index 25f7f6269e37..e68182fac3df 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergProjectionPushdownPlans.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergProjectionPushdownPlans.java @@ -23,7 +23,7 @@ import io.trino.metadata.TestingFunctionResolution; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.function.OperatorType; import io.trino.spi.predicate.Domain; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestMetadataQueryOptimization.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestMetadataQueryOptimization.java index f38d6fafb324..446b0d5b4c70 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestMetadataQueryOptimization.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestMetadataQueryOptimization.java @@ -18,7 +18,7 @@ import io.trino.Session; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.spi.security.PrincipalType; import io.trino.sql.ir.Constant; import io.trino.sql.planner.assertions.BasePushdownPlanTest; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java index 25a343fe398f..fc02c4b7f82c 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java @@ -18,8 +18,8 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.units.Duration; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/optimizer/TestConnectorPushdownRulesWithIceberg.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/optimizer/TestConnectorPushdownRulesWithIceberg.java index 676f4cf1f7d2..37ebe5d04f8d 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/optimizer/TestConnectorPushdownRulesWithIceberg.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/optimizer/TestConnectorPushdownRulesWithIceberg.java @@ -24,8 +24,8 @@ import io.trino.metadata.TestingFunctionResolution; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.HiveTransactionHandle; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.iceberg.ColumnIdentity; import io.trino.plugin.iceberg.IcebergColumnHandle; import io.trino.plugin.iceberg.IcebergConnector; diff --git a/testing/trino-tests/src/test/java/io/trino/sql/planner/IcebergCostBasedPlanTestSetup.java b/testing/trino-tests/src/test/java/io/trino/sql/planner/IcebergCostBasedPlanTestSetup.java index 574f02faa61b..3d446863f4f0 100644 --- a/testing/trino-tests/src/test/java/io/trino/sql/planner/IcebergCostBasedPlanTestSetup.java +++ b/testing/trino-tests/src/test/java/io/trino/sql/planner/IcebergCostBasedPlanTestSetup.java @@ -19,8 +19,8 @@ import io.airlift.log.Logger; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.iceberg.IcebergConnector; import io.trino.plugin.iceberg.IcebergConnectorFactory; import io.trino.spi.connector.Connector; From 522b7591e1a14a47564fcb41b77a05dbbc26abc0 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 00:22:47 -0700 Subject: [PATCH 122/158] Cleanup binding of FlushMetadataCacheProcedure --- .../metastore/DeltaLakeMetastoreModule.java | 2 +- .../TestingDeltaLakeMetastoreModule.java | 2 +- .../metastore/CachingHiveMetastoreModule.java | 19 ------------------- .../hive/metastore/HiveMetastoreModule.java | 2 +- .../hive/procedure/HiveProcedureModule.java | 7 +++++++ .../IcebergFileMetastoreCatalogModule.java | 2 +- .../IcebergHiveMetastoreCatalogModule.java | 2 +- ...tingIcebergFileMetastoreCatalogModule.java | 2 +- 8 files changed, 13 insertions(+), 25 deletions(-) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/DeltaLakeMetastoreModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/DeltaLakeMetastoreModule.java index 4eb4f8fe3ab7..6bfd78808c7a 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/DeltaLakeMetastoreModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/DeltaLakeMetastoreModule.java @@ -39,7 +39,7 @@ protected void setup(Binder binder) bindMetastoreModule("glue", new DeltaLakeGlueMetastoreModule()); bindMetastoreModule("glue-v1", new DeltaLakeGlueV1MetastoreModule()); - install(new CachingHiveMetastoreModule(false)); + install(new CachingHiveMetastoreModule()); } private void bindMetastoreModule(String name, Module module) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java index 98488d0330e4..2c772a0aef39 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java @@ -42,7 +42,7 @@ public TestingDeltaLakeMetastoreModule(HiveMetastore metastore) public void setup(Binder binder) { binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore)); - install(new CachingHiveMetastoreModule(false)); + install(new CachingHiveMetastoreModule()); binder.bind(DeltaLakeTableOperationsProvider.class).to(DeltaLakeFileMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(Key.get(boolean.class, HideDeltaLakeTables.class)).toInstance(false); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java index 3d6182349366..b0ba34f6e411 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java @@ -19,33 +19,20 @@ import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastoreFactory; -import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.metastore.cache.ImpersonationCachingConfig; import io.trino.plugin.hive.metastore.cache.SharedHiveMetastoreCache; import io.trino.plugin.hive.metastore.cache.SharedHiveMetastoreCache.CachingHiveMetastoreFactory; -import io.trino.plugin.hive.metastore.glue.GlueCache; -import io.trino.plugin.hive.procedure.FlushMetadataCacheProcedure; -import io.trino.spi.procedure.Procedure; import java.util.Optional; -import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; import static io.airlift.configuration.ConfigBinder.configBinder; import static org.weakref.jmx.guice.ExportBinder.newExporter; public class CachingHiveMetastoreModule extends AbstractConfigurationAwareModule { - private final boolean installFlushMetadataCacheProcedure; - - public CachingHiveMetastoreModule(boolean installFlushMetadataCacheProcedure) - { - this.installFlushMetadataCacheProcedure = installFlushMetadataCacheProcedure; - } - @Override protected void setup(Binder binder) { @@ -56,12 +43,6 @@ protected void setup(Binder binder) // export under the old name, for backwards compatibility newExporter(binder).export(HiveMetastoreFactory.class) .as(generator -> generator.generatedNameOf(CachingHiveMetastore.class)); - - if (installFlushMetadataCacheProcedure) { - newOptionalBinder(binder, GlueCache.class); - newOptionalBinder(binder, DirectoryLister.class); - newSetBinder(binder, Procedure.class).addBinding().toProvider(FlushMetadataCacheProcedure.class).in(Scopes.SINGLETON); - } } @Provides diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java index 9703885f428b..15f153afe7f7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java @@ -55,7 +55,7 @@ protected void setup(Binder binder) bindMetastoreModule("glue-v1", new io.trino.plugin.hive.metastore.glue.v1.GlueMetastoreModule()); } - install(new CachingHiveMetastoreModule(true)); + install(new CachingHiveMetastoreModule()); } private void bindMetastoreModule(String name, Module module) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/HiveProcedureModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/HiveProcedureModule.java index 465d4d70ebe0..1f7cb0bd32dc 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/HiveProcedureModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/HiveProcedureModule.java @@ -17,10 +17,13 @@ import com.google.inject.Module; import com.google.inject.Scopes; import com.google.inject.multibindings.Multibinder; +import io.trino.plugin.hive.fs.DirectoryLister; +import io.trino.plugin.hive.metastore.glue.GlueCache; import io.trino.spi.connector.TableProcedureMetadata; import io.trino.spi.procedure.Procedure; import static com.google.inject.multibindings.Multibinder.newSetBinder; +import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; public class HiveProcedureModule implements Module @@ -34,8 +37,12 @@ public void configure(Binder binder) procedures.addBinding().toProvider(UnregisterPartitionProcedure.class).in(Scopes.SINGLETON); procedures.addBinding().toProvider(SyncPartitionMetadataProcedure.class).in(Scopes.SINGLETON); procedures.addBinding().toProvider(DropStatsProcedure.class).in(Scopes.SINGLETON); + procedures.addBinding().toProvider(FlushMetadataCacheProcedure.class).in(Scopes.SINGLETON); Multibinder tableProcedures = newSetBinder(binder, TableProcedureMetadata.class); tableProcedures.addBinding().toProvider(OptimizeTableProcedure.class).in(Scopes.SINGLETON); + + newOptionalBinder(binder, GlueCache.class); + newOptionalBinder(binder, DirectoryLister.class); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java index cef2b7927262..97a334c34420 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java @@ -47,7 +47,7 @@ protected void setup(Binder binder) binder.bind(TrinoCatalogFactory.class).to(TrinoHiveCatalogFactory.class).in(Scopes.SINGLETON); binder.bind(MetastoreValidator.class).asEagerSingleton(); binder.bind(Key.get(boolean.class, HideDeltaLakeTables.class)).toInstance(HIDE_DELTA_LAKE_TABLES_IN_ICEBERG); - install(new CachingHiveMetastoreModule(false)); + install(new CachingHiveMetastoreModule()); configBinder(binder).bindConfigDefaults(CachingHiveMetastoreConfig.class, config -> { // ensure caching metastore wrapper isn't created, as it's not leveraged by Iceberg diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java index b254247f9828..c5ee875fda0e 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java @@ -49,7 +49,7 @@ protected void setup(Binder binder) binder.bind(MetastoreValidator.class).asEagerSingleton(); binder.bind(Key.get(boolean.class, TranslateHiveViews.class)).toInstance(false); binder.bind(Key.get(boolean.class, HideDeltaLakeTables.class)).toInstance(HIDE_DELTA_LAKE_TABLES_IN_ICEBERG); - install(new CachingHiveMetastoreModule(false)); + install(new CachingHiveMetastoreModule()); configBinder(binder).bindConfigDefaults(CachingHiveMetastoreConfig.class, config -> { // ensure caching metastore wrapper isn't created, as it's not leveraged by Iceberg diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java index fc02c4b7f82c..695ae115c5c5 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java @@ -45,7 +45,7 @@ public TestingIcebergFileMetastoreCatalogModule(HiveMetastore metastore) protected void setup(Binder binder) { binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore)); - install(new CachingHiveMetastoreModule(false)); + install(new CachingHiveMetastoreModule()); binder.bind(IcebergTableOperationsProvider.class).to(FileMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(TrinoCatalogFactory.class).to(TrinoHiveCatalogFactory.class).in(Scopes.SINGLETON); From fc78d2c248dc50a1b43a6f8d86feac2c2e0727f0 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Tue, 30 Jul 2024 23:51:20 -0700 Subject: [PATCH 123/158] Move CachingHiveMetastore to metastore module --- lib/trino-metastore/pom.xml | 40 +++++++++++++++++++ .../metastore/RawHiveMetastoreFactory.java | 2 +- .../metastore/cache/CachingHiveMetastore.java | 17 ++++---- .../cache/CachingHiveMetastoreConfig.java | 2 +- .../metastore/cache}/HivePartitionName.java | 2 +- .../trino/metastore/cache}/HiveTableName.java | 2 +- .../cache/ImpersonationCachingConfig.java | 2 +- .../metastore/cache}/PartitionFilter.java | 4 +- .../cache/ReentrantBoundedExecutor.java | 2 +- .../cache/SharedHiveMetastoreCache.java | 4 +- .../cache/TestCachingHiveMetastoreConfig.java | 4 +- .../cache/TestImpersonationCachingConfig.java | 2 +- .../cache/TestReentrantBoundedExecutor.java | 2 +- .../deltalake/DeltaLakeMetadataFactory.java | 4 +- .../FlushMetadataCacheProcedure.java | 2 +- .../deltalake/TestDeltaLakeMetadata.java | 2 +- .../TestingDeltaLakeMetastoreModule.java | 2 +- .../plugin/hive/HiveMetadataFactory.java | 2 +- .../plugin/hive/HivePageSinkProvider.java | 4 +- .../metastore/CachingHiveMetastoreModule.java | 11 ++--- .../hive/metastore/HiveMetastoreModule.java | 1 + .../metastore/file/FileMetastoreModule.java | 2 +- .../metastore/glue/GlueMetastoreModule.java | 4 +- .../metastore/glue/InMemoryGlueCache.java | 2 +- .../glue/v1/GlueMetastoreModule.java | 2 +- .../thrift/ThriftMetastoreModule.java | 2 +- .../FlushMetadataCacheProcedure.java | 2 +- .../cache/TestCachingHiveMetastore.java | 3 +- ...stCachingHiveMetastoreWithQueryRunner.java | 2 +- .../plugin/hudi/HudiMetadataFactory.java | 4 +- .../iceberg/IcebergMetadataFactory.java | 2 +- .../trino/plugin/iceberg/IcebergModule.java | 2 +- .../iceberg/catalog/MetastoreValidator.java | 2 +- .../file/FileMetastoreTableOperations.java | 2 +- .../IcebergFileMetastoreCatalogModule.java | 2 +- .../hms/AbstractMetastoreTableOperations.java | 2 +- .../hms/HiveMetastoreTableOperations.java | 2 +- .../IcebergHiveMetastoreCatalogModule.java | 2 +- .../iceberg/catalog/hms/TrinoHiveCatalog.java | 2 +- .../catalog/hms/TrinoHiveCatalogFactory.java | 4 +- .../iceberg/procedure/MigrateProcedure.java | 2 +- .../plugin/iceberg/IcebergTestUtils.java | 4 +- .../iceberg/TestIcebergMergeAppend.java | 4 +- .../TestIcebergOrcMetricsCollection.java | 4 +- .../iceberg/TestIcebergSplitSource.java | 8 ++-- .../TestAbstractIcebergTableOperations.java | 2 +- ...TestTrinoHiveCatalogWithFileMetastore.java | 4 +- ...tingIcebergFileMetastoreCatalogModule.java | 4 +- ...TestTrinoHiveCatalogWithHiveMetastore.java | 4 +- .../product/deltalake/TestDeltaLakeJmx.java | 4 +- 50 files changed, 119 insertions(+), 79 deletions(-) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino}/metastore/RawHiveMetastoreFactory.java (96%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino}/metastore/cache/CachingHiveMetastore.java (98%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino}/metastore/cache/CachingHiveMetastoreConfig.java (99%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore => lib/trino-metastore/src/main/java/io/trino/metastore/cache}/HivePartitionName.java (98%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore => lib/trino-metastore/src/main/java/io/trino/metastore/cache}/HiveTableName.java (98%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino}/metastore/cache/ImpersonationCachingConfig.java (97%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore => lib/trino-metastore/src/main/java/io/trino/metastore/cache}/PartitionFilter.java (96%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino}/metastore/cache/ReentrantBoundedExecutor.java (97%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino}/metastore/cache/SharedHiveMetastoreCache.java (99%) rename {plugin/trino-hive/src/test/java/io/trino/plugin/hive => lib/trino-metastore/src/test/java/io/trino}/metastore/cache/TestCachingHiveMetastoreConfig.java (96%) rename {plugin/trino-hive/src/test/java/io/trino/plugin/hive => lib/trino-metastore/src/test/java/io/trino}/metastore/cache/TestImpersonationCachingConfig.java (97%) rename {plugin/trino-hive/src/test/java/io/trino/plugin/hive => lib/trino-metastore/src/test/java/io/trino}/metastore/cache/TestReentrantBoundedExecutor.java (97%) diff --git a/lib/trino-metastore/pom.xml b/lib/trino-metastore/pom.xml index c0549ec5c888..a4415f39af65 100644 --- a/lib/trino-metastore/pom.xml +++ b/lib/trino-metastore/pom.xml @@ -33,11 +33,31 @@ guava + + com.google.inject + guice + + + + io.airlift + concurrent + + + + io.airlift + configuration + + io.airlift slice + + io.airlift + units + + io.opentelemetry opentelemetry-api @@ -53,11 +73,31 @@ opentelemetry-semconv + + io.trino + trino-cache + + io.trino trino-spi + + jakarta.annotation + jakarta.annotation-api + + + + jakarta.validation + jakarta.validation-api + + + + org.weakref + jmxutils + + io.airlift junit-extensions diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/RawHiveMetastoreFactory.java b/lib/trino-metastore/src/main/java/io/trino/metastore/RawHiveMetastoreFactory.java similarity index 96% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/RawHiveMetastoreFactory.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/RawHiveMetastoreFactory.java index a9d7b5b793b0..573bb4b23efd 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/RawHiveMetastoreFactory.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/RawHiveMetastoreFactory.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore; +package io.trino.metastore; import com.google.inject.BindingAnnotation; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastore.java similarity index 98% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastore.java index b8c44879c49f..0c22b77d39f9 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastore.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import com.google.common.cache.Cache; import com.google.common.cache.CacheLoader; @@ -44,9 +44,6 @@ import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.metastore.HivePartitionName; -import io.trino.plugin.hive.metastore.HiveTableName; -import io.trino.plugin.hive.metastore.PartitionFilter; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.function.LanguageFunction; @@ -86,12 +83,12 @@ import static io.trino.cache.CacheUtils.invalidateAllIf; import static io.trino.cache.CacheUtils.uncheckedCacheGet; import static io.trino.metastore.Partitions.makePartName; -import static io.trino.plugin.hive.metastore.HivePartitionName.hivePartitionName; -import static io.trino.plugin.hive.metastore.HiveTableName.hiveTableName; -import static io.trino.plugin.hive.metastore.PartitionFilter.partitionFilter; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType.OTHER; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType.PARTITION; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType.STATS; +import static io.trino.metastore.cache.CachingHiveMetastore.ObjectType.OTHER; +import static io.trino.metastore.cache.CachingHiveMetastore.ObjectType.PARTITION; +import static io.trino.metastore.cache.CachingHiveMetastore.ObjectType.STATS; +import static io.trino.metastore.cache.HivePartitionName.hivePartitionName; +import static io.trino.metastore.cache.HiveTableName.hiveTableName; +import static io.trino.metastore.cache.PartitionFilter.partitionFilter; import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastoreConfig.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastoreConfig.java similarity index 99% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastoreConfig.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastoreConfig.java index 2f97cbc454ed..b69c62d6b131 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastoreConfig.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastoreConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import io.airlift.configuration.Config; import io.airlift.units.Duration; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HivePartitionName.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/HivePartitionName.java similarity index 98% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HivePartitionName.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/HivePartitionName.java index 9aeba984c4fa..dab9f21cfa00 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HivePartitionName.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/HivePartitionName.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore; +package io.trino.metastore.cache; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveTableName.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/HiveTableName.java similarity index 98% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveTableName.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/HiveTableName.java index dde5609a0bac..85281946e1a8 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveTableName.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/HiveTableName.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore; +package io.trino.metastore.cache; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/ImpersonationCachingConfig.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/ImpersonationCachingConfig.java similarity index 97% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/ImpersonationCachingConfig.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/ImpersonationCachingConfig.java index 15eb31787d94..29268b002c1d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/ImpersonationCachingConfig.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/ImpersonationCachingConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import io.airlift.configuration.Config; import io.airlift.units.Duration; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/PartitionFilter.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/PartitionFilter.java similarity index 96% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/PartitionFilter.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/PartitionFilter.java index 883b9a1641c2..73695573927c 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/PartitionFilter.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/PartitionFilter.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore; +package io.trino.metastore.cache; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -23,7 +23,7 @@ import java.util.Objects; import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.plugin.hive.metastore.HiveTableName.hiveTableName; +import static io.trino.metastore.cache.HiveTableName.hiveTableName; import static java.util.Objects.requireNonNull; @Immutable diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/ReentrantBoundedExecutor.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/ReentrantBoundedExecutor.java similarity index 97% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/ReentrantBoundedExecutor.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/ReentrantBoundedExecutor.java index 6005d3c77d6d..0480695a8f00 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/ReentrantBoundedExecutor.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/ReentrantBoundedExecutor.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import io.airlift.concurrent.BoundedExecutor; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/SharedHiveMetastoreCache.java similarity index 99% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/cache/SharedHiveMetastoreCache.java index 8e2c731b8d92..68adfd506063 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/SharedHiveMetastoreCache.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -25,7 +25,7 @@ import io.airlift.units.Duration; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.ObjectType; +import io.trino.metastore.cache.CachingHiveMetastore.ObjectType; import io.trino.spi.NodeManager; import io.trino.spi.TrinoException; import io.trino.spi.catalog.CatalogName; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreConfig.java b/lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestCachingHiveMetastoreConfig.java similarity index 96% rename from plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreConfig.java rename to lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestCachingHiveMetastoreConfig.java index 38d59830b42b..5047c09a1301 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreConfig.java +++ b/lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestCachingHiveMetastoreConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; @@ -22,7 +22,7 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig.DEFAULT_STATS_CACHE_TTL; +import static io.trino.metastore.cache.CachingHiveMetastoreConfig.DEFAULT_STATS_CACHE_TTL; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestImpersonationCachingConfig.java b/lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestImpersonationCachingConfig.java similarity index 97% rename from plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestImpersonationCachingConfig.java rename to lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestImpersonationCachingConfig.java index 63edc9a23e1e..7127c141532d 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestImpersonationCachingConfig.java +++ b/lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestImpersonationCachingConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestReentrantBoundedExecutor.java b/lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestReentrantBoundedExecutor.java similarity index 97% rename from plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestReentrantBoundedExecutor.java rename to lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestReentrantBoundedExecutor.java index 3f4c04ae283c..feadf2e49a18 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestReentrantBoundedExecutor.java +++ b/lib/trino-metastore/src/test/java/io/trino/metastore/cache/TestReentrantBoundedExecutor.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.cache; +package io.trino.metastore.cache; import com.google.common.util.concurrent.SettableFuture; import org.junit.jupiter.api.Test; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java index dc1241ed504d..34b048049ab1 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java @@ -18,6 +18,7 @@ import io.airlift.json.JsonCodec; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.deltalake.metastore.DeltaLakeTableMetadataScheduler; import io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore; import io.trino.plugin.deltalake.statistics.CachingExtendedStatisticsAccess; @@ -27,7 +28,6 @@ import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.security.AccessControlMetadata; import io.trino.spi.NodeManager; import io.trino.spi.security.ConnectorIdentity; @@ -38,7 +38,7 @@ import java.util.concurrent.ExecutorService; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static java.util.Objects.requireNonNull; public class DeltaLakeMetadataFactory diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/FlushMetadataCacheProcedure.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/FlushMetadataCacheProcedure.java index 220636d141f1..97fa7e73c306 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/FlushMetadataCacheProcedure.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/procedure/FlushMetadataCacheProcedure.java @@ -16,9 +16,9 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import com.google.inject.Provider; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.deltalake.statistics.CachingExtendedStatisticsAccess; import io.trino.plugin.deltalake.transactionlog.TransactionLogAccess; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.spi.TrinoException; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.SchemaTableName; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java index 8ffa699674f9..f2527040d271 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java @@ -31,13 +31,13 @@ import io.trino.hdfs.TrinoHdfsFileSystemStats; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastore; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule; import io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore; import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; import io.trino.plugin.hive.NodeVersion; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.spi.NodeManager; import io.trino.spi.PageIndexerFactory; import io.trino.spi.TrinoException; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java index 2c772a0aef39..07bf6000f02d 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java @@ -19,12 +19,12 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.deltalake.AllowDeltaLakeManagedTableRename; import io.trino.plugin.deltalake.MaxTableParameterLength; import io.trino.plugin.deltalake.metastore.file.DeltaLakeFileMetastoreTableOperationsProvider; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import static java.util.Objects.requireNonNull; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java index f63c5dad2b42..b8fd3fc20f59 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java @@ -39,7 +39,7 @@ import java.util.concurrent.ScheduledExecutorService; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static java.util.Objects.requireNonNull; public class HiveMetadataFactory diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java index d1030f73de35..d71d24935d34 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSinkProvider.java @@ -23,8 +23,8 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.SortingColumn; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.HivePageSinkMetadataProvider; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.spi.PageIndexerFactory; import io.trino.spi.PageSorter; import io.trino.spi.connector.ConnectorInsertTableHandle; @@ -48,7 +48,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java index b0ba34f6e411..5b2e0cc2c7ae 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java @@ -19,11 +19,12 @@ import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastoreFactory; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; -import io.trino.plugin.hive.metastore.cache.ImpersonationCachingConfig; -import io.trino.plugin.hive.metastore.cache.SharedHiveMetastoreCache; -import io.trino.plugin.hive.metastore.cache.SharedHiveMetastoreCache.CachingHiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; +import io.trino.metastore.cache.CachingHiveMetastore; +import io.trino.metastore.cache.CachingHiveMetastoreConfig; +import io.trino.metastore.cache.ImpersonationCachingConfig; +import io.trino.metastore.cache.SharedHiveMetastoreCache; +import io.trino.metastore.cache.SharedHiveMetastoreCache.CachingHiveMetastoreFactory; import java.util.Optional; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java index 15f153afe7f7..7e247958ab31 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java @@ -21,6 +21,7 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.AllowHiveTableRename; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.file.FileMetastoreModule; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java index e990434bea34..2490ec55ea4c 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileMetastoreModule.java @@ -18,8 +18,8 @@ import com.google.inject.Module; import com.google.inject.Scopes; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.AllowHiveTableRename; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import static io.airlift.configuration.ConfigBinder.configBinder; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java index d1431bb30de7..b4fe87a8ee7e 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreModule.java @@ -28,10 +28,10 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.awssdk.v2_2.AwsSdkTelemetry; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; +import io.trino.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.AllowHiveTableRename; import io.trino.plugin.hive.HideDeltaLakeTables; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.spi.NodeManager; import io.trino.spi.catalog.CatalogName; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/InMemoryGlueCache.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/InMemoryGlueCache.java index fe1980b1f69f..dc627293c35e 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/InMemoryGlueCache.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/InMemoryGlueCache.java @@ -26,7 +26,7 @@ import io.trino.metastore.Partition; import io.trino.metastore.Table; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.metastore.cache.ReentrantBoundedExecutor; +import io.trino.metastore.cache.ReentrantBoundedExecutor; import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.function.LanguageFunction; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java index eddec0839eb6..1d1bbf384cf3 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java @@ -32,8 +32,8 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.awssdk.v1_11.AwsSdkTelemetry; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.AllowHiveTableRename; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; import java.lang.annotation.Annotation; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java index 8c5f0b71b37c..c83d47e60aff 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java @@ -22,10 +22,10 @@ import com.google.inject.multibindings.OptionalBinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.base.security.UserNameProvider; import io.trino.plugin.hive.AllowHiveTableRename; import io.trino.plugin.hive.ForHiveMetastore; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import java.util.concurrent.ExecutorService; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java index 8cf5ae65c4c7..fffea2dcd713 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java @@ -20,9 +20,9 @@ import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.HiveErrorCode; import io.trino.plugin.hive.fs.DirectoryLister; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.glue.GlueCache; import io.trino.plugin.hive.metastore.glue.PartitionName; import io.trino.spi.StandardErrorCode; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java index 148ec287cd1f..c9e62ef88fae 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java @@ -31,6 +31,7 @@ import io.trino.metastore.PartitionStatistics; import io.trino.metastore.Table; import io.trino.metastore.TableInfo; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; @@ -72,12 +73,12 @@ import static io.trino.metastore.HiveColumnStatistics.createIntegerColumnStatistics; import static io.trino.metastore.HiveType.HIVE_STRING; import static io.trino.metastore.StatisticsUpdateMode.MERGE_INCREMENTAL; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static io.trino.plugin.hive.HiveColumnHandle.createBaseColumn; import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder; import static io.trino.plugin.hive.metastore.MetastoreUtil.computePartitionKeyFilter; import static io.trino.plugin.hive.metastore.MetastoreUtil.makePartitionName; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.hive.metastore.thrift.MockThriftMetastoreClient.BAD_DATABASE; import static io.trino.plugin.hive.metastore.thrift.MockThriftMetastoreClient.BAD_PARTITION; import static io.trino.plugin.hive.metastore.thrift.MockThriftMetastoreClient.PARTITION_COLUMN_NAMES; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java index 6acda90db491..ea53c3d45836 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java @@ -20,8 +20,8 @@ import io.trino.Session; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.HiveQueryRunner; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.spi.security.Identity; import io.trino.spi.security.SelectedRole; import io.trino.testing.AbstractTestQueryFramework; diff --git a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java index 1489d191d0a0..444d20e6a833 100644 --- a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java +++ b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java @@ -16,13 +16,13 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastoreFactory; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; import java.util.Optional; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static java.util.Objects.requireNonNull; public class HudiMetadataFactory diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java index e7fbb29f4a55..6437912766b5 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadataFactory.java @@ -19,7 +19,7 @@ import io.airlift.concurrent.BoundedExecutor; import io.airlift.json.JsonCodec; import io.trino.metastore.HiveMetastoreFactory; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.spi.connector.CatalogHandle; import io.trino.spi.security.ConnectorIdentity; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index 254a994cb673..ed5b6b10fb86 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -23,6 +23,7 @@ import com.google.inject.multibindings.Multibinder; import io.trino.filesystem.cache.CacheKeyProvider; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSinkProvider; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProviderFactory; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; @@ -31,7 +32,6 @@ import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.hive.SortingFileWriterConfig; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.hive.orc.OrcReaderConfig; import io.trino.plugin.hive.orc.OrcWriterConfig; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/MetastoreValidator.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/MetastoreValidator.java index 4cde7c705eec..c0a1fe3869b8 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/MetastoreValidator.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/MetastoreValidator.java @@ -14,7 +14,7 @@ package io.trino.plugin.iceberg.catalog; import com.google.inject.Inject; -import io.trino.plugin.hive.metastore.cache.SharedHiveMetastoreCache; +import io.trino.metastore.cache.SharedHiveMetastoreCache; public class MetastoreValidator { diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperations.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperations.java index eacdccf9b731..751372cbcfc6 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperations.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperations.java @@ -16,8 +16,8 @@ import io.trino.annotation.NotThreadSafe; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.MetastoreUtil; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.catalog.hms.AbstractMetastoreTableOperations; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java index 97a334c34420..9c4039a7b393 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java @@ -19,9 +19,9 @@ import com.google.inject.multibindings.Multibinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.units.Duration; +import io.trino.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.metastore.file.FileMetastoreModule; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.MetastoreValidator; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/AbstractMetastoreTableOperations.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/AbstractMetastoreTableOperations.java index afdc4dbd211f..b0dc44310858 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/AbstractMetastoreTableOperations.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/AbstractMetastoreTableOperations.java @@ -16,8 +16,8 @@ import io.trino.annotation.NotThreadSafe; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.MetastoreUtil; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.CreateTableException; import io.trino.plugin.iceberg.UnknownTableTypeException; import io.trino.plugin.iceberg.catalog.AbstractIcebergTableOperations; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperations.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperations.java index 65d4dac8f7a0..905a1e4c834c 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperations.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperations.java @@ -18,8 +18,8 @@ import io.trino.metastore.AcidTransactionOwner; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.MetastoreUtil; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.ThriftMetastore; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.TableNotFoundException; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java index c5ee875fda0e..bcad82df2c53 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java @@ -19,9 +19,9 @@ import com.google.inject.multibindings.Multibinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.units.Duration; +import io.trino.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreModule; import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java index 908d7cc6dde9..665a050f69b3 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java @@ -28,10 +28,10 @@ import io.trino.metastore.HivePrincipal; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.TableInfo; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.HiveSchemaProperties; import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.MetastoreUtil; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.util.HiveUtil; import io.trino.plugin.iceberg.IcebergTableName; import io.trino.plugin.iceberg.UnknownTableTypeException; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java index d18785562efa..2c32fc663f8f 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java @@ -17,9 +17,9 @@ import io.airlift.concurrent.BoundedExecutor; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.ForIcebergMetadata; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergSecurityConfig; @@ -35,7 +35,7 @@ import java.util.concurrent.ExecutorService; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.iceberg.IcebergSecurityConfig.IcebergSecurity.SYSTEM; import static io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog.TRINO_CREATED_BY_VALUE; import static java.util.Objects.requireNonNull; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java index b56f2b8289f7..a8eceeddaeed 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrateProcedure.java @@ -26,9 +26,9 @@ import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Partition; import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.metastore.Storage; import io.trino.plugin.hive.HiveStorageFormat; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergFileFormat; import io.trino.plugin.iceberg.IcebergSecurityConfig; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java index c1d4f0a0b485..17d8695f61f5 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java @@ -23,6 +23,7 @@ import io.trino.filesystem.TrinoInputFile; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.orc.OrcDataSource; import io.trino.orc.OrcReader; import io.trino.orc.OrcReaderOptions; @@ -36,7 +37,6 @@ import io.trino.parquet.reader.MetadataReader; import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.parquet.TrinoParquetDataSource; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -64,7 +64,7 @@ import static com.google.common.collect.Iterators.getOnlyElement; import static com.google.common.collect.MoreCollectors.onlyElement; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; import static io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable; import static io.trino.plugin.iceberg.util.FileOperationUtils.FileType.METADATA_JSON; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java index cce80e901972..43a255b7c10c 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java @@ -15,8 +15,8 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.file.FileMetastoreTableOperationsProvider; @@ -32,7 +32,7 @@ import org.junit.jupiter.api.Test; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; import static io.trino.plugin.iceberg.IcebergTestUtils.getHiveMetastore; import static org.assertj.core.api.Assertions.assertThat; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java index 02efbf39d572..bd47288fec81 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java @@ -17,8 +17,8 @@ import io.trino.Session; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.file.FileMetastoreTableOperationsProvider; @@ -47,7 +47,7 @@ import static io.trino.SystemSessionProperties.TASK_CONCURRENCY; import static io.trino.SystemSessionProperties.TASK_MAX_WRITER_COUNT; import static io.trino.SystemSessionProperties.TASK_MIN_WRITER_COUNT; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.iceberg.DataFileRecord.toDataFileRecord; import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java index 62be3d6672c3..a7fb5fc85859 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java @@ -20,8 +20,8 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.cache.DefaultCachingHostAddressProvider; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.orc.OrcReaderConfig; import io.trino.plugin.hive.orc.OrcWriterConfig; import io.trino.plugin.hive.parquet.ParquetReaderConfig; @@ -84,7 +84,7 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.iceberg.IcebergSplitSource.createFileStatisticsDomain; import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; import static io.trino.plugin.iceberg.IcebergTestUtils.getHiveMetastore; @@ -406,14 +406,14 @@ public void testSplitWeight() // Write position delete file FileIO fileIo = new ForwardingFileIo(fileSystemFactory.create(SESSION)); - PositionDeleteWriter writer = Parquet.writeDeletes(fileIo.newOutputFile("local:///delete_file_" + UUID.randomUUID())) + PositionDeleteWriter writer = Parquet.writeDeletes(fileIo.newOutputFile("local:///delete_file_" + UUID.randomUUID())) .createWriterFunc(GenericParquetWriter::buildWriter) .forTable(nationTable) .overwrite() .rowSchema(nationTable.schema()) .withSpec(PartitionSpec.unpartitioned()) .buildPositionWriter(); - PositionDelete positionDelete = PositionDelete.create(); + PositionDelete positionDelete = PositionDelete.create(); PositionDelete record = positionDelete.set(dataFilePath, 0, GenericRecord.create(nationTable.schema())); try (Closeable ignored = writer) { writer.write(record); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestAbstractIcebergTableOperations.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestAbstractIcebergTableOperations.java index 3f12c1b9e42a..ec4beb206993 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestAbstractIcebergTableOperations.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestAbstractIcebergTableOperations.java @@ -28,7 +28,7 @@ import java.nio.file.Path; import java.util.Optional; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.hive.metastore.file.TestingFileHiveMetastore.createTestingFileHiveMetastore; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA; import static io.trino.testing.TestingConnectorSession.SESSION; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java index c8641e0773f0..9e630d451430 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java @@ -20,8 +20,8 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.local.LocalFileSystemFactory; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.TrinoViewHiveMetastore; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.catalog.BaseTrinoCatalogTest; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -48,7 +48,7 @@ import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.hive.metastore.file.TestingFileHiveMetastore.createTestingFileHiveMetastore; import static io.trino.plugin.iceberg.IcebergFileFormat.PARQUET; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java index 695ae115c5c5..d88c0ac316be 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java @@ -19,9 +19,9 @@ import io.airlift.units.Duration; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.RawHiveMetastoreFactory; +import io.trino.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalogFactory; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index c0f80a947eeb..63d841b90083 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -32,11 +32,11 @@ import io.trino.hdfs.s3.TrinoS3ConfigurationInitializer; import io.trino.metastore.Table; import io.trino.metastore.TableInfo; +import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveMinioDataLake; -import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.ThriftMetastore; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreConfig; @@ -71,9 +71,9 @@ import static com.google.common.base.Verify.verify; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.trino.metastore.PrincipalPrivileges.NO_PRIVILEGES; +import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder; import static io.trino.plugin.hive.containers.HiveHadoop.HIVE3_IMAGE; -import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.iceberg.IcebergFileFormat.PARQUET; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java index fa7f14eaba86..6e92f6953bf4 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java @@ -35,7 +35,7 @@ public void testJmxTablesExposedByDeltaLakeConnectorBackedByGlueMetastore() { assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( row("io.trino.filesystem.s3:name=delta,type=s3filesystemstats"), - row("io.trino.plugin.hive.metastore.cache:name=delta,type=cachinghivemetastore"), + row("io.trino.metastore.cache:name=delta,type=cachinghivemetastore"), row("io.trino.plugin.hive.metastore.glue:name=delta,type=gluehivemetastore"), row("io.trino.plugin.hive.metastore.glue:name=delta,type=gluemetastorestats"), row("io.trino.plugin.base.metrics:catalog=delta,name=delta,type=fileformatdatasourcestats"), @@ -48,7 +48,7 @@ public void testJmxTablesExposedByDeltaLakeConnectorBackedByThriftMetastore() { assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( row("io.trino.filesystem.s3:name=delta,type=s3filesystemstats"), - row("io.trino.plugin.hive.metastore.cache:name=delta,type=cachinghivemetastore"), + row("io.trino.metastore.cache:name=delta,type=cachinghivemetastore"), row("io.trino.plugin.hive.metastore.thrift:name=delta,type=thrifthivemetastore"), row("io.trino.plugin.hive.metastore.thrift:name=delta,type=thriftmetastorestats"), row("io.trino.plugin.base.metrics:catalog=delta,name=delta,type=fileformatdatasourcestats"), From 010e727a09c030d46918f633c4cec78ab3ccffbf Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 00:30:51 -0700 Subject: [PATCH 124/158] Move Glue v1 converters into Glue package --- ...eltaLakeGlueMetastoreTableFilterProvider.java | 2 +- .../DeltaLakeGlueV1MetastoreTableOperations.java | 4 ++-- .../v1/DefaultGlueColumnStatisticsProvider.java | 4 ++-- .../DefaultGlueMetastoreTableFilterProvider.java | 2 +- .../metastore/glue/v1/GlueHiveMetastore.java | 16 +++++++--------- .../v1/{converter => }/GlueInputConverter.java | 8 ++++---- .../v1/{converter => }/GlueStatConverter.java | 2 +- .../v1/{converter => }/GlueToTrinoConverter.java | 4 ++-- .../glue/v1/{converter => }/Memoizers.java | 2 +- .../glue/v1/TestGlueInputConverter.java | 2 -- .../glue/v1/TestGlueToTrinoConverter.java | 11 +++++------ .../iceberg/catalog/AbstractTrinoCatalog.java | 2 +- .../catalog/glue/GlueIcebergTableOperations.java | 6 +++--- .../iceberg/catalog/glue/TrinoGlueCatalog.java | 10 +++++----- ...TestIcebergGlueCatalogConnectorSmokeTest.java | 6 +++--- .../TestIcebergGlueCatalogMaterializedView.java | 2 +- .../glue/TestIcebergGlueCatalogSkipArchive.java | 2 +- .../TestDatabricksWithGlueMetastoreCleanUp.java | 2 +- 18 files changed, 41 insertions(+), 46 deletions(-) rename plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/{converter => }/GlueInputConverter.java (95%) rename plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/{converter => }/GlueStatConverter.java (99%) rename plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/{converter => }/GlueToTrinoConverter.java (99%) rename plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/{converter => }/Memoizers.java (97%) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueMetastoreTableFilterProvider.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueMetastoreTableFilterProvider.java index f44eb3f1c3be..654b251bbb8f 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueMetastoreTableFilterProvider.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueMetastoreTableFilterProvider.java @@ -21,7 +21,7 @@ import java.util.function.Predicate; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; public class DeltaLakeGlueMetastoreTableFilterProvider implements Provider> diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueV1MetastoreTableOperations.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueV1MetastoreTableOperations.java index 1e1df8668529..d3e0eed81a98 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueV1MetastoreTableOperations.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueV1MetastoreTableOperations.java @@ -28,8 +28,8 @@ import java.util.Optional; import static io.trino.plugin.deltalake.metastore.DeltaLakeTableMetadataScheduler.tableMetadataParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueInputConverter.convertGlueTableToTableInput; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueInputConverter.convertGlueTableToTableInput; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; import static java.util.Objects.requireNonNull; public class DeltaLakeGlueV1MetastoreTableOperations diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java index fb3e912956a2..7fe6b886bf62 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueColumnStatisticsProvider.java @@ -56,8 +56,8 @@ import static io.trino.metastore.Partitions.toPartitionValues; import static io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static io.trino.plugin.hive.HiveErrorCode.HIVE_PARTITION_NOT_FOUND; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueStatConverter.fromGlueColumnStatistics; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueStatConverter.toGlueColumnStatistics; +import static io.trino.plugin.hive.metastore.glue.v1.GlueStatConverter.fromGlueColumnStatistics; +import static io.trino.plugin.hive.metastore.glue.v1.GlueStatConverter.toGlueColumnStatistics; import static java.util.concurrent.CompletableFuture.allOf; import static java.util.concurrent.CompletableFuture.runAsync; import static java.util.concurrent.CompletableFuture.supplyAsync; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueMetastoreTableFilterProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueMetastoreTableFilterProvider.java index a072f2682239..54f81a1eb184 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueMetastoreTableFilterProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/DefaultGlueMetastoreTableFilterProvider.java @@ -20,7 +20,7 @@ import java.util.function.Predicate; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; import static io.trino.plugin.hive.util.HiveUtil.isDeltaLakeTable; import static java.util.function.Predicate.not; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java index 10ad2d54c40b..79d5ec45ce24 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java @@ -96,9 +96,7 @@ import io.trino.plugin.hive.metastore.glue.AwsApiCallStats; import io.trino.plugin.hive.metastore.glue.GlueExpressionUtil; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; -import io.trino.plugin.hive.metastore.glue.v1.converter.GlueInputConverter; -import io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter; -import io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.GluePartitionConverter; +import io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.GluePartitionConverter; import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnNotFoundException; import io.trino.spi.connector.SchemaNotFoundException; @@ -150,12 +148,12 @@ import static io.trino.plugin.hive.metastore.MetastoreUtil.updateStatisticsParameters; import static io.trino.plugin.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static io.trino.plugin.hive.metastore.glue.v1.AwsSdkUtil.getPaginatedResults; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueInputConverter.convertFunction; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueInputConverter.convertGlueTableToTableInput; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueInputConverter.convertPartition; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableType; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.mappedCopy; +import static io.trino.plugin.hive.metastore.glue.v1.GlueInputConverter.convertFunction; +import static io.trino.plugin.hive.metastore.glue.v1.GlueInputConverter.convertGlueTableToTableInput; +import static io.trino.plugin.hive.metastore.glue.v1.GlueInputConverter.convertPartition; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableType; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.mappedCopy; import static io.trino.plugin.hive.util.HiveUtil.escapeSchemaName; import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; import static io.trino.spi.StandardErrorCode.FUNCTION_NOT_FOUND; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueInputConverter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueInputConverter.java similarity index 95% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueInputConverter.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueInputConverter.java index 1d12277e87d4..9d16be3760ce 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueInputConverter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueInputConverter.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.glue.v1.converter; +package io.trino.plugin.hive.metastore.glue.v1; import com.amazonaws.services.glue.model.DatabaseInput; import com.amazonaws.services.glue.model.Order; @@ -48,9 +48,9 @@ import static io.trino.plugin.hive.metastore.MetastoreUtil.metastoreFunctionName; import static io.trino.plugin.hive.metastore.MetastoreUtil.toResourceUris; import static io.trino.plugin.hive.metastore.MetastoreUtil.updateStatisticsParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getStorageDescriptor; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableTypeNullable; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getStorageDescriptor; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableTypeNullable; public final class GlueInputConverter { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueStatConverter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueStatConverter.java similarity index 99% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueStatConverter.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueStatConverter.java index 13ccbb855996..fc71676c5a36 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueStatConverter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueStatConverter.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.glue.v1.converter; +package io.trino.plugin.hive.metastore.glue.v1; import com.amazonaws.services.glue.model.BinaryColumnStatisticsData; import com.amazonaws.services.glue.model.BooleanColumnStatisticsData; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueToTrinoConverter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueToTrinoConverter.java similarity index 99% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueToTrinoConverter.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueToTrinoConverter.java index 53bed07ae493..3ab9906c170c 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/GlueToTrinoConverter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueToTrinoConverter.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.glue.v1.converter; +package io.trino.plugin.hive.metastore.glue.v1; import com.amazonaws.services.glue.model.SerDeInfo; import com.amazonaws.services.glue.model.StorageDescriptor; @@ -55,7 +55,7 @@ import static io.trino.plugin.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; import static io.trino.plugin.hive.ViewReaderUtil.isTrinoMaterializedView; -import static io.trino.plugin.hive.metastore.glue.v1.converter.Memoizers.memoizeLast; +import static io.trino.plugin.hive.metastore.glue.v1.Memoizers.memoizeLast; import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.decodeFunction; import static io.trino.plugin.hive.util.HiveUtil.isDeltaLakeTable; import static io.trino.plugin.hive.util.HiveUtil.isIcebergTable; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/Memoizers.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/Memoizers.java similarity index 97% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/Memoizers.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/Memoizers.java index 3ae763a83f4e..43d4af02a23b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/converter/Memoizers.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/Memoizers.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.glue.v1.converter; +package io.trino.plugin.hive.metastore.glue.v1; import java.util.Objects; import java.util.function.BiFunction; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueInputConverter.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueInputConverter.java index e3d713056449..b2c3c42da904 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueInputConverter.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueInputConverter.java @@ -27,8 +27,6 @@ import io.trino.metastore.Partition; import io.trino.metastore.Storage; import io.trino.metastore.Table; -import io.trino.plugin.hive.metastore.glue.v1.converter.GlueInputConverter; -import io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter; import io.trino.spi.function.LanguageFunction; import org.junit.jupiter.api.Test; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueToTrinoConverter.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueToTrinoConverter.java index e7231d6ee928..49356c25abe9 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueToTrinoConverter.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/v1/TestGlueToTrinoConverter.java @@ -22,8 +22,7 @@ import io.trino.metastore.Column; import io.trino.metastore.HiveBucketProperty; import io.trino.metastore.Storage; -import io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter; -import io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.GluePartitionConverter; +import io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.GluePartitionConverter; import io.trino.spi.security.PrincipalType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -37,16 +36,16 @@ import static com.amazonaws.util.CollectionUtils.isNullOrEmpty; import static io.trino.metastore.HiveType.HIVE_STRING; import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getPartitionParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getStorageDescriptor; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableTypeNullable; import static io.trino.plugin.hive.metastore.glue.v1.TestingMetastoreObjects.getGlueTestColumn; import static io.trino.plugin.hive.metastore.glue.v1.TestingMetastoreObjects.getGlueTestDatabase; import static io.trino.plugin.hive.metastore.glue.v1.TestingMetastoreObjects.getGlueTestPartition; import static io.trino.plugin.hive.metastore.glue.v1.TestingMetastoreObjects.getGlueTestStorageDescriptor; import static io.trino.plugin.hive.metastore.glue.v1.TestingMetastoreObjects.getGlueTestTable; import static io.trino.plugin.hive.metastore.glue.v1.TestingMetastoreObjects.getGlueTestTrinoMaterializedView; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getPartitionParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getStorageDescriptor; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableTypeNullable; import static io.trino.plugin.hive.util.HiveUtil.DELTA_LAKE_PROVIDER; import static io.trino.plugin.hive.util.HiveUtil.ICEBERG_TABLE_TYPE_NAME; import static io.trino.plugin.hive.util.HiveUtil.ICEBERG_TABLE_TYPE_VALUE; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java index 4daa639b8ce5..55406ef7c86d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java @@ -71,7 +71,7 @@ import static io.trino.metastore.TableInfo.ICEBERG_MATERIALIZED_VIEW_COMMENT; import static io.trino.plugin.hive.HiveMetadata.STORAGE_TABLE; import static io.trino.plugin.hive.ViewReaderUtil.PRESTO_VIEW_FLAG; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.mappedCopy; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.mappedCopy; import static io.trino.plugin.hive.util.HiveUtil.escapeTableName; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_FILESYSTEM_ERROR; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperations.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperations.java index 60aef66c65b8..ec88e7f0618f 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperations.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperations.java @@ -48,9 +48,9 @@ import static com.google.common.base.Verify.verify; import static io.trino.plugin.hive.ViewReaderUtil.isTrinoMaterializedView; import static io.trino.plugin.hive.ViewReaderUtil.isTrinoView; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getStorageDescriptor; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableType; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getStorageDescriptor; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableType; import static io.trino.plugin.hive.util.HiveUtil.isIcebergTable; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_COMMIT_ERROR; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index 6c8a3dfa44bb..0fe8f2fcf278 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -130,11 +130,11 @@ import static io.trino.plugin.hive.ViewReaderUtil.isTrinoMaterializedView; import static io.trino.plugin.hive.ViewReaderUtil.isTrinoView; import static io.trino.plugin.hive.metastore.glue.v1.AwsSdkUtil.getPaginatedResults; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getColumnParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getStorageDescriptor; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableType; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableTypeNullable; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getColumnParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getStorageDescriptor; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableType; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableTypeNullable; import static io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema; import static io.trino.plugin.hive.util.HiveUtil.isIcebergTable; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_BAD_DATA; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java index d88407b09309..24efa6b0e4db 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java @@ -53,9 +53,9 @@ import java.util.List; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getStorageDescriptor; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableType; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getStorageDescriptor; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableType; import static io.trino.plugin.iceberg.IcebergTestUtils.checkParquetFileSorting; import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingConnectorSession.SESSION; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedView.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedView.java index 9f3eeceab96d..299f7720af5e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedView.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedView.java @@ -40,7 +40,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.trino.plugin.base.util.Closables.closeAllSuppress; import static io.trino.plugin.hive.metastore.glue.v1.AwsSdkUtil.getPaginatedResults; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; import static io.trino.testing.TestingNames.randomNameSuffix; import static org.apache.iceberg.BaseMetastoreTableOperations.METADATA_LOCATION_PROP; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java index 69be06111502..81e86808e7db 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java @@ -48,7 +48,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; import static io.trino.plugin.hive.metastore.glue.v1.AwsSdkUtil.getPaginatedResults; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters; import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; import static io.trino.plugin.iceberg.catalog.glue.GlueIcebergUtil.getTableInput; import static io.trino.testing.TestingConnectorSession.SESSION; diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksWithGlueMetastoreCleanUp.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksWithGlueMetastoreCleanUp.java index bb600aebc6ef..70fea807582c 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksWithGlueMetastoreCleanUp.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksWithGlueMetastoreCleanUp.java @@ -34,7 +34,7 @@ import java.util.stream.Collectors; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableType; +import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableType; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; From dc191365d0e146fd133cd3209c6725123593ccf0 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 10:26:02 -0700 Subject: [PATCH 125/158] Move already exists exceptions to metastore module --- .../io/trino/metastore}/SchemaAlreadyExistsException.java | 2 +- .../java/io/trino/metastore}/TableAlreadyExistsException.java | 2 +- .../src/main/java/io/trino/plugin/hive/HiveMetadata.java | 1 + .../java/io/trino/plugin/hive/TrinoViewHiveMetastore.java | 1 + .../plugin/hive/metastore/SemiTransactionalHiveMetastore.java | 2 +- .../trino/plugin/hive/metastore/file/FileHiveMetastore.java | 4 ++-- .../trino/plugin/hive/metastore/glue/GlueHiveMetastore.java | 4 ++-- .../plugin/hive/metastore/glue/v1/GlueHiveMetastore.java | 4 ++-- .../plugin/hive/metastore/thrift/BridgingHiveMetastore.java | 4 ++-- .../plugin/hive/metastore/thrift/ThriftHiveMetastore.java | 4 ++-- .../plugin/hive/metastore/AbstractTestHiveMetastore.java | 4 ++-- .../hive/metastore/thrift/TestBridgingHiveMetastore.java | 4 ++-- .../trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java | 2 +- 13 files changed, 20 insertions(+), 18 deletions(-) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino/metastore}/SchemaAlreadyExistsException.java (97%) rename {plugin/trino-hive/src/main/java/io/trino/plugin/hive => lib/trino-metastore/src/main/java/io/trino/metastore}/TableAlreadyExistsException.java (97%) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/SchemaAlreadyExistsException.java b/lib/trino-metastore/src/main/java/io/trino/metastore/SchemaAlreadyExistsException.java similarity index 97% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/SchemaAlreadyExistsException.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/SchemaAlreadyExistsException.java index 8bff8da39a12..df46c9e8705a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/SchemaAlreadyExistsException.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/SchemaAlreadyExistsException.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive; +package io.trino.metastore; import io.trino.spi.TrinoException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TableAlreadyExistsException.java b/lib/trino-metastore/src/main/java/io/trino/metastore/TableAlreadyExistsException.java similarity index 97% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/TableAlreadyExistsException.java rename to lib/trino-metastore/src/main/java/io/trino/metastore/TableAlreadyExistsException.java index ff0e9fdae2bb..83f293809b76 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TableAlreadyExistsException.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/TableAlreadyExistsException.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive; +package io.trino.metastore; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaTableName; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 73ff0d872439..17d4542a0a0f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -50,6 +50,7 @@ import io.trino.metastore.SortingColumn; import io.trino.metastore.StorageFormat; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; import io.trino.plugin.base.projection.ApplyProjectionUtil; import io.trino.plugin.base.projection.ApplyProjectionUtil.ProjectedColumnRepresentation; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewHiveMetastore.java index df99de97138c..1e485066bdba 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewHiveMetastore.java @@ -19,6 +19,7 @@ import io.trino.metastore.HiveMetastore; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java index 070ea9e0b631..e055c767de1c 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java @@ -51,12 +51,12 @@ import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; import io.trino.plugin.hive.HiveTableHandle; import io.trino.plugin.hive.LocationHandle.WriteMode; import io.trino.plugin.hive.PartitionNotFoundException; import io.trino.plugin.hive.PartitionUpdateAndMergeResults; -import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.plugin.hive.TableInvalidationCallback; import io.trino.plugin.hive.acid.AcidTransaction; import io.trino.plugin.hive.projection.PartitionProjection; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java index 68b3b0edcc2f..9ba3811bea5f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java @@ -44,13 +44,13 @@ import io.trino.metastore.PartitionStatistics; import io.trino.metastore.PartitionWithStatistics; import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.PartitionNotFoundException; -import io.trino.plugin.hive.SchemaAlreadyExistsException; -import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.plugin.hive.TableType; import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig.VersionCompatibility; import io.trino.spi.TrinoException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java index eee2a234a016..a7103e4326cc 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java @@ -35,13 +35,13 @@ import io.trino.metastore.PartitionStatistics; import io.trino.metastore.PartitionWithStatistics; import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; import io.trino.plugin.hive.HivePartitionManager; import io.trino.plugin.hive.PartitionNotFoundException; -import io.trino.plugin.hive.SchemaAlreadyExistsException; -import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.spi.ErrorCode; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaNotFoundException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java index 79d5ec45ce24..8e24cdfcf873 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastore.java @@ -87,12 +87,12 @@ import io.trino.metastore.PartitionWithStatistics; import io.trino.metastore.Partitions; import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; import io.trino.plugin.hive.PartitionNotFoundException; -import io.trino.plugin.hive.SchemaAlreadyExistsException; -import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.plugin.hive.metastore.glue.AwsApiCallStats; import io.trino.plugin.hive.metastore.glue.GlueExpressionUtil; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java index ecb79587b170..807b39fac17e 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java @@ -32,11 +32,11 @@ import io.trino.metastore.PartitionWithStatistics; import io.trino.metastore.Partitions; import io.trino.metastore.PrincipalPrivileges; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.SchemaAlreadyExistsException; -import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaNotFoundException; import io.trino.spi.connector.SchemaTableName; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java index 7b07889050cd..ef91a6a263f6 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java @@ -69,10 +69,10 @@ import io.trino.metastore.HiveType; import io.trino.metastore.PartitionStatistics; import io.trino.metastore.PartitionWithStatistics; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.StatisticsUpdateMode; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.plugin.hive.PartitionNotFoundException; -import io.trino.plugin.hive.SchemaAlreadyExistsException; -import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.plugin.hive.util.RetryDriver; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaNotFoundException; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/AbstractTestHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/AbstractTestHiveMetastore.java index 0f7e63dcbae6..ffb7e8a2451a 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/AbstractTestHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/AbstractTestHiveMetastore.java @@ -16,9 +16,9 @@ import io.trino.metastore.Column; import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.Table; -import io.trino.plugin.hive.SchemaAlreadyExistsException; -import io.trino.plugin.hive.TableAlreadyExistsException; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.spi.connector.TableNotFoundException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestBridgingHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestBridgingHiveMetastore.java index 3c92e0f43aaa..7184f5732c51 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestBridgingHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestBridgingHiveMetastore.java @@ -15,10 +15,10 @@ import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.Table; +import io.trino.metastore.TableAlreadyExistsException; import io.trino.plugin.base.util.AutoCloseableCloser; -import io.trino.plugin.hive.SchemaAlreadyExistsException; -import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.plugin.hive.containers.HiveHadoop; import io.trino.plugin.hive.metastore.AbstractTestHiveMetastore; import org.junit.jupiter.api.AfterAll; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index 0fe8f2fcf278..12782f9957ea 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -46,8 +46,8 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.metastore.SchemaAlreadyExistsException; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.SchemaAlreadyExistsException; import io.trino.plugin.hive.TrinoViewUtil; import io.trino.plugin.hive.ViewAlreadyExistsException; import io.trino.plugin.hive.ViewReaderUtil; From 09d706203988f8c03a4add5ec3d4a10fdbdc6254 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 10:12:44 -0700 Subject: [PATCH 126/158] Move ForHiveMetastore annotation to thrift package --- .../plugin/hive/{ => metastore/thrift}/ForHiveMetastore.java | 2 +- .../metastore/thrift/KerberosHiveMetastoreAuthentication.java | 1 - .../metastore/thrift/ThriftMetastoreAuthenticationModule.java | 1 - .../plugin/hive/metastore/thrift/ThriftMetastoreModule.java | 1 - .../metastore/thrift/TokenFetchingMetastoreClientFactory.java | 1 - .../hive/metastore/thrift/UgiBasedMetastoreClientFactory.java | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) rename plugin/trino-hive/src/main/java/io/trino/plugin/hive/{ => metastore/thrift}/ForHiveMetastore.java (95%) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/ForHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ForHiveMetastore.java similarity index 95% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/ForHiveMetastore.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ForHiveMetastore.java index 1b5ca893bff4..f54ed16dc51d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/ForHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ForHiveMetastore.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive; +package io.trino.plugin.hive.metastore.thrift; import com.google.inject.BindingAnnotation; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/KerberosHiveMetastoreAuthentication.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/KerberosHiveMetastoreAuthentication.java index 834928ea07d0..5f0da82d63f0 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/KerberosHiveMetastoreAuthentication.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/KerberosHiveMetastoreAuthentication.java @@ -19,7 +19,6 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.Slices; import io.trino.plugin.base.authentication.CachingKerberosAuthentication; -import io.trino.plugin.hive.ForHiveMetastore; import org.apache.thrift.transport.TSaslClientTransport; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java index 066c6628e01f..b244f7f0cdcb 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java @@ -21,7 +21,6 @@ import io.trino.plugin.base.authentication.CachingKerberosAuthentication; import io.trino.plugin.base.authentication.KerberosAuthentication; import io.trino.plugin.base.authentication.KerberosConfiguration; -import io.trino.plugin.hive.ForHiveMetastore; import static com.google.inject.Scopes.SINGLETON; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java index c83d47e60aff..4bef109c2f4d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreModule.java @@ -25,7 +25,6 @@ import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.plugin.base.security.UserNameProvider; import io.trino.plugin.hive.AllowHiveTableRename; -import io.trino.plugin.hive.ForHiveMetastore; import java.util.concurrent.ExecutorService; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/TokenFetchingMetastoreClientFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/TokenFetchingMetastoreClientFactory.java index ab95396c19e1..3f68f3499633 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/TokenFetchingMetastoreClientFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/TokenFetchingMetastoreClientFactory.java @@ -23,7 +23,6 @@ import dev.failsafe.function.CheckedSupplier; import io.trino.cache.NonEvictableLoadingCache; import io.trino.plugin.base.security.UserNameProvider; -import io.trino.plugin.hive.ForHiveMetastore; import io.trino.spi.TrinoException; import io.trino.spi.security.ConnectorIdentity; import org.apache.thrift.TException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/UgiBasedMetastoreClientFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/UgiBasedMetastoreClientFactory.java index f69bd4575a94..5408300b951d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/UgiBasedMetastoreClientFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/UgiBasedMetastoreClientFactory.java @@ -15,7 +15,6 @@ import com.google.inject.Inject; import io.trino.plugin.base.security.UserNameProvider; -import io.trino.plugin.hive.ForHiveMetastore; import io.trino.spi.security.ConnectorIdentity; import org.apache.thrift.TException; From 9beb0bca7fe923bc67a04a87bb54a01a46cd3ae7 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 10:38:17 -0700 Subject: [PATCH 127/158] Move RetryDriver to thrift package --- .../plugin/hive/{util => metastore/thrift}/RetryDriver.java | 2 +- .../trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java | 1 - .../hive/metastore/thrift/ThriftHttpMetastoreFactory.java | 1 - .../plugin/hive/metastore/thrift/ThriftMetastoreConfig.java | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) rename plugin/trino-hive/src/main/java/io/trino/plugin/hive/{util => metastore/thrift}/RetryDriver.java (99%) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/RetryDriver.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/RetryDriver.java similarity index 99% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/RetryDriver.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/RetryDriver.java index 10f31de8482d..7d1aecdd9924 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/RetryDriver.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/RetryDriver.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.util; +package io.trino.plugin.hive.metastore.thrift; import com.google.common.collect.ImmutableList; import io.airlift.log.Logger; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java index ef91a6a263f6..41fdbffd42ad 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java @@ -73,7 +73,6 @@ import io.trino.metastore.StatisticsUpdateMode; import io.trino.metastore.TableAlreadyExistsException; import io.trino.plugin.hive.PartitionNotFoundException; -import io.trino.plugin.hive.util.RetryDriver; import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaNotFoundException; import io.trino.spi.connector.SchemaTableName; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHttpMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHttpMetastoreFactory.java index bfc8e8c7820f..26e13e5abae0 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHttpMetastoreFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHttpMetastoreFactory.java @@ -16,7 +16,6 @@ import com.google.inject.Inject; import io.airlift.units.Duration; import io.trino.filesystem.TrinoFileSystemFactory; -import io.trino.plugin.hive.util.RetryDriver; import io.trino.spi.security.ConnectorIdentity; import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreConfig.java index b4511e45c17b..ae4c96b10b83 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreConfig.java @@ -22,7 +22,6 @@ import io.airlift.configuration.validation.FileExists; import io.airlift.units.Duration; import io.airlift.units.MinDuration; -import io.trino.plugin.hive.util.RetryDriver; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; From 6e795b155c5bc21c32dc897f9a350551b1f71000 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 31 Jul 2024 10:52:25 -0700 Subject: [PATCH 128/158] Move Avro utility method to ThriftMetastoreUtil --- .../trino/plugin/hive/metastore/MetastoreUtil.java | 12 ------------ .../hive/metastore/thrift/BridgingHiveMetastore.java | 1 - .../hive/metastore/thrift/ThriftMetastoreUtil.java | 9 +++++++++ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java index 561a4a336ff7..fa851d819ed8 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/MetastoreUtil.java @@ -89,10 +89,7 @@ import static io.trino.hive.thrift.metastore.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS; import static io.trino.hive.thrift.metastore.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES; import static io.trino.metastore.Partitions.makePartName; -import static io.trino.plugin.hive.HiveMetadata.AVRO_SCHEMA_LITERAL_KEY; -import static io.trino.plugin.hive.HiveMetadata.AVRO_SCHEMA_URL_KEY; import static io.trino.plugin.hive.HiveSplitManager.PRESTO_OFFLINE; -import static io.trino.plugin.hive.HiveStorageFormat.AVRO; import static io.trino.plugin.hive.metastore.SparkMetastoreUtil.getSparkBasicStatistics; import static io.trino.plugin.hive.util.SerdeConstants.LIST_COLUMN_COMMENTS; import static io.trino.plugin.hive.util.SerdeConstants.SERIALIZATION_LIB; @@ -243,15 +240,6 @@ public static ProtectMode getProtectMode(Table table) return getProtectMode(table.getParameters()); } - public static boolean isAvroTableWithSchemaSet(Table table) - { - return AVRO.getSerde().equals(table.getStorage().getStorageFormat().getSerDeNullable()) && - ((table.getParameters().get(AVRO_SCHEMA_URL_KEY) != null || - (table.getStorage().getSerdeParameters().get(AVRO_SCHEMA_URL_KEY) != null)) || - (table.getParameters().get(AVRO_SCHEMA_LITERAL_KEY) != null || - (table.getStorage().getSerdeParameters().get(AVRO_SCHEMA_LITERAL_KEY) != null))); - } - public static String makePartitionName(Table table, Partition partition) { return makePartitionName(table.getPartitionColumns(), partition.getValues()); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java index 807b39fac17e..e66c5fdf76e1 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/BridgingHiveMetastore.java @@ -57,7 +57,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.metastore.Table.TABLE_COMMENT; import static io.trino.plugin.hive.HiveMetadata.TRINO_QUERY_ID_NAME; -import static io.trino.plugin.hive.metastore.MetastoreUtil.isAvroTableWithSchemaSet; import static io.trino.plugin.hive.metastore.MetastoreUtil.metastoreFunctionName; import static io.trino.plugin.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.csvSchemaFields; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreUtil.java index 7fd79ede9317..1e377956c03d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreUtil.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreUtil.java @@ -974,4 +974,13 @@ public static DataOperationType toDataOperationType(AcidOperation acidOperation) default -> throw new IllegalStateException("No metastore operation for ACID operation " + acidOperation); }; } + + public static boolean isAvroTableWithSchemaSet(Table table) + { + return AVRO.getSerde().equals(table.getStorage().getStorageFormat().getSerDeNullable()) && + ((table.getParameters().get(AVRO_SCHEMA_URL_KEY) != null || + (table.getStorage().getSerdeParameters().get(AVRO_SCHEMA_URL_KEY) != null)) || + (table.getParameters().get(AVRO_SCHEMA_LITERAL_KEY) != null || + (table.getStorage().getSerdeParameters().get(AVRO_SCHEMA_LITERAL_KEY) != null))); + } } From cdd0a9f0709cacbe4048a2ecc48095330271ed32 Mon Sep 17 00:00:00 2001 From: Anu Sudarsan Date: Mon, 23 Dec 2024 13:54:37 -0600 Subject: [PATCH 129/158] Add SSE-C option on native-filesystem security mapping --- .../sphinx/object-storage/file-system-s3.md | 5 + .../io/trino/filesystem/s3/S3Context.java | 11 ++ .../filesystem/s3/S3FileSystemLoader.java | 6 + .../filesystem/s3/S3SecurityMapping.java | 20 +++ .../s3/S3SecurityMappingConfig.java | 14 ++ .../s3/S3SecurityMappingProvider.java | 31 ++++ .../s3/S3SecurityMappingResult.java | 2 + .../filesystem/s3/TestS3SecurityMapping.java | 147 ++++++++++++++++-- .../s3/TestS3SecurityMappingConfig.java | 5 + .../trino/filesystem/s3/security-mapping.json | 20 +++ 10 files changed, 251 insertions(+), 10 deletions(-) diff --git a/docs/src/main/sphinx/object-storage/file-system-s3.md b/docs/src/main/sphinx/object-storage/file-system-s3.md index 926d31cb870d..e18f188cdcc8 100644 --- a/docs/src/main/sphinx/object-storage/file-system-s3.md +++ b/docs/src/main/sphinx/object-storage/file-system-s3.md @@ -174,6 +174,9 @@ The security mapping must provide one or more configuration settings: - `kmsKeyId`: ID of KMS-managed key to be used for client-side encryption. - `allowedKmsKeyIds`: KMS-managed key IDs that are allowed to be specified as an extra credential. If list cotains `*`, then any key can be specified via extra credential. +- `sseCustomerKey`: The customer provided key (SSE-C) for server-side encryption. +- `allowedSseCustomerKey`: The SSE-C keys that are allowed to be specified as an extra + credential. If list cotains `*`, then any key can be specified via extra credential. - `endpoint`: The S3 storage endpoint server. This optional property can be used to override S3 endpoints on a per-bucket basis. - `region`: The S3 region to connect to. This optional property can be used @@ -262,6 +265,8 @@ Example JSON configuration: - The name of the *extra credential* used to provide the IAM role. * - `s3.security-mapping.kms-key-id-credential-name` - The name of the *extra credential* used to provide the KMS-managed key ID. +* - `s3.security-mapping.sse-customer-key-credential-name` + - The name of the *extra credential* used to provide the server-side encryption with customer-provided keys (SSE-C). * - `s3.security-mapping.refresh-period` - How often to refresh the security mapping configuration, specified as a {ref}`prop-type-duration`. By default, the configuration is not refreshed. diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3Context.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3Context.java index eeb33295d832..cc9dbf4ffeeb 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3Context.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3Context.java @@ -25,6 +25,7 @@ import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; +import static io.trino.filesystem.s3.S3FileSystemConfig.S3SseType.CUSTOMER; import static io.trino.filesystem.s3.S3FileSystemConfig.S3SseType.KMS; import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_ACCESS_KEY_PROPERTY; import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_SECRET_KEY_PROPERTY; @@ -70,6 +71,11 @@ public S3Context withCredentials(ConnectorIdentity identity) return this; } + public S3Context withSseCustomerKey(String key) + { + return new S3Context(partSize, requesterPays, S3SseContext.withSseCustomerKey(key), credentialsProviderOverride, cannedAcl, exclusiveWriteSupported); + } + public S3Context withCredentialsProviderOverride(AwsCredentialsProvider credentialsProviderOverride) { return new S3Context( @@ -109,5 +115,10 @@ public static S3SseContext withKmsKeyId(String kmsKeyId) { return new S3SseContext(KMS, Optional.ofNullable(kmsKeyId), Optional.empty()); } + + public static S3SseContext withSseCustomerKey(String key) + { + return new S3SseContext(CUSTOMER, Optional.empty(), Optional.ofNullable(key).map(S3SseCustomerKey::onAes256)); + } } } diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java index bb011a336f5c..be1531030181 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java @@ -46,6 +46,7 @@ import java.util.concurrent.ExecutorService; import java.util.function.Function; +import static com.google.common.base.Preconditions.checkState; import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.trino.filesystem.s3.S3FileSystemConfig.RetryMode.getRetryStrategy; import static java.lang.Math.toIntExact; @@ -108,9 +109,14 @@ public TrinoFileSystemFactory apply(Location location) S3Context context = this.context.withCredentials(identity); if (mapping.isPresent() && mapping.get().kmsKeyId().isPresent()) { + checkState(mapping.get().sseCustomerKey().isEmpty(), "Both SSE-C and KMS-managed keys cannot be used at the same time"); context = context.withKmsKeyId(mapping.get().kmsKeyId().get()); } + if (mapping.isPresent() && mapping.get().sseCustomerKey().isPresent()) { + context = context.withSseCustomerKey(mapping.get().sseCustomerKey().get()); + } + return new S3FileSystem(uploadExecutor, client, preSigner, context); }; } diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMapping.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMapping.java index 4b65e5fce887..064040d06295 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMapping.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMapping.java @@ -42,6 +42,8 @@ public final class S3SecurityMapping private final Set allowedIamRoles; private final Optional kmsKeyId; private final Set allowedKmsKeyIds; + private final Optional sseCustomerKey; + private final Set allowedSseCustomerKeys; private final Optional credentials; private final boolean useClusterDefault; private final Optional endpoint; @@ -57,6 +59,8 @@ public S3SecurityMapping( @JsonProperty("allowedIamRoles") Optional> allowedIamRoles, @JsonProperty("kmsKeyId") Optional kmsKeyId, @JsonProperty("allowedKmsKeyIds") Optional> allowedKmsKeyIds, + @JsonProperty("sseCustomerKey") Optional sseCustomerKey, + @JsonProperty("allowedSseCustomerKeys") Optional> allowedSseCustomerKeys, @JsonProperty("accessKey") Optional accessKey, @JsonProperty("secretKey") Optional secretKey, @JsonProperty("useClusterDefault") Optional useClusterDefault, @@ -86,6 +90,10 @@ public S3SecurityMapping( this.allowedKmsKeyIds = ImmutableSet.copyOf(allowedKmsKeyIds.orElse(ImmutableList.of())); + this.sseCustomerKey = requireNonNull(sseCustomerKey, "sseCustomerKey is null"); + + this.allowedSseCustomerKeys = allowedSseCustomerKeys.map(ImmutableSet::copyOf).orElse(ImmutableSet.of()); + requireNonNull(accessKey, "accessKey is null"); requireNonNull(secretKey, "secretKey is null"); checkArgument(accessKey.isPresent() == secretKey.isPresent(), "accessKey and secretKey must be provided together"); @@ -96,6 +104,8 @@ public S3SecurityMapping( checkArgument(this.useClusterDefault != roleOrCredentialsArePresent, "must either allow useClusterDefault role or provide role and/or credentials"); checkArgument(!this.useClusterDefault || this.kmsKeyId.isEmpty(), "KMS key ID cannot be provided together with useClusterDefault"); + checkArgument(!this.useClusterDefault || this.sseCustomerKey.isEmpty(), "SSE Customer key cannot be provided together with useClusterDefault"); + checkArgument(this.kmsKeyId.isEmpty() || this.sseCustomerKey.isEmpty(), "SSE Customer key cannot be provided together with KMS key ID"); this.endpoint = requireNonNull(endpoint, "endpoint is null"); this.region = requireNonNull(region, "region is null"); @@ -133,6 +143,16 @@ public Set allowedKmsKeyIds() return allowedKmsKeyIds; } + public Optional sseCustomerKey() + { + return sseCustomerKey; + } + + public Set allowedSseCustomerKeys() + { + return allowedSseCustomerKeys; + } + public Optional credentials() { return credentials; diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingConfig.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingConfig.java index 2f1d6062f7cb..8bf2b8cbe499 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingConfig.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingConfig.java @@ -31,6 +31,7 @@ public class S3SecurityMappingConfig private String jsonPointer = ""; private String roleCredentialName; private String kmsKeyIdCredentialName; + private String sseCustomerKeyCredentialName; private Duration refreshPeriod; private String colonReplacement; @@ -100,6 +101,19 @@ public S3SecurityMappingConfig setKmsKeyIdCredentialName(String kmsKeyIdCredenti return this; } + public Optional getSseCustomerKeyCredentialName() + { + return Optional.ofNullable(sseCustomerKeyCredentialName); + } + + @Config("s3.security-mapping.sse-customer-key-credential-name") + @ConfigDescription("Name of the extra credential used to provide SSE Customer key") + public S3SecurityMappingConfig setSseCustomerKeyCredentialName(String sseCustomerKeyCredentialName) + { + this.sseCustomerKeyCredentialName = sseCustomerKeyCredentialName; + return this; + } + public Optional getRefreshPeriod() { return Optional.ofNullable(refreshPeriod); diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingProvider.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingProvider.java index e0823c2c5fcc..008fb835f6bf 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingProvider.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingProvider.java @@ -33,6 +33,7 @@ final class S3SecurityMappingProvider private final Supplier mappingsProvider; private final Optional roleCredentialName; private final Optional kmsKeyIdCredentialName; + private final Optional sseCustomerKeyCredentialName; private final Optional colonReplacement; @Inject @@ -41,6 +42,7 @@ public S3SecurityMappingProvider(S3SecurityMappingConfig config, Supplier mappingsProvider, Optional roleCredentialName, Optional kmsKeyIdCredentialName, + Optional sseCustomerKeyCredentialName, Optional colonReplacement) { this.mappingsProvider = requireNonNull(mappingsProvider, "mappingsProvider is null"); this.roleCredentialName = requireNonNull(roleCredentialName, "roleCredentialName is null"); this.kmsKeyIdCredentialName = requireNonNull(kmsKeyIdCredentialName, "kmsKeyIdCredentialName is null"); + this.sseCustomerKeyCredentialName = requireNonNull(sseCustomerKeyCredentialName, "customerKeyCredentialName is null"); this.colonReplacement = requireNonNull(colonReplacement, "colonReplacement is null"); } @@ -70,6 +74,7 @@ public Optional getMapping(ConnectorIdentity identity, selectRole(mapping, identity), mapping.roleSessionName().map(name -> name.replace("${USER}", identity.getUser())), selectKmsKeyId(mapping, identity), + getSseCustomerKey(mapping, identity), mapping.endpoint(), mapping.region())); } @@ -131,6 +136,32 @@ private Optional getKmsKeyIdFromExtraCredential(ConnectorIdentity identi return kmsKeyIdCredentialName.map(name -> identity.getExtraCredentials().get(name)); } + private Optional getSseCustomerKey(S3SecurityMapping mapping, ConnectorIdentity identity) + { + Optional providedKey = getSseCustomerKeyFromExtraCredential(identity); + + if (providedKey.isEmpty()) { + return mapping.sseCustomerKey(); + } + if (mapping.sseCustomerKey().isPresent() && mapping.allowedSseCustomerKeys().isEmpty()) { + throw new AccessDeniedException("allowedSseCustomerKeys must be set if sseCustomerKey is provided"); + } + + String selected = providedKey.get(); + + if (selected.equals(mapping.sseCustomerKey().orElse(null)) || + mapping.allowedSseCustomerKeys().contains(selected) || + mapping.allowedSseCustomerKeys().contains("*")) { + return providedKey; + } + throw new AccessDeniedException("Provided SSE Customer Key is not allowed"); + } + + private Optional getSseCustomerKeyFromExtraCredential(ConnectorIdentity identity) + { + return sseCustomerKeyCredentialName.map(name -> identity.getExtraCredentials().get(name)); + } + private static Supplier mappingsProvider(Supplier supplier, Optional refreshPeriod) { return refreshPeriod diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingResult.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingResult.java index 23514f2d1a83..2d2ef7dc2e92 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingResult.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3SecurityMappingResult.java @@ -26,6 +26,7 @@ record S3SecurityMappingResult( Optional iamRole, Optional roleSessionName, Optional kmsKeyId, + Optional sseCustomerKey, Optional endpoint, Optional region) { @@ -35,6 +36,7 @@ record S3SecurityMappingResult( requireNonNull(iamRole, "iamRole is null"); requireNonNull(roleSessionName, "roleSessionName is null"); requireNonNull(kmsKeyId, "kmsKeyId is null"); + requireNonNull(sseCustomerKey, "sseCustomerKey is null"); requireNonNull(endpoint, "endpoint is null"); requireNonNull(region, "region is null"); } diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMapping.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMapping.java index f301a83d9234..b3b0809ec581 100644 --- a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMapping.java +++ b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMapping.java @@ -37,6 +37,7 @@ public class TestS3SecurityMapping { private static final String IAM_ROLE_CREDENTIAL_NAME = "IAM_ROLE_CREDENTIAL_NAME"; private static final String KMS_KEY_ID_CREDENTIAL_NAME = "KMS_KEY_ID_CREDENTIAL_NAME"; + private static final String CUSTOMER_KEY_CREDENTIAL_NAME = "CUSTOMER_KEY_CREDENTIAL_NAME"; private static final String DEFAULT_PATH = "s3://default/"; private static final String DEFAULT_USER = "testuser"; @@ -47,6 +48,7 @@ public void testMapping() .setConfigFile(getResourceFile("security-mapping.json")) .setRoleCredentialName(IAM_ROLE_CREDENTIAL_NAME) .setKmsKeyIdCredentialName(KMS_KEY_ID_CREDENTIAL_NAME) + .setSseCustomerKeyCredentialName(CUSTOMER_KEY_CREDENTIAL_NAME) .setColonReplacement("#"); var provider = new S3SecurityMappingProvider(mappingConfig, new S3SecurityMappingsFileSource(mappingConfig)); @@ -104,6 +106,45 @@ public void testMapping() credentials("AKIAxxxaccess", "iXbXxxxsecret") .withKmsKeyId("kmsKey_12")); + // matches prefix exactly -- mapping provides credentials, customer key from extra credentials matching default + assertMapping( + provider, + path("s3://baz/") + .withExtraCredentialCustomerKey("customerKey_10"), + credentials("AKIAxxxaccess", "iXbXxxxsecret") + .withSseCustomerKey("customerKey_10")); + + // matches prefix exactly -- mapping provides credentials, customer key from extra credentials, allowed, different from default + assertMapping( + provider, + path("s3://baz/") + .withExtraCredentialCustomerKey("customerKey_11"), + credentials("AKIAxxxaccess", "iXbXxxxsecret") + .withSseCustomerKey("customerKey_11")); + + // matches prefix exactly -- mapping provides credentials, customer key from extra credentials, not allowed + assertMappingFails( + provider, + path("s3://baz/") + .withExtraCredentialCustomerKey("customerKey_not_allowed"), + "Provided SSE Customer Key is not allowed"); + + // matches prefix exactly -- mapping provides credentials, customer key from extra credentials, all keys are allowed, different from default + assertMapping( + provider, + path("s3://baz_all_customer_keys_allowed/") + .withExtraCredentialCustomerKey("customerKey_777"), + credentials("AKIAxxxaccess", "iXbXxxxsecret") + .withSseCustomerKey("customerKey_777")); + + // matches prefix exactly -- mapping provides credentials, customer key from extra credentials, allowed, no default key + assertMapping( + provider, + path("s3://baz_no_customer_default_key/") + .withExtraCredentialCustomerKey("customerKey_12"), + credentials("AKIAxxxaccess", "iXbXxxxsecret") + .withSseCustomerKey("customerKey_12")); + // no role selected and mapping has no default role assertMappingFails( provider, @@ -320,6 +361,8 @@ public void testMappingWithoutRoleCredentialsFallbackShouldFail() Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), Optional.empty())) .isInstanceOf(IllegalArgumentException.class) .hasMessage("must either allow useClusterDefault role or provide role and/or credentials"); @@ -343,6 +386,8 @@ public void testMappingWithRoleAndFallbackShouldFail() Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), useClusterDefault, Optional.empty(), Optional.empty())) @@ -368,6 +413,8 @@ public void testMappingWithEncryptionKeysAndFallbackShouldFail() Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), useClusterDefault, Optional.empty(), Optional.empty())) @@ -375,6 +422,60 @@ public void testMappingWithEncryptionKeysAndFallbackShouldFail() .hasMessage("KMS key ID cannot be provided together with useClusterDefault"); } + @Test + public void testMappingWithSseCustomerKeyAndFallbackShouldFail() + { + Optional useClusterDefault = Optional.of(true); + Optional sseCustomerKey = Optional.of("CLIENT_S3CRT_CUSTOMER_KEY"); + + assertThatThrownBy(() -> + new S3SecurityMapping( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + sseCustomerKey, + Optional.empty(), + Optional.empty(), + Optional.empty(), + useClusterDefault, + Optional.empty(), + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("SSE Customer key cannot be provided together with useClusterDefault"); + } + + @Test + public void testMappingWithSseCustomerAndKMSKeysShouldFail() + { + Optional kmsKeyId = Optional.of("CLIENT_S3CRT_KEY_ID"); + Optional sseCustomerKey = Optional.of("CLIENT_S3CRT_CUSTOMER_KEY"); + + assertThatThrownBy(() -> + new S3SecurityMapping( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of("arn:aws:iam::123456789101:role/allow_path"), + Optional.empty(), + Optional.empty(), + kmsKeyId, + Optional.empty(), + sseCustomerKey, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("SSE Customer key cannot be provided together with KMS key ID"); + } + @Test public void testMappingWithRoleSessionNameWithoutIamRoleShouldFail() { @@ -394,6 +495,8 @@ public void testMappingWithRoleSessionNameWithoutIamRoleShouldFail() Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), Optional.empty())) .isInstanceOf(IllegalArgumentException.class) .hasMessage("iamRole must be provided when roleSessionName is provided"); @@ -414,6 +517,7 @@ private static void assertMapping(S3SecurityMappingProvider provider, MappingSel assertThat(actual.iamRole()).isEqualTo(expected.iamRole()); assertThat(actual.roleSessionName()).isEqualTo(expected.roleSessionName()); assertThat(actual.kmsKeyId()).isEqualTo(expected.kmsKeyId()); + assertThat(actual.sseCustomerKey()).isEqualTo(expected.sseCustomerKey()); assertThat(actual.endpoint()).isEqualTo(expected.endpoint()); assertThat(actual.region()).isEqualTo(expected.region()); }); @@ -440,7 +544,7 @@ public static MappingSelector empty() public static MappingSelector path(String location) { - return new MappingSelector(DEFAULT_USER, ImmutableSet.of(), Location.of(location), Optional.empty(), Optional.empty()); + return new MappingSelector(DEFAULT_USER, ImmutableSet.of(), Location.of(location), Optional.empty(), Optional.empty(), Optional.empty()); } private final String user; @@ -448,14 +552,16 @@ public static MappingSelector path(String location) private final Location location; private final Optional extraCredentialIamRole; private final Optional extraCredentialKmsKeyId; + private final Optional extraCredentialCustomerKey; - private MappingSelector(String user, Set groups, Location location, Optional extraCredentialIamRole, Optional extraCredentialKmsKeyId) + private MappingSelector(String user, Set groups, Location location, Optional extraCredentialIamRole, Optional extraCredentialKmsKeyId, Optional extraCredentialCustomerKey) { this.user = requireNonNull(user, "user is null"); this.groups = ImmutableSet.copyOf(requireNonNull(groups, "groups is null")); this.location = requireNonNull(location, "location is null"); this.extraCredentialIamRole = requireNonNull(extraCredentialIamRole, "extraCredentialIamRole is null"); this.extraCredentialKmsKeyId = requireNonNull(extraCredentialKmsKeyId, "extraCredentialKmsKeyId is null"); + this.extraCredentialCustomerKey = requireNonNull(extraCredentialCustomerKey, "extraCredentialCustomerKey is null"); } public Location location() @@ -465,22 +571,27 @@ public Location location() public MappingSelector withExtraCredentialIamRole(String role) { - return new MappingSelector(user, groups, location, Optional.of(role), extraCredentialKmsKeyId); + return new MappingSelector(user, groups, location, Optional.of(role), extraCredentialKmsKeyId, extraCredentialCustomerKey); } public MappingSelector withExtraCredentialKmsKeyId(String kmsKeyId) { - return new MappingSelector(user, groups, location, extraCredentialIamRole, Optional.of(kmsKeyId)); + return new MappingSelector(user, groups, location, extraCredentialIamRole, Optional.of(kmsKeyId), Optional.empty()); + } + + public MappingSelector withExtraCredentialCustomerKey(String customerKey) + { + return new MappingSelector(user, groups, location, extraCredentialIamRole, Optional.empty(), Optional.of(customerKey)); } public MappingSelector withUser(String user) { - return new MappingSelector(user, groups, location, extraCredentialIamRole, extraCredentialKmsKeyId); + return new MappingSelector(user, groups, location, extraCredentialIamRole, extraCredentialKmsKeyId, extraCredentialCustomerKey); } public MappingSelector withGroups(String... groups) { - return new MappingSelector(user, ImmutableSet.copyOf(groups), location, extraCredentialIamRole, extraCredentialKmsKeyId); + return new MappingSelector(user, ImmutableSet.copyOf(groups), location, extraCredentialIamRole, extraCredentialKmsKeyId, extraCredentialCustomerKey); } public ConnectorIdentity identity() @@ -488,6 +599,7 @@ public ConnectorIdentity identity() Map extraCredentials = new HashMap<>(); extraCredentialIamRole.ifPresent(role -> extraCredentials.put(IAM_ROLE_CREDENTIAL_NAME, role)); extraCredentialKmsKeyId.ifPresent(kmsKeyId -> extraCredentials.put(KMS_KEY_ID_CREDENTIAL_NAME, kmsKeyId)); + extraCredentialCustomerKey.ifPresent(customerKey -> extraCredentials.put(CUSTOMER_KEY_CREDENTIAL_NAME, customerKey)); return ConnectorIdentity.forUser(user) .withGroups(groups) @@ -507,6 +619,7 @@ public static MappingResult credentials(String accessKey, String secretKey) Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); } @@ -519,6 +632,7 @@ public static MappingResult iamRole(String role) Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); } @@ -527,6 +641,7 @@ public static MappingResult iamRole(String role) private final Optional iamRole; private final Optional roleSessionName; private final Optional kmsKeyId; + private final Optional sseCustomerKey; private final Optional endpoint; private final Optional region; @@ -536,6 +651,7 @@ private MappingResult( Optional iamRole, Optional roleSessionName, Optional kmsKeyId, + Optional sseCustomerKey, Optional endpoint, Optional region) { @@ -543,6 +659,7 @@ private MappingResult( this.secretKey = requireNonNull(secretKey, "secretKey is null"); this.iamRole = requireNonNull(iamRole, "role is null"); this.kmsKeyId = requireNonNull(kmsKeyId, "kmsKeyId is null"); + this.sseCustomerKey = requireNonNull(sseCustomerKey, "sseCustomerKey is null"); this.endpoint = requireNonNull(endpoint, "endpoint is null"); this.roleSessionName = requireNonNull(roleSessionName, "roleSessionName is null"); this.region = requireNonNull(region, "region is null"); @@ -550,22 +667,27 @@ private MappingResult( public MappingResult withEndpoint(String endpoint) { - return new MappingResult(accessKey, secretKey, iamRole, Optional.empty(), kmsKeyId, Optional.of(endpoint), region); + return new MappingResult(accessKey, secretKey, iamRole, Optional.empty(), kmsKeyId, sseCustomerKey, Optional.of(endpoint), region); } public MappingResult withKmsKeyId(String kmsKeyId) { - return new MappingResult(accessKey, secretKey, iamRole, Optional.empty(), Optional.of(kmsKeyId), endpoint, region); + return new MappingResult(accessKey, secretKey, iamRole, Optional.empty(), Optional.of(kmsKeyId), Optional.empty(), endpoint, region); + } + + public MappingResult withSseCustomerKey(String customerKey) + { + return new MappingResult(accessKey, secretKey, iamRole, Optional.empty(), Optional.empty(), Optional.of(customerKey), endpoint, region); } public MappingResult withRegion(String region) { - return new MappingResult(accessKey, secretKey, iamRole, Optional.empty(), kmsKeyId, endpoint, Optional.of(region)); + return new MappingResult(accessKey, secretKey, iamRole, Optional.empty(), kmsKeyId, sseCustomerKey, endpoint, Optional.of(region)); } public MappingResult withRoleSessionName(String roleSessionName) { - return new MappingResult(accessKey, secretKey, iamRole, Optional.of(roleSessionName), kmsKeyId, Optional.empty(), region); + return new MappingResult(accessKey, secretKey, iamRole, Optional.of(roleSessionName), kmsKeyId, sseCustomerKey, Optional.empty(), region); } public Optional accessKey() @@ -593,6 +715,11 @@ public Optional kmsKeyId() return kmsKeyId; } + public Optional sseCustomerKey() + { + return sseCustomerKey; + } + public Optional endpoint() { return endpoint; diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMappingConfig.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMappingConfig.java index 0aabecd00745..ec51e87fd5bf 100644 --- a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMappingConfig.java +++ b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3SecurityMappingConfig.java @@ -44,6 +44,7 @@ public void testDefaults() .setConfigUri(null) .setRoleCredentialName(null) .setKmsKeyIdCredentialName(null) + .setSseCustomerKeyCredentialName(null) .setRefreshPeriod(null) .setColonReplacement(null)); } @@ -59,6 +60,7 @@ public void testExplicitPropertyMappingsWithFile() .put("s3.security-mapping.json-pointer", "/data") .put("s3.security-mapping.iam-role-credential-name", "iam-role-credential-name") .put("s3.security-mapping.kms-key-id-credential-name", "kms-key-id-credential-name") + .put("s3.security-mapping.sse-customer-key-credential-name", "sse-customer-key-credential-name") .put("s3.security-mapping.refresh-period", "13s") .put("s3.security-mapping.colon-replacement", "#") .buildOrThrow(); @@ -71,6 +73,7 @@ public void testExplicitPropertyMappingsWithFile() assertThat(config.getJsonPointer()).isEqualTo("/data"); assertThat(config.getRoleCredentialName()).contains("iam-role-credential-name"); assertThat(config.getKmsKeyIdCredentialName()).contains("kms-key-id-credential-name"); + assertThat(config.getSseCustomerKeyCredentialName()).contains("sse-customer-key-credential-name"); assertThat(config.getRefreshPeriod()).contains(new Duration(13, SECONDS)); assertThat(config.getColonReplacement()).contains("#"); } @@ -83,6 +86,7 @@ public void testExplicitPropertyMappingsWithUrl() .put("s3.security-mapping.json-pointer", "/data") .put("s3.security-mapping.iam-role-credential-name", "iam-role-credential-name") .put("s3.security-mapping.kms-key-id-credential-name", "kms-key-id-credential-name") + .put("s3.security-mapping.sse-customer-key-credential-name", "sse-customer-key-credential-name") .put("s3.security-mapping.refresh-period", "13s") .put("s3.security-mapping.colon-replacement", "#") .buildOrThrow(); @@ -95,6 +99,7 @@ public void testExplicitPropertyMappingsWithUrl() assertThat(config.getJsonPointer()).isEqualTo("/data"); assertThat(config.getRoleCredentialName()).contains("iam-role-credential-name"); assertThat(config.getKmsKeyIdCredentialName()).contains("kms-key-id-credential-name"); + assertThat(config.getSseCustomerKeyCredentialName()).contains("sse-customer-key-credential-name"); assertThat(config.getRefreshPeriod()).contains(new Duration(13, SECONDS)); assertThat(config.getColonReplacement()).contains("#"); } diff --git a/lib/trino-filesystem-s3/src/test/resources/io/trino/filesystem/s3/security-mapping.json b/lib/trino-filesystem-s3/src/test/resources/io/trino/filesystem/s3/security-mapping.json index a9ddf6e5638d..44e0d58d4a59 100644 --- a/lib/trino-filesystem-s3/src/test/resources/io/trino/filesystem/s3/security-mapping.json +++ b/lib/trino-filesystem-s3/src/test/resources/io/trino/filesystem/s3/security-mapping.json @@ -40,6 +40,26 @@ "secretKey": "iXbXxxxsecret", "allowedKmsKeyIds": ["kmsKey_11", "kmsKey_12"] }, + { + "prefix": "s3://baz/", + "accessKey": "AKIAxxxaccess", + "secretKey": "iXbXxxxsecret", + "sseCustomerKey": "customerKey_10", + "allowedSseCustomerKeys": ["customerKey_11"] + }, + { + "prefix": "s3://baz_all_customer_keys_allowed/", + "accessKey": "AKIAxxxaccess", + "secretKey": "iXbXxxxsecret", + "sseCustomerKey": "customerKey_10", + "allowedSseCustomerKeys": ["*"] + }, + { + "prefix": "s3://baz_no_customer_default_key/", + "accessKey": "AKIAxxxaccess", + "secretKey": "iXbXxxxsecret", + "allowedSseCustomerKeys": ["customerKey_11", "customerKey_12"] + }, { "user": "alice", "iamRole": "alice_role" From ac7316301399ef830a702e064a264245d10d37f8 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:26:46 -0800 Subject: [PATCH 130/158] Remove SPI exclusion from previous release --- core/trino-spi/pom.xml | 43 ------------------------------------------ 1 file changed, 43 deletions(-) diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index da80d6442716..aaf6a9892f99 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -212,49 +212,6 @@ - - true - java.method.returnTypeTypeParametersChanged - method java.lang.Iterable<io.trino.spi.protocol.SpoolingManagerFactory> io.trino.spi.Plugin::getSpoolingManagerFactories() - method java.lang.Iterable<io.trino.spi.spool.SpoolingManagerFactory> io.trino.spi.Plugin::getSpoolingManagerFactories() - Spooling package renamed - - - true - java.class.removed - interface io.trino.spi.protocol.SpoolingManager - Spooling package renamed - - - java.method.noLongerDefault - method io.trino.spi.security.SystemAccessControl io.trino.spi.security.SystemAccessControlFactory::create(java.util.Map<java.lang.String, java.lang.String>, io.trino.spi.security.SystemAccessControlFactory.SystemAccessControlContext) - method io.trino.spi.security.SystemAccessControl io.trino.spi.security.SystemAccessControlFactory::create(java.util.Map<java.lang.String, java.lang.String>, io.trino.spi.security.SystemAccessControlFactory.SystemAccessControlContext) - Old variant was removed and this becomes the new non-default one - - - java.method.nowAbstract - method io.trino.spi.security.SystemAccessControl io.trino.spi.security.SystemAccessControlFactory::create(java.util.Map<java.lang.String, java.lang.String>, io.trino.spi.security.SystemAccessControlFactory.SystemAccessControlContext) - method io.trino.spi.security.SystemAccessControl io.trino.spi.security.SystemAccessControlFactory::create(java.util.Map<java.lang.String, java.lang.String>, io.trino.spi.security.SystemAccessControlFactory.SystemAccessControlContext) - Old variant was removed and this becomes the new non-default one - - - true - java.method.numberOfParametersChanged - method io.trino.spi.eventlistener.EventListener io.trino.spi.eventlistener.EventListenerFactory::create(java.util.Map<java.lang.String, java.lang.String>) - method io.trino.spi.eventlistener.EventListener io.trino.spi.eventlistener.EventListenerFactory::create(java.util.Map<java.lang.String, java.lang.String>, io.trino.spi.eventlistener.EventListenerFactory.EventListenerContext) - Added EventListenerContext - - - java.method.removed - method io.trino.spi.connector.ConnectorTableHandle io.trino.spi.connector.ConnectorMetadata::makeCompatiblePartitioning(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableHandle, io.trino.spi.connector.ConnectorPartitioningHandle) - - - true - java.method.numberOfParametersChanged - method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Optional<java.lang.String>) - method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Optional<java.lang.String>) - Added output buffer metrics - From ad43316c65b916582b02b7152e0cf6880044f387 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:31:06 -0800 Subject: [PATCH 131/158] Remove support for connector event listeners --- .../io/trino/connector/ConnectorServices.java | 12 --- .../src/main/java/io/trino/server/Server.java | 27 ------ .../server/testing/TestingTrinoServer.java | 15 ---- .../io/trino/connector/MockConnector.java | 10 --- .../trino/connector/MockConnectorFactory.java | 23 ----- .../TestConnectorEventListener.java | 72 ---------------- core/trino-spi/pom.xml | 6 ++ .../io/trino/spi/connector/Connector.java | 9 -- .../execution/TestConnectorEventListener.java | 86 ------------------- 9 files changed, 6 insertions(+), 254 deletions(-) delete mode 100644 core/trino-main/src/test/java/io/trino/eventlistener/TestConnectorEventListener.java delete mode 100644 testing/trino-tests/src/test/java/io/trino/execution/TestConnectorEventListener.java diff --git a/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java b/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java index 898f4b60c49b..5da2fb9fd07e 100644 --- a/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java +++ b/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java @@ -13,7 +13,6 @@ */ package io.trino.connector; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import io.airlift.log.Logger; @@ -37,7 +36,6 @@ import io.trino.spi.connector.SchemaRoutineName; import io.trino.spi.connector.SystemTable; import io.trino.spi.connector.TableProcedureMetadata; -import io.trino.spi.eventlistener.EventListener; import io.trino.spi.function.FunctionKind; import io.trino.spi.function.FunctionProvider; import io.trino.spi.function.table.ArgumentSpecification; @@ -83,7 +81,6 @@ public class ConnectorServices private final Optional indexProvider; private final Optional partitioningProvider; private final Optional accessControl; - private final List eventListeners; private final Map> sessionProperties; private final Map> tableProperties; private final Map> viewProperties; @@ -184,10 +181,6 @@ public ConnectorServices(Tracer tracer, CatalogHandle catalogHandle, Connector c verifyAccessControl(accessControl); this.accessControl = Optional.ofNullable(accessControl); - Iterable eventListeners = connector.getEventListeners(); - requireNonNull(eventListeners, format("Connector '%s' returned a null event listeners iterable", eventListeners)); - this.eventListeners = ImmutableList.copyOf(eventListeners); - List> sessionProperties = connector.getSessionProperties(); requireNonNull(sessionProperties, format("Connector '%s' returned a null system properties set", catalogHandle)); this.sessionProperties = Maps.uniqueIndex(sessionProperties, PropertyMetadata::getName); @@ -297,11 +290,6 @@ public Optional getAccessControl() return accessControl; } - public List getEventListeners() - { - return eventListeners; - } - public Map> getSessionProperties() { return sessionProperties; diff --git a/core/trino-main/src/main/java/io/trino/server/Server.java b/core/trino-main/src/main/java/io/trino/server/Server.java index fd7bf81b84bb..a0a56b78bc06 100644 --- a/core/trino-main/src/main/java/io/trino/server/Server.java +++ b/core/trino-main/src/main/java/io/trino/server/Server.java @@ -13,7 +13,6 @@ */ package io.trino.server; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.StandardSystemProperty; import com.google.common.collect.ImmutableList; import com.google.inject.Injector; @@ -44,7 +43,6 @@ import io.trino.connector.CatalogManagerConfig.CatalogMangerKind; import io.trino.connector.CatalogManagerModule; import io.trino.connector.CatalogStoreManager; -import io.trino.connector.ConnectorServices; import io.trino.connector.ConnectorServicesProvider; import io.trino.eventlistener.EventListenerManager; import io.trino.eventlistener.EventListenerModule; @@ -72,7 +70,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -84,7 +81,6 @@ import static io.trino.server.TrinoSystemRequirements.verifySystemRequirements; import static java.lang.String.format; import static java.nio.file.LinkOption.NOFOLLOW_LINKS; -import static java.util.function.Predicate.not; import static java.util.stream.Collectors.joining; public class Server @@ -157,14 +153,8 @@ private void doStart(String trinoVersion) // Only static catalog manager announces catalogs // Connector event listeners are only supported for statically loaded catalogs - // TODO: remove connector event listeners or add support for dynamic loading from connector if (injector.getInstance(CatalogManagerConfig.class).getCatalogMangerKind() == CatalogMangerKind.STATIC) { CatalogManager catalogManager = injector.getInstance(CatalogManager.class); - addConnectorEventListeners( - catalogManager, - injector.getInstance(ConnectorServicesProvider.class), - injector.getInstance(EventListenerManager.class)); - // TODO: remove this huge hack updateConnectorIds(injector.getInstance(Announcer.class), catalogManager); } @@ -211,23 +201,6 @@ private void doStart(String trinoVersion) } } - @VisibleForTesting - public static void addConnectorEventListeners( - CatalogManager catalogManager, - ConnectorServicesProvider connectorServicesProvider, - EventListenerManager eventListenerManager) - { - catalogManager.getCatalogNames().stream() - .map(catalogManager::getCatalog) - .flatMap(Optional::stream) - .filter(not(Catalog::isFailed)) - .map(Catalog::getCatalogHandle) - .map(connectorServicesProvider::getConnectorServices) - .map(ConnectorServices::getEventListeners) - .flatMap(Collection::stream) - .forEach(eventListenerManager::addEventListener); - } - private static void addMessages(StringBuilder output, String type, List messages) { if (messages.isEmpty()) { diff --git a/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java b/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java index 321628772c14..54fe75471c24 100644 --- a/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java +++ b/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java @@ -79,7 +79,6 @@ import io.trino.server.PluginInstaller; import io.trino.server.PrefixObjectNameGeneratorModule; import io.trino.server.QuerySessionSupplier; -import io.trino.server.Server; import io.trino.server.ServerMainModule; import io.trino.server.SessionContext; import io.trino.server.SessionPropertyDefaults; @@ -529,20 +528,6 @@ public void loadSpoolingManager(String name, Map properties) spoolingManagerRegistry.loadSpoolingManager(name, properties); } - /** - * Add the event listeners from connectors. Connector event listeners are - * only supported for statically loaded catalogs, and this doesn't match up - * with the model of the testing Trino server. This method should only be - * called once after all catalogs are added. - */ - public void addConnectorEventListeners() - { - Server.addConnectorEventListeners( - injector.getInstance(CatalogManager.class), - injector.getInstance(ConnectorServicesProvider.class), - injector.getInstance(EventListenerManager.class)); - } - public Path getBaseDataDir() { return baseDataDir; diff --git a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java index 345ed0c580f4..948218479a0d 100644 --- a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java +++ b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java @@ -88,7 +88,6 @@ import io.trino.spi.connector.TableScanRedirectApplicationResult; import io.trino.spi.connector.TopNApplicationResult; import io.trino.spi.connector.WriterScalingOptions; -import io.trino.spi.eventlistener.EventListener; import io.trino.spi.expression.ConnectorExpression; import io.trino.spi.function.BoundSignature; import io.trino.spi.function.FunctionDependencyDeclaration; @@ -174,7 +173,6 @@ public class MockConnector private final BiFunction> getSupportedType; private final BiFunction getTableProperties; private final BiFunction> listTablePrivileges; - private final Supplier> eventListeners; private final Collection functions; private final MockConnectorFactory.ListRoleGrants roleGrants; private final Optional partitioningProvider; @@ -229,7 +227,6 @@ public class MockConnector BiFunction> getSupportedType, BiFunction getTableProperties, BiFunction> listTablePrivileges, - Supplier> eventListeners, Collection functions, ListRoleGrants roleGrants, Optional partitioningProvider, @@ -282,7 +279,6 @@ public class MockConnector this.getSupportedType = requireNonNull(getSupportedType, "getSupportedType is null"); this.getTableProperties = requireNonNull(getTableProperties, "getTableProperties is null"); this.listTablePrivileges = requireNonNull(listTablePrivileges, "listTablePrivileges is null"); - this.eventListeners = requireNonNull(eventListeners, "eventListeners is null"); this.functions = ImmutableList.copyOf(functions); this.roleGrants = requireNonNull(roleGrants, "roleGrants is null"); this.partitioningProvider = requireNonNull(partitioningProvider, "partitioningProvider is null"); @@ -367,12 +363,6 @@ public ConnectorNodePartitioningProvider getNodePartitioningProvider() return partitioningProvider.orElseThrow(UnsupportedOperationException::new); } - @Override - public Iterable getEventListeners() - { - return eventListeners.get(); - } - @Override public ConnectorAccessControl getAccessControl() { diff --git a/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java b/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java index 7a0c8e090360..2ff7a826f9fb 100644 --- a/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java +++ b/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java @@ -53,7 +53,6 @@ import io.trino.spi.connector.TableScanRedirectApplicationResult; import io.trino.spi.connector.TopNApplicationResult; import io.trino.spi.connector.WriterScalingOptions; -import io.trino.spi.eventlistener.EventListener; import io.trino.spi.expression.ConnectorExpression; import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.FunctionProvider; @@ -125,7 +124,6 @@ public class MockConnectorFactory private final BiFunction> getSupportedType; private final BiFunction getTableProperties; private final BiFunction> listTablePrivileges; - private final Supplier> eventListeners; private final Collection functions; private final Function>> data; private final Function metrics; @@ -183,7 +181,6 @@ private MockConnectorFactory( BiFunction> getSupportedType, BiFunction getTableProperties, BiFunction> listTablePrivileges, - Supplier> eventListeners, Collection functions, Function>> data, Function metrics, @@ -237,7 +234,6 @@ private MockConnectorFactory( this.getSupportedType = requireNonNull(getSupportedType, "getSupportedType is null"); this.getTableProperties = requireNonNull(getTableProperties, "getTableProperties is null"); this.listTablePrivileges = requireNonNull(listTablePrivileges, "listTablePrivileges is null"); - this.eventListeners = requireNonNull(eventListeners, "eventListeners is null"); this.functions = ImmutableList.copyOf(functions); this.analyzeProperties = requireNonNull(analyzeProperties, "analyzeProperties is null"); this.schemaProperties = requireNonNull(schemaProperties, "schemaProperties is null"); @@ -301,7 +297,6 @@ public Connector create(String catalogName, Map config, Connecto getSupportedType, getTableProperties, listTablePrivileges, - eventListeners, functions, roleGrants, partitioningProvider, @@ -446,7 +441,6 @@ public static final class Builder private BiFunction> getSupportedType = (session, type) -> Optional.empty(); private BiFunction getTableProperties = defaultGetTableProperties(); private BiFunction> listTablePrivileges = defaultListTablePrivileges(); - private Supplier> eventListeners = ImmutableList::of; private Collection functions = ImmutableList.of(); private ApplyTopN applyTopN = (session, handle, topNCount, sortItems, assignments) -> Optional.empty(); private ApplyFilter applyFilter = (session, handle, constraint) -> Optional.empty(); @@ -683,22 +677,6 @@ public Builder withListTablePrivileges(BiFunction listener); - return this; - } - - public Builder withEventListener(Supplier listenerFactory) - { - requireNonNull(listenerFactory, "listenerFactory is null"); - - this.eventListeners = () -> ImmutableList.of(listenerFactory.get()); - return this; - } - public Builder withFunctions(Collection functions) { requireNonNull(functions, "functions is null"); @@ -882,7 +860,6 @@ public MockConnectorFactory build() getSupportedType, getTableProperties, listTablePrivileges, - eventListeners, functions, data, metrics, diff --git a/core/trino-main/src/test/java/io/trino/eventlistener/TestConnectorEventListener.java b/core/trino-main/src/test/java/io/trino/eventlistener/TestConnectorEventListener.java deleted file mode 100644 index 18c4fda71301..000000000000 --- a/core/trino-main/src/test/java/io/trino/eventlistener/TestConnectorEventListener.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.eventlistener; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.Key; -import io.trino.connector.MockConnectorFactory; -import io.trino.connector.MockConnectorPlugin; -import io.trino.spi.eventlistener.EventListener; -import io.trino.testing.QueryRunner; -import io.trino.testing.StandaloneQueryRunner; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; - -import static io.trino.testing.TestingSession.testSessionBuilder; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; - -public class TestConnectorEventListener -{ - @Test - public void testConnectorWithoutEventListener() - { - QueryRunner queryRunner = new StandaloneQueryRunner(testSessionBuilder().build()); - - queryRunner.getCoordinator().getInstance(Key.get(EventListenerManager.class)).loadEventListeners(); - - assertThatCode(() -> queryRunner.execute("SELECT 1")) - .doesNotThrowAnyException(); - } - - @Test - public void testConnectorWithEventListener() - { - MockEventListenerFactory listenerFactory = new MockEventListenerFactory(); - QueryRunner queryRunner = new StandaloneQueryRunner(testSessionBuilder().build()); - queryRunner.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder() - .withEventListener(listenerFactory) - .build())); - queryRunner.createCatalog("event_listening", "mock", ImmutableMap.of()); - - queryRunner.getCoordinator().getInstance(Key.get(EventListenerManager.class)).loadEventListeners(); - - assertThat(listenerFactory.getEventListenerInvocationCounter).hasValue(1); - } - - private static class MockEventListenerFactory - implements Supplier - { - private final AtomicLong getEventListenerInvocationCounter = new AtomicLong(0); - - @Override - public EventListener get() - { - getEventListenerInvocationCounter.incrementAndGet(); - return new EventListener() {}; - } - } -} diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index aaf6a9892f99..94a08c016a45 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -212,6 +212,12 @@ + + true + java.method.removed + method java.lang.Iterable<io.trino.spi.eventlistener.EventListener> io.trino.spi.connector.Connector::getEventListeners() + Remove connector event listeners + diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java b/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java index a6bf5755b44b..6273a335f64b 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java @@ -14,7 +14,6 @@ package io.trino.spi.connector; import io.trino.spi.Experimental; -import io.trino.spi.eventlistener.EventListener; import io.trino.spi.function.FunctionProvider; import io.trino.spi.function.table.ConnectorTableFunction; import io.trino.spi.procedure.Procedure; @@ -231,14 +230,6 @@ default ConnectorAccessControl getAccessControl() throw new UnsupportedOperationException(); } - /** - * @return the event listeners provided by this connector - */ - default Iterable getEventListeners() - { - return emptySet(); - } - /** * Commit the transaction. Will be called at most once and will not be called if * {@link #rollback} is called. diff --git a/testing/trino-tests/src/test/java/io/trino/execution/TestConnectorEventListener.java b/testing/trino-tests/src/test/java/io/trino/execution/TestConnectorEventListener.java deleted file mode 100644 index df05f48d7907..000000000000 --- a/testing/trino-tests/src/test/java/io/trino/execution/TestConnectorEventListener.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.execution; - -import com.google.common.collect.ImmutableList; -import com.google.common.io.Closer; -import io.trino.connector.MockConnectorFactory; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.testing.DistributedQueryRunner; -import io.trino.testing.QueryRunner; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.IOException; - -import static io.trino.SessionTestUtils.TEST_SESSION; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; - -@TestInstance(PER_CLASS) -@Execution(CONCURRENT) -public class TestConnectorEventListener -{ - private final EventsCollector generatedEvents = new EventsCollector(); - - private Closer closer; - private EventsAwaitingQueries queries; - - @BeforeAll - public void setUp() - throws Exception - { - closer = Closer.create(); - QueryRunner queryRunner = DistributedQueryRunner.builder(TEST_SESSION) - .setWorkerCount(0) - .build(); - closer.register(queryRunner); - - queryRunner.installPlugin(new Plugin() - { - @Override - public Iterable getConnectorFactories() - { - return ImmutableList.of(MockConnectorFactory.builder() - .withEventListener(new TestingEventListener(generatedEvents)) - .build()); - } - }); - queryRunner.createCatalog("mock-catalog", "mock"); - - queryRunner.getCoordinator().addConnectorEventListeners(); - queries = new EventsAwaitingQueries(generatedEvents, queryRunner); - } - - @AfterAll - public void tearDown() - throws IOException - { - if (closer != null) { - closer.close(); - } - closer = null; - } - - @Test - public void testConnectorEventHandlerReceivingEvents() - throws Exception - { - queries.runQueryAndWaitForEvents("SELECT 1", TEST_SESSION).getQueryEvents(); - } -} From d2bed4765dc48403e32e7986e004bd64722836ba Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:16:45 -0800 Subject: [PATCH 132/158] Inline partition projection name in HiveConfig --- .../java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java | 3 +-- .../src/main/java/io/trino/plugin/hive/HiveConfig.java | 4 +--- .../src/main/java/io/trino/plugin/hive/HiveMetadata.java | 2 +- .../src/test/java/io/trino/plugin/hive/TestHiveConfig.java | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java index d5c20b7dda9d..2ebc98f68e5b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java @@ -15,7 +15,6 @@ import com.google.common.collect.ImmutableMap; import io.airlift.bootstrap.ApplicationConfigurationException; -import io.trino.plugin.hive.HiveConfig; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorFactory; import io.trino.testing.TestingConnectorContext; @@ -165,7 +164,7 @@ public void testHiveConfigIsNotBound() ImmutableMap.of( "hive.metastore.uri", "thrift://foo:1234", // Try setting any property provided by HiveConfig class - HiveConfig.CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED, "true", + "hive.partition-projection-enabled", "true", "bootstrap.quiet", "true"), new TestingConnectorContext())) .hasMessageContaining("Error: Configuration property 'hive.partition-projection-enabled' was not used"); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java index e27da88622d9..5b392f4bf14f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java @@ -66,8 +66,6 @@ }) public class HiveConfig { - public static final String CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED = "hive.partition-projection-enabled"; - private boolean singleStatementWritesOnly; private DataSize maxSplitSize = DataSize.of(64, MEGABYTE); @@ -1249,7 +1247,7 @@ public boolean isPartitionProjectionEnabled() return partitionProjectionEnabled; } - @Config(CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED) + @Config("hive.partition-projection-enabled") @ConfigDescription("Enables AWS Athena partition projection") public HiveConfig setPartitionProjectionEnabled(boolean enabledAthenaPartitionProjection) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 17d4542a0a0f..2f843c606892 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -1281,7 +1281,7 @@ else if (avroSchemaLiteral != null) { else if (arePartitionProjectionPropertiesSet(tableMetadata)) { throw new TrinoException( INVALID_COLUMN_PROPERTY, - "Partition projection is disabled. Enable it in configuration by setting " + HiveConfig.CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED + "=true"); + "Partition projection is disabled. Enable it in configuration by setting " + "hive.partition-projection-enabled" + "=true"); } Map baseProperties = tableProperties.buildOrThrow(); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java index cfeedd94941c..d07f3a58b821 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java @@ -29,7 +29,6 @@ import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.plugin.hive.HiveConfig.CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED; import static io.trino.plugin.hive.HiveSessionProperties.InsertExistingPartitionsBehavior.APPEND; import static io.trino.plugin.hive.HiveSessionProperties.InsertExistingPartitionsBehavior.OVERWRITE; import static io.trino.plugin.hive.util.TestHiveUtil.nonDefaultTimeZone; @@ -203,7 +202,7 @@ public void testExplicitPropertyMappings() .put("hive.delta-lake-catalog-name", "delta") .put("hive.hudi-catalog-name", "hudi") .put("hive.auto-purge", "true") - .put(CONFIGURATION_HIVE_PARTITION_PROJECTION_ENABLED, "true") + .put("hive.partition-projection-enabled", "true") .put("hive.s3.storage-class-filter", "READ_NON_GLACIER_AND_RESTORED") .put("hive.metadata.parallelism", "10") .buildOrThrow(); From acf6ccc325d73bf16ce53a6ddff38537900be6ca Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:38:17 -0800 Subject: [PATCH 133/158] Remove shadowed field from MockConnectorMetadata --- .../src/test/java/io/trino/connector/MockConnector.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java index 948218479a0d..bb7b5a3ca1b0 100644 --- a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java +++ b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java @@ -317,7 +317,7 @@ public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel @Override public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transaction) { - return metadataWrapper.apply(new MockConnectorMetadata(allowSplittingReadIntoMultipleSubQueries)); + return metadataWrapper.apply(new MockConnectorMetadata()); } @Override @@ -438,13 +438,6 @@ public Set getCapabilities() private class MockConnectorMetadata implements ConnectorMetadata { - private final boolean allowSplittingReadIntoMultipleSubQueries; - - public MockConnectorMetadata(boolean allowSplittingReadIntoMultipleSubQueries) - { - this.allowSplittingReadIntoMultipleSubQueries = allowSplittingReadIntoMultipleSubQueries; - } - @Override public boolean schemaExists(ConnectorSession session, String schemaName) { From 386a6fcad6f5caf22e97cb47fece6eba898cd7e6 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:39:23 -0800 Subject: [PATCH 134/158] Remove optional binder for HivePageSourceProvider --- .../src/main/java/io/trino/plugin/hive/HiveModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java index 4aedf0a0786e..70d49953cde2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java @@ -108,7 +108,7 @@ public void configure(Binder binder) binder.bind(HiveTransactionManager.class).in(Scopes.SINGLETON); binder.bind(ConnectorSplitManager.class).to(HiveSplitManager.class).in(Scopes.SINGLETON); newExporter(binder).export(ConnectorSplitManager.class).as(generator -> generator.generatedNameOf(HiveSplitManager.class)); - newOptionalBinder(binder, ConnectorPageSourceProvider.class).setDefault().to(HivePageSourceProvider.class).in(Scopes.SINGLETON); + binder.bind(ConnectorPageSourceProvider.class).to(HivePageSourceProvider.class).in(Scopes.SINGLETON); binder.bind(ConnectorPageSinkProvider.class).to(HivePageSinkProvider.class).in(Scopes.SINGLETON); binder.bind(ConnectorNodePartitioningProvider.class).to(HiveNodePartitioningProvider.class).in(Scopes.SINGLETON); From bc446bbca88a5528b5a8854831472fe2a0ae9073 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:41:10 -0800 Subject: [PATCH 135/158] Remove unused HiveMaterializedViewPropertiesProvider --- .../io/trino/plugin/hive/HiveConnector.java | 9 ------- .../plugin/hive/HiveConnectorFactory.java | 2 -- ...iveMaterializedViewPropertiesProvider.java | 24 ------------------- .../java/io/trino/plugin/hive/HiveModule.java | 3 --- 4 files changed, 38 deletions(-) delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMaterializedViewPropertiesProvider.java diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java index 578a6f1e9c93..c3248b7646ff 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java @@ -62,7 +62,6 @@ public class HiveConnector private final List> viewProperties; private final List> columnProperties; private final List> analyzeProperties; - private final List> materializedViewProperties; private final Optional accessControl; private final ClassLoader classLoader; @@ -88,7 +87,6 @@ public HiveConnector( List> viewProperties, List> columnProperties, List> analyzeProperties, - List> materializedViewProperties, Optional accessControl, Set connectorTableFunctions, FunctionProvider functionProvider, @@ -112,7 +110,6 @@ public HiveConnector( this.viewProperties = ImmutableList.copyOf(requireNonNull(viewProperties, "viewProperties is null")); this.columnProperties = ImmutableList.copyOf(requireNonNull(columnProperties, "columnProperties is null")); this.analyzeProperties = ImmutableList.copyOf(requireNonNull(analyzeProperties, "analyzeProperties is null")); - this.materializedViewProperties = requireNonNull(materializedViewProperties, "materializedViewProperties is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); this.connectorTableFunctions = ImmutableSet.copyOf(requireNonNull(connectorTableFunctions, "connectorTableFunctions is null")); this.functionProvider = requireNonNull(functionProvider, "functionProvider is null"); @@ -194,12 +191,6 @@ public List> getColumnProperties() return this.columnProperties; } - @Override - public List> getMaterializedViewProperties() - { - return materializedViewProperties; - } - @Override public ConnectorAccessControl getAccessControl() { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java index 2b1eba0220f2..d9983da28fd9 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java @@ -137,7 +137,6 @@ public static Connector createConnector( HiveViewProperties hiveViewProperties = injector.getInstance(HiveViewProperties.class); HiveColumnProperties hiveColumnProperties = injector.getInstance(HiveColumnProperties.class); HiveAnalyzeProperties hiveAnalyzeProperties = injector.getInstance(HiveAnalyzeProperties.class); - HiveMaterializedViewPropertiesProvider hiveMaterializedViewPropertiesProvider = injector.getInstance(HiveMaterializedViewPropertiesProvider.class); Set procedures = injector.getInstance(new Key<>() {}); Set tableProcedures = injector.getInstance(new Key<>() {}); Set systemTableProviders = injector.getInstance(new Key<>() {}); @@ -161,7 +160,6 @@ public static Connector createConnector( hiveViewProperties.getViewProperties(), hiveColumnProperties.getColumnProperties(), hiveAnalyzeProperties.getAnalyzeProperties(), - hiveMaterializedViewPropertiesProvider.getMaterializedViewProperties(), hiveAccessControl, injector.getInstance(new Key<>() {}), injector.getInstance(FunctionProvider.class), diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMaterializedViewPropertiesProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMaterializedViewPropertiesProvider.java deleted file mode 100644 index 829465375eab..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMaterializedViewPropertiesProvider.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.trino.plugin.hive; - -import io.trino.spi.session.PropertyMetadata; - -import java.util.List; - -public interface HiveMaterializedViewPropertiesProvider -{ - List> getMaterializedViewProperties(); -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java index 70d49953cde2..8eddcea048c4 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java @@ -13,7 +13,6 @@ */ package io.trino.plugin.hive; -import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; @@ -86,8 +85,6 @@ public void configure(Binder binder) binder.bind(HiveViewProperties.class).in(Scopes.SINGLETON); binder.bind(HiveColumnProperties.class).in(Scopes.SINGLETON); binder.bind(HiveAnalyzeProperties.class).in(Scopes.SINGLETON); - newOptionalBinder(binder, HiveMaterializedViewPropertiesProvider.class) - .setDefault().toInstance(ImmutableList::of); binder.bind(CachingDirectoryLister.class).in(Scopes.SINGLETON); newExporter(binder).export(CachingDirectoryLister.class).withGeneratedName(); From 8a32336a6e03064d2f70d7a6a131fd9419b74d95 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:43:05 -0800 Subject: [PATCH 136/158] Remove unused HiveRedirectionsProvider --- .../io/trino/plugin/hive/HiveMetadata.java | 10 ------- .../plugin/hive/HiveMetadataFactory.java | 6 ---- .../java/io/trino/plugin/hive/HiveModule.java | 2 -- .../plugin/hive/HiveRedirectionsProvider.java | 25 ---------------- .../hive/NoneHiveRedirectionsProvider.java | 30 ------------------- 5 files changed, 73 deletions(-) delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRedirectionsProvider.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/NoneHiveRedirectionsProvider.java diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 2f843c606892..c253267694ec 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -109,7 +109,6 @@ import io.trino.spi.connector.SystemTable; import io.trino.spi.connector.TableColumnsMetadata; import io.trino.spi.connector.TableNotFoundException; -import io.trino.spi.connector.TableScanRedirectApplicationResult; import io.trino.spi.connector.ViewNotFoundException; import io.trino.spi.connector.WriterScalingOptions; import io.trino.spi.expression.ConnectorExpression; @@ -414,7 +413,6 @@ public class HiveMetadata private final boolean hideDeltaLakeTables; private final String trinoVersion; private final HiveStatisticsProvider hiveStatisticsProvider; - private final HiveRedirectionsProvider hiveRedirectionsProvider; private final Set systemTableProviders; private final AccessControlMetadata accessControlMetadata; private final DirectoryLister directoryLister; @@ -442,7 +440,6 @@ public HiveMetadata( JsonCodec partitionUpdateCodec, String trinoVersion, HiveStatisticsProvider hiveStatisticsProvider, - HiveRedirectionsProvider hiveRedirectionsProvider, Set systemTableProviders, AccessControlMetadata accessControlMetadata, DirectoryLister directoryLister, @@ -469,7 +466,6 @@ public HiveMetadata( this.hideDeltaLakeTables = hideDeltaLakeTables; this.trinoVersion = requireNonNull(trinoVersion, "trinoVersion is null"); this.hiveStatisticsProvider = requireNonNull(hiveStatisticsProvider, "hiveStatisticsProvider is null"); - this.hiveRedirectionsProvider = requireNonNull(hiveRedirectionsProvider, "hiveRedirectionsProvider is null"); this.systemTableProviders = requireNonNull(systemTableProviders, "systemTableProviders is null"); this.accessControlMetadata = requireNonNull(accessControlMetadata, "accessControlMetadata is null"); this.directoryLister = requireNonNull(directoryLister, "directoryLister is null"); @@ -3256,12 +3252,6 @@ private HiveColumnHandle createProjectedColumnHandle(HiveColumnHandle column, Li column.getComment()); } - @Override - public Optional applyTableScanRedirect(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return hiveRedirectionsProvider.getTableScanRedirection(session, (HiveTableHandle) tableHandle); - } - @Override public Optional applyPartitioning(ConnectorSession session, ConnectorTableHandle tableHandle, Optional partitioningHandle, List columns) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java index b8fd3fc20f59..72b5aebe420f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java @@ -68,7 +68,6 @@ public class HiveMetadataFactory private final Executor updateExecutor; private final long maxPartitionDropsPerQuery; private final String trinoVersion; - private final HiveRedirectionsProvider hiveRedirectionsProvider; private final Set systemTableProviders; private final AccessControlMetadataFactory accessControlMetadataFactory; private final Optional hiveTransactionHeartbeatInterval; @@ -96,7 +95,6 @@ public HiveMetadataFactory( LocationService locationService, JsonCodec partitionUpdateCodec, NodeVersion nodeVersion, - HiveRedirectionsProvider hiveRedirectionsProvider, Set systemTableProviders, AccessControlMetadataFactory accessControlMetadataFactory, DirectoryLister directoryLister, @@ -130,7 +128,6 @@ public HiveMetadataFactory( executorService, heartbeatService, nodeVersion.toString(), - hiveRedirectionsProvider, systemTableProviders, accessControlMetadataFactory, directoryLister, @@ -168,7 +165,6 @@ public HiveMetadataFactory( ExecutorService executorService, ScheduledExecutorService heartbeatService, String trinoVersion, - HiveRedirectionsProvider hiveRedirectionsProvider, Set systemTableProviders, AccessControlMetadataFactory accessControlMetadataFactory, DirectoryLister directoryLister, @@ -198,7 +194,6 @@ public HiveMetadataFactory( this.locationService = requireNonNull(locationService, "locationService is null"); this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null"); this.trinoVersion = requireNonNull(trinoVersion, "trinoVersion is null"); - this.hiveRedirectionsProvider = requireNonNull(hiveRedirectionsProvider, "hiveRedirectionsProvider is null"); this.systemTableProviders = requireNonNull(systemTableProviders, "systemTableProviders is null"); this.accessControlMetadataFactory = requireNonNull(accessControlMetadataFactory, "accessControlMetadataFactory is null"); this.hiveTransactionHeartbeatInterval = requireNonNull(hiveTransactionHeartbeatInterval, "hiveTransactionHeartbeatInterval is null"); @@ -266,7 +261,6 @@ public TransactionalMetadata create(ConnectorIdentity identity, boolean autoComm partitionUpdateCodec, trinoVersion, new MetastoreHiveStatisticsProvider(metastore), - hiveRedirectionsProvider, systemTableProviders, accessControlMetadataFactory.create(metastore), directoryLister, diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java index 8eddcea048c4..72d97e1fc2bf 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java @@ -97,8 +97,6 @@ public void configure(Binder binder) Multibinder systemTableProviders = newSetBinder(binder, SystemTableProvider.class); systemTableProviders.addBinding().to(PartitionsSystemTableProvider.class).in(Scopes.SINGLETON); systemTableProviders.addBinding().to(PropertiesSystemTableProvider.class).in(Scopes.SINGLETON); - newOptionalBinder(binder, HiveRedirectionsProvider.class) - .setDefault().to(NoneHiveRedirectionsProvider.class).in(Scopes.SINGLETON); newOptionalBinder(binder, TransactionalMetadataFactory.class) .setDefault().to(HiveMetadataFactory.class).in(Scopes.SINGLETON); binder.bind(TransactionScopeCachingDirectoryListerFactory.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRedirectionsProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRedirectionsProvider.java deleted file mode 100644 index 6d595a964be7..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRedirectionsProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.trino.plugin.hive; - -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.TableScanRedirectApplicationResult; - -import java.util.Optional; - -public interface HiveRedirectionsProvider -{ - Optional getTableScanRedirection(ConnectorSession session, HiveTableHandle tableHandle); -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/NoneHiveRedirectionsProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/NoneHiveRedirectionsProvider.java deleted file mode 100644 index 124fe56dd358..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/NoneHiveRedirectionsProvider.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.trino.plugin.hive; - -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.TableScanRedirectApplicationResult; - -import java.util.Optional; - -public class NoneHiveRedirectionsProvider - implements HiveRedirectionsProvider -{ - @Override - public Optional getTableScanRedirection(ConnectorSession session, HiveTableHandle tableHandle) - { - return Optional.empty(); - } -} From a7e0d5e9c6a64576189339711ea9822428c10599 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:44:23 -0800 Subject: [PATCH 137/158] Remove unused DeltaLakeRedirectionsProvider --- .../io/trino/plugin/deltalake/DeltaLakeMetadata.java | 10 ---------- .../plugin/deltalake/DeltaLakeMetadataFactory.java | 4 ---- .../io/trino/plugin/deltalake/DeltaLakeModule.java | 3 --- .../deltalake/DeltaLakeRedirectionsProvider.java | 2 -- .../plugin/deltalake/TestDeltaLakeSplitManager.java | 1 - 5 files changed, 20 deletions(-) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java index 658ecec123cf..2d0648961b2c 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java @@ -125,7 +125,6 @@ import io.trino.spi.connector.SystemTable; import io.trino.spi.connector.TableColumnsMetadata; import io.trino.spi.connector.TableNotFoundException; -import io.trino.spi.connector.TableScanRedirectApplicationResult; import io.trino.spi.connector.ViewNotFoundException; import io.trino.spi.connector.WriterScalingOptions; import io.trino.spi.expression.ConnectorExpression; @@ -438,7 +437,6 @@ public class DeltaLakeMetadata private final String nodeVersion; private final String nodeId; private final AtomicReference rollbackAction = new AtomicReference<>(); - private final DeltaLakeRedirectionsProvider deltaLakeRedirectionsProvider; private final CachingExtendedStatisticsAccess statisticsAccess; private final boolean deleteSchemaLocationsFallback; private final boolean useUniqueTableLocation; @@ -474,7 +472,6 @@ public DeltaLakeMetadata( CheckpointWriterManager checkpointWriterManager, long defaultCheckpointInterval, boolean deleteSchemaLocationsFallback, - DeltaLakeRedirectionsProvider deltaLakeRedirectionsProvider, CachingExtendedStatisticsAccess statisticsAccess, DeltaLakeTableMetadataScheduler metadataScheduler, boolean useUniqueTableLocation, @@ -497,7 +494,6 @@ public DeltaLakeMetadata( this.nodeId = nodeManager.getCurrentNode().getNodeIdentifier(); this.checkpointWriterManager = requireNonNull(checkpointWriterManager, "checkpointWriterManager is null"); this.defaultCheckpointInterval = defaultCheckpointInterval; - this.deltaLakeRedirectionsProvider = requireNonNull(deltaLakeRedirectionsProvider, "deltaLakeRedirectionsProvider is null"); this.statisticsAccess = requireNonNull(statisticsAccess, "statisticsAccess is null"); this.deleteSchemaLocationsFallback = deleteSchemaLocationsFallback; this.metadataScheduler = requireNonNull(metadataScheduler, "metadataScheduler is null"); @@ -3576,12 +3572,6 @@ public void validateScan(ConnectorSession session, ConnectorTableHandle handle) } } - @Override - public Optional applyTableScanRedirect(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return deltaLakeRedirectionsProvider.getTableScanRedirection(session, (DeltaLakeTableHandle) tableHandle); - } - @Override public ConnectorAnalyzeMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, Map analyzeProperties) { diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java index 34b048049ab1..b238c6c82c4d 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java @@ -53,7 +53,6 @@ public class DeltaLakeMetadataFactory private final TransactionLogWriterFactory transactionLogWriterFactory; private final NodeManager nodeManager; private final CheckpointWriterManager checkpointWriterManager; - private final DeltaLakeRedirectionsProvider deltaLakeRedirectionsProvider; private final CachingExtendedStatisticsAccess statisticsAccess; private final int domainCompactionThreshold; private final boolean unsafeWritesEnabled; @@ -79,7 +78,6 @@ public DeltaLakeMetadataFactory( TransactionLogWriterFactory transactionLogWriterFactory, NodeManager nodeManager, CheckpointWriterManager checkpointWriterManager, - DeltaLakeRedirectionsProvider deltaLakeRedirectionsProvider, CachingExtendedStatisticsAccess statisticsAccess, @AllowDeltaLakeManagedTableRename boolean allowManagedTableRename, NodeVersion nodeVersion, @@ -96,7 +94,6 @@ public DeltaLakeMetadataFactory( this.transactionLogWriterFactory = requireNonNull(transactionLogWriterFactory, "transactionLogWriterFactory is null"); this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.checkpointWriterManager = requireNonNull(checkpointWriterManager, "checkpointWriterManager is null"); - this.deltaLakeRedirectionsProvider = requireNonNull(deltaLakeRedirectionsProvider, "deltaLakeRedirectionsProvider is null"); this.statisticsAccess = requireNonNull(statisticsAccess, "statisticsAccess is null"); this.domainCompactionThreshold = deltaLakeConfig.getDomainCompactionThreshold(); this.unsafeWritesEnabled = deltaLakeConfig.getUnsafeWritesEnabled(); @@ -148,7 +145,6 @@ public DeltaLakeMetadata create(ConnectorIdentity identity) checkpointWriterManager, checkpointWritingInterval, deleteSchemaLocationsFallback, - deltaLakeRedirectionsProvider, statisticsAccess, metadataScheduler, useUniqueTableLocation, diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java index 7c62bdb87765..a4939050b7bc 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java @@ -127,9 +127,6 @@ public void setup(Binder binder) binder.bind(NoIsolationSynchronizer.class).in(Scopes.SINGLETON); newMapBinder(binder, String.class, TransactionLogSynchronizer.class); - newOptionalBinder(binder, DeltaLakeRedirectionsProvider.class) - .setDefault().toInstance(DeltaLakeRedirectionsProvider.NOOP); - jsonCodecBinder(binder).bindJsonCodec(DataFileInfo.class); jsonCodecBinder(binder).bindJsonCodec(DeltaLakeMergeResult.class); binder.bind(DeltaLakeWriterStats.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeRedirectionsProvider.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeRedirectionsProvider.java index b371d787b2af..090d91b26ce4 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeRedirectionsProvider.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeRedirectionsProvider.java @@ -20,7 +20,5 @@ public interface DeltaLakeRedirectionsProvider { - DeltaLakeRedirectionsProvider NOOP = (session, tableHandle) -> Optional.empty(); - Optional getTableScanRedirection(ConnectorSession session, DeltaLakeTableHandle tableHandle); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java index 15ca606fb0e6..a5ddd9368b7b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java @@ -230,7 +230,6 @@ public Stream getActiveFiles( new TransactionLogSynchronizerManager(ImmutableMap.of(), new NoIsolationSynchronizer(hdfsFileSystemFactory))), new TestingNodeManager(), checkpointWriterManager, - DeltaLakeRedirectionsProvider.NOOP, new CachingExtendedStatisticsAccess(new MetaDirStatisticsAccess(HDFS_FILE_SYSTEM_FACTORY, new JsonCodecFactory().jsonCodec(ExtendedStatistics.class))), true, new NodeVersion("test_version"), From c24794763bcdcfb0eedfd0ff91ae3674c59cb4cb Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:49:29 -0800 Subject: [PATCH 138/158] Remove unused function providers from Hive connector --- .../io/trino/plugin/hive/HiveConnector.java | 20 ------------------- .../plugin/hive/HiveConnectorFactory.java | 3 --- .../java/io/trino/plugin/hive/HiveModule.java | 5 ----- 3 files changed, 28 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java index c3248b7646ff..5f0505f1431f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java @@ -29,8 +29,6 @@ import io.trino.spi.connector.ConnectorSplitManager; import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.TableProcedureMetadata; -import io.trino.spi.function.FunctionProvider; -import io.trino.spi.function.table.ConnectorTableFunction; import io.trino.spi.procedure.Procedure; import io.trino.spi.session.PropertyMetadata; import io.trino.spi.transaction.IsolationLevel; @@ -67,8 +65,6 @@ public class HiveConnector private final ClassLoader classLoader; private final HiveTransactionManager transactionManager; - private final Set connectorTableFunctions; - private final FunctionProvider functionProvider; private final boolean singleStatementWritesOnly; public HiveConnector( @@ -88,8 +84,6 @@ public HiveConnector( List> columnProperties, List> analyzeProperties, Optional accessControl, - Set connectorTableFunctions, - FunctionProvider functionProvider, boolean singleStatementWritesOnly, ClassLoader classLoader) { @@ -111,8 +105,6 @@ public HiveConnector( this.columnProperties = ImmutableList.copyOf(requireNonNull(columnProperties, "columnProperties is null")); this.analyzeProperties = ImmutableList.copyOf(requireNonNull(analyzeProperties, "analyzeProperties is null")); this.accessControl = requireNonNull(accessControl, "accessControl is null"); - this.connectorTableFunctions = ImmutableSet.copyOf(requireNonNull(connectorTableFunctions, "connectorTableFunctions is null")); - this.functionProvider = requireNonNull(functionProvider, "functionProvider is null"); this.singleStatementWritesOnly = singleStatementWritesOnly; this.classLoader = requireNonNull(classLoader, "classLoader is null"); } @@ -230,18 +222,6 @@ public final void shutdown() lifeCycleManager.stop(); } - @Override - public Optional getFunctionProvider() - { - return Optional.of(functionProvider); - } - - @Override - public Set getTableFunctions() - { - return connectorTableFunctions; - } - @Override public Set getTableProcedures() { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java index d9983da28fd9..21299c48f0ff 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java @@ -55,7 +55,6 @@ import io.trino.spi.connector.ConnectorSplitManager; import io.trino.spi.connector.MetadataProvider; import io.trino.spi.connector.TableProcedureMetadata; -import io.trino.spi.function.FunctionProvider; import io.trino.spi.procedure.Procedure; import org.weakref.jmx.guice.MBeanModule; @@ -161,8 +160,6 @@ public static Connector createConnector( hiveColumnProperties.getColumnProperties(), hiveAnalyzeProperties.getAnalyzeProperties(), hiveAccessControl, - injector.getInstance(new Key<>() {}), - injector.getInstance(FunctionProvider.class), injector.getInstance(HiveConfig.class).isSingleStatementWritesOnly(), classLoader); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java index 72d97e1fc2bf..253cd16e01b7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java @@ -54,8 +54,6 @@ import io.trino.spi.connector.ConnectorPageSinkProvider; import io.trino.spi.connector.ConnectorPageSourceProvider; import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.function.FunctionProvider; -import io.trino.spi.function.table.ConnectorTableFunction; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; @@ -143,9 +141,6 @@ public void configure(Binder binder) configBinder(binder).bindConfig(ParquetWriterConfig.class); fileWriterFactoryBinder.addBinding().to(ParquetFileWriterFactory.class).in(Scopes.SINGLETON); - newOptionalBinder(binder, FunctionProvider.class).setDefault().toInstance(new NoopFunctionProvider()); - newSetBinder(binder, ConnectorTableFunction.class); - closingBinder(binder).registerExecutor(ExecutorService.class); closingBinder(binder).registerExecutor(Key.get(ScheduledExecutorService.class, ForHiveTransactionHeartbeats.class)); } From afd2f96d8fb9a48adeb205e05c7075dfeb5b7de2 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 17:57:41 -0800 Subject: [PATCH 139/158] Remove deprecated ConnectorMetadata.getTableHandleForExecute --- .../tracing/TracingConnectorMetadata.java | 9 ------- .../spi/connector/ConnectorMetadata.java | 25 +------------------ .../ClassLoaderSafeConnectorMetadata.java | 8 ------ 3 files changed, 1 insertion(+), 41 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java b/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java index c4e2ecc31266..42eed75af618 100644 --- a/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java +++ b/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java @@ -143,15 +143,6 @@ public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTable } } - @Override - public Optional getTableHandleForExecute(ConnectorSession session, ConnectorTableHandle tableHandle, String procedureName, Map executeProperties, RetryMode retryMode) - { - Span span = startSpan("getTableHandleForExecute", tableHandle); - try (var _ = scopedSpan(span)) { - return delegate.getTableHandleForExecute(session, tableHandle, procedureName, executeProperties, retryMode); - } - } - @Override public Optional getTableHandleForExecute(ConnectorSession session, ConnectorAccessControl accessControl, ConnectorTableHandle tableHandle, String procedureName, Map executeProperties, RetryMode retryMode) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java index 4696d8c03ea4..0d1dbae97867 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java @@ -116,29 +116,6 @@ default ConnectorTableHandle getTableHandle( throw new TrinoException(GENERIC_INTERNAL_ERROR, "ConnectorMetadata getTableHandle() is not implemented"); } - /** - * Create initial handle for execution of table procedure. The handle will be used through planning process. It will be converted to final - * handle used for execution via @{link {@link ConnectorMetadata#beginTableExecute} - *

- * If connector does not support execution with retries, the method should throw: - * 

-     *     new TrinoException(NOT_SUPPORTED, "This connector does not support query retries")
-     *
- * unless {@code retryMode} is set to {@code NO_RETRIES}. - * - * @deprecated {Use {@link #getTableHandleForExecute(ConnectorSession, ConnectorAccessControl, ConnectorTableHandle, String, Map, RetryMode)}} - */ - @Deprecated - default Optional getTableHandleForExecute( - ConnectorSession session, - ConnectorTableHandle tableHandle, - String procedureName, - Map executeProperties, - RetryMode retryMode) - { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support table procedures"); - } - /** * Create initial handle for execution of table procedure. The handle will be used through planning process. It will be converted to final * handle used for execution via @{link {@link ConnectorMetadata#beginTableExecute} @@ -157,7 +134,7 @@ default Optional getTableHandleForExecute( Map executeProperties, RetryMode retryMode) { - return getTableHandleForExecute(session, tableHandle, procedureName, executeProperties, retryMode); + throw new TrinoException(NOT_SUPPORTED, "This connector does not support table procedures"); } default Optional getLayoutForTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java index 52f09777e537..88320ad219eb 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java @@ -201,14 +201,6 @@ public List listSchemaNames(ConnectorSession session) } } - @Override - public Optional getTableHandleForExecute(ConnectorSession session, ConnectorTableHandle tableHandle, String procedureName, Map executeProperties, RetryMode retryMode) - { - try (ThreadContextClassLoader _ = new ThreadContextClassLoader(classLoader)) { - return delegate.getTableHandleForExecute(session, tableHandle, procedureName, executeProperties, retryMode); - } - } - @Override public Optional getTableHandleForExecute(ConnectorSession session, ConnectorAccessControl accessControl, ConnectorTableHandle tableHandle, String procedureName, Map executeProperties, RetryMode retryMode) { From ede0fdd2656f4486fb0c63beaeb82a7373d349d4 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 18:02:12 -0800 Subject: [PATCH 140/158] Remove deprecated ConnectorMetadata.beginMerge --- .../io/trino/tracing/TracingConnectorMetadata.java | 9 --------- .../java/io/trino/connector/MockConnector.java | 2 +- .../io/trino/spi/connector/ConnectorMetadata.java | 14 +------------- .../ClassLoaderSafeConnectorMetadata.java | 8 -------- .../trino/plugin/blackhole/BlackHoleMetadata.java | 2 +- .../trino/plugin/deltalake/DeltaLakeMetadata.java | 2 +- .../java/io/trino/plugin/hive/HiveMetadata.java | 2 +- .../io/trino/plugin/iceberg/IcebergMetadata.java | 2 +- 8 files changed, 6 insertions(+), 35 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java b/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java index 42eed75af618..acaf65954dc7 100644 --- a/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java +++ b/core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java @@ -752,15 +752,6 @@ public Optional getUpdateLayout(ConnectorSession se } } - @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) - { - Span span = startSpan("beginMerge", tableHandle); - try (var _ = scopedSpan(span)) { - return delegate.beginMerge(session, tableHandle, retryMode); - } - } - @Override public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { diff --git a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java index bb7b5a3ca1b0..d62355323467 100644 --- a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java +++ b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java @@ -859,7 +859,7 @@ public ColumnHandle getMergeRowIdColumnHandle(ConnectorSession session, Connecto } @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) + public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { return new MockConnectorMergeTableHandle((MockConnectorTableHandle) tableHandle); } diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java index 0d1dbae97867..ce45ca7415f0 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java @@ -862,25 +862,13 @@ default Optional getUpdateLayout(ConnectorSession s return Optional.empty(); } - /** - * Do whatever is necessary to start an MERGE query, returning the {@link ConnectorMergeTableHandle} - * instance that will be passed to the PageSink, and to the {@link #finishMerge} method. - * - * @deprecated {Use {@link #beginMerge(ConnectorSession, ConnectorTableHandle, Map, RetryMode)}} - */ - @Deprecated - default ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) - { - throw new TrinoException(NOT_SUPPORTED, MODIFYING_ROWS_MESSAGE); - } - /** * Do whatever is necessary to start an MERGE query, returning the {@link ConnectorMergeTableHandle} * instance that will be passed to the PageSink, and to the {@link #finishMerge} method. */ default ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { - return beginMerge(session, tableHandle, retryMode); + throw new TrinoException(NOT_SUPPORTED, MODIFYING_ROWS_MESSAGE); } /** diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java index 88320ad219eb..ddf3dcba315f 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java @@ -1198,14 +1198,6 @@ public Optional getUpdateLayout(ConnectorSession se return delegate.getUpdateLayout(session, tableHandle); } - @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) - { - try (ThreadContextClassLoader _ = new ThreadContextClassLoader(classLoader)) { - return delegate.beginMerge(session, tableHandle, retryMode); - } - } - @Override public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { diff --git a/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHoleMetadata.java b/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHoleMetadata.java index ece7a0f39afc..9d4220ee7b43 100644 --- a/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHoleMetadata.java +++ b/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHoleMetadata.java @@ -401,7 +401,7 @@ public ColumnHandle getMergeRowIdColumnHandle(ConnectorSession session, Connecto } @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) + public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { return new BlackHoleMergeTableHandle((BlackHoleTableHandle) tableHandle); } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java index 2d0648961b2c..4f3da6a15d7b 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java @@ -2432,7 +2432,7 @@ public Optional getUpdateLayout(ConnectorSession se } @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) + public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { DeltaLakeTableHandle handle = (DeltaLakeTableHandle) tableHandle; if (isAppendOnly(handle.getMetadataEntry(), handle.getProtocolEntry())) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index c253267694ec..254eda802723 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -2061,7 +2061,7 @@ public RowChangeParadigm getRowChangeParadigm(ConnectorSession session, Connecto } @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) + public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle; SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index b34240379f8a..2149fb71ed23 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -2861,7 +2861,7 @@ public Optional getUpdateLayout(ConnectorSession se } @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) + public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, Map> updateCaseColumns, RetryMode retryMode) { IcebergTableHandle table = (IcebergTableHandle) tableHandle; verifyTableVersionForUpdate(table); From ffe42ed032de51533ffd400b50b6af99a3556860 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 1 Jan 2025 18:07:50 -0800 Subject: [PATCH 141/158] Remove optional binder for IcebergPageSourceProviderFactory --- .../src/main/java/io/trino/plugin/iceberg/IcebergModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index ed5b6b10fb86..b0c653ab55a6 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -95,7 +95,7 @@ public void configure(Binder binder) binder.bind(ConnectorSplitManager.class).annotatedWith(ForClassLoaderSafe.class).to(IcebergSplitManager.class).in(Scopes.SINGLETON); binder.bind(ConnectorSplitManager.class).to(ClassLoaderSafeConnectorSplitManager.class).in(Scopes.SINGLETON); - newOptionalBinder(binder, Key.get(ConnectorPageSourceProviderFactory.class, ForClassLoaderSafe.class)).setDefault().to(IcebergPageSourceProviderFactory.class).in(Scopes.SINGLETON); + binder.bind(ConnectorPageSourceProviderFactory.class).annotatedWith(ForClassLoaderSafe.class).to(IcebergPageSourceProviderFactory.class).in(Scopes.SINGLETON); binder.bind(IcebergPageSourceProviderFactory.class).in(Scopes.SINGLETON); binder.bind(ConnectorPageSourceProviderFactory.class).to(ClassLoaderSafeConnectorPageSourceProviderFactory.class).in(Scopes.SINGLETON); binder.bind(ConnectorPageSinkProvider.class).annotatedWith(ForClassLoaderSafe.class).to(IcebergPageSinkProvider.class).in(Scopes.SINGLETON); From c03c50d482228d7d701bc075c4aaf3fec2e8e071 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 22 Nov 2024 13:23:00 +0100 Subject: [PATCH 142/158] Make SqlVarbinary map serialization consistent with other types Other Sql* classes are serialized to JSON on-the-wire format, using @JsonValue annotated toString methods, except for SqlVarbinary which was serialized using its' getBytes() method that was Base64-encoded to a map key. --- .../io/trino/server/protocol/JsonEncodingUtils.java | 9 ++------- .../TestQuantileDigestAggregationFunction.java | 8 ++++---- .../aggregation/TestQuantileDigestFunctions.java | 2 +- core/trino-spi/pom.xml | 8 ++++++++ .../src/main/java/io/trino/spi/type/SqlVarbinary.java | 9 ++++++--- .../test/java/io/trino/spi/type/TestSqlVarbinary.java | 11 +++++++---- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/server/protocol/JsonEncodingUtils.java b/core/trino-main/src/main/java/io/trino/server/protocol/JsonEncodingUtils.java index d930cbb82cd3..57c671267ed5 100644 --- a/core/trino-main/src/main/java/io/trino/server/protocol/JsonEncodingUtils.java +++ b/core/trino-main/src/main/java/io/trino/server/protocol/JsonEncodingUtils.java @@ -49,7 +49,6 @@ import java.io.IOException; import java.math.BigDecimal; -import java.util.Base64; import java.util.List; import java.util.function.Consumer; @@ -368,14 +367,10 @@ public void encode(JsonGenerator generator, ConnectorSession session, Block bloc verify(keyBlock.getPositionCount() == valueBlock.getPositionCount(), "Key and value blocks have different number of positions"); generator.writeStartObject(); for (int i = 0; i < map.getSize(); i++) { - // Map keys are always serialized as strings for backward compatibility with existing clients, - // except for SqlVarbinary type which is encoded as base64-encoded string. + // Map keys are always serialized as strings for backward compatibility with existing clients. // Map values are always properly encoded using their types. // TODO: improve in v2 JSON format - switch (mapType.getKeyType().getObjectValue(session, keyBlock, offset + i)) { - case SqlVarbinary varbinary -> generator.writeFieldName(Base64.getEncoder().encodeToString(varbinary.getBytes())); - case Object value -> generator.writeFieldName(value.toString()); - } + generator.writeFieldName(mapType.getKeyType().getObjectValue(session, keyBlock, offset + i).toString()); valueEncoder.encode(generator, session, valueBlock, offset + i); } generator.writeEndObject(); diff --git a/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestAggregationFunction.java b/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestAggregationFunction.java index 59ad5770b07e..b0dd6f4f760e 100644 --- a/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestAggregationFunction.java +++ b/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestAggregationFunction.java @@ -363,11 +363,11 @@ private void assertPercentileWithinError(String type, SqlVarbinary binary, doubl // Check that the chosen quantile is within the upper and lower bound of the error assertThat(assertions.expression( format("value_at_quantile(CAST(a AS qdigest(%s)), %s) >= %s", type, percentile, lowerBound)) - .binding("a", "X'%s'".formatted(binary.toString().replaceAll("\\s+", " ")))) + .binding("a", "X'%s'".formatted(binary.toHexString().replaceAll("\\s+", " ")))) .isEqualTo(true); assertThat(assertions.expression( format("value_at_quantile(CAST(a AS qdigest(%s)), %s) <= %s", type, percentile, upperBound)) - .binding("a", "X'%s'".formatted(binary.toString().replaceAll("\\s+", " ")))) + .binding("a", "X'%s'".formatted(binary.toHexString().replaceAll("\\s+", " ")))) .isEqualTo(true); } @@ -384,7 +384,7 @@ private void assertPercentilesWithinError(String type, SqlVarbinary binary, doub type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(lowerBounds))) - .binding("a", "X'%s'".formatted(binary.toString().replaceAll("\\s+", " ")))) + .binding("a", "X'%s'".formatted(binary.toHexString().replaceAll("\\s+", " ")))) .hasType(new ArrayType(BOOLEAN)) .isEqualTo(Collections.nCopies(percentiles.length, true)); @@ -395,7 +395,7 @@ private void assertPercentilesWithinError(String type, SqlVarbinary binary, doub type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(upperBounds))) - .binding("a", "X'%s'".formatted(binary.toString().replaceAll("\\s+", " ")))) + .binding("a", "X'%s'".formatted(binary.toHexString().replaceAll("\\s+", " ")))) .hasType(new ArrayType(BOOLEAN)) .isEqualTo(Collections.nCopies(percentiles.length, true)); } diff --git a/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestFunctions.java b/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestFunctions.java index e029d8f0496c..d12931da47f4 100644 --- a/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestFunctions.java +++ b/core/trino-main/src/test/java/io/trino/operator/aggregation/TestQuantileDigestFunctions.java @@ -140,6 +140,6 @@ private static void addAll(QuantileDigest digest, long... values) private static String toHexString(QuantileDigest qdigest) { - return new SqlVarbinary(qdigest.serialize().getBytes()).toString().replaceAll("\\s+", " "); + return new SqlVarbinary(qdigest.serialize().getBytes()).toHexString().replaceAll("\\s+", " "); } } diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index 94a08c016a45..704f9e702b47 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -218,6 +218,14 @@ method java.lang.Iterable<io.trino.spi.eventlistener.EventListener> io.trino.spi.connector.Connector::getEventListeners() Remove connector event listeners + + true + java.annotation.removed + method byte[] io.trino.spi.type.SqlVarbinary::getBytes() + method byte[] io.trino.spi.type.SqlVarbinary::getBytes() + @com.fasterxml.jackson.annotation.JsonValue + On-the-wire representation shouldn't rely on the Jackson format + diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SqlVarbinary.java b/core/trino-spi/src/main/java/io/trino/spi/type/SqlVarbinary.java index 5c54a95fe15f..673c14fc35f6 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SqlVarbinary.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SqlVarbinary.java @@ -13,9 +13,8 @@ */ package io.trino.spi.type; -import com.fasterxml.jackson.annotation.JsonValue; - import java.util.Arrays; +import java.util.Base64; import java.util.HexFormat; import static java.lang.Math.min; @@ -42,7 +41,6 @@ public int compareTo(SqlVarbinary obj) return Arrays.compare(bytes, obj.bytes); } - @JsonValue public byte[] getBytes() { return bytes; @@ -69,6 +67,11 @@ public boolean equals(Object obj) @Override public String toString() + { + return Base64.getEncoder().encodeToString(bytes); + } + + public String toHexString() { if (bytes.length == 0) { return ""; diff --git a/core/trino-spi/src/test/java/io/trino/spi/type/TestSqlVarbinary.java b/core/trino-spi/src/test/java/io/trino/spi/type/TestSqlVarbinary.java index 54da3087a7b6..8d4bc970035a 100644 --- a/core/trino-spi/src/test/java/io/trino/spi/type/TestSqlVarbinary.java +++ b/core/trino-spi/src/test/java/io/trino/spi/type/TestSqlVarbinary.java @@ -15,24 +15,27 @@ import org.junit.jupiter.api.Test; +import java.util.Base64; + import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; public class TestSqlVarbinary { @Test - public void testToString() + public void testToHexString() { for (int lines = 0; lines < 5; lines++) { for (int lastLineBytes = 0; lastLineBytes < 32; lastLineBytes++) { byte[] bytes = createBytes(lines, lastLineBytes); - String expected = simpleToString(bytes); - assertThat(expected).isEqualTo(new SqlVarbinary(bytes).toString()); + String expectedHex = simpleToHex(bytes); + assertThat(expectedHex).isEqualTo(new SqlVarbinary(bytes).toHexString()); + assertThat(Base64.getEncoder().encodeToString(bytes)).isEqualTo(new SqlVarbinary(bytes).toString()); } } } - private static String simpleToString(byte[] bytes) + private static String simpleToHex(byte[] bytes) { StringBuilder builder = new StringBuilder(); From 1a4bdbf808476498db65a4612d1b1827b312b131 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 22 Nov 2024 13:29:33 +0100 Subject: [PATCH 143/158] Decouple Sql types from JSON serialization The new JSON serialization is not using ObjectMapper to serialize these values anymore. We want to decouple SPI types from JSON representation to be able to introduce alternative encoding formats. --- core/trino-spi/pom.xml | 48 +++++++++++++++++++ .../main/java/io/trino/spi/type/SqlDate.java | 3 -- .../java/io/trino/spi/type/SqlDecimal.java | 3 -- .../main/java/io/trino/spi/type/SqlTime.java | 3 -- .../trino/spi/type/SqlTimeWithTimeZone.java | 3 -- .../java/io/trino/spi/type/SqlTimestamp.java | 3 -- .../spi/type/SqlTimestampWithTimeZone.java | 3 -- .../trino/plugin/base/util/JsonTypeUtil.java | 31 +++++++++++- .../plugin/postgresql/PostgreSqlClient.java | 8 +++- 9 files changed, 83 insertions(+), 22 deletions(-) diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index 704f9e702b47..e6358e816586 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -226,6 +226,54 @@ @com.fasterxml.jackson.annotation.JsonValue On-the-wire representation shouldn't rely on the Jackson format + + true + java.annotation.removed + method java.lang.String io.trino.spi.type.SqlDate::toString() + method java.lang.String io.trino.spi.type.SqlDate::toString() + @com.fasterxml.jackson.annotation.JsonValue + On-the-wire representation shouldn't rely on the Jackson format + + + true + java.annotation.removed + method java.lang.String io.trino.spi.type.SqlDecimal::toString() + method java.lang.String io.trino.spi.type.SqlDecimal::toString() + @com.fasterxml.jackson.annotation.JsonValue + On-the-wire representation shouldn't rely on the Jackson format + + + true + java.annotation.removed + method java.lang.String io.trino.spi.type.SqlTime::toString() + method java.lang.String io.trino.spi.type.SqlTime::toString() + @com.fasterxml.jackson.annotation.JsonValue + On-the-wire representation shouldn't rely on the Jackson format + + + true + java.annotation.removed + method java.lang.String io.trino.spi.type.SqlTimeWithTimeZone::toString() + method java.lang.String io.trino.spi.type.SqlTimeWithTimeZone::toString() + @com.fasterxml.jackson.annotation.JsonValue + On-the-wire representation shouldn't rely on the Jackson format + + + true + java.annotation.removed + method java.lang.String io.trino.spi.type.SqlTimestamp::toString() + method java.lang.String io.trino.spi.type.SqlTimestamp::toString() + @com.fasterxml.jackson.annotation.JsonValue + On-the-wire representation shouldn't rely on the Jackson format + + + true + java.annotation.removed + method java.lang.String io.trino.spi.type.SqlTimestampWithTimeZone::toString() + method java.lang.String io.trino.spi.type.SqlTimestampWithTimeZone::toString() + @com.fasterxml.jackson.annotation.JsonValue + On-the-wire representation shouldn't rely on the Jackson format + diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SqlDate.java b/core/trino-spi/src/main/java/io/trino/spi/type/SqlDate.java index d8f092448de3..2ca4cecab692 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SqlDate.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SqlDate.java @@ -13,8 +13,6 @@ */ package io.trino.spi.type; -import com.fasterxml.jackson.annotation.JsonValue; - import java.time.LocalDate; public final class SqlDate @@ -51,7 +49,6 @@ public boolean equals(Object obj) return days == other.days; } - @JsonValue @Override public String toString() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SqlDecimal.java b/core/trino-spi/src/main/java/io/trino/spi/type/SqlDecimal.java index f84341736ad6..d25369fe6782 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SqlDecimal.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SqlDecimal.java @@ -13,8 +13,6 @@ */ package io.trino.spi.type; -import com.fasterxml.jackson.annotation.JsonValue; - import java.math.BigDecimal; import java.math.BigInteger; import java.math.MathContext; @@ -81,7 +79,6 @@ public int hashCode() return Objects.hash(unscaledValue, precision, scale); } - @JsonValue @Override public String toString() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTime.java b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTime.java index a70a50a1875b..44113f652cb0 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTime.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTime.java @@ -13,8 +13,6 @@ */ package io.trino.spi.type; -import com.fasterxml.jackson.annotation.JsonValue; - import java.util.Objects; import static io.trino.spi.type.TimeType.MAX_PRECISION; @@ -85,7 +83,6 @@ public int hashCode() return Objects.hash(precision, picos); } - @JsonValue @Override public String toString() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimeWithTimeZone.java b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimeWithTimeZone.java index dc039a09bf4a..c37c18a32b6c 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimeWithTimeZone.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimeWithTimeZone.java @@ -13,8 +13,6 @@ */ package io.trino.spi.type; -import com.fasterxml.jackson.annotation.JsonValue; - import java.util.Objects; import static io.trino.spi.type.TimeType.MAX_PRECISION; @@ -99,7 +97,6 @@ public int hashCode() return Objects.hash(precision, picos, offsetMinutes); } - @JsonValue @Override public String toString() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestamp.java b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestamp.java index bab2fadfda55..1a2b2558adff 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestamp.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestamp.java @@ -13,8 +13,6 @@ */ package io.trino.spi.type; -import com.fasterxml.jackson.annotation.JsonValue; - import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.Objects; @@ -149,7 +147,6 @@ public int hashCode() return Objects.hash(epochMicros, picosOfMicros, precision); } - @JsonValue @Override public String toString() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestampWithTimeZone.java b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestampWithTimeZone.java index 7c1b6155f5ce..9eaf8133e80c 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestampWithTimeZone.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SqlTimestampWithTimeZone.java @@ -13,8 +13,6 @@ */ package io.trino.spi.type; -import com.fasterxml.jackson.annotation.JsonValue; - import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -132,7 +130,6 @@ public SqlTimestampWithTimeZone roundTo(int precision) return newInstanceWithRounding(precision, epochMillis, picosOfMilli, timeZoneKey); } - @JsonValue @Override public String toString() { diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/JsonTypeUtil.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/JsonTypeUtil.java index 46150e5975df..ec110a2407e7 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/JsonTypeUtil.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/JsonTypeUtil.java @@ -22,10 +22,19 @@ import io.airlift.slice.SliceOutput; import io.airlift.slice.Slices; import io.trino.spi.TrinoException; +import io.trino.spi.type.SqlDate; +import io.trino.spi.type.SqlDecimal; +import io.trino.spi.type.SqlTime; +import io.trino.spi.type.SqlTimeWithTimeZone; +import io.trino.spi.type.SqlTimestamp; +import io.trino.spi.type.SqlTimestampWithTimeZone; +import io.trino.spi.type.SqlVarbinary; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; +import java.util.List; +import java.util.StringJoiner; import static com.fasterxml.jackson.core.JsonFactory.Feature.CANONICALIZE_FIELD_NAMES; import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS; @@ -63,10 +72,28 @@ public static Slice jsonParse(Slice slice) } } - public static Slice toJsonValue(Object value) + public static Slice toJsonValue(List values) throws IOException { - return Slices.wrappedBuffer(SORTED_MAPPER.writeValueAsBytes(value)); + if (values == null) { + return Slices.utf8Slice("[]"); + } + + StringJoiner joiner = new StringJoiner(",", "[", "]"); + for (Object value : values) { + joiner.add(switch (value) { + case null -> "null"; + case SqlDate _, + SqlTime _, + SqlVarbinary _, + SqlTimeWithTimeZone _, + SqlDecimal _, + SqlTimestamp _, + SqlTimestampWithTimeZone _ -> SORTED_MAPPER.writeValueAsString(value.toString()); + default -> SORTED_MAPPER.writeValueAsString(value); + }); + } + return Slices.utf8Slice(joiner.toString()); } private static JsonParser createJsonParser(JsonFactory factory, Slice json) diff --git a/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java b/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java index 7cc262b79ea9..68bd81cb23ba 100644 --- a/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java +++ b/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java @@ -1509,11 +1509,15 @@ private static SliceReadFunction arrayAsJsonReadFunction(ConnectorSession sessio type.writeObject(builder, block); Object value = type.getObjectValue(session, builder.build(), 0); + if (!(value instanceof List list)) { + throw new TrinoException(JDBC_ERROR, "Unexpected JSON object value for " + type.getDisplayName() + " expected List, got " + value.getClass().getSimpleName()); + } + try { - return toJsonValue(value); + return toJsonValue(list); } catch (IOException e) { - throw new TrinoException(JDBC_ERROR, "Conversion to JSON failed for " + type.getDisplayName(), e); + throw new TrinoException(JDBC_ERROR, "Conversion to JSON failed for " + type.getDisplayName(), e); } }; } From c06f1e1755d629972ca34f3fcfca486f28411320 Mon Sep 17 00:00:00 2001 From: Jungwoo Lee Date: Tue, 31 Dec 2024 19:03:05 +0000 Subject: [PATCH 144/158] Derive aws sdk retry count from request count --- .../trino/hdfs/s3/AwsSdkClientCoreStats.java | 20 +++++++++++++++---- .../glue/v1/AwsSdkClientCoreStats.java | 20 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/s3/AwsSdkClientCoreStats.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/s3/AwsSdkClientCoreStats.java index 88c27e50ca34..af139c41d922 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/s3/AwsSdkClientCoreStats.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/s3/AwsSdkClientCoreStats.java @@ -44,6 +44,7 @@ public final class AwsSdkClientCoreStats { private final CounterStat awsRequestCount = new CounterStat(); private final CounterStat awsRetryCount = new CounterStat(); + private final CounterStat awsHttpClientRetryCount = new CounterStat(); private final CounterStat awsThrottleExceptions = new CounterStat(); private final TimeStat awsRequestTime = new TimeStat(MILLISECONDS); private final TimeStat awsClientExecuteTime = new TimeStat(MILLISECONDS); @@ -66,6 +67,13 @@ public CounterStat getAwsRetryCount() return awsRetryCount; } + @Managed + @Nested + public CounterStat getAwsHttpClientRetryCount() + { + return awsHttpClientRetryCount; + } + @Managed @Nested public CounterStat getAwsThrottleExceptions() @@ -134,12 +142,16 @@ public void collectMetrics(Request request, Response response) Number requestCounts = timingInfo.getCounter(RequestCount.name()); if (requestCounts != null) { - stats.awsRequestCount.update(requestCounts.longValue()); + long count = requestCounts.longValue(); + stats.awsRequestCount.update(count); + if (count > 1) { + stats.awsRetryCount.update(count - 1); + } } - Number retryCounts = timingInfo.getCounter(HttpClientRetryCount.name()); - if (retryCounts != null) { - stats.awsRetryCount.update(retryCounts.longValue()); + Number httpClientRetryCounts = timingInfo.getCounter(HttpClientRetryCount.name()); + if (httpClientRetryCounts != null) { + stats.awsHttpClientRetryCount.update(httpClientRetryCounts.longValue()); } Number throttleExceptions = timingInfo.getCounter(ThrottleException.name()); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/AwsSdkClientCoreStats.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/AwsSdkClientCoreStats.java index 8d48c88582d0..95517512bbc2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/AwsSdkClientCoreStats.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/AwsSdkClientCoreStats.java @@ -44,6 +44,7 @@ public final class AwsSdkClientCoreStats { private final CounterStat awsRequestCount = new CounterStat(); private final CounterStat awsRetryCount = new CounterStat(); + private final CounterStat awsHttpClientRetryCount = new CounterStat(); private final CounterStat awsThrottleExceptions = new CounterStat(); private final TimeStat awsRequestTime = new TimeStat(MILLISECONDS); private final TimeStat awsClientExecuteTime = new TimeStat(MILLISECONDS); @@ -66,6 +67,13 @@ public CounterStat getAwsRetryCount() return awsRetryCount; } + @Managed + @Nested + public CounterStat getAwsHttpClientRetryCount() + { + return awsHttpClientRetryCount; + } + @Managed @Nested public CounterStat getAwsThrottleExceptions() @@ -134,12 +142,16 @@ public void collectMetrics(Request request, Response response) Number requestCounts = timingInfo.getCounter(RequestCount.name()); if (requestCounts != null) { - stats.awsRequestCount.update(requestCounts.longValue()); + long count = requestCounts.longValue(); + stats.awsRequestCount.update(count); + if (count > 1) { + stats.awsRetryCount.update(count - 1); + } } - Number retryCounts = timingInfo.getCounter(HttpClientRetryCount.name()); - if (retryCounts != null) { - stats.awsRetryCount.update(retryCounts.longValue()); + Number httpClientRetryCounts = timingInfo.getCounter(HttpClientRetryCount.name()); + if (httpClientRetryCounts != null) { + stats.awsHttpClientRetryCount.update(httpClientRetryCounts.longValue()); } Number throttleExceptions = timingInfo.getCounter(ThrottleException.name()); From 0ccb5071e4779702cea71a6450d052aa9cce0f70 Mon Sep 17 00:00:00 2001 From: Anu Sudarsan Date: Fri, 3 Jan 2025 00:27:35 -0600 Subject: [PATCH 145/158] Minor cleanup in Hive procedures --- .../procedure/CreateEmptyPartitionProcedure.java | 8 +++++--- .../plugin/hive/procedure/DropStatsProcedure.java | 15 ++++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java index 97044f95cdc6..21d2792ce39f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/CreateEmptyPartitionProcedure.java @@ -23,7 +23,6 @@ import io.trino.plugin.base.util.UncheckedCloseable; import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.HiveInsertTableHandle; -import io.trino.plugin.hive.HiveTableHandle; import io.trino.plugin.hive.LocationService; import io.trino.plugin.hive.LocationService.WriteInfo; import io.trino.plugin.hive.PartitionUpdate; @@ -34,6 +33,7 @@ import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.ConnectorAccessControl; import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.procedure.Procedure; import io.trino.spi.procedure.Procedure.Argument; @@ -112,14 +112,16 @@ private void doCreateEmptyPartition(ConnectorSession session, ConnectorAccessCon TransactionalMetadata hiveMetadata = hiveMetadataFactory.create(session.getIdentity(), true); hiveMetadata.beginQuery(session); try (UncheckedCloseable ignore = () -> hiveMetadata.cleanupQuery(session)) { - HiveTableHandle tableHandle = (HiveTableHandle) hiveMetadata.getTableHandle(session, new SchemaTableName(schemaName, tableName), Optional.empty(), Optional.empty()); + ConnectorTableHandle tableHandle = hiveMetadata.getTableHandle(session, new SchemaTableName(schemaName, tableName), Optional.empty(), Optional.empty()); if (tableHandle == null) { throw new TrinoException(INVALID_PROCEDURE_ARGUMENT, format("Table '%s' does not exist", new SchemaTableName(schemaName, tableName))); } accessControl.checkCanInsertIntoTable(null, new SchemaTableName(schemaName, tableName)); - List actualPartitionColumnNames = tableHandle.getPartitionColumns().stream() + List actualPartitionColumnNames = hiveMetadata.getColumnHandles(session, tableHandle).values().stream() + .map(HiveColumnHandle.class::cast) + .filter(HiveColumnHandle::isPartitionKey) .map(HiveColumnHandle::getName) .collect(toImmutableList()); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java index add2f10235e8..ad85ca0018c8 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/DropStatsProcedure.java @@ -21,7 +21,6 @@ import io.trino.metastore.PartitionStatistics; import io.trino.plugin.base.util.UncheckedCloseable; import io.trino.plugin.hive.HiveColumnHandle; -import io.trino.plugin.hive.HiveTableHandle; import io.trino.plugin.hive.TransactionalMetadata; import io.trino.plugin.hive.TransactionalMetadataFactory; import io.trino.spi.TrinoException; @@ -29,6 +28,7 @@ import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorAccessControl; import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.predicate.TupleDomain; @@ -107,12 +107,13 @@ private void doDropStats(ConnectorSession session, ConnectorAccessControl access TransactionalMetadata hiveMetadata = hiveMetadataFactory.create(session.getIdentity(), true); hiveMetadata.beginQuery(session); try (UncheckedCloseable ignore = () -> hiveMetadata.cleanupQuery(session)) { - HiveTableHandle handle = (HiveTableHandle) hiveMetadata.getTableHandle(session, new SchemaTableName(schema, table), Optional.empty(), Optional.empty()); + SchemaTableName schemaTableName = new SchemaTableName(schema, table); + ConnectorTableHandle handle = hiveMetadata.getTableHandle(session, schemaTableName, Optional.empty(), Optional.empty()); if (handle == null) { - throw new TrinoException(INVALID_PROCEDURE_ARGUMENT, format("Table '%s' does not exist", new SchemaTableName(schema, table))); + throw new TrinoException(INVALID_PROCEDURE_ARGUMENT, format("Table '%s' does not exist", schemaTableName)); } - accessControl.checkCanInsertIntoTable(null, new SchemaTableName(schema, table)); + accessControl.checkCanInsertIntoTable(null, schemaTableName); Map columns = hiveMetadata.getColumnHandles(session, handle); List partitionColumns = columns.values().stream() @@ -131,7 +132,7 @@ private void doDropStats(ConnectorSession session, ConnectorAccessControl access partitionStringValues.forEach(values -> metastore.updatePartitionStatistics( metastore.getTable(schema, table) - .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(schema, table))), + .orElseThrow(() -> new TableNotFoundException(schemaTableName)), CLEAR_ALL, ImmutableMap.of( makePartName(partitionColumns, values), @@ -150,10 +151,10 @@ private void doDropStats(ConnectorSession session, ConnectorAccessControl access } else { // the table is partitioned; remove stats for every partition - hiveMetadata.getMetastore().getPartitionNamesByFilter(handle.getSchemaName(), handle.getTableName(), partitionColumns, TupleDomain.all()) + hiveMetadata.getMetastore().getPartitionNamesByFilter(schemaTableName.getSchemaName(), schemaTableName.getTableName(), partitionColumns, TupleDomain.all()) .ifPresent(partitions -> partitions.forEach(partitionName -> metastore.updatePartitionStatistics( metastore.getTable(schema, table) - .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(schema, table))), + .orElseThrow(() -> new TableNotFoundException(schemaTableName)), CLEAR_ALL, ImmutableMap.of( partitionName, From dbcfc5f3593be12b895156247e665dd1310640e8 Mon Sep 17 00:00:00 2001 From: Jinyang Li Date: Fri, 6 Dec 2024 15:08:57 -0800 Subject: [PATCH 146/158] Move createParquetMetadata to ParquetMetadata --- .../parquet/metadata/ParquetMetadata.java | 181 ++++++++++++++++++ .../trino/parquet/reader/MetadataReader.java | 176 +---------------- .../plugin/deltalake/DeltaLakeWriter.java | 3 +- .../iceberg/IcebergParquetFileWriter.java | 3 +- 4 files changed, 184 insertions(+), 179 deletions(-) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java index 9363add2ca6c..0636cb1207c1 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java @@ -13,10 +13,49 @@ */ package io.trino.parquet.metadata; +import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; +import io.trino.parquet.ParquetCorruptionException; +import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.reader.MetadataReader; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.format.ColumnChunk; +import org.apache.parquet.format.ColumnMetaData; +import org.apache.parquet.format.FileMetaData; +import org.apache.parquet.format.KeyValue; +import org.apache.parquet.format.RowGroup; +import org.apache.parquet.format.SchemaElement; +import org.apache.parquet.hadoop.metadata.ColumnPath; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.trino.parquet.ParquetMetadataConverter.convertEncodingStats; +import static io.trino.parquet.ParquetMetadataConverter.getEncoding; +import static io.trino.parquet.ParquetMetadataConverter.getLogicalTypeAnnotation; +import static io.trino.parquet.ParquetMetadataConverter.getPrimitive; +import static io.trino.parquet.ParquetMetadataConverter.toColumnIndexReference; +import static io.trino.parquet.ParquetMetadataConverter.toOffsetIndexReference; +import static io.trino.parquet.ParquetValidationUtils.validateParquet; public class ParquetMetadata { + private static final Logger log = Logger.get(ParquetMetadata.class); + private final FileMetadata fileMetaData; private final List blocks; @@ -41,4 +80,146 @@ public String toString() { return "ParquetMetaData{" + fileMetaData + ", blocks: " + blocks + "}"; } + + public static ParquetMetadata createParquetMetadata(FileMetaData fileMetaData, ParquetDataSourceId dataSourceId) + throws ParquetCorruptionException + { + List schema = fileMetaData.getSchema(); + validateParquet(!schema.isEmpty(), dataSourceId, "Schema is empty"); + + MessageType messageType = readParquetSchema(schema); + List blocks = new ArrayList<>(); + List rowGroups = fileMetaData.getRow_groups(); + if (rowGroups != null) { + for (RowGroup rowGroup : rowGroups) { + List columns = rowGroup.getColumns(); + validateParquet(!columns.isEmpty(), dataSourceId, "No columns in row group: %s", rowGroup); + String filePath = columns.get(0).getFile_path(); + ImmutableList.Builder columnMetadataBuilder = ImmutableList.builderWithExpectedSize(columns.size()); + for (ColumnChunk columnChunk : columns) { + validateParquet( + (filePath == null && columnChunk.getFile_path() == null) + || (filePath != null && filePath.equals(columnChunk.getFile_path())), + dataSourceId, + "all column chunks of the same row group must be in the same file"); + ColumnMetaData metaData = columnChunk.meta_data; + String[] path = metaData.path_in_schema.stream() + .map(value -> value.toLowerCase(Locale.ENGLISH)) + .toArray(String[]::new); + ColumnPath columnPath = ColumnPath.get(path); + PrimitiveType primitiveType = messageType.getType(columnPath.toArray()).asPrimitiveType(); + ColumnChunkMetadata column = ColumnChunkMetadata.get( + columnPath, + primitiveType, + CompressionCodecName.fromParquet(metaData.codec), + convertEncodingStats(metaData.encoding_stats), + readEncodings(metaData.encodings), + MetadataReader.readStats(Optional.ofNullable(fileMetaData.getCreated_by()), Optional.ofNullable(metaData.statistics), primitiveType), + metaData.data_page_offset, + metaData.dictionary_page_offset, + metaData.num_values, + metaData.total_compressed_size, + metaData.total_uncompressed_size); + column.setColumnIndexReference(toColumnIndexReference(columnChunk)); + column.setOffsetIndexReference(toOffsetIndexReference(columnChunk)); + column.setBloomFilterOffset(metaData.bloom_filter_offset); + columnMetadataBuilder.add(column); + } + blocks.add(new BlockMetadata(rowGroup.getNum_rows(), columnMetadataBuilder.build())); + } + } + + Map keyValueMetaData = new HashMap<>(); + List keyValueList = fileMetaData.getKey_value_metadata(); + if (keyValueList != null) { + for (KeyValue keyValue : keyValueList) { + keyValueMetaData.put(keyValue.key, keyValue.value); + } + } + FileMetadata parquetFileMetadata = new FileMetadata( + messageType, + keyValueMetaData, + fileMetaData.getCreated_by()); + return new ParquetMetadata(parquetFileMetadata, blocks); + } + + private static MessageType readParquetSchema(List schema) + { + Iterator schemaIterator = schema.iterator(); + SchemaElement rootSchema = schemaIterator.next(); + Types.MessageTypeBuilder builder = Types.buildMessage(); + readTypeSchema(builder, schemaIterator, rootSchema.getNum_children()); + return builder.named(rootSchema.name); + } + + private static void readTypeSchema(Types.GroupBuilder builder, Iterator schemaIterator, int typeCount) + { + for (int i = 0; i < typeCount; i++) { + SchemaElement element = schemaIterator.next(); + Types.Builder typeBuilder; + if (element.type == null) { + typeBuilder = builder.group(Type.Repetition.valueOf(element.repetition_type.name())); + readTypeSchema((Types.GroupBuilder) typeBuilder, schemaIterator, element.num_children); + } + else { + Types.PrimitiveBuilder primitiveBuilder = builder.primitive(getPrimitive(element.type), Type.Repetition.valueOf(element.repetition_type.name())); + if (element.isSetType_length()) { + primitiveBuilder.length(element.type_length); + } + if (element.isSetPrecision()) { + primitiveBuilder.precision(element.precision); + } + if (element.isSetScale()) { + primitiveBuilder.scale(element.scale); + } + typeBuilder = primitiveBuilder; + } + + // Reading of element.logicalType and element.converted_type corresponds to parquet-mr's code at + // https://github.com/apache/parquet-mr/blob/apache-parquet-1.12.0/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java#L1568-L1582 + LogicalTypeAnnotation annotationFromLogicalType = null; + if (element.isSetLogicalType()) { + annotationFromLogicalType = getLogicalTypeAnnotation(element.logicalType); + typeBuilder.as(annotationFromLogicalType); + } + if (element.isSetConverted_type()) { + LogicalTypeAnnotation annotationFromConvertedType = getLogicalTypeAnnotation(element.converted_type, element); + if (annotationFromLogicalType != null) { + // Both element.logicalType and element.converted_type set + if (annotationFromLogicalType.toOriginalType() == annotationFromConvertedType.toOriginalType()) { + // element.converted_type matches element.logicalType, even though annotationFromLogicalType may differ from annotationFromConvertedType + // Following parquet-mr behavior, we favor LogicalTypeAnnotation derived from element.logicalType, as potentially containing more information. + } + else { + // Following parquet-mr behavior, issue warning and let converted_type take precedence. + log.warn("Converted type and logical type metadata map to different OriginalType (convertedType: %s, logical type: %s). Using value in converted type.", + element.converted_type, element.logicalType); + // parquet-mr reads only OriginalType from converted_type. We retain full LogicalTypeAnnotation + // 1. for compatibility, as previous Trino reader code would read LogicalTypeAnnotation from element.converted_type and some additional fields. + // 2. so that we override LogicalTypeAnnotation annotation read from element.logicalType in case of mismatch detected. + typeBuilder.as(annotationFromConvertedType); + } + } + else { + // parquet-mr reads only OriginalType from converted_type. We retain full LogicalTypeAnnotation for compatibility, as previous + // Trino reader code would read LogicalTypeAnnotation from element.converted_type and some additional fields. + typeBuilder.as(annotationFromConvertedType); + } + } + + if (element.isSetField_id()) { + typeBuilder.id(element.field_id); + } + typeBuilder.named(element.name.toLowerCase(Locale.ENGLISH)); + } + } + + private static Set readEncodings(List encodings) + { + Set columnEncodings = new HashSet<>(); + for (org.apache.parquet.format.Encoding encoding : encodings) { + columnEncodings.add(getEncoding(encoding)); + } + return Collections.unmodifiableSet(columnEncodings); + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java index fe0635646f98..9c1ca8c42729 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java @@ -13,57 +13,27 @@ */ package io.trino.parquet.reader; -import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.ParquetDataSource; import io.trino.parquet.ParquetDataSourceId; import io.trino.parquet.ParquetWriteValidation; -import io.trino.parquet.metadata.BlockMetadata; -import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.parquet.metadata.FileMetadata; import io.trino.parquet.metadata.ParquetMetadata; import org.apache.parquet.CorruptStatistics; import org.apache.parquet.column.statistics.BinaryStatistics; -import org.apache.parquet.format.ColumnChunk; -import org.apache.parquet.format.ColumnMetaData; -import org.apache.parquet.format.Encoding; import org.apache.parquet.format.FileMetaData; -import org.apache.parquet.format.KeyValue; -import org.apache.parquet.format.RowGroup; -import org.apache.parquet.format.SchemaElement; import org.apache.parquet.format.Statistics; -import org.apache.parquet.hadoop.metadata.ColumnPath; -import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.schema.LogicalTypeAnnotation; -import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type.Repetition; -import org.apache.parquet.schema.Types; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; import java.util.Optional; -import java.util.Set; -import static io.trino.parquet.ParquetMetadataConverter.convertEncodingStats; import static io.trino.parquet.ParquetMetadataConverter.fromParquetStatistics; -import static io.trino.parquet.ParquetMetadataConverter.getEncoding; -import static io.trino.parquet.ParquetMetadataConverter.getLogicalTypeAnnotation; -import static io.trino.parquet.ParquetMetadataConverter.getPrimitive; -import static io.trino.parquet.ParquetMetadataConverter.toColumnIndexReference; -import static io.trino.parquet.ParquetMetadataConverter.toOffsetIndexReference; import static io.trino.parquet.ParquetValidationUtils.validateParquet; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; @@ -73,8 +43,6 @@ public final class MetadataReader { - private static final Logger log = Logger.get(MetadataReader.class); - private static final Slice MAGIC = Slices.utf8Slice("PAR1"); private static final int POST_SCRIPT_SIZE = Integer.BYTES + MAGIC.length(); // Typical 1GB files produced by Trino were found to have footer size between 30-40KB @@ -119,144 +87,11 @@ public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional< InputStream metadataStream = buffer.slice(buffer.length() - completeFooterSize, metadataLength).getInput(); FileMetaData fileMetaData = readFileMetaData(metadataStream); - ParquetMetadata parquetMetadata = createParquetMetadata(fileMetaData, dataSource.getId()); + ParquetMetadata parquetMetadata = ParquetMetadata.createParquetMetadata(fileMetaData, dataSource.getId()); validateFileMetadata(dataSource.getId(), parquetMetadata.getFileMetaData(), parquetWriteValidation); return parquetMetadata; } - public static ParquetMetadata createParquetMetadata(FileMetaData fileMetaData, ParquetDataSourceId dataSourceId) - throws ParquetCorruptionException - { - List schema = fileMetaData.getSchema(); - validateParquet(!schema.isEmpty(), dataSourceId, "Schema is empty"); - - MessageType messageType = readParquetSchema(schema); - List blocks = new ArrayList<>(); - List rowGroups = fileMetaData.getRow_groups(); - if (rowGroups != null) { - for (RowGroup rowGroup : rowGroups) { - List columns = rowGroup.getColumns(); - validateParquet(!columns.isEmpty(), dataSourceId, "No columns in row group: %s", rowGroup); - String filePath = columns.get(0).getFile_path(); - ImmutableList.Builder columnMetadataBuilder = ImmutableList.builderWithExpectedSize(columns.size()); - for (ColumnChunk columnChunk : columns) { - validateParquet( - (filePath == null && columnChunk.getFile_path() == null) - || (filePath != null && filePath.equals(columnChunk.getFile_path())), - dataSourceId, - "all column chunks of the same row group must be in the same file"); - ColumnMetaData metaData = columnChunk.meta_data; - String[] path = metaData.path_in_schema.stream() - .map(value -> value.toLowerCase(Locale.ENGLISH)) - .toArray(String[]::new); - ColumnPath columnPath = ColumnPath.get(path); - PrimitiveType primitiveType = messageType.getType(columnPath.toArray()).asPrimitiveType(); - ColumnChunkMetadata column = ColumnChunkMetadata.get( - columnPath, - primitiveType, - CompressionCodecName.fromParquet(metaData.codec), - convertEncodingStats(metaData.encoding_stats), - readEncodings(metaData.encodings), - readStats(Optional.ofNullable(fileMetaData.getCreated_by()), Optional.ofNullable(metaData.statistics), primitiveType), - metaData.data_page_offset, - metaData.dictionary_page_offset, - metaData.num_values, - metaData.total_compressed_size, - metaData.total_uncompressed_size); - column.setColumnIndexReference(toColumnIndexReference(columnChunk)); - column.setOffsetIndexReference(toOffsetIndexReference(columnChunk)); - column.setBloomFilterOffset(metaData.bloom_filter_offset); - columnMetadataBuilder.add(column); - } - blocks.add(new BlockMetadata(rowGroup.getNum_rows(), columnMetadataBuilder.build())); - } - } - - Map keyValueMetaData = new HashMap<>(); - List keyValueList = fileMetaData.getKey_value_metadata(); - if (keyValueList != null) { - for (KeyValue keyValue : keyValueList) { - keyValueMetaData.put(keyValue.key, keyValue.value); - } - } - FileMetadata parquetFileMetadata = new FileMetadata( - messageType, - keyValueMetaData, - fileMetaData.getCreated_by()); - return new ParquetMetadata(parquetFileMetadata, blocks); - } - - private static MessageType readParquetSchema(List schema) - { - Iterator schemaIterator = schema.iterator(); - SchemaElement rootSchema = schemaIterator.next(); - Types.MessageTypeBuilder builder = Types.buildMessage(); - readTypeSchema(builder, schemaIterator, rootSchema.getNum_children()); - return builder.named(rootSchema.name); - } - - private static void readTypeSchema(Types.GroupBuilder builder, Iterator schemaIterator, int typeCount) - { - for (int i = 0; i < typeCount; i++) { - SchemaElement element = schemaIterator.next(); - Types.Builder typeBuilder; - if (element.type == null) { - typeBuilder = builder.group(Repetition.valueOf(element.repetition_type.name())); - readTypeSchema((Types.GroupBuilder) typeBuilder, schemaIterator, element.num_children); - } - else { - Types.PrimitiveBuilder primitiveBuilder = builder.primitive(getPrimitive(element.type), Repetition.valueOf(element.repetition_type.name())); - if (element.isSetType_length()) { - primitiveBuilder.length(element.type_length); - } - if (element.isSetPrecision()) { - primitiveBuilder.precision(element.precision); - } - if (element.isSetScale()) { - primitiveBuilder.scale(element.scale); - } - typeBuilder = primitiveBuilder; - } - - // Reading of element.logicalType and element.converted_type corresponds to parquet-mr's code at - // https://github.com/apache/parquet-mr/blob/apache-parquet-1.12.0/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java#L1568-L1582 - LogicalTypeAnnotation annotationFromLogicalType = null; - if (element.isSetLogicalType()) { - annotationFromLogicalType = getLogicalTypeAnnotation(element.logicalType); - typeBuilder.as(annotationFromLogicalType); - } - if (element.isSetConverted_type()) { - LogicalTypeAnnotation annotationFromConvertedType = getLogicalTypeAnnotation(element.converted_type, element); - if (annotationFromLogicalType != null) { - // Both element.logicalType and element.converted_type set - if (annotationFromLogicalType.toOriginalType() == annotationFromConvertedType.toOriginalType()) { - // element.converted_type matches element.logicalType, even though annotationFromLogicalType may differ from annotationFromConvertedType - // Following parquet-mr behavior, we favor LogicalTypeAnnotation derived from element.logicalType, as potentially containing more information. - } - else { - // Following parquet-mr behavior, issue warning and let converted_type take precedence. - log.warn("Converted type and logical type metadata map to different OriginalType (convertedType: %s, logical type: %s). Using value in converted type.", - element.converted_type, element.logicalType); - // parquet-mr reads only OriginalType from converted_type. We retain full LogicalTypeAnnotation - // 1. for compatibility, as previous Trino reader code would read LogicalTypeAnnotation from element.converted_type and some additional fields. - // 2. so that we override LogicalTypeAnnotation annotation read from element.logicalType in case of mismatch detected. - typeBuilder.as(annotationFromConvertedType); - } - } - else { - // parquet-mr reads only OriginalType from converted_type. We retain full LogicalTypeAnnotation for compatibility, as previous - // Trino reader code would read LogicalTypeAnnotation from element.converted_type and some additional fields. - typeBuilder.as(annotationFromConvertedType); - } - } - - if (element.isSetField_id()) { - typeBuilder.id(element.field_id); - } - typeBuilder.named(element.name.toLowerCase(Locale.ENGLISH)); - } - } - public static org.apache.parquet.column.statistics.Statistics readStats(Optional fileCreatedBy, Optional statisticsFromFile, PrimitiveType type) { Statistics statistics = statisticsFromFile.orElse(null); @@ -352,15 +187,6 @@ private static int commonPrefix(byte[] a, byte[] b) return commonPrefixLength; } - private static Set readEncodings(List encodings) - { - Set columnEncodings = new HashSet<>(); - for (Encoding encoding : encodings) { - columnEncodings.add(getEncoding(encoding)); - } - return Collections.unmodifiableSet(columnEncodings); - } - private static void validateFileMetadata(ParquetDataSourceId dataSourceId, FileMetadata fileMetaData, Optional parquetWriteValidation) throws ParquetCorruptionException { diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java index 8f686205e239..b33ea8e369a4 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java @@ -22,7 +22,6 @@ import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.parquet.metadata.ParquetMetadata; -import io.trino.parquet.reader.MetadataReader; import io.trino.plugin.deltalake.DataFileInfo.DataFileType; import io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeJsonFileStatistics; import io.trino.plugin.hive.FileWriter; @@ -184,7 +183,7 @@ public DataFileInfo getDataFileInfo() { Location path = rootTableLocation.appendPath(relativeFilePath); FileMetaData fileMetaData = fileWriter.getFileMetadata(); - ParquetMetadata parquetMetadata = MetadataReader.createParquetMetadata(fileMetaData, new ParquetDataSourceId(path.toString())); + ParquetMetadata parquetMetadata = ParquetMetadata.createParquetMetadata(fileMetaData, new ParquetDataSourceId(path.toString())); return new DataFileInfo( relativeFilePath, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java index 7f0716b66188..f03ad35e82dc 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java @@ -33,7 +33,6 @@ import java.util.Optional; import java.util.stream.Stream; -import static io.trino.parquet.reader.MetadataReader.createParquetMetadata; import static io.trino.plugin.iceberg.util.ParquetUtil.footerMetrics; import static io.trino.plugin.iceberg.util.ParquetUtil.getSplitOffsets; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -83,7 +82,7 @@ public FileMetrics getFileMetrics() { ParquetMetadata parquetMetadata; try { - parquetMetadata = createParquetMetadata(parquetFileWriter.getFileMetadata(), new ParquetDataSourceId(location.toString())); + parquetMetadata = ParquetMetadata.createParquetMetadata(parquetFileWriter.getFileMetadata(), new ParquetDataSourceId(location.toString())); } catch (IOException e) { throw new TrinoException(GENERIC_INTERNAL_ERROR, format("Error creating metadata for Parquet file %s", location), e); From 358633f27639b6975b552766a04148cf7a19c25e Mon Sep 17 00:00:00 2001 From: Jinyang Li Date: Fri, 6 Dec 2024 15:39:02 -0800 Subject: [PATCH 147/158] Parse parquet footer row groups lazily --- .../parquet/metadata/ParquetMetadata.java | 76 ++++++++++++------- .../trino/parquet/reader/MetadataReader.java | 2 +- .../plugin/deltalake/DeltaLakeWriter.java | 2 +- .../iceberg/IcebergParquetFileWriter.java | 4 +- .../plugin/iceberg/util/ParquetUtil.java | 9 ++- .../plugin/iceberg/IcebergTestUtils.java | 6 +- 6 files changed, 62 insertions(+), 37 deletions(-) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java index 0636cb1207c1..3eecc23a41fd 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java @@ -14,6 +14,7 @@ package io.trino.parquet.metadata; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.ParquetDataSourceId; @@ -35,7 +36,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -44,6 +44,8 @@ import java.util.Optional; import java.util.Set; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.trino.parquet.ParquetMetadataConverter.convertEncodingStats; import static io.trino.parquet.ParquetMetadataConverter.getEncoding; import static io.trino.parquet.ParquetMetadataConverter.getLogicalTypeAnnotation; @@ -51,45 +53,49 @@ import static io.trino.parquet.ParquetMetadataConverter.toColumnIndexReference; import static io.trino.parquet.ParquetMetadataConverter.toOffsetIndexReference; import static io.trino.parquet.ParquetValidationUtils.validateParquet; +import static java.util.Objects.requireNonNull; public class ParquetMetadata { private static final Logger log = Logger.get(ParquetMetadata.class); - private final FileMetadata fileMetaData; - private final List blocks; + private final FileMetaData parquetMetadata; + private final ParquetDataSourceId dataSourceId; + private final FileMetadata fileMetadata; - public ParquetMetadata(FileMetadata fileMetaData, List blocks) - { - this.fileMetaData = fileMetaData; - this.blocks = blocks; - } - - public List getBlocks() + public ParquetMetadata(FileMetaData parquetMetadata, ParquetDataSourceId dataSourceId) + throws ParquetCorruptionException { - return blocks; + this.fileMetadata = new FileMetadata( + readMessageType(parquetMetadata, dataSourceId), + keyValueMetaData(parquetMetadata), + parquetMetadata.getCreated_by()); + this.parquetMetadata = parquetMetadata; + this.dataSourceId = requireNonNull(dataSourceId, "dataSourceId is null"); } public FileMetadata getFileMetaData() { - return fileMetaData; + return fileMetadata; } @Override public String toString() { - return "ParquetMetaData{" + fileMetaData + ", blocks: " + blocks + "}"; + return toStringHelper(this) + .add("parquetMetadata", parquetMetadata) + .toString(); } - public static ParquetMetadata createParquetMetadata(FileMetaData fileMetaData, ParquetDataSourceId dataSourceId) + public List getBlocks() throws ParquetCorruptionException { - List schema = fileMetaData.getSchema(); + List schema = parquetMetadata.getSchema(); validateParquet(!schema.isEmpty(), dataSourceId, "Schema is empty"); MessageType messageType = readParquetSchema(schema); List blocks = new ArrayList<>(); - List rowGroups = fileMetaData.getRow_groups(); + List rowGroups = parquetMetadata.getRow_groups(); if (rowGroups != null) { for (RowGroup rowGroup : rowGroups) { List columns = rowGroup.getColumns(); @@ -114,7 +120,7 @@ public static ParquetMetadata createParquetMetadata(FileMetaData fileMetaData, P CompressionCodecName.fromParquet(metaData.codec), convertEncodingStats(metaData.encoding_stats), readEncodings(metaData.encodings), - MetadataReader.readStats(Optional.ofNullable(fileMetaData.getCreated_by()), Optional.ofNullable(metaData.statistics), primitiveType), + MetadataReader.readStats(Optional.ofNullable(parquetMetadata.getCreated_by()), Optional.ofNullable(metaData.statistics), primitiveType), metaData.data_page_offset, metaData.dictionary_page_offset, metaData.num_values, @@ -129,18 +135,7 @@ public static ParquetMetadata createParquetMetadata(FileMetaData fileMetaData, P } } - Map keyValueMetaData = new HashMap<>(); - List keyValueList = fileMetaData.getKey_value_metadata(); - if (keyValueList != null) { - for (KeyValue keyValue : keyValueList) { - keyValueMetaData.put(keyValue.key, keyValue.value); - } - } - FileMetadata parquetFileMetadata = new FileMetadata( - messageType, - keyValueMetaData, - fileMetaData.getCreated_by()); - return new ParquetMetadata(parquetFileMetadata, blocks); + return blocks; } private static MessageType readParquetSchema(List schema) @@ -222,4 +217,27 @@ private static Set readEncodings(List schema = parquetMetadata.getSchema(); + validateParquet(!schema.isEmpty(), dataSourceId, "Schema is empty"); + + Iterator schemaIterator = schema.iterator(); + SchemaElement rootSchema = schemaIterator.next(); + Types.MessageTypeBuilder builder = Types.buildMessage(); + readTypeSchema(builder, schemaIterator, rootSchema.getNum_children()); + return builder.named(rootSchema.name); + } + + private static Map keyValueMetaData(FileMetaData parquetMetadata) + { + if (parquetMetadata.getKey_value_metadata() == null) { + return ImmutableMap.of(); + } + return parquetMetadata.getKey_value_metadata() + .stream() + .collect(toImmutableMap(KeyValue::getKey, KeyValue::getValue, (_, second) -> second)); + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java index 9c1ca8c42729..e543841f20eb 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java @@ -87,7 +87,7 @@ public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional< InputStream metadataStream = buffer.slice(buffer.length() - completeFooterSize, metadataLength).getInput(); FileMetaData fileMetaData = readFileMetaData(metadataStream); - ParquetMetadata parquetMetadata = ParquetMetadata.createParquetMetadata(fileMetaData, dataSource.getId()); + ParquetMetadata parquetMetadata = new ParquetMetadata(fileMetaData, dataSource.getId()); validateFileMetadata(dataSource.getId(), parquetMetadata.getFileMetaData(), parquetWriteValidation); return parquetMetadata; } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java index b33ea8e369a4..c247261db38d 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java @@ -183,7 +183,7 @@ public DataFileInfo getDataFileInfo() { Location path = rootTableLocation.appendPath(relativeFilePath); FileMetaData fileMetaData = fileWriter.getFileMetadata(); - ParquetMetadata parquetMetadata = ParquetMetadata.createParquetMetadata(fileMetaData, new ParquetDataSourceId(path.toString())); + ParquetMetadata parquetMetadata = new ParquetMetadata(fileMetaData, new ParquetDataSourceId(path.toString())); return new DataFileInfo( relativeFilePath, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java index f03ad35e82dc..06cccccebe3c 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java @@ -82,12 +82,12 @@ public FileMetrics getFileMetrics() { ParquetMetadata parquetMetadata; try { - parquetMetadata = ParquetMetadata.createParquetMetadata(parquetFileWriter.getFileMetadata(), new ParquetDataSourceId(location.toString())); + parquetMetadata = new ParquetMetadata(parquetFileWriter.getFileMetadata(), new ParquetDataSourceId(location.toString())); + return new FileMetrics(footerMetrics(parquetMetadata, Stream.empty(), metricsConfig), Optional.of(getSplitOffsets(parquetMetadata))); } catch (IOException e) { throw new TrinoException(GENERIC_INTERNAL_ERROR, format("Error creating metadata for Parquet file %s", location), e); } - return new FileMetrics(footerMetrics(parquetMetadata, Stream.empty(), metricsConfig), Optional.of(getSplitOffsets(parquetMetadata))); } @Override diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java index 0a676ca339ca..98f50940b419 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java @@ -15,6 +15,7 @@ package io.trino.plugin.iceberg.util; import com.google.common.collect.ImmutableList; +import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.parquet.metadata.ParquetMetadata; @@ -69,6 +70,7 @@ public final class ParquetUtil private ParquetUtil() {} public static Metrics footerMetrics(ParquetMetadata metadata, Stream> fieldMetrics, MetricsConfig metricsConfig) + throws ParquetCorruptionException { return footerMetrics(metadata, fieldMetrics, metricsConfig, null); } @@ -78,6 +80,7 @@ public static Metrics footerMetrics( Stream> fieldMetrics, MetricsConfig metricsConfig, NameMapping nameMapping) + throws ParquetCorruptionException { requireNonNull(fieldMetrics, "fieldMetrics should not be null"); @@ -156,9 +159,11 @@ public static Metrics footerMetrics( } public static List getSplitOffsets(ParquetMetadata metadata) + throws ParquetCorruptionException { - List splitOffsets = new ArrayList<>(metadata.getBlocks().size()); - for (BlockMetadata blockMetaData : metadata.getBlocks()) { + List blocks = metadata.getBlocks(); + List splitOffsets = new ArrayList<>(blocks.size()); + for (BlockMetadata blockMetaData : blocks) { splitOffsets.add(blockMetaData.getStartingPos()); } Collections.sort(splitOffsets); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java index 17d8695f61f5..35f514551533 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java @@ -139,18 +139,20 @@ private static boolean checkOrcFileSorting(Supplier dataSourceSup public static boolean checkParquetFileSorting(TrinoInputFile inputFile, String sortColumnName) { ParquetMetadata parquetMetadata; + List blocks; try { parquetMetadata = MetadataReader.readFooter( new TrinoParquetDataSource(inputFile, new ParquetReaderOptions(), new FileFormatDataSourceStats()), Optional.empty()); + blocks = parquetMetadata.getBlocks(); } catch (IOException e) { throw new UncheckedIOException(e); } Comparable previousMax = null; - verify(parquetMetadata.getBlocks().size() > 1, "Test must produce at least two row groups"); - for (BlockMetadata blockMetaData : parquetMetadata.getBlocks()) { + verify(blocks.size() > 1, "Test must produce at least two row groups"); + for (BlockMetadata blockMetaData : blocks) { ColumnChunkMetadata columnMetadata = blockMetaData.columns().stream() .filter(column -> getOnlyElement(column.getPath().iterator()).equalsIgnoreCase(sortColumnName)) .collect(onlyElement()); From 8bf43b5aaeb1818ba04f31cf0719a1678aedc17f Mon Sep 17 00:00:00 2001 From: Jinyang Li Date: Fri, 3 Jan 2025 09:45:41 +0530 Subject: [PATCH 148/158] Write row group fileOffset in parquet file footer --- .../parquet/metadata/ParquetMetadata.java | 7 +++ .../trino/parquet/writer/ParquetWriter.java | 8 ++-- .../parquet/writer/TestParquetWriter.java | 33 +++++++++++++ ...stDeltaLakeAlluxioCacheFileOperations.java | 48 +++++++++---------- ...LakeAlluxioCacheMutableTransactionLog.java | 16 +++---- 5 files changed, 77 insertions(+), 35 deletions(-) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java index 3eecc23a41fd..13de4ecc01fc 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java @@ -13,6 +13,7 @@ */ package io.trino.parquet.metadata; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; @@ -138,6 +139,12 @@ public List getBlocks() return blocks; } + @VisibleForTesting + public FileMetaData getParquetMetadata() + { + return parquetMetadata; + } + private static MessageType readParquetSchema(List schema) { Iterator schemaIterator = schema.iterator(); diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java b/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java index 1eed8ba73ef1..3bf3ab7010df 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java @@ -350,7 +350,7 @@ private void flush() columnMetaDataBuilder.add(columnMetaData); currentOffset += columnMetaData.getTotal_compressed_size(); } - updateRowGroups(columnMetaDataBuilder.build()); + updateRowGroups(columnMetaDataBuilder.build(), outputStream.longSize()); // flush pages for (BufferData bufferData : bufferDataList) { @@ -409,12 +409,14 @@ private void writeBloomFilters(List rowGroups, List columnMetaData) + private void updateRowGroups(List columnMetaData, long fileOffset) { long totalCompressedBytes = columnMetaData.stream().mapToLong(ColumnMetaData::getTotal_compressed_size).sum(); long totalBytes = columnMetaData.stream().mapToLong(ColumnMetaData::getTotal_uncompressed_size).sum(); ImmutableList columnChunks = columnMetaData.stream().map(ParquetWriter::toColumnChunk).collect(toImmutableList()); - fileFooter.addRowGroup(new RowGroup(columnChunks, totalBytes, rows).setTotal_compressed_size(totalCompressedBytes)); + fileFooter.addRowGroup(new RowGroup(columnChunks, totalBytes, rows) + .setTotal_compressed_size(totalCompressedBytes) + .setFile_offset(fileOffset)); } private static Slice serializeFooter(FileMetaData fileMetaData) diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java b/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java index a80cbbcd00d7..2d3b7f6c1ac6 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java @@ -46,6 +46,7 @@ import org.apache.parquet.format.CompressionCodec; import org.apache.parquet.format.PageHeader; import org.apache.parquet.format.PageType; +import org.apache.parquet.format.RowGroup; import org.apache.parquet.format.Util; import org.apache.parquet.schema.PrimitiveType; import org.assertj.core.data.Percentage; @@ -379,6 +380,38 @@ public void testDictionaryPageOffset() } } + @Test + void testRowGroupOffset() + throws IOException + { + // Write a file with 100 rows per row-group + List columnNames = ImmutableList.of("columnA", "columnB"); + List types = ImmutableList.of(INTEGER, BIGINT); + + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder() + .setMaxBlockSize(DataSize.ofBytes(1000)) + .build(), + types, + columnNames, + generateInputPages(types, 100, 10)), + new ParquetReaderOptions()); + + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + List blocks = parquetMetadata.getBlocks(); + assertThat(blocks.size()).isGreaterThan(1); + + List rowGroups = parquetMetadata.getParquetMetadata().getRow_groups(); + assertThat(rowGroups.size()).isEqualTo(blocks.size()); + for (int rowGroupIndex = 0; rowGroupIndex < rowGroups.size(); rowGroupIndex++) { + RowGroup rowGroup = rowGroups.get(rowGroupIndex); + assertThat(rowGroup.isSetFile_offset()).isTrue(); + BlockMetadata blockMetadata = blocks.get(rowGroupIndex); + assertThat(blockMetadata.getStartingPos()).isEqualTo(rowGroup.getFile_offset()); + } + } + @ParameterizedTest @MethodSource("testWriteBloomFiltersParams") public void testWriteBloomFilters(Type type, List data) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java index d7e92d3a063e..dc9f72d210e6 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheFileOperations.java @@ -95,12 +95,12 @@ public void testCacheFileOperations() .add(new CacheOperation("Alluxio.writeCache", "00000000000000000002.json", 0, 658)) .add(new CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) - .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) - .add(new CacheOperation("Input.readFully", "key=p1/", 0, 227)) - .add(new CacheOperation("Input.readFully", "key=p2/", 0, 227)) - .add(new CacheOperation("Alluxio.writeCache", "key=p1/", 0, 227)) - .add(new CacheOperation("Alluxio.writeCache", "key=p2/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 229)) + .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 229)) + .add(new CacheOperation("Input.readFully", "key=p1/", 0, 229)) + .add(new CacheOperation("Input.readFully", "key=p2/", 0, 229)) + .add(new CacheOperation("Alluxio.writeCache", "key=p1/", 0, 229)) + .add(new CacheOperation("Alluxio.writeCache", "key=p2/", 0, 229)) .build()); assertFileSystemAccesses( "SELECT * FROM test_cache_file_operations", @@ -113,8 +113,8 @@ public void testCacheFileOperations() .add(new CacheOperation("InputFile.length", "00000000000000000002.json")) .add(new CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) - .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 229)) + .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 229)) .build()); assertUpdate("INSERT INTO test_cache_file_operations VALUES ('p3', '3-xyz')", 1); assertUpdate("INSERT INTO test_cache_file_operations VALUES ('p4', '4-xyz')", 1); @@ -139,17 +139,17 @@ public void testCacheFileOperations() .add(new CacheOperation("InputFile.length", "00000000000000000005.json")) .add(new CacheOperation("InputFile.length", "00000000000000000006.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) - .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) - .add(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 227)) - .add(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 227)) - .add(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 227)) - .add(new CacheOperation("Input.readFully", "key=p3/", 0, 227)) - .add(new CacheOperation("Input.readFully", "key=p4/", 0, 227)) - .add(new CacheOperation("Input.readFully", "key=p5/", 0, 227)) - .add(new CacheOperation("Alluxio.writeCache", "key=p3/", 0, 227)) - .add(new CacheOperation("Alluxio.writeCache", "key=p4/", 0, 227)) - .add(new CacheOperation("Alluxio.writeCache", "key=p5/", 0, 227)) + .add(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 229)) + .add(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 229)) + .add(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 229)) + .add(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 229)) + .add(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 229)) + .add(new CacheOperation("Input.readFully", "key=p3/", 0, 229)) + .add(new CacheOperation("Input.readFully", "key=p4/", 0, 229)) + .add(new CacheOperation("Input.readFully", "key=p5/", 0, 229)) + .add(new CacheOperation("Alluxio.writeCache", "key=p3/", 0, 229)) + .add(new CacheOperation("Alluxio.writeCache", "key=p4/", 0, 229)) + .add(new CacheOperation("Alluxio.writeCache", "key=p5/", 0, 229)) .build()); assertFileSystemAccesses( "SELECT * FROM test_cache_file_operations", @@ -168,11 +168,11 @@ public void testCacheFileOperations() .add(new CacheOperation("InputFile.length", "00000000000000000005.json")) .add(new CacheOperation("InputFile.length", "00000000000000000006.json")) .add(new CacheOperation("InputFile.newStream", "_last_checkpoint")) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 227), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 227), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 227), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 227), 1) - .addCopies(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 227), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p1/", 0, 229), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p2/", 0, 229), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p3/", 0, 229), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p4/", 0, 229), 1) + .addCopies(new CacheOperation("Alluxio.readCached", "key=p5/", 0, 229), 1) .build()); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java index 04b3cbfba622..a69d2df8a19d 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeAlluxioCacheMutableTransactionLog.java @@ -79,12 +79,12 @@ public void testTableDataCachedWhileTransactionLogNotCached() .addCopies(new CacheFileSystemTraceUtils.CacheOperation("Input.readTail", "00000000000000000002.checkpoint.parquet"), 2) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p1/", 0, 227)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p2/", 0, 227)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p1/", 0, 227)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p2/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 229)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 229)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p1/", 0, 229)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Input.readFully", "key=p2/", 0, 229)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p1/", 0, 229)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.writeCache", "key=p2/", 0, 229)) .build()); assertFileSystemAccesses( "SELECT * FROM test_transaction_log_not_cached", @@ -93,8 +93,8 @@ public void testTableDataCachedWhileTransactionLogNotCached() .addCopies(new CacheFileSystemTraceUtils.CacheOperation("Input.readTail", "00000000000000000002.checkpoint.parquet"), 2) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.length", "00000000000000000003.json")) .add(new CacheFileSystemTraceUtils.CacheOperation("InputFile.newStream", "_last_checkpoint")) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 227)) - .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 227)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p1/", 0, 229)) + .add(new CacheFileSystemTraceUtils.CacheOperation("Alluxio.readCached", "key=p2/", 0, 229)) .build()); } From 2ef6dc1c9484b6726a44e15fdaeb36853e2bcd35 Mon Sep 17 00:00:00 2001 From: Jinyang Li Date: Mon, 9 Dec 2024 10:44:42 -0800 Subject: [PATCH 149/158] Parse only required row groups from parquet footer --- .../parquet/metadata/ParquetMetadata.java | 14 +++++++ .../parquet/predicate/PredicateUtils.java | 5 ++- .../io/trino/parquet/ParquetTestUtils.java | 2 +- .../parquet/reader/TestParquetReader.java | 39 +++++++++++++++++++ .../parquet/ParquetPageSourceFactory.java | 2 +- .../plugin/hudi/HudiPageSourceProvider.java | 2 +- .../iceberg/IcebergPageSourceProvider.java | 2 +- 7 files changed, 60 insertions(+), 6 deletions(-) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java index 13de4ecc01fc..56091d962a08 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java @@ -90,6 +90,12 @@ public String toString() public List getBlocks() throws ParquetCorruptionException + { + return getBlocks(0, Long.MAX_VALUE); + } + + public List getBlocks(long splitStart, long splitLength) + throws ParquetCorruptionException { List schema = parquetMetadata.getSchema(); validateParquet(!schema.isEmpty(), dataSourceId, "Schema is empty"); @@ -99,6 +105,14 @@ public List getBlocks() List rowGroups = parquetMetadata.getRow_groups(); if (rowGroups != null) { for (RowGroup rowGroup : rowGroups) { + if (rowGroup.isSetFile_offset()) { + long rowGroupStart = rowGroup.getFile_offset(); + boolean splitContainsRowGroup = splitStart <= rowGroupStart && rowGroupStart < splitStart + splitLength; + if (!splitContainsRowGroup) { + continue; + } + } + List columns = rowGroup.getColumns(); validateParquet(!columns.isEmpty(), dataSourceId, "No columns in row group: %s", rowGroup); String filePath = columns.get(0).getFile_path(); diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java b/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java index 6901bb23a4e6..978e915a7a56 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java @@ -27,6 +27,7 @@ import io.trino.parquet.ParquetReaderOptions; import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ColumnChunkMetadata; +import io.trino.parquet.metadata.ParquetMetadata; import io.trino.parquet.metadata.PrunedBlockMetadata; import io.trino.parquet.reader.RowGroupInfo; import io.trino.spi.predicate.TupleDomain; @@ -183,7 +184,7 @@ public static List getFilteredRowGroups( long splitStart, long splitLength, ParquetDataSource dataSource, - List blocksMetaData, + ParquetMetadata parquetMetadata, List> parquetTupleDomains, List parquetPredicates, Map, ColumnDescriptor> descriptorsByPath, @@ -194,7 +195,7 @@ public static List getFilteredRowGroups( { long fileRowCount = 0; ImmutableList.Builder rowGroupInfoBuilder = ImmutableList.builder(); - for (BlockMetadata block : blocksMetaData) { + for (BlockMetadata block : parquetMetadata.getBlocks(splitStart, splitLength)) { long blockStart = block.getStartingPos(); boolean splitContainsBlock = splitStart <= blockStart && blockStart < splitStart + splitLength; if (splitContainsBlock) { diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java b/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java index aeda237c642f..0528adcd1166 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java @@ -155,7 +155,7 @@ public static ParquetReader createParquetReader( 0, input.getEstimatedSize(), input, - parquetMetadata.getBlocks(), + parquetMetadata, ImmutableList.of(parquetTupleDomain), ImmutableList.of(parquetPredicate), descriptorsByPath, diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java index 00ecd8388857..db8b4225770a 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java @@ -20,6 +20,7 @@ import io.trino.memory.context.AggregatedMemoryContext; import io.trino.parquet.ParquetDataSource; import io.trino.parquet.ParquetReaderOptions; +import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ParquetMetadata; import io.trino.parquet.writer.ParquetWriterOptions; import io.trino.spi.Page; @@ -186,6 +187,44 @@ public void testBackwardsCompatibleRepeatedPrimitiveFieldDefinedAsPrimitive() .isInstanceOf(TrinoException.class); } + @Test + void testReadMetadataWithSplitOffset() + throws IOException + { + // Write a file with 100 rows per row-group + List columnNames = ImmutableList.of("columna", "columnb"); + List types = ImmutableList.of(INTEGER, BIGINT); + + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder() + .setMaxBlockSize(DataSize.ofBytes(1000)) + .build(), + types, + columnNames, + generateInputPages(types, 100, 5)), + new ParquetReaderOptions()); + + // Read both columns, 1 row group + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + List columnBlocks = parquetMetadata.getBlocks(0, 800); + assertThat(columnBlocks.size()).isEqualTo(1); + assertThat(columnBlocks.getFirst().columns().size()).isEqualTo(2); + assertThat(columnBlocks.getFirst().rowCount()).isEqualTo(100); + + // Read both columns, half row groups + parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + columnBlocks = parquetMetadata.getBlocks(0, 2500); + assertThat(columnBlocks.stream().allMatch(block -> block.columns().size() == 2)).isTrue(); + assertThat(columnBlocks.stream().mapToLong(BlockMetadata::rowCount).sum()).isEqualTo(300); + + // Read both columns, all row groups + parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + columnBlocks = parquetMetadata.getBlocks(); + assertThat(columnBlocks.stream().allMatch(block -> block.columns().size() == 2)).isTrue(); + assertThat(columnBlocks.stream().mapToLong(BlockMetadata::rowCount).sum()).isEqualTo(500); + } + private void testReadingOldParquetFiles(File file, List columnNames, Type columnType, List expectedValues) throws IOException { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java index dc072f0967a0..f364a9e18739 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java @@ -253,7 +253,7 @@ public static ReaderPageSource createPageSource( start, length, dataSource, - parquetMetadata.getBlocks(), + parquetMetadata, parquetTupleDomains, parquetPredicates, descriptorsByPath, diff --git a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java index 3bb7b1c42a79..83d62c9ab160 100644 --- a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java +++ b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java @@ -218,7 +218,7 @@ private static ConnectorPageSource createPageSource( start, length, dataSource, - parquetMetadata.getBlocks(), + parquetMetadata, ImmutableList.of(parquetTupleDomain), ImmutableList.of(parquetPredicate), descriptorsByPath, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java index 4b5abd936d98..1911d5d4426d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java @@ -891,7 +891,7 @@ private static ReaderPageSourceWithRowPositions createParquetPageSource( start, length, dataSource, - parquetMetadata.getBlocks(), + parquetMetadata, ImmutableList.of(parquetTupleDomain), ImmutableList.of(parquetPredicate), descriptorsByPath, From 27519398c41e67532a11c15a840cfcd4e086c375 Mon Sep 17 00:00:00 2001 From: Mayank Vadariya Date: Thu, 2 Jan 2025 11:07:41 -0500 Subject: [PATCH 150/158] Extend AbstractTestQueryFramework in TestRedshiftUnload --- .../java/io/trino/plugin/redshift/TestRedshiftUnload.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java index 433830c495d0..fd86bdcef45c 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java @@ -17,7 +17,7 @@ import io.trino.Session; import io.trino.operator.OperatorInfo; import io.trino.operator.SplitOperatorInfo; -import io.trino.testing.AbstractTestQueries; +import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; import io.trino.testing.sql.SqlExecutor; import io.trino.testing.sql.TestTable; @@ -40,6 +40,7 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.TestingProperties.requiredNonEmptySystemProperty; import static io.trino.testing.TestingSession.testSessionBuilder; +import static io.trino.tpch.TpchTable.NATION; import static java.util.Locale.ENGLISH; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -48,7 +49,7 @@ @TestInstance(PER_CLASS) @Execution(CONCURRENT) final class TestRedshiftUnload - extends AbstractTestQueries + extends AbstractTestQueryFramework { private static final String S3_UNLOAD_ROOT = requiredNonEmptySystemProperty("test.redshift.s3.unload.root"); private static final String AWS_REGION = requiredNonEmptySystemProperty("test.redshift.aws.region"); @@ -67,7 +68,7 @@ protected QueryRunner createQueryRunner() "s3.region", AWS_REGION, "s3.aws-access-key", AWS_ACCESS_KEY, "s3.aws-secret-key", AWS_SECRET_KEY)) - .setInitialTables(REQUIRED_TPCH_TABLES) + .setInitialTables(List.of(NATION)) .build(); } From 9e3aebb213d5651c0e9ff7d08f46cb323755d80d Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 6 Jan 2025 14:38:00 +0900 Subject: [PATCH 151/158] Use correct table operations provider for Thrift metastore in Delta --- .../metastore/thrift/DeltaLakeThriftMetastoreModule.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java index 8946e5ec9c58..0ec996a93eae 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java @@ -20,7 +20,6 @@ import io.trino.plugin.deltalake.AllowDeltaLakeManagedTableRename; import io.trino.plugin.deltalake.MaxTableParameterLength; import io.trino.plugin.deltalake.metastore.DeltaLakeTableOperationsProvider; -import io.trino.plugin.deltalake.metastore.file.DeltaLakeFileMetastoreTableOperationsProvider; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreModule; public class DeltaLakeThriftMetastoreModule @@ -30,7 +29,7 @@ public class DeltaLakeThriftMetastoreModule protected void setup(Binder binder) { install(new ThriftMetastoreModule()); - binder.bind(DeltaLakeTableOperationsProvider.class).to(DeltaLakeFileMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); + binder.bind(DeltaLakeTableOperationsProvider.class).to(DeltaLakeThriftMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(Key.get(boolean.class, AllowDeltaLakeManagedTableRename.class)).toInstance(false); // Limit per Hive metastore code (https://github.com/apache/hive/tree/7f6367e0c6e21b11ef62da1ea6681a54d547de07/standalone-metastore/metastore-server/src/main/sql as of this writing) // - MySQL: mediumtext (16777215) From 34a6a7320079fe55c856933115960048beb25579 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 3 Jan 2025 10:01:31 +0100 Subject: [PATCH 152/158] Update zstd-jni to 1.5.6-9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5c79c3ca8c63..359906395ad7 100644 --- a/pom.xml +++ b/pom.xml @@ -505,7 +505,7 @@ com.github.luben zstd-jni - 1.5.6-8 + 1.5.6-9 From a3212f421b1b7dcc1d5d7d4b8839dfd45671caa9 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 3 Jan 2025 10:01:49 +0100 Subject: [PATCH 153/158] Update nimbus-jose-jwt to 10.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 359906395ad7..fce39010a448 100644 --- a/pom.xml +++ b/pom.xml @@ -604,7 +604,7 @@ com.nimbusds nimbus-jose-jwt - 9.48 + 10.0 From 0f703ca7bd3d3df255517eaa7acb8e221fd68814 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 3 Jan 2025 10:02:27 +0100 Subject: [PATCH 154/158] Update AWS SDK v2 to 2.29.44 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fce39010a448..3d59e0397fbd 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ software.amazon.awssdk bom - 2.29.43 + 2.29.44 pom import From eb686065711f636e22d42f9bd8022bc26ee6759f Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 3 Jan 2025 10:03:20 +0100 Subject: [PATCH 155/158] Update nessie to 0.101.3 --- plugin/trino-iceberg/pom.xml | 2 +- .../io/trino/plugin/iceberg/containers/NessieContainer.java | 2 +- .../env/environment/EnvSinglenodeSparkIcebergNessie.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index f07debe57d23..9fbd26c7f844 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -15,7 +15,7 @@ - 0.101.1 + 0.101.3 diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java index b327be7c862d..ff5484d3fa04 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java @@ -28,7 +28,7 @@ public class NessieContainer { private static final Logger log = Logger.get(NessieContainer.class); - public static final String DEFAULT_IMAGE = "ghcr.io/projectnessie/nessie:0.101.1"; + public static final String DEFAULT_IMAGE = "ghcr.io/projectnessie/nessie:0.101.3"; public static final String DEFAULT_HOST_NAME = "nessie"; public static final String VERSION_STORE_TYPE = "IN_MEMORY"; diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java index 3b929d278ad7..9c53e019d8f3 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java @@ -43,7 +43,7 @@ public class EnvSinglenodeSparkIcebergNessie private static final int SPARK_THRIFT_PORT = 10213; private static final int NESSIE_PORT = 19120; - private static final String NESSIE_VERSION = "0.101.1"; + private static final String NESSIE_VERSION = "0.101.3"; private static final String SPARK = "spark"; private final DockerFiles dockerFiles; From 9817f41e17ecfe4d148a7de479d7c52a06c9aae2 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 3 Jan 2025 21:32:06 +0100 Subject: [PATCH 156/158] Allow configuring gcs service endpoint --- .../trino/filesystem/gcs/GcsFileSystemConfig.java | 15 +++++++++++++++ .../trino/filesystem/gcs/GcsStorageFactory.java | 5 +++++ .../filesystem/gcs/TestGcsFileSystemConfig.java | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java index 18cb104e348c..f78d1a51e658 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java @@ -26,6 +26,7 @@ import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; +import java.util.Optional; import java.util.concurrent.TimeUnit; import static com.google.common.base.Preconditions.checkState; @@ -39,6 +40,7 @@ public class GcsFileSystemConfig private int batchSize = 100; private String projectId; + private Optional endpoint = Optional.empty(); private boolean useGcsAccessToken; private String jsonKey; @@ -120,6 +122,19 @@ public GcsFileSystemConfig setProjectId(String projectId) return this; } + public Optional getEndpoint() + { + return endpoint; + } + + @ConfigDescription("Endpoint to use for GCS requests") + @Config("gcs.endpoint") + public GcsFileSystemConfig setEndpoint(Optional endpoint) + { + this.endpoint = endpoint; + return this; + } + public boolean isUseGcsAccessToken() { return useGcsAccessToken; diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java index f57dcb9f32dd..f8ea12c4520a 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java @@ -41,6 +41,7 @@ public class GcsStorageFactory public static final String GCS_OAUTH_KEY = "gcs.oauth"; public static final List DEFAULT_SCOPES = ImmutableList.of("https://www.googleapis.com/auth/cloud-platform"); private final String projectId; + private final Optional endpoint; private final boolean useGcsAccessToken; private final Optional jsonGoogleCredential; private final int maxRetries; @@ -56,6 +57,7 @@ public GcsStorageFactory(GcsFileSystemConfig config) { config.validate(); projectId = config.getProjectId(); + endpoint = config.getEndpoint(); useGcsAccessToken = config.isUseGcsAccessToken(); String jsonKey = config.getJsonKey(); String jsonKeyFilePath = config.getJsonKeyFilePath(); @@ -105,6 +107,9 @@ public Storage create(ConnectorIdentity identity) if (projectId != null) { storageOptionsBuilder.setProjectId(projectId); } + + endpoint.ifPresent(storageOptionsBuilder::setHost); + // Note: without uniform strategy we cannot retry idempotent operations. // The trino-filesystem api does not violate the conditions for idempotency, see https://cloud.google.com/storage/docs/retry-strategy#java for details. return storageOptionsBuilder diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java index 22ab1f7f6212..fd9f5cddfd70 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java @@ -23,6 +23,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.Optional; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; @@ -44,6 +45,7 @@ void testDefaults() .setPageSize(100) .setBatchSize(100) .setProjectId(null) + .setEndpoint(Optional.empty()) .setUseGcsAccessToken(false) .setJsonKey(null) .setJsonKeyFilePath(null) @@ -67,6 +69,7 @@ void testExplicitPropertyMappings() .put("gcs.page-size", "10") .put("gcs.batch-size", "11") .put("gcs.project-id", "project") + .put("gcs.endpoint", "http://custom.dns.org:8000") .put("gcs.use-access-token", "true") .put("gcs.json-key", "{}") .put("gcs.json-key-file-path", jsonKeyFile.toString()) @@ -84,6 +87,7 @@ void testExplicitPropertyMappings() .setPageSize(10) .setBatchSize(11) .setProjectId("project") + .setEndpoint(Optional.of("http://custom.dns.org:8000")) .setUseGcsAccessToken(true) .setJsonKey("{}") .setJsonKeyFilePath(jsonKeyFile.toString()) From bdd45038abbb0667068243bdbf9cbcbea82de807 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Tue, 7 Jan 2025 08:42:57 +0900 Subject: [PATCH 157/158] Update delta-kernel to 3.3.0 --- plugin/trino-delta-lake/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-delta-lake/pom.xml b/plugin/trino-delta-lake/pom.xml index fd35145fc80a..efc785b7db2e 100644 --- a/plugin/trino-delta-lake/pom.xml +++ b/plugin/trino-delta-lake/pom.xml @@ -88,7 +88,7 @@ io.delta delta-kernel-api - 3.2.1 + 3.3.0 From d2436a10a3ef2ca3ec9bdbd8b45bba5b4483fc35 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Thu, 26 Dec 2024 23:00:16 +0900 Subject: [PATCH 158/158] Allow configuring orc_bloom_filter_columns table property in Iceberg --- .../trino/plugin/iceberg/IcebergMetadata.java | 18 +++++ .../io/trino/plugin/iceberg/IcebergUtil.java | 6 +- .../iceberg/BaseIcebergConnectorTest.java | 4 -- .../TestIcebergOrcWithBloomFilters.java | 66 ++++++++++++++++--- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 2149fb71ed23..2b01414e1c10 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -282,6 +282,7 @@ import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.OBJECT_STORE_LAYOUT_ENABLED_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.ORC_BLOOM_FILTER_COLUMNS_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.SORTED_BY_PROPERTY; @@ -307,6 +308,7 @@ import static io.trino.plugin.iceberg.IcebergUtil.getTopLevelColumns; import static io.trino.plugin.iceberg.IcebergUtil.newCreateTableTransaction; import static io.trino.plugin.iceberg.IcebergUtil.schemaFromMetadata; +import static io.trino.plugin.iceberg.IcebergUtil.validateOrcBloomFilterColumns; import static io.trino.plugin.iceberg.IcebergUtil.validateParquetBloomFilterColumns; import static io.trino.plugin.iceberg.IcebergUtil.verifyExtraProperties; import static io.trino.plugin.iceberg.PartitionFields.parsePartitionFields; @@ -374,6 +376,7 @@ import static org.apache.iceberg.TableProperties.DELETE_ISOLATION_LEVEL_DEFAULT; import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.OBJECT_STORE_ENABLED; +import static org.apache.iceberg.TableProperties.ORC_BLOOM_FILTER_COLUMNS; import static org.apache.iceberg.TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX; import static org.apache.iceberg.TableProperties.WRITE_DATA_LOCATION; import static org.apache.iceberg.TableProperties.WRITE_LOCATION_PROVIDER_IMPL; @@ -397,6 +400,7 @@ public class IcebergMetadata .add(FORMAT_VERSION_PROPERTY) .add(OBJECT_STORE_LAYOUT_ENABLED_PROPERTY) .add(DATA_LOCATION_PROPERTY) + .add(ORC_BLOOM_FILTER_COLUMNS_PROPERTY) .add(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY) .add(PARTITIONING_PROPERTY) .add(SORTED_BY_PROPERTY) @@ -2295,6 +2299,20 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta parquetBloomFilterColumns.forEach(column -> updateProperties.set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column, "true")); } + if (properties.containsKey(ORC_BLOOM_FILTER_COLUMNS_PROPERTY)) { + checkFormatForProperty(getFileFormat(icebergTable).toIceberg(), FileFormat.ORC, ORC_BLOOM_FILTER_COLUMNS_PROPERTY); + //noinspection unchecked + List orcBloomFilterColumns = (List) properties.get(ORC_BLOOM_FILTER_COLUMNS_PROPERTY) + .orElseThrow(() -> new IllegalArgumentException("The orc_bloom_filter_columns property cannot be empty")); + if (orcBloomFilterColumns.isEmpty()) { + updateProperties.remove(ORC_BLOOM_FILTER_COLUMNS); + } + else { + validateOrcBloomFilterColumns(getColumnMetadatas(SchemaParser.fromJson(table.getTableSchemaJson()), typeManager), orcBloomFilterColumns); + updateProperties.set(ORC_BLOOM_FILTER_COLUMNS, Joiner.on(",").join(orcBloomFilterColumns)); + } + } + if (properties.containsKey(FILE_FORMAT_PROPERTY)) { IcebergFileFormat fileFormat = (IcebergFileFormat) properties.get(FILE_FORMAT_PROPERTY) .orElseThrow(() -> new IllegalArgumentException("The format property cannot be empty")); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java index ee3a2160eb63..65922abca244 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java @@ -867,7 +867,7 @@ public static Map createTableProperties(ConnectorTableMetadata t List orcBloomFilterColumns = IcebergTableProperties.getOrcBloomFilterColumns(tableMetadata.getProperties()); if (!orcBloomFilterColumns.isEmpty()) { checkFormatForProperty(fileFormat.toIceberg(), FileFormat.ORC, ORC_BLOOM_FILTER_COLUMNS_PROPERTY); - validateOrcBloomFilterColumns(tableMetadata, orcBloomFilterColumns); + validateOrcBloomFilterColumns(tableMetadata.getColumns(), orcBloomFilterColumns); propertiesBuilder.put(ORC_BLOOM_FILTER_COLUMNS, Joiner.on(",").join(orcBloomFilterColumns)); propertiesBuilder.put(ORC_BLOOM_FILTER_FPP, String.valueOf(IcebergTableProperties.getOrcBloomFilterFpp(tableMetadata.getProperties()))); } @@ -993,9 +993,9 @@ public static void checkFormatForProperty(FileFormat actualStorageFormat, FileFo } } - private static void validateOrcBloomFilterColumns(ConnectorTableMetadata tableMetadata, List orcBloomFilterColumns) + public static void validateOrcBloomFilterColumns(List columns, List orcBloomFilterColumns) { - Set allColumns = tableMetadata.getColumns().stream() + Set allColumns = columns.stream() .map(ColumnMetadata::getName) .collect(toImmutableSet()); if (!allColumns.containsAll(orcBloomFilterColumns)) { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java index 8f56f7fede55..4902182aeb87 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java @@ -7475,12 +7475,8 @@ public void testAlterTableWithUnsupportedProperties() assertUpdate("CREATE TABLE " + tableName + " (a bigint)"); - assertQueryFails("ALTER TABLE " + tableName + " SET PROPERTIES orc_bloom_filter_columns = ARRAY['a']", - "The following properties cannot be updated: orc_bloom_filter_columns"); assertQueryFails("ALTER TABLE " + tableName + " SET PROPERTIES location = '/var/data/table/', orc_bloom_filter_fpp = 0.5", "The following properties cannot be updated: location, orc_bloom_filter_fpp"); - assertQueryFails("ALTER TABLE " + tableName + " SET PROPERTIES format = 'ORC', orc_bloom_filter_columns = ARRAY['a']", - "The following properties cannot be updated: orc_bloom_filter_columns"); assertUpdate("DROP TABLE " + tableName); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcWithBloomFilters.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcWithBloomFilters.java index 5e396b78ad68..91f223faf028 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcWithBloomFilters.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcWithBloomFilters.java @@ -14,12 +14,13 @@ package io.trino.plugin.iceberg; import io.trino.testing.BaseOrcWithBloomFiltersTest; -import io.trino.testing.MaterializedResult; import io.trino.testing.QueryRunner; +import io.trino.testing.sql.TestTable; import org.junit.jupiter.api.Test; -import static io.trino.testing.MaterializedResult.resultBuilder; -import static io.trino.testing.QueryAssertions.assertContains; +import java.util.Map; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; @@ -32,6 +33,7 @@ protected QueryRunner createQueryRunner() throws Exception { return IcebergQueryRunner.builder() + .addIcebergProperty("iceberg.file-format", "ORC") .addIcebergProperty("hive.orc.bloom-filters.enabled", "true") .addIcebergProperty("hive.orc.default-bloom-filter-fpp", "0.001") .build(); @@ -55,14 +57,60 @@ public void testBloomFilterPropertiesArePersistedDuringCreate() "orc_bloom_filter_columns = array['a','b']," + "orc_bloom_filter_fpp = 0.1)"); - MaterializedResult actualProperties = computeActual("SELECT * FROM \"" + tableName + "$properties\""); - assertThat(actualProperties).isNotNull(); - MaterializedResult expectedProperties = resultBuilder(getSession()) - .row("write.orc.bloom.filter.columns", "a,b") - .row("write.orc.bloom.filter.fpp", "0.1").build(); - assertContains(actualProperties, expectedProperties); + assertThat(getTableProperties(tableName)) + .containsEntry("write.orc.bloom.filter.columns", "a,b") + .containsEntry("write.orc.bloom.filter.fpp", "0.1"); assertThat((String) computeScalar("SHOW CREATE TABLE " + tableName)) .contains("orc_bloom_filter_columns", "orc_bloom_filter_fpp"); } + + @Test + void testBloomFilterPropertiesArePersistedDuringSetProperties() + { + String tableName = "test_metadata_write_properties_" + randomNameSuffix(); + assertQuerySucceeds("CREATE TABLE " + tableName + "(A bigint, b bigint, c bigint)"); + + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES orc_bloom_filter_columns = ARRAY['a','B']"); + assertThat(getTableProperties(tableName)) + .containsEntry("write.orc.bloom.filter.columns", "a,b"); + + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES orc_bloom_filter_columns = ARRAY['a']"); + assertThat(getTableProperties(tableName)) + .containsEntry("write.orc.bloom.filter.columns", "a"); + + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES orc_bloom_filter_columns = ARRAY[]"); + assertThat(getTableProperties(tableName)) + .doesNotContainKey("write.orc.bloom.filter.columns"); + } + + @Test + void testInvalidBloomFilterProperties() + { + String tableName = "test_invalid_bloom_filter_properties_" + randomNameSuffix(); + assertQueryFails( + "CREATE TABLE " + tableName + "(x int) WITH (orc_bloom_filter_columns = ARRAY['missing_column'])", + "\\QOrc bloom filter columns [missing_column] not present in schema"); + + assertQuerySucceeds("CREATE TABLE " + tableName + "(x array(integer))"); + assertQueryFails( + "ALTER TABLE " + tableName + " SET PROPERTIES orc_bloom_filter_columns = ARRAY['missing_column']", + "\\QOrc bloom filter columns [missing_column] not present in schema"); + } + + @Test + void testInvalidOrcBloomFilterPropertiesOnParquet() + { + try (TestTable table = new TestTable(getQueryRunner()::execute, "test_orc_bloom_filter", "(x int) WITH (format = 'PARQUET')")) { + assertQueryFails( + "ALTER TABLE " + table.getName() + " SET PROPERTIES orc_bloom_filter_columns = ARRAY['x']", + "Cannot specify orc_bloom_filter_columns table property for storage format: PARQUET"); + } + } + + private Map getTableProperties(String tableName) + { + return computeActual("SELECT key, value FROM \"" + tableName + "$properties\"").getMaterializedRows().stream() + .collect(toImmutableMap(row -> (String) row.getField(0), row -> (String) row.getField(1))); + } }