From 1798601e7ec04ea9211a53fc7b4bdbb9f235a7de Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:05:26 -0800 Subject: [PATCH 1/3] update libpq-dev dependency to 13.18-0+deb11u1 (#1420) * use dynamic schema in test_grant_access_to.py * use dynamic schema in test_grant_access_to.py * revert setup * update libpq-dev dependency to 13.18-0+deb11u1 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bda507dc5..8f371d6b4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update \ build-essential=12.9 \ ca-certificates=20210119 \ git=1:2.30.2-1+deb11u2 \ - libpq-dev=13.14-0+deb11u1 \ + libpq-dev=13.18-0+deb11u1 \ make=4.3-4.1 \ openssh-client=1:8.4p1-5+deb11u3 \ software-properties-common=0.96.20.2-2.1 \ From 26c19e911f6d493397578c05dd218496de03a334 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:47:35 -0500 Subject: [PATCH 2/3] Remove custom retry in get_table call (#1423) * remove custom retry in get_table call * changelog --- .changes/unreleased/Fixes-20241205-133606.yaml | 6 ++++++ dbt/adapters/bigquery/connections.py | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241205-133606.yaml diff --git a/.changes/unreleased/Fixes-20241205-133606.yaml b/.changes/unreleased/Fixes-20241205-133606.yaml new file mode 100644 index 000000000..b88a0981c --- /dev/null +++ b/.changes/unreleased/Fixes-20241205-133606.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix issue where rate limit errors on table service calls are not retried +time: 2024-12-05T13:36:06.436005-05:00 +custom: + Author: mikealfare + Issue: "1423" diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 61fa87d40..bb062f330 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -519,10 +519,7 @@ def get_bq_table(self, database, schema, identifier) -> Table: # backwards compatibility: fill in with defaults if not specified database = database or conn.credentials.database schema = schema or conn.credentials.schema - return client.get_table( - table=self.table_ref(database, schema, identifier), - retry=self._retry.create_reopen_with_deadline(conn), - ) + return client.get_table(self.table_ref(database, schema, identifier)) def drop_dataset(self, database, schema) -> None: conn = self.get_thread_connection() From 4d255b2f854d21d5d8871bdaa8d7ab47e7e863a3 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 5 Dec 2024 16:26:52 -0500 Subject: [PATCH 3/3] Fix: Cast to timestamp prior to event time comparison (#1422) --- .../unreleased/Fixes-20241204-105846.yaml | 7 +++++ dbt/adapters/bigquery/relation.py | 25 +++++++++++++++- .../incremental_strategy_fixtures.py | 20 ++++++++++++- .../test_incremental_microbatch.py | 30 +++++++++++++++++-- 4 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241204-105846.yaml diff --git a/.changes/unreleased/Fixes-20241204-105846.yaml b/.changes/unreleased/Fixes-20241204-105846.yaml new file mode 100644 index 000000000..2693e4513 --- /dev/null +++ b/.changes/unreleased/Fixes-20241204-105846.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Cast `event_time` to a timestamp prior to comparing against microbatch start/end + time +time: 2024-12-04T10:58:46.573608-05:00 +custom: + Author: michelleark + Issue: "1422" diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 4edc8d7ac..037761918 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -4,7 +4,12 @@ from dbt_common.exceptions import CompilationError from dbt_common.utils.dict import filter_null_values -from dbt.adapters.base.relation import BaseRelation, ComponentName, InformationSchema +from dbt.adapters.base.relation import ( + BaseRelation, + ComponentName, + InformationSchema, + EventTimeFilter, +) from dbt.adapters.contracts.relation import RelationConfig, RelationType from dbt.adapters.relation_configs import RelationConfigChangeAction @@ -116,6 +121,24 @@ def materialized_view_config_changeset( def information_schema(self, identifier: Optional[str] = None) -> "BigQueryInformationSchema": return BigQueryInformationSchema.from_relation(self, identifier) + def _render_event_time_filtered(self, event_time_filter: EventTimeFilter) -> str: + """ + Returns "" if start and end are both None + """ + filter = "" + if event_time_filter.start and event_time_filter.end: + filter = f"cast({event_time_filter.field_name} as timestamp) >= '{event_time_filter.start}' and cast({event_time_filter.field_name} as timestamp) < '{event_time_filter.end}'" + elif event_time_filter.start: + filter = ( + f"cast({event_time_filter.field_name} as timestamp) >= '{event_time_filter.start}'" + ) + elif event_time_filter.end: + filter = ( + f"cast({event_time_filter.field_name} as timestamp) < '{event_time_filter.end}'" + ) + + return filter + @dataclass(frozen=True, eq=False, repr=False) class BigQueryInformationSchema(InformationSchema): diff --git a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py index 02efbb6c2..365aba8c8 100644 --- a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py +++ b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py @@ -570,7 +570,7 @@ begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0) ) }} -select * from {{ ref('input_model') }} +select id, cast(event_time as timestamp) as event_time from {{ ref('input_model') }} """ microbatch_input_sql = """ @@ -582,6 +582,24 @@ select 3 as id, TIMESTAMP '2020-01-03 00:00:00-0' as event_time """ +microbatch_input_event_time_date_sql = """ +{{ config(materialized='table', event_time='event_time') }} +select 1 as id, DATE '2020-01-01' as event_time +union all +select 2 as id, DATE '2020-01-02' as event_time +union all +select 3 as id, DATE '2020-01-03' as event_time +""" + +microbatch_input_event_time_datetime_sql = """ +{{ config(materialized='table', event_time='event_time') }} +select 1 as id, DATETIME '2020-01-01' as event_time +union all +select 2 as id, DATETIME '2020-01-02' as event_time +union all +select 3 as id, DATETIME '2020-01-03' as event_time +""" + microbatch_model_no_partition_by_sql = """ {{ config( materialized='incremental', diff --git a/tests/functional/adapter/incremental/test_incremental_microbatch.py b/tests/functional/adapter/incremental/test_incremental_microbatch.py index d1bbbcea3..d0f8b62b7 100644 --- a/tests/functional/adapter/incremental/test_incremental_microbatch.py +++ b/tests/functional/adapter/incremental/test_incremental_microbatch.py @@ -13,6 +13,8 @@ microbatch_input_sql, microbatch_model_no_partition_by_sql, microbatch_model_invalid_partition_by_sql, + microbatch_input_event_time_date_sql, + microbatch_input_event_time_datetime_sql, ) @@ -22,6 +24,32 @@ def microbatch_model_sql(self) -> str: return microbatch_model_no_unique_id_sql +class TestBigQueryMicrobatchInputWithDate(TestBigQueryMicrobatch): + @pytest.fixture(scope="class") + def input_model_sql(self) -> str: + return microbatch_input_event_time_date_sql + + @pytest.fixture(scope="class") + def insert_two_rows_sql(self, project) -> str: + test_schema_relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + return f"insert into {test_schema_relation}.input_model (id, event_time) values (4, DATE '2020-01-04'), (5, DATE '2020-01-05')" + + +class TestBigQueryMicrobatchInputWithDatetime(TestBigQueryMicrobatch): + @pytest.fixture(scope="class") + def input_model_sql(self) -> str: + return microbatch_input_event_time_datetime_sql + + @pytest.fixture(scope="class") + def insert_two_rows_sql(self, project) -> str: + test_schema_relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + return f"insert into {test_schema_relation}.input_model (id, event_time) values (4, DATETIME '2020-01-04'), (5, DATETIME '2020-01-05')" + + class TestBigQueryMicrobatchMissingPartitionBy: @pytest.fixture(scope="class") def models(self) -> str: @@ -30,7 +58,6 @@ def models(self) -> str: "input_model.sql": microbatch_input_sql, } - @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"}) def test_execution_failure_no_partition_by(self, project): with patch_microbatch_end_time("2020-01-03 13:57:00"): _, stdout = run_dbt_and_capture(["run"], expect_pass=False) @@ -45,7 +72,6 @@ def models(self) -> str: "input_model.sql": microbatch_input_sql, } - @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"}) def test_execution_failure_no_partition_by(self, project): with patch_microbatch_end_time("2020-01-03 13:57:00"): _, stdout = run_dbt_and_capture(["run"], expect_pass=False)