From 759603a6e16c5eec2eb6854ddc0ffdbd1ec19dfe Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Wed, 29 Jan 2025 07:35:24 -0800 Subject: [PATCH] look for new data in current max partition also (#784) --- models/bronze/stage/bronze__stage_block_txs_2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/bronze/stage/bronze__stage_block_txs_2.sql b/models/bronze/stage/bronze__stage_block_txs_2.sql index ae72749c..60466b13 100644 --- a/models/bronze/stage/bronze__stage_block_txs_2.sql +++ b/models/bronze/stage/bronze__stage_block_txs_2.sql @@ -13,7 +13,7 @@ FROM {{ ref('bronze__streamline_block_txs_2') }} WHERE {% if is_incremental() %} - _partition_id > (SELECT max(_partition_id) FROM {{ this }}) + _partition_id >= (SELECT max(_partition_id) FROM {{ this }}) AND _inserted_timestamp >= (SELECT max(_inserted_timestamp) FROM {{ this }}) {% else %} _partition_id = (SELECT max(_partition_id) FROM {{ source('solana_streamline', 'complete_block_txs_2') }}) -- Reference this once to get a starting point