Give wiggle room for selecting SLA policy names if timestamps are off…

… by milliseconds (#174) * stage for PR * docs and tests * remove redundant comment * release review
fivetran · Oct 24, 2024 · 7fa81ff · 7fa81ff
1 parent a9be770
commit 7fa81ff
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,12 @@
+# dbt_zendesk v0.18.1
+[PR #174](https://github.com/fivetran/dbt_zendesk/pull/174) includes the following changes:
+
+## Bug Fix
+- Addressed an issue in which some records in `zendesk__sla_policies` might erroneously have a null `sla_policy_name` due to system-generated millisecond-long gaps in timestamps. The package now compares timestamps to the nearest `second` when selecting valid SLA policy names in [int_zendesk__sla_policy_applied](https://github.com/fivetran/dbt_zendesk/blob/main/models/sla_policy/int_zendesk__sla_policy_applied.sql).
+
+## Under the Hood
+- Updated `consistency_sla_policies` and `sla_count_match` data validation tests to account for the above change.
+
 # dbt_zendesk v0.18.0
 [PR #171](https://github.com/fivetran/dbt_zendesk/pull/171) includes the following changes:
 

diff --git a/README.md b/README.md
@@ -93,6 +93,7 @@ vars:
 ```
 
 ### (Optional) Step 5: Additional configurations
+<details open><summary>Expand/Collapse details</summary>
 
 #### Enabling the unstructured document model for NLP
 This package includes the `zendesk__document` model, which processes and segments Zendesk text data for vectorization, making it suitable for NLP workflows. The model outputs structured chunks of text with associated document IDs, segment indices, and token counts. For definitions and more information, refer to [zendesk__document](https://fivetran.github.io/dbt_zendesk/#!/model/model.zendesk.zendesk__document) in our dbt docs.
@@ -171,9 +172,6 @@ This package will create a row in `zendesk__ticket_field_history` for each day t
 ```yml
 # dbt_project.yml
 
-...
-config-version: 2
-
 vars:
   zendesk:
     ticket_field_history_extension_months: integer_number_of_months # default = 0 
@@ -184,9 +182,6 @@ Conversely, you may want to only track the past X years of ticket field history.
 ```yml
 # dbt_project.yml
 
-...
-config-version: 2
-
 vars:
   zendesk:
     ticket_field_history_timeframe_years: integer_number_of_years # default = 50 (everything)
@@ -219,6 +214,8 @@ vars:
     zendesk_<default_source_table_name>_identifier: your_table_name 
 ```
 
+</details>
+
 ### (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™
 <details><summary>Expand for details</summary>
 <br>

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'zendesk'
-version: '0.18.0'
+version: '0.18.1'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'zendesk_integration_tests'
-version: '0.18.0'
+version: '0.18.1'
 
 profile: 'integration_tests'
 
@@ -38,6 +38,7 @@ vars:
   # using_organization_tags: false
   # fivetran_integrity_sla_metric_parity_exclusion_tickets: (56,80)
   # fivetran_integrity_sla_first_reply_time_exclusion_tickets: (56,80)
+  # fivetran_consistency_sla_policies_exclusion_tickets: (55,58) # can remove after this release
 
 models:
   +schema: "zendesk_{{ var('directed_schema','dev') }}"

diff --git a/integration_tests/tests/consistency/consistency_sla_policies.sql b/integration_tests/tests/consistency/consistency_sla_policies.sql
@@ -13,7 +13,7 @@ with prod as (
         target,
         in_business_hours,
         sla_breach_at,
-        round(sla_elapsed_time, -1) as sla_elapsed_time, --round to the nearest tens
+        sla_elapsed_time,
         is_active_sla,
         is_sla_breach
     from {{ target.schema }}_zendesk_prod.zendesk__sla_policies
@@ -28,7 +28,7 @@ dev as (
         target,
         in_business_hours,
         sla_breach_at,
-        round(sla_elapsed_time, -1) as sla_elapsed_time, --round to the nearest tens
+        sla_elapsed_time,
         is_active_sla,
         is_sla_breach
     from {{ target.schema }}_zendesk_dev.zendesk__sla_policies
@@ -48,7 +48,7 @@ dev_not_in_prod as (
     select * from prod
 ),
 
-final as (
+combine as (
     select
         *,
         'from prod' as source
@@ -60,8 +60,20 @@ final as (
         *,
         'from dev' as source
     from dev_not_in_prod
+),
+
+final as (
+    select 
+        *,
+        max(sla_elapsed_time) over (partition by ticket_id, metric, sla_applied_at) as max_sla_elapsed_time,
+        min(sla_elapsed_time) over (partition by ticket_id, metric, sla_applied_at) as min_sla_elapsed_time
+
+    from combine 
+    {{ "where ticket_id not in " ~ var('fivetran_consistency_sla_policies_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_sla_policies_exclusion_tickets',[]) }}
 )
 
 select *
 from final
-{{ "where ticket_id not in " ~ var('fivetran_consistency_sla_policies_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_sla_policies_exclusion_tickets',[]) }}
+where 
+    {# Take differences in runtime into account #}
+    max_sla_elapsed_time - min_sla_elapsed_time > 2 
diff --git a/integration_tests/tests/integrity/sla_count_match.sql b/integration_tests/tests/integrity/sla_count_match.sql
@@ -8,7 +8,7 @@
 with source as (
     select
         *,
-        case when field_name = 'first_reply_time' then row_number() over (partition by ticket_id, field_name order by valid_starting_at desc) else 1 end as latest_sla
+        case when field_name = 'first_reply_time' then row_number() over (partition by ticket_id, field_name order by valid_starting_at) else 1 end as latest_sla
     from {{ ref('stg_zendesk__ticket_field_history') }}
 ),
 

diff --git a/models/sla_policy/int_zendesk__sla_policy_applied.sql b/models/sla_policy/int_zendesk__sla_policy_applied.sql
@@ -46,8 +46,8 @@ with ticket_field_history as (
   from sla_policy_applied
   left join sla_policy_name
     on sla_policy_name.ticket_id = sla_policy_applied.ticket_id
-      and sla_policy_applied.valid_starting_at >= sla_policy_name.valid_starting_at
-      and sla_policy_applied.valid_starting_at < coalesce(sla_policy_name.valid_ending_at, {{ dbt.current_timestamp() }}) 
+      and {{ dbt.date_trunc("second", "sla_policy_applied.valid_starting_at") }} >= {{ dbt.date_trunc("second", "sla_policy_name.valid_starting_at") }}
+      and {{ dbt.date_trunc("second", "sla_policy_applied.valid_starting_at") }} < coalesce({{ dbt.date_trunc("second", "sla_policy_name.valid_ending_at") }}, {{ dbt.current_timestamp() }})
   where sla_policy_applied.latest_sla = 1
 )