Skip to content

Commit

Permalink
use set write_method instead of inline conditional
Browse files Browse the repository at this point in the history
  • Loading branch information
colin-rogers-dbt committed Oct 29, 2024
1 parent a97abb1 commit ec96883
Showing 1 changed file with 10 additions and 4 deletions.
14 changes: 10 additions & 4 deletions dbt/include/bigquery/macros/materializations/table.sql
Original file line number Diff line number Diff line change
Expand Up @@ -113,23 +113,29 @@ else:
msg = f"{type(df)} is not a supported type for dbt Python materialization"
raise Exception(msg)

# For writeMethod we need to use "indirect" if materializing a partitioned table
# otherwise we can use "direct". Note that indirect will fail if the GCS bucket has a retention policy set on it.
{%- if partition_config is not none %}
{%- set write_method = 'indirect' -%}
{%- else %}
{% set write_method = 'direct' -%}
{%- endif %}

df.write \
.mode("overwrite") \
.format("bigquery") \
.option("writeMethod", "{{ write_method }}") \
.option("writeDisposition", 'WRITE_TRUNCATE') \
{%- if partition_config is not none %}
.option("writeMethod", "indirect") \
{%- if partition_config.data_type | lower in ('date','timestamp','datetime') %}
.option("partitionField", "{{- partition_config.field -}}") \
{%- if partition_config.granularity is not none %}
.option("partitionType", "{{- partition_config.granularity| upper -}}") \
{%- endif %}
{%- endif %}
{% else %}
.option("writeMethod", "direct") \
{%- endif %}
{%- if raw_cluster_by is not none %}
.option("clusteredFields", "{{- raw_cluster_by | join(',') -}}") \
{%- endif %}
.option("writeDisposition", 'WRITE_TRUNCATE') \
.save("{{target_relation}}")
{% endmacro %}

0 comments on commit ec96883

Please sign in to comment.