Skip to content

Commit

Permalink
add defer_to_manifest in before_run to fix faulty deferred docs gener…
Browse files Browse the repository at this point in the history
…ate (#6488)

* add defer_to_manifest in before_run to fix faulty deferred docs generate

* add a changelog

* add declaration of defer_to_manifest to FreshnessTask and GraphRunnableTask

* fix: add defer_to_manifest method  to ListTask
  • Loading branch information
mivanicova authored Jan 10, 2023
1 parent 5da6360 commit 0fc080d
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 0 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20221226-010211.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: fix docs generate --defer by adding defer_to_manifest to before_run
time: 2022-12-26T01:02:11.630614+01:00
custom:
Author: mivanicova
Issue: "6488"
4 changes: 4 additions & 0 deletions core/dbt/task/freshness.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ def node_is_match(self, node):


class FreshnessTask(GraphRunnableTask):
def defer_to_manifest(self, adapter, selected_uids):
# freshness don't defer
return

def result_path(self):
if self.args.output:
return os.path.realpath(self.args.output)
Expand Down
4 changes: 4 additions & 0 deletions core/dbt/task/list.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,10 @@ def selection_arg(self):
else:
return self.args.select

def defer_to_manifest(self, adapter, selected_uids):
# list don't defer
return

def get_node_selector(self):
if self.manifest is None or self.graph is None:
raise InternalException("manifest and graph must be set to get perform node selection")
Expand Down
5 changes: 5 additions & 0 deletions core/dbt/task/runnable.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,10 @@ def get_selection_spec(self) -> SelectionSpec:
def get_node_selector(self) -> NodeSelector:
raise NotImplementedException(f"get_node_selector not implemented for task {type(self)}")

@abstractmethod
def defer_to_manifest(self, adapter, selected_uids: AbstractSet[str]):
raise NotImplementedException(f"defer_to_manifest not implemented for task {type(self)}")

def get_graph_queue(self) -> GraphQueue:
selector = self.get_node_selector()
spec = self.get_selection_spec()
Expand Down Expand Up @@ -419,6 +423,7 @@ def populate_adapter_cache(self, adapter, required_schemas: Set[BaseRelation] =
def before_run(self, adapter, selected_uids: AbstractSet[str]):
with adapter.connection_named("master"):
self.populate_adapter_cache(adapter)
self.defer_to_manifest(adapter, selected_uids)

def after_run(self, adapter, results):
pass
Expand Down
10 changes: 10 additions & 0 deletions test/integration/062_defer_state_tests/test_defer_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,22 @@ def run_and_defer(self):
# no state, wrong schema, failure.
self.run_dbt(['test', '--target', 'otherschema'], expect_pass=False)

# test generate docs
# no state, wrong schema, empty nodes
catalog = self.run_dbt(['docs','generate','--target', 'otherschema'])
assert not catalog.nodes

# no state, run also fails
self.run_dbt(['run', '--target', 'otherschema'], expect_pass=False)

# defer test, it succeeds
results = self.run_dbt(['test', '-m', 'view_model+', '--state', 'state', '--defer', '--target', 'otherschema'])

# defer docs generate with state, catalog refers schema from the happy times
catalog = self.run_dbt(['docs','generate', '-m', 'view_model+', '--state', 'state', '--defer','--target', 'otherschema'])
assert self.other_schema not in catalog.nodes["seed.test.seed"].metadata.schema
assert self.unique_schema() in catalog.nodes["seed.test.seed"].metadata.schema

# with state it should work though
results = self.run_dbt(['run', '-m', 'view_model', '--state', 'state', '--defer', '--target', 'otherschema'])
assert self.other_schema not in results[0].node.compiled_code
Expand Down

0 comments on commit 0fc080d

Please sign in to comment.