From f2d1b6e13acb4882b1e8f9d30522959e4bef35d7 Mon Sep 17 00:00:00 2001 From: Justin Carter Date: Mon, 27 Jan 2025 15:55:04 +0100 Subject: [PATCH] refactor: rename dataform_output -> dbt_output for reports (#1323) --- .github/workflows/bats.yml | 4 ---- .github/workflows/cypress.yml | 5 ----- Makefile | 16 ---------------- bats/lana-sim-time.yml | 4 +--- bats/lana.yml | 4 +--- lana/app/src/report/config.rs | 10 ++-------- lana/app/src/report/error.rs | 4 ---- lana/app/src/report/upload.rs | 4 ++-- 8 files changed, 6 insertions(+), 45 deletions(-) diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml index 5a9d1ffc1..9c112ee07 100644 --- a/.github/workflows/bats.yml +++ b/.github/workflows/bats.yml @@ -26,10 +26,6 @@ jobs: uses: "google-github-actions/auth@v2" with: credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" - - name: Sync dataform-branch - run: nix develop -c make push-dataform-branch - env: - DATAFORM_BRANCH: "gha-dataform" - name: Run e2e tests run: nix develop -c make e2e env: diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 1fab4bec8..9175e5346 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -31,11 +31,6 @@ jobs: - name: build backend run: nix develop -c make build-for-tests - - name: sync dataform-branch - run: nix develop -c make push-dataform-branch - env: - DATAFORM_BRANCH: "gha-dataform" - - name: keep tilt running run: nix develop -c make tilt-in-ci env: diff --git a/Makefile b/Makefile index 4a4133b1b..d7914cb52 100644 --- a/Makefile +++ b/Makefile @@ -111,19 +111,3 @@ tilt-in-ci: test-cypress-in-ci-through-browserstack: cd apps/admin-panel && pnpm cypress:run browserstack - -push-dataform-branch: - git push -f origin HEAD:${DATAFORM_BRANCH} - -dataform-install: - yarn install - node_modules/.bin/dataform install - -dataform-run: - node_modules/.bin/dataform run --timeout 5m --schema-suffix=${DATAFORM_SCHEMA_SUFFIX} --vars=${DATAFORM_VARS} - -dataform-run-dev: - node_modules/.bin/dataform run --timeout 5m --schema-suffix=${DATAFORM_SCHEMA_SUFFIX} --vars=${DATAFORM_VARS} --tags=current-dev - -dataform-run-staging: - node_modules/.bin/dataform run --timeout 5m --schema-suffix=${DATAFORM_SCHEMA_SUFFIX} --vars="executionEnv=galoy-staging,devUser=${TF_VAR_name_prefix}" diff --git a/bats/lana-sim-time.yml b/bats/lana-sim-time.yml index 20bd28188..69c80f585 100644 --- a/bats/lana-sim-time.yml +++ b/bats/lana-sim-time.yml @@ -9,9 +9,7 @@ app: bucket_name: "gha-lana-documents" root_folder: "gha" report: - dataform_repo: "gha-repo" - dataform_output_dataset: "dataform_gha" - dataform_release_config: "gha-release" + dbt_output_dataset: "dbt_gha" dev_disable_auto_create: true time: realtime: false diff --git a/bats/lana.yml b/bats/lana.yml index 0aa3c6994..5b5a78540 100644 --- a/bats/lana.yml +++ b/bats/lana.yml @@ -9,6 +9,4 @@ app: bucket_name: "gha-lana-documents" root_folder: "gha" report: - dataform_repo: "gha-repo" - dataform_output_dataset: "dataform_gha" - dataform_release_config: "gha-release" + dbt_output_dataset: "dbt_gha" diff --git a/lana/app/src/report/config.rs b/lana/app/src/report/config.rs index 24b2befac..f9534b9fe 100644 --- a/lana/app/src/report/config.rs +++ b/lana/app/src/report/config.rs @@ -5,11 +5,7 @@ use crate::service_account::ServiceAccountConfig; #[derive(Clone, Debug, Deserialize, Serialize, Default)] pub struct ReportConfig { #[serde(default)] - pub dataform_repo: String, - #[serde(default)] - pub dataform_output_dataset: String, - #[serde(default)] - pub dataform_release_config: String, + pub dbt_output_dataset: String, #[serde(default)] pub dev_disable_auto_create: bool, @@ -24,9 +20,7 @@ impl ReportConfig { dev_disable_auto_create: bool, ) -> ReportConfig { Self { - dataform_repo: format!("{}-repo", name_prefix), - dataform_output_dataset: format!("dbt_{}", name_prefix), - dataform_release_config: format!("{}-release", name_prefix), + dbt_output_dataset: format!("dbt_{}", name_prefix), dev_disable_auto_create, service_account: Some(service_account), } diff --git a/lana/app/src/report/error.rs b/lana/app/src/report/error.rs index 85c3f96ba..373dfd15f 100644 --- a/lana/app/src/report/error.rs +++ b/lana/app/src/report/error.rs @@ -10,10 +10,6 @@ pub enum ReportError { CursorDestructureError(#[from] es_entity::CursorDestructureError), #[error("ReportError - SerdeJson: {0}")] Deserialization(#[from] serde_json::Error), - #[error("ReportError - DataformCompilation: {0}")] - DataformCompilation(String), - #[error("ReportError - DataformInvocation: {0}")] - DataformInvocation(String), #[error("ReportError - Reqwest: {0}")] Reqwest(#[from] reqwest::Error), #[error("ReportError - GCPAuth: {0}")] diff --git a/lana/app/src/report/upload.rs b/lana/app/src/report/upload.rs index 41122b6de..da25ce68c 100644 --- a/lana/app/src/report/upload.rs +++ b/lana/app/src/report/upload.rs @@ -107,7 +107,7 @@ pub(super) mod bq { .table() .list( &config.service_account().gcp_project, - &config.dataform_output_dataset, + &config.dbt_output_dataset, ListOptions::default(), ) .await?; @@ -135,7 +135,7 @@ pub(super) mod bq { let gcp_project = &config.service_account().gcp_project; let query = format!( "SELECT * FROM `{}.{}.{}`", - gcp_project, config.dataform_output_dataset, report + gcp_project, config.dbt_output_dataset, report ); let res = client .job()