copy github data directly from DB (#875)

guardian · Mar 26, 2024 · 2606331 · 2606331
1 parent ee0cbad
commit 2606331
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 63 deletions.
diff --git a/packages/dev-environment/dev-config/cloudquery.yaml b/packages/dev-environment/dev-config/cloudquery.yaml
@@ -1,53 +1,3 @@
----
-kind: source
-spec:
-  # Source spec section
-  name: 'github'
-  path: 'cloudquery/github'
-  version: v${CQ_GITHUB}
-  skip_dependent_tables: true
-  tables:
-    - github_repositories
-    - github_repository_branches
-    - github_workflows
-  destinations: ['postgresql']
-  spec:
-    app_auth:
-      - org: guardian
-        private_key_path: /private-key.pem
-        app_id: '${file:/app-id}'
-        installation_id: '${file:/installation-id}'
-    repos: [
-        # example devX owned repos
-        'guardian/cdk',
-        'guardian/service-catalogue',
-
-        # example non-devX repo
-        'guardian/dotcom-rendering',
-
-        # example private repo
-        'guardian/tracker',
-      ]
----
-kind: source
-spec:
-  name: 'github-teams'
-  path: 'cloudquery/github'
-  version: v${CQ_GITHUB}
-  skip_dependent_tables: true
-  tables:
-    - github_teams
-    - github_team_repositories
-  destinations: ['postgresql']
-  spec:
-    app_auth:
-      - org: guardian
-        private_key_path: /private-key.pem
-        app_id: '${file:/app-id}'
-        installation_id: '${file:/installation-id}'
-    orgs:
-      - guardian
----
 kind: source
 spec:
   name: 'aws'
@@ -72,16 +22,6 @@ spec:
       - id: 'deployTools'
         local_profile: 'deployTools'
 ---
-kind: source
-spec:
-  name: 'github-languages'
-  # https://github.com/guardian/cq-source-github-languages
-  path: 'guardian/github-languages'
-  registry: 'github'
-  version: v0.0.4
-  tables: ['*']
-  destinations: ['postgresql']
----
 kind: destination
 spec:
   name: 'postgresql'

diff --git a/packages/dev-environment/script/start b/packages/dev-environment/script/start
@@ -49,10 +49,10 @@ copy(){
     step "Copying $1 data to local database"
     PGPASSWORD=${CODE_DB_PASSWORD} pg_dump \
     -U postgres -h "${CODE_HOST}" -p 5432 -d "${DATABASE_NAME}" -t "$1*" -f "$1.sql" \
-    --no-owner --no-privileges --verbose --inserts
+    --no-owner --no-privileges --inserts
 
     PGPASSWORD=${DATABASE_PASSWORD}  psql \
-    -U postgres -h "${DATABASE_HOSTNAME}" -p 5432 -d "${DATABASE_NAME}" -f "$1.sql"
+    -U postgres -h "${DATABASE_HOSTNAME}" -p 5432 -d "${DATABASE_NAME}" -f "$1.sql" -q
     rm "$1.sql"
 }
 
@@ -83,6 +83,9 @@ source "$MAIN_ENV_FILE"
 
 copy "snyk"
 copy "galaxies"
-
+copy "github_team"
+copy "github_repo"
+copy "github_workflows"
+copy "github_languages"
 unset CODE_DB_PASSWORD
 unset CODE_HOST