fix(ext-drievr-bq): Fix pr suggestion, including typo, document layou…

…t and testing
Canner · Nov 3, 2022 · 90d145c · 90d145c
1 parent 2c58dc0
commit 90d145c
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 146 deletions.
diff --git a/packages/doc/docs/connectors.mdx b/packages/doc/docs/connectors.mdx
@@ -7,7 +7,7 @@ We support the following data warehouses to connect with, you can choose multipl
 | [PostgreSQL](./connectors/postgresql) | ✅ Yes | ✅ Yes      | ❌ No             |
 | [DuckDB](./connectors/duckdb)         | ✅ Yes | ✅ Yes      | ❌ No             |
 | [Snowflake](./connectors/snowflake)   | ✅ Yes | ✅ Yes      | ❌ No             |
-| BigQuery                              | ✅ Yes | ✅ Yes      | ❌ No             |
+| [BigQuery]](./connectors/bigquery)    | ✅ Yes | ✅ Yes      | ❌ No             |
 
 \* Fetching rows only when we need them, it has better performance with large query results.
 

diff --git a/packages/doc/docs/connectors/bigquery.mdx b/packages/doc/docs/connectors/bigquery.mdx
@@ -26,34 +26,42 @@ Connect with your bigquery servers via the official [Node.js Driver](https://clo
 
 3. Create a new profile in `profiles.yaml` or in your profile files. For example:
    :::info
-   You can choose one from `keyFilename` or `credentials` to use.
+
+   - Your service account must have the following permissions to successfully execute queries.
+   - - BigQuery Data Viewer
+   - - BigQuery Job User
+
+   - You can choose one from `keyFilename` or `credentials` to use.
+
    For details, please refer to [here](https://cloud.google.com/docs/authentication#service-accounts)
    :::
 
-wish keyFilename:
+with keyFilename:
 
 ```yaml
-- name: bq # profile name
-  type: bq
-  connection:
-    location: ''
-    projectId: 'your-project-id'
-    keyFilename: '/path/to/keyfile.json'
-  allow: '*'
+name: bq # profile name
+type: bq
+connection:
+location: ''
+projectId: 'your-project-id'
+keyFilename: '/path/to/keyfile.json'
+allow: '*'
 ```
 
-wish credential:
+with credential:
 
 ```yaml
-- name: bq # profile name
-  type: bq
-  connection:
-  location: US
-  projectId: 'your-project-id'
-  credential:
-    client_email: [email protected]
-    private_key: '-----BEGIN PRIVATE KEY----- XXXXX -----END PRIVATE KEY-----\n'
-  allow: '*'
+name: bq # profile name
+type: bq
+connection:
+location: US
+projectId: 'your-project-id'
+credential:
+client_email: [email protected]
+private_key: '-----BEGIN PRIVATE KEY----- XXXXX -----END PRIVATE KEY-----\n'
+allow: '*'
+```
+
 ```
 
 ## Connection Configuration
@@ -63,8 +71,9 @@ Please check [Interface BigQueryOptions](https://cloud.google.com/nodejs/docs/re
 | Name                     | Required | Default | Description                                                                                                                                                                                                                                                                                                                                                                                       |
 | ------------------------ | -------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | location                 | N        | US      | Location must match that of the dataset(s) referenced in the query.                                                                                                                                                                                                                                                                                                                               |
-| projectId                | N        |         | The project ID from the Google Developer's Console, e.g. 'grape-spaceship-123'. We will also check the environment variable `GCLOUD_PROJECT` for your project ID. If your app is running in an environment which [supports](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application Application Default Credentials), your project ID will be detected. |
+| projectId                | N        |         | The project ID from the Google Developer's Console, e.g. 'grape-spaceship-123'. We will also check the environment variable `GCLOUD_PROJECT` for your project ID. If your app is running in an environment which [supports](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application) Application Default Credentials), your project ID will be detected. |
 | keyFilename              | N        |         | Full path to the a .json, .pem, or .p12 key downloaded from the Google Developers Console. If you provide a path to a JSON file, the `projectId` option above is not necessary. NOTE: .pem and .p12 require you to specify the `email` option as well.                                                                                                                                            |
 | credentials              | N        |         | Credentials object.                                                                                                                                                                                                                                                                                                                                                                               |
 | credentials.client_email | N        |         | Your service account.                                                                                                                                                                                                                                                                                                                                                                             |
 | credentials.private_key  | N        |         | Your service account's private key.                                                                                                                                                                                                                                                                                                                                                               |
+```
diff --git a/packages/extension-driver-bq/README.md b/packages/extension-driver-bq/README.md
@@ -19,23 +19,23 @@
 
 3. Create a new profile in `profiles.yaml` or in your profiles' paths.
 
-> ⚠️ Your service account must have the following permissions to successfully execute queries...
+> ⚠️ Your service account must have the following permissions to successfully execute queries.
 >
 > - BigQuery Data Viewer
 > - BigQuery Job User
 
 ```yaml
-- name: bq # profile name
-  type: bq
-  connection:
-    # Location must match that of the dataset(s) referenced in the query.
-    location: US
-    # Optional: The max rows we should fetch once.
-    chunkSize: 100
-    # The project ID from the Google Developer's Console, e.g. 'grape-spaceship-123'. We will also check the environment variable `GCLOUD_PROJECT` for your project ID. If your app is running in an environment which [supports](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application Application Default Credentials), your project ID will be detected.
-    projectId: 'your-project-id'
-    # Full path to the a .json, .pem, or .p12 key downloaded from the Google Developers Console. If you provide a path to a JSON file, the `projectId` option above is not necessary. NOTE: .pem and .p12 require you to specify the `email` option as well.
-    keyFilename: '/path/to/keyfile.json'
+name: bq # profile name
+type: bq
+connection:
+  # Location must match that of the dataset(s) referenced in the query.
+  location: US
+  # Optional: The max rows we should fetch once.
+  chunkSize: 100
+  # The project ID from the Google Developer's Console, e.g. 'grape-spaceship-123'. We will also check the environment variable `GCLOUD_PROJECT` for your project ID. If your app is running in an environment which [supports](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application) Application Default Credentials), your project ID will be detected.
+  projectId: 'your-project-id'
+  # Full path to the a .json, .pem, or .p12 key downloaded from the Google Developers Console. If you provide a path to a JSON file, the `projectId` option above is not necessary. NOTE: .pem and .p12 require you to specify the `email` option as well.
+  keyFilename: '/path/to/keyfile.json'
 ```
 
 ## Testing

diff --git a/packages/extension-driver-bq/package.json b/packages/extension-driver-bq/package.json
@@ -15,8 +15,8 @@
     "data-warehouse",
     "data-lake",
     "api-builder",
-    "postgres",
-    "pg"
+    "bigquery",
+    "bq"
   ],
   "repository": {
     "type": "git",

diff --git a/packages/extension-driver-bq/project.json b/packages/extension-driver-bq/project.json
@@ -3,14 +3,33 @@
   "sourceRoot": "packages/extension-driver-bq/src",
   "targets": {
     "build": {
+      "executor": "@nrwl/js:tsc",
+      "options": {
+        "command": "yarn ts-node ./tools/scripts/replaceAlias.ts extension-driver-bq"
+      },
+      "dependsOn": [
+        {
+          "projects": "self",
+          "target": "tsc"
+        }
+      ]
+    },
+    "tsc": {
       "executor": "@nrwl/js:tsc",
       "outputs": ["{options.outputPath}"],
       "options": {
         "outputPath": "dist/packages/extension-driver-bq",
         "main": "packages/extension-driver-bq/src/index.ts",
         "tsConfig": "packages/extension-driver-bq/tsconfig.lib.json",
-        "assets": ["packages/extension-driver-bq/*.md"]
-      }
+        "assets": ["packages/extension-driver-bq/*.md"],
+        "buildableProjectDepsInPackageJsonType": "dependencies"
+      },
+      "dependsOn": [
+        {
+          "projects": "dependencies",
+          "target": "build"
+        }
+      ]
     },
     "publish": {
       "executor": "@nrwl/workspace:run-commands",

diff --git a/packages/extension-driver-bq/src/lib/bqDataSource.ts b/packages/extension-driver-bq/src/lib/bqDataSource.ts
@@ -26,13 +26,14 @@ export class BQDataSource extends DataSource<any, BQOptions> {
     const profiles = this.getProfiles().values();
     for (const profile of profiles) {
       this.logger.debug(
-        `Initializing profile: ${profile.name} using pg driver`
+        `Initializing profile: ${profile.name} using bq driver`
       );
       const bigqueryClient = new BigQuery(profile.connection);
       // https://cloud.google.com/nodejs/docs/reference/bigquery/latest
 
       this.bqMapping.set(profile.name, {
         bq: bigqueryClient,
+        options: profile.connection,
       });
 
       // Testing connection
@@ -51,9 +52,7 @@ export class BQDataSource extends DataSource<any, BQOptions> {
       throw new InternalError(`Profile instance ${profileName} not found`);
     }
     const { bq: client, options } = this.bqMapping.get(profileName)!;
-    this.logger.debug(`Acquiring connection from ${profileName}`);
 
-    origin;
     const params: Record<string, any> = {};
     bindParams.forEach((value, key) => {
       params[key.replace('@', '')] = value;
@@ -70,7 +69,6 @@ export class BQDataSource extends DataSource<any, BQOptions> {
 
       const [job] = await client.createQueryJob(queryOptions);
 
-      // All promises MUST fulfilled in this function or we are not able to release the connection when error occurred
       return await this.getResultFromQueryJob(job, options);
     } catch (e: any) {
       this.logger.debug(
@@ -89,8 +87,8 @@ export class BQDataSource extends DataSource<any, BQOptions> {
     options?: BQOptions
   ): Promise<DataResult> {
     const { chunkSize = 100 } = options || {};
-    const jobDataRead = this.jobDataRead.bind(this);
-    const firstChunk = await jobDataRead(queryJob, chunkSize);
+    const fetchJobResult = this.fetchJobResult.bind(this);
+    const firstChunk = await fetchJobResult(queryJob, chunkSize);
 
     // save first chunk in buffer for incoming requests
     let bufferedRows = [...firstChunk.rows];
@@ -101,7 +99,7 @@ export class BQDataSource extends DataSource<any, BQOptions> {
       if (bufferReadIndex >= bufferedRows.length) {
         if (nextQuery == null) return null;
 
-        const fetchData = await jobDataRead(queryJob, chunkSize, nextQuery);
+        const fetchData = await fetchJobResult(queryJob, chunkSize, nextQuery);
         bufferedRows = fetchData.rows;
         nextQuery = fetchData.nextQuery;
         bufferReadIndex = 0;
@@ -122,10 +120,6 @@ export class BQDataSource extends DataSource<any, BQOptions> {
             this.destroy(error);
           });
       },
-      destroy(error: Error | null, cb: (error: Error | null) => void) {
-        // Send done event to notify upstream to release the connection.
-        cb(error);
-      },
       // automatically destroy() the stream when it emits 'finish' or errors. Node > 10.16
       autoDestroy: true,
     });
@@ -141,7 +135,7 @@ export class BQDataSource extends DataSource<any, BQOptions> {
     };
   }
 
-  public async jobDataRead(
+  public async fetchJobResult(
     queryJob: Job,
     chunkSize: number,
     nextQuery?: Query | null | undefined
@@ -151,7 +145,7 @@ export class BQDataSource extends DataSource<any, BQOptions> {
       nextQuery: Query | null | undefined;
       apiResponse: bigquery.IGetQueryResultsResponse | null | undefined;
     }>((resolve, reject) => {
-      return queryJob.getQueryResults(
+      queryJob.getQueryResults(
         nextQuery || { maxResults: chunkSize },
         (err, rows, nextQuery, apiResponse) => {
           if (err) {