From c3f4b9eafa1b05d87b51dd7ba2545d603f14565b Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 17:00:51 -0700 Subject: [PATCH 01/14] change break indicators in output --- src/io.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io.ts b/src/io.ts index a894896..2383185 100644 --- a/src/io.ts +++ b/src/io.ts @@ -8,7 +8,7 @@ export function readJsonFileContent(path: string) { export function writeToStdOut(data: {parentSql: string, childQueries: string[]}) { console.log(data.parentSql) data.childQueries.forEach(q => { - console.log("--------") + console.log("/**/") console.log(q) }) } From 99932a2f2abf4a6267092e7023201eff2a3408b2 Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 17:15:21 -0700 Subject: [PATCH 02/14] update readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8521b93..278ee74 100644 --- a/README.md +++ b/README.md @@ -10,18 +10,18 @@ This CLI tool reads a JSON file and produces BigQuery compatible SQL views from ## Usage ```bash -npx @nealwp/blobview +npx @nealwp/blobview@latest ``` ## Examples: Default output to STDOUT: ```bash -npx @nealwp/blobview ./path/to/file.json +npx @nealwp/blobview@latest ./path/to/file.json ``` Redirect output to file: ```bash -npx @nealwp/blobview ./path/to/file.json > my-view-file.sql +npx @nealwp/blobview@latest ./path/to/file.json > my-view-file.sql ``` ## Features @@ -101,17 +101,17 @@ SELECT , CAST(JSON_VALUE(json_blob.decimalField) as DECIMAL) as decimal_field , TO_JSON_STRING(json_blob.exampleGeoJson) as example_geo_json FROM .. --------- +/**/ SELECT CAST(JSON_VALUE(json_blob.childField1.gender) as STRING) as gender , CAST(JSON_VALUE(json_blob.childField1.latitude) as DECIMAL) as latitude FROM .. --------- +/**/ SELECT CAST(JSON_VALUE(json_blob.childField2.favoriteFruit) as STRING) as favorite_fruit , CAST(JSON_VALUE(json_blob.childField2.longitude) as DECIMAL) as longitude FROM .. --------- +/**/ SELECT CAST(JSON_VALUE(json_blob.childWithNestedObject.isNormal) as STRING) as is_normal , TO_JSON_STRING(json_blob.childWithNestedObject.nestedObject) as nested_object From 23c565529f3173188a32c1642c207972802f3cdd Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 17:26:14 -0700 Subject: [PATCH 03/14] rename datastream to dataset, dataset to table --- src/parser.spec.ts | 18 +++++++++--------- src/parser.ts | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/parser.spec.ts b/src/parser.spec.ts index 2692c91..b0d56f2 100644 --- a/src/parser.spec.ts +++ b/src/parser.spec.ts @@ -12,7 +12,7 @@ describe('parser', () => { } } - const expectedResult = ['SELECT\n\tCAST(JSON_VALUE(json_blob.topKey.nestedKey1) as INTEGER) as nested_key_1\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey2) as STRING) as nested_key_2\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey3) as DECIMAL) as nested_key_3\nFROM ..'] + const expectedResult = ['SELECT\n\tCAST(JSON_VALUE(json_blob.topKey.nestedKey1) as INTEGER) as nested_key_1\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey2) as STRING) as nested_key_2\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey3) as DECIMAL) as nested_key_3\nFROM ..'] const output = jsonToSqlView(testObj) expect(output.childQueries).toEqual(expectedResult) }) @@ -26,9 +26,9 @@ describe('parser', () => { } const expectedResult = [ - 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey1.nestedKey1) as STRING) as nested_key_1\nFROM ..', - 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey2.nestedKey2) as INTEGER) as nested_key_2\nFROM ..', - 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey3.nestedKey3) as DECIMAL) as nested_key_3\nFROM ..', + 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey1.nestedKey1) as STRING) as nested_key_1\nFROM ..
', + 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey2.nestedKey2) as INTEGER) as nested_key_2\nFROM ..
', + 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey3.nestedKey3) as DECIMAL) as nested_key_3\nFROM ..
', ] const output = jsonToSqlView(testObj) expect(output.childQueries).toEqual(expectedResult) @@ -40,7 +40,7 @@ describe('parser', () => { key2: 'abcd', key3: 3.14 } - const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.key1) as INTEGER) as key_1\n\t, CAST(JSON_VALUE(json_blob.key2) as STRING) as key_2\n\t, CAST(JSON_VALUE(json_blob.key3) as DECIMAL) as key_3\nFROM ..' + const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.key1) as INTEGER) as key_1\n\t, CAST(JSON_VALUE(json_blob.key2) as STRING) as key_2\n\t, CAST(JSON_VALUE(json_blob.key3) as DECIMAL) as key_3\nFROM ..
' const output = jsonToSqlView(testObj) expect(output.parentSql).toEqual(expectedResult) }) @@ -56,7 +56,7 @@ describe('parser', () => { features: ["feature", "collection", "here"] } } - const expectedResult = 'SELECT\n\tTO_JSON_STRING(json_blob.geoJsonThing1) as geo_json_thing_1\n\t, TO_JSON_STRING(json_blob.geoJsonThing2) as geo_json_thing_2\nFROM ..' + const expectedResult = 'SELECT\n\tTO_JSON_STRING(json_blob.geoJsonThing1) as geo_json_thing_1\n\t, TO_JSON_STRING(json_blob.geoJsonThing2) as geo_json_thing_2\nFROM ..
' const output = jsonToSqlView(testObj) expect(output.parentSql).toEqual(expectedResult) }) @@ -64,7 +64,7 @@ describe('parser', () => { it('should json string deeply nested objects', () => { const testObj = { nestedKey: { deeplyNestedKey: { some: 'key' } } } - const expectedResult = ['SELECT\n\tTO_JSON_STRING(json_blob.nestedKey.deeplyNestedKey) as deeply_nested_key\nFROM ..'] + const expectedResult = ['SELECT\n\tTO_JSON_STRING(json_blob.nestedKey.deeplyNestedKey) as deeply_nested_key\nFROM ..
'] const output = jsonToSqlView(testObj) expect(output.childQueries).toEqual(expectedResult) }) @@ -80,7 +80,7 @@ describe('parser', () => { } } - const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey.nestedKey1) as INTEGER) as nested_key_1\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey2) as STRING) as nested_key_2\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey3) as DECIMAL) as nested_key_3\nFROM ..' + const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey.nestedKey1) as INTEGER) as nested_key_1\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey2) as STRING) as nested_key_2\n\t, CAST(JSON_VALUE(json_blob.topKey.nestedKey3) as DECIMAL) as nested_key_3\nFROM ..
' const output = parseNestedKey(testObj.topKey, 'topKey') expect(output).toEqual(expectedResult) }) @@ -88,7 +88,7 @@ describe('parser', () => { it('should json string deeply nested objects', () => { const testObj = { nestedKey: { deeplyNestedKey: { some: 'key' } } } - const expectedResult = 'SELECT\n\tTO_JSON_STRING(json_blob.nestedKey.deeplyNestedKey) as deeply_nested_key\nFROM ..' + const expectedResult = 'SELECT\n\tTO_JSON_STRING(json_blob.nestedKey.deeplyNestedKey) as deeply_nested_key\nFROM ..
' const output = parseNestedKey(testObj.nestedKey, 'nestedKey') expect(output).toEqual(expectedResult) }) diff --git a/src/parser.ts b/src/parser.ts index f8a734e..d1f1467 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -59,7 +59,7 @@ export function jsonToSqlView(json: any) { parentSql = `${parentSql}\n\t${commaIfNeeded(i)}CAST(JSON_VALUE(json_blob.${k}) as ${type}) as ${snakeCase(k)}` } } - parentSql = `${parentSql}\nFROM ..` + parentSql = `${parentSql}\nFROM ..
` return { parentSql, childQueries } } @@ -77,6 +77,6 @@ export function parseNestedKey(json: any, keyName: string) { sql = `${sql}\n\t${commaIfNeeded(i)}CAST(JSON_VALUE(json_blob.${keyName}.${col}) as ${type}) as ${snakeCase(col)}` } } - sql = `${sql}\nFROM ..` + sql = `${sql}\nFROM ..
` return sql } From e5abca848d5788eeb550a925680fc26154e24225 Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:01:30 -0700 Subject: [PATCH 04/14] table and dataset name as input. default values --- src/parser.spec.ts | 18 ++++++++++++++++++ src/parser.ts | 15 +++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/parser.spec.ts b/src/parser.spec.ts index b0d56f2..948b963 100644 --- a/src/parser.spec.ts +++ b/src/parser.spec.ts @@ -45,6 +45,24 @@ describe('parser', () => { expect(output.parentSql).toEqual(expectedResult) }) + it('should use input table name', () => { + const testObj = { key1: 'abcdefg' } + const options = { table: 'myTableName'} + + const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.key1) as STRING) as key_1\nFROM ..myTableName' + const output = jsonToSqlView(testObj, options) + expect(output.parentSql).toEqual(expectedResult) + }) + + it('should use input dataset name', () => { + const testObj = { key1: 'abcdefg' } + const options = { dataset: 'myDataset'} + + const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.key1) as STRING) as key_1\nFROM .myDataset.
' + const output = jsonToSqlView(testObj, options) + expect(output.parentSql).toEqual(expectedResult) + }) + it('should keep geoJson collections in parent query as json strings', () => { const testObj = { geoJsonThing1: { diff --git a/src/parser.ts b/src/parser.ts index d1f1467..f5a1de1 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -43,8 +43,19 @@ export function snakeCase(str: string) { }); } -export function jsonToSqlView(json: any) { +type Options = { + table?: string, + dataset?: string +} + +const defaultOptions: Options = { + table: '
', + dataset: '' +} + +export function jsonToSqlView(json: any, options: Options = defaultOptions) { const keys = Object.keys(json) + const { table = '
', dataset = '' } = options const childQueries = []; let parentSql = `SELECT`; @@ -59,7 +70,7 @@ export function jsonToSqlView(json: any) { parentSql = `${parentSql}\n\t${commaIfNeeded(i)}CAST(JSON_VALUE(json_blob.${k}) as ${type}) as ${snakeCase(k)}` } } - parentSql = `${parentSql}\nFROM ..
` + parentSql = `${parentSql}\nFROM .${dataset}.${table}` return { parentSql, childQueries } } From ccaeb2684898a890f3717542979658c268aaaf3c Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:04:44 -0700 Subject: [PATCH 05/14] add options for nested dataset and table inputs --- src/parser.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index f5a1de1..b6d13cf 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -74,9 +74,10 @@ export function jsonToSqlView(json: any, options: Options = defaultOptions) { return { parentSql, childQueries } } -export function parseNestedKey(json: any, keyName: string) { +export function parseNestedKey(json: any, keyName: string, options: Options = defaultOptions) { const columns = Object.keys(json) const values = Object.values(json) + const { table = '
', dataset = '' } = options let sql = `SELECT` for(let i=0; i..
` + sql = `${sql}\nFROM .${dataset}.${table}` return sql } From 4201ec8a794643c6a2086b50a8223a71026dd3ef Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:08:16 -0700 Subject: [PATCH 06/14] nested takes table name --- src/parser.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/parser.spec.ts b/src/parser.spec.ts index 948b963..8517c24 100644 --- a/src/parser.spec.ts +++ b/src/parser.spec.ts @@ -103,6 +103,15 @@ describe('parser', () => { expect(output).toEqual(expectedResult) }) + it('should use input table name', () => { + const testObj = { topKey: { nestedKey1: 'abdcd' } } + const options = { table: 'myTableName' } + + const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey.nestedKey1) as STRING) as nested_key_1\nFROM ..myTableName' + const output = parseNestedKey(testObj.topKey, 'topKey', options) + expect(output).toEqual(expectedResult) + }) + it('should json string deeply nested objects', () => { const testObj = { nestedKey: { deeplyNestedKey: { some: 'key' } } } From 0ceed3b15d77c1d7aced1f54044ec59c17035365 Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:10:30 -0700 Subject: [PATCH 07/14] parse nested takes dataset name input --- src/parser.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/parser.spec.ts b/src/parser.spec.ts index 8517c24..0191acd 100644 --- a/src/parser.spec.ts +++ b/src/parser.spec.ts @@ -112,6 +112,15 @@ describe('parser', () => { expect(output).toEqual(expectedResult) }) + it('should use input dataset name', () => { + const testObj = { topKey: { nestedKey1: 'abdcd' } } + const options = { dataset: 'myDatasetName' } + + const expectedResult = 'SELECT\n\tCAST(JSON_VALUE(json_blob.topKey.nestedKey1) as STRING) as nested_key_1\nFROM .myDatasetName.
' + const output = parseNestedKey(testObj.topKey, 'topKey', options) + expect(output).toEqual(expectedResult) + }) + it('should json string deeply nested objects', () => { const testObj = { nestedKey: { deeplyNestedKey: { some: 'key' } } } From edeae6d1d96bcd8d4855236145d8b42f9c6756d1 Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:34:46 -0700 Subject: [PATCH 08/14] accepts table and dataset from cli --- package-lock.json | 20 +++++++++++++++++--- package.json | 4 ++++ src/index.spec.ts | 3 ++- src/index.ts | 26 +++++++++++++++++++++----- src/parser.ts | 2 +- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4e0c3ec..72f28b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,11 +8,15 @@ "name": "@nealwp/blobview", "version": "0.1.0", "license": "ISC", + "dependencies": { + "minimist": "^1.2.8" + }, "bin": { "blobview": "index.js" }, "devDependencies": { "@types/jest": "^29.5.0", + "@types/minimist": "^1.2.2", "@typescript-eslint/eslint-plugin": "^5.57.1", "@typescript-eslint/parser": "^5.57.1", "copyfiles": "^2.4.1", @@ -1535,6 +1539,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/minimist": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", + "dev": true + }, "node_modules/@types/node": { "version": "20.5.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.6.tgz", @@ -4950,7 +4960,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -7856,6 +7865,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "@types/minimist": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", + "dev": true + }, "@types/node": { "version": "20.5.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.6.tgz", @@ -10325,8 +10340,7 @@ "minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" }, "minipass": { "version": "7.0.3", diff --git a/package.json b/package.json index b08a919..5929757 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "license": "ISC", "devDependencies": { "@types/jest": "^29.5.0", + "@types/minimist": "^1.2.2", "@typescript-eslint/eslint-plugin": "^5.57.1", "@typescript-eslint/parser": "^5.57.1", "copyfiles": "^2.4.1", @@ -36,5 +37,8 @@ "ts-jest": "^29.1.0", "ts-node": "^10.9.1", "typescript": "^5.0.3" + }, + "dependencies": { + "minimist": "^1.2.8" } } diff --git a/src/index.spec.ts b/src/index.spec.ts index 72097f1..167c102 100644 --- a/src/index.spec.ts +++ b/src/index.spec.ts @@ -5,7 +5,8 @@ describe('main', () => { it('should log error if no filepath is passed', () => { process.argv = ['foo', 'bar'] console.log = jest.fn() - main() + const args = {_: []} + main(args) expect(console.log).toHaveBeenCalled() }) }) diff --git a/src/index.ts b/src/index.ts index d520e55..a4dee98 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,20 +1,31 @@ #!/usr/bin/env node +import minimist from 'minimist' import { readJsonFileContent, writeToStdOut } from "./io" import { jsonToSqlView } from "./parser" -export const main = () => { - const args = process.argv.slice(2) - const inputFilePath = args[0] +export const main = (args: minimist.ParsedArgs) => { + const inputFilePath = args._[0] + + let table = '
' + let dataset = '' if(!inputFilePath) { printHelp(); return } + if (args.table) { + table = args.table + } + + if (args.dataset) { + dataset = args.dataset + } + try { const fileContent = readJsonFileContent(inputFilePath) - const sqlOutput = jsonToSqlView(fileContent) + const sqlOutput = jsonToSqlView(fileContent, {table, dataset}) writeToStdOut(sqlOutput) } catch(err) { console.log(err) @@ -26,5 +37,10 @@ function printHelp() { console.log(msg) } -main(); +const args = minimist(process.argv.slice(2), { + stopEarly: true, + boolean: true +}) + +main(args); diff --git a/src/parser.ts b/src/parser.ts index b6d13cf..03f4385 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -63,7 +63,7 @@ export function jsonToSqlView(json: any, options: Options = defaultOptions) { if (typeof json[k] === 'object' && isGeoJsonFeatureCollection(json[k])) { parentSql = `${parentSql}\n\t${commaIfNeeded(i)}TO_JSON_STRING(json_blob.${k}) as ${snakeCase(k)}` } else if (typeof json[k] === 'object' && !isGeoJsonFeatureCollection(json[k])){ - const query = parseNestedKey(json[k], k) + const query = parseNestedKey(json[k], k, options) childQueries.push(query) } else { const type = datatype(json[k]) From 768671c27a34749c27c5907205c8850f1157f248 Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:47:58 -0700 Subject: [PATCH 09/14] add help command --- src/index.ts | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index a4dee98..e8b808c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,16 +10,21 @@ export const main = (args: minimist.ParsedArgs) => { let table = '
' let dataset = '' + if(args.help || args.h) { + printHelp() + return + } + if(!inputFilePath) { - printHelp(); + printError("No file path provided!"); return } - if (args.table) { + if (args.table || args.t) { table = args.table } - if (args.dataset) { + if (args.dataset || args.d) { dataset = args.dataset } @@ -33,8 +38,37 @@ export const main = (args: minimist.ParsedArgs) => { } function printHelp() { - const msg = `Error: no file path provided.\nUsage:\n\nnpx @nealwp/blobview \n` - console.log(msg) + const helpText = ` +================================= + @nealwp/blobview +================================= + +Generate BigQuery SQL views from JSON. + +Usage: + + @nealwp/blobview [options] + +Arguments: + + filepath path to valid JSON file + +Options: + + --help, -h show help + + --dataset=DATASET, specify a dataset to use in FROM clause. default: "" + -d DATASET + + --table=TABLE, specify a table name to use in FROM clause. default: "
" + -t TABLE +` + console.log(helpText) +} + +function printError(message: string) { + console.log(`Error: ${message}`) + console.log('Run "@nealwp/blobview --help" to display help.') } const args = minimist(process.argv.slice(2), { From 73f71dfb832989e4c6d20b8e6bf1b862079b75ed Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:48:39 -0700 Subject: [PATCH 10/14] turn off boolean flags --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index e8b808c..e064cea 100644 --- a/src/index.ts +++ b/src/index.ts @@ -73,7 +73,7 @@ function printError(message: string) { const args = minimist(process.argv.slice(2), { stopEarly: true, - boolean: true + boolean: false }) main(args); From 16077d999537d9716b4e8c3af895d1104ff0f13f Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:55:43 -0700 Subject: [PATCH 11/14] cleanup help --- src/index.ts | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/src/index.ts b/src/index.ts index e064cea..1c26077 100644 --- a/src/index.ts +++ b/src/index.ts @@ -38,31 +38,19 @@ export const main = (args: minimist.ParsedArgs) => { } function printHelp() { - const helpText = ` -================================= - @nealwp/blobview -================================= + const helpText = `Generate BigQuery SQL views from JSON. -Generate BigQuery SQL views from JSON. - -Usage: - - @nealwp/blobview [options] +Usage: + @nealwp/blobview [options] Arguments: - - filepath path to valid JSON file + filepath path to valid JSON file Options: + -t TABLE, --table=TABLE specify a table name to use in FROM clause. default: "
" + -d DATASET, --dataset=DATASET specify a dataset to use in FROM clause. default: "" + -h, --help show help` - --help, -h show help - - --dataset=DATASET, specify a dataset to use in FROM clause. default: "" - -d DATASET - - --table=TABLE, specify a table name to use in FROM clause. default: "
" - -t TABLE -` console.log(helpText) } From de38b6f8993865cd95c0d841c863f0cb778edbaa Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 18:59:34 -0700 Subject: [PATCH 12/14] fix input args --- src/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 1c26077..6faee2c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,12 +20,16 @@ export const main = (args: minimist.ParsedArgs) => { return } - if (args.table || args.t) { + if (args.table) { table = args.table + } else if (args.t) { + table = args.t } - if (args.dataset || args.d) { + if (args.dataset) { dataset = args.dataset + } else if (args.d) { + dataset = args.d } try { From 784bf52c7e6d232b29a787b266f60ef1c97a70f5 Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 19:07:54 -0700 Subject: [PATCH 13/14] update readme --- README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 278ee74..f96ec6b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,15 @@ This CLI tool reads a JSON file and produces BigQuery compatible SQL views from ## Usage ```bash -npx @nealwp/blobview@latest +npx @nealwp/blobview [options] + +Arguments: + filepath path to valid JSON file + +Options: + -t TABLE, --table=TABLE specify a table name to use in FROM clause. default: "
" + -d DATASET, --dataset=DATASET specify a dataset to use in FROM clause. default: "" + -h, --help show help ``` ## Examples: @@ -19,6 +27,13 @@ Default output to STDOUT: npx @nealwp/blobview@latest ./path/to/file.json ``` +Dataset and table as input options: +```bash +npx @nealwp/blobview@latest --dataset=myDataset --table=myTable ./path/to/file.json +# shorthand options +npx @nealwp/blobview@latest -d myDataset -t myTable ./path/to/file.json +``` + Redirect output to file: ```bash npx @nealwp/blobview@latest ./path/to/file.json > my-view-file.sql @@ -32,6 +47,7 @@ npx @nealwp/blobview@latest ./path/to/file.json > my-view-file.sql * Auto-formats column names to snake_case from camelCase and PascalCase * Detects deeply-nested objects and formats to JSON string * Pre-populates FROM clause with BigQuery-style placeholders +* BigQuery dataset and table name can be supplied as input options ## Limitations * Does not detect DATE or TIMESTAMP types, or other types like BOOLEAN @@ -40,7 +56,7 @@ npx @nealwp/blobview@latest ./path/to/file.json > my-view-file.sql * Does not create SQL views in any syntax other than BigQuery * Requires a local JSON file to read * Does not include option to write queries to separate files instead of STDOUT -* BigQuery project, dataset, and datastream names cannot be supplied as input +* BigQuery project name cannot be supplied as input ## Example Output @@ -100,20 +116,52 @@ SELECT , CAST(JSON_VALUE(json_blob.integerField) as INTEGER) as integer_field , CAST(JSON_VALUE(json_blob.decimalField) as DECIMAL) as decimal_field , TO_JSON_STRING(json_blob.exampleGeoJson) as example_geo_json -FROM .. +FROM ..
+/**/ +SELECT + CAST(JSON_VALUE(json_blob.childField1.gender) as STRING) as gender + , CAST(JSON_VALUE(json_blob.childField1.latitude) as DECIMAL) as latitude +FROM ..
+/**/ +SELECT + CAST(JSON_VALUE(json_blob.childField2.favoriteFruit) as STRING) as favorite_fruit + , CAST(JSON_VALUE(json_blob.childField2.longitude) as DECIMAL) as longitude +FROM ..
+/**/ +SELECT + CAST(JSON_VALUE(json_blob.childWithNestedObject.isNormal) as STRING) as is_normal + , TO_JSON_STRING(json_blob.childWithNestedObject.nestedObject) as nested_object +FROM ..
+``` + +```bash +# terminal command with input options +npx @nealwp/blobview --dataset=myDataset --table=myTable sample-data.json +``` + +Will produce the following output: + +```sql +/* stdout */ +SELECT + CAST(JSON_VALUE(json_blob.stringField) as STRING) as string_field + , CAST(JSON_VALUE(json_blob.integerField) as INTEGER) as integer_field + , CAST(JSON_VALUE(json_blob.decimalField) as DECIMAL) as decimal_field + , TO_JSON_STRING(json_blob.exampleGeoJson) as example_geo_json +FROM .myDataset.myTable /**/ SELECT CAST(JSON_VALUE(json_blob.childField1.gender) as STRING) as gender , CAST(JSON_VALUE(json_blob.childField1.latitude) as DECIMAL) as latitude -FROM .. +FROM .myDataset.myTable /**/ SELECT CAST(JSON_VALUE(json_blob.childField2.favoriteFruit) as STRING) as favorite_fruit , CAST(JSON_VALUE(json_blob.childField2.longitude) as DECIMAL) as longitude -FROM .. +FROM .myDataset.myTable /**/ SELECT CAST(JSON_VALUE(json_blob.childWithNestedObject.isNormal) as STRING) as is_normal , TO_JSON_STRING(json_blob.childWithNestedObject.nestedObject) as nested_object -FROM .. +FROM .myDataset.myTable ``` From 714804a34abe0ecc72a9e3a9dd4acfb9ef1c0a24 Mon Sep 17 00:00:00 2001 From: Preston Neal Date: Sun, 27 Aug 2023 21:04:48 -0700 Subject: [PATCH 14/14] bump version and update readme --- README.md | 10 ++++++---- package.json | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f96ec6b..cb77bf3 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,10 @@ This CLI tool reads a JSON file and produces BigQuery compatible SQL views from ## Usage ```bash -npx @nealwp/blobview [options] +npx @nealwp/blobview@latest [options] +``` +```text Arguments: filepath path to valid JSON file @@ -24,14 +26,14 @@ Options: ## Examples: Default output to STDOUT: ```bash -npx @nealwp/blobview@latest ./path/to/file.json +npx @nealwp/blobview ./path/to/file.json ``` Dataset and table as input options: ```bash -npx @nealwp/blobview@latest --dataset=myDataset --table=myTable ./path/to/file.json +npx @nealwp/blobview --dataset=myDataset --table=myTable ./path/to/file.json # shorthand options -npx @nealwp/blobview@latest -d myDataset -t myTable ./path/to/file.json +npx @nealwp/blobview -d myDataset -t myTable ./path/to/file.json ``` Redirect output to file: diff --git a/package.json b/package.json index 5929757..b92f442 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@nealwp/blobview", - "version": "0.1.0", + "version": "0.2.0", "description": "Generate BigQuery SQL views from JSON", "bin": { "@nealwp/blobview": "./index.js"