From cadb489311724fc7acc6cf06aeb2ae61e30bdbd1 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Sun, 14 Apr 2024 23:27:43 +0300 Subject: [PATCH 1/2] Make `lz4` module optional Signed-off-by: Levko Kravets --- lib/DBSQLSession.ts | 4 ++-- lib/result/ArrowResultHandler.ts | 9 +++++-- lib/result/CloudFetchResultHandler.ts | 9 +++++-- lib/utils/index.ts | 3 ++- lib/utils/lz4.ts | 16 +++++++++++++ package-lock.json | 34 ++++++++++++++++++++------- package.json | 4 +++- 7 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 lib/utils/lz4.ts diff --git a/lib/DBSQLSession.ts b/lib/DBSQLSession.ts index 9863bc0c..68c82ef1 100644 --- a/lib/DBSQLSession.ts +++ b/lib/DBSQLSession.ts @@ -27,7 +27,7 @@ import IOperation from './contracts/IOperation'; import DBSQLOperation from './DBSQLOperation'; import Status from './dto/Status'; import InfoValue from './dto/InfoValue'; -import { definedOrError } from './utils'; +import { definedOrError, LZ4 } from './utils'; import CloseableCollection from './utils/CloseableCollection'; import { LogLevel } from './contracts/IDBSQLLogger'; import HiveDriverError from './errors/HiveDriverError'; @@ -184,7 +184,7 @@ export default class DBSQLSession implements IDBSQLSession { ...getArrowOptions(clientConfig), canDownloadResult: options.useCloudFetch ?? clientConfig.useCloudFetch, parameters: getQueryParameters(this.sessionHandle, options.namedParameters, options.ordinalParameters), - canDecompressLZ4Result: clientConfig.useLZ4Compression, + canDecompressLZ4Result: clientConfig.useLZ4Compression && Boolean(LZ4), }); const response = await this.handleResponse(operationPromise); const operation = this.createOperation(response); diff --git a/lib/result/ArrowResultHandler.ts b/lib/result/ArrowResultHandler.ts index 601432e8..108f3365 100644 --- a/lib/result/ArrowResultHandler.ts +++ b/lib/result/ArrowResultHandler.ts @@ -1,8 +1,9 @@ -import LZ4 from 'lz4'; import { TGetResultSetMetadataResp, TRowSet } from '../../thrift/TCLIService_types'; +import HiveDriverError from '../errors/HiveDriverError'; import IClientContext from '../contracts/IClientContext'; import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider'; import { ArrowBatch, hiveSchemaToArrowSchema } from './utils'; +import { LZ4 } from '../utils'; export default class ArrowResultHandler implements IResultsProvider { protected readonly context: IClientContext; @@ -24,6 +25,10 @@ export default class ArrowResultHandler implements IResultsProvider // so it's possible to infer Arrow schema from Hive schema ignoring `useArrowNativeTypes` option this.arrowSchema = arrowSchema ?? hiveSchemaToArrowSchema(schema); this.isLZ4Compressed = lz4Compressed ?? false; + + if (this.isLZ4Compressed && !LZ4) { + throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed'); + } } public async hasMore() { @@ -47,7 +52,7 @@ export default class ArrowResultHandler implements IResultsProvider let totalRowCount = 0; rowSet?.arrowBatches?.forEach(({ batch, rowCount }) => { if (batch) { - batches.push(this.isLZ4Compressed ? LZ4.decode(batch) : batch); + batches.push(this.isLZ4Compressed ? LZ4!.decode(batch) : batch); totalRowCount += rowCount.toNumber(true); } }); diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 39ef6f94..c0450aef 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,9 +1,10 @@ -import LZ4 from 'lz4'; import fetch, { RequestInfo, RequestInit, Request } from 'node-fetch'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; +import HiveDriverError from '../errors/HiveDriverError'; import IClientContext from '../contracts/IClientContext'; import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider'; import { ArrowBatch } from './utils'; +import { LZ4 } from '../utils'; export default class CloudFetchResultHandler implements IResultsProvider { protected readonly context: IClientContext; @@ -24,6 +25,10 @@ export default class CloudFetchResultHandler implements IResultsProvider LZ4.decode(buffer)); + batch.batches = batch.batches.map((buffer) => LZ4!.decode(buffer)); } return batch; } diff --git a/lib/utils/index.ts b/lib/utils/index.ts index 4603277a..963f6b05 100644 --- a/lib/utils/index.ts +++ b/lib/utils/index.ts @@ -1,5 +1,6 @@ import definedOrError from './definedOrError'; import buildUserAgentString from './buildUserAgentString'; import formatProgress, { ProgressUpdateTransformer } from './formatProgress'; +import LZ4 from './lz4'; -export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer }; +export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer, LZ4 }; diff --git a/lib/utils/lz4.ts b/lib/utils/lz4.ts new file mode 100644 index 00000000..8186024d --- /dev/null +++ b/lib/utils/lz4.ts @@ -0,0 +1,16 @@ +import type LZ4Namespace from 'lz4'; + +type LZ4Module = typeof LZ4Namespace; + +function tryLoadLZ4Module(): LZ4Module | undefined { + try { + return require('lz4'); // eslint-disable-line global-require + } catch (err) { + const isModuleNotFoundError = err instanceof Error && 'code' in err && err.code === 'MODULE_NOT_FOUND'; + if (!isModuleNotFoundError) { + throw err; + } + } +} + +export default tryLoadLZ4Module(); diff --git a/package-lock.json b/package-lock.json index 3259493a..ce4ee92a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "dependencies": { "apache-arrow": "^13.0.0", "commander": "^9.3.0", - "lz4": "^0.6.5", "node-fetch": "^2.6.12", "node-int64": "^0.4.0", "open": "^8.4.2", @@ -48,6 +47,9 @@ }, "engines": { "node": ">=14.0.0" + }, + "optionalDependencies": { + "lz4": "^0.6.5" } }, "node_modules/@75lb/deep-merge": { @@ -1486,7 +1488,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "optional": true }, "node_modules/basic-ftp": { "version": "5.0.3", @@ -1584,6 +1587,7 @@ "url": "https://feross.org/support" } ], + "optional": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -1941,7 +1945,8 @@ "node_modules/cuint": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", - "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==" + "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==", + "optional": true }, "node_modules/damerau-levenshtein": { "version": "1.0.8", @@ -3335,7 +3340,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "optional": true }, "node_modules/ignore": { "version": "5.2.0", @@ -4093,6 +4099,7 @@ "resolved": "https://registry.npmjs.org/lz4/-/lz4-0.6.5.tgz", "integrity": "sha512-KSZcJU49QZOlJSItaeIU3p8WoAvkTmD9fJqeahQXNu1iQ/kR0/mQLdbrK8JY9MY8f6AhJoMrihp1nu1xDbscSQ==", "hasInstallScript": true, + "optional": true, "dependencies": { "buffer": "^5.2.1", "cuint": "^0.2.2", @@ -4278,7 +4285,8 @@ "node_modules/nan": { "version": "2.18.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==" + "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", + "optional": true }, "node_modules/nanoid": { "version": "3.3.3", @@ -6106,6 +6114,7 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==", + "optional": true, "dependencies": { "cuint": "^0.2.2" } @@ -7294,7 +7303,8 @@ "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "optional": true }, "basic-ftp": { "version": "5.0.3", @@ -7353,6 +7363,7 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "optional": true, "requires": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -7630,7 +7641,8 @@ "cuint": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", - "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==" + "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==", + "optional": true }, "damerau-levenshtein": { "version": "1.0.8", @@ -8644,7 +8656,8 @@ "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "optional": true }, "ignore": { "version": "5.2.0", @@ -9207,6 +9220,7 @@ "version": "0.6.5", "resolved": "https://registry.npmjs.org/lz4/-/lz4-0.6.5.tgz", "integrity": "sha512-KSZcJU49QZOlJSItaeIU3p8WoAvkTmD9fJqeahQXNu1iQ/kR0/mQLdbrK8JY9MY8f6AhJoMrihp1nu1xDbscSQ==", + "optional": true, "requires": { "buffer": "^5.2.1", "cuint": "^0.2.2", @@ -9349,7 +9363,8 @@ "nan": { "version": "2.18.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==" + "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", + "optional": true }, "nanoid": { "version": "3.3.3", @@ -10702,6 +10717,7 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==", + "optional": true, "requires": { "cuint": "^0.2.2" } diff --git a/package.json b/package.json index 4c1b0ec8..87e6833a 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,6 @@ "dependencies": { "apache-arrow": "^13.0.0", "commander": "^9.3.0", - "lz4": "^0.6.5", "node-fetch": "^2.6.12", "node-int64": "^0.4.0", "open": "^8.4.2", @@ -83,5 +82,8 @@ "thrift": "^0.16.0", "uuid": "^9.0.0", "winston": "^3.8.2" + }, + "optionalDependencies": { + "lz4": "^0.6.5" } } From d111da5100958bc955aeb1db1747899009d97285 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Sun, 14 Apr 2024 23:41:48 +0300 Subject: [PATCH 2/2] Fix lint errors Signed-off-by: Levko Kravets --- .eslintrc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.eslintrc b/.eslintrc index 88b52f9b..33499af2 100644 --- a/.eslintrc +++ b/.eslintrc @@ -15,7 +15,14 @@ "@typescript-eslint/no-throw-literal": "off", "no-restricted-syntax": "off", "no-case-declarations": "off", - "max-classes-per-file": "off" + "max-classes-per-file": "off", + "import/no-extraneous-dependencies": [ + "error", + { + "devDependencies": true, + "optionalDependencies": true + } + ] } } ]