diff --git a/asset/asset.json b/asset/asset.json index bc6e61e2..5eadb930 100644 --- a/asset/asset.json +++ b/asset/asset.json @@ -1,5 +1,5 @@ { "name": "elasticsearch", - "version": "4.0.6", + "version": "4.0.7", "minimum_teraslice_version": "2.0.0" } diff --git a/asset/package.json b/asset/package.json index 9cc2334e..7ba11164 100644 --- a/asset/package.json +++ b/asset/package.json @@ -1,7 +1,7 @@ { "name": "asset", "displayName": "Asset", - "version": "4.0.6", + "version": "4.0.7", "private": true, "description": "", "license": "MIT", @@ -20,7 +20,7 @@ "dependencies": { "@terascope/data-mate": "~1.7.2", "@terascope/elasticsearch-api": "~4.7.1", - "@terascope/elasticsearch-asset-apis": "~1.0.5", + "@terascope/elasticsearch-asset-apis": "~1.0.6", "@terascope/job-components": "~1.9.1", "@terascope/teraslice-state-storage": "~1.7.1", "@terascope/utils": "~1.7.1", diff --git a/asset/src/elasticsearch_reader_api/schema.ts b/asset/src/elasticsearch_reader_api/schema.ts index a4df94a8..fbab3f03 100644 --- a/asset/src/elasticsearch_reader_api/schema.ts +++ b/asset/src/elasticsearch_reader_api/schema.ts @@ -1,7 +1,7 @@ import { ConvictSchema, AnyObject, ValidatedJobConfig, toNumber, isString, isNumber, getTypeOf, - isNotNil, has + isNotNil, has, isKey } from '@terascope/job-components'; import elasticAPI from '@terascope/elasticsearch-api'; import moment from 'moment'; @@ -151,7 +151,7 @@ export const schema = { ms: 'ms' }; if (!isString(val)) throw new Error(`Invalid parameter time_resolution, it must be of type string, was given ${getTypeOf(val)}`); - if (!obj[val]) throw new Error('Invalid time_resolution, must be set in either "s"[seconds] or "ms"[milliseconds]'); + if (!isKey(obj, val)) throw new Error('Invalid time_resolution, must be set in either "s"[seconds] or "ms"[milliseconds]'); return obj[val]; } @@ -193,7 +193,7 @@ export const schema = { if (val) { const options = { asc: true, desc: true }; if (typeof val !== 'string') throw new Error('Invalid geo_sort_order parameter, must be a string IF specified'); - if (!options[val]) throw new Error('If geo_sort_order is specified it must be either "asc" or "desc"'); + if (!isKey(options, val)) throw new Error('If geo_sort_order is specified it must be either "asc" or "desc"'); } } }, diff --git a/package.json b/package.json index feadc911..0b8ac340 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-assets", "displayName": "Elasticsearch Assets", - "version": "4.0.6", + "version": "4.0.7", "private": true, "description": "bundle of processors for teraslice", "homepage": "https://github.com/terascope/elasticsearch-assets#readme", @@ -46,7 +46,7 @@ "devDependencies": { "@terascope/data-types": "~1.7.1", "@terascope/elasticsearch-api": "~4.7.1", - "@terascope/elasticsearch-asset-apis": "~1.0.5", + "@terascope/elasticsearch-asset-apis": "~1.0.6", "@terascope/eslint-config": "~1.1.4", "@terascope/job-components": "~1.9.1", "@terascope/scripts": "~1.9.0", @@ -65,8 +65,9 @@ "node-notifier": "~10.0.1", "teraslice-test-harness": "~1.3.1", "ts-jest": "~29.2.5", - "typescript": "~5.2.2" + "typescript": "~5.7.3" }, + "packageManager": "yarn@4.6.0", "engines": { "node": ">=18.0.0", "yarn": ">=1.22.19" @@ -88,6 +89,5 @@ "npm": { "registry": "https://registry.npmjs.org/" } - }, - "packageManager": "yarn@4.6.0" + } } diff --git a/packages/elasticsearch-asset-apis/package.json b/packages/elasticsearch-asset-apis/package.json index 2a4d173a..4824b796 100644 --- a/packages/elasticsearch-asset-apis/package.json +++ b/packages/elasticsearch-asset-apis/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/elasticsearch-asset-apis", "displayName": "Elasticsearch Asset Apis", - "version": "1.0.5", + "version": "1.0.6", "description": "Elasticsearch reader and sender apis", "homepage": "https://github.com/terascope/elasticsearch-assets", "repository": "git@github.com:terascope/elasticsearch-assets.git", diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts index 858f127b..00784f5c 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts @@ -1,5 +1,5 @@ import elasticAPI from '@terascope/elasticsearch-api'; -import { isNil, isPlainObject, isString } from '@terascope/utils'; +import { isKey, isNil, isPlainObject, isString } from '@terascope/utils'; import { ElasticsearchBulkSender } from './ElasticsearchBulkSender.js'; import { ElasticsearchSenderConfig } from './interfaces.js'; @@ -33,7 +33,7 @@ function validateConfig(input: unknown): ElasticsearchSenderConfig { const actionSet = new Set(); // only one of these should be set to true at a time ['delete', 'create', 'update', 'index', 'upsert'].forEach((key: string) => { - if (config[key] === true) actionSet.add(key); + if (isKey(config, key) && config[key] === true) actionSet.add(key); if (actionSet.size > 1) { const actions = Array.from(actionSet).join(', '); const msg = `Invalid parameters, only one of "${actions}" may be set at a time`; diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts index 190f4364..9b14454c 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts @@ -4,9 +4,13 @@ import { getTypeOf, Logger, isSimpleObject, isNumber, isValidDate, isFunction, isString, isWildCardString, matchWildcard, - pRetry, toIntegerOrThrow, + pRetry, toIntegerOrThrow, isKey, } from '@terascope/utils'; -import { ClientParams, ClientResponse } from '@terascope/types'; +import { + ClientParams, ClientResponse, + IndicesIndexSettings, + IndicesIndexStatePrefixedSettings +} from '@terascope/types'; import { DataFrame } from '@terascope/data-mate'; import { DataTypeConfig } from '@terascope/data-types'; import moment from 'moment'; @@ -663,7 +667,7 @@ export class ElasticsearchReaderAPI { // we have a date, parse and return it if (date) return parseDate(date); // we are in auto, so we determine each part - const sortObj = {}; + const sortObj: Record = {}; const sortOrder = order === 'start' ? 'asc' : 'desc'; sortObj[this.config.date_field_name] = { order: sortOrder }; @@ -710,13 +714,40 @@ export class ElasticsearchReaderAPI { return this.client.getSettings(index); } + /** + * Typeguard to differentiate IndicesIndexSettings + * from IndicesIndexStatePrefixedSettings + */ + private _isIndicesIndexStatePrefixedSettings( + input: unknown + ): input is IndicesIndexStatePrefixedSettings { + if (!isObject(input)) return false; + if (isKey(input as object, 'index')) return true; + + return false; + } + + private _getMaxResultWindowFromSettings( + settings: IndicesIndexSettings | IndicesIndexStatePrefixedSettings | undefined + ) { + const window = 'index.max_result_window'; + let windowSize; + if (settings) { + if (!this._isIndicesIndexStatePrefixedSettings(settings)) { + windowSize = settings[window]; + } else { + windowSize = settings.index[window]; + } + } + return windowSize; + } + /** * This used verify the index.max_result_window size * will be big enough to fix the within the requested * slice size */ async getWindowSize(): Promise { - const window = 'index.max_result_window'; const { index } = this.config; const settings = await this.getSettings(index); @@ -724,8 +755,9 @@ export class ElasticsearchReaderAPI { for (const [key, configs] of Object.entries(settings)) { if (matcher(key)) { - const defaultPath = configs.defaults![window]; - const configPath = configs.settings![window]; + const defaultPath = this._getMaxResultWindowFromSettings(configs.defaults); + const configPath = this._getMaxResultWindowFromSettings(configs.settings); + // config goes first as it overrides an defaults if (configPath) return toIntegerOrThrow(configPath); if (defaultPath) return toIntegerOrThrow(defaultPath); diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts index 0078e162..d38b396e 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts @@ -2,6 +2,7 @@ import tls from 'tls'; import { Logger, TSError, get, isNil, AnyObject, withoutNil, DataEntity, + isKey, } from '@terascope/utils'; import { ClientParams, ClientResponse } from '@terascope/types'; import { DataTypeConfig } from '@terascope/data-types'; @@ -159,9 +160,9 @@ export class SpacesReaderClient implements ReaderClient { if (mustArray) { mustArray.forEach((queryAction) => { for (const [key, qConfig] of Object.entries(queryAction)) { - const queryFn = queryOptions[key]; - if (queryFn) { - let queryStr = queryFn(qConfig); + if (isKey(queryOptions, key) && queryOptions[key]) { + const queryFn = queryOptions[key]; + let queryStr = queryFn(qConfig as Record); if (key !== 'range') queryStr = `(${queryStr})`; if (luceneQuery.length) { diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/WindowState.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/WindowState.ts index 8027f66c..c590a2d8 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/WindowState.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/WindowState.ts @@ -9,16 +9,16 @@ interface WindowMeta { * This is used track the slicer state when running persistent mode */ export class WindowState { - _windowState: Record = {}; + _windowState = new Map(); constructor(numOfSlicers: number) { times(numOfSlicers, (id) => { - this._windowState[id] = { hasCalled: false, canRestart: false }; + this._windowState.set(id, { hasCalled: false, canRestart: false }); }); } private _checkState(value: boolean) { - return Object.values(this._windowState).every((meta) => meta.hasCalled === value); + return Array.from(this._windowState.values()).every((meta) => meta.hasCalled === value); } /** @@ -28,16 +28,19 @@ export class WindowState { * processing */ checkin(id: number): boolean { - const meta = this._windowState[id]; + const meta = this._windowState.get(id); + if (!meta) { + throw new Error(`Window metadata for id ${id} is not defined`); + } let bool = false; meta.hasCalled = true; const allDone = this._checkState(true); if (allDone) { - for (const key of Object.keys(this._windowState)) { - this._windowState[key].canRestart = true; - this._windowState[key].hasCalled = false; + for (const value of this._windowState.values()) { + value.canRestart = true; + value.hasCalled = false; } } diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts index 8f075c5b..e332b32a 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts @@ -1,4 +1,4 @@ -import { times, toIntegerOrThrow } from '@terascope/utils'; +import { isKey, times, toIntegerOrThrow } from '@terascope/utils'; import moment from 'moment'; import fs from 'node:fs'; // @ts-expect-error @@ -11,7 +11,7 @@ import { } from '../interfaces.js'; export function dateOptions(value: string): moment.unitOfTime.Base { - const options = { + const options: Record = { year: 'y', years: 'y', y: 'y', @@ -46,7 +46,7 @@ export function dateOptions(value: string): moment.unitOfTime.Base { ms: 'ms' }; - if (options[value]) { + if (isKey(options, value)) { return options[value]; } @@ -120,7 +120,7 @@ export function existsSync(filename: string): boolean { } } -export function getMilliseconds(interval: any[]): number { +export function getMilliseconds(interval: [number, string]): number { const conversions = { d: 86400000, h: 3600000, @@ -129,7 +129,7 @@ export function getMilliseconds(interval: any[]): number { ms: 1 }; - return interval[0] * conversions[interval[1]]; + return interval[0] * conversions[interval[1] as keyof typeof conversions]; } export function parseDate(date: string): moment.Moment { diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts index 6456b345..a5b2d9d3 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts @@ -1,4 +1,4 @@ -import { AnyObject, GeoPoint, ClientParams } from '@terascope/types'; +import { AnyObject, GeoPoint, ClientParams, GeoQuery } from '@terascope/types'; import { isString, parseGeoPoint } from '@terascope/utils'; import { ESReaderOptions, ReaderSlice } from './interfaces.js'; @@ -36,7 +36,7 @@ function _buildRangeQuery( }; // is a range type query if (params.start && params.end) { - const dateObj = {}; + const dateObj: Record = {}; const { date_field_name: dateFieldName } = opConfig; dateObj[dateFieldName] = { gte: params.start, @@ -155,8 +155,8 @@ export function geoSearch(opConfig: ESReaderOptions): AnyObject { function createGeoSortQuery(location: GeoPoint) { const sortedSearch: AnyObject = { _geo_distance: {} }; sortedSearch._geo_distance[opConfig.geo_field as string] = { - lat: location[0], - lon: location[1], + lat: location.lat, + lon: location.lon, }; sortedSearch._geo_distance.order = geoSortOrder; sortedSearch._geo_distance.unit = geoSortUnit; @@ -174,18 +174,18 @@ export function geoSearch(opConfig: ESReaderOptions): AnyObject { const topLeft = parseGeoPoint(geoBoxTopLeft); const bottomRight = parseGeoPoint(geoBoxBottomRight as string); - const searchQuery = { + const searchQuery: GeoQuery = { geo_bounding_box: {}, }; - searchQuery.geo_bounding_box[opConfig.geo_field as string] = { + searchQuery.geo_bounding_box![opConfig.geo_field as string] = { top_left: { - lat: topLeft[0], - lon: topLeft[1], + lat: topLeft.lat, + lon: topLeft.lon, }, bottom_right: { - lat: bottomRight[0], - lon: bottomRight[1], + lat: bottomRight.lat, + lon: bottomRight.lon, }, }; @@ -200,15 +200,15 @@ export function geoSearch(opConfig: ESReaderOptions): AnyObject { if (geoDistance) { const location = parseGeoPoint(geoPoint as string); - const searchQuery = { + const searchQuery: GeoQuery = { geo_distance: { distance: geoDistance, }, }; - searchQuery.geo_distance[opConfig.geo_field as string] = { - lat: location[0], - lon: location[1], + searchQuery.geo_distance![opConfig.geo_field as string] = { + lat: location.lat, + lon: location.lon, }; queryResults.query = searchQuery; diff --git a/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts b/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts index 7178379d..6fbbea28 100644 --- a/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts +++ b/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts @@ -1,6 +1,5 @@ import 'jest-extended'; import { debugLogger, AnyObject, DataEntity } from '@terascope/utils'; -import { WorkerTestHarness } from 'teraslice-test-harness'; import { isOpensearch2, isElasticsearch8 } from 'elasticsearch-store'; import elasticAPI from '@terascope/elasticsearch-api'; import { @@ -15,7 +14,6 @@ describe('elasticsearch bulk sender module', () => { const senderIndex = `${TEST_INDEX_PREFIX}_sender_api_`; let apiClient: elasticAPI.Client; - let harness: WorkerTestHarness; let client: any; let type: string | undefined; @@ -40,10 +38,6 @@ describe('elasticsearch bulk sender module', () => { await cleanupIndex(client, `${senderIndex}*`); }); - afterEach(async () => { - if (harness) await harness.shutdown(); - }); - function createSender(config: AnyObject = {}) { const senderConfig = Object.assign( {}, diff --git a/packages/elasticsearch-asset-apis/test/window_state-spec.ts b/packages/elasticsearch-asset-apis/test/window_state-spec.ts index 6ed140ea..2d2e6690 100644 --- a/packages/elasticsearch-asset-apis/test/window_state-spec.ts +++ b/packages/elasticsearch-asset-apis/test/window_state-spec.ts @@ -6,7 +6,7 @@ describe('WindowState', () => { const numOfSlicers = 3; const state = new WindowState(numOfSlicers); - expect(Object.keys(state._windowState)).toBeArrayOfSize(numOfSlicers); + expect(state._windowState.size).toBe(numOfSlicers); expect(state.checkin).toBeFunction(); }); diff --git a/tsconfig.json b/tsconfig.json index f4507b74..628c8344 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,6 @@ "esModuleInterop": true, "resolveJsonModule": true, "forceConsistentCasingInFileNames": true, - "suppressImplicitAnyIndexErrors": true, "composite": true, "declaration": true, "declarationMap": true, diff --git a/yarn.lock b/yarn.lock index ea457be9..6255d4b3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1171,7 +1171,7 @@ __metadata: languageName: node linkType: hard -"@terascope/elasticsearch-asset-apis@npm:~1.0.5, @terascope/elasticsearch-asset-apis@workspace:packages/elasticsearch-asset-apis": +"@terascope/elasticsearch-asset-apis@npm:~1.0.6, @terascope/elasticsearch-asset-apis@workspace:packages/elasticsearch-asset-apis": version: 0.0.0-use.local resolution: "@terascope/elasticsearch-asset-apis@workspace:packages/elasticsearch-asset-apis" dependencies: @@ -2426,7 +2426,7 @@ __metadata: dependencies: "@terascope/data-mate": "npm:~1.7.2" "@terascope/elasticsearch-api": "npm:~4.7.1" - "@terascope/elasticsearch-asset-apis": "npm:~1.0.5" + "@terascope/elasticsearch-asset-apis": "npm:~1.0.6" "@terascope/job-components": "npm:~1.9.1" "@terascope/teraslice-state-storage": "npm:~1.7.1" "@terascope/utils": "npm:~1.7.1" @@ -3457,7 +3457,7 @@ __metadata: dependencies: "@terascope/data-types": "npm:~1.7.1" "@terascope/elasticsearch-api": "npm:~4.7.1" - "@terascope/elasticsearch-asset-apis": "npm:~1.0.5" + "@terascope/elasticsearch-asset-apis": "npm:~1.0.6" "@terascope/eslint-config": "npm:~1.1.4" "@terascope/job-components": "npm:~1.9.1" "@terascope/scripts": "npm:~1.9.0" @@ -3476,7 +3476,7 @@ __metadata: node-notifier: "npm:~10.0.1" teraslice-test-harness: "npm:~1.3.1" ts-jest: "npm:~29.2.5" - typescript: "npm:~5.2.2" + typescript: "npm:~5.7.3" languageName: unknown linkType: soft @@ -9093,17 +9093,7 @@ __metadata: languageName: node linkType: hard -"typescript@npm:~5.2.2": - version: 5.2.2 - resolution: "typescript@npm:5.2.2" - bin: - tsc: bin/tsc - tsserver: bin/tsserver - checksum: 10c0/91ae3e6193d0ddb8656d4c418a033f0f75dec5e077ebbc2bd6d76439b93f35683936ee1bdc0e9cf94ec76863aa49f27159b5788219b50e1cd0cd6d110aa34b07 - languageName: node - linkType: hard - -"typescript@npm:~5.7.2": +"typescript@npm:~5.7.2, typescript@npm:~5.7.3": version: 5.7.3 resolution: "typescript@npm:5.7.3" bin: @@ -9113,17 +9103,7 @@ __metadata: languageName: node linkType: hard -"typescript@patch:typescript@npm%3A~5.2.2#optional!builtin": - version: 5.2.2 - resolution: "typescript@patch:typescript@npm%3A5.2.2#optional!builtin::version=5.2.2&hash=f3b441" - bin: - tsc: bin/tsc - tsserver: bin/tsserver - checksum: 10c0/062c1cee1990e6b9419ce8a55162b8dc917eb87f807e4de0327dbc1c2fa4e5f61bc0dd4e034d38ff541d1ed0479b53bcee8e4de3a4075c51a1724eb6216cb6f5 - languageName: node - linkType: hard - -"typescript@patch:typescript@npm%3A~5.7.2#optional!builtin": +"typescript@patch:typescript@npm%3A~5.7.2#optional!builtin, typescript@patch:typescript@npm%3A~5.7.3#optional!builtin": version: 5.7.3 resolution: "typescript@patch:typescript@npm%3A5.7.3#optional!builtin::version=5.7.3&hash=5786d5" bin: