diff --git a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 253b1953..58b99fa5 100644 --- a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -60,7 +60,10 @@ describe('test create saved search report', () => { test('create report with expected file name', async () => { const hits: Array<{ _source: any }> = []; const client = mockOpenSearchClient(hits); - const { timeCreated, fileName } = await createSavedSearchReport( + const { + timeCreated: _timeCreated, + fileName, + } = await createSavedSearchReport( input, client, mockDateFormat, @@ -909,12 +912,30 @@ describe('test create saved search report', () => { 'geoip.location': { lon: -0.1, lat: 51.5 }, customer_birth_date: '2023-04-26T04:34:32Z', order_date: '2023-04-26T04:34:32Z', - products: { created_on: '2023-04-26T04:34:32Z' }, + products: [ + { + created_on: '2023-04-26T04:34:32Z', + price: 100, + category: 'Electronics', + }, + { + created_on: '2023-05-01T08:22:00Z', + price: 50, + category: 'Books', + }, + ], + customer: { + name: 'John Doe', + address: { city: 'London', postcode: 'SW1A 1AA' }, + }, }, { customer_birth_date: '2023-04-26T04:34:32Z', order_date: '2023-04-26T04:34:32Z', 'products.created_on': '2023-04-26T04:34:32Z', + 'customer.name': 'John Doe', + 'customer.address.city': 'London', + 'customer.address.postcode': 'SW1A 1AA', } ), hit( @@ -923,20 +944,46 @@ describe('test create saved search report', () => { 'geoip.city_name': 'New York', 'geoip.location': { lon: -74, lat: 40.8 }, customer_birth_date: '2023-04-26T04:34:32Z', - order_date: '2023-04-26T04:34:32Z', - products: { created_on: '2023-04-26T04:34:32Z' }, + products: [ + { + created_on: '2023-06-10T14:30:00Z', + price: 150, + category: 'Furniture', + }, + ], + customer: { + name: 'Jane Smith', + address: { city: 'New York', postcode: '10001' }, + }, }, { customer_birth_date: '2023-04-26T04:34:32Z', order_date: '2023-04-26T04:34:32Z', 'products.created_on': '2023-04-26T04:34:32Z', + 'customer.name': 'Jane Smith', + 'customer.address.city': 'New York', + 'customer.address.postcode': '10001', } ), + hit( + { + 'geoip.country_iso_code': 'CA', + 'geoip.city_name': 'Toronto', + 'geoip.location': { lon: -79.38, lat: 43.65 }, + customer: { + name: 'Alice Johnson', + address: { city: 'Toronto', postcode: 'M5H 2N2' }, + }, + }, + {} + ), ]; + const client = mockOpenSearchClient( hits, - '"geoip.country_iso_code", "geoip.city_name", "geoip.location"' + '"geoip.country_iso_code", "geoip.city_name", "geoip.location", "customer.name", "customer.address.city", "customer.address.postcode"' ); + const { dataUrl } = await createSavedSearchReport( input, client, @@ -949,9 +996,10 @@ describe('test create saved search report', () => { ); expect(dataUrl).toEqual( - 'geoip\\.country_iso_code,geoip\\.location\\.lon,geoip\\.location\\.lat,geoip\\.city_name\n' + - 'GB,-0.1,51.5, \n' + - 'US,-74,40.8,New York' + 'geoip\\.country_iso_code,products\\.created_on,products\\.price,products\\.category,geoip\\.location\\.lon,geoip\\.location\\.lat,customer\\.name,customer\\.address\\.city,customer\\.address\\.postcode,geoip\\.city_name\n' + + 'GB,"[""2023-04-26T04:34:32Z"",""2023-05-01T08:22:00Z""]","[100,50]","[""Electronics"",""Books""]",-0.1,51.5,John Doe,London,SW1A 1AA, \n' + + 'US,"[""2023-06-10T14:30:00Z""]",[150],"[""Furniture""]",-74,40.8,Jane Smith,New York,10001,New York\n' + + 'CA, , , ,-79.38,43.65,Alice Johnson,Toronto,M5H 2N2,Toronto' ); }, 20000); @@ -1213,7 +1261,7 @@ test('create report for data set contains null field value', async () => { test('create report for data set with metadata fields', async () => { const metadataFields = { _index: 'nameofindex', _id: 'someid' }; - let hits = [ + const hits = [ hit( { category: 'c1', @@ -1357,7 +1405,7 @@ test('create report with Etc/GMT-2 Timezone', async () => { true, undefined, mockLogger, - "Etc/GMT-2" + 'Etc/GMT-2' ); expect(dataUrl).toEqual( @@ -1451,6 +1499,85 @@ test('create report with empty/one/multiple(list) date values', async () => { ); }, 20000); +test('create report for deeply nested inventory data set with escaped field names', async () => { + const hits = [ + hit( + { + inventory: { + categories: { + subcategories: [ + { + items: [{ price: 100 }, { price: 200 }], + }, + { + items: [{ price: 300 }, { price: 400 }], + }, + ], + }, + }, + }, + { + 'inventory.categories.subcategories.items': `[[{"price":100},{"price":200}],[{"price":300},{"price":400}]]`, + } + ), + hit( + { + inventory: { + categories: { + subcategories: [ + { + items: [{ price: 500 }, { price: 600 }], + }, + { + items: [{ price: 700 }, { price: 800 }], + }, + ], + }, + }, + }, + { + 'inventory.categories.subcategories.items': `[[{"price":500},{"price":600}],[{"price":700},{"price":800}]]`, + } + ), + hit( + { + inventory: { + categories: { + subcategories: [ + { + items: [{ price: 900 }], + }, + ], + }, + }, + }, + { + 'inventory.categories.subcategories.items': `[[{"price":900}]]`, + } + ), + ]; + + const client = mockOpenSearchClient(hits); + + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + '|', + true, + undefined, + mockLogger, + mockTimezone + ); + + expect(dataUrl).toEqual( + 'inventory\\.categories\\.subcategories\\.items\n' + + '"[[{""price"":100},{""price"":200}],[{""price"":300},{""price"":400}]]"\n' + + '"[[{""price"":500},{""price"":600}],[{""price"":700},{""price"":800}]]"\n' + + '"[[{""price"":900}]]"' + ); +}, 20000); + /** * Mock Elasticsearch client and return different mock objects based on endpoint and parameters. */ @@ -1495,7 +1622,9 @@ function mockOpenSearchClient( case 'clearScroll': return null; default: - fail('Fail due to unexpected function call on client', endpoint); + throw new Error( + `Fail due to unexpected function call on client: ${endpoint}` + ); } }); return client; @@ -1573,9 +1702,9 @@ function mockIndexSettings() { `); } -function hit(source_kv: any, fields_kv = {}) { +function hit(sourceKv: any, fieldsKv = {}) { return { - _source: source_kv, - fields: fields_kv, + _source: sourceKv, + fields: fieldsKv, }; } diff --git a/server/routes/utils/dataReportHelpers.ts b/server/routes/utils/dataReportHelpers.ts index 1a64eaee..e71acd6d 100644 --- a/server/routes/utils/dataReportHelpers.ts +++ b/server/routes/utils/dataReportHelpers.ts @@ -235,20 +235,19 @@ export const convertToCSV = async (dataset, csvSeparator) => { return convertedData; }; -function flattenHits(hits, result = {}, prefix = '') { - for (const [key, value] of Object.entries(hits)) { - if (!hits.hasOwnProperty(key)) continue; +function flattenHits(hits: any, result: { [key: string]: any } = {}, prefix = '') { + Object.entries(hits).forEach(([key, value]) => { if ( - value != null && + value !== null && typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length > 0 ) { - flattenHits(value, result, prefix + key + '.'); + flattenHits(value, result, `${prefix}${key}.`); } else { - result[prefix.replace(/^_source\./, '') + key] = value; + result[`${prefix.replace(/^_source\./, '')}${key}`] = value; } - } + }); return result; } @@ -297,18 +296,44 @@ export const convertToExcel = async (dataset: any) => { }; //Return only the selected fields -function traverse(data, keys, result = {}) { +function traverse(data: any, keys: string[], result: { [key: string]: any } = {}) { + // Flatten the data if necessary (ensure all nested fields are at the top level) data = flattenHits(data); - const sourceKeys = Object.keys(data); + keys.forEach((key) => { const value = _.get(data, key, undefined); - if (value !== undefined) result[key] = value; - else { - Object.keys(data) - .filter((sourceKey) => sourceKey.startsWith(key + '.')) - .forEach((sourceKey) => (result[sourceKey] = data[sourceKey])); + + if (value !== undefined) { + result[key] = value; + } else { + const flattenedValues: { [key: string]: any[] } = {}; + + Object.keys(data).forEach((dataKey) => { + if (dataKey.startsWith(key + '.')) { + result[dataKey] = data[dataKey]; + } + const arrayValue = data[dataKey]; + if (Array.isArray(arrayValue)) { + arrayValue.forEach((item) => { + if (typeof item === 'object' && item !== null) { + Object.keys(item).forEach((subKey) => { + const newKey = `${dataKey}.${subKey}`; + if (!flattenedValues[newKey]) { + flattenedValues[newKey] = []; + } + flattenedValues[newKey].push(item[subKey]); + }); + } + }); + } + }); + + Object.keys(flattenedValues).forEach((newKey) => { + result[newKey] = flattenedValues[newKey]; + }); } }); + return result; }