[BUG]Csv report generation had missing nested Fields

server/routes/utils/__tests__/savedSearchReportHelper.test.ts

@@ -6,7 +6,7 @@
 import 'regenerator-runtime/runtime';
 import { createSavedSearchReport } from '../savedSearchReportHelper';
 import { reportSchema } from '../../../model';
 import { mockLogger } from '../../../../test/__mocks__/loggerMock';
 import _ from 'lodash';
 
 /**
@@ -52,15 +52,18 @@
 const maxResultSize = 5;
 
 describe('test create saved search report', () => {
   test('create report with valid input', async () => {
     // Check if the assumption of input is up-to-date
     reportSchema.validate(input);
   }, 20000);
 
   test('create report with expected file name', async () => {
     const hits: Array<{ _source: any }> = [];
     const client = mockOpenSearchClient(hits);
-    const { timeCreated, fileName } = await createSavedSearchReport(
+    const {
+      timeCreated: _timeCreated,
+      fileName,
+    } = await createSavedSearchReport(
       input,
       client,
       mockDateFormat,
@@ -103,7 +106,7 @@
   }, 20000);
 
   test('create report for empty data set', async () => {
     const hits: Array<{ _source: any }> = [];
     const client = mockOpenSearchClient(hits);
     const { dataUrl } = await createSavedSearchReport(
       input,
@@ -909,12 +912,30 @@
           'geoip.location': { lon: -0.1, lat: 51.5 },
           customer_birth_date: '2023-04-26T04:34:32Z',
           order_date: '2023-04-26T04:34:32Z',
-          products: { created_on: '2023-04-26T04:34:32Z' },
+          products: [
+            {
+              created_on: '2023-04-26T04:34:32Z',
+              price: 100,
+              category: 'Electronics',
+            },
+            {
+              created_on: '2023-05-01T08:22:00Z',
+              price: 50,
+              category: 'Books',
+            },
+          ],
+          customer: {
+            name: 'John Doe',
+            address: { city: 'London', postcode: 'SW1A 1AA' },
+          },
         },
         {
           customer_birth_date: '2023-04-26T04:34:32Z',
           order_date: '2023-04-26T04:34:32Z',
           'products.created_on': '2023-04-26T04:34:32Z',
+          'customer.name': 'John Doe',
+          'customer.address.city': 'London',
+          'customer.address.postcode': 'SW1A 1AA',
         }
       ),
       hit(
@@ -923,20 +944,46 @@
           'geoip.city_name': 'New York',
           'geoip.location': { lon: -74, lat: 40.8 },
           customer_birth_date: '2023-04-26T04:34:32Z',
-          order_date: '2023-04-26T04:34:32Z',
-          products: { created_on: '2023-04-26T04:34:32Z' },
+          products: [
+            {
+              created_on: '2023-06-10T14:30:00Z',
+              price: 150,
+              category: 'Furniture',
+            },
+          ],
+          customer: {
+            name: 'Jane Smith',
+            address: { city: 'New York', postcode: '10001' },
+          },
         },
         {
           customer_birth_date: '2023-04-26T04:34:32Z',
           order_date: '2023-04-26T04:34:32Z',
           'products.created_on': '2023-04-26T04:34:32Z',
+          'customer.name': 'Jane Smith',
+          'customer.address.city': 'New York',
+          'customer.address.postcode': '10001',
         }
       ),
+      hit(
+        {
+          'geoip.country_iso_code': 'CA',
+          'geoip.city_name': 'Toronto',
+          'geoip.location': { lon: -79.38, lat: 43.65 },
+          customer: {
+            name: 'Alice Johnson',
+            address: { city: 'Toronto', postcode: 'M5H 2N2' },
+          },
+        },
+        {}
+      ),
     ];
+
     const client = mockOpenSearchClient(
       hits,
-      '"geoip.country_iso_code", "geoip.city_name", "geoip.location"'
+      '"geoip.country_iso_code", "geoip.city_name", "geoip.location", "customer.name", "customer.address.city", "customer.address.postcode"'
     );
+
     const { dataUrl } = await createSavedSearchReport(
       input,
       client,
@@ -949,9 +996,10 @@
     );
 
     expect(dataUrl).toEqual(
-      'geoip\\.country_iso_code,geoip\\.location\\.lon,geoip\\.location\\.lat,geoip\\.city_name\n' +
-        'GB,-0.1,51.5, \n' +
-        'US,-74,40.8,New York'
+      'geoip\\.country_iso_code,products\\.created_on,products\\.price,products\\.category,geoip\\.location\\.lon,geoip\\.location\\.lat,customer\\.name,customer\\.address\\.city,customer\\.address\\.postcode,geoip\\.city_name\n' +
+        'GB,"[""2023-04-26T04:34:32Z"",""2023-05-01T08:22:00Z""]","[100,50]","[""Electronics"",""Books""]",-0.1,51.5,John Doe,London,SW1A 1AA, \n' +
+        'US,"[""2023-06-10T14:30:00Z""]",[150],"[""Furniture""]",-74,40.8,Jane Smith,New York,10001,New York\n' +
+        'CA, , , ,-79.38,43.65,Alice Johnson,Toronto,M5H 2N2,Toronto'
     );
   }, 20000);
 
@@ -1213,7 +1261,7 @@
 
 test('create report for data set with metadata fields', async () => {
   const metadataFields = { _index: 'nameofindex', _id: 'someid' };
-  let hits = [
+  const hits = [
     hit(
       {
         category: 'c1',
@@ -1357,7 +1405,7 @@
     true,
     undefined,
     mockLogger,
-    "Etc/GMT-2"
+    'Etc/GMT-2'
   );
 
   expect(dataUrl).toEqual(
@@ -1451,18 +1499,97 @@
   );
 }, 20000);
 
+test('create report for deeply nested inventory data set with escaped field names', async () => {
+  const hits = [
+    hit(
+      {
+        inventory: {
+          categories: {
+            subcategories: [
+              {
+                items: [{ price: 100 }, { price: 200 }],
+              },
+              {
+                items: [{ price: 300 }, { price: 400 }],
+              },
+            ],
+          },
+        },
+      },
+      {
+        'inventory.categories.subcategories.items': `[[{"price":100},{"price":200}],[{"price":300},{"price":400}]]`,
+      }
+    ),
+    hit(
+      {
+        inventory: {
+          categories: {
+            subcategories: [
+              {
+                items: [{ price: 500 }, { price: 600 }],
+              },
+              {
+                items: [{ price: 700 }, { price: 800 }],
+              },
+            ],
+          },
+        },
+      },
+      {
+        'inventory.categories.subcategories.items': `[[{"price":500},{"price":600}],[{"price":700},{"price":800}]]`,
+      }
+    ),
+    hit(
+      {
+        inventory: {
+          categories: {
+            subcategories: [
+              {
+                items: [{ price: 900 }],
+              },
+            ],
+          },
+        },
+      },
+      {
+        'inventory.categories.subcategories.items': `[[{"price":900}]]`,
+      }
+    ),
+  ];
+
+  const client = mockOpenSearchClient(hits);
+
+  const { dataUrl } = await createSavedSearchReport(
+    input,
+    client,
+    mockDateFormat,
+    '|',
+    true,
+    undefined,
+    mockLogger,
+    mockTimezone
+  );
+
+  expect(dataUrl).toEqual(
+    'inventory\\.categories\\.subcategories\\.items\n' +
+      '"[[{""price"":100},{""price"":200}],[{""price"":300},{""price"":400}]]"\n' +
+      '"[[{""price"":500},{""price"":600}],[{""price"":700},{""price"":800}]]"\n' +
+      '"[[{""price"":900}]]"'
+  );
+}, 20000);
+
 /**
  * Mock Elasticsearch client and return different mock objects based on endpoint and parameters.
  */
 function mockOpenSearchClient(
   mockHits: Array<{ _source: any; fields: any }>,
   columns = '"category", "customer_gender"'
 ) {
   let call = 0;
   const client = jest.fn();
   client.callAsInternalUser = jest
     .fn()
     .mockImplementation((endpoint: string, params: any) => {
       switch (endpoint) {
         case 'get':
           return {
@@ -1495,7 +1622,9 @@
         case 'clearScroll':
           return null;
         default:
-          fail('Fail due to unexpected function call on client', endpoint);
+          throw new Error(
+            `Fail due to unexpected function call on client: ${endpoint}`
+          );
       }
     });
   return client;
@@ -1573,9 +1702,9 @@
   `);
 }
 
-function hit(source_kv: any, fields_kv = {}) {
+function hit(sourceKv: any, fieldsKv = {}) {
   return {
-    _source: source_kv,
-    fields: fields_kv,
+    _source: sourceKv,
+    fields: fieldsKv,
   };
 }

server/routes/utils/dataReportHelpers.ts

@@ -235,20 +235,19 @@ export const convertToCSV = async (dataset, csvSeparator) => {
   return convertedData;
 };
 
-function flattenHits(hits, result = {}, prefix = '') {
-  for (const [key, value] of Object.entries(hits)) {
-    if (!hits.hasOwnProperty(key)) continue;
+function flattenHits(hits: any, result: { [key: string]: any } = {}, prefix = '') {
+  Object.entries(hits).forEach(([key, value]) => {
     if (
-      value != null &&
+      value !== null &&
       typeof value === 'object' &&
       !Array.isArray(value) &&
       Object.keys(value).length > 0
     ) {
-      flattenHits(value, result, prefix + key + '.');
+      flattenHits(value, result, `${prefix}${key}.`);
     } else {
-      result[prefix.replace(/^_source\./, '') + key] = value;
+      result[`${prefix.replace(/^_source\./, '')}${key}`] = value;
     }
-  }
+  });
   return result;
 }
 
@@ -297,18 +296,44 @@ export const convertToExcel = async (dataset: any) => {
 };
 
 //Return only the selected fields
-function traverse(data, keys, result = {}) {
+function traverse(data: any, keys: string[], result: { [key: string]: any } = {}) {
+  // Flatten the data if necessary (ensure all nested fields are at the top level)
   data = flattenHits(data);
-  const sourceKeys = Object.keys(data);
+
   keys.forEach((key) => {
     const value = _.get(data, key, undefined);
-    if (value !== undefined) result[key] = value;
-    else {
-      Object.keys(data)
-        .filter((sourceKey) => sourceKey.startsWith(key + '.'))
-        .forEach((sourceKey) => (result[sourceKey] = data[sourceKey]));
+
+    if (value !== undefined) {
+      result[key] = value;
+    } else {
+      const flattenedValues: { [key: string]: any[] } = {};
+
+      Object.keys(data).forEach((dataKey) => {
+        if (dataKey.startsWith(key + '.')) {
+          result[dataKey] = data[dataKey];
+        }
+        const arrayValue = data[dataKey];
+        if (Array.isArray(arrayValue)) {
+          arrayValue.forEach((item) => {
+            if (typeof item === 'object' && item !== null) {
+              Object.keys(item).forEach((subKey) => {
+                const newKey = `${dataKey}.${subKey}`;
+                if (!flattenedValues[newKey]) {
+                  flattenedValues[newKey] = [];
+                }
+                flattenedValues[newKey].push(item[subKey]);
+              });
+            }
+          });
+        }
+      });
+
+      Object.keys(flattenedValues).forEach((newKey) => {
+        result[newKey] = flattenedValues[newKey];
+      });
     }
   });
+
   return result;
 }