Skip to content

Commit

Permalink
[BUG]Csv report generation had missing nested Fields (#502) (#503)
Browse files Browse the repository at this point in the history 
(cherry picked from commit c2ea8be)

Signed-off-by: sumukhswamy <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@opensearch-trigger-bot @github-actions
opensearch-trigger-bot[bot] and github-actions[bot] authored Jan 28, 2025
1 parent c9913d0 commit 6cd6bdf
Show file tree
Hide file tree
Showing 2 changed files with 182 additions and 28 deletions.
157 changes: 143 additions & 14 deletions server/routes/utils/__tests__/savedSearchReportHelper.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,10 @@ describe('test create saved search report', () => {
test('create report with expected file name', async () => {
const hits: Array<{ _source: any }> = [];
const client = mockOpenSearchClient(hits);
const { timeCreated, fileName } = await createSavedSearchReport(
const {
timeCreated: _timeCreated,
fileName,
} = await createSavedSearchReport(
input,
client,
mockDateFormat,
Expand Down Expand Up @@ -909,12 +912,30 @@ describe('test create saved search report', () => {
'geoip.location': { lon: -0.1, lat: 51.5 },
customer_birth_date: '2023-04-26T04:34:32Z',
order_date: '2023-04-26T04:34:32Z',
products: { created_on: '2023-04-26T04:34:32Z' },
products: [
{
created_on: '2023-04-26T04:34:32Z',
price: 100,
category: 'Electronics',
},
{
created_on: '2023-05-01T08:22:00Z',
price: 50,
category: 'Books',
},
],
customer: {
name: 'John Doe',
address: { city: 'London', postcode: 'SW1A 1AA' },
},
},
{
customer_birth_date: '2023-04-26T04:34:32Z',
order_date: '2023-04-26T04:34:32Z',
'products.created_on': '2023-04-26T04:34:32Z',
'customer.name': 'John Doe',
'customer.address.city': 'London',
'customer.address.postcode': 'SW1A 1AA',
}
),
hit(
Expand All @@ -923,20 +944,46 @@ describe('test create saved search report', () => {
'geoip.city_name': 'New York',
'geoip.location': { lon: -74, lat: 40.8 },
customer_birth_date: '2023-04-26T04:34:32Z',
order_date: '2023-04-26T04:34:32Z',
products: { created_on: '2023-04-26T04:34:32Z' },
products: [
{
created_on: '2023-06-10T14:30:00Z',
price: 150,
category: 'Furniture',
},
],
customer: {
name: 'Jane Smith',
address: { city: 'New York', postcode: '10001' },
},
},
{
customer_birth_date: '2023-04-26T04:34:32Z',
order_date: '2023-04-26T04:34:32Z',
'products.created_on': '2023-04-26T04:34:32Z',
'customer.name': 'Jane Smith',
'customer.address.city': 'New York',
'customer.address.postcode': '10001',
}
),
hit(
{
'geoip.country_iso_code': 'CA',
'geoip.city_name': 'Toronto',
'geoip.location': { lon: -79.38, lat: 43.65 },
customer: {
name: 'Alice Johnson',
address: { city: 'Toronto', postcode: 'M5H 2N2' },
},
},
{}
),
];

const client = mockOpenSearchClient(
hits,
'"geoip.country_iso_code", "geoip.city_name", "geoip.location"'
'"geoip.country_iso_code", "geoip.city_name", "geoip.location", "customer.name", "customer.address.city", "customer.address.postcode"'
);

const { dataUrl } = await createSavedSearchReport(
input,
client,
Expand All @@ -949,9 +996,10 @@ describe('test create saved search report', () => {
);

expect(dataUrl).toEqual(
'geoip\\.country_iso_code,geoip\\.location\\.lon,geoip\\.location\\.lat,geoip\\.city_name\n' +
'GB,-0.1,51.5, \n' +
'US,-74,40.8,New York'
'geoip\\.country_iso_code,products\\.created_on,products\\.price,products\\.category,geoip\\.location\\.lon,geoip\\.location\\.lat,customer\\.name,customer\\.address\\.city,customer\\.address\\.postcode,geoip\\.city_name\n' +
'GB,"[""2023-04-26T04:34:32Z"",""2023-05-01T08:22:00Z""]","[100,50]","[""Electronics"",""Books""]",-0.1,51.5,John Doe,London,SW1A 1AA, \n' +
'US,"[""2023-06-10T14:30:00Z""]",[150],"[""Furniture""]",-74,40.8,Jane Smith,New York,10001,New York\n' +
'CA, , , ,-79.38,43.65,Alice Johnson,Toronto,M5H 2N2,Toronto'
);
}, 20000);

Expand Down Expand Up @@ -1213,7 +1261,7 @@ test('create report for data set contains null field value', async () => {

test('create report for data set with metadata fields', async () => {
const metadataFields = { _index: 'nameofindex', _id: 'someid' };
let hits = [
const hits = [
hit(
{
category: 'c1',
Expand Down Expand Up @@ -1357,7 +1405,7 @@ test('create report with Etc/GMT-2 Timezone', async () => {
true,
undefined,
mockLogger,
"Etc/GMT-2"
'Etc/GMT-2'
);

expect(dataUrl).toEqual(
Expand Down Expand Up @@ -1451,6 +1499,85 @@ test('create report with empty/one/multiple(list) date values', async () => {
);
}, 20000);

test('create report for deeply nested inventory data set with escaped field names', async () => {
const hits = [
hit(
{
inventory: {
categories: {
subcategories: [
{
items: [{ price: 100 }, { price: 200 }],
},
{
items: [{ price: 300 }, { price: 400 }],
},
],
},
},
},
{
'inventory.categories.subcategories.items': `[[{"price":100},{"price":200}],[{"price":300},{"price":400}]]`,
}
),
hit(
{
inventory: {
categories: {
subcategories: [
{
items: [{ price: 500 }, { price: 600 }],
},
{
items: [{ price: 700 }, { price: 800 }],
},
],
},
},
},
{
'inventory.categories.subcategories.items': `[[{"price":500},{"price":600}],[{"price":700},{"price":800}]]`,
}
),
hit(
{
inventory: {
categories: {
subcategories: [
{
items: [{ price: 900 }],
},
],
},
},
},
{
'inventory.categories.subcategories.items': `[[{"price":900}]]`,
}
),
];

const client = mockOpenSearchClient(hits);

const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
'|',
true,
undefined,
mockLogger,
mockTimezone
);

expect(dataUrl).toEqual(
'inventory\\.categories\\.subcategories\\.items\n' +
'"[[{""price"":100},{""price"":200}],[{""price"":300},{""price"":400}]]"\n' +
'"[[{""price"":500},{""price"":600}],[{""price"":700},{""price"":800}]]"\n' +
'"[[{""price"":900}]]"'
);
}, 20000);

/**
* Mock Elasticsearch client and return different mock objects based on endpoint and parameters.
*/
Expand Down Expand Up @@ -1495,7 +1622,9 @@ function mockOpenSearchClient(
case 'clearScroll':
return null;
default:
fail('Fail due to unexpected function call on client', endpoint);
throw new Error(
`Fail due to unexpected function call on client: ${endpoint}`
);
}
});
return client;
Expand Down Expand Up @@ -1573,9 +1702,9 @@ function mockIndexSettings() {
`);
}

function hit(source_kv: any, fields_kv = {}) {
function hit(sourceKv: any, fieldsKv = {}) {
return {
_source: source_kv,
fields: fields_kv,
_source: sourceKv,
fields: fieldsKv,
};
}
53 changes: 39 additions & 14 deletions server/routes/utils/dataReportHelpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,20 +235,19 @@ export const convertToCSV = async (dataset, csvSeparator) => {
return convertedData;
};

function flattenHits(hits, result = {}, prefix = '') {
for (const [key, value] of Object.entries(hits)) {
if (!hits.hasOwnProperty(key)) continue;
function flattenHits(hits: any, result: { [key: string]: any } = {}, prefix = '') {
Object.entries(hits).forEach(([key, value]) => {
if (
value != null &&
value !== null &&
typeof value === 'object' &&
!Array.isArray(value) &&
Object.keys(value).length > 0
) {
flattenHits(value, result, prefix + key + '.');
flattenHits(value, result, `${prefix}${key}.`);
} else {
result[prefix.replace(/^_source\./, '') + key] = value;
result[`${prefix.replace(/^_source\./, '')}${key}`] = value;
}
}
});
return result;
}

Expand Down Expand Up @@ -297,18 +296,44 @@ export const convertToExcel = async (dataset: any) => {
};

//Return only the selected fields
function traverse(data, keys, result = {}) {
function traverse(data: any, keys: string[], result: { [key: string]: any } = {}) {
// Flatten the data if necessary (ensure all nested fields are at the top level)
data = flattenHits(data);
const sourceKeys = Object.keys(data);

keys.forEach((key) => {
const value = _.get(data, key, undefined);
if (value !== undefined) result[key] = value;
else {
Object.keys(data)
.filter((sourceKey) => sourceKey.startsWith(key + '.'))
.forEach((sourceKey) => (result[sourceKey] = data[sourceKey]));

if (value !== undefined) {
result[key] = value;
} else {
const flattenedValues: { [key: string]: any[] } = {};

Object.keys(data).forEach((dataKey) => {
if (dataKey.startsWith(key + '.')) {
result[dataKey] = data[dataKey];
}
const arrayValue = data[dataKey];
if (Array.isArray(arrayValue)) {
arrayValue.forEach((item) => {
if (typeof item === 'object' && item !== null) {
Object.keys(item).forEach((subKey) => {
const newKey = `${dataKey}.${subKey}`;
if (!flattenedValues[newKey]) {
flattenedValues[newKey] = [];
}
flattenedValues[newKey].push(item[subKey]);
});
}
});
}
});

Object.keys(flattenedValues).forEach((newKey) => {
result[newKey] = flattenedValues[newKey];
});
}
});

return result;
}

Expand Down

0 comments on commit 6cd6bdf

Please sign in to comment.