Skip to content

Commit

Permalink
[backend] Improve prepare log metadata to clean inside error attributes
Browse files Browse the repository at this point in the history
  • Loading branch information
@richard-julien
richard-julien committed Jan 13, 2025
1 parent 008cc21 commit 8997343
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 148 deletions.
31 changes: 22 additions & 9 deletions opencti-platform/opencti-graphql/src/config/conf.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ const appLogLevel = nconf.get('app:app_logs:logs_level');
const appLogFileTransport = booleanConf('app:app_logs:logs_files', true);
const appLogConsoleTransport = booleanConf('app:app_logs:logs_console', true);
export const appLogLevelMaxDepthSize = nconf.get('app:app_logs:control:max_depth_size') ?? 5;
export const appLogLevelMaxDepthKeys = nconf.get('app:app_logs:control:max_depth_keys') ?? 20;
export const appLogLevelMaxDepthKeys = nconf.get('app:app_logs:control:max_depth_keys') ?? 30;
export const appLogLevelMaxArraySize = nconf.get('app:app_logs:control:max_array_size') ?? 50;
export const appLogLevelMaxStringSize = nconf.get('app:app_logs:control:max_string_size') ?? 5000;
export const appLogExtendedErrors = booleanConf('app:app_logs:extended_error_message', false);
Expand All @@ -108,11 +108,11 @@ export const extendedErrors = (metaExtension) => {
}
return {};
};
const convertErrorObject = (error) => {
const convertErrorObject = (error, acc, current_depth) => {
if (error instanceof GraphQLError) {
const extensions = error.extensions ?? {};
const extensionsData = extensions.data ?? {};
const { ...attributes } = extensionsData;
const attributes = prepareLogMetadataComplexityWrapper(extensionsData, acc, current_depth);
return { name: extensions.code ?? error.name, code: extensions.code, message: error.message, stack: error.stack, attributes };
}
if (error instanceof Error) {
Expand All @@ -121,25 +121,34 @@ const convertErrorObject = (error) => {
return error;
};
const prepareLogMetadataComplexityWrapper = (obj, acc, current_depth = 0) => {
const noMaxDepth = current_depth < appLogLevelMaxDepthSize;
const noMaxKeys = acc.current_nb_key < appLogLevelMaxDepthKeys;
const isNotAKeyFunction = typeof obj !== 'function';
if (obj !== null && noMaxDepth && noMaxKeys && isNotAKeyFunction) {
const maxDepth = current_depth > appLogLevelMaxDepthSize;
const maxKeys = acc.current_nb_key > appLogLevelMaxDepthKeys;
const isAKeyFunction = typeof obj === 'function';
if (obj !== null) {
// If complexity is too much or function found.
// return null value
if (maxDepth || maxKeys || isAKeyFunction) {
return null;
}
// If array, try to limit the number of elements
if (Array.isArray(obj)) {
// Create a new array with a limited size
const limitedArray = obj.slice(0, appLogLevelMaxArraySize);
// Recursively process each item in the truncated array
const processedArray = [];
for (let i = 0; i < limitedArray.length; i += 1) {
processedArray[i] = prepareLogMetadataComplexityWrapper(limitedArray[i], acc, current_depth);
const cleanItem = prepareLogMetadataComplexityWrapper(limitedArray[i], acc, current_depth);
if (cleanItem) {
processedArray[i] = cleanItem;
}
}
return processedArray;
}
if (typeof obj === 'string' && obj.length > appLogLevelMaxStringSize) {
return `${obj.substring(0, appLogLevelMaxStringSize - 3)}...`;
}
if (typeof obj === 'object') {
const workingObject = convertErrorObject(obj);
const workingObject = convertErrorObject(obj, acc, current_depth);
// Create a new object to hold the processed properties
const limitedObject = {};
const keys = Object.keys(workingObject); // Get the keys of the object
Expand All @@ -148,6 +157,10 @@ const prepareLogMetadataComplexityWrapper = (obj, acc, current_depth = 0) => {
acc.current_nb_key += 1;
const key = keys[i];
limitedObject[key] = prepareLogMetadataComplexityWrapper(workingObject[key], acc, newDepth);
// If data is null, remove the key
if (!limitedObject[key]) {
delete limitedObject[key];
}
}
return limitedObject;
}
Expand Down
207 changes: 68 additions & 139 deletions opencti-platform/opencti-graphql/tests/01-unit/utils/logger-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,144 +34,73 @@ const CLASSIC_OBJECT = {
timestamp: '2025-01-09T20:57:05.422Z',
version: '6.4.6'
};
const CATEGORY_TO_LIMIT = [
'Item 1',
'Item 2',
'Item 3',
'Item 4',
'Item 5',
'Item 6',
'Item 7',
'Item 8',
'Item 9',
'Item 10',
'Item 11',
'Item 12',
'Item 13',
'Item 14',
'Item 15',
'Item 16',
'Item 17',
'Item 18',
'Item 19',
'Item 20',
'Item 21',
'Item 22',
'Item 23',
'Item 24',
'Item 25',
'Item 26',
'Item 27',
'Item 28',
'Item 29',
'Item 30',
'Item 31',
'Item 32',
'Item 33',
'Item 34',
'Item 35',
'Item 36',
'Item 37',
'Item 38',
'Item 39',
'Item 40',
'Item 41',
'Item 42',
'Item 43',
'Item 44',
'Item 45',
'Item 46',
'Item 47',
'Item 48',
'Item 49',
'Item 50',
'Item 51',
'Item 52',
'Item 53',
'Item 54',
'Item 55',
'Item 56'
];
const TOO_COMPLEX_OBJECT = {
category: 'APP',
cause: FunctionalError('my error', { category_to_limit: CATEGORY_TO_LIMIT }),
errors: [
{
attributes: {
category_to_limit: [
'Item 1',
'Item 2',
'Item 3',
'Item 4',
'Item 5',
'Item 6',
'Item 7',
'Item 8',
'Item 9',
'Item 10',
'Item 11',
'Item 12',
'Item 13',
'Item 14',
'Item 15',
'Item 16',
'Item 17',
'Item 18',
'Item 19',
'Item 20',
'Item 21',
'Item 22',
'Item 23',
'Item 24',
'Item 25',
'Item 26',
'Item 27',
'Item 28',
'Item 29',
'Item 30',
'Item 31',
'Item 32',
'Item 33',
'Item 34',
'Item 35',
'Item 36',
'Item 37',
'Item 38',
'Item 39',
'Item 40',
'Item 41',
'Item 42',
'Item 43',
'Item 44',
'Item 45',
'Item 46',
'Item 47',
'Item 48',
'Item 49',
'Item 50',
'Item 51',
'Item 52',
'Item 53',
'Item 54',
'Item 55',
'Item 56'
]
},
message: 'Bulk indexing fail',
name: 'DATABASE_ERROR',
departments: [
{
departmentName: 'Research and Development',
teams: [
{
teamName: 'AI Team',
category_to_limit: [
'Item 1',
'Item 2',
'Item 3',
'Item 4',
'Item 5',
'Item 6',
'Item 7',
'Item 8',
'Item 9',
'Item 10',
'Item 11',
'Item 12',
'Item 13',
'Item 14',
'Item 15',
'Item 16',
'Item 17',
'Item 18',
'Item 19',
'Item 20',
'Item 21',
'Item 22',
'Item 23',
'Item 24',
'Item 25',
'Item 26',
'Item 27',
'Item 28',
'Item 29',
'Item 30',
'Item 31',
'Item 32',
'Item 33',
'Item 34',
'Item 35',
'Item 36',
'Item 37',
'Item 38',
'Item 39',
'Item 40',
'Item 41',
'Item 42',
'Item 43',
'Item 44',
'Item 45',
'Item 46',
'Item 47',
'Item 48',
'Item 49',
'Item 50',
'Item 51',
'Item 52',
'Item 53',
'Item 54',
'Item 55',
'Item 56'
]
},
{
teamName: 'Robotics Team',
category_to_limit: ['2', '1', '3'],
}
]
}
]
category_to_limit: CATEGORY_TO_LIMIT
},
{
category_to_limit: ['2', '1', '3'],
}
],
id: '3f001108-c42c-4131-b3a3-583a98043c15',
Expand Down Expand Up @@ -201,18 +130,18 @@ describe('Logger test suite', () => {
});

it('Log object is correctly limited', () => {
let initialSize = TOO_COMPLEX_OBJECT.errors[0].attributes.category_to_limit.length;
let initialSize = CATEGORY_TO_LIMIT.length;
const start = new Date().getTime();
const cleanObject = prepareLogMetadata(TOO_COMPLEX_OBJECT);
const parsingTimeMs = new Date().getTime() - start;
expect(parsingTimeMs).to.be.lt(2);
let cleanedSize = cleanObject.errors[0].attributes.category_to_limit.length;
expect(parsingTimeMs).to.be.lt(5);
let cleanedSize = cleanObject.cause.attributes.category_to_limit.length;
expect(initialSize).not.toEqual(cleanedSize);
expect(initialSize).to.be.gt(appLogLevelMaxArraySize);
expect(cleanedSize).to.be.eq(appLogLevelMaxArraySize);
// check more inside look
initialSize = TOO_COMPLEX_OBJECT.errors[0].departments[0].teams[0].category_to_limit.length;
cleanedSize = cleanObject.errors[0].departments[0].teams[0].category_to_limit.length;
initialSize = TOO_COMPLEX_OBJECT.errors[0].category_to_limit.length;
cleanedSize = cleanObject.errors[0].category_to_limit.length;
expect(initialSize).not.toEqual(cleanedSize);
expect(initialSize).to.be.gt(appLogLevelMaxArraySize);
expect(cleanedSize).to.be.eq(appLogLevelMaxArraySize);
Expand Down

0 comments on commit 8997343

Please sign in to comment.