[backend] Global rework on application logging

opencti-platform/opencti-graphql/src/config/conf.js

@@ -9,7 +9,7 @@ import DailyRotateFile from 'winston-daily-rotate-file';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import { HttpProxyAgent } from 'http-proxy-agent';
 import { v4 as uuid } from 'uuid';
-import { GraphQLError } from 'graphql/error';
+import { GraphQLError } from 'graphql/index';
 import * as O from '../schema/internalObject';
 import * as M from '../schema/stixMetaObject';
 import {
@@ -30,7 +30,7 @@ import { ENTITY_TYPE_ENTITY_SETTING } from '../modules/entitySetting/entitySetti
 import { ENTITY_TYPE_MANAGER_CONFIGURATION } from '../modules/managerConfiguration/managerConfiguration-types';
 import { ENTITY_TYPE_WORKSPACE } from '../modules/workspace/workspace-types';
 import { ENTITY_TYPE_NOTIFIER } from '../modules/notifier/notifier-types';
-import { UnknownError, UnsupportedError } from './errors';
+import { UNKNOWN_ERROR, UnknownError, UnsupportedError } from './errors';
 import { ENTITY_TYPE_PUBLIC_DASHBOARD } from '../modules/publicDashboard/publicDashboard-types';
 import { AI_BUS } from '../modules/ai/ai-types';
 import { SUPPORT_BUS } from '../modules/support/support-types';
@@ -47,8 +47,9 @@ const LINUX_CERTFILES = [
 
 const DEFAULT_ENV = 'production';
 export const OPENCTI_SESSION = 'opencti_session';
-
 export const PLATFORM_VERSION = pjson.version;
+const LOG_APP = 'APP';
+const LOG_AUDIT = 'AUDIT';
 
 export const booleanConf = (key, defaultValue = true) => {
   const configValue = nconf.get(key);
@@ -107,7 +108,19 @@ export const extendedErrors = (metaExtension) => {
   }
   return {};
 };
-const limitMetaErrorComplexityWrapper = (obj, acc, current_depth = 0) => {
+const convertErrorObject = (error) => {
+  if (error instanceof GraphQLError) {
+    const extensions = error.extensions ?? {};
+    const extensionsData = extensions.data ?? {};
+    const { ...attributes } = extensionsData;
+    return { name: extensions.code ?? error.name, code: extensions.code, message: error.message, stack: error.stack, attributes };
+  }
+  if (error instanceof Error) {
+    return { name: error.name, code: UNKNOWN_ERROR, message: error.message, stack: error.stack };
+  }
+  return error;
+};
+const prepareLogMetadataComplexityWrapper = (obj, acc, current_depth = 0) => {
   const noMaxDepth = current_depth < appLogLevelMaxDepthSize;
   const noMaxKeys = acc.current_nb_key < appLogLevelMaxDepthKeys;
   const isNotAKeyFunction = typeof obj !== 'function';
@@ -118,31 +131,34 @@ const limitMetaErrorComplexityWrapper = (obj, acc, current_depth = 0) => {
       // Recursively process each item in the truncated array
       const processedArray = [];
       for (let i = 0; i < limitedArray.length; i += 1) {
-        processedArray[i] = limitMetaErrorComplexityWrapper(limitedArray[i], acc, current_depth);
+        processedArray[i] = prepareLogMetadataComplexityWrapper(limitedArray[i], acc, current_depth);
       }
       return processedArray;
     }
     if (typeof obj === 'string' && obj.length > appLogLevelMaxStringSize) {
       return `${obj.substring(0, appLogLevelMaxStringSize - 3)}...`;
     }
     if (typeof obj === 'object') {
+      const workingObject = convertErrorObject(obj);
       // Create a new object to hold the processed properties
       const limitedObject = {};
-      const keys = Object.keys(obj); // Get the keys of the object
+      const keys = Object.keys(workingObject); // Get the keys of the object
       const newDepth = current_depth + 1;
       for (let i = 0; i < keys.length; i += 1) {
         acc.current_nb_key += 1;
         const key = keys[i];
-        limitedObject[key] = limitMetaErrorComplexityWrapper(obj[key], acc, newDepth);
+        limitedObject[key] = prepareLogMetadataComplexityWrapper(workingObject[key], acc, newDepth);
       }
       return limitedObject;
     }
   }
   return obj;
 };
-export const limitMetaErrorComplexity = (obj) => {
+// Prepare the data - Format the errors and limit complexity
+export const prepareLogMetadata = (obj, extra = {}) => {
   const acc = { current_nb_key: 0 };
-  return limitMetaErrorComplexityWrapper(obj, acc);
+  const protectedObj = prepareLogMetadataComplexityWrapper(obj, acc);
+  return { ...extra, ...protectedObj, version: PLATFORM_VERSION };
 };
 
 const appLogTransports = [];
@@ -230,87 +246,50 @@ const telemetryLogger = winston.createLogger({
   transports: telemetryLogTransports,
 });
 
-// Specific case to fail any test that produce an error log
-const LOG_APP = 'APP';
-const buildMetaErrors = (error) => {
-  const errors = [];
-  if (error instanceof GraphQLError) {
-    const extensions = error.extensions ?? {};
-    const extensionsData = extensions.data ?? {};
-    const { cause: _, ...attributes } = extensionsData;
-    const baseError = { name: extensions.code ?? error.name, message: error.message, stack: error.stack, attributes };
-    errors.push(baseError);
-    if (extensionsData.cause && extensionsData.cause instanceof Error) {
-      errors.push(...buildMetaErrors(extensionsData.cause));
-    }
-  } else if (error instanceof Error) {
-    const baseError = { name: error.name, message: error.message, stack: error.stack };
-    errors.push(baseError);
-  }
-  return errors;
-};
-const addBasicMetaInformation = (category, error, meta) => {
-  const logMeta = { ...meta };
-  if (error) logMeta.errors = buildMetaErrors(error);
-  return { category, version: PLATFORM_VERSION, ...logMeta };
-};
-
 export const logS3Debug = {
   debug: (message, detail) => {
-    logApp._log('info', message, null, { detail });
+    logApp._log('info', message, { detail });
   },
 };
 
 export const logApp = {
-  _log: (level, message, error, meta = {}) => {
+  _log: (level, message, meta = {}) => {
     if (appLogTransports.length > 0 && appLogger.isLevelEnabled(level)) {
-      const data = addBasicMetaInformation(LOG_APP, error, { ...meta, source: 'backend' });
-      // Prevent meta information to be too massive.
-      const limitedData = limitMetaErrorComplexity(data);
-      appLogger.log(level, message, limitedData);
-    }
-  },
-  _logWithError: (level, messageOrError, meta = {}) => {
-    const isError = messageOrError instanceof Error;
-    const message = isError ? messageOrError.message : messageOrError;
-    let error = null;
-    if (isError) {
-      if (messageOrError instanceof GraphQLError) {
-        error = messageOrError;
-      } else {
-        error = UnknownError(message, { cause: messageOrError });
+      const data = prepareLogMetadata(meta, { category: LOG_APP, source: 'backend' });
+      appLogger.log(level, message, data);
+      // Only add in support package starting warn level
+      if (appLogger.isLevelEnabled('warn')) {
+        supportLogger.log(level, message, data);
       }
     }
-    logApp._log(level, message, error, meta);
-    supportLogger.log(level, message, addBasicMetaInformation(LOG_APP, error, { ...meta, source: 'backend' }));
   },
-  debug: (message, meta = {}) => logApp._log('debug', message, null, meta),
-  info: (message, meta = {}) => logApp._log('info', message, null, meta),
-  warn: (messageOrError, meta = {}) => logApp._logWithError('warn', messageOrError, meta),
-  error: (messageOrError, meta = {}) => logApp._logWithError('error', messageOrError, meta),
+  debug: (message, meta = {}) => logApp._log('debug', message, meta),
+  info: (message, meta = {}) => logApp._log('info', message, meta),
+  warn: (message, meta = {}) => logApp._log('warn', message, meta),
+  error: (message, meta = {}) => logApp._log('error', message, meta),
   query: (options, errCallback) => appLogger.query(options, errCallback),
 };
 
-const LOG_AUDIT = 'AUDIT';
 export const logAudit = {
   _log: (level, user, operation, meta = {}) => {
     if (auditLogTransports.length > 0) {
       const metaUser = { email: user.user_email, ...user.origin };
       const logMeta = isEmpty(meta) ? { auth: metaUser } : { resource: meta, auth: metaUser };
-      auditLogger.log(level, operation, addBasicMetaInformation(LOG_AUDIT, null, logMeta));
+      const data = prepareLogMetadata(logMeta, { category: LOG_AUDIT, source: 'backend' });
+      auditLogger.log(level, operation, data);
     }
   },
   info: (user, operation, meta = {}) => logAudit._log('info', user, operation, meta),
   error: (user, operation, meta = {}) => logAudit._log('error', user, operation, meta),
 };
 
 export const logFrontend = {
-  _log: (level, message, error, meta = {}) => {
-    const info = { ...meta, source: 'frontend' };
-    appLogger.log(level, message, addBasicMetaInformation(LOG_APP, error, info));
-    supportLogger.log(level, message, addBasicMetaInformation(LOG_APP, error, info));
+  _log: (level, message, meta = {}) => {
+    const data = prepareLogMetadata(meta, { category: LOG_APP, source: 'frontend' });
+    appLogger.log(level, message, data);
+    supportLogger.log(level, message, data);
   },
-  error: (message, meta = {}) => logFrontend._log('error', message, null, meta),
+  error: (message, meta = {}) => logFrontend._log('error', message, meta),
 };
 
 export const logTelemetry = {

opencti-platform/opencti-graphql/src/config/credentials.ts

@@ -41,7 +41,7 @@ export const enrichWithRemoteCredentials = async (prefix: string, baseConfigurat
           return secretResult;
         }
       } catch (e: any) {
-        logApp.error('[OPENCTI] Remote credentials data fail to fetch, fallback', { error: e, provider, source: prefix });
+        logApp.error('[OPENCTI] Remote credentials data fail to fetch, fallback', { cause: e, provider, source: prefix });
       }
     }
     // No compatible provider available

opencti-platform/opencti-graphql/src/config/errors.js

@@ -87,7 +87,7 @@ export const ConfigurationError = (reason, data) => error(CONFIGURATION_ERROR, r
   ...data,
 });
 
-const UNKNOWN_ERROR = 'UNKNOWN_ERROR';
+export const UNKNOWN_ERROR = 'UNKNOWN_ERROR';
 export const UnknownError = (reason, data) => error(UNKNOWN_ERROR, reason || 'An unknown error has occurred', {
   http_status: 500,
   genre: CATEGORY_TECHNICAL,
@@ -148,6 +148,12 @@ export const ValidationError = (message, field, data) => error(VALIDATION_ERROR,
   ...(data ?? {}),
 });
 
+export const RESOURCE_NOT_FOUND_ERROR = 'RESOURCE_NOT_FOUND';
+export const ResourceNotFoundError = (data) => error(RESOURCE_NOT_FOUND_ERROR, 'Resource not found', {
+  http_status: 404,
+  ...data,
+});
+
 const TYPE_LOCK = 'LOCK_ERROR';
 export const TYPE_LOCK_ERROR = 'ExecutionError';
 export const LockTimeoutError = (data, reason) => error(TYPE_LOCK, reason ?? 'Execution timeout, too many concurrent call on the same entities', {
@@ -161,6 +167,7 @@ export const FUNCTIONAL_ERRORS = [
   ALREADY_DELETED_ERROR,
   MISSING_REF_ERROR,
   VALIDATION_ERROR,
+  RESOURCE_NOT_FOUND_ERROR,
   TYPE_LOCK_ERROR
 ];
 // endregion

opencti-platform/opencti-graphql/src/connector/importCsv/importCsv-connector.ts

@@ -46,7 +46,7 @@ const processCSVforWorkbench = async (context: AuthContext, fileId: string, opts
       hasError = true;
       const errorData = { error: error.message, source: fileId };
       await reportExpectation(context, applicantUser, workId, errorData);
-      logApp.error(`${LOG_PREFIX} Error streaming the CSV data`, { error });
+      logApp.error(`${LOG_PREFIX} Error streaming the CSV data`, { cause: error });
     }).on('end', async () => {
       if (!hasError) {
         // it's fine to use deprecated bundleProcess since this whole method is also deprecated for drafts.
@@ -132,12 +132,12 @@ export const processCSVforWorkers = async (context: AuthContext, fileId: string,
             totalBundlesCount += bundleCount;
           } catch (error: any) {
             const errorData = { error: error.message, source: `${fileId}, from ${lineNumber} and ${BULK_LINE_PARSING_NUMBER} following lines.` };
-            logApp.error(`${LOG_PREFIX} CSV line parsing error`, { error: errorData });
+            logApp.error(`${LOG_PREFIX} CSV line parsing error`, { cause: errorData });
             await reportExpectation(context, applicantUser, workId, errorData);
           }
         }
       } catch (error: any) {
-        logApp.error(`${LOG_PREFIX} CSV global parsing error`, { error });
+        logApp.error(`${LOG_PREFIX} CSV global parsing error`, { cause: error });
         const errorData = { error: error.message, source: fileId };
         await reportExpectation(context, applicantUser, workId, errorData);
         // circuit breaker
@@ -204,7 +204,7 @@ const consumeQueueCallback = async (context: AuthContext, message: string) => {
       await processCSVforWorkers(context, fileId, opts);
     }
   } catch (error: any) {
-    logApp.error(`${LOG_PREFIX} CSV global parsing error`, { error, source: fileId });
+    logApp.error(`${LOG_PREFIX} CSV global parsing error`, { cause: error, source: fileId });
     const errorData = { error: error.stack, source: fileId };
     await reportExpectation(context, applicantUser, workId, errorData);
   }

opencti-platform/opencti-graphql/src/database/ai-llm.ts

@@ -57,7 +57,7 @@ export const queryMistralAi = async (busId: string | null, question: string, use
     logApp.error('[AI] No response from MistralAI', { busId, question });
     return 'No response from MistralAI';
   } catch (err) {
-    logApp.error('[AI] Cannot query MistralAI', { error: err });
+    logApp.error('[AI] Cannot query MistralAI', { cause: err });
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-expect-error
     return `An error occurred: ${err.toString()}`;
@@ -92,7 +92,7 @@ export const queryChatGpt = async (busId: string | null, question: string, user:
     logApp.error('[AI] No response from OpenAI', { busId, question });
     return 'No response from OpenAI';
   } catch (err) {
-    logApp.error('[AI] Cannot query OpenAI', { error: err });
+    logApp.error('[AI] Cannot query OpenAI', { cause: err });
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-expect-error
     return `An error occurred: ${err.toString()}`;

opencti-platform/opencti-graphql/src/database/engine.js

@@ -1602,7 +1602,7 @@ export const elFindByIds = async (context, user, ids, opts = {}) => {
       });
       const elements = data.hits.hits;
       if (elements.length > workingIds.length) {
-        logApp.warn('Search query returned more elements than expected', workingIds);
+        logApp.warn('Search query returned more elements than expected', { ids: workingIds });
       }
       if (elements.length > 0) {
         const convertedHits = await elConvertHits(elements, { withoutRels });

opencti-platform/opencti-graphql/src/database/file-search.js

@@ -66,11 +66,11 @@ export const elIndexFiles = async (context, user, files) => {
         await elIndex(INDEX_FILES, documentBody, { pipeline: 'attachment' });
       } catch (err) {
         // catch & log error
-        logApp.error('Error on file indexing', { message: err.message, causeStack: err.data?.cause?.stack, stack: err.stack, file_id });
+        logApp.error('Error on file indexing', { cause: err, file_id });
         // try to index without file content
         const documentWithoutFileData = R.dissoc('file_data', documentBody);
         await elIndex(INDEX_FILES, documentWithoutFileData).catch((e) => {
-          logApp.error('Error in fallback file indexing', { message: e.message, cause: e.cause, file_id });
+          logApp.error('Error in fallback file indexing', { message: e.message, cause: e, file_id });
         });
       }
     }

opencti-platform/opencti-graphql/src/database/file-storage-helper.ts

@@ -9,6 +9,7 @@ import { allFilesForPaths, EXPORT_STORAGE_PATH, FROM_TEMPLATE_STORAGE_PATH, IMPO
 import { deleteWorkForSource } from '../domain/work';
 import { ENTITY_TYPE_SUPPORT_PACKAGE } from '../modules/support/support-types';
 import { getDraftContext } from '../utils/draftContext';
+import { UnsupportedError } from '../config/errors';
 
 interface FileUploadOpts {
   entity?:BasicStoreBase | unknown, // entity on which the file is uploaded
@@ -54,7 +55,9 @@ interface S3File {
  * @param opts
  */
 export const uploadToStorage = (context: AuthContext, user: AuthUser, filePath: string, fileUpload: FileUploadData, opts: FileUploadOpts) => {
-  if (getDraftContext(context, user)) throw new Error('Cannot upload file in draft context');
+  if (getDraftContext(context, user)) {
+    throw UnsupportedError('Cannot upload file in draft context');
+  }
   return upload(context, user, filePath, fileUpload, opts);
 };
 

opencti-platform/opencti-graphql/src/database/file-storage.js

@@ -173,7 +173,7 @@ export const downloadFile = async (id) => {
     }
     return object.Body;
   } catch (err) {
-    logApp.error('[FILE STORAGE] Cannot retrieve file from S3', { error: err, fileId: id });
+    logApp.error('[FILE STORAGE] Cannot retrieve file from S3', { cause: err, fileId: id });
     return null;
   }
 };
@@ -213,7 +213,7 @@ export const copyFile = async (context, copyProps) => {
     logApp.info('[FILE STORAGE] Copy file to S3 in success', { document: file, sourceId, targetId });
     return file;
   } catch (err) {
-    logApp.error('[FILE STORAGE] Cannot copy file in S3', { error: err, sourceId, targetId });
+    logApp.error('[FILE STORAGE] Cannot copy file in S3', { cause: err, sourceId, targetId });
     return null;
   }
 };

opencti-platform/opencti-graphql/src/domain/backgroundTask-common.js

@@ -306,7 +306,9 @@ const authorizedMembersForTask = (user, scope) => {
 };
 
 export const createListTask = async (context, user, input) => {
-  if (getDraftContext(context, user)) throw new Error('Cannot create background task in draft');
+  if (getDraftContext(context, user)) {
+    throw UnsupportedError('Cannot create background task in draft');
+  }
   const { actions, ids, scope } = input;
   await checkActionValidity(context, user, input, scope, TASK_TYPE_LIST);
   const task = createDefaultTask(user, input, TASK_TYPE_LIST, ids.length, scope);

opencti-platform/opencti-graphql/src/domain/backgroundTask.js

@@ -9,7 +9,7 @@ import { ABSTRACT_STIX_CORE_OBJECT, ABSTRACT_STIX_CORE_RELATIONSHIP, RULE_PREFIX
 import { buildEntityFilters, listEntities, storeLoadById } from '../database/middleware-loader';
 import { checkActionValidity, createDefaultTask, TASK_TYPE_QUERY, TASK_TYPE_RULE } from './backgroundTask-common';
 import { publishUserAction } from '../listener/UserActionListener';
-import { ForbiddenAccess } from '../config/errors';
+import { ForbiddenAccess, UnsupportedError } from '../config/errors';
 import { STIX_SIGHTING_RELATIONSHIP } from '../schema/stixSightingRelationship';
 import { ENTITY_TYPE_VOCABULARY } from '../modules/vocabulary/vocabulary-types';
 import { ENTITY_TYPE_NOTIFICATION } from '../modules/notification/notification-types';
@@ -130,7 +130,9 @@ export const createRuleTask = async (context, user, ruleDefinition, input) => {
 };
 
 export const createQueryTask = async (context, user, input) => {
-  if (getDraftContext(context, user)) throw new Error('Cannot create background task in draft');
+  if (getDraftContext(context, user)) {
+    throw UnsupportedError('Cannot create background task in draft');
+  }
   const { actions, filters, excluded_ids = [], search = null, scope } = input;
   await checkActionValidity(context, user, input, scope, TASK_TYPE_QUERY);
   const queryData = await executeTaskQuery(context, user, filters, search, scope);