From 0a4ed6169904555b243c345267d1f476b5a02348 Mon Sep 17 00:00:00 2001 From: Ievgen Sorokopud Date: Fri, 5 Jul 2024 13:29:22 +0200 Subject: [PATCH] [Security Solution][Detections][BUG] ES|QL rule execution error when source document has a non-ECS compliant sub-field with data under event field (#187384) (#187549) ## Summary Ticket https://github.com/elastic/kibana/issues/187384 These changes fix the error on saving the alert > An error occurred during rule execution: message: "[1:6778] failed to parse field [kibana.alert.original_event.action] of type [keyword] in document with id '027b925ae2799635a0dee97a6aa9d58dc87d9771'." which happens due to not stripping non-ECS compliant sub-fields of the `event.action` field. See the main ticket for steps to reproduce the issue. --- .../rule_types/__mocks__/es_results.ts | 18 ++++++++ .../factories/utils/build_bulk_body.test.ts | 45 +++++++++++++++++++ .../factories/utils/build_bulk_body.ts | 7 ++- .../ecs_non_compliant/mappings.json | 9 ++++ .../execution_logic/esql.ts | 34 ++++++++++++++ 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts index 57892d65da35f..0867245a40933 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts @@ -359,6 +359,24 @@ export const sampleDocNoSortIdWithTimestamp = ( }; }; +export const sampleDocWithNonEcsCompliantFields = ( + someUuid: string = sampleIdGuid, + nonEcsFields: Record +): SignalSourceHit & { + _id: Required['_id']; + _source: Required['_source'] & { '@timestamp': string }; +} => { + const doc = sampleDocNoSortId(someUuid); + return { + ...doc, + _source: { + ...doc._source, + ...nonEcsFields, + '@timestamp': new Date().toISOString(), + }, + }; +}; + export const sampleAlertDocNoSortIdWithTimestamp = ( someUuid: string = sampleIdGuid, ip?: string diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts new file mode 100644 index 0000000000000..b2426ceda9767 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { sampleDocWithNonEcsCompliantFields } from '../../__mocks__/es_results'; +import { buildBulkBody } from './build_bulk_body'; +import { getCompleteRuleMock, getEsqlRuleParams } from '../../../rule_schema/mocks'; +import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks'; + +const SPACE_ID = 'space'; +const publicBaseUrl = 'testKibanaBasePath.com'; +const alertUuid = 'test-uuid'; +const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + +describe('buildBulkBody', () => { + test('should strip non-ECS compliant sub-fields of `event.action` field', () => { + const doc = sampleDocWithNonEcsCompliantFields(docId, { + 'event.action': 'process', + 'event.action.keyword': 'process', + }); + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + const buildReasonMessageStub = jest.fn(); + const alert = buildBulkBody( + SPACE_ID, + completeRule, + doc, + 'missingFields', + [], + true, + buildReasonMessageStub, + [], + undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl + ); + + expect(alert['kibana.alert.original_event.action']).toEqual('process'); + expect(alert['kibana.alert.original_event.action.keyword']).toBeUndefined(); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts index fb1aa57fdb82d..9294cc7159c12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts @@ -104,6 +104,7 @@ export const buildBulkBody = ( mergedDoc, }); + const thresholdResult = mergedDoc._source?.threshold_result; if (isSourceDoc(mergedDoc)) { return { ...validatedSource, @@ -121,7 +122,11 @@ export const buildBulkBody = ( ), ...additionalAlertFields({ ...mergedDoc, - _source: { ...mergedDoc._source, ...validatedEventFields }, + _source: { + ...validatedSource, + ...validatedEventFields, + threshold_result: thresholdResult, + }, }), }; } diff --git a/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json b/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json index ea4f271af0ffc..7a76d0da64667 100644 --- a/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json @@ -42,6 +42,15 @@ "properties": { "created": { "type": "keyword" + }, + "action": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } } } }, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts index cbc7f43cfe6cc..69795e04cd351 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts @@ -1103,6 +1103,40 @@ export default ({ getService }: FtrProviderContext) => { 'random non-ecs field' ); }); + + it('creates alert if `event.action` ECS field has non-ECS sub-field', async () => { + // The issue was found by customer and reported in + // https://github.com/elastic/sdh-security-team/issues/1015 + const id = uuidv4(); + const interval: [string, string] = [ + '2020-10-28T06:00:00.000Z', + '2020-10-28T06:10:00.000Z', + ]; + const doc1 = { + 'event.action': 'process', + }; + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_non_compliant metadata _id ${internalIdPipe(id)}`, + from: 'now-1h', + interval: '1h', + }; + + await indexEnhancedDocumentsToNonEcs({ + documents: [doc1], + interval, + id, + }); + + const { logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + }); + + expect(logs[0].errors.length).toEqual(0); + }); }); }); });