[Security Solution][Detections][BUG] ES|QL rule execution error when …

…source document has a non-ECS compliant sub-field with data under event field (elastic#187384) (elastic#187549)

## Summary

Ticket elastic#187384

These changes fix the error on saving the alert
> An error occurred during rule execution: message: "[1:6778] failed to
parse field [kibana.alert.original_event.action] of type [keyword] in
document with id '027b925ae2799635a0dee97a6aa9d58dc87d9771'."

which happens due to not stripping non-ECS compliant sub-fields of the
`event.action` field.

See the main ticket for steps to reproduce the issue.

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/es_results.ts

@@ -359,6 +359,24 @@ export const sampleDocNoSortIdWithTimestamp = (
   };
 };
 
+export const sampleDocWithNonEcsCompliantFields = (
+  someUuid: string = sampleIdGuid,
+  nonEcsFields: Record<string, string>
+): SignalSourceHit & {
+  _id: Required<SignalSourceHit>['_id'];
+  _source: Required<SignalSourceHit>['_source'] & { '@timestamp': string };
+} => {
+  const doc = sampleDocNoSortId(someUuid);
+  return {
+    ...doc,
+    _source: {
+      ...doc._source,
+      ...nonEcsFields,
+      '@timestamp': new Date().toISOString(),
+    },
+  };
+};
+
 export const sampleAlertDocNoSortIdWithTimestamp = (
   someUuid: string = sampleIdGuid,
   ip?: string

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { sampleDocWithNonEcsCompliantFields } from '../../__mocks__/es_results';
+import { buildBulkBody } from './build_bulk_body';
+import { getCompleteRuleMock, getEsqlRuleParams } from '../../../rule_schema/mocks';
+import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks';
+
+const SPACE_ID = 'space';
+const publicBaseUrl = 'testKibanaBasePath.com';
+const alertUuid = 'test-uuid';
+const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71';
+const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create();
+
+describe('buildBulkBody', () => {
+  test('should strip non-ECS compliant sub-fields of `event.action` field', () => {
+    const doc = sampleDocWithNonEcsCompliantFields(docId, {
+      'event.action': 'process',
+      'event.action.keyword': 'process',
+    });
+    const completeRule = getCompleteRuleMock(getEsqlRuleParams());
+    const buildReasonMessageStub = jest.fn();
+    const alert = buildBulkBody(
+      SPACE_ID,
+      completeRule,
+      doc,
+      'missingFields',
+      [],
+      true,
+      buildReasonMessageStub,
+      [],
+      undefined,
+      ruleExecutionLogger,
+      alertUuid,
+      publicBaseUrl
+    );
+
+    expect(alert['kibana.alert.original_event.action']).toEqual('process');
+    expect(alert['kibana.alert.original_event.action.keyword']).toBeUndefined();
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts

@@ -104,6 +104,7 @@ export const buildBulkBody = (
     mergedDoc,
   });
 
+  const thresholdResult = mergedDoc._source?.threshold_result;
   if (isSourceDoc(mergedDoc)) {
     return {
       ...validatedSource,
@@ -121,7 +122,11 @@ export const buildBulkBody = (
       ),
       ...additionalAlertFields({
         ...mergedDoc,
-        _source: { ...mergedDoc._source, ...validatedEventFields },
+        _source: {
+          ...validatedSource,
+          ...validatedEventFields,
+          threshold_result: thresholdResult,
+        },
       }),
     };
   }

x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json

@@ -42,6 +42,15 @@
             "properties": {
               "created": {
                 "type": "keyword"
+              },
+              "action": {
+                "type": "text",
+                "fields": {
+                  "keyword": {
+                    "type": "keyword",
+                    "ignore_above": 256
+                  }
+                }
               }
             }
           },

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts

@@ -1103,6 +1103,40 @@ export default ({ getService }: FtrProviderContext) => {
             'random non-ecs field'
           );
         });
+
+        it('creates alert if `event.action` ECS field has non-ECS sub-field', async () => {
+          // The issue was found by customer and reported in
+          // https://github.com/elastic/sdh-security-team/issues/1015
+          const id = uuidv4();
+          const interval: [string, string] = [
+            '2020-10-28T06:00:00.000Z',
+            '2020-10-28T06:10:00.000Z',
+          ];
+          const doc1 = {
+            'event.action': 'process',
+          };
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_non_compliant metadata _id ${internalIdPipe(id)}`,
+            from: 'now-1h',
+            interval: '1h',
+          };
+
+          await indexEnhancedDocumentsToNonEcs({
+            documents: [doc1],
+            interval,
+            id,
+          });
+
+          const { logs } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          });
+
+          expect(logs[0].errors.length).toEqual(0);
+        });
       });
     });
   });