From 6a666ca605e1c6b61b80eacc7ff3784d29b74c85 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 16 Jan 2024 14:42:14 -0800 Subject: [PATCH 1/5] support object fields in aggregation based sigma rules (#789) Signed-off-by: Subhobrata Dey --- .../aggregation/AggregationLexer.java | 88 ++--- src/main/grammars/Aggregation.g4 | 2 +- .../rules/backend/AggregationBuilders.java | 2 +- .../rules/backend/OSQueryBackend.java | 27 +- .../securityanalytics/TestHelpers.java | 373 ++++++++++++++++-- .../resthandler/DetectorMonitorRestApiIT.java | 239 ++++++++++- .../aggregation/AggregationBackendTests.java | 86 +++- 7 files changed, 714 insertions(+), 103 deletions(-) diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java index fcf7ae82a..115766bb1 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) { "\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+ "\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+ "\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+ - "\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+ - "\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+ - "\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+ - "\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+ - "\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+ - "\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+ - "\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+ - "\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+ - "\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+ - "\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+ - "&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+ - "\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+ - "\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+ - "\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+ - "\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+ - "\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+ - ">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+ - "=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+ - "\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+ - "\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+ - "\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+ - "u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+ - "\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+ - "\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+ - "\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+ - "\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+ - "\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+ - "C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+ - "F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+ - "\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+ - "-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+ - "\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+ - "PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+ - "\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+ - "\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+ - "\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+ - "\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+ - "\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+ - "\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+ - "\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+ - "\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+ - "\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+ - "\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+ - "\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; + "\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+ + "z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+ + "\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+ + "\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+ + "\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+ + "\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+ + "\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+ + "\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+ + "\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+ + "\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+ + "\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+ + "\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+ + "\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+ + "@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+ + "\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+ + "\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+ + "\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+ + "$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+ + "\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+ + "*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+ + "-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+ + "01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+ + "\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+ + "\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+ + "i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+ + "m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+ + "\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+ + "\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+ + "\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+ + "\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+ + "KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+ + "MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+ + "\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+ + "\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+ + "\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+ + "\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+ + "YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+ + "\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+ + "\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+ + "_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+ + "\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+ + "\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+ + "fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+ + "\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/src/main/grammars/Aggregation.g4 b/src/main/grammars/Aggregation.g4 index 00303b68a..be395c5ae 100644 --- a/src/main/grammars/Aggregation.g4 +++ b/src/main/grammars/Aggregation.g4 @@ -21,7 +21,7 @@ RPAREN : ')' ; DECIMAL : '-'?[0-9]+('.'[0-9]+)? ; -IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ; +IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ; WS : [ \r\t\u000C\n]+ -> skip ; comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java index 3927186fb..c0f6bbb7a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java @@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat aggregationBuilder = new TermsAggregationBuilder(name).field(name); break; case "count": - aggregationBuilder = new ValueCountAggregationBuilder(name).field(name); + aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name); break; default: throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction)); diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 560bd47bf..814f32652 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -361,14 +361,15 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { BucketSelectorExtAggregationBuilder condition; String bucketTriggerSelectorId = UUIDs.base64UUID(); - if (aggregation.getAggFunction().equals("count")) { + if (aggregation.getAggFunction().equals("count") && aggregation.getAggField().equals("*")) { String fieldName; - if (aggregation.getAggField().equals("*") && aggregation.getGroupByField() == null) { + if (aggregation.getGroupByField() == null) { fieldName = "_index"; fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index"); } else { - fieldName = aggregation.getGroupByField(); - fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", aggregation.getGroupByField()); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fieldName = mappedGroupByField; + fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", mappedGroupByField); } aggBuilder.field(fieldName); fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, "_cnt", "_count", "result_agg", "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()); @@ -376,17 +377,23 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold())); condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_count"), script, "result_agg", null); } else { - fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField(), aggregation.getAggFunction(), aggregation.getAggField()); - fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField(), aggregation.getAggField(), "result_agg", aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold()); + /** + * removing dots to eliminate dots in aggregation names + */ + String mappedAggField = getFinalField(aggregation.getAggField()); + String mappedAggFieldUpdated = mappedAggField.replace(".", "_"); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", mappedGroupByField, mappedAggFieldUpdated, aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), mappedAggField); + fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, mappedAggFieldUpdated, mappedAggField, "result_agg", mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold()); // Add subaggregation - AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField()); + AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), mappedAggField); if (subAgg != null) { - aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg); + aggBuilder.field(mappedGroupByField).subAggregation(subAgg); } - Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold())); - condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField(), aggregation.getAggField()), script, "result_agg", null); + Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold())); + condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(mappedAggFieldUpdated, mappedAggFieldUpdated), script, "result_agg", null); } AggregationQueries aggregationQueries = new AggregationQueries(); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 7be4febff..843abbf3c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -86,7 +86,7 @@ public static Detector randomDetectorWithTriggers(List rules, List rules, List triggers, List inputIndices) { DetectorInput input = new DetectorInput("windows detector for security analytics", inputIndices, Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); - return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null); + return randomDetector(null, null, null, List.of(input), triggers, null, true, null, null); } public static Detector randomDetectorWithTriggersAndScheduleAndEnabled(List rules, List triggers, Schedule schedule, boolean enabled) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), @@ -168,44 +168,6 @@ public static CustomLogType randomCustomLogType(String name, String description, return new CustomLogType(null, null, name, description, category, source, null); } - public static String randomDocWithNullField() { - return "{\n" + - "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + - "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + - "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + - "\"Keywords\":\"9223372036854775808\",\n" + - "\"SeverityValue\":2,\n" + - "\"Severity\":\"INFO\",\n" + - "\"EventID\":22,\n" + - "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + - "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + - "\"Version\":5,\n" + - "\"TaskValue\":22,\n" + - "\"OpcodeValue\":0,\n" + - "\"RecordNumber\":null,\n" + - "\"ExecutionProcessID\":1996,\n" + - "\"ExecutionThreadID\":2616,\n" + - "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + - "\"Domain\":\"NTAUTHORITY\",\n" + - "\"AccountName\":\"SYSTEM\",\n" + - "\"UserID\":\"S-1-5-18\",\n" + - "\"AccountType\":\"User\",\n" + - "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + - "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + - "\"Opcode\":\"Info\",\n" + - "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + - "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + - "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + - "\"QueryResults\":\"172.31.46.38;\",\n" + - "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + - "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + - "\"SourceModuleName\":\"in\",\n" + - "\"SourceModuleType\":\"im_msvistalog\",\n" + - "\"CommandLine\": \"eachtest\",\n" + - "\"Initiated\": \"true\"\n" + - "}"; - } - public static Detector randomDetectorWithNoUser() { String name = OpenSearchRestTestCase.randomAlphaOfLength(10); String detectorType = randomDetectorType(); @@ -336,6 +298,29 @@ public static String randomRuleForMappingView(String field) { "level: high"; } + public static String randomCloudtrailRuleForCorrelations(String value) { + return "id: 5f92fff9-82e2-48ab-8fc1-8b133556a551\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: AWS User Created\n" + + "description: AWS User Created\n" + + "tags:\n" + + " - attack.test1\n" + + "falsepositives:\n" + + " - Legit User Account Administration\n" + + "level: high\n" + + "date: 2022/01/01\n" + + "status: experimental\n" + + "references:\n" + + " - 'https://github.com/RhinoSecurityLabs/AWS-IAM-Privilege-Escalation'\n" + + "author: toffeebr33k\n" + + "detection:\n" + + " condition: selection_source\n" + + " selection_source:\n" + + " EventName:\n" + + " - " + value; + } + public static String randomRuleForCustomLogType() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -879,6 +864,109 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } + public static String randomCloudtrailAggrRule() { + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " EventName:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by AccountName >= 2"; + } + + public static String randomCloudtrailAggrRuleWithDotFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String randomCloudtrailAggrRuleWithEcsFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(eventName) by awsRegion > 1\n" + + " selection1:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String cloudtrailOcsfMappings() { + return "\"properties\": {\n" + + " \"time\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"cloud.region\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"api\": {\n" + + " \"properties\": {\n" + + " \"operation\": {\"type\": \"keyword\"},\n" + + " \"service\": {\n" + + " \"properties\": {\n" + + " \"name\": {\"type\": \"text\"}\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + public static String windowsIndexMapping() { return "\"properties\": {\n" + " \"@timestamp\": {\"type\":\"date\"},\n" + @@ -892,7 +980,10 @@ public static String windowsIndexMapping() { " \"type\": \"text\"\n" + " },\n" + " \"AccountName\": {\n" + - " \"type\": \"text\"\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"EventName\": {\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"AccountType\": {\n" + " \"type\": \"text\",\n" + @@ -1033,7 +1124,7 @@ public static String windowsIndexMapping() { " \"type\": \"date\"\n" + " },\n" + " \"EventType\": {\n" + - " \"type\": \"integer\"\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"ExecutionProcessID\": {\n" + " \"type\": \"long\"\n" + @@ -1544,6 +1635,43 @@ public static String windowsIndexMappingOnlyNumericAndText() { " }"; } + public static String randomDocWithNullField() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":null,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + @@ -1616,6 +1744,7 @@ public static String randomDoc() { "\"Severity\":\"INFO\",\n" + "\"EventID\":22,\n" + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"SourceIp\":\"1.2.3.4\",\n" + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + "\"Version\":5,\n" + "\"TaskValue\":22,\n" + @@ -1666,6 +1795,160 @@ public static String randomAdLdapDoc() { "}"; } + public static String randomCloudtrailOcsfDoc() { + return "{\n" + + " \"activity_id\": 8,\n" + + " \"activity_name\": \"Detach Policy\",\n" + + " \"actor\": {\n" + + " \"idp\": {\n" + + " \"name\": null\n" + + " },\n" + + " \"invoked_by\": null,\n" + + " \"session\": {\n" + + " \"created_time\": 1702510696000,\n" + + " \"issuer\": \"arn\",\n" + + " \"mfa\": false\n" + + " },\n" + + " \"user\": {\n" + + " \"account_uid\": \"\",\n" + + " \"credential_uid\": \"\",\n" + + " \"name\": null,\n" + + " \"type\": \"AssumedRole\",\n" + + " \"uid\": \"\",\n" + + " \"uuid\": \"\"\n" + + " }\n" + + " },\n" + + " \"api\": {\n" + + " \"operation\": \"CreateFunction\",\n" + + " \"request\": {\n" + + " \"uid\": \"0966237c-6279-43f4-a9d7-1eb416fca17d\"\n" + + " },\n" + + " \"response\": {\n" + + " \"error\": null,\n" + + " \"message\": null\n" + + " },\n" + + " \"service\": {\n" + + " \"name\": \"lambda.amazonaws.com\"\n" + + " },\n" + + " \"version\": null\n" + + " },\n" + + " \"category_name\": \"Audit Activity\",\n" + + " \"category_uid\": 3,\n" + + " \"class_name\": \"account_change\",\n" + + " \"class_uid\": 3001,\n" + + " \"cloud\": {\n" + + " \"provider\": \"AWS\",\n" + + " \"region\": \"us-east-1\"\n" + + " },\n" + + " \"dst_endpoint\": null,\n" + + " \"http_request\": {\n" + + " \"user_agent\": \"Boto3/1.26.90 Python/3.7.17 Linux/test.amzn2.x86_64 exec-env/AWS_Lambda_python3.7 Botocore/1.29.90\"\n" + + " },\n" + + " \"metadata\": {\n" + + " \"product\": {\n" + + " \"feature\": {\n" + + " \"name\": \"Management\"\n" + + " },\n" + + " \"name\": \"cloudtrail\",\n" + + " \"vendor_name\": \"AWS\",\n" + + " \"version\": \"1.08\"\n" + + " },\n" + + " \"profiles\": [\n" + + " \"cloud\"\n" + + " ],\n" + + " \"uid\": \"\",\n" + + " \"version\": \"1.0.0-rc.2\"\n" + + " },\n" + + " \"mfa\": null,\n" + + " \"resources\": null,\n" + + " \"severity\": \"Informational\",\n" + + " \"severity_id\": 1,\n" + + " \"src_endpoint\": {\n" + + " \"domain\": null,\n" + + " \"ip\": \"\",\n" + + " \"uid\": null\n" + + " },\n" + + " \"status\": \"Success\",\n" + + " \"status_id\": 1,\n" + + " \"time\": 1702952105000,\n" + + " \"type_name\": \"Account Change: Detach Policy\",\n" + + " \"type_uid\": 300108,\n" + + " \"unmapped\": {\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": \"true\",\n" + + " \"readOnly\": \"false\",\n" + + " \"recipientAccountId\": \"\",\n" + + " \"requestParameters.instanceProfileName\": \"\",\n" + + " \"tlsDetails.cipherSuite\": \"\",\n" + + " \"tlsDetails.clientProvidedHostHeader\": \"iam.amazonaws.com\",\n" + + " \"tlsDetails.tlsVersion\": \"TLSv1.2\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.accountId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.principalId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.type\": \"Role\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.userName\": \"\"\n" + + " },\n" + + " \"user\": {\n" + + " \"name\": \"\",\n" + + " \"uid\": null,\n" + + " \"uuid\": null\n" + + " }\n" + + "}"; + } + + public static String randomCloudtrailDoc(String user, String event) { + return "{\n" + + " \"eventVersion\": \"1.08\",\n" + + " \"userIdentity\": {\n" + + " \"type\": \"IAMUser\",\n" + + " \"principalId\": \"AIDA6ON6E4XEGITEXAMPLE\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Mary\",\n" + + " \"accountId\": \"888888888888\",\n" + + " \"accessKeyId\": \"AKIAIOSFODNN7EXAMPLE\",\n" + + " \"userName\": \"Mary\",\n" + + " \"sessionContext\": {\n" + + " \"sessionIssuer\": {},\n" + + " \"webIdFederationData\": {},\n" + + " \"attributes\": {\n" + + " \"creationDate\": \"2023-07-19T21:11:57Z\",\n" + + " \"mfaAuthenticated\": \"false\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": \"2023-07-19T21:25:09Z\",\n" + + " \"eventSource\": \"iam.amazonaws.com\",\n" + + " \"EventName\": \"" + event + "\",\n" + + " \"awsRegion\": \"us-east-1\",\n" + + " \"sourceIPAddress\": \"192.0.2.0\",\n" + + " \"AccountName\": \"" + user + "\",\n" + + " \"userAgent\": \"aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-user\",\n" + + " \"requestParameters\": {\n" + + " \"userName\": \"" + user + "\"\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"user\": {\n" + + " \"path\": \"/\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Richard\",\n" + + " \"userId\": \"AIDA6ON6E4XEP7EXAMPLE\",\n" + + " \"createDate\": \"Jul 19, 2023 9:25:09 PM\",\n" + + " \"userName\": \"Richard\"\n" + + " }\n" + + " },\n" + + " \"requestID\": \"2d528c76-329e-410b-9516-EXAMPLE565dc\",\n" + + " \"eventID\": \"ba0801a1-87ec-4d26-be87-EXAMPLE75bbb\",\n" + + " \"readOnly\": false,\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": true,\n" + + " \"recipientAccountId\": \"888888888888\",\n" + + " \"eventCategory\": \"Management\",\n" + + " \"tlsDetails\": {\n" + + " \"tlsVersion\": \"TLSv1.2\",\n" + + " \"cipherSuite\": \"ECDHE-RSA-AES128-GCM-SHA256\",\n" + + " \"clientProvidedHostHeader\": \"iam.amazonaws.com\"\n" + + " },\n" + + " \"sessionCredentialFromConsole\": \"true\"\n" + + "}"; + } + public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + @@ -1756,6 +2039,14 @@ public static String vpcFlowMappings() { " }"; } + private static String randomString() { + return OpenSearchTestCase.randomAlphaOfLengthBetween(2, 16); + } + + public static String randomLowerCaseString() { + return randomString().toLowerCase(Locale.ROOT); + } + public static XContentParser parser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 56ff61983..392909cfb 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.resthandler; import org.apache.http.HttpStatus; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; import org.junit.Assert; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -28,12 +30,20 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import static java.util.Collections.emptyList; + +import static org.opensearch.securityanalytics.TestHelpers.cloudtrailOcsfMappings; import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithDotFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithEcsFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailOcsfDoc; import static org.opensearch.securityanalytics.TestHelpers.randomDetector; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; @@ -51,7 +61,6 @@ import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; - import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1989,6 +1998,234 @@ public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutDateMappin assertEquals(0, noOfSigmaRuleMatches); } + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRule() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + indexDoc(index, "0", randomCloudtrailDoc("A12346", "CREATED")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomCloudtrailAggrRule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("A12345", "CREATED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + indexDoc(index, "2", randomCloudtrailDoc("A12345", "DELETED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithDotFields() throws IOException { + String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithDotFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailOcsfDoc()); + indexDoc(index, "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOException { + String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"api.operation\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.event_source\": {\n" + + " \"path\": \"api.service.name\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.aws_region\": {\n" + + " \"path\": \"cloud.region\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithEcsFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailOcsfDoc()); + indexDoc(index, "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index 395f15a79..71b855711 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -82,7 +82,7 @@ public void testCountAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -116,7 +116,7 @@ public void testSumAggregationWithGroupBy() throws IOException, SigmaError { // inputs.query.aggregations -> Query - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); // triggers.bucket_level_trigger.condition -> Condition Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -149,7 +149,7 @@ public void testMinAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -181,7 +181,7 @@ public void testMaxAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -213,7 +213,83 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } + + public void testCloudtrailAggregationRule() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " event:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by accountid > 2", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("(event: \"CREATED\") OR (event: \"DELETED\")", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"accountid\"}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 2.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } + + public void testCloudtrailAggregationRuleWithDotFields() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"CreateFunction\")) OR ((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"Invoke\"))", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"cloud.region\"},\"aggs\":{\"api_operation\":{\"value_count\":{\"field\":\"api.operation\"}}}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"api_operation\":\"api.operation\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.api_operation > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } } \ No newline at end of file From 1b948ac8a142977385445837f55116f8e60a8101 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 7 Mar 2024 09:40:44 -0800 Subject: [PATCH 2/5] Fix duplicate ecs mappings which returns incorrect log index field in mapping view API (#786) (#788) * field mapping changes Signed-off-by: Joanne Wang * add integ test Signed-off-by: Joanne Wang * turn unmappedfieldaliases as set and add integ test Signed-off-by: Joanne Wang * add comments Signed-off-by: Joanne Wang * fix integ tests Signed-off-by: Joanne Wang * moved logic to method for better readability Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang --- .../mapper/MapperService.java | 48 ++++- .../mapper/MapperRestApiIT.java | 169 ++++++++++++++++++ .../resthandler/OCSFDetectorRestApiIT.java | 4 +- 3 files changed, 210 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index e50af189a..a1d7e5746 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -18,7 +18,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.OpenSearchStatusException; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; @@ -477,13 +476,16 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { String rawPath = requiredField.getRawField(); String ocsfPath = requiredField.getOcsf(); if (allFieldsFromIndex.contains(rawPath)) { - if (alias != null) { - // Maintain list of found paths in index - applyableAliases.add(alias); - } else { - applyableAliases.add(rawPath); + // if the alias was already added into applyable aliases, then skip to avoid duplicates + if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) { + if (alias != null) { + // Maintain list of found paths in index + applyableAliases.add(alias); + } else { + applyableAliases.add(rawPath); + } + pathsOfApplyableAliases.add(rawPath); } - pathsOfApplyableAliases.add(rawPath); } else if (allFieldsFromIndex.contains(ocsfPath)) { applyableAliases.add(alias); pathsOfApplyableAliases.add(ocsfPath); @@ -497,13 +499,21 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { } } + // turn unmappedFieldAliases into a set to remove duplicates + Set setOfUnmappedFieldAliases = new HashSet<>(unmappedFieldAliases); + + // filter out aliases that were included in applyableAliases already + List filteredUnmappedFieldAliases = setOfUnmappedFieldAliases.stream() + .filter(e -> false == applyableAliases.contains(e)) + .collect(Collectors.toList()); + Map> aliasMappingFields = new HashMap<>(); XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject(); for (LogType.Mapping mapping: requiredFields) { if (allFieldsFromIndex.contains(mapping.getOcsf())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); } else if (mapping.getEcs() != null) { - aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases); } else if (mapping.getEcs() == null) { aliasMappingFields.put(mapping.getRawField(), Map.of("type", "alias", "path", mapping.getRawField())); } @@ -520,7 +530,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { .collect(Collectors.toList()); actionListener.onResponse( - new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, unmappedFieldAliases) + new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, filteredUnmappedFieldAliases) ); } catch (Exception e) { actionListener.onFailure(e); @@ -534,6 +544,26 @@ public void onFailure(Exception e) { }); } + /** + * Only updates the alias mapping fields if the ecs key has not been mapped yet + * or if pathOfApplyableAliases contains the raw field + * + * @param mapping + * @param aliasMappingFields + * @param pathsOfApplyableAliases + */ + private static void shouldUpdateEcsMappingAndMaybeUpdates(LogType.Mapping mapping, Map> aliasMappingFields, List pathsOfApplyableAliases) { + // check if aliasMappingFields already contains a key + if (aliasMappingFields.containsKey(mapping.getEcs())) { + // if the pathOfApplyableAliases contains the raw field, then override the existing map + if (pathsOfApplyableAliases.contains(mapping.getRawField())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + } + } else { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + } + } + /** * Given index name, resolves it to single concrete index, depending on what initial indexName is. * In case of Datastream or Alias, WriteIndex would be returned. In case of index pattern, newest index by creation date would be returned. diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 91057069f..f3d23ae34 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -390,6 +390,112 @@ public void testGetMappingsViewLinuxSuccess() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + // Tests mappings where multiple raw fields correspond to one ecs value + public void testGetMappingsViewWindowsSuccess() throws IOException { + + String testIndexName = "get_mappings_view_index"; + + createSampleWindex(testIndexName); + + // Execute GetMappingsViewAction to add alias mapping for index + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", testIndexName); + request.addParameter("rule_topic", "windows"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + assertEquals(3, props.size()); + assertTrue(props.containsKey("winlog.event_data.LogonType")); + assertTrue(props.containsKey("winlog.provider_name")); + assertTrue(props.containsKey("host.hostname")); + + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(3, unmappedIndexFields.size()); + assert(unmappedIndexFields.contains("plain1")); + assert(unmappedIndexFields.contains("ParentUser.first")); + assert(unmappedIndexFields.contains("ParentUser.last")); + + // Verify unmapped field aliases + List filteredUnmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(191, filteredUnmappedFieldAliases.size()); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_data.LogonType")); + assert(!filteredUnmappedFieldAliases.contains("winlog.provider_name")); + assert(!filteredUnmappedFieldAliases.contains("host.hostname")); + + // Index a doc for a field with multiple raw fields corresponding to one ecs field + indexDoc(testIndexName, "1", "{ \"EventID\": 1 }"); + // Execute GetMappingsViewAction to add alias mapping for index + request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", testIndexName); + request.addParameter("rule_topic", "windows"); + response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + respMap = responseAsMap(response); + + // Verify alias mappings + props = (Map) respMap.get("properties"); + assertEquals(4, props.size()); + assertTrue(props.containsKey("winlog.event_id")); + + // verify unmapped index fields + unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(3, unmappedIndexFields.size()); + + // verify unmapped field aliases + filteredUnmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(190, filteredUnmappedFieldAliases.size()); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_id")); + } + + // Tests mappings where multiple raw fields correspond to one ecs value and all fields are present in the index + public void testGetMappingsViewMulitpleRawFieldsSuccess() throws IOException { + + String testIndexName = "get_mappings_view_index"; + + createSampleWindex(testIndexName); + String sampleDoc = "{" + + " \"EventID\": 1," + + " \"EventId\": 2," + + " \"event_uid\": 3" + + "}"; + indexDoc(testIndexName, "1", sampleDoc); + + // Execute GetMappingsViewAction to add alias mapping for index + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", testIndexName); + request.addParameter("rule_topic", "windows"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + assertEquals(4, props.size()); + assertTrue(props.containsKey("winlog.event_data.LogonType")); + assertTrue(props.containsKey("winlog.provider_name")); + assertTrue(props.containsKey("host.hostname")); + assertTrue(props.containsKey("winlog.event_id")); + + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(5, unmappedIndexFields.size()); + + // Verify unmapped field aliases + List filteredUnmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(190, filteredUnmappedFieldAliases.size()); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_data.LogonType")); + assert(!filteredUnmappedFieldAliases.contains("winlog.provider_name")); + assert(!filteredUnmappedFieldAliases.contains("host.hostname")); + assert(!filteredUnmappedFieldAliases.contains("winlog.event_id")); + } + public void testCreateMappings_withDatastream_success() throws IOException { String datastream = "test_datastream"; @@ -1273,6 +1379,69 @@ private void createSampleIndex(String indexName, Settings settings, String alias assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + private void createSampleWindex(String indexName) throws IOException { + createSampleWindex(indexName, Settings.EMPTY, null); + } + + private void createSampleWindex(String indexName, Settings settings, String aliases) throws IOException { + String indexMapping = + " \"properties\": {" + + " \"LogonType\": {" + + " \"type\": \"integer\"" + + " }," + + " \"Provider\": {" + + " \"type\": \"text\"" + + " }," + + " \"hostname\": {" + + " \"type\": \"text\"" + + " }," + + " \"plain1\": {" + + " \"type\": \"integer\"" + + " }," + + " \"ParentUser\":{" + + " \"type\":\"nested\"," + + " \"properties\":{" + + " \"first\":{" + + " \"type\":\"text\"," + + " \"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + "}" + + "}" + + "}," + + " \"last\":{" + + "\"type\":\"text\"," + + "\"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + "}" + + "}" + + "}" + + "}" + + "}" + + " }"; + + createIndex(indexName, settings, indexMapping, aliases); + + // Insert sample doc with event_uid not explicitly mapped + String sampleDoc = "{" + + " \"LogonType\":1," + + " \"Provider\":\"Microsoft-Windows-Security-Auditing\"," + + " \"hostname\":\"FLUXCAPACITOR\"" + + "}"; + + // Index doc + Request indexRequest = new Request("POST", indexName + "/_doc?refresh=wait_for"); + indexRequest.setJsonEntity(sampleDoc); + Response response = client().performRequest(indexRequest); + assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + // Refresh everything + response = client().performRequest(new Request("POST", "_refresh")); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + private void createSampleDatastream(String datastreamName) throws IOException { String indexMapping = " \"properties\": {" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index 248bb8798..07ab7164d 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -436,7 +436,7 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException { assertEquals(20, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(25, unmappedFieldAliases.size()); + assertEquals(24, unmappedFieldAliases.size()); } @SuppressWarnings("unchecked") @@ -502,7 +502,7 @@ public void testRawCloudtrailGetMappingsViewApi() throws IOException { assertEquals(17, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(26, unmappedFieldAliases.size()); + assertEquals(25, unmappedFieldAliases.size()); } @SuppressWarnings("unchecked") From 840a8bf696cd5d1acef325eda464cb23e661f823 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Sun, 10 Mar 2024 17:56:14 -0700 Subject: [PATCH 3/5] Fix get mappings view API incorrectly returning ecs path (#867) * add logic and integ tests to not add duplicate to log-types-config index Signed-off-by: Joanne Wang * change naming for existingFieldMapping and change contains to equals Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang --- .../logtype/LogTypeService.java | 23 ++-- .../securityanalytics/TestHelpers.java | 102 +++++++++++++++--- .../resthandler/OCSFDetectorRestApiIT.java | 46 ++++++++ 3 files changed, 148 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index 85f2242ad..2a0665b23 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -348,15 +348,26 @@ private List mergeFieldMappings(List existingF List newFieldMappings = new ArrayList<>(); fieldMappingDocs.forEach( newFieldMapping -> { Optional foundFieldMappingDoc = Optional.empty(); - for (FieldMappingDoc e: existingFieldMappings) { - if (e.getRawField().equals(newFieldMapping.getRawField())) { + for (FieldMappingDoc existingFieldMapping: existingFieldMappings) { + if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) { if (( - e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null && - e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField)) + existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null && + existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField)) ) || ( - e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null + existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null )) { - foundFieldMappingDoc = Optional.of(e); + foundFieldMappingDoc = Optional.of(existingFieldMapping); + } + // Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings + // Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed + } else { + String id = existingFieldMapping.getId(); + int indexOfPipe = id.indexOf("|"); + if (indexOfPipe != -1) { + String ecsIdField = id.substring(indexOfPipe + 1); + if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) { + foundFieldMappingDoc = Optional.of(existingFieldMapping); + } } } } diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 843abbf3c..2794651bd 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -239,8 +239,8 @@ public static String randomRule() { "level: high"; } - public static String randomNullRule() { - return "title: null field\n" + + public static String randomRuleWithRawField() { + return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + "references:\n" + @@ -258,18 +258,50 @@ public static String randomNullRule() { "logsource:\n" + " product: rpc_firewall\n" + " category: application\n" + - " definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + "detection:\n" + " selection:\n" + - " EventID: 22\n" + - " RecordNumber: null\n" + + " eventName: testinghere\n" + " condition: selection\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + "level: high"; } - public static String randomRuleForMappingView(String field) { + public static String randomRuleWithNotCondition() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection1:\n" + + " AccountType: TestAccountType\n" + + " selection2:\n" + + " AccountName: TestAccountName\n" + + " selection3:\n" + + " EventID: 22\n" + + " condition: (not selection1 and not selection2) and selection3\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomRuleWithNotConditionBoolAndNum() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -290,8 +322,42 @@ public static String randomRuleForMappingView(String field) { " category: application\n" + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + "detection:\n" + + " selection1:\n" + + " Initiated: \"false\"\n" + + " selection2:\n" + + " AccountName: TestAccountName\n" + + " selection3:\n" + + " EventID: 21\n" + + " condition: not selection1 and not selection3\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomNullRule() { + return "title: null field\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + " selection:\n" + - " "+ field + ": 'ACL'\n" + + " EventID: 22\n" + + " RecordNumber: null\n" + " condition: selection\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + @@ -1635,31 +1701,31 @@ public static String windowsIndexMappingOnlyNumericAndText() { " }"; } - public static String randomDocWithNullField() { - return "{\n" + - "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + + public static String randomDoc(int severity, int version, String opCode) { + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + - "\"SeverityValue\":2,\n" + + "\"SeverityValue\":%s,\n" + "\"Severity\":\"INFO\",\n" + "\"EventID\":22,\n" + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + - "\"Version\":5,\n" + + "\"Version\":%s,\n" + "\"TaskValue\":22,\n" + "\"OpcodeValue\":0,\n" + - "\"RecordNumber\":null,\n" + + "\"RecordNumber\":9532,\n" + "\"ExecutionProcessID\":1996,\n" + "\"ExecutionThreadID\":2616,\n" + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + - "\"Domain\":\"NTAUTHORITY\",\n" + + "\"Domain\":\"NT AUTHORITY\",\n" + "\"AccountName\":\"SYSTEM\",\n" + "\"UserID\":\"S-1-5-18\",\n" + "\"AccountType\":\"User\",\n" + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + - "\"Opcode\":\"Info\",\n" + + "\"Opcode\":\"%s\",\n" + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + @@ -1671,9 +1737,11 @@ public static String randomDocWithNullField() { "\"CommandLine\": \"eachtest\",\n" + "\"Initiated\": \"true\"\n" + "}"; + return String.format(Locale.ROOT, doc, severity, version, opCode); + } - public static String randomDoc(int severity, int version, String opCode) { + public static String randomDocForNotCondition(int severity, int version, String opCode) { String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + @@ -1691,7 +1759,6 @@ public static String randomDoc(int severity, int version, String opCode) { "\"ExecutionThreadID\":2616,\n" + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + "\"Domain\":\"NT AUTHORITY\",\n" + - "\"AccountName\":\"SYSTEM\",\n" + "\"UserID\":\"S-1-5-18\",\n" + "\"AccountType\":\"User\",\n" + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + @@ -1783,6 +1850,7 @@ public static String randomVpcFlowDoc() { " \"srcport\": 9000,\n" + " \"dstport\": 8000,\n" + " \"severity_id\": \"-1\",\n" + + " \"id.orig_h\": \"1.2.3.4\",\n" + " \"class_name\": \"Network Activity\"\n" + "}"; } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index 07ab7164d..8c1a14b52 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -439,6 +439,52 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException { assertEquals(24, unmappedFieldAliases.size()); } + @SuppressWarnings("unchecked") + public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOException { + String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "cloudtrail"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(18, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(20, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(25, unmappedFieldAliases.size()); + + // create a cloudtrail rule with a raw field + String rule = randomRuleWithRawField(); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + // check the mapping view API again to ensure it's the same after rule is created + Response response2 = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response2.getStatusLine().getStatusCode()); + Map respMap2 = responseAsMap(response2); + // Verify alias mappings + Map props2 = (Map) respMap2.get("properties"); + Assert.assertEquals(18, props2.size()); + // Verify unmapped index fields + List unmappedIndexFields2 = (List) respMap2.get("unmapped_index_fields"); + assertEquals(20, unmappedIndexFields2.size()); + // Verify unmapped field aliases + List unmappedFieldAliases2 = (List) respMap2.get("unmapped_field_aliases"); + assertEquals(25, unmappedFieldAliases2.size()); + // Verify that first response and second response are the same after rule was indexed + assertEquals(props, props2); + assertEquals(unmappedIndexFields, unmappedIndexFields2); + assertEquals(unmappedFieldAliases, unmappedFieldAliases2); + } + @SuppressWarnings("unchecked") public void testOCSFVpcflowGetMappingsViewApi() throws IOException { String index = createTestIndex("vpcflow", ocsfVpcflowMappings()); From 09e4b6cc4fe62d94a6ffb3b65a6606f1b43f3bcb Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Wed, 13 Mar 2024 15:56:06 -0700 Subject: [PATCH 4/5] fix integ test (#918) Signed-off-by: Joanne Wang --- .../securityanalytics/resthandler/OCSFDetectorRestApiIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index 8c1a14b52..f22f70877 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -458,7 +458,7 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept assertEquals(20, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(25, unmappedFieldAliases.size()); + assertEquals(24, unmappedFieldAliases.size()); // create a cloudtrail rule with a raw field String rule = randomRuleWithRawField(); @@ -478,7 +478,7 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept assertEquals(20, unmappedIndexFields2.size()); // Verify unmapped field aliases List unmappedFieldAliases2 = (List) respMap2.get("unmapped_field_aliases"); - assertEquals(25, unmappedFieldAliases2.size()); + assertEquals(24, unmappedFieldAliases2.size()); // Verify that first response and second response are the same after rule was indexed assertEquals(props, props2); assertEquals(unmappedIndexFields, unmappedIndexFields2); From e5fd3a9a9988e15c4507082568a2a6300e7e8bf0 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Wed, 13 Mar 2024 22:49:35 -0700 Subject: [PATCH 5/5] fix integ tests Signed-off-by: Joanne Wang --- .../securityanalytics/TestHelpers.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 2794651bd..2d001bc85 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -387,6 +387,35 @@ public static String randomCloudtrailRuleForCorrelations(String value) { " - " + value; } + public static String randomRuleForMappingView(String field) { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " "+ field + ": 'ACL'\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String randomRuleForCustomLogType() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -1701,6 +1730,43 @@ public static String windowsIndexMappingOnlyNumericAndText() { " }"; } + public static String randomDocWithNullField() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":null,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" +