From 1d4f5d7e4d161a423e9869995ee49f3fbbe38743 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Sat, 13 Jan 2024 09:08:56 +0000 Subject: [PATCH] support object fields in aggregation based sigma rules Signed-off-by: Subhobrata Dey --- .../rules/backend/OSQueryBackend.java | 23 +++-- .../securityanalytics/TestHelpers.java | 40 +++++++- .../resthandler/DetectorMonitorRestApiIT.java | 93 +++++++++++++++++++ .../aggregation/AggregationBackendTests.java | 10 +- 4 files changed, 151 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 8c750bf05..2d1763a43 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -367,8 +367,9 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { fieldName = "_index"; fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index"); } else { - fieldName = aggregation.getGroupByField(); - fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", aggregation.getGroupByField()); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fieldName = mappedGroupByField; + fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", mappedGroupByField); } aggBuilder.field(fieldName); fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, "_cnt", "_count", "result_agg", "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()); @@ -376,17 +377,23 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold())); condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_count"), script, "result_agg", null); } else { - fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField().replace(".", "_"), aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), aggregation.getAggField()); - fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField().replace(".", "_"), aggregation.getAggField(), "result_agg", aggregation.getAggField().replace(".", "_"), aggregation.getCompOperator(), aggregation.getThreshold()); + /** + * removing dots to eliminate dots in aggregation names + */ + String mappedAggField = getFinalField(aggregation.getAggField()); + String mappedAggFieldUpdated = mappedAggField.replace(".", "_"); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", mappedGroupByField, mappedAggFieldUpdated, aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), mappedAggField); + fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, mappedAggFieldUpdated, mappedAggField, "result_agg", mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold()); // Add subaggregation - AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField()); + AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), mappedAggField); if (subAgg != null) { - aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg); + aggBuilder.field(mappedGroupByField).subAggregation(subAgg); } - Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField().replace(".", "_"), aggregation.getCompOperator(), aggregation.getThreshold())); - condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField().replace(".", "_"), aggregation.getAggField().replace(".", "_")), script, "result_agg", null); + Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold())); + condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(mappedAggFieldUpdated, mappedAggFieldUpdated), script, "result_agg", null); } AggregationQueries aggregationQueries = new AggregationQueries(); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 8f84de039..d907b797c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -942,6 +942,35 @@ public static String randomCloudtrailAggrRuleWithDotFields() { " - attack.t1078"; } + public static String randomCloudtrailAggrRuleWithEcsFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(eventName) by awsRegion > 1\n" + + " selection1:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + public static String cloudtrailOcsfMappings() { return "\"properties\": {\n" + " \"time\": {\n" + @@ -950,8 +979,15 @@ public static String cloudtrailOcsfMappings() { " \"cloud.region\": {\n" + " \"type\": \"keyword\"\n" + " },\n" + - " \"api.operation\": {\n" + - " \"type\": \"keyword\"\n" + + " \"api\": {\n" + + " \"properties\": {\n" + + " \"operation\": {\"type\": \"keyword\"},\n" + + " \"service\": {\n" + + " \"properties\": {\n" + + " \"name\": {\"type\": \"text\"}\n" + + " }\n" + + " }\n" + + " }\n" + " }\n" + " }\n" + " }"; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 73558047b..3a11300ee 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -41,6 +41,7 @@ import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithDotFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithEcsFields; import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailOcsfDoc; import static org.opensearch.securityanalytics.TestHelpers.randomDetector; @@ -2046,6 +2047,98 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithDotFields() throws IOExc assertEquals(1, getFindingsBody.get("total_findings")); } + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOException { + String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"api.operation\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.event_source\": {\n" + + " \"path\": \"api.service.name\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.aws_region\": {\n" + + " \"path\": \"cloud.region\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithEcsFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailOcsfDoc()); + indexDoc(index, "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum(); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index acf88aca8..71b855711 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -82,7 +82,7 @@ public void testCountAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -116,7 +116,7 @@ public void testSumAggregationWithGroupBy() throws IOException, SigmaError { // inputs.query.aggregations -> Query - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); // triggers.bucket_level_trigger.condition -> Condition Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -149,7 +149,7 @@ public void testMinAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -181,7 +181,7 @@ public void testMaxAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -213,7 +213,7 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); }