Skip to content

Commit

Permalink
support object fields in aggregation based sigma rules
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@sbcd90
sbcd90 committed Jan 13, 2024
1 parent d626a95 commit 1d4f5d7
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 15 deletions.
23 changes: 15 additions & 8 deletions src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,26 +367,33 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) {
fieldName = "_index";
fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index");
} else {
fieldName = aggregation.getGroupByField();
fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", aggregation.getGroupByField());
String mappedGroupByField = getMappedField(aggregation.getGroupByField());
fieldName = mappedGroupByField;
fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", mappedGroupByField);
}
aggBuilder.field(fieldName);
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, "_cnt", "_count", "result_agg", "_cnt", aggregation.getCompOperator(), aggregation.getThreshold());

Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_count"), script, "result_agg", null);
} else {
fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField().replace(".", "_"), aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), aggregation.getAggField());
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField().replace(".", "_"), aggregation.getAggField(), "result_agg", aggregation.getAggField().replace(".", "_"), aggregation.getCompOperator(), aggregation.getThreshold());
/**
* removing dots to eliminate dots in aggregation names
*/
String mappedAggField = getFinalField(aggregation.getAggField());
String mappedAggFieldUpdated = mappedAggField.replace(".", "_");
String mappedGroupByField = getMappedField(aggregation.getGroupByField());
fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", mappedGroupByField, mappedAggFieldUpdated, aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), mappedAggField);
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, mappedAggFieldUpdated, mappedAggField, "result_agg", mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold());

// Add subaggregation
AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField());
AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), mappedAggField);
if (subAgg != null) {
aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg);
aggBuilder.field(mappedGroupByField).subAggregation(subAgg);
}

Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField().replace(".", "_"), aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField().replace(".", "_"), aggregation.getAggField().replace(".", "_")), script, "result_agg", null);
Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(mappedAggFieldUpdated, mappedAggFieldUpdated), script, "result_agg", null);
}

AggregationQueries aggregationQueries = new AggregationQueries();
Expand Down
40 changes: 38 additions & 2 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -942,6 +942,35 @@ public static String randomCloudtrailAggrRuleWithDotFields() {
" - attack.t1078";
}

public static String randomCloudtrailAggrRuleWithEcsFields() {
return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" +
"title: test\n" +
"description: Detects when an user creates or invokes a lambda function.\n" +
"status: experimental\n" +
"author: deysubho\n" +
"date: 2023/12/07\n" +
"modified: 2023/12/07\n" +
"logsource:\n" +
" category: cloudtrail\n" +
"level: low\n" +
"detection:\n" +
" condition: selection1 or selection2 | count(eventName) by awsRegion > 1\n" +
" selection1:\n" +
" eventSource:\n" +
" - lambda.amazonaws.com\n" +
" eventName:\n" +
" - CreateFunction\n" +
" selection2:\n" +
" eventSource:\n" +
" - lambda.amazonaws.com\n" +
" eventName: \n" +
" - Invoke\n" +
" timeframe: 20m\n" +
" tags:\n" +
" - attack.privilege_escalation\n" +
" - attack.t1078";
}

public static String cloudtrailOcsfMappings() {
return "\"properties\": {\n" +
" \"time\": {\n" +
Expand All @@ -950,8 +979,15 @@ public static String cloudtrailOcsfMappings() {
" \"cloud.region\": {\n" +
" \"type\": \"keyword\"\n" +
" },\n" +
" \"api.operation\": {\n" +
" \"type\": \"keyword\"\n" +
" \"api\": {\n" +
" \"properties\": {\n" +
" \"operation\": {\"type\": \"keyword\"},\n" +
" \"service\": {\n" +
" \"properties\": {\n" +
" \"name\": {\"type\": \"text\"}\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }";
Expand Down
93 changes: 93 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule;
import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule;
import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithDotFields;
import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithEcsFields;
import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc;
import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailOcsfDoc;
import static org.opensearch.securityanalytics.TestHelpers.randomDetector;
Expand Down Expand Up @@ -2046,6 +2047,98 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithDotFields() throws IOExc
assertEquals(1, getFindingsBody.get("total_findings"));
}

@SuppressWarnings("unchecked")
public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOException {
String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings());

// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{\n" +
" \"index_name\": \"" + index + "\",\n" +
" \"rule_topic\": \"cloudtrail\",\n" +
" \"partial\": true,\n" +
" \"alias_mappings\": {\n" +
" \"properties\": {\n" +
" \"aws.cloudtrail.event_name\": {\n" +
" \"path\": \"api.operation\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"aws.cloudtrail.event_source\": {\n" +
" \"path\": \"api.service.name\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"aws.cloudtrail.aws_region\": {\n" +
" \"path\": \"cloud.region\",\n" +
" \"type\": \"alias\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}"
);

Response createMappingResponse = client().performRequest(createMappingRequest);

assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());
indexDoc(index, "0", randomCloudtrailOcsfDoc());

String rule = randomCloudtrailAggrRuleWithEcsFields();

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"),
new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse));
Map<String, Object> responseBody = asMap(createResponse);
String createdId = responseBody.get("_id").toString();

DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)),
List.of());
Detector detector = randomDetectorWithInputsAndTriggers(List.of(input),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of())));

createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

responseBody = asMap(createResponse);

createdId = responseBody.get("_id").toString();
int createdVersion = Integer.parseInt(responseBody.get("_version").toString());
Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId);
Assert.assertTrue("incorrect version", createdVersion > 0);
Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location"));
Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("rule_topic_index"));
Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("findings_index"));
Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("alert_index"));

String detectorTypeInResponse = (String) ((Map<String, Object>)responseBody.get("detector")).get("detector_type");
Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse);

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

String workflowId = ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0);

indexDoc(index, "1", randomCloudtrailOcsfDoc());
indexDoc(index, "2", randomCloudtrailOcsfDoc());
executeAlertingWorkflow(workflowId, Collections.emptyMap());

Map<String, String> params = new HashMap<>();
params.put("detector_id", createdId);
Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);

// Assert findings
assertNotNull(getFindingsBody);
assertEquals(1, getFindingsBody.get("total_findings"));
}

private static void assertRuleMonitorFinding(Map<String, Object> executeResults, String ruleId, int expectedDocCount, List<String> expectedTriggerResult) {
List<Map<String, Object>> buckets = ((List<Map<String, Object>>) (((Map<String, Object>) ((Map<String, Object>) ((Map<String, Object>) ((List<Object>) ((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets")));
Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum();
Expand Down
10 changes: 5 additions & 5 deletions ...test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void testCountAggregationWithGroupBy() throws IOException, SigmaError {
String aggQuery = aggQueries.getAggQuery();
String bucketTriggerQuery = aggQueries.getBucketTriggerQuery();

Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"}}}", aggQuery);
Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"}}}", aggQuery);
Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery);
}

Expand Down Expand Up @@ -116,7 +116,7 @@ public void testSumAggregationWithGroupBy() throws IOException, SigmaError {


// inputs.query.aggregations -> Query
Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery);
Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery);
// triggers.bucket_level_trigger.condition -> Condition
Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery);
}
Expand Down Expand Up @@ -149,7 +149,7 @@ public void testMinAggregationWithGroupBy() throws IOException, SigmaError {
String aggQuery = aggQueries.getAggQuery();
String bucketTriggerQuery = aggQueries.getBucketTriggerQuery();

Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery);
Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery);
Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery);
}

Expand Down Expand Up @@ -181,7 +181,7 @@ public void testMaxAggregationWithGroupBy() throws IOException, SigmaError {
String aggQuery = aggQueries.getAggQuery();
String bucketTriggerQuery = aggQueries.getBucketTriggerQuery();

Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery);
Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery);
Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery);
}

Expand Down Expand Up @@ -213,7 +213,7 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError {
String aggQuery = aggQueries.getAggQuery();
String bucketTriggerQuery = aggQueries.getBucketTriggerQuery();

Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery);
Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery);
Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery);
}

Expand Down

0 comments on commit 1d4f5d7

Please sign in to comment.