Skip to content

Commit

Permalink
support object fields in aggregation based sigma rules
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@sbcd90
sbcd90 committed Dec 19, 2023
1 parent c373343 commit 7f1b86a
Show file tree
Hide file tree
Showing 7 changed files with 302 additions and 53 deletions.
88 changes: 44 additions & 44 deletions ...erated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) {
"\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+
"\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+
"\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+
"\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+
"\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+
"\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+
"\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+
"\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+
"\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+
"\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+
"\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+
"\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+
"\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+
"&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+
"\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+
"\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+
"\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+
"\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+
"\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+
">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+
"=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+
"\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+
"\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+
"\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+
"u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+
"\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+
"\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+
"\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+
"\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+
"\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+
"C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+
"F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+
"\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+
"-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+
"\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+
"PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+
"\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+
"\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+
"\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+
"\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+
"\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+
"\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+
"\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+
"\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+
"\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+
"\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+
"\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
"\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+
"z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+
"\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+
"\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+
"\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+
"\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+
"\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+
"\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+
"\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+
"\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+
"\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+
"\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+
"\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+
"@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+
"\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+
"\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+
"\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+
"$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+
"\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+
"*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+
"-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+
"01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+
"\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+
"\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+
"i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+
"m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+
"\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+
"\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+
"\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+
"\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+
"KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+
"MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+
"\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+
"\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+
"\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+
"\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+
"YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+
"\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+
"\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+
"_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+
"\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+
"\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+
"fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+
"\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
public static final ATN _ATN =
new ATNDeserializer().deserialize(_serializedATN.toCharArray());
static {
Expand Down
2 changes: 1 addition & 1 deletion src/main/grammars/Aggregation.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ RPAREN : ')' ;

DECIMAL : '-'?[0-9]+('.'[0-9]+)? ;

IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ;
IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ;
WS : [ \r\t\u000C\n]+ -> skip ;

comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat
aggregationBuilder = new TermsAggregationBuilder(name).field(name);
break;
case "count":
aggregationBuilder = new ValueCountAggregationBuilder(name).field(name);
aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name);
break;
default:
throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction));
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,9 +361,9 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) {
BucketSelectorExtAggregationBuilder condition;
String bucketTriggerSelectorId = UUIDs.base64UUID();

if (aggregation.getAggFunction().equals("count")) {
if (aggregation.getAggFunction().equals("count") && aggregation.getAggField().equals("*")) {
String fieldName;
if (aggregation.getAggField().equals("*") && aggregation.getGroupByField() == null) {
if (aggregation.getGroupByField() == null) {
fieldName = "_index";
fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index");
} else {
Expand All @@ -376,17 +376,17 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) {
Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_count"), script, "result_agg", null);
} else {
fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField(), aggregation.getAggFunction(), aggregation.getAggField());
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField(), aggregation.getAggField(), "result_agg", aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold());
fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField().replace(".", "_"), aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), aggregation.getAggField());
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField().replace(".", "_"), aggregation.getAggField(), "result_agg", aggregation.getAggField().replace(".", "_"), aggregation.getCompOperator(), aggregation.getThreshold());

// Add subaggregation
AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField());
if (subAgg != null) {
aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg);
}

Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField(), aggregation.getAggField()), script, "result_agg", null);
Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField().replace(".", "_"), aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField().replace(".", "_"), aggregation.getAggField().replace(".", "_")), script, "result_agg", null);
}

AggregationQueries aggregationQueries = new AggregationQueries();
Expand Down
144 changes: 144 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -913,6 +913,50 @@ public static String randomCloudtrailAggrRule() {
" condition: selection | count(*) by AccountName >= 2";
}

public static String randomCloudtrailAggrRuleWithDotFields() {
return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" +
"title: test\n" +
"description: Detects when an user creates or invokes a lambda function.\n" +
"status: experimental\n" +
"author: deysubho\n" +
"date: 2023/12/07\n" +
"modified: 2023/12/07\n" +
"logsource:\n" +
" category: cloudtrail\n" +
"level: low\n" +
"detection:\n" +
" condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" +
" selection1:\n" +
" api.service.name:\n" +
" - lambda.amazonaws.com\n" +
" api.operation:\n" +
" - CreateFunction\n" +
" selection2:\n" +
" api.service.name:\n" +
" - lambda.amazonaws.com\n" +
" api.operation: \n" +
" - Invoke\n" +
" timeframe: 20m\n" +
" tags:\n" +
" - attack.privilege_escalation\n" +
" - attack.t1078";
}

public static String cloudtrailOcsfMappings() {
return "\"properties\": {\n" +
" \"time\": {\n" +
" \"type\": \"date\"\n" +
" },\n" +
" \"cloud.region\": {\n" +
" \"type\": \"keyword\"\n" +
" },\n" +
" \"api.operation\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }";
}

public static String windowsIndexMapping() {
return "\"properties\": {\n" +
" \"@timestamp\": {\"type\":\"date\"},\n" +
Expand Down Expand Up @@ -1789,6 +1833,106 @@ public static String randomAdLdapDoc() {
"}";
}

public static String randomCloudtrailOcsfDoc() {
return "{\n" +
" \"activity_id\": 8,\n" +
" \"activity_name\": \"Detach Policy\",\n" +
" \"actor\": {\n" +
" \"idp\": {\n" +
" \"name\": null\n" +
" },\n" +
" \"invoked_by\": null,\n" +
" \"session\": {\n" +
" \"created_time\": 1702510696000,\n" +
" \"issuer\": \"arn\",\n" +
" \"mfa\": false\n" +
" },\n" +
" \"user\": {\n" +
" \"account_uid\": \"\",\n" +
" \"credential_uid\": \"\",\n" +
" \"name\": null,\n" +
" \"type\": \"AssumedRole\",\n" +
" \"uid\": \"\",\n" +
" \"uuid\": \"\"\n" +
" }\n" +
" },\n" +
" \"api\": {\n" +
" \"operation\": \"CreateFunction\",\n" +
" \"request\": {\n" +
" \"uid\": \"0966237c-6279-43f4-a9d7-1eb416fca17d\"\n" +
" },\n" +
" \"response\": {\n" +
" \"error\": null,\n" +
" \"message\": null\n" +
" },\n" +
" \"service\": {\n" +
" \"name\": \"lambda.amazonaws.com\"\n" +
" },\n" +
" \"version\": null\n" +
" },\n" +
" \"category_name\": \"Audit Activity\",\n" +
" \"category_uid\": 3,\n" +
" \"class_name\": \"account_change\",\n" +
" \"class_uid\": 3001,\n" +
" \"cloud\": {\n" +
" \"provider\": \"AWS\",\n" +
" \"region\": \"us-east-1\"\n" +
" },\n" +
" \"dst_endpoint\": null,\n" +
" \"http_request\": {\n" +
" \"user_agent\": \"Boto3/1.26.90 Python/3.7.17 Linux/test.amzn2.x86_64 exec-env/AWS_Lambda_python3.7 Botocore/1.29.90\"\n" +
" },\n" +
" \"metadata\": {\n" +
" \"product\": {\n" +
" \"feature\": {\n" +
" \"name\": \"Management\"\n" +
" },\n" +
" \"name\": \"cloudtrail\",\n" +
" \"vendor_name\": \"AWS\",\n" +
" \"version\": \"1.08\"\n" +
" },\n" +
" \"profiles\": [\n" +
" \"cloud\"\n" +
" ],\n" +
" \"uid\": \"\",\n" +
" \"version\": \"1.0.0-rc.2\"\n" +
" },\n" +
" \"mfa\": null,\n" +
" \"resources\": null,\n" +
" \"severity\": \"Informational\",\n" +
" \"severity_id\": 1,\n" +
" \"src_endpoint\": {\n" +
" \"domain\": null,\n" +
" \"ip\": \"\",\n" +
" \"uid\": null\n" +
" },\n" +
" \"status\": \"Success\",\n" +
" \"status_id\": 1,\n" +
" \"time\": 1702952105000,\n" +
" \"type_name\": \"Account Change: Detach Policy\",\n" +
" \"type_uid\": 300108,\n" +
" \"unmapped\": {\n" +
" \"eventType\": \"AwsApiCall\",\n" +
" \"managementEvent\": \"true\",\n" +
" \"readOnly\": \"false\",\n" +
" \"recipientAccountId\": \"\",\n" +
" \"requestParameters.instanceProfileName\": \"\",\n" +
" \"tlsDetails.cipherSuite\": \"\",\n" +
" \"tlsDetails.clientProvidedHostHeader\": \"iam.amazonaws.com\",\n" +
" \"tlsDetails.tlsVersion\": \"TLSv1.2\",\n" +
" \"userIdentity.sessionContext.sessionIssuer.accountId\": \"\",\n" +
" \"userIdentity.sessionContext.sessionIssuer.principalId\": \"\",\n" +
" \"userIdentity.sessionContext.sessionIssuer.type\": \"Role\",\n" +
" \"userIdentity.sessionContext.sessionIssuer.userName\": \"\"\n" +
" },\n" +
" \"user\": {\n" +
" \"name\": \"\",\n" +
" \"uid\": null,\n" +
" \"uuid\": null\n" +
" }\n" +
"}";
}

public static String randomCloudtrailDoc(String user, String event) {
return "{\n" +
" \"eventVersion\": \"1.08\",\n" +
Expand Down
Loading

0 comments on commit 7f1b86a

Please sign in to comment.