diff --git a/src/main/java/org/opensearch/securityanalytics/model/LogType.java b/src/main/java/org/opensearch/securityanalytics/model/LogType.java index 7acc0d1f3..a983b592f 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/LogType.java +++ b/src/main/java/org/opensearch/securityanalytics/model/LogType.java @@ -12,8 +12,6 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; public class LogType implements Writeable { @@ -25,12 +23,16 @@ public class LogType implements Writeable { private static final String RAW_FIELD = "raw_field"; public static final String ECS = "ecs"; public static final String OCSF = "ocsf"; + public static final String IOC_FIELDS = "ioc_fields"; + public static final String IOC = "ioc"; + public static final String FIELDS = "fields"; private String id; private String name; private String description; private Boolean isBuiltIn; private List mappings; + private List iocFieldsList; public LogType(StreamInput sin) throws IOException { this.id = sin.readString(); @@ -38,14 +40,16 @@ public LogType(StreamInput sin) throws IOException { this.name = sin.readString(); this.description = sin.readString(); this.mappings = sin.readList(Mapping::readFrom); + this.iocFieldsList = sin.readList(IocFields::readFrom); } - public LogType(String id, String name, String description, boolean isBuiltIn, List mappings) { + public LogType(String id, String name, String description, boolean isBuiltIn, List mappings, List iocFieldsList) { this.id = id; this.name = name; this.description = description; this.isBuiltIn = isBuiltIn; this.mappings = mappings == null ? List.of() : mappings; + this.iocFieldsList = iocFieldsList == null ? List.of() : iocFieldsList; } public LogType(Map logTypeAsMap) { @@ -62,6 +66,14 @@ public LogType(Map logTypeAsMap) { new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF)) ).collect(Collectors.toList()); } + + List> iocFieldsList = (List>)logTypeAsMap.get(IOC_FIELDS); + if (iocFieldsList.size() > 0) { + this.iocFieldsList = new ArrayList<>(mappings.size()); + this.iocFieldsList = iocFieldsList.stream().map(e -> + new IocFields(e.get(IOC).toString(), (List)e.get(FIELDS)) + ).collect(Collectors.toList()); + } } public String getName() { @@ -74,6 +86,10 @@ public String getDescription() { public boolean getIsBuiltIn() { return isBuiltIn; } + public List getIocFieldsList() { + return iocFieldsList; + } + public List getMappings() { return mappings; } @@ -85,6 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(description); out.writeCollection(mappings); + out.writeCollection(iocFieldsList); } @Override @@ -134,4 +151,43 @@ public static Mapping readFrom(StreamInput sin) throws IOException { } } + /** + * stores information of list of field names that contain information for given IoC (Indicator of Compromise). + */ + public static class IocFields implements Writeable { + private final String ioc; + + private final List fields; + + public IocFields(String ioc, List fields) { + this.ioc = ioc; + this.fields = fields; + } + + public IocFields(StreamInput sin) throws IOException { + this.ioc = sin.readString(); + this.fields = sin.readStringList(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(ioc); + out.writeStringCollection(fields); + } + + public String getIoc() { + return ioc; + } + + public List getFields() { + return fields; + } + + + public static IocFields readFrom(StreamInput sin) throws IOException { + return new IocFields(sin); + } + } + + } \ No newline at end of file diff --git a/src/main/resources/OSMapping/ad_ldap_logtype.json b/src/main/resources/OSMapping/ad_ldap_logtype.json index e3434bca5..be2dd5488 100644 --- a/src/main/resources/OSMapping/ad_ldap_logtype.json +++ b/src/main/resources/OSMapping/ad_ldap_logtype.json @@ -2,7 +2,8 @@ "name": "ad_ldap", "description": "AD/LDAP", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"TargetUserName", "ecs":"azure.signinlogs.properties.user_id" diff --git a/src/main/resources/OSMapping/apache_access_logtype.json b/src/main/resources/OSMapping/apache_access_logtype.json index 7753c8440..714fa2acb 100644 --- a/src/main/resources/OSMapping/apache_access_logtype.json +++ b/src/main/resources/OSMapping/apache_access_logtype.json @@ -2,5 +2,6 @@ "name": "apache_access", "description": "Apache Access Log type", "is_builtin": true, - "mappings": [] + "ioc_fields" : [], + "mappings":[] } diff --git a/src/main/resources/OSMapping/azure_logtype.json b/src/main/resources/OSMapping/azure_logtype.json index ec9ae0502..bb55dbe5f 100644 --- a/src/main/resources/OSMapping/azure_logtype.json +++ b/src/main/resources/OSMapping/azure_logtype.json @@ -2,7 +2,8 @@ "name": "azure", "description": "Azure Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"Resultdescription", "ecs":"azure.signinlogs.result_description" diff --git a/src/main/resources/OSMapping/cloudtrail_logtype.json b/src/main/resources/OSMapping/cloudtrail_logtype.json index 389652373..8c2ea3b3a 100644 --- a/src/main/resources/OSMapping/cloudtrail_logtype.json +++ b/src/main/resources/OSMapping/cloudtrail_logtype.json @@ -2,7 +2,15 @@ "name": "cloudtrail", "description": "Cloudtrail Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "src_endpoint.ip" + ] + } + ], + "mappings":[ { "raw_field":"eventName", "ecs":"aws.cloudtrail.event_name", diff --git a/src/main/resources/OSMapping/dns_logtype.json b/src/main/resources/OSMapping/dns_logtype.json index ca2f5451a..ef012407f 100644 --- a/src/main/resources/OSMapping/dns_logtype.json +++ b/src/main/resources/OSMapping/dns_logtype.json @@ -2,7 +2,15 @@ "name": "dns", "description": "DNS Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "src_endpoint.ip" + ] + } + ], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type", diff --git a/src/main/resources/OSMapping/github_logtype.json b/src/main/resources/OSMapping/github_logtype.json index 6369e2949..31ec6ee59 100644 --- a/src/main/resources/OSMapping/github_logtype.json +++ b/src/main/resources/OSMapping/github_logtype.json @@ -2,7 +2,8 @@ "name": "github", "description": "Github Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"action", "ecs":"github.action" diff --git a/src/main/resources/OSMapping/gworkspace_logtype.json b/src/main/resources/OSMapping/gworkspace_logtype.json index b0006b6a3..7c5766895 100644 --- a/src/main/resources/OSMapping/gworkspace_logtype.json +++ b/src/main/resources/OSMapping/gworkspace_logtype.json @@ -2,7 +2,8 @@ "name": "gworkspace", "description": "GWorkspace Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventSource", "ecs":"google_workspace.admin.service.name" diff --git a/src/main/resources/OSMapping/linux_logtype.json b/src/main/resources/OSMapping/linux_logtype.json index f719913c0..5b77de6b3 100644 --- a/src/main/resources/OSMapping/linux_logtype.json +++ b/src/main/resources/OSMapping/linux_logtype.json @@ -2,7 +2,8 @@ "name": "linux", "description": "Linux Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"name", "ecs":"user.filesystem.name" diff --git a/src/main/resources/OSMapping/m365_logtype.json b/src/main/resources/OSMapping/m365_logtype.json index 6547d3d63..e19c2418e 100644 --- a/src/main/resources/OSMapping/m365_logtype.json +++ b/src/main/resources/OSMapping/m365_logtype.json @@ -2,7 +2,8 @@ "name": "m365", "description": "Microsoft 365 Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventSource", "ecs":"rsa.misc.event_source" diff --git a/src/main/resources/OSMapping/netflow_logtype.json b/src/main/resources/OSMapping/netflow_logtype.json index d8ec32632..9dc015198 100644 --- a/src/main/resources/OSMapping/netflow_logtype.json +++ b/src/main/resources/OSMapping/netflow_logtype.json @@ -2,7 +2,16 @@ "name": "netflow", "description": "Netflow Log Type used only in Integration Tests", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "destination.ip", + "source.ip" + ] + } + ], + "mappings":[ { "raw_field":"netflow.source_ipv4_address", "ecs":"source.ip" diff --git a/src/main/resources/OSMapping/network_logtype.json b/src/main/resources/OSMapping/network_logtype.json index 90f0b2ee6..2ca92a1ad 100644 --- a/src/main/resources/OSMapping/network_logtype.json +++ b/src/main/resources/OSMapping/network_logtype.json @@ -2,7 +2,16 @@ "name": "network", "description": "Network Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "destination.ip", + "source.ip" + ] + } + ], + "mappings":[ { "raw_field":"action", "ecs":"netflow.firewall_event" diff --git a/src/main/resources/OSMapping/okta_logtype.json b/src/main/resources/OSMapping/okta_logtype.json index 8038b7f01..e73a0c273 100644 --- a/src/main/resources/OSMapping/okta_logtype.json +++ b/src/main/resources/OSMapping/okta_logtype.json @@ -2,7 +2,8 @@ "name": "okta", "description": "Okta Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventtype", "ecs":"okta.event_type" diff --git a/src/main/resources/OSMapping/others_application_logtype.json b/src/main/resources/OSMapping/others_application_logtype.json index d7faf8c94..4008602d4 100644 --- a/src/main/resources/OSMapping/others_application_logtype.json +++ b/src/main/resources/OSMapping/others_application_logtype.json @@ -2,7 +2,8 @@ "name": "others_application", "description": "others_application", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_apt_logtype.json b/src/main/resources/OSMapping/others_apt_logtype.json index ace55cbc3..1a4ca711f 100644 --- a/src/main/resources/OSMapping/others_apt_logtype.json +++ b/src/main/resources/OSMapping/others_apt_logtype.json @@ -2,7 +2,8 @@ "name": "others_apt", "description": "others_apt", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_cloud_logtype.json b/src/main/resources/OSMapping/others_cloud_logtype.json index b5da3e005..64cbc7935 100644 --- a/src/main/resources/OSMapping/others_cloud_logtype.json +++ b/src/main/resources/OSMapping/others_cloud_logtype.json @@ -2,7 +2,8 @@ "name": "others_cloud", "description": "others_cloud", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_compliance_logtype.json b/src/main/resources/OSMapping/others_compliance_logtype.json index 6f362d589..6e065795a 100644 --- a/src/main/resources/OSMapping/others_compliance_logtype.json +++ b/src/main/resources/OSMapping/others_compliance_logtype.json @@ -2,7 +2,8 @@ "name": "others_compliance", "description": "others_compliance", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_macos_logtype.json b/src/main/resources/OSMapping/others_macos_logtype.json index 50d1c2160..6b6452100 100644 --- a/src/main/resources/OSMapping/others_macos_logtype.json +++ b/src/main/resources/OSMapping/others_macos_logtype.json @@ -2,7 +2,8 @@ "name": "others_macos", "description": "others_macos", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_proxy_logtype.json b/src/main/resources/OSMapping/others_proxy_logtype.json index aca4529d1..a2b0794a4 100644 --- a/src/main/resources/OSMapping/others_proxy_logtype.json +++ b/src/main/resources/OSMapping/others_proxy_logtype.json @@ -2,7 +2,8 @@ "name": "others_proxy", "description": "others_proxy", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_web_logtype.json b/src/main/resources/OSMapping/others_web_logtype.json index ae8262d52..b46adc6a4 100644 --- a/src/main/resources/OSMapping/others_web_logtype.json +++ b/src/main/resources/OSMapping/others_web_logtype.json @@ -2,7 +2,8 @@ "name": "others_web", "description": "others_web", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/s3_logtype.json b/src/main/resources/OSMapping/s3_logtype.json index 58c546258..20c896df6 100644 --- a/src/main/resources/OSMapping/s3_logtype.json +++ b/src/main/resources/OSMapping/s3_logtype.json @@ -2,7 +2,8 @@ "name": "s3", "description": "S3 Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventName", "ecs":"aws.cloudtrail.event_name" diff --git a/src/main/resources/OSMapping/test_windows_logtype.json b/src/main/resources/OSMapping/test_windows_logtype.json index 7491a954c..59e9cecad 100644 --- a/src/main/resources/OSMapping/test_windows_logtype.json +++ b/src/main/resources/OSMapping/test_windows_logtype.json @@ -2,7 +2,8 @@ "name": "test_windows", "description": "Test Log Type used by tests. It is created as a lightweight log type for integration tests", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"EventID", "ecs":"event_uid" diff --git a/src/main/resources/OSMapping/vpcflow_logtype.json b/src/main/resources/OSMapping/vpcflow_logtype.json index c55305b6d..29d9f38c2 100644 --- a/src/main/resources/OSMapping/vpcflow_logtype.json +++ b/src/main/resources/OSMapping/vpcflow_logtype.json @@ -2,7 +2,16 @@ "name": "vpcflow", "description": "VPC Flow Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "dst_endpoint.ip", + "src_endpoint.ip" + ] + } + ], + "mappings":[ { "raw_field":"version", "ecs":"netflow.version", diff --git a/src/main/resources/OSMapping/waf_logtype.json b/src/main/resources/OSMapping/waf_logtype.json index 5eed2c2fb..3e5b1f4f1 100644 --- a/src/main/resources/OSMapping/waf_logtype.json +++ b/src/main/resources/OSMapping/waf_logtype.json @@ -2,7 +2,8 @@ "name": "waf", "description": "Web Application Firewall Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"cs-method", "ecs":"waf.request.method" diff --git a/src/main/resources/OSMapping/windows_logtype.json b/src/main/resources/OSMapping/windows_logtype.json index a5fef8ea7..ec9b3ed1a 100644 --- a/src/main/resources/OSMapping/windows_logtype.json +++ b/src/main/resources/OSMapping/windows_logtype.json @@ -2,7 +2,13 @@ "name": "windows", "description": "Windows Log Type", "is_builtin": true, - "mappings":[ + "ioc_fields" : [ + { + "ioc": "ip", + "fields": ["destination.ip","source.ip"] + } + ], + "mappings": [ { "raw_field":"AccountName", "ecs":"winlog.computerObject.name" diff --git a/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java index 8eb717e60..64288f669 100644 --- a/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java @@ -50,7 +50,8 @@ protected void beforeTest() throws Exception { new LogType.Mapping("rawFld1", "ecsFld1", "ocsfFld1"), new LogType.Mapping("rawFld2", "ecsFld2", "ocsfFld2"), new LogType.Mapping("rawFld3", "ecsFld3", "ocsfFld3") - ) + ), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ) ); when(builtinLogTypeLoader.getAllLogTypes()).thenReturn(dummyLogTypes); diff --git a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java index e82911c1b..7c16e5f6f 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java @@ -50,7 +50,8 @@ public void testEmptyUserAsStream() throws IOException { public void testLogTypeAsStreamRawFieldOnly() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", null, null)) + List.of(new LogType.Mapping("rawField", null, null)), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -66,7 +67,8 @@ public void testLogTypeAsStreamRawFieldOnly() throws IOException { public void testLogTypeAsStreamFull() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")) + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -80,7 +82,7 @@ public void testLogTypeAsStreamFull() throws IOException { } public void testLogTypeAsStreamNoMappings() throws IOException { - LogType logType = new LogType("1", "my_log_type", "description", false, null); + LogType logType = new LogType("1", "my_log_type", "description", false, null, null); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); diff --git a/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java index 4ede7891b..d9d592641 100644 --- a/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java +++ b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java @@ -21,7 +21,8 @@ public class LogTypeTests { public void testLogTypeAsStreamRawFieldOnly() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", null, null)) + List.of(new LogType.Mapping("rawField", null, null)), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -32,13 +33,16 @@ public void testLogTypeAsStreamRawFieldOnly() throws IOException { assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + assertEquals(logType.getIocFieldsList().get(0).getFields().get(0), newLogType.getIocFieldsList().get(0).getFields().get(0)); + assertEquals(logType.getIocFieldsList().get(0).getIoc(), newLogType.getIocFieldsList().get(0).getIoc()); } @Test public void testLogTypeAsStreamFull() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")) + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -49,11 +53,14 @@ public void testLogTypeAsStreamFull() throws IOException { assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + assertEquals(logType.getIocFieldsList().get(0).getFields().get(0), newLogType.getIocFieldsList().get(0).getFields().get(0)); + assertEquals(logType.getIocFieldsList().get(0).getIoc(), newLogType.getIocFieldsList().get(0).getIoc()); + } @Test public void testLogTypeAsStreamNoMappings() throws IOException { - LogType logType = new LogType("1", "my_log_type", "description", false, null); + LogType logType = new LogType("1", "my_log_type", "description", false, null, null); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes);