adds ioc fields list in log type config files and ioc fields object i…

…n LogType POJO
opensearch-project · Oct 16, 2023 · 9bb5ecc · 9bb5ecc
1 parent d53085b
commit 9bb5ecc
Show file tree

Hide file tree

Showing 28 changed files with 167 additions and 34 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/model/LogType.java b/src/main/java/org/opensearch/securityanalytics/model/LogType.java
@@ -12,8 +12,6 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.core.xcontent.ToXContentObject;
-import org.opensearch.core.xcontent.XContentBuilder;
 
 public class LogType implements Writeable {
 
@@ -25,27 +23,33 @@ public class LogType implements Writeable {
     private static final String RAW_FIELD = "raw_field";
     public static final String ECS = "ecs";
     public static final String OCSF = "ocsf";
+    public static final String IOC_FIELDS = "ioc_fields";
+    public static final String IOC = "ioc";
+    public static final String FIELDS = "fields";
 
     private String id;
     private String name;
     private String description;
     private Boolean isBuiltIn;
     private List<Mapping> mappings;
+    private List<IocFields> iocFieldsList;
 
     public LogType(StreamInput sin) throws IOException {
         this.id = sin.readString();
         this.isBuiltIn = sin.readOptionalBoolean();
         this.name = sin.readString();
         this.description = sin.readString();
         this.mappings = sin.readList(Mapping::readFrom);
+        this.iocFieldsList = sin.readList(IocFields::readFrom);
     }
 
-    public LogType(String id, String name, String description, boolean isBuiltIn, List<Mapping> mappings) {
+    public LogType(String id, String name, String description, boolean isBuiltIn, List<Mapping> mappings, List<IocFields> iocFieldsList) {
         this.id = id;
         this.name = name;
         this.description = description;
         this.isBuiltIn = isBuiltIn;
         this.mappings = mappings == null ? List.of() : mappings;
+        this.iocFieldsList = iocFieldsList == null ? List.of() : iocFieldsList;
     }
 
     public LogType(Map<String, Object> logTypeAsMap) {
@@ -62,6 +66,14 @@ public LogType(Map<String, Object> logTypeAsMap) {
                     new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF))
             ).collect(Collectors.toList());
         }
+
+        List<Map<String, Object>> iocFieldsList = (List<Map<String, Object>>)logTypeAsMap.get(IOC_FIELDS);
+        if (iocFieldsList.size() > 0) {
+            this.iocFieldsList = new ArrayList<>(mappings.size());
+            this.iocFieldsList = iocFieldsList.stream().map(e ->
+                    new IocFields(e.get(IOC).toString(), (List<String>)e.get(FIELDS))
+            ).collect(Collectors.toList());
+        }
     }
 
     public String getName() {
@@ -74,6 +86,10 @@ public String getDescription() {
 
     public boolean getIsBuiltIn() { return isBuiltIn; }
 
+    public List<IocFields> getIocFieldsList() {
+        return iocFieldsList;
+    }
+
     public List<Mapping> getMappings() {
         return mappings;
     }
@@ -85,6 +101,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeString(description);
         out.writeCollection(mappings);
+        out.writeCollection(iocFieldsList);
     }
 
     @Override
@@ -134,4 +151,43 @@ public static Mapping readFrom(StreamInput sin) throws IOException {
         }
     }
 
+    /**
+     * stores information of list of field names that contain information for given IoC (Indicator of Compromise).
+     */
+    public static class IocFields implements Writeable {
+        private final String ioc;
+
+        private final List<String> fields;
+
+        public IocFields(String ioc, List<String> fields) {
+            this.ioc = ioc;
+            this.fields = fields;
+        }
+
+        public IocFields(StreamInput sin) throws IOException {
+            this.ioc = sin.readString();
+            this.fields = sin.readStringList();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(ioc);
+            out.writeStringCollection(fields);
+        }
+
+        public String getIoc() {
+            return ioc;
+        }
+
+        public List<String> getFields() {
+            return fields;
+        }
+
+
+        public static IocFields readFrom(StreamInput sin) throws IOException {
+            return new IocFields(sin);
+        }
+    }
+
+
 }
diff --git a/src/main/resources/OSMapping/ad_ldap_logtype.json b/src/main/resources/OSMapping/ad_ldap_logtype.json
@@ -2,7 +2,8 @@
   "name": "ad_ldap",
   "description": "AD/LDAP",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"TargetUserName",
       "ecs":"azure.signinlogs.properties.user_id"

diff --git a/src/main/resources/OSMapping/apache_access_logtype.json b/src/main/resources/OSMapping/apache_access_logtype.json
@@ -2,5 +2,6 @@
   "name": "apache_access",
   "description": "Apache Access Log type",
   "is_builtin": true,
-  "mappings": []
+  "ioc_fields" : [],
+  "mappings":[]
 }
diff --git a/src/main/resources/OSMapping/azure_logtype.json b/src/main/resources/OSMapping/azure_logtype.json
@@ -2,7 +2,8 @@
   "name": "azure",
   "description": "Azure Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"Resultdescription",
       "ecs":"azure.signinlogs.result_description"

diff --git a/src/main/resources/OSMapping/cloudtrail_logtype.json b/src/main/resources/OSMapping/cloudtrail_logtype.json
@@ -2,7 +2,15 @@
   "name": "cloudtrail",
   "description": "Cloudtrail Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields": [
+    {
+      "ioc": "ip",
+      "fields": [
+        "src_endpoint.ip"
+      ]
+    }
+  ],
+  "mappings":[
     {
       "raw_field":"eventName",
       "ecs":"aws.cloudtrail.event_name",

diff --git a/src/main/resources/OSMapping/dns_logtype.json b/src/main/resources/OSMapping/dns_logtype.json
@@ -2,7 +2,15 @@
   "name": "dns",
   "description": "DNS Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields": [
+    {
+      "ioc": "ip",
+      "fields": [
+        "src_endpoint.ip"
+      ]
+    }
+  ],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type",

diff --git a/src/main/resources/OSMapping/github_logtype.json b/src/main/resources/OSMapping/github_logtype.json
@@ -2,7 +2,8 @@
   "name": "github",
   "description": "Github Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"action",
       "ecs":"github.action"

diff --git a/src/main/resources/OSMapping/gworkspace_logtype.json b/src/main/resources/OSMapping/gworkspace_logtype.json
@@ -2,7 +2,8 @@
   "name": "gworkspace",
   "description": "GWorkspace Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"eventSource",
       "ecs":"google_workspace.admin.service.name"

diff --git a/src/main/resources/OSMapping/linux_logtype.json b/src/main/resources/OSMapping/linux_logtype.json
@@ -2,7 +2,8 @@
   "name": "linux",
   "description": "Linux Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"name",
       "ecs":"user.filesystem.name"

diff --git a/src/main/resources/OSMapping/m365_logtype.json b/src/main/resources/OSMapping/m365_logtype.json
@@ -2,7 +2,8 @@
   "name": "m365",
   "description": "Microsoft 365 Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"eventSource",
       "ecs":"rsa.misc.event_source"

diff --git a/src/main/resources/OSMapping/netflow_logtype.json b/src/main/resources/OSMapping/netflow_logtype.json
@@ -2,7 +2,16 @@
   "name": "netflow",
   "description": "Netflow Log Type used only in Integration Tests",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields": [
+    {
+      "ioc": "ip",
+      "fields": [
+        "destination.ip",
+        "source.ip"
+      ]
+    }
+  ],
+  "mappings":[
     {
       "raw_field":"netflow.source_ipv4_address",
       "ecs":"source.ip"

diff --git a/src/main/resources/OSMapping/network_logtype.json b/src/main/resources/OSMapping/network_logtype.json
@@ -2,7 +2,16 @@
   "name": "network",
   "description": "Network Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields": [
+    {
+      "ioc": "ip",
+      "fields": [
+        "destination.ip",
+        "source.ip"
+      ]
+    }
+  ],
+  "mappings":[
     {
       "raw_field":"action",
       "ecs":"netflow.firewall_event"

diff --git a/src/main/resources/OSMapping/okta_logtype.json b/src/main/resources/OSMapping/okta_logtype.json
@@ -2,7 +2,8 @@
   "name": "okta",
   "description": "Okta Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"eventtype",
       "ecs":"okta.event_type"

diff --git a/src/main/resources/OSMapping/others_application_logtype.json b/src/main/resources/OSMapping/others_application_logtype.json
@@ -2,7 +2,8 @@
   "name": "others_application",
   "description": "others_application",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type"

diff --git a/src/main/resources/OSMapping/others_apt_logtype.json b/src/main/resources/OSMapping/others_apt_logtype.json
@@ -2,7 +2,8 @@
   "name": "others_apt",
   "description": "others_apt",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type"

diff --git a/src/main/resources/OSMapping/others_cloud_logtype.json b/src/main/resources/OSMapping/others_cloud_logtype.json
@@ -2,7 +2,8 @@
   "name": "others_cloud",
   "description": "others_cloud",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type"

diff --git a/src/main/resources/OSMapping/others_compliance_logtype.json b/src/main/resources/OSMapping/others_compliance_logtype.json
@@ -2,7 +2,8 @@
   "name": "others_compliance",
   "description": "others_compliance",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type"

diff --git a/src/main/resources/OSMapping/others_macos_logtype.json b/src/main/resources/OSMapping/others_macos_logtype.json
@@ -2,7 +2,8 @@
   "name": "others_macos",
   "description": "others_macos",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type"

diff --git a/src/main/resources/OSMapping/others_proxy_logtype.json b/src/main/resources/OSMapping/others_proxy_logtype.json
@@ -2,7 +2,8 @@
   "name": "others_proxy",
   "description": "others_proxy",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type"

diff --git a/src/main/resources/OSMapping/others_web_logtype.json b/src/main/resources/OSMapping/others_web_logtype.json
@@ -2,7 +2,8 @@
   "name": "others_web",
   "description": "others_web",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"record_type",
       "ecs":"dns.answers.type"

diff --git a/src/main/resources/OSMapping/s3_logtype.json b/src/main/resources/OSMapping/s3_logtype.json
@@ -2,7 +2,8 @@
   "name": "s3",
   "description": "S3 Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"eventName",
       "ecs":"aws.cloudtrail.event_name"

diff --git a/src/main/resources/OSMapping/test_windows_logtype.json b/src/main/resources/OSMapping/test_windows_logtype.json
@@ -2,7 +2,8 @@
   "name": "test_windows",
   "description": "Test Log Type used by tests. It is created as a lightweight log type for integration tests",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"EventID",
       "ecs":"event_uid"

diff --git a/src/main/resources/OSMapping/vpcflow_logtype.json b/src/main/resources/OSMapping/vpcflow_logtype.json
@@ -2,7 +2,16 @@
   "name": "vpcflow",
   "description": "VPC Flow Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields": [
+    {
+      "ioc": "ip",
+      "fields": [
+        "dst_endpoint.ip",
+        "src_endpoint.ip"
+      ]
+    }
+  ],
+  "mappings":[
     {
       "raw_field":"version",
       "ecs":"netflow.version",

diff --git a/src/main/resources/OSMapping/waf_logtype.json b/src/main/resources/OSMapping/waf_logtype.json
@@ -2,7 +2,8 @@
   "name": "waf",
   "description": "Web Application Firewall Log Type",
   "is_builtin": true,
-  "mappings": [
+  "ioc_fields" : [],
+  "mappings":[
     {
       "raw_field":"cs-method",
       "ecs":"waf.request.method"