for idaholab#465, work in progress handling uploading evtx files

dashboards/templates/composable/component/evtx.json

@@ -43,7 +43,6 @@
                 "EventData.ChannelName": { "type": "keyword" },
                 "EventData.ClientIP": { "type": "keyword" },
                 "EventData.ClientMode": { "type": "integer" },
-                "EventData.CommandLine": { "type": "keyword" },
                 "EventData.Company": { "type": "keyword" },
                 "EventData.ComplexData": { "type": "keyword" },
                 "EventData.ComplexData_attributes.Name": { "type": "keyword" },
@@ -55,7 +54,6 @@
                 "EventData.ConnType": { "type": "keyword" },
                 "EventData.Consumer": { "type": "keyword" },
                 "EventData.CreationUtcTime": { "type": "date" },
-                "EventData.CurrentDirectory": { "type": "keyword" },
                 "EventData.CustomLevel": { "type": "keyword" },
                 "EventData.Data": { "type": "keyword" },
                 "EventData.Description": { "type": "keyword" },
@@ -113,7 +111,6 @@
                 "EventData.GUID": { "type": "keyword" },
                 "EventData.HandleId": { "type": "keyword" },
                 "EventData.Hash": { "type": "keyword" },
-                "EventData.Hashes": { "type": "keyword" },
                 "EventData.HomeDirectory": { "type": "keyword" },
                 "EventData.HomePath": { "type": "keyword" },
                 "EventData.hr": { "type": "long" },
@@ -358,6 +355,7 @@
                 "System.Execution_attributes.KernelTime": { "type": "integer" },
                 "System.Execution_attributes.ProcessID": { "type": "keyword" },
                 "System.Execution_attributes.ProcessorID": { "type": "integer" },
+                "System.Execution_attributes.ThreadID": { "type": "keyword" },
                 "System.Execution_attributes.UserTime": { "type": "integer" },
                 "System.Provider_attributes.EventSourceName": { "type": "keyword" },
                 "System.TimeCreated_attributes.SystemTime": { "type": "date" },

dashboards/templates/composable/component/miscbeat.json

@@ -269,17 +269,14 @@
                 "EventType" : { "type": "keyword" },
                 "Keywords" : { "type": "keyword" },
                 "Level" : { "type": "integer" },
-                "Message" : { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } },
                 "Opcode" : { "type": "integer" },
-                "ProcessID" : { "type": "integer" },
                 "ProviderGuid" : { "type": "keyword" },
                 "ProviderName" : { "type": "keyword" },
                 "Qualifiers" : { "type": "integer" },
                 "RelatedActivityID" : { "type": "keyword" },
                 "Sid" : { "type": "keyword" },
                 "StringInserts" : { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } },
                 "Task" : { "type": "integer" },
-                "ThreadID" : { "type": "integer" },
                 "TimeGenerated" : { "type": "date" },
                 "TimeWritten" : { "type": "date" },
                 "UserID" : { "type": "keyword" },

dashboards/templates/malcolm_beats_template.json

@@ -13,6 +13,7 @@
     "ecs_http",
     "ecs_log",
     "ecs_network",
+    "ecs_os",
     "ecs_process",
     "ecs_related",
     "ecs_server",

docs/third-party-logs.md

@@ -21,6 +21,7 @@ The types of third-party logs and metrics discussed in this document are *not* t
     - [Convenience Script for Linux/macOS](#FluentBitBash)
     - [Convenience Script for Windows](#FluentBitPowerShell)
 * [Beats](#Beats)
+* [Uploading Third-Party Logs](#ThirdPartyUpload)
 * [Data Format and Visualization](#Data)
 * [Document Indices](#Indices)
 
@@ -308,6 +309,12 @@ The important bits to note in this example are the settings under [`output.logst
 
 Most Beats forwarders can use [processors](https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html) to filter, transform, and enhance data prior to sending it to Malcolm. Consult each forwarder's [documentation](https://www.elastic.co/beats/) to learn more about what processors are available and how to configure them. Use the [Console output](https://www.elastic.co/guide/en/beats/filebeat/current/console-output.html) for debugging and experimenting with how Beats forwarders format the logs they generate.
 
+## <a name="ThirdPartyUpload"></a>Uploading Third-Party Logs
+
+### Microsoft Windows Event Logs
+
+Microsoft Windows [event log files](https://learn.microsoft.com/en-us/windows/win32/eventlog/event-log-file-format) (with a `.evtx` file extension) can also be [uploaded](upload.md#Upload) via the artifact upload interface, either singly or in archive files (`application/gzip`, `application/x-gzip`, `application/x-7z-compressed`, `application/x-bzip2`, `application/x-cpio`, `application/x-lzip`, `application/x-lzma`, `application/x-rar-compressed`, `application/x-tar`, `application/x-xz`, or `application/zip`). These files are processed using [evtx](https://github.com/omerbenamram/evtx) and indexed as similarly as possible to the way forwarded Windows event logs are indexed.
+
 ## <a name="Data"></a>Data Format and Visualization
 
 Because Malcolm could receive logs or metrics from virtually any provider, Malcolm most likely does not have prebuilt dashboards and visualizations for third-party logs. Luckily, [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/index/) provides visualization tools that can be used with whatever data is stored in Malcolm's OpenSearch document store. Here are some resources covering OpenSearch Dashboards and building custom visualizations:

logstash/pipelines/beats/11_beats_logs.conf

@@ -946,8 +946,6 @@ filter {
              rename => { "[evtx][Event][System][EventID]" => "[event][id]" }
              rename => { "[evtx][Event][System][EventID_attributes][Qualifiers]" => "[miscbeat][winlog][Qualifiers]" }
              rename => { "[evtx][Event][System][EventRecordID]" => "[miscbeat][winlog][EventRecordID]" }
-             rename => { "[evtx][Event][System][Execution_attributes][ProcessId]" => "[miscbeat][winlog][ProcessID]" }
-             rename => { "[evtx][Event][System][Execution_attributes][ThreadID]" => "[miscbeat][winlog][ThreadID]" }
              rename => { "[evtx][Event][System][Keywords]" => "[miscbeat][winlog][Keywords]" }
              rename => { "[evtx][Event][System][Level]" => "[miscbeat][winlog][Level]" }
              rename => { "[evtx][Event][System][Opcode]" => "[miscbeat][winlog][Opcode]" }
@@ -960,24 +958,39 @@ filter {
              rename => { "[evtx][Event][EventData][CurrentDirectory]" => "[process][working_directory]" }
     }
 
-    # there is some inconsistency across windows event log providers...
-    if ([evtx][Event][EventData][ProcessId]) {
-      mutate { id => "mutate_rename_evtx_eventdata_processid_1"
-               rename => { "[evtx][Event][EventData][ProcessId]" => "[process][pid]" } }
-    } else if ([evtx][Event][EventData][ProcessID]) {
-      mutate { id => "mutate_rename_evtx_eventdata_processid_2"
-               rename => { "[evtx][Event][EventData][ProcessID]" => "[process][pid]" } }
-    } else if ([evtx][Event][EventData][processId]) {
-      mutate { id => "mutate_rename_evtx_eventdata_processid_3"
-               rename => { "[evtx][Event][EventData][processId]" => "[process][pid]" } }
-    }
-    if ([process][pid] =~ /^0x/) {
-      ruby {
-          id => "ruby_evtx_process_pid_from_hex"
-          code => "
-            event.set('[process][pid]', event.get('[process][pid]').to_s.hex)
-          "
-      }
+    # there is some inconsistency across windows event log providers about how to name things...
+    ruby {
+        id => "ruby_miscbeat_evtx_process_id"
+        code => "
+          pids = Array.new
+          tids = Array.new
+          ['[evtx][Event][EventData][CallerProcessId]',
+           '[evtx][Event][EventData][NewProcessId]',
+           '[evtx][Event][EventData][ParentProcessId]',
+           '[evtx][Event][EventData][processId]',
+           '[evtx][Event][EventData][ProcessId]',
+           '[evtx][Event][EventData][ProcessID]',
+           '[evtx][Event][EventData][SourceProcessId]',
+           '[evtx][Event][EventData][TargetProcessId]',
+           '[evtx][Event][System][Execution_attributes]',
+           '[evtx][Event][System][Execution_attributes][ProcessID]',
+           '[evtx][Event][UserData][CompatibilityFixEvent][ProcessId]'].each {|fname|
+            if (pidstr = event.get(fname).to_s) then
+              pidint = pidstr.start_with?('0x') ? pidstr.hex : pidstr.to_i
+              pids.push(pidint) if pidint > 0
+            end
+          }
+          ['[evtx][Event][EventData][NewThreadId]',
+           '[evtx][Event][EventData][SourceThreadId]',
+           '[evtx][Event][System][Execution_attributes][ThreadID]'].each {|fname|
+           if (tidstr = event.get(fname).to_s) then
+             tidint = tidstr.start_with?('0x') ? tidstr.hex : tidstr.to_i
+             tids.push(tidint) if tidint > 0
+           end
+          }
+          event.set('[process][pid]', pids.uniq) unless (pids.length == 0)
+          event.set('[process][thread][id]', tids.uniq) unless (tids.length == 0)
+        "
     }
 
     # map error description/code to event.result
@@ -1231,6 +1244,8 @@ filter {
         mutate { id => "mutate_merge_evtx_related_hash_imphash"
                  merge => { "[related][hash]" => "[pe][imphash]" } }
       }
+      mutate { id => "mutate_miscbeat_remove_eventdata_hashes"
+               remove_field => [ "[evtx][Event][EventData][Hashes]" ] }
     }
 
     # ECS dll