[windows] Add system tests for httpjson input (elastic#1044)

* Add system tests for httpjson input * Use new mock server
build-security · Jun 8, 2021 · e4c86d8 · e4c86d8
1 parent b431c43
commit e4c86d8
Show file tree

Hide file tree

Showing 21 changed files with 1,982 additions and 360 deletions.
diff --git a/packages/windows/_dev/build/docs/README.md b/packages/windows/_dev/build/docs/README.md
@@ -46,18 +46,24 @@ channel specific datasets.
 The Windows `powershell` dataset provides events from the Windows
 `Windows PowerShell` event log.
 
+{{event "powershell"}}
+
 {{fields "powershell"}}
 
 ### Powershell/Operational
 
 The Windows `powershell_operational` dataset provides events from the Windows
 `Microsoft-Windows-PowerShell/Operational` event log.
 
+{{event "powershell_operational"}}
+
 {{fields "powershell_operational"}}
 
 ### Sysmon/Operational
 
 The Windows `sysmon_operational` dataset provides events from the Windows
 `Microsoft-Windows-Sysmon/Operational` event log.
 
+{{event "sysmon_operational"}}
+
 {{fields "sysmon_operational"}}
diff --git a/packages/windows/_dev/deploy/docker/docker-compose.yml b/packages/windows/_dev/deploy/docker/docker-compose.yml
@@ -0,0 +1,12 @@
+version: '2.3'
+services:
+  splunk-mock:
+    image: docker.elastic.co/observability/stream:v0.5.0
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml
diff --git a/packages/windows/_dev/deploy/docker/files/config.yml b/packages/windows/_dev/deploy/docker/files/config.yml
@@ -0,0 +1,177 @@
+rules:
+  - path: /services/search/jobs/export
+    user: test
+    password: test
+    methods:
+      - POST
+    query_params:
+      index_earliest: "{index_earliest:[0-9]+}"
+      index_latest: "{index_latest:[0-9]+}"
+      output_mode: json
+      search: 'search sourcetype="XmlWinEventLog:ForwardedEvents" | streamstats max(_indextime) AS max_indextime'
+    request_headers:
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {
+              "preview": false,
+              "offset": 194,
+              "lastrow": true,
+              "result": {
+                  "_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
+                  "_cd": "0:315",
+                  "_indextime": "1622471463",
+                  "_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/><EventID>4105</EventID><Version>1</Version><Level>5</Level><Task>102</Task><Opcode>15</Opcode><Keywords>0x0</Keywords><TimeCreated SystemTime='2020-05-13T09:04:04.755232500Z'/><EventRecordID>790</EventRecordID><Correlation ActivityID='{dd68516a-2930-0000-5962-68dd3029d601}'/><Execution ProcessID='4204' ThreadID='1476'/><Channel>Microsoft-Windows-PowerShell/Operational</Channel><Computer>vagrant</Computer><Security UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/></System><EventData><Data Name='ScriptBlockId'>f4a378ab-b74f-41a7-a5ef-6dd55562fdb9</Data><Data Name='RunspaceId'>9c031e5c-8d5a-4b91-a12e-b3624970b623</Data></EventData></Event>",
+                  "_serial": "194",
+                  "_si": [
+                      "69819b6ce1bd",
+                      "main"
+                  ],
+                  "_sourcetype": "XmlWinEventLog:Security",
+                  "_time": "2021-05-25 13:11:45.000 UTC",
+                  "host": "VAGRANT",
+                  "index": "main",
+                  "linecount": "1",
+                  "max_indextime": "1622471606",
+                  "source": "WinEventLog:Security",
+                  "sourcetype": "XmlWinEventLog:Security",
+                  "splunk_server": "69819b6ce1bd"
+              }
+          }
+  - path: /services/search/jobs/export
+    user: test
+    password: test
+    methods:
+      - post
+    query_params:
+      index_earliest: "{index_earliest:[0-9]+}"
+      index_latest: "{index_latest:[0-9]+}"
+      output_mode: json
+      search: 'search sourcetype="XmlWinEventLog:Windows PowerShell" | streamstats max(_indextime) AS max_indextime'
+    request_headers:
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {
+              "preview": false,
+              "offset": 194,
+              "lastrow": true,
+              "result": {
+                  "_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
+                  "_cd": "0:315",
+                  "_indextime": "1622471463",
+                  "_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='PowerShell'/><EventID Qualifiers='0'>600</EventID><Level>4</Level><Task>6</Task><Keywords>0x80000000000000</Keywords><TimeCreated SystemTime='2020-05-13T13:21:43.183180900Z'/><EventRecordID>1089</EventRecordID><Channel>Windows PowerShell</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data>Certificate</Data><Data>Started</Data><Data>\tProviderName=Certificate\n\tNewProviderState=Started\n\n\tSequenceNumber=35\n\n\tHostName=Windows PowerShell ISE Host\n\tHostVersion=5.1.17763.1007\n\tHostId=86edc16f-6943-469e-8bd8-ef1857080206\n\tHostApplication=C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell_ise.exe C:\\Users\\vagrant\\Desktop\\lateral.ps1\n\tEngineVersion=5.1.17763.1007\n\tRunspaceId=9d21da0b-e402-40e1-92ff-98c5ab1137a9\n\tPipelineId=15\n\tCommandName=\n\tCommandType=\n\tScriptName=\n\tCommandPath=\n\tCommandLine=</Data></EventData></Event>\n<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='PowerShell'/><EventID Qualifiers='0'>600</EventID><Level>4</Level><Task>6</Task><Keywords>0x80000000000000</Keywords><TimeCreated SystemTime='2020-05-13T13:25:04.656426900Z'/><EventRecordID>1266</EventRecordID><Channel>Windows PowerShell</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data>Registry</Data><Data>Started</Data><Data>\tProviderName=Registry\n\tNewProviderState=Started\n\n\tSequenceNumber=1\n\n\tHostName=ConsoleHost\n\tHostVersion=5.1.17763.1007\n\tHostId=44b8d66c-f5a2-4abb-ac7d-6db73990a6d3\n\tHostApplication=C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe -noexit -command 'C:\\Gopath\\src\\github.com\\elastic\\beats'\n\tEngineVersion=\n\tRunspaceId=\n\tPipelineId=\n\tCommandName=\n\tCommandType=\n\tScriptName=\n\tCommandPath=\n\tCommandLine=</Data></EventData></Event>\n<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='PowerShell'/><EventID Qualifiers='0'>600</EventID><Level>4</Level><Task>6</Task><Keywords>0x80000000000000</Keywords><TimeCreated SystemTime='2020-06-04T07:25:04.857430200Z'/><EventRecordID>18640</EventRecordID><Channel>Windows PowerShell</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data>Certificate</Data><Data>Started</Data><Data>\tProviderName=Certificate\n\tNewProviderState=Started\n\n\tSequenceNumber=8\n\n\tHostName=ConsoleHost\n\tHostVersion=2.0\n\tHostId=99a16837-7392-463d-afe5-5f3ed24bd358\n\tEngineVersion=\n\tRunspaceId=\n\tPipelineId=\n\tCommandName=\n\tCommandType=\n\tScriptName=\n\tCommandPath=\n\tCommandLine=</Data></EventData></Event>",
+                  "_serial": "194",
+                  "_si": [
+                      "69819b6ce1bd",
+                      "main"
+                  ],
+                  "_sourcetype": "XmlWinEventLog:Security",
+                  "_time": "2021-05-25 13:11:45.000 UTC",
+                  "host": "VAGRANT",
+                  "index": "main",
+                  "linecount": "1",
+                  "max_indextime": "1622471606",
+                  "source": "WinEventLog:Security",
+                  "sourcetype": "XmlWinEventLog:Security",
+                  "splunk_server": "69819b6ce1bd"
+              }
+          }
+  - path: /services/search/jobs/export
+    user: test
+    password: test
+    methods:
+      - post
+    query_params:
+      index_earliest: "{index_earliest:[0-9]+}"
+      index_latest: "{index_latest:[0-9]+}"
+      output_mode: json
+      search: 'search sourcetype="XmlWinEventLog:Microsoft-Windows-Powershell/Operational" | streamstats max(_indextime) AS max_indextime'
+    request_headers:
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {
+              "preview": false,
+              "offset": 194,
+              "lastrow": true,
+              "result": {
+                  "_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
+                  "_cd": "0:315",
+                  "_indextime": "1622471463",
+                  "_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/><EventID>4105</EventID><Version>1</Version><Level>5</Level><Task>102</Task><Opcode>15</Opcode><Keywords>0x0</Keywords><TimeCreated SystemTime='2020-05-13T09:04:04.755232500Z'/><EventRecordID>790</EventRecordID><Correlation ActivityID='{dd68516a-2930-0000-5962-68dd3029d601}'/><Execution ProcessID='4204' ThreadID='1476'/><Channel>Microsoft-Windows-PowerShell/Operational</Channel><Computer>vagrant</Computer><Security UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/></System><EventData><Data Name='ScriptBlockId'>f4a378ab-b74f-41a7-a5ef-6dd55562fdb9</Data><Data Name='RunspaceId'>9c031e5c-8d5a-4b91-a12e-b3624970b623</Data></EventData></Event>",
+                  "_serial": "194",
+                  "_si": [
+                      "69819b6ce1bd",
+                      "main"
+                  ],
+                  "_sourcetype": "XmlWinEventLog:Security",
+                  "_time": "2021-05-25 13:11:45.000 UTC",
+                  "host": "VAGRANT",
+                  "index": "main",
+                  "linecount": "1",
+                  "max_indextime": "1622471606",
+                  "source": "WinEventLog:Security",
+                  "sourcetype": "XmlWinEventLog:Security",
+                  "splunk_server": "69819b6ce1bd"
+              }
+          }
+  - path: /services/search/jobs/export
+    user: test
+    password: test
+    methods:
+      - post
+    query_params:
+      index_earliest: "{index_earliest:[0-9]+}"
+      index_latest: "{index_latest:[0-9]+}"
+      output_mode: json
+      search: 'search sourcetype="XmlWinEventLog:Microsoft-Windows-Sysmon/Operational" | streamstats max(_indextime) AS max_indextime'
+    request_headers:
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {
+              "preview": false,
+              "offset": 194,
+              "lastrow": true,
+              "result": {
+                  "_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
+                  "_cd": "0:315",
+                  "_indextime": "1622471463",
+                  "_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Sysmon' Guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}'/><EventID>22</EventID><Version>5</Version><Level>4</Level><Task>22</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime='2019-07-18T03:34:02.025237700Z'/><EventRecordID>67</EventRecordID><Correlation/><Execution ProcessID='2828' ThreadID='1684'/><Channel>Microsoft-Windows-Sysmon/Operational</Channel><Computer>vagrant-2016</Computer><Security UserID='S-1-5-18'/></System><EventData><Data Name='RuleName'></Data><Data Name='UtcTime'>2019-07-18 03:34:01.261</Data><Data Name='ProcessGuid'>{fa4a0de6-e8a9-5d2f-0000-001053699900}</Data><Data Name='ProcessId'>2736</Data><Data Name='QueryName'>www.msn.com</Data><Data Name='QueryStatus'>0</Data><Data Name='QueryResults'>type:  5 www-msn-com.a-0003.a-msedge.net;type:  5 a-0003.a-msedge.net;::ffff:204.79.197.203;</Data><Data Name='Image'>C:\\Program Files (x86)\\Internet Explorer\\iexplore.exe</Data></EventData></Event>",
+                  "_serial": "194",
+                  "_si": [
+                      "69819b6ce1bd",
+                      "main"
+                  ],
+                  "_sourcetype": "XmlWinEventLog:Security",
+                  "_time": "2021-05-25 13:11:45.000 UTC",
+                  "host": "VAGRANT",
+                  "index": "main",
+                  "linecount": "1",
+                  "max_indextime": "1622471606",
+                  "source": "WinEventLog:Security",
+                  "sourcetype": "XmlWinEventLog:Security",
+                  "splunk_server": "69819b6ce1bd"
+              }
+          }
diff --git a/packages/windows/changelog.yml b/packages/windows/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "0.8.2"
+  changes:
+    - description: Add system tests for Splunk http inputs and improve README.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1044
+    - description: Fix sysmon pipeline when processing `dns.resolved_ip`.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1044
 - version: "0.8.1"
   changes:
     - description: Fix security pipeline to support string event.code.

diff --git a/packages/windows/data_stream/forwarded/_dev/test/system/test-default-config.yml b/packages/windows/data_stream/forwarded/_dev/test/system/test-default-config.yml
@@ -0,0 +1,9 @@
+input: httpjson
+service: splunk-mock
+vars:
+  url: http://{{Hostname}}:{{Port}}
+  username: test
+  password: test
+data_stream:
+  vars:
+    preserve_original_event: true
diff --git a/packages/windows/data_stream/forwarded/fields/beats.yml b/packages/windows/data_stream/forwarded/fields/beats.yml
@@ -0,0 +1,3 @@
+- name: input.type
+  type: keyword
+  description: Type of Filebeat input.
diff --git a/packages/windows/data_stream/forwarded/sample_event.json b/packages/windows/data_stream/forwarded/sample_event.json
@@ -0,0 +1,75 @@
+{
+    "@timestamp": "2020-05-13T09:04:04.755Z",
+    "agent": {
+        "ephemeral_id": "6d6bb3f5-f905-4ee5-8bee-c719616f8b6b",
+        "hostname": "docker-fleet-agent",
+        "id": "6fe85b08-7c10-4f55-ba4e-eeb75fdd6fdf",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "7.13.0"
+    },
+    "data_stream": {
+        "dataset": "windows.forwarded",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "1.9.0"
+    },
+    "elastic_agent": {
+        "id": "6bdfe1ae-64c3-4177-8b0c-2380a6e01322",
+        "snapshot": true,
+        "version": "7.13.0"
+    },
+    "event": {
+        "category": "process",
+        "code": "4105",
+        "created": "2021-06-01T10:22:56.365Z",
+        "dataset": "windows.forwarded",
+        "ingested": "2021-06-01T10:22:57.387144900Z",
+        "kind": "event",
+        "original": "\u003cEvent xmlns='http://schemas.microsoft.com/win/2004/08/events/event'\u003e\u003cSystem\u003e\u003cProvider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/\u003e\u003cEventID\u003e4105\u003c/EventID\u003e\u003cVersion\u003e1\u003c/Version\u003e\u003cLevel\u003e5\u003c/Level\u003e\u003cTask\u003e102\u003c/Task\u003e\u003cOpcode\u003e15\u003c/Opcode\u003e\u003cKeywords\u003e0x0\u003c/Keywords\u003e\u003cTimeCreated SystemTime='2020-05-13T09:04:04.755232500Z'/\u003e\u003cEventRecordID\u003e790\u003c/EventRecordID\u003e\u003cCorrelation ActivityID='{dd68516a-2930-0000-5962-68dd3029d601}'/\u003e\u003cExecution ProcessID='4204' ThreadID='1476'/\u003e\u003cChannel\u003eMicrosoft-Windows-PowerShell/Operational\u003c/Channel\u003e\u003cComputer\u003evagrant\u003c/Computer\u003e\u003cSecurity UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name='ScriptBlockId'\u003ef4a378ab-b74f-41a7-a5ef-6dd55562fdb9\u003c/Data\u003e\u003cData Name='RunspaceId'\u003e9c031e5c-8d5a-4b91-a12e-b3624970b623\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
+        "provider": "Microsoft-Windows-PowerShell",
+        "type": "start"
+    },
+    "host": {
+        "name": "vagrant"
+    },
+    "input": {
+        "type": "httpjson"
+    },
+    "log": {
+        "level": "verbose"
+    },
+    "powershell": {
+        "file": {
+            "script_block_id": "f4a378ab-b74f-41a7-a5ef-6dd55562fdb9"
+        },
+        "runspace_id": "9c031e5c-8d5a-4b91-a12e-b3624970b623"
+    },
+    "tags": [
+        "forwarded"
+    ],
+    "user": {
+        "id": "S-1-5-21-1350058589-2282154016-2764056528-1000"
+    },
+    "winlog": {
+        "activity_id": "{dd68516a-2930-0000-5962-68dd3029d601}",
+        "channel": "Microsoft-Windows-PowerShell/Operational",
+        "computer_name": "vagrant",
+        "event_id": "4105",
+        "process": {
+            "pid": 4204,
+            "thread": {
+                "id": 1476
+            }
+        },
+        "provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
+        "provider_name": "Microsoft-Windows-PowerShell",
+        "record_id": "790",
+        "user": {
+            "identifier": "S-1-5-21-1350058589-2282154016-2764056528-1000"
+        },
+        "version": 1
+    }
+}
diff --git a/packages/windows/data_stream/powershell/_dev/test/system/test-default-config.yml b/packages/windows/data_stream/powershell/_dev/test/system/test-default-config.yml
@@ -0,0 +1,9 @@
+input: httpjson
+service: splunk-mock
+vars:
+  url: http://{{Hostname}}:{{Port}}
+  username: test
+  password: test
+data_stream:
+  vars:
+    preserve_original_event: true
diff --git a/packages/windows/data_stream/powershell/fields/beats.yml b/packages/windows/data_stream/powershell/fields/beats.yml
@@ -0,0 +1,3 @@
+- name: input.type
+  type: keyword
+  description: Type of Filebeat input.