Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[windows] Add system tests for httpjson input #1044

Merged
merged 2 commits into from
Jun 8, 2021
Merged

[windows] Add system tests for httpjson input #1044

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
65ceace
Add system tests for httpjson input
marc-gr Jun 1, 2021
176458d
Use new mock server
marc-gr Jun 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions packages/windows/_dev/build/docs/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,24 @@ channel specific datasets.
The Windows `powershell` dataset provides events from the Windows
`Windows PowerShell` event log.

{{event "powershell"}}

{{fields "powershell"}}

### Powershell/Operational

The Windows `powershell_operational` dataset provides events from the Windows
`Microsoft-Windows-PowerShell/Operational` event log.

{{event "powershell_operational"}}

{{fields "powershell_operational"}}

### Sysmon/Operational

The Windows `sysmon_operational` dataset provides events from the Windows
`Microsoft-Windows-Sysmon/Operational` event log.

{{event "sysmon_operational"}}

{{fields "sysmon_operational"}}
12 changes: 12 additions & 0 deletions packages/windows/_dev/deploy/docker/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
version: '2.3'
services:
splunk-mock:
image: docker.elastic.co/observability/stream:v0.5.0
ports:
- 8080
volumes:
- ./files:/files:ro
command:
- http-server
- --addr=:8080
- --config=/files/config.yml
177 changes: 177 additions & 0 deletions packages/windows/_dev/deploy/docker/files/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
rules:
- path: /services/search/jobs/export
user: test
password: test
methods:
- POST
query_params:
index_earliest: "{index_earliest:[0-9]+}"
index_latest: "{index_latest:[0-9]+}"
output_mode: json
search: 'search sourcetype="XmlWinEventLog:ForwardedEvents" | streamstats max(_indextime) AS max_indextime'
request_headers:
Content-Type:
- "application/x-www-form-urlencoded"
responses:
- status_code: 200
headers:
Content-Type:
- "application/json"
body: |-
{
"preview": false,
"offset": 194,
"lastrow": true,
"result": {
"_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
"_cd": "0:315",
"_indextime": "1622471463",
"_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/><EventID>4105</EventID><Version>1</Version><Level>5</Level><Task>102</Task><Opcode>15</Opcode><Keywords>0x0</Keywords><TimeCreated SystemTime='2020-05-13T09:04:04.755232500Z'/><EventRecordID>790</EventRecordID><Correlation ActivityID='{dd68516a-2930-0000-5962-68dd3029d601}'/><Execution ProcessID='4204' ThreadID='1476'/><Channel>Microsoft-Windows-PowerShell/Operational</Channel><Computer>vagrant</Computer><Security UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/></System><EventData><Data Name='ScriptBlockId'>f4a378ab-b74f-41a7-a5ef-6dd55562fdb9</Data><Data Name='RunspaceId'>9c031e5c-8d5a-4b91-a12e-b3624970b623</Data></EventData></Event>",
"_serial": "194",
"_si": [
"69819b6ce1bd",
"main"
],
"_sourcetype": "XmlWinEventLog:Security",
"_time": "2021-05-25 13:11:45.000 UTC",
"host": "VAGRANT",
"index": "main",
"linecount": "1",
"max_indextime": "1622471606",
"source": "WinEventLog:Security",
"sourcetype": "XmlWinEventLog:Security",
"splunk_server": "69819b6ce1bd"
}
}
- path: /services/search/jobs/export
user: test
password: test
methods:
- post
query_params:
index_earliest: "{index_earliest:[0-9]+}"
index_latest: "{index_latest:[0-9]+}"
output_mode: json
search: 'search sourcetype="XmlWinEventLog:Windows PowerShell" | streamstats max(_indextime) AS max_indextime'
request_headers:
Content-Type:
- "application/x-www-form-urlencoded"
responses:
- status_code: 200
headers:
Content-Type:
- "application/json"
body: |-
{
"preview": false,
"offset": 194,
"lastrow": true,
"result": {
"_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
"_cd": "0:315",
"_indextime": "1622471463",
"_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='PowerShell'/><EventID Qualifiers='0'>600</EventID><Level>4</Level><Task>6</Task><Keywords>0x80000000000000</Keywords><TimeCreated SystemTime='2020-05-13T13:21:43.183180900Z'/><EventRecordID>1089</EventRecordID><Channel>Windows PowerShell</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data>Certificate</Data><Data>Started</Data><Data>\tProviderName=Certificate\n\tNewProviderState=Started\n\n\tSequenceNumber=35\n\n\tHostName=Windows PowerShell ISE Host\n\tHostVersion=5.1.17763.1007\n\tHostId=86edc16f-6943-469e-8bd8-ef1857080206\n\tHostApplication=C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell_ise.exe C:\\Users\\vagrant\\Desktop\\lateral.ps1\n\tEngineVersion=5.1.17763.1007\n\tRunspaceId=9d21da0b-e402-40e1-92ff-98c5ab1137a9\n\tPipelineId=15\n\tCommandName=\n\tCommandType=\n\tScriptName=\n\tCommandPath=\n\tCommandLine=</Data></EventData></Event>\n<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='PowerShell'/><EventID Qualifiers='0'>600</EventID><Level>4</Level><Task>6</Task><Keywords>0x80000000000000</Keywords><TimeCreated SystemTime='2020-05-13T13:25:04.656426900Z'/><EventRecordID>1266</EventRecordID><Channel>Windows PowerShell</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data>Registry</Data><Data>Started</Data><Data>\tProviderName=Registry\n\tNewProviderState=Started\n\n\tSequenceNumber=1\n\n\tHostName=ConsoleHost\n\tHostVersion=5.1.17763.1007\n\tHostId=44b8d66c-f5a2-4abb-ac7d-6db73990a6d3\n\tHostApplication=C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe -noexit -command 'C:\\Gopath\\src\\github.com\\elastic\\beats'\n\tEngineVersion=\n\tRunspaceId=\n\tPipelineId=\n\tCommandName=\n\tCommandType=\n\tScriptName=\n\tCommandPath=\n\tCommandLine=</Data></EventData></Event>\n<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='PowerShell'/><EventID Qualifiers='0'>600</EventID><Level>4</Level><Task>6</Task><Keywords>0x80000000000000</Keywords><TimeCreated SystemTime='2020-06-04T07:25:04.857430200Z'/><EventRecordID>18640</EventRecordID><Channel>Windows PowerShell</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data>Certificate</Data><Data>Started</Data><Data>\tProviderName=Certificate\n\tNewProviderState=Started\n\n\tSequenceNumber=8\n\n\tHostName=ConsoleHost\n\tHostVersion=2.0\n\tHostId=99a16837-7392-463d-afe5-5f3ed24bd358\n\tEngineVersion=\n\tRunspaceId=\n\tPipelineId=\n\tCommandName=\n\tCommandType=\n\tScriptName=\n\tCommandPath=\n\tCommandLine=</Data></EventData></Event>",
"_serial": "194",
"_si": [
"69819b6ce1bd",
"main"
],
"_sourcetype": "XmlWinEventLog:Security",
"_time": "2021-05-25 13:11:45.000 UTC",
"host": "VAGRANT",
"index": "main",
"linecount": "1",
"max_indextime": "1622471606",
"source": "WinEventLog:Security",
"sourcetype": "XmlWinEventLog:Security",
"splunk_server": "69819b6ce1bd"
}
}
- path: /services/search/jobs/export
user: test
password: test
methods:
- post
query_params:
index_earliest: "{index_earliest:[0-9]+}"
index_latest: "{index_latest:[0-9]+}"
output_mode: json
search: 'search sourcetype="XmlWinEventLog:Microsoft-Windows-Powershell/Operational" | streamstats max(_indextime) AS max_indextime'
request_headers:
Content-Type:
- "application/x-www-form-urlencoded"
responses:
- status_code: 200
headers:
Content-Type:
- "application/json"
body: |-
{
"preview": false,
"offset": 194,
"lastrow": true,
"result": {
"_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
"_cd": "0:315",
"_indextime": "1622471463",
"_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/><EventID>4105</EventID><Version>1</Version><Level>5</Level><Task>102</Task><Opcode>15</Opcode><Keywords>0x0</Keywords><TimeCreated SystemTime='2020-05-13T09:04:04.755232500Z'/><EventRecordID>790</EventRecordID><Correlation ActivityID='{dd68516a-2930-0000-5962-68dd3029d601}'/><Execution ProcessID='4204' ThreadID='1476'/><Channel>Microsoft-Windows-PowerShell/Operational</Channel><Computer>vagrant</Computer><Security UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/></System><EventData><Data Name='ScriptBlockId'>f4a378ab-b74f-41a7-a5ef-6dd55562fdb9</Data><Data Name='RunspaceId'>9c031e5c-8d5a-4b91-a12e-b3624970b623</Data></EventData></Event>",
"_serial": "194",
"_si": [
"69819b6ce1bd",
"main"
],
"_sourcetype": "XmlWinEventLog:Security",
"_time": "2021-05-25 13:11:45.000 UTC",
"host": "VAGRANT",
"index": "main",
"linecount": "1",
"max_indextime": "1622471606",
"source": "WinEventLog:Security",
"sourcetype": "XmlWinEventLog:Security",
"splunk_server": "69819b6ce1bd"
}
}
- path: /services/search/jobs/export
user: test
password: test
methods:
- post
query_params:
index_earliest: "{index_earliest:[0-9]+}"
index_latest: "{index_latest:[0-9]+}"
output_mode: json
search: 'search sourcetype="XmlWinEventLog:Microsoft-Windows-Sysmon/Operational" | streamstats max(_indextime) AS max_indextime'
request_headers:
Content-Type:
- "application/x-www-form-urlencoded"
responses:
- status_code: 200
headers:
Content-Type:
- "application/json"
body: |-
{
"preview": false,
"offset": 194,
"lastrow": true,
"result": {
"_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
"_cd": "0:315",
"_indextime": "1622471463",
"_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Sysmon' Guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}'/><EventID>22</EventID><Version>5</Version><Level>4</Level><Task>22</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime='2019-07-18T03:34:02.025237700Z'/><EventRecordID>67</EventRecordID><Correlation/><Execution ProcessID='2828' ThreadID='1684'/><Channel>Microsoft-Windows-Sysmon/Operational</Channel><Computer>vagrant-2016</Computer><Security UserID='S-1-5-18'/></System><EventData><Data Name='RuleName'></Data><Data Name='UtcTime'>2019-07-18 03:34:01.261</Data><Data Name='ProcessGuid'>{fa4a0de6-e8a9-5d2f-0000-001053699900}</Data><Data Name='ProcessId'>2736</Data><Data Name='QueryName'>www.msn.com</Data><Data Name='QueryStatus'>0</Data><Data Name='QueryResults'>type: 5 www-msn-com.a-0003.a-msedge.net;type: 5 a-0003.a-msedge.net;::ffff:204.79.197.203;</Data><Data Name='Image'>C:\\Program Files (x86)\\Internet Explorer\\iexplore.exe</Data></EventData></Event>",
"_serial": "194",
"_si": [
"69819b6ce1bd",
"main"
],
"_sourcetype": "XmlWinEventLog:Security",
"_time": "2021-05-25 13:11:45.000 UTC",
"host": "VAGRANT",
"index": "main",
"linecount": "1",
"max_indextime": "1622471606",
"source": "WinEventLog:Security",
"sourcetype": "XmlWinEventLog:Security",
"splunk_server": "69819b6ce1bd"
}
}
8 changes: 8 additions & 0 deletions packages/windows/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
# newer versions go on top
- version: "0.8.2"
changes:
- description: Add system tests for Splunk http inputs and improve README.
type: enhancement
link: https://github.com/elastic/integrations/pull/1044
- description: Fix sysmon pipeline when processing `dns.resolved_ip`.
type: bugfix
link: https://github.com/elastic/integrations/pull/1044
- version: "0.8.1"
changes:
- description: Fix security pipeline to support string event.code.
Expand Down
9 changes: 9 additions & 0 deletions packages/windows/data_stream/forwarded/_dev/test/system/test-default-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
input: httpjson
service: splunk-mock
vars:
url: http://{{Hostname}}:{{Port}}
username: test
password: test
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an enhancement we should make stream verify the basic auth.

data_stream:
vars:
preserve_original_event: true
3 changes: 3 additions & 0 deletions packages/windows/data_stream/forwarded/fields/beats.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
- name: input.type
type: keyword
description: Type of Filebeat input.
75 changes: 75 additions & 0 deletions packages/windows/data_stream/forwarded/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{
"@timestamp": "2020-05-13T09:04:04.755Z",
"agent": {
"ephemeral_id": "6d6bb3f5-f905-4ee5-8bee-c719616f8b6b",
"hostname": "docker-fleet-agent",
"id": "6fe85b08-7c10-4f55-ba4e-eeb75fdd6fdf",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "7.13.0"
},
"data_stream": {
"dataset": "windows.forwarded",
"namespace": "ep",
"type": "logs"
},
"ecs": {
"version": "1.9.0"
},
"elastic_agent": {
"id": "6bdfe1ae-64c3-4177-8b0c-2380a6e01322",
"snapshot": true,
"version": "7.13.0"
},
"event": {
"category": "process",
"code": "4105",
"created": "2021-06-01T10:22:56.365Z",
"dataset": "windows.forwarded",
"ingested": "2021-06-01T10:22:57.387144900Z",
"kind": "event",
"original": "\u003cEvent xmlns='http://schemas.microsoft.com/win/2004/08/events/event'\u003e\u003cSystem\u003e\u003cProvider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/\u003e\u003cEventID\u003e4105\u003c/EventID\u003e\u003cVersion\u003e1\u003c/Version\u003e\u003cLevel\u003e5\u003c/Level\u003e\u003cTask\u003e102\u003c/Task\u003e\u003cOpcode\u003e15\u003c/Opcode\u003e\u003cKeywords\u003e0x0\u003c/Keywords\u003e\u003cTimeCreated SystemTime='2020-05-13T09:04:04.755232500Z'/\u003e\u003cEventRecordID\u003e790\u003c/EventRecordID\u003e\u003cCorrelation ActivityID='{dd68516a-2930-0000-5962-68dd3029d601}'/\u003e\u003cExecution ProcessID='4204' ThreadID='1476'/\u003e\u003cChannel\u003eMicrosoft-Windows-PowerShell/Operational\u003c/Channel\u003e\u003cComputer\u003evagrant\u003c/Computer\u003e\u003cSecurity UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name='ScriptBlockId'\u003ef4a378ab-b74f-41a7-a5ef-6dd55562fdb9\u003c/Data\u003e\u003cData Name='RunspaceId'\u003e9c031e5c-8d5a-4b91-a12e-b3624970b623\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
"provider": "Microsoft-Windows-PowerShell",
"type": "start"
},
"host": {
"name": "vagrant"
},
"input": {
"type": "httpjson"
},
"log": {
"level": "verbose"
},
"powershell": {
"file": {
"script_block_id": "f4a378ab-b74f-41a7-a5ef-6dd55562fdb9"
},
"runspace_id": "9c031e5c-8d5a-4b91-a12e-b3624970b623"
},
"tags": [
"forwarded"
],
"user": {
"id": "S-1-5-21-1350058589-2282154016-2764056528-1000"
},
"winlog": {
"activity_id": "{dd68516a-2930-0000-5962-68dd3029d601}",
"channel": "Microsoft-Windows-PowerShell/Operational",
"computer_name": "vagrant",
"event_id": "4105",
"process": {
"pid": 4204,
"thread": {
"id": 1476
}
},
"provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
"provider_name": "Microsoft-Windows-PowerShell",
"record_id": "790",
"user": {
"identifier": "S-1-5-21-1350058589-2282154016-2764056528-1000"
},
"version": 1
}
}
9 changes: 9 additions & 0 deletions packages/windows/data_stream/powershell/_dev/test/system/test-default-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
input: httpjson
service: splunk-mock
vars:
url: http://{{Hostname}}:{{Port}}
username: test
password: test
data_stream:
vars:
preserve_original_event: true
3 changes: 3 additions & 0 deletions packages/windows/data_stream/powershell/fields/beats.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
- name: input.type
type: keyword
description: Type of Filebeat input.
Loading