Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Duplicate system.process.cmdline field with process.command_line ECS field name #22325

Merged
merged 7 commits into from
Nov 3, 2020
Merged

Duplicate system.process.cmdline field with process.command_line ECS field name #22325

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3894a04
Use process.command_line ECS field name for system.process.cmdline field
kaiyan-sheng Oct 30, 2020
d652677
add field to ecs-migration.yml
kaiyan-sheng Oct 30, 2020
bcf09d4
add changelog
kaiyan-sheng Nov 2, 2020
dac430c
duplicate system.process.cmdline
kaiyan-sheng Nov 3, 2020
1f9a3bb
regenerate data.json for process metricset
kaiyan-sheng Nov 3, 2020
0e75081
update changelog
kaiyan-sheng Nov 3, 2020
a0a3134
update changelog
kaiyan-sheng Nov 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,6 +778,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
- Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381]
- Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703]
- Replace system.process.cmdline field name with process.command_line ECS field name. {pull}22325[22325]

*Packetbeat*

Expand Down
5 changes: 5 additions & 0 deletions dev-tools/ecs-migration.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1826,6 +1826,11 @@
alias: true
beat: metricbeat

- from: system.process.cmdline
to: process.command_line
alias: true
beat: metricbeat

- from: system.process.username
to: user.name
alias: true
Expand Down
5 changes: 2 additions & 3 deletions metricbeat/docs/fields.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41747,10 +41747,9 @@ alias to: process.pgid
*`system.process.cmdline`*::
+
--
The full command-line used to start the process, including the arguments separated by space.

type: alias

type: keyword
alias to: process.command_line

--

Expand Down
2 changes: 1 addition & 1 deletion metricbeat/module/system/fields.go
View file
Open in desktop

Large diffs are not rendered by default.

195 changes: 20 additions & 175 deletions metricbeat/module/system/process/_meta/data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,200 +11,45 @@
},
"process": {
"args": [
"/usr/lib/systemd/systemd",
"--switched-root",
"--system",
"--deserialize",
"28"
"/private/var/folders/gj/50xyx1cj35z_348989k485th0000gn/T/___TestData_in_github_com_elastic_beats_v7_metricbeat_module_system_process",
"-test.v",
"-test.run",
"^\\QTestData\\E$"
],
"name": "systemd",
"pgid": 1,
"pid": 1,
"ppid": 0
"command_line": "/private/var/folders/gj/50xyx1cj35z_348989k485th0000gn/T/___TestData_in_github_com_elastic_beats_v7_metricbeat_module_system_process -test.v -test.run ^\\QTestData\\E$",
"executable": "/private/var/folders/gj/50xyx1cj35z_348989k485th0000gn/T/___TestData_in_github_com_elastic_beats_v7_metricbeat_module_system_process",
"name": "___TestData_in_",
"pgid": 89312,
"pid": 89314,
"ppid": 89313
},
"service": {
"type": "system"
},
"system": {
"process": {
kaiyan-sheng marked this conversation as resolved.
Show resolved Hide resolved
"cgroup": {
"blkio": {
"id": "init.scope",
"path": "/init.scope",
"total": {
"bytes": 7453696,
"ios": 548
}
},
"cpu": {
"cfs": {
"period": {
"us": 100000
},
"quota": {
"us": 0
},
"shares": 1024
},
"id": "init.scope",
"path": "/init.scope",
"rt": {
"period": {
"us": 0
},
"runtime": {
"us": 0
}
},
"stats": {
"periods": 0,
"throttled": {
"ns": 0,
"periods": 0
}
}
},
"cpuacct": {
"id": "init.scope",
"path": "/init.scope",
"percpu": {
"1": 3930656993407,
"2": 4025787490535,
"3": 4064460082910,
"4": 3387847262532
},
"stats": {
"system": {
"ns": 4996000000000
},
"user": {
"ns": 10329380000000
}
},
"total": {
"ns": 15408751829384
}
},
"id": "init.scope",
"memory": {
"id": "init.scope",
"kmem": {
"failures": 0,
"limit": {
"bytes": 9223372036854771712
},
"usage": {
"bytes": 9404416,
"max": {
"bytes": 14987264
}
}
},
"kmem_tcp": {
"failures": 0,
"limit": {
"bytes": 9223372036854771712
},
"usage": {
"bytes": 0,
"max": {
"bytes": 0
}
}
},
"mem": {
"failures": 0,
"limit": {
"bytes": 9223372036854771712
},
"usage": {
"bytes": 29437952,
"max": {
"bytes": 70705152
}
}
},
"memsw": {
"failures": 0,
"limit": {
"bytes": 9223372036854771712
},
"usage": {
"bytes": 30392320,
"max": {
"bytes": 70705152
}
}
},
"path": "/init.scope",
"stats": {
"active_anon": {
"bytes": 3444736
},
"active_file": {
"bytes": 10563584
},
"cache": {
"bytes": 10752000
},
"hierarchical_memory_limit": {
"bytes": 9223372036854771712
},
"hierarchical_memsw_limit": {
"bytes": 9223372036854771712
},
"inactive_anon": {
"bytes": 6197248
},
"inactive_file": {
"bytes": 327680
},
"major_page_faults": 198,
"mapped_file": {
"bytes": 9867264
},
"page_faults": 3626304,
"pages_in": 1095732,
"pages_out": 1090806,
"rss": {
"bytes": 9592832
},
"rss_huge": {
"bytes": 0
},
"swap": {
"bytes": 675840
},
"unevictable": {
"bytes": 0
}
}
},
"path": "/init.scope"
},
"cmdline": "/usr/lib/systemd/systemd --switched-root --system --deserialize 28",
"cpu": {
"start_time": "2020-08-27T01:05:16.000Z",
"start_time": "2020-10-30T17:50:57.446Z",
"total": {
"norm": {
"pct": 0.0056
"pct": 0.0153
},
"pct": 0.0222,
"value": 15389060
"pct": 0.1834,
"value": 207
}
},
"memory": {
"rss": {
"bytes": 12853248,
"pct": 0.0008
"bytes": 15990784,
"pct": 0.0005
},
"share": 7118848,
"size": 176881664
"share": 0,
"size": 5139021824
},
"state": "sleeping"
"state": "running"
}
},
"user": {
"name": "root"
"name": "kaiyansheng"
}
}
8 changes: 3 additions & 5 deletions metricbeat/module/system/process/_meta/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,9 @@
path: process.pgid
migration: true
- name: cmdline
type: keyword
description: >
The full command-line used to start the process, including the
arguments separated by space.
ignore_above: 2048
type: alias
path: process.command_line
migration: true
- name: username
type: alias
path: user.name
Expand Down
9 changes: 5 additions & 4 deletions metricbeat/module/system/process/process.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,11 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
for _, proc := range procs {
rootFields := common.MapStr{
"process": common.MapStr{
"name": getAndRemove(proc, "name"),
"pid": getAndRemove(proc, "pid"),
"ppid": getAndRemove(proc, "ppid"),
"pgid": getAndRemove(proc, "pgid"),
"name": getAndRemove(proc, "name"),
"pid": getAndRemove(proc, "pid"),
"ppid": getAndRemove(proc, "ppid"),
"pgid": getAndRemove(proc, "pgid"),
"command_line": getAndRemove(proc, "cmdline"),
kaiyan-sheng marked this conversation as resolved.
Show resolved Hide resolved
},
"user": common.MapStr{
"name": getAndRemove(proc, "username"),
Expand Down
11 changes: 2 additions & 9 deletions metricbeat/module/system/test_system.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@

SYSTEM_DISK_HOST_FIELDS = ["read.bytes", "write.bytes"]

# cmdline is also part of the system process fields, but it may not be present
# for some kernel level processes. fd is also part of the system process, but
# fd is also part of the system process, but
# is not available on all OSes and requires root to read for all processes.
# cgroup is only available on linux.
SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state"]
Expand Down Expand Up @@ -442,10 +441,8 @@ def test_process(self):
output = self.read_output_json()
self.assertGreater(len(output), 0)

found_cmdline = False
for evt in output:
process = evt["system"]["process"]
found_cmdline |= "cmdline" in process

# Remove 'env' prior to checking documented fields because its keys are dynamic.
process.pop("env", None)
Expand All @@ -454,12 +451,9 @@ def test_process(self):
# Remove optional keys.
process.pop("cgroup", None)
process.pop("fd", None)
process.pop("cmdline", None)

self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys())

self.assertTrue(found_cmdline, "cmdline not found in any process events")

@unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os")
def test_process_unix(self):
"""
Expand Down Expand Up @@ -512,7 +506,6 @@ def test_process_unix(self):

# Remove optional keys.
process.pop("cgroup", None)
process.pop("cmdline", None)
process.pop("fd", None)

self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys())
Expand Down Expand Up @@ -543,7 +536,7 @@ def test_process_metricbeat(self):
output = self.read_output()[0]

assert re.match("(?i)metricbeat.test(.exe)?", output["process.name"])
assert re.match("(?i).*metricbeat.test(.exe)? -systemTest", output["system.process.cmdline"])
assert re.match("(?i).*metricbeat.test(.exe)? -systemTest", output["process.command_line"])
assert isinstance(output["system.process.state"], six.string_types)
assert isinstance(output["system.process.cpu.start_time"], six.string_types)
self.check_username(output["user.name"])
Expand Down
2 changes: 1 addition & 1 deletion metricbeat/tests/system/test_processors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def test_dropevent_with_complex_condition(self):
}],
processors=[{
"drop_event": {
"when.not": "contains.system.process.cmdline: metricbeat.test",
"when.not": "contains.system.process.state: running",
},
}]
)
Expand Down