[microsoft_dhcp] changes/fix to set values for source and the right values for host; #…

packages/microsoft_dhcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.19.0"
+  changes:
+    - description: Fix the values of host.* and introduce source.*
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7633
 - version: "1.18.0"
   changes:
     - description: Add QResult description based on numerical value

packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json

@@ -80,12 +80,12 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "test.com",
+            "source": {
+                "domain": "057182593757.test.com",
                 "ip": [
                     "172.28.43.169"
                 ],
-                "name": "057182593757.test.com"
+                "address": "057182593757.test.com"
             },
             "log": {
                 "file": {
@@ -125,12 +125,12 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "test.com",
+            "source": {
+                "address": "1-07.test.com",
+                "domain": "1-07.test.com",
                 "ip": [
                     "172.28.53.173"
-                ],
-                "name": "1-07.test.com"
+                ]
             },
             "log": {
                 "file": {
@@ -171,11 +171,12 @@
                 ]
             },
             "host": {
-                "domain": "test.com",
+                "address": "3-07.test.com",
+                "domain": "3-07.test.com",
                 "ip": [
                     "172.28.53.36"
-                ],
-                "name": "3-07.test.com"
+                ]
+
             },
             "log": {
                 "file": {
@@ -215,8 +216,7 @@
                     "denied"
                 ]
             },
-            "host": {
-                "id": "76691ED45C90",
+            "source": {
                 "ip": [
                     "172.28.52.0"
                 ],
@@ -262,12 +262,12 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "test.com",
+            "source": {
+                "address": "035856103966.test.com",
+                "domain": "035856103966.test.com",
                 "ip": [
                     "172.28.43.159"
-                ],
-                "name": "035856103966.test.com"
+                ]
             },
             "log": {
                 "file": {
@@ -307,12 +307,12 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "test.com",
+            "source": {
+                "address": "001100581357.test.com",
+                "domain": "001100581357.test.com",
                 "ip": [
                     "172.28.40.35"
-                ],
-                "name": "001100581357.test.com"
+                ]
             },
             "log": {
                 "file": {
@@ -353,16 +353,15 @@
                     "denied"
                 ]
             },
-            "host": {
-                "domain": "test.com",
-                "id": "000000000000",
+            "source": {
+                "address": "host.test.com",
+                "domain": "host.test.com",
                 "ip": [
                     "192.168.2.1"
                 ],
                 "mac": [
                     "00-00-00-00-00-00"
-                ],
-                "name": "host.test.com"
+                ]
             },
             "log": {
                 "file": {
@@ -395,16 +394,15 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "test.com",
-                "id": "000000000000",
+            "source": {
+                "address": "host.test.com",
+                "domain": "host.test.com",
                 "ip": [
                     "192.168.2.10"
                 ],
                 "mac": [
                     "00-00-00-00-00-00"
-                ],
-                "name": "host.test.com"
+                ]
             },
             "log": {
                 "file": {
@@ -444,16 +442,16 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "test.com",
+            "source": {
+                "address": "host.test.com",
+                "domain": "host.test.com",
                 "id": "000000000000",
                 "ip": [
                     "192.168.2.20"
                 ],
                 "mac": [
                     "00-00-00-00-00-00"
-                ],
-                "name": "host.test.com"
+                ]
             },
             "log": {
                 "file": {
@@ -535,12 +533,12 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "test.com",
+            "source": {
+                "address": "hostname.test.com",
+                "domain": "hostname.test.com",
                 "ip": [
                     "10.10.10.10"
-                ],
-                "name": "hostname.test.com"
+                ]
             },
             "log": {
                 "file": {
@@ -625,9 +623,9 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "local",
-                "name": "domain.local"
+            "source": {
+                "address": "domain.local",
+                "domain": "domain.local"
             },
             "log": {
                 "file": {
@@ -660,9 +658,9 @@
                     "connection"
                 ]
             },
-            "host": {
-                "domain": "local",
-                "name": "domain.local"
+            "source": {
+                "address": "domain.local",
+                "domain": "domain.local"
             },
             "log": {
                 "file": {
@@ -726,8 +724,7 @@
                     "connection"
                 ]
             },
-            "host": {
-                "id": "653445372C132434342A381337302B566C616D31",
+            "source": {
                 "ip": [
                     "192.168.10.40"
                 ]

packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -28,10 +28,10 @@ processors:
   - pipeline:
       name: '{{ IngestPipeline "dhcpv6" }}'
       if: "ctx.log?.file?.path != null && ctx.log.file.path.contains('V6')"
-  - append:
+  - set:
       field: host.ip
-      value: '{{{_tmp_.host.ip}}}'
-      if: ctx._tmp_?.host?.ip != null
+      override: false
+      copy_from: observer.ip
   - foreach:
       field: observer.mac
       ignore_missing: true
@@ -46,6 +46,26 @@ processors:
       processor:
         uppercase:
           field: _ingest._value
+  - set:
+      field: host.mac
+      override: false
+      copy_from: observer.mac
+  - set:
+      field: host.name
+      override: false
+      copy_from: agent.name
+  - grok:
+      field: host.name
+      if: 'ctx.host?.name != null && ctx.host.name.contains(".")'
+      patterns:
+        - "%{HOSTNAME}\\.%{GREEDYDATA:host.domain}"
+      pattern_definitions:
+        "HOSTNAME": "[^.]+"
+      ignore_failure: true
+  - set:
+      field: host.hostname
+      override: false
+      copy_from: agent.hostname
   - remove:
       field: event.original
       if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"

packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml

@@ -9,9 +9,9 @@ processors:
         - _tmp_.date
         - _tmp_.time
         - message
-        - _tmp_.host.ip
-        - host.name
-        - _tmp_.mac
+        - source.ip
+        - source.address
+        - _tmp_.source.mac
         - user.name
         - microsoft.dhcp.transaction_id
         - microsoft.dhcp.result
@@ -26,16 +26,12 @@ processors:
         - microsoft.dhcp.dns_error_code
       ignore_failure: true
   - lowercase:
-      field: host.name
-      if: "ctx.host?.name != null"
-  - grok:
-      field: host.name
-      if: 'ctx.host?.name != null && ctx.host.name.contains(".")'
-      patterns:
-        - "%{HOSTNAME}\\.%{GREEDYDATA:host.domain}"
-      pattern_definitions:
-        "HOSTNAME": "[^.]+"
-      ignore_failure: true
+      field: source.address
+      if: "ctx.source?.address != null"
+  - set:
+      field: source.domain
+      if: "ctx.source?.address != null"
+      copy_from: source.address
   - set:
       field: _tmp_.timestamp
       value: "{{{_tmp_.date}}} {{{_tmp_.time}}}"
@@ -401,21 +397,17 @@ processors:
       value: '{{{event.reason}}}'
       if: ctx.event?.reasson != null
   - uppercase:
-      field: _tmp_.mac
+      field: _tmp_.source.mac
       ignore_missing: true
-  - set:
-      if: ctx._tmp_?.mac != null
-      field: host.id
-      copy_from: _tmp_.mac
   - gsub:
-      field: _tmp_.mac
+      field: _tmp_.source.mac
       pattern: '(..)(?!$)'
       replacement: '$1-'
       ignore_missing: true
-  - append:
-      if: 'ctx.host?.id != null && ctx.host?.id.length() == 12'
-      field: host.mac
-      value: '{{{_tmp_.mac}}}'
+  - set:
+      if: ctx._tmp_?.source.mac != null
+      field: source.mac
+      copy_from: _tmp_.source.mac
 on_failure:
   - set:
       field: event.kind

packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml

@@ -8,8 +8,8 @@ processors:
         - _tmp_.date
         - _tmp_.time
         - message
-        - _tmp_.host.ip
-        - host.name
+        - source.ip
+        - source.address
         - microsoft.dhcp.error_code
         - microsoft.dhcp.duid.length
         - microsoft.dhcp.duid.hex
@@ -18,16 +18,12 @@ processors:
         - microsoft.dhcp.subnet_prefix
       ignore_failure: true
   - lowercase:
-      field: host.name
-      if: "ctx.host?.name != null"
-  - grok:
-      field: host.name
-      if: 'ctx.host?.name != null && ctx.host.name.contains(".")'
-      patterns:
-        - "%{HOSTNAME}\\.%{GREEDYDATA:host.domain}"
-      pattern_definitions:
-        "HOSTNAME": "[^.]+"
-      ignore_failure: true
+      field: source.address
+      if: "ctx.source?.address != null"
+  - set:
+      field: source.domain
+      if: "ctx.source?.address != null"
+      copy_from: source.address
   - set:
       field: _tmp_.timestamp
       value: "{{{_tmp_.date}}} {{{_tmp_.time}}}"

packages/microsoft_dhcp/data_stream/log/fields/ecs.yml

@@ -42,3 +42,11 @@
   name: tags
 - external: ecs
   name: user.name
+- external: ecs
+  name: source.address
+- external: ecs
+  name: source.ip
+- external: ecs
+  name: source.domain
+- external: ecs
+  name: source.mac

packages/microsoft_dhcp/data_stream/log/fields/fields.yml

@@ -64,4 +64,4 @@
     - name: subnet_prefix
       type: keyword
       description: |
-        The number of bits for the subnet prefix.
+        The number of bits for the subnet prefix.

packages/microsoft_dhcp/docs/README.md

@@ -146,7 +146,10 @@ An example event for `log` looks as following:
 | observer.hostname | Hostname of the observer. | keyword |
 | observer.ip | IP addresses of the observer. | ip |
 | observer.mac | MAC addresses of the observer. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
+| source.address| FQDN of the client who is the DHCP requestor (sender)| keyword |
+| source.mac| MAC of the client who is the DHCP requestor (sender). The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
+| source.ip| IP address of the client who is the DHCP requestor (sender)| ip |
+| source.domain| FQDN of the client who is the DHCP requestor (sender)| keyword |
 | tags | List of keywords used to tag each event. | keyword |
 | user.name | Short name or login of the user. | keyword |
-| user.name.text | Multi-field of `user.name`. | match_only_text |
-
+| user.name.text | Multi-field of `user.name`. | match_only_text |