diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json index 86da7729891..fb9e8fa34ed 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json @@ -38,7 +38,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-08-10T12:16:43.225275533Z", + "ingested": "2021-08-26T12:37:08.253067515Z", "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":true,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -117,7 +117,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-08-10T12:16:43.225299658Z", + "ingested": "2021-08-26T12:37:08.253113730Z", "original": "{\"ts\":1547188416.857497,\"uid\":\"CAcJw21BbVedgFnYH4\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38340,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -211,7 +211,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-08-10T12:16:43.225307774Z", + "ingested": "2021-08-26T12:37:08.253137894Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -275,7 +275,7 @@ "ip": "192.0.2.205" }, "event": { - "ingested": "2021-08-10T12:16:43.225310341Z", + "ingested": "2021-08-26T12:37:08.253152215Z", "original": "{\"ts\":1551399000.57855,\"uid\":\"Cc6NJ3GRlfjE44I3h\",\"id.orig_h\":\"192.0.2.205\",\"id.orig_p\":3,\"id.resp_h\":\"198.51.100.249\",\"id.resp_p\":3,\"proto\":\"icmp\",\"conn_state\":\"OTH\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"orig_pkts\":1,\"orig_ip_bytes\":107,\"resp_pkts\":0,\"resp_ip_bytes\":0,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -354,7 +354,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-08-10T12:16:43.225312808Z", + "ingested": "2021-08-26T12:37:08.253166885Z", "original": "{\"ts\":1617062400.404645,\"uid\":\"CCicIg43lOtCQOxXnb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":56190,\"id.resp_h\":\"46.101.87.151\",\"id.resp_p\":443,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -434,7 +434,7 @@ }, "event": { "duration": 103708982, - "ingested": "2021-08-10T12:16:43.225315271Z", + "ingested": "2021-08-26T12:37:08.253183090Z", "original": "{\"ts\":1617062100.419397,\"uid\":\"C52mXBCPJ4pPGkhr1\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60810,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10370898246765137,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -514,7 +514,7 @@ }, "event": { "duration": 104128838, - "ingested": "2021-08-10T12:16:43.225317743Z", + "ingested": "2021-08-26T12:37:08.253198430Z", "original": "{\"ts\":1617062100.419603,\"uid\":\"CTzCky2CyLT5JJvHck\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60804,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10412883758544922,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -594,7 +594,7 @@ }, "event": { "duration": 104333878, - "ingested": "2021-08-10T12:16:43.225320249Z", + "ingested": "2021-08-26T12:37:08.253214584Z", "original": "{\"ts\":1617062100.419826,\"uid\":\"CIkS28PDxqQnN49m2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60802,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10433387756347656,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -656,7 +656,7 @@ }, "event": { "duration": 26802063, - "ingested": "2021-08-10T12:16:43.225322698Z", + "ingested": "2021-08-26T12:37:08.253228176Z", "original": "{\"ts\":1617062390.563187,\"uid\":\"CezEGe4jeLNkayV976\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":38948,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.02680206298828125,\"orig_bytes\":0,\"resp_bytes\":241,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":269}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -719,7 +719,7 @@ }, "event": { "duration": 25056124, - "ingested": "2021-08-10T12:16:43.225325161Z", + "ingested": "2021-08-26T12:37:08.253241303Z", "original": "{\"ts\":1617062390.563442,\"uid\":\"CKSr3w18mmW6t7bXC4\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":40080,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.025056123733520509,\"orig_bytes\":0,\"resp_bytes\":276,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":304}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -782,7 +782,7 @@ }, "event": { "duration": 3319979, - "ingested": "2021-08-10T12:16:43.225327579Z", + "ingested": "2021-08-26T12:37:08.253258868Z", "original": "{\"ts\":1617062390.667048,\"uid\":\"CGUiHy4kLIF2ml95eg\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":41407,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.003319978713989258,\"orig_bytes\":0,\"resp_bytes\":133,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -845,7 +845,7 @@ }, "event": { "duration": 1111984, - "ingested": "2021-08-10T12:16:43.225330390Z", + "ingested": "2021-08-26T12:37:08.253273597Z", "original": "{\"ts\":1617062390.698943,\"uid\":\"CAOZZi4Qrio7gUVgVc\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":50487,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0011119842529296876,\"orig_bytes\":0,\"resp_bytes\":202,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":230}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -908,7 +908,7 @@ }, "event": { "duration": 908852, - "ingested": "2021-08-10T12:16:43.225332830Z", + "ingested": "2021-08-26T12:37:08.253287095Z", "original": "{\"ts\":1617062390.699227,\"uid\":\"Chx5fs3xQ5ALB72i4e\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":49647,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0009088516235351563,\"orig_bytes\":0,\"resp_bytes\":145,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":173}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -970,7 +970,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-08-10T12:16:43.225335244Z", + "ingested": "2021-08-26T12:37:08.253306054Z", "original": "{\"ts\":1617062400.703865,\"uid\":\"C3pPjh1YRYcVDiZD3\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44944,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1031,7 +1031,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-08-10T12:16:43.225337683Z", + "ingested": "2021-08-26T12:37:08.253319736Z", "original": "{\"ts\":1617062400.703851,\"uid\":\"ChUxTmYLG37oO5qUb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44942,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1092,7 +1092,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-08-10T12:16:43.225340124Z", + "ingested": "2021-08-26T12:37:08.253332989Z", "original": "{\"ts\":1617062400.704467,\"uid\":\"CpeAOT3B11CTXJgzw2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44946,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1205,7 +1205,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-08-10T12:16:43.225342661Z", + "ingested": "2021-08-26T12:37:08.253346340Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1270,7 +1270,7 @@ "ip": "10.0.2.15" }, "event": { - "ingested": "2021-08-10T12:16:43.225345083Z", + "ingested": "2021-08-26T12:37:08.253359559Z", "original": "{\"ts\":\"2021-06-09T20:55:13.160328Z\",\"uid\":\"C2KP1V3alRLoxl4JB9\",\"id.orig_h\":\"10.0.2.15\",\"id.orig_p\":46408,\"id.resp_h\":\"172.217.9.68\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json index cb59e72ff23..d89a329fbab 100644 --- a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json +++ b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json @@ -64,7 +64,7 @@ "ip": "130.118.205.62" }, "event": { - "ingested": "2021-08-10T12:16:45.963639174Z", + "ingested": "2021-08-26T12:37:14.367022936Z", "original": "{\"ts\":1602116947.977,\"uid\":\"CqlPpF1AQVLMPgGiL5\",\"id.orig_h\":\"130.118.205.62\",\"id.orig_p\":38461,\"id.resp_h\":\"208.79.89.249\",\"id.resp_p\":123,\"version\":4,\"mode\":3,\"stratum\":0,\"poll\":1,\"precision\":1,\"root_delay\":0,\"root_disp\":0,\"ref_id\":\"\\\\x00\\\\x00\\\\x00\\\\x00\",\"ref_time\":0,\"org_time\":0,\"rec_time\":0,\"xmt_time\":1602116947.215,\"num_exts\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -150,7 +150,7 @@ "ip": "130.118.205.62" }, "event": { - "ingested": "2021-08-10T12:16:45.963645447Z", + "ingested": "2021-08-26T12:37:14.367053691Z", "original": "{\"ts\":1602116948.081,\"uid\":\"CqlPpF1AQVLMPgGiL5\",\"id.orig_h\":\"130.118.205.62\",\"id.orig_p\":38461,\"id.resp_h\":\"208.79.89.249\",\"id.resp_p\":123,\"version\":4,\"mode\":4,\"stratum\":2,\"poll\":8,\"precision\":5.960464477539063e-8,\"root_delay\":0.00921630859375,\"root_disp\":0.0212249755859375,\"ref_id\":\"127.67.113.92\",\"ref_time\":1602116655.942,\"org_time\":1602116947.215,\"rec_time\":1602116947.964,\"xmt_time\":1602116947.964,\"num_exts\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ntp/fields/base-fields.yml b/packages/zeek/data_stream/ntp/fields/base-fields.yml index 9790a9113a2..048a36b4f4c 100644 --- a/packages/zeek/data_stream/ntp/fields/base-fields.yml +++ b/packages/zeek/data_stream/ntp/fields/base-fields.yml @@ -14,7 +14,7 @@ - name: event.dataset type: constant_keyword description: Event dataset - value: zeek.connection + value: zeek.ntp - name: '@timestamp' type: date description: Event timestamp. diff --git a/packages/zeek/data_stream/ntp/fields/ecs.yml b/packages/zeek/data_stream/ntp/fields/ecs.yml index 6bae2883f4f..3a5f3e6922c 100644 --- a/packages/zeek/data_stream/ntp/fields/ecs.yml +++ b/packages/zeek/data_stream/ntp/fields/ecs.yml @@ -1,227 +1,100 @@ -- description: Destination network address. - ignore_above: 1024 - name: destination.address - type: keyword -- description: Unique number allocated to the autonomous system. - example: 15169 - name: destination.as.number - type: long -- description: Organization name. - example: Google LLC - ignore_above: 1024 - multi_fields: - - flat_name: destination.as.organization.name.text - name: text - norms: false - type: text - name: destination.as.organization.name - type: keyword -- description: Bytes sent from the destination to the source. - example: 184 - name: destination.bytes - type: long -- description: City name. - example: Montreal - ignore_above: 1024 - name: destination.geo.city_name - type: keyword -- description: Name of the continent. - example: North America - ignore_above: 1024 - name: destination.geo.continent_name - type: keyword -- description: Country ISO code. - example: CA - ignore_above: 1024 - name: destination.geo.country_iso_code - type: keyword -- description: Country name. - example: Canada - ignore_above: 1024 - name: destination.geo.country_name - type: keyword -- description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - name: destination.geo.location - type: geo_point -- description: User-defined description of a location. - example: boston-dc - ignore_above: 1024 - name: destination.geo.name - type: keyword -- description: Region ISO code. - example: CA-QC - ignore_above: 1024 - name: destination.geo.region_iso_code - type: keyword -- description: Region name. - example: Quebec - ignore_above: 1024 - name: destination.geo.region_name - type: keyword -- description: IP address of the destination. - name: destination.ip - type: ip -- description: MAC address of the destination. - ignore_above: 1024 - name: destination.mac - type: keyword -- description: Packets sent from the destination to the source. - example: 12 - name: destination.packets - type: long -- description: Port of the destination. - name: destination.port - type: long -- description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 - name: ecs.version - type: keyword -- description: Error message. - name: error.message - type: text -- description: Event category. The second categorization field in the hierarchy. - example: authentication - ignore_above: 1024 - name: event.category - type: keyword -- description: Time when the event was first read by an agent or by your pipeline. - example: "2016-05-23T08:05:34.857Z" - name: event.created - type: date -- description: Duration of the event in nanoseconds. - name: event.duration - type: long -- description: Unique ID to describe the event. - example: 8a4f500d - ignore_above: 1024 - name: event.id - type: keyword -- description: Timestamp when an event arrived in the central data store. - example: "2016-05-23T08:05:35.101Z" - name: event.ingested - type: date -- description: The kind of the event. The highest categorization field in the hierarchy. - example: alert - ignore_above: 1024 - name: event.kind - type: keyword -- description: Event type. The third categorization field in the hierarchy. - ignore_above: 1024 - name: event.type - type: keyword -- description: Host ip addresses. - name: host.ip - type: ip -- description: Total bytes transferred in both directions. - example: 368 - name: network.bytes - type: long -- description: A hash of source and destination IPs and ports. - example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= - ignore_above: 1024 - name: network.community_id - type: keyword -- description: Direction of the network traffic. - example: inbound - ignore_above: 1024 - name: network.direction - type: keyword -- description: Total packets transferred in both directions. - example: 24 - name: network.packets - type: long -- description: L7 Network protocol name. - example: http - ignore_above: 1024 - name: network.protocol - type: keyword -- description: Protocol Name corresponding to the field `iana_number`. - example: tcp - ignore_above: 1024 - name: network.transport - type: keyword -- description: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc - example: ipv4 - ignore_above: 1024 - name: network.type - type: keyword -- description: All of the IPs seen on your event. - name: related.ip - type: ip -- description: Source network address. - ignore_above: 1024 - name: source.address - type: keyword -- description: Unique number allocated to the autonomous system. - example: 15169 - name: source.as.number - type: long -- description: Organization name. - example: Google LLC - ignore_above: 1024 - multi_fields: - - flat_name: source.as.organization.name.text - name: text - norms: false - type: text - name: source.as.organization.name - type: keyword -- description: Bytes sent from the source to the destination. - example: 184 - name: source.bytes - type: long -- description: City name. - example: Montreal - ignore_above: 1024 - name: source.geo.city_name - type: keyword -- description: Name of the continent. - example: North America - ignore_above: 1024 - name: source.geo.continent_name - type: keyword -- description: Country ISO code. - example: CA - ignore_above: 1024 - name: source.geo.country_iso_code - type: keyword -- description: Country name. - example: Canada - ignore_above: 1024 - name: source.geo.country_name - type: keyword -- description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - name: source.geo.location - type: geo_point -- description: User-defined description of a location. - example: boston-dc - ignore_above: 1024 - name: source.geo.name - type: keyword -- description: Region ISO code. - example: CA-QC - ignore_above: 1024 - name: source.geo.region_iso_code - type: keyword -- description: Region name. - example: Quebec - ignore_above: 1024 - name: source.geo.region_name - type: keyword -- description: IP address of the source. - name: source.ip - type: ip -- description: MAC address of the source. - ignore_above: 1024 - name: source.mac - type: keyword -- description: Packets sent from the source to the destination. - example: 12 - name: source.packets - type: long -- description: Port of the source. - name: source.port - type: long +- name: destination.address + external: ecs +- name: destination.as.number + external: ecs +- name: destination.as.organization.name + external: ecs +- name: destination.bytes + external: ecs +- name: destination.geo.city_name + external: ecs +- name: destination.geo.continent_name + external: ecs +- name: destination.geo.country_iso_code + external: ecs +- name: destination.geo.country_name + external: ecs +- name: destination.geo.location + external: ecs +- name: destination.geo.name + external: ecs +- name: destination.geo.region_iso_code + external: ecs +- name: destination.geo.region_name + external: ecs +- name: destination.ip + external: ecs +- name: destination.mac + external: ecs +- name: destination.packets + external: ecs +- name: destination.port + external: ecs +- name: ecs.version + external: ecs +- name: error.message + external: ecs +- name: event.category + external: ecs +- name: event.created + external: ecs +- name: event.duration + external: ecs +- name: event.id + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: host.ip + external: ecs +- name: network.bytes + external: ecs +- name: network.community_id + external: ecs +- name: network.direction + external: ecs +- name: network.packets + external: ecs +- name: network.protocol + external: ecs +- name: network.transport + external: ecs +- name: network.type + external: ecs +- name: related.ip + external: ecs +- name: source.address + external: ecs +- name: source.as.number + external: ecs +- name: source.as.organization.name + external: ecs +- name: source.bytes + external: ecs +- name: source.geo.city_name + external: ecs +- name: source.geo.continent_name + external: ecs +- name: source.geo.country_iso_code + external: ecs +- name: source.geo.country_name + external: ecs +- name: source.geo.location + external: ecs +- name: source.geo.name + external: ecs +- name: source.geo.region_iso_code + external: ecs +- name: source.geo.region_name + external: ecs +- name: source.ip + external: ecs +- name: source.mac + external: ecs +- name: source.packets + external: ecs +- name: source.port + external: ecs diff --git a/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-expected.json b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-expected.json index 44dd953d78b..9850d1fc4b4 100644 --- a/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-expected.json +++ b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-expected.json @@ -72,7 +72,7 @@ ] }, "event": { - "ingested": "2021-08-10T12:16:47.210647593Z", + "ingested": "2021-08-26T12:37:15.323536262Z", "original": "{\"ts\": 1611852809.869245,\"uid\": \"CbjAXE4CBxJ8W7VoJg\",\"src_addr\": \"124.51.137.154\",\"src_port\": 51617,\"dst_addr\": \"160.218.27.63\",\"dst_port\": 445,\"note\": \"Signatures::Sensitive_Signature\",\"sig_id\": \"my-second-sig\",\"event_msg\": \"124.51.137.154: TCP traffic\",\"sub_msg\": \"\"}", "id": "CbjAXE4CBxJ8W7VoJg", "category": "network", diff --git a/packages/zeek/data_stream/signature/fields/base-fields.yml b/packages/zeek/data_stream/signature/fields/base-fields.yml index 9790a9113a2..a1d0bd61fdd 100644 --- a/packages/zeek/data_stream/signature/fields/base-fields.yml +++ b/packages/zeek/data_stream/signature/fields/base-fields.yml @@ -14,7 +14,7 @@ - name: event.dataset type: constant_keyword description: Event dataset - value: zeek.connection + value: zeek.signature - name: '@timestamp' type: date description: Event timestamp. diff --git a/packages/zeek/data_stream/signature/fields/ecs.yml b/packages/zeek/data_stream/signature/fields/ecs.yml index 8cf67091cf1..c598ec29a9f 100644 --- a/packages/zeek/data_stream/signature/fields/ecs.yml +++ b/packages/zeek/data_stream/signature/fields/ecs.yml @@ -1,235 +1,104 @@ -- description: Destination network address. - ignore_above: 1024 - name: destination.address - type: keyword -- description: Unique number allocated to the autonomous system. - example: 15169 - name: destination.as.number - type: long -- description: Organization name. - example: Google LLC - ignore_above: 1024 - multi_fields: - - flat_name: destination.as.organization.name.text - name: text - norms: false - type: text - name: destination.as.organization.name - type: keyword -- description: Bytes sent from the destination to the source. - example: 184 - name: destination.bytes - type: long -- description: City name. - example: Montreal - ignore_above: 1024 - name: destination.geo.city_name - type: keyword -- description: Name of the continent. - example: North America - ignore_above: 1024 - name: destination.geo.continent_name - type: keyword -- description: Country ISO code. - example: CA - ignore_above: 1024 - name: destination.geo.country_iso_code - type: keyword -- description: Country name. - example: Canada - ignore_above: 1024 - name: destination.geo.country_name - type: keyword -- description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - name: destination.geo.location - type: geo_point -- description: User-defined description of a location. - example: boston-dc - ignore_above: 1024 - name: destination.geo.name - type: keyword -- description: Region ISO code. - example: CA-QC - ignore_above: 1024 - name: destination.geo.region_iso_code - type: keyword -- description: Region name. - example: Quebec - ignore_above: 1024 - name: destination.geo.region_name - type: keyword -- description: IP address of the destination. - name: destination.ip - type: ip -- description: MAC address of the destination. - ignore_above: 1024 - name: destination.mac - type: keyword -- description: Packets sent from the destination to the source. - example: 12 - name: destination.packets - type: long -- description: Port of the destination. - name: destination.port - type: long -- description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 - name: ecs.version - type: keyword -- description: Error message. - name: error.message - type: text -- description: Event category. The second categorization field in the hierarchy. - example: authentication - ignore_above: 1024 - name: event.category - type: keyword -- description: Time when the event was first read by an agent or by your pipeline. - example: "2016-05-23T08:05:34.857Z" - name: event.created - type: date -- description: Duration of the event in nanoseconds. - name: event.duration - type: long -- description: Unique ID to describe the event. - example: 8a4f500d - ignore_above: 1024 - name: event.id - type: keyword -- description: Timestamp when an event arrived in the central data store. - example: "2016-05-23T08:05:35.101Z" - name: event.ingested - type: date -- description: The kind of the event. The highest categorization field in the hierarchy. - example: alert - ignore_above: 1024 - name: event.kind - type: keyword -- description: Event type. The third categorization field in the hierarchy. - ignore_above: 1024 - name: event.type - type: keyword -- description: Host ip addresses. - name: host.ip - type: ip -- description: Total bytes transferred in both directions. - example: 368 - name: network.bytes - type: long -- description: A hash of source and destination IPs and ports. - example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= - ignore_above: 1024 - name: network.community_id - type: keyword -- description: Direction of the network traffic. - example: inbound - ignore_above: 1024 - name: network.direction - type: keyword -- description: Total packets transferred in both directions. - example: 24 - name: network.packets - type: long -- description: L7 Network protocol name. - example: http - ignore_above: 1024 - name: network.protocol - type: keyword -- description: Protocol Name corresponding to the field `iana_number`. - example: tcp - ignore_above: 1024 - name: network.transport - type: keyword -- description: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc - example: ipv4 - ignore_above: 1024 - name: network.type - type: keyword -- description: All of the IPs seen on your event. - name: related.ip - type: ip -- description: Source network address. - ignore_above: 1024 - name: source.address - type: keyword -- description: Unique number allocated to the autonomous system. - example: 15169 - name: source.as.number - type: long -- description: Organization name. - example: Google LLC - ignore_above: 1024 - multi_fields: - - flat_name: source.as.organization.name.text - name: text - norms: false - type: text - name: source.as.organization.name - type: keyword -- description: Bytes sent from the source to the destination. - example: 184 - name: source.bytes - type: long -- description: City name. - example: Montreal - ignore_above: 1024 - name: source.geo.city_name - type: keyword -- description: Name of the continent. - example: North America - ignore_above: 1024 - name: source.geo.continent_name - type: keyword -- description: Country ISO code. - example: CA - ignore_above: 1024 - name: source.geo.country_iso_code - type: keyword -- description: Country name. - example: Canada - ignore_above: 1024 - name: source.geo.country_name - type: keyword -- description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - name: source.geo.location - type: geo_point -- description: User-defined description of a location. - example: boston-dc - ignore_above: 1024 - name: source.geo.name - type: keyword -- description: Region ISO code. - example: CA-QC - ignore_above: 1024 - name: source.geo.region_iso_code - type: keyword -- description: Region name. - example: Quebec - ignore_above: 1024 - name: source.geo.region_name - type: keyword -- description: IP address of the source. - name: source.ip - type: ip -- description: MAC address of the source. - ignore_above: 1024 - name: source.mac - type: keyword -- description: Packets sent from the source to the destination. - example: 12 - name: source.packets - type: long -- description: Port of the source. - name: source.port - type: long -- description: A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. - ignore_above: 1024 - name: rule.id - type: keyword -- description: The description of the rule generating the event. - ignore_above: 1024 - name: rule.description - type: keyword +- name: destination.address + external: ecs +- name: destination.as.number + external: ecs +- name: destination.as.organization.name + external: ecs +- name: destination.bytes + external: ecs +- name: destination.geo.city_name + external: ecs +- name: destination.geo.continent_name + external: ecs +- name: destination.geo.country_iso_code + external: ecs +- name: destination.geo.country_name + external: ecs +- name: destination.geo.location + external: ecs +- name: destination.geo.name + external: ecs +- name: destination.geo.region_iso_code + external: ecs +- name: destination.geo.region_name + external: ecs +- name: destination.ip + external: ecs +- name: destination.mac + external: ecs +- name: destination.packets + external: ecs +- name: destination.port + external: ecs +- name: ecs.version + external: ecs +- name: error.message + external: ecs +- name: event.category + external: ecs +- name: event.created + external: ecs +- name: event.duration + external: ecs +- name: event.id + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: host.ip + external: ecs +- name: network.bytes + external: ecs +- name: network.community_id + external: ecs +- name: network.direction + external: ecs +- name: network.packets + external: ecs +- name: network.protocol + external: ecs +- name: network.transport + external: ecs +- name: network.type + external: ecs +- name: related.ip + external: ecs +- name: source.address + external: ecs +- name: source.as.number + external: ecs +- name: source.as.organization.name + external: ecs +- name: source.bytes + external: ecs +- name: source.geo.city_name + external: ecs +- name: source.geo.continent_name + external: ecs +- name: source.geo.country_iso_code + external: ecs +- name: source.geo.country_name + external: ecs +- name: source.geo.location + external: ecs +- name: source.geo.name + external: ecs +- name: source.geo.region_iso_code + external: ecs +- name: source.geo.region_name + external: ecs +- name: source.ip + external: ecs +- name: source.mac + external: ecs +- name: source.packets + external: ecs +- name: source.port + external: ecs +- name: rule.id + external: ecs +- name: rule.description + external: ecs diff --git a/packages/zeek/docs/README.md b/packages/zeek/docs/README.md index fd611310305..10e7871080f 100644 --- a/packages/zeek/docs/README.md +++ b/packages/zeek/docs/README.md @@ -1707,8 +1707,8 @@ NTP data. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Destination network address. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. | long | +| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | destination.as.organization.name | Organization name. | keyword | | destination.bytes | Bytes sent from the destination to the source. | long | | destination.geo.city_name | City name. | keyword | @@ -1716,24 +1716,24 @@ NTP data. | destination.geo.country_iso_code | Country ISO code. | keyword | | destination.geo.country_name | Country name. | keyword | | destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location. | keyword | +| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | | destination.geo.region_iso_code | Region ISO code. | keyword | | destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination. | ip | -| destination.mac | MAC address of the destination. | keyword | +| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | +| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | destination.packets | Packets sent from the destination to the source. | long | | destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | error.message | Error message. | text | -| event.category | Event category. The second categorization field in the hierarchy. | keyword | -| event.created | Time when the event was first read by an agent or by your pipeline. | date | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. | long | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. | date | -| event.kind | The kind of the event. The highest categorization field in the hierarchy. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.type | Event type. The third categorization field in the hierarchy. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1754,16 +1754,16 @@ NTP data. | log.file.path | Full path to the log file this event came from. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| network.bytes | Total bytes transferred in both directions. | long | -| network.community_id | A hash of source and destination IPs and ports. | keyword | -| network.direction | Direction of the network traffic. | keyword | -| network.packets | Total packets transferred in both directions. | long | -| network.protocol | L7 Network protocol name. | keyword | -| network.transport | Protocol Name corresponding to the field `iana_number`. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | +| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | +| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | +| network.direction | Direction of the network traffic. Recommended values are: \* ingress \* egress \* inbound \* outbound \* internal \* external \* unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | +| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | +| network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | | related.ip | All of the IPs seen on your event. | ip | -| source.address | Source network address. | keyword | -| source.as.number | Unique number allocated to the autonomous system. | long | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.bytes | Bytes sent from the source to the destination. | long | | source.geo.city_name | City name. | keyword | @@ -1771,11 +1771,11 @@ NTP data. | source.geo.country_iso_code | Country ISO code. | keyword | | source.geo.country_name | Country name. | keyword | | source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location. | keyword | +| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | | source.geo.region_iso_code | Region ISO code. | keyword | | source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source. | ip | -| source.mac | MAC address of the source. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | source.packets | Packets sent from the source to the destination. | long | | source.port | Port of the source. | long | | tags | List of keywords used to tag each event. | keyword | @@ -2379,8 +2379,8 @@ Zeek signature matches. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Destination network address. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. | long | +| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | destination.as.organization.name | Organization name. | keyword | | destination.bytes | Bytes sent from the destination to the source. | long | | destination.geo.city_name | City name. | keyword | @@ -2388,24 +2388,24 @@ Zeek signature matches. | destination.geo.country_iso_code | Country ISO code. | keyword | | destination.geo.country_name | Country name. | keyword | | destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location. | keyword | +| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | | destination.geo.region_iso_code | Region ISO code. | keyword | | destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination. | ip | -| destination.mac | MAC address of the destination. | keyword | +| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | +| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | destination.packets | Packets sent from the destination to the source. | long | | destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | error.message | Error message. | text | -| event.category | Event category. The second categorization field in the hierarchy. | keyword | -| event.created | Time when the event was first read by an agent or by your pipeline. | date | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. | long | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. | date | -| event.kind | The kind of the event. The highest categorization field in the hierarchy. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.type | Event type. The third categorization field in the hierarchy. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -2426,18 +2426,18 @@ Zeek signature matches. | log.file.path | Full path to the log file this event came from. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| network.bytes | Total bytes transferred in both directions. | long | -| network.community_id | A hash of source and destination IPs and ports. | keyword | -| network.direction | Direction of the network traffic. | keyword | -| network.packets | Total packets transferred in both directions. | long | -| network.protocol | L7 Network protocol name. | keyword | -| network.transport | Protocol Name corresponding to the field `iana_number`. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | +| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | +| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | +| network.direction | Direction of the network traffic. Recommended values are: \* ingress \* egress \* inbound \* outbound \* internal \* external \* unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | +| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | +| network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | | related.ip | All of the IPs seen on your event. | ip | | rule.description | The description of the rule generating the event. | keyword | | rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| source.address | Source network address. | keyword | -| source.as.number | Unique number allocated to the autonomous system. | long | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.bytes | Bytes sent from the source to the destination. | long | | source.geo.city_name | City name. | keyword | @@ -2445,11 +2445,11 @@ Zeek signature matches. | source.geo.country_iso_code | Country ISO code. | keyword | | source.geo.country_name | Country name. | keyword | | source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location. | keyword | +| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | | source.geo.region_iso_code | Region ISO code. | keyword | | source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source. | ip | -| source.mac | MAC address of the source. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | source.packets | Packets sent from the source to the destination. | long | | source.port | Port of the source. | long | | tags | List of keywords used to tag each event. | keyword |