Skip to content

Commit

Permalink
Update Zeek DNS pipeline with ECS DNS fields (#13324)
Browse files Browse the repository at this point in the history
This adds ECS DNS fields to the Zeek DNS fileset (but does not change or remove any existing ones).

Use event.original
Add registered_domain
Add dns.resolved_ip

Relates #13320
  • Loading branch information
andrewkroh authored Aug 27, 2019
1 parent fa17a55 commit 3a69204
Show file tree
Hide file tree
Showing 9 changed files with 379 additions and 126 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add aws module s3access metricset. {pull}13170[13170] {issue}12880[12880]
- Update Suricata module to populate ECS DNS fields and handle EVE DNS version 2. {issue}13320[13320] {pull}13329[13329]
- Update PAN-OS fileset to use the ECS NAT fields. {issue}13320[13320] {pull}13330[13330]
- Add fields to the Zeek DNS fileset for ECS DNS. {issue}13320[13320] {pull}13324[13324]

*Heartbeat*

Expand Down
1 change: 1 addition & 0 deletions libbeat/processors/script/javascript/module/include.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package module
import (
// Register javascript modules.
_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/console"
_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/net"
_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/path"
_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor"
_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/require"
Expand Down
68 changes: 68 additions & 0 deletions libbeat/processors/script/javascript/module/net/net.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package net

import (
"net"

"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/require"
)

// Require registers the net module that provides utilities for working with IP
// addresses. It can be accessed using:
//
// // javascript
// var net = require('net');
//
func Require(vm *goja.Runtime, module *goja.Object) {
o := module.Get("exports").(*goja.Object)
o.Set("isIP", isIP)
o.Set("isIPv4", isIPv4)
o.Set("isIPv6", isIPv6)
}

func isIP(input string) int32 {
ip := net.ParseIP(input)
if ip == nil {
return 0
}

if ip.To4() != nil {
return 4
}

return 6
}

func isIPv4(input string) bool {
return 4 == isIP(input)
}

func isIPv6(input string) bool {
return 6 == isIP(input)
}

// Enable adds net to the given runtime.
func Enable(runtime *goja.Runtime) {
runtime.Set("net", require.Require(runtime, "net"))
}

func init() {
require.RegisterNativeModule("net", Require)
}
98 changes: 98 additions & 0 deletions libbeat/processors/script/javascript/module/net/net_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package net_test

import (
"testing"

"github.com/stretchr/testify/assert"

"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/processors/script/javascript"

_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/net"
_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/require"
)

func TestNetIsIP(t *testing.T) {
const script = `
var net = require('net');
function process(evt) {
var ip = evt.Get("ip");
var ipType = net.isIP(ip);
switch (ipType) {
case 4:
evt.Put("network.type", "ipv4");
break
case 6:
evt.Put("network.type", "ipv6");
break
}
}
`

p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil)
if err != nil {
t.Fatal(err)
}

for ip, typ := range map[string]interface{}{
"192.168.0.1": "ipv4",
"::ffff:192.168.0.1": "ipv4",
"2001:0db8:0000:0000:0000:ff00:0042:8329": "ipv6",
"2001:db8:0:0:0:ff00:42:8329": "ipv6",
"2001:db8::ff00:42:8329": "ipv6",
"www.elastic.co": nil,
} {
evt, err := p.Run(&beat.Event{Fields: common.MapStr{"ip": ip}})
if err != nil {
t.Fatal(err)
}

fields := evt.Fields.Flatten()
assert.Equal(t, typ, fields["network.type"])
}
}

func TestNetIsIPvN(t *testing.T) {
const script = `
var net = require('net');
function process(evt) {
if (net.isIPv4("192.168.0.1") !== true) {
throw "isIPv4 failed";
}
if (net.isIPv6("2001:db8::ff00:42:8329") !== true) {
throw "isIPv6 failed";
}
}
`

p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil)
if err != nil {
t.Fatal(err)
}

_, err = p.Run(&beat.Event{Fields: common.MapStr{}})
if err != nil {
t.Fatal(err)
}
}
152 changes: 128 additions & 24 deletions x-pack/filebeat/module/zeek/dns/config/dns.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,143 @@ paths:
exclude_files: [".gz$"]
tags: {{.tags}}

json.keys_under_root: false

processors:
- drop_fields:
fields: ["json.Z","json.auth","json.addl"]
- rename:
fields:
- from: "json"
to: "zeek.dns"
- {from: message, to: event.original}
- decode_json_fields:
fields: [event.original]
target: zeek.dns
- script:
lang: javascript
id: zeek_dns_flags
source: >
var net = require("net");
- from: "zeek.dns.id.orig_h"
to: "source.address"
function addDnsHeaderFlags(evt) {
var flag = evt.Get("zeek.dns.AA");
if (flag === true) {
evt.AppendTo("dns.header_flags", "AA");
}
flag = evt.Get("zeek.dns.TC");
if (flag === true) {
evt.AppendTo("dns.header_flags", "TC");
}
flag = evt.Get("zeek.dns.RD");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RD");
}
flag = evt.Get("zeek.dns.RA");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RA");
}
}
- from: "zeek.dns.id.orig_p"
to: "source.port"
function addDnsQuestionClass(evt) {
var qclass = evt.Get("zeek.dns.qclass");
if (!qclass) {
return;
}
switch (qclass) {
case 1:
qclass = "IN";
break;
case 3:
qclass = "CH";
break;
case 4:
qclass = "HS";
break;
case 254:
qclass = "NONE";
break;
case 255:
qclass = "ANY";
break;
}
evt.Put("dns.question.class", qclass);
}
- from: "zeek.dns.id.resp_h"
to: "destination.address"
function addDnsAnswers(evt) {
var answers = evt.Get("zeek.dns.answers");
var ttls = evt.Get("zeek.dns.TTLs");
if (!answers || !ttls || answers.length != ttls.length) {
return;
}
- from: "zeek.dns.id.resp_p"
to: "destination.port"
var resolvedIps = [];
var answersObjs = [];
for (var i = 0; i < answers.length; i++) {
var answer = answers[i];
answersObjs.push({
data: answer,
ttl: ttls[i],
})
if (net.isIP(answer)) {
resolvedIps.push(answer);
}
}
evt.Put("dns.answers", answersObjs);
if (resolvedIps.length > 0) {
evt.Put("dns.resolved_ip", resolvedIps);
}
}
- from: "zeek.dns.proto"
to: "network.transport"
function addEventDuration(evt) {
var rttSec = evt.Get("zeek.dns.rtt");
if (!rttSec) {
return;
}
evt.Put("event.duration", rttSec * 1000000000);
}
- from: "zeek.dns.uid"
to: "zeek.session_id"

function process(evt) {
addDnsHeaderFlags(evt);
addDnsQuestionClass(evt);
addDnsAnswers(evt);
addEventDuration(evt);
}
- convert:
ignore_missing: true
fail_on_error: false
{{ if .community_id }}
- community_id:
ignore_failure: true
mode: rename
fields:
source_ip: source.address
destination_ip: destination.address
- {from: zeek.dns.id.orig_h, to: source.address}
- {from: zeek.dns.id.orig_p, to: source.port, type: long}
- {from: zeek.dns.id.resp_h, to: destination.address}
- {from: zeek.dns.id.resp_p, to: destination.port, type: long}
- {from: zeek.dns.uid, to: zeek.session_id}
- {from: zeek.dns.proto, to: network.transport}
- convert:
ignore_missing: true
ignore_failure: true
mode: copy
fields:
- {from: source.address, to: source.ip, type: ip}
- {from: destination.address, to: destination.ip, type: ip}
- {from: zeek.session_id, to: event.id}
- {from: '@timestamp', to: event.created}
- {from: zeek.dns.trans_id, to: dns.id}
- {from: zeek.dns.query, to: dns.question.name}
- {from: zeek.dns.qtype_name, to: dns.question.type}
- {from: zeek.dns.rcode_name, to: dns.response_code}
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
{{ if .community_id }}
- community_id: ~
{{ end }}
- timestamp:
ignore_missing: true
field: zeek.dns.ts
layouts:
- UNIX
- drop_fields:
ignore_missing: true
fields:
- zeek.dns.Z
- zeek.dns.auth
- zeek.dns.addl
- zeek.dns.ts
Loading

0 comments on commit 3a69204

Please sign in to comment.