From e268f01fd38657ff176471e66cecf1f37ff4a377 Mon Sep 17 00:00:00 2001 From: McStork Date: Tue, 8 Dec 2015 16:27:11 -0500 Subject: [PATCH] DNS - implementing Parse for Tcp. * Implement DNS over TCP * Publish Notes when a response fails to decode in Gap and Fin --- packetbeat/protos/dns/dns.go | 205 +++++++- packetbeat/protos/dns/dns_test.go | 480 +++++++++++++++++- .../tests/system/pcaps/dns_tcp_axfr.pcap | Bin 0 -> 915 bytes packetbeat/tests/system/test_0032_dns.py | 24 + tests/system/pcaps/dns_tcp_axfr.pcap | Bin 0 -> 915 bytes 5 files changed, 691 insertions(+), 18 deletions(-) create mode 100644 packetbeat/tests/system/pcaps/dns_tcp_axfr.pcap create mode 100644 tests/system/pcaps/dns_tcp_axfr.pcap diff --git a/packetbeat/protos/dns/dns.go b/packetbeat/protos/dns/dns.go index 777ae7e03e7..2067ff3e826 100644 --- a/packetbeat/protos/dns/dns.go +++ b/packetbeat/protos/dns/dns.go @@ -6,8 +6,7 @@ // messages. // // Future Additions: -// * Implement TcpProtocolPlugin. -// * Publish a message when packets are received that cannot be decoded. +// * Publish a message when Query packets are received that cannot be decoded. // * Add EDNS and DNSSEC support (consider using miekg/dns instead // of gopacket). // * Consider adding ICMP support to @@ -30,6 +29,7 @@ import ( "github.com/elastic/beats/packetbeat/config" "github.com/elastic/beats/packetbeat/procs" "github.com/elastic/beats/packetbeat/protos" + "github.com/elastic/beats/packetbeat/protos/tcp" "github.com/tsg/gopacket" "github.com/tsg/gopacket/layers" @@ -45,8 +45,10 @@ const ( // Notes that are added to messages during exceptional conditions. const ( - NonDnsPacketMsg = "Packet's data could not be decoded as DNS." - DuplicateQueryMsg = "Another query with the same DNS ID from this client " + + NonDnsPacketMsg = "Packet's data could not be decoded as DNS." + NonDnsCompleteMsg = "Message's data could not be decoded as DNS." + NonDnsResponsePacketMsg = "Response packet's data could not be decoded as DNS" + DuplicateQueryMsg = "Another query with the same DNS ID from this client " + "was received so this query was closed without receiving a response." OrphanedResponseMsg = "Response was received without an associated query." NoResponse = "No response to this query was received." @@ -60,6 +62,8 @@ const ( TransportUdp ) +const DecodeOffset = 2 + var TransportNames = []string{ "tcp", "udp", @@ -166,7 +170,7 @@ type DnsMessage struct { // DnsStream contains DNS data from one side of a TCP transmission. A pair // of DnsStream's are used to represent the full conversation. type DnsStream struct { - tcptuple *common.TcpTuple + tcpTuple *common.TcpTuple data []byte @@ -318,7 +322,7 @@ func (dns *Dns) ParseUdp(pkt *protos.Packet) { logp.Debug("dns", "Parsing packet addressed with %s of length %d.", pkt.Tuple.String(), len(pkt.Payload)) - dnsPkt, err := decodeDnsPacket(pkt.Payload) + dnsPkt, err := decodeDnsData(pkt.Payload) if err != nil { // This means that malformed requests or responses are being sent or // that someone is attempting to the DNS port for non-DNS traffic. Both @@ -344,6 +348,10 @@ func (dns *Dns) ParseUdp(pkt *protos.Packet) { } } +func (dns *Dns) ConnectionTimeout() time.Duration { + return dns.transactionTimeout +} + func (dns *Dns) receivedDnsRequest(tuple *DnsTuple, msg *DnsMessage) { logp.Debug("dns", "Processing query. %s", tuple) @@ -379,6 +387,11 @@ func (dns *Dns) receivedDnsResponse(tuple *DnsTuple, msg *DnsMessage) { } func (dns *Dns) publishTransaction(t *DnsTransaction) { + var offset int + if t.Transport == TransportTcp { + offset = DecodeOffset + } + if dns.results == nil { return } @@ -402,8 +415,8 @@ func (dns *Dns) publishTransaction(t *DnsTransaction) { event["dns"] = dnsEvent if t.Request != nil && t.Response != nil { - event["bytes_in"] = t.Request.Length - event["bytes_out"] = t.Response.Length + event["bytes_in"] = t.Request.Length + offset + event["bytes_out"] = t.Response.Length + offset event["responsetime"] = int32(t.Response.Ts.Sub(t.ts).Nanoseconds() / 1e6) event["method"] = dnsOpCodeToString(t.Request.Data.OpCode) if len(t.Request.Data.Questions) > 0 { @@ -424,7 +437,7 @@ func (dns *Dns) publishTransaction(t *DnsTransaction) { event["response"] = dnsToString(t.Response.Data) } } else if t.Request != nil { - event["bytes_in"] = t.Request.Length + event["bytes_in"] = t.Request.Length + offset event["method"] = dnsOpCodeToString(t.Request.Data.OpCode) if len(t.Request.Data.Questions) > 0 { event["query"] = dnsQuestionToString(t.Request.Data.Questions[0]) @@ -437,7 +450,7 @@ func (dns *Dns) publishTransaction(t *DnsTransaction) { event["request"] = dnsToString(t.Request.Data) } } else if t.Response != nil { - event["bytes_out"] = t.Response.Length + event["bytes_out"] = t.Response.Length + offset event["method"] = dnsOpCodeToString(t.Response.Data.OpCode) if len(t.Response.Data.Questions) > 0 { event["query"] = dnsQuestionToString(t.Response.Data.Questions[0]) @@ -690,16 +703,17 @@ func nameToString(name []byte) string { return string(s) } -// decodeDnsPacket decodes a byte array into a DNS struct. If an error occurs +// decodeDnsData decodes a byte array into a DNS struct. If an error occurs // then the returnd dns pointer will be nil. This method recovers from panics // and is concurrency-safe. -func decodeDnsPacket(data []byte) (dns *layers.DNS, err error) { +func decodeDnsData(data []byte) (dns *layers.DNS, err error) { // Recover from any panics that occur while parsing a packet. defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: %v", r) } }() + d := &layers.DNS{} err = d.DecodeFromBytes(data, gopacket.NilDecodeFeedback) if err != nil { @@ -707,3 +721,170 @@ func decodeDnsPacket(data []byte) (dns *layers.DNS, err error) { } return d, nil } + +// TCP implementation + +func (dns *Dns) Parse(pkt *protos.Packet, tcpTuple *common.TcpTuple, dir uint8, private protos.ProtocolData) protos.ProtocolData { + defer logp.Recover("DNS ParseTcp") + + logp.Debug("dns", "Parsing packet addressed with %s of length %d.", + pkt.Tuple.String(), len(pkt.Payload)) + + priv := dnsPrivateData{} + + if private != nil { + var ok bool + priv, ok = private.(dnsPrivateData) + if !ok { + priv = dnsPrivateData{} + } + } + + var payload []byte + + // Offset is critical + if len(pkt.Payload) > DecodeOffset { + payload = pkt.Payload[DecodeOffset:] + } + + stream := priv.Data[dir] + if stream == nil { + stream = &DnsStream{ + tcpTuple: tcpTuple, + data: payload, + message: &DnsMessage{Ts: pkt.Ts, Tuple: pkt.Tuple}, + } + } else { + stream.data = append(stream.data, payload...) + if len(stream.data) > tcp.TCP_MAX_DATA_IN_STREAM { + logp.Debug("dns", "Stream data too large, dropping DNS stream") + stream = nil + return priv + } + } + + priv.Data[dir] = stream + data, err := decodeDnsData(stream.data) + + if err != nil { + logp.Debug("dns", NonDnsCompleteMsg+" addresses %s, length %d", + tcpTuple.String(), len(stream.data)) + + // wait for decoding with the next segment + return priv + } + dns.messageComplete(tcpTuple, dir, stream, data) + + return priv +} + +func (dns *Dns) messageComplete(tcpTuple *common.TcpTuple, dir uint8, s *DnsStream, decodedData *layers.DNS) { + dns.handleDns(s.message, tcpTuple, dir, s.data, decodedData) + + s.PrepareForNewMessage() +} + +func (dns *Dns) handleDns(m *DnsMessage, tcpTuple *common.TcpTuple, dir uint8, data []byte, decodedData *layers.DNS) { + dnsTuple := DnsTupleFromIpPort(&m.Tuple, TransportTcp, decodedData.ID) + m.CmdlineTuple = procs.ProcWatcher.FindProcessesTuple(tcpTuple.IpPort()) + m.Data = decodedData + m.Length = len(data) + + if decodedData.QR == Query { + dns.receivedDnsRequest(&dnsTuple, m) + } else /* Response */ { + dns.receivedDnsResponse(&dnsTuple, m) + } +} + +func (stream *DnsStream) PrepareForNewMessage() { + stream.message = nil +} + +func (dns *Dns) ReceivedFin(tcpTuple *common.TcpTuple, dir uint8, private protos.ProtocolData) protos.ProtocolData { + if private == nil { + return private + } + dnsData, ok := private.(dnsPrivateData) + if !ok { + return private + } + if dnsData.Data[dir] == nil { + return dnsData + } + stream := dnsData.Data[dir] + if stream.message != nil { + decodedData, err := decodeDnsData(stream.data) + if err == nil { + dns.messageComplete(tcpTuple, dir, stream, decodedData) + } else /*Failed decode */ { + if dir == tcp.TcpDirectionReverse { + dns.publishDecodeFailureNotes(dnsData) + stream.PrepareForNewMessage() + } + logp.Debug("dns", NonDnsCompleteMsg+" addresses %s, length %d", + tcpTuple.String(), len(stream.data)) + } + } + + return dnsData +} + +func (dns *Dns) GapInStream(tcpTuple *common.TcpTuple, dir uint8, nbytes int, private protos.ProtocolData) (priv protos.ProtocolData, drop bool) { + dnsData, ok := private.(dnsPrivateData) + + if !ok { + return private, false + } + + stream := dnsData.Data[dir] + + if stream == nil || stream.message == nil { + return private, false + } + + decodedData, err := decodeDnsData(stream.data) + + // Add Notes if the failed stream is the response + if err != nil { + if dir == tcp.TcpDirectionReverse { + dns.publishDecodeFailureNotes(dnsData) + } + + // drop the stream because it is binary and it would be rare to have a decodable message later + logp.Debug("dns", NonDnsCompleteMsg+" addresses %s, length %d", + tcpTuple.String(), len(stream.data)) + return private, true + } + + // publish and ignore the gap. No case should reach this code though ... + dns.messageComplete(tcpTuple, dir, stream, decodedData) + return private, false +} + +// Add Notes to the query stream about a failure to decode the response +func (dns *Dns) publishDecodeFailureNotes(dnsData dnsPrivateData) { + streamOrigin := dnsData.Data[tcp.TcpDirectionOriginal] + streamReverse := dnsData.Data[tcp.TcpDirectionReverse] + + dataOrigin, err := decodeDnsData(streamOrigin.data) + tupleReverse := streamReverse.message.Tuple + + if err == nil { + dnsTupleReverse := DnsTupleFromIpPort(&tupleReverse, TransportTcp, dataOrigin.ID) + hashDnsTupleOrigin := (&dnsTupleReverse).RevHashable() + + trans := dns.deleteTransaction(hashDnsTupleOrigin) + + if trans == nil { // happens when a Gap is followed by Fin + return + } + + trans.Notes = append(trans.Notes, NonDnsResponsePacketMsg) + + dns.publishTransaction(trans) + dns.deleteTransaction(hashDnsTupleOrigin) + } else { + logp.Debug("dns", "Unabled to decode response with adresses %s has no associated query", streamReverse.tcpTuple.String()) + } +} diff --git a/packetbeat/protos/dns/dns_test.go b/packetbeat/protos/dns/dns_test.go index 0a3eb4b349f..f3f1fc4d736 100644 --- a/packetbeat/protos/dns/dns_test.go +++ b/packetbeat/protos/dns/dns_test.go @@ -3,7 +3,7 @@ // The byte array test data was generated from pcap files using the gopacket // test_creator.py script contained in the gopacket repository. The script was // modified to drop the Ethernet, IP, and UDP headers from the byte arrays -// (skip the first 42 bytes). +// (skip the first 42 bytes for UDP packets and the first 54 bytes for TCP packets). // // TODO: // * Add test validation for responsetime to make sure unit conversion @@ -25,6 +25,7 @@ import ( "time" "github.com/elastic/beats/packetbeat/protos" + "github.com/elastic/beats/packetbeat/protos/tcp" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -68,6 +69,12 @@ var ( githubPtr, sophosTxt, } + messagesTcp = []DnsTestMessage{ + elasticATcp, + zoneAxfrTcp, + githubPtrTcp, + sophosTxtTcp, + } elasticA = DnsTestMessage{ id: 8529, @@ -90,6 +97,37 @@ var ( }, } + elasticATcp = DnsTestMessage{ + id: 11674, + opcode: "QUERY", + flags: []string{"rd", "ra"}, + rcode: "NOERROR", + q_class: "IN", + q_type: "A", + q_name: "elastic.co", + answers: []string{"54.201.204.244", "54.200.185.88"}, + authorities: []string{"NS-835.AWSDNS-40.NET", "NS-1183.AWSDNS-19.ORG", "NS-2007.AWSDNS-58.CO.UK", "NS-66.AWSDNS-08.COM"}, + request: []byte{ + 0x00, 0x1c, 0x2d, 0x9a, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x65, + 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x02, 0x63, 0x6f, 0x00, 0x00, 0x01, 0x00, 0x01, + }, + response: []byte{ + 0x00, 0xc7, 0x2d, 0x9a, 0x81, 0x80, 0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x00, 0x07, 0x65, + 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x02, 0x63, 0x6f, 0x00, 0x00, 0x01, 0x00, 0x01, 0xc0, 0x0c, + 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x04, 0x36, 0xc8, 0xb9, 0x58, 0xc0, 0x0c, + 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x04, 0x36, 0xc9, 0xcc, 0xf4, 0xc0, 0x0c, + 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x16, 0x82, 0x00, 0x16, 0x06, 0x4e, 0x53, 0x2d, 0x38, 0x33, + 0x35, 0x09, 0x41, 0x57, 0x53, 0x44, 0x4e, 0x53, 0x2d, 0x34, 0x30, 0x03, 0x4e, 0x45, 0x54, 0x00, + 0xc0, 0x0c, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x16, 0x82, 0x00, 0x17, 0x07, 0x4e, 0x53, 0x2d, + 0x31, 0x31, 0x38, 0x33, 0x09, 0x41, 0x57, 0x53, 0x44, 0x4e, 0x53, 0x2d, 0x31, 0x39, 0x03, 0x4f, + 0x52, 0x47, 0x00, 0xc0, 0x0c, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x16, 0x82, 0x00, 0x19, 0x07, + 0x4e, 0x53, 0x2d, 0x32, 0x30, 0x30, 0x37, 0x09, 0x41, 0x57, 0x53, 0x44, 0x4e, 0x53, 0x2d, 0x35, + 0x38, 0x02, 0x43, 0x4f, 0x02, 0x55, 0x4b, 0x00, 0xc0, 0x0c, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, + 0x16, 0x82, 0x00, 0x15, 0x05, 0x4e, 0x53, 0x2d, 0x36, 0x36, 0x09, 0x41, 0x57, 0x53, 0x44, 0x4e, + 0x53, 0x2d, 0x30, 0x38, 0x03, 0x43, 0x4f, 0x4d, 0x00, + }, + } + zoneIxfr = DnsTestMessage{ id: 16384, opcode: "QUERY", @@ -126,6 +164,35 @@ var ( }, } + zoneAxfrTcp = DnsTestMessage{ + id: 0, + opcode: "QUERY", + rcode: "NOERROR", + q_class: "IN", + q_type: "AXFR", + q_name: "etas.com", + answers: []string{"training2003p", "training2003p", "1.1.1.1", "training2003p"}, + request: []byte{ + 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x65, + 0x74, 0x61, 0x73, 0x03, 0x63, 0x6f, 0x6d, 0x00, 0x00, 0xfc, 0x00, 0x01, 0x4d, 0x53, + }, + response: []byte{ + 0x00, 0xc3, 0x00, 0x00, 0x80, 0x80, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x65, + 0x74, 0x61, 0x73, 0x03, 0x63, 0x6f, 0x6d, 0x00, 0x00, 0xfc, 0x00, 0x01, 0xc0, 0x0c, 0x00, 0x06, + 0x00, 0x01, 0x00, 0x00, 0x0e, 0x10, 0x00, 0x2f, 0x0d, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, + 0x67, 0x32, 0x30, 0x30, 0x33, 0x70, 0x00, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x73, 0x74, + 0x65, 0x72, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x02, 0x58, 0x00, + 0x01, 0x51, 0x80, 0x00, 0x00, 0x0e, 0x10, 0xc0, 0x0c, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x0e, + 0x10, 0x00, 0x0f, 0x0d, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x32, 0x30, 0x30, 0x33, + 0x70, 0x00, 0x07, 0x77, 0x65, 0x6c, 0x63, 0x6f, 0x6d, 0x65, 0xc0, 0x0c, 0x00, 0x01, 0x00, 0x01, + 0x00, 0x00, 0x0e, 0x10, 0x00, 0x04, 0x01, 0x01, 0x01, 0x01, 0xc0, 0x0c, 0x00, 0x06, 0x00, 0x01, + 0x00, 0x00, 0x0e, 0x10, 0x00, 0x2f, 0x0d, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x32, + 0x30, 0x30, 0x33, 0x70, 0x00, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x02, 0x58, 0x00, 0x01, 0x51, + 0x80, 0x00, 0x00, 0x0e, 0x10, + }, + } + githubPtr = DnsTestMessage{ id: 344, opcode: "QUERY", @@ -166,6 +233,35 @@ var ( }, } + githubPtrTcp = DnsTestMessage{ + id: 6766, + opcode: "QUERY", + flags: []string{"rd", "ra"}, + rcode: "NOERROR", + q_class: "IN", + q_type: "PTR", + q_name: "131.252.30.192.in-addr.arpa", + answers: []string{"github.com"}, + authorities: []string{"ns1.p16.dynect.net", "ns3.p16.dynect.net", "ns4.p16.dynect.net", "ns2.p16.dynect.net"}, + request: []byte{ + 0x00, 0x2d, 0x1a, 0x6e, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x31, + 0x33, 0x31, 0x03, 0x32, 0x35, 0x32, 0x02, 0x33, 0x30, 0x03, 0x31, 0x39, 0x32, 0x07, 0x69, 0x6e, + 0x2d, 0x61, 0x64, 0x64, 0x72, 0x04, 0x61, 0x72, 0x70, 0x61, 0x00, 0x00, 0x0c, 0x00, 0x01, + }, + response: []byte{ + 0x00, 0x9b, 0x1a, 0x6e, 0x81, 0x80, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x03, 0x31, + 0x33, 0x31, 0x03, 0x32, 0x35, 0x32, 0x02, 0x33, 0x30, 0x03, 0x31, 0x39, 0x32, 0x07, 0x69, 0x6e, + 0x2d, 0x61, 0x64, 0x64, 0x72, 0x04, 0x61, 0x72, 0x70, 0x61, 0x00, 0x00, 0x0c, 0x00, 0x01, 0xc0, + 0x0c, 0x00, 0x0c, 0x00, 0x01, 0x00, 0x00, 0x0e, 0x07, 0x00, 0x0c, 0x06, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x03, 0x63, 0x6f, 0x6d, 0x00, 0xc0, 0x10, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x51, + 0x77, 0x00, 0x14, 0x03, 0x6e, 0x73, 0x31, 0x03, 0x70, 0x31, 0x36, 0x06, 0x64, 0x79, 0x6e, 0x65, + 0x63, 0x74, 0x03, 0x6e, 0x65, 0x74, 0x00, 0xc0, 0x10, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x51, + 0x77, 0x00, 0x06, 0x03, 0x6e, 0x73, 0x33, 0xc0, 0x55, 0xc0, 0x10, 0x00, 0x02, 0x00, 0x01, 0x00, + 0x01, 0x51, 0x77, 0x00, 0x06, 0x03, 0x6e, 0x73, 0x34, 0xc0, 0x55, 0xc0, 0x10, 0x00, 0x02, 0x00, + 0x01, 0x00, 0x01, 0x51, 0x77, 0x00, 0x06, 0x03, 0x6e, 0x73, 0x32, 0xc0, 0x55, + }, + } + sophosTxt = DnsTestMessage{ id: 8238, opcode: "QUERY", @@ -211,6 +307,52 @@ var ( 0x6f, 0x73, 0x78, 0x6c, 0x03, 0x6e, 0x65, 0x74, 0x00, 0x00, 0x10, 0x00, 0x01, }, } + + sophosTxtTcp = DnsTestMessage{ + id: 35009, + opcode: "QUERY", + flags: []string{"rd", "ra"}, + rcode: "NXDOMAIN", + q_class: "IN", + q_type: "TXT", + q_name: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" + + "648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" + + "1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net", + request: []byte{ + 0x00, 0xed, 0x88, 0xc1, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, + 0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, + 0x71, 0x70, 0x33, 0x37, 0x35, 0x73, 0x70, 0x34, 0x39, 0x72, 0x38, 0x33, 0x30, 0x6e, 0x32, 0x6e, + 0x34, 0x6e, 0x39, 0x32, 0x33, 0x73, 0x38, 0x38, 0x33, 0x39, 0x30, 0x35, 0x32, 0x73, 0x37, 0x70, + 0x37, 0x37, 0x36, 0x38, 0x73, 0x35, 0x33, 0x33, 0x36, 0x35, 0x32, 0x32, 0x36, 0x70, 0x70, 0x33, + 0x3f, 0x36, 0x35, 0x39, 0x70, 0x31, 0x72, 0x37, 0x34, 0x31, 0x6f, 0x73, 0x33, 0x37, 0x33, 0x39, + 0x33, 0x36, 0x34, 0x38, 0x73, 0x32, 0x33, 0x34, 0x38, 0x6f, 0x37, 0x36, 0x32, 0x71, 0x31, 0x30, + 0x36, 0x36, 0x71, 0x35, 0x33, 0x72, 0x71, 0x35, 0x70, 0x34, 0x36, 0x31, 0x34, 0x72, 0x31, 0x71, + 0x34, 0x37, 0x38, 0x31, 0x71, 0x70, 0x72, 0x31, 0x36, 0x6e, 0x38, 0x30, 0x39, 0x71, 0x70, 0x34, + 0x1a, 0x38, 0x37, 0x39, 0x6f, 0x33, 0x6f, 0x37, 0x33, 0x34, 0x71, 0x39, 0x73, 0x6e, 0x73, 0x30, + 0x30, 0x35, 0x6f, 0x33, 0x70, 0x70, 0x37, 0x36, 0x71, 0x38, 0x33, 0x28, 0x32, 0x71, 0x36, 0x35, + 0x71, 0x6e, 0x73, 0x33, 0x73, 0x70, 0x6e, 0x73, 0x31, 0x30, 0x38, 0x31, 0x73, 0x35, 0x72, 0x6e, + 0x35, 0x73, 0x72, 0x37, 0x34, 0x6f, 0x70, 0x71, 0x72, 0x71, 0x6e, 0x70, 0x71, 0x36, 0x72, 0x6e, + 0x33, 0x72, 0x6f, 0x35, 0x01, 0x69, 0x02, 0x30, 0x30, 0x03, 0x6d, 0x61, 0x63, 0x08, 0x73, 0x6f, + 0x70, 0x68, 0x6f, 0x73, 0x78, 0x6c, 0x03, 0x6e, 0x65, 0x74, 0x00, 0x00, 0x10, 0x00, 0x01, + }, + response: []byte{ + 0x00, 0xed, 0x88, 0xc1, 0x81, 0x83, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, + 0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, + 0x71, 0x70, 0x33, 0x37, 0x35, 0x73, 0x70, 0x34, 0x39, 0x72, 0x38, 0x33, 0x30, 0x6e, 0x32, 0x6e, + 0x34, 0x6e, 0x39, 0x32, 0x33, 0x73, 0x38, 0x38, 0x33, 0x39, 0x30, 0x35, 0x32, 0x73, 0x37, 0x70, + 0x37, 0x37, 0x36, 0x38, 0x73, 0x35, 0x33, 0x33, 0x36, 0x35, 0x32, 0x32, 0x36, 0x70, 0x70, 0x33, + 0x3f, 0x36, 0x35, 0x39, 0x70, 0x31, 0x72, 0x37, 0x34, 0x31, 0x6f, 0x73, 0x33, 0x37, 0x33, 0x39, + 0x33, 0x36, 0x34, 0x38, 0x73, 0x32, 0x33, 0x34, 0x38, 0x6f, 0x37, 0x36, 0x32, 0x71, 0x31, 0x30, + 0x36, 0x36, 0x71, 0x35, 0x33, 0x72, 0x71, 0x35, 0x70, 0x34, 0x36, 0x31, 0x34, 0x72, 0x31, 0x71, + 0x34, 0x37, 0x38, 0x31, 0x71, 0x70, 0x72, 0x31, 0x36, 0x6e, 0x38, 0x30, 0x39, 0x71, 0x70, 0x34, + 0x1a, 0x38, 0x37, 0x39, 0x6f, 0x33, 0x6f, 0x37, 0x33, 0x34, 0x71, 0x39, 0x73, 0x6e, 0x73, 0x30, + 0x30, 0x35, 0x6f, 0x33, 0x70, 0x70, 0x37, 0x36, 0x71, 0x38, 0x33, 0x28, 0x32, 0x71, 0x36, 0x35, + 0x71, 0x6e, 0x73, 0x33, 0x73, 0x70, 0x6e, 0x73, 0x31, 0x30, 0x38, 0x31, 0x73, 0x35, 0x72, 0x6e, + 0x35, 0x73, 0x72, 0x37, 0x34, 0x6f, 0x70, 0x71, 0x72, 0x71, 0x6e, 0x70, 0x71, 0x36, 0x72, 0x6e, + 0x33, 0x72, 0x6f, 0x35, 0x01, 0x69, 0x02, 0x30, 0x30, 0x03, 0x6d, 0x61, 0x63, 0x08, 0x73, 0x6f, + 0x70, 0x68, 0x6f, 0x73, 0x78, 0x6c, 0x03, 0x6e, 0x65, 0x74, 0x00, 0x00, 0x10, 0x00, 0x01, + }, + } ) // Request and response addresses. @@ -225,9 +367,7 @@ var ( // Verify that the interfaces for UDP and TCP have been satisfied. var _ protos.UdpProtocolPlugin = &Dns{} - -// TODO: Uncomment when TCP is implemented. -//var _ protos.TcpProtocolPlugin = &Dns{} +var _ protos.TcpProtocolPlugin = &Dns{} func newDns(verbose bool) *Dns { if verbose { @@ -316,7 +456,7 @@ func TestParseUdp_responseOnly(t *testing.T) { assertMapStrData(t, m, q) } -// Verify that the first request if published without a response and that +// Verify that the first request is published without a response and that // the status is error. This second packet will remain in the transaction // map awaiting a response. func TestParseUdp_duplicateRequests(t *testing.T) { @@ -424,7 +564,7 @@ func BenchmarkUdpSophosTxt(b *testing.B) { benchmarkUdp(b, sophosTxt) } // Benchmark that runs with parallelism to help find concurrency related // issues. To run with parallelism, the 'go test' cpu flag must be set // greater than 1, otherwise it just runs concurrently but not in parallel. -func BenchmarkParallelParse(b *testing.B) { +func BenchmarkParallelUdpParse(b *testing.B) { rand.Seed(22) numMessages := len(messages) dns := newDns(false) @@ -649,3 +789,331 @@ func assertAddress(t testing.TB, expected common.IpPortTuple, endpoint interface assert.Equal(t, expected.Src_ip.String(), e.Ip) assert.Equal(t, expected.Src_port, e.Port) } + +// TCP tests + +func testTcpTuple() *common.TcpTuple { + t := &common.TcpTuple{ + Ip_length: 4, + Src_ip: net.IPv4(192, 168, 0, 1), Dst_ip: net.IPv4(192, 168, 0, 2), + Src_port: ClientPort, Dst_port: ServerPort, + } + t.ComputeHashebles() + return t +} + +// Verify that an empty packet is safely handled (no panics). +func TestParseTcp_emptyPacket(t *testing.T) { + dns := newDns(testing.Verbose()) + packet := newPacket(forward, []byte{}) + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Empty(t, dns.transactions.Size(), "There should be no transactions.") + client := dns.results.(publisher.ChanClient) + close(client.Channel) + assert.Nil(t, <-client.Channel, "No result should have been published.") +} + +// Verify that a malformed packet is safely handled (no panics). +func TestParseTcp_malformedPacket(t *testing.T) { + dns := newDns(testing.Verbose()) + garbage := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13} + tcptuple := testTcpTuple() + packet := newPacket(forward, garbage) + private := protos.ProtocolData(new(dnsPrivateData)) + + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Empty(t, dns.transactions.Size(), "There should be no transactions.") +} + +// Verify that the lone request packet is parsed. +func TestParseTcp_requestPacket(t *testing.T) { + dns := newDns(testing.Verbose()) + packet := newPacket(forward, elasticATcp.request) + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + client := dns.results.(publisher.ChanClient) + close(client.Channel) + assert.Nil(t, <-client.Channel, "No result should have been published.") +} + +// Verify that the lone response packet is parsed and that an error +// result is published. +func TestParseTcp_responseOnly(t *testing.T) { + dns := newDns(testing.Verbose()) + q := elasticATcp + packet := newPacket(reverse, q.response) + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + m := expectResult(t, dns) + assert.Equal(t, "tcp", mapValue(t, m, "transport")) + assert.Nil(t, mapValue(t, m, "bytes_in")) + assert.Equal(t, len(q.response), mapValue(t, m, "bytes_out")) + assert.Nil(t, mapValue(t, m, "responsetime")) + assert.Equal(t, common.ERROR_STATUS, mapValue(t, m, "status")) + assert.Equal(t, OrphanedResponseMsg, mapValue(t, m, "notes")) + assertMapStrData(t, m, q) +} + +// Verify that the first request is published without a response and that +// the status is error. This second packet will remain in the transaction +// map awaiting a response. +func TestParseTcp_duplicateRequests(t *testing.T) { + dns := newDns(testing.Verbose()) + q := elasticATcp + packet := newPacket(forward, q.request) + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + packet = newPacket(forward, q.request) + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + + m := expectResult(t, dns) + assert.Equal(t, "tcp", mapValue(t, m, "transport")) + assert.Equal(t, len(q.request), mapValue(t, m, "bytes_in")) + assert.Nil(t, mapValue(t, m, "bytes_out")) + assert.Nil(t, mapValue(t, m, "responsetime")) + assert.Equal(t, common.ERROR_STATUS, mapValue(t, m, "status")) + assert.Equal(t, DuplicateQueryMsg, mapValue(t, m, "notes")) +} + +// parseTcpRequestResponse parses a request then a response packet and validates +// the published result. +func parseTcpRequestResponse(t testing.TB, dns *Dns, q DnsTestMessage) { + packet := newPacket(forward, q.request) + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + + packet = newPacket(reverse, q.response) + dns.Parse(packet, tcptuple, tcp.TcpDirectionReverse, private) + + assert.Empty(t, dns.transactions.Size(), "There should be no transactions.") + + m := expectResult(t, dns) + assert.Equal(t, "tcp", mapValue(t, m, "transport")) + assert.Equal(t, len(q.request), mapValue(t, m, "bytes_in")) + assert.Equal(t, len(q.response), mapValue(t, m, "bytes_out")) + assert.NotNil(t, mapValue(t, m, "responsetime")) + + if assert.ObjectsAreEqual("NOERROR", mapValue(t, m, "dns.response_code")) { + assert.Equal(t, common.OK_STATUS, mapValue(t, m, "status")) + } else { + assert.Equal(t, common.ERROR_STATUS, mapValue(t, m, "status")) + } + + assert.Nil(t, mapValue(t, m, "notes")) + assertMapStrData(t, m, q) +} + +// Verify that the split lone request packet is parsed. +func TestParseTcpSplitRequest(t *testing.T) { + stream := &DnsStream{data: sophosTxtTcp.request[2:10], message: new(DnsMessage)} + _, err := decodeDnsData(stream.data) + + assert.NotNil(t, err, "Not expecting a complete message yet") + + stream.data = append(stream.data, sophosTxtTcp.request[10:]...) + _, err = decodeDnsData(stream.data) + + assert.Nil(t, err, "Message should be complete") +} + +func TestGapRequestDrop(t *testing.T) { + dns := newDns(testing.Verbose()) + q := sophosTxtTcp.request[:10] + + packet := newPacket(forward, q) + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + private = dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + + private, drop := dns.GapInStream(tcptuple, tcp.TcpDirectionOriginal, 10, private) + + assert.Equal(t, true, drop) + + private = dns.ReceivedFin(tcptuple, tcp.TcpDirectionOriginal, private) + + client := dns.results.(publisher.ChanClient) + close(client.Channel) + mapStr := <-client.Channel + assert.Nil(t, mapStr, "No result should have been published.") +} + +// Verify that a gap during the response publish the request with Notes +func TestGapResponse(t *testing.T) { + dns := newDns(testing.Verbose()) + q := sophosTxtTcp.request + r := sophosTxtTcp.response[:10] + + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + packet := newPacket(forward, q) + private = dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + + packet = newPacket(reverse, r) + private = dns.Parse(packet, tcptuple, tcp.TcpDirectionReverse, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + + private, drop := dns.GapInStream(tcptuple, tcp.TcpDirectionReverse, 10, private) + assert.Equal(t, true, drop) + + private = dns.ReceivedFin(tcptuple, tcp.TcpDirectionReverse, private) + + client := dns.results.(publisher.ChanClient) + close(client.Channel) + mapStr := <-client.Channel + assert.NotNil(t, mapStr, "One result should have been published.") + assert.Equal(t, mapStr["notes"], "Response packet's data could not be decoded as DNS") + assert.Nil(t, mapStr["answers"]) +} + +// Verify that a gap/fin happening after a valid query create only one tansaction +func TestGapFinValidMessage(t *testing.T) { + dns := newDns(testing.Verbose()) + q := sophosTxtTcp.request + + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + packet := newPacket(forward, q) + private = dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + + private, drop := dns.GapInStream(tcptuple, tcp.TcpDirectionOriginal, 10, private) + assert.Equal(t, false, drop) + + private = dns.ReceivedFin(tcptuple, tcp.TcpDirectionReverse, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + + client := dns.results.(publisher.ChanClient) + close(client.Channel) + mapStr := <-client.Channel + assert.Nil(t, mapStr, "No result should have been published.") + assert.Empty(t, mapStr["notes"], "There should be no notes") +} + +// Verify that a Fin during the response publish the request with Notes +func TestFinResponse(t *testing.T) { + dns := newDns(testing.Verbose()) + q := zoneAxfrTcp.request + r := zoneAxfrTcp.response[:10] + + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + + packet := newPacket(forward, q) + private = dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + + packet = newPacket(reverse, r) + private = dns.Parse(packet, tcptuple, tcp.TcpDirectionReverse, private) + assert.Equal(t, 1, dns.transactions.Size(), "There should be one transaction.") + + private = dns.ReceivedFin(tcptuple, tcp.TcpDirectionReverse, private) + + client := dns.results.(publisher.ChanClient) + close(client.Channel) + mapStr := <-client.Channel + assert.NotNil(t, mapStr, "One result should have been published.") + assert.Equal(t, mapStr["notes"], "Response packet's data could not be decoded as DNS") + assert.Nil(t, mapStr["answers"]) +} + +// Verify that the request/response pair are parsed and that a result +// is published. +func TestParseTcp_requestResponse(t *testing.T) { + parseTcpRequestResponse(t, newDns(testing.Verbose()), elasticATcp) +} + +// Verify all DNS TCP test messages are parsed correctly. +func TestParseTcp_allTestMessages(t *testing.T) { + dns := newDns(testing.Verbose()) + for _, q := range messagesTcp { + t.Logf("Testing with query for %s", q.q_name) + parseTcpRequestResponse(t, dns, q) + } +} + +// Benchmarks TCP parsing for the given test message. +func benchmarkTcp(b *testing.B, q DnsTestMessage) { + dns := newDns(false) + for i := 0; i < b.N; i++ { + packet := newPacket(forward, q.request) + tcptuple := testTcpTuple() + private := protos.ProtocolData(new(dnsPrivateData)) + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + + packet = newPacket(reverse, q.response) + tcptuple = testTcpTuple() + private = protos.ProtocolData(new(dnsPrivateData)) + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + + client := dns.results.(publisher.ChanClient) + <-client.Channel + } +} + +// Benchmark Tcp parsing against each test message. +func BenchmarkTcpElasticA(b *testing.B) { benchmarkTcp(b, elasticATcp) } +func BenchmarkTcpZoneIxfr(b *testing.B) { benchmarkTcp(b, zoneAxfrTcp) } +func BenchmarkTcpGithubPtr(b *testing.B) { benchmarkTcp(b, githubPtrTcp) } +func BenchmarkTcpSophosTxt(b *testing.B) { benchmarkTcp(b, sophosTxtTcp) } + +// Benchmark that runs with parallelism to help find concurrency related +// issues. To run with parallelism, the 'go test' cpu flag must be set +// greater than 1, otherwise it just runs concurrently but not in parallel. +func BenchmarkParallelTcpParse(b *testing.B) { + rand.Seed(22) + numMessages := len(messagesTcp) + dns := newDns(false) + client := dns.results.(publisher.ChanClient) + + // Drain the results channal while the test is running. + go func() { + totalMessages := 0 + for r := range client.Channel { + _ = r + totalMessages++ + } + fmt.Printf("Parsed %d messages.\n", totalMessages) + }() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + // Each iteration parses one message, either a request or a response. + // The request and response could be parsed on different goroutines. + for pb.Next() { + q := messagesTcp[rand.Intn(numMessages)] + var packet *protos.Packet + var tcptuple *common.TcpTuple + var private protos.ProtocolData + + if rand.Intn(2) == 0 { + packet = newPacket(forward, q.request) + tcptuple = testTcpTuple() + private = protos.ProtocolData(new(dnsPrivateData)) + } else { + packet = newPacket(reverse, q.response) + tcptuple = testTcpTuple() + private = protos.ProtocolData(new(dnsPrivateData)) + } + dns.Parse(packet, tcptuple, tcp.TcpDirectionOriginal, private) + } + }) + + defer close(client.Channel) +} diff --git a/packetbeat/tests/system/pcaps/dns_tcp_axfr.pcap b/packetbeat/tests/system/pcaps/dns_tcp_axfr.pcap new file mode 100644 index 0000000000000000000000000000000000000000..0291cbf166d4a79a659a1fc599495abee3c0113f GIT binary patch literal 915 zcmbu8Pe@cz7{$+hb2ZVXo>Qp{T{}Y1P`s&yR2$m~FHCch3_}emq`qdJ)^bqVyQ@t> zo5)>A%Y?+Gn<8CA6wOkdkv2o&1h@0uc@yZwY`XBxAMX8r-#O>Lx4pUc(t$=}83QQ( zc~L$yetAmcDt}9LVDQS93O}D-8|lCZkiy<|6x*wxl~N8%=>)uwzczY=6gY=J%AWv7 zx4q?#BedWs&2s8Yp~Y{3w4p#qTn;>)$W&jyBe6F~7*iS$TI^Yawwuuy<|L^mZAt7~ zZQq3?xJX+siR->wTX8*6i6>@El)cJoaJ;Q1)(w;`YvZ4+zS>ILG+X-xXM^LH&V|`R z?C$(5@EdB>&UkZAqsBd{Mv8ykSK{O1KDj1Zxf<@*>E84EOzHUtwr%oEku&UN9i8$1 zB-&;ADHj*wVLm%EH#2wtVk&hhK(HWbxT08Bml$i>cTK5_U83k`*phPFQWolvW0Y=F>f|dUl=F{U`Z}<_zKRYZ TMZVJXwd}j}b&g_yJ7(+;z{TAd literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/test_0032_dns.py b/packetbeat/tests/system/test_0032_dns.py index 7755f43db10..51366385774 100644 --- a/packetbeat/tests/system/test_0032_dns.py +++ b/packetbeat/tests/system/test_0032_dns.py @@ -179,3 +179,27 @@ def test_send_request_response(self): assert "response" in o assert "elastic.co" in o["request"] assert "include:_spf.google.com" in o["response"] + + def test_tcp_axfr(self): + """ + Should correctly interpret a TCP AXFR query + """ + self.render_config_template( + dns_ports=[53], + dns_send_request=True, + dns_send_response=True + ) + self.run_packetbeat(pcap="dns_tcp_axfr.pcap") + + objs = self.read_output() + assert len(objs) == 1 + o = objs[0] + + assert o["type"] == "dns" + assert o["transport"] == "tcp" + assert o["method"] == "QUERY" + assert o["query"] == "class IN, type AXFR, etas.com" + assert o["dns.question.type"] == "AXFR" + assert o["status"] == "OK" + assert len(o["dns.answers"]) == 4 + assert all("etas.com" in x["name"] for x in o["dns.answers"]) diff --git a/tests/system/pcaps/dns_tcp_axfr.pcap b/tests/system/pcaps/dns_tcp_axfr.pcap new file mode 100644 index 0000000000000000000000000000000000000000..0291cbf166d4a79a659a1fc599495abee3c0113f GIT binary patch literal 915 zcmbu8Pe@cz7{$+hb2ZVXo>Qp{T{}Y1P`s&yR2$m~FHCch3_}emq`qdJ)^bqVyQ@t> zo5)>A%Y?+Gn<8CA6wOkdkv2o&1h@0uc@yZwY`XBxAMX8r-#O>Lx4pUc(t$=}83QQ( zc~L$yetAmcDt}9LVDQS93O}D-8|lCZkiy<|6x*wxl~N8%=>)uwzczY=6gY=J%AWv7 zx4q?#BedWs&2s8Yp~Y{3w4p#qTn;>)$W&jyBe6F~7*iS$TI^Yawwuuy<|L^mZAt7~ zZQq3?xJX+siR->wTX8*6i6>@El)cJoaJ;Q1)(w;`YvZ4+zS>ILG+X-xXM^LH&V|`R z?C$(5@EdB>&UkZAqsBd{Mv8ykSK{O1KDj1Zxf<@*>E84EOzHUtwr%oEku&UN9i8$1 zB-&;ADHj*wVLm%EH#2wtVk&hhK(HWbxT08Bml$i>cTK5_U83k`*phPFQWolvW0Y=F>f|dUl=F{U`Z}<_zKRYZ TMZVJXwd}j}b&g_yJ7(+;z{TAd literal 0 HcmV?d00001