diff --git a/configure.ac b/configure.ac index 93e728fc2246..91731c7fad8c 100644 --- a/configure.ac +++ b/configure.ac @@ -2415,6 +2415,35 @@ if test "${enable_ebpf}" = "yes" || test "${enable_nfqueue}" = "yes" || test "${ AC_DEFINE([CAPTURE_OFFLOAD], [1],[Building flow capture bypass code]) fi +AC_ARG_ENABLE(ndpi, AS_HELP_STRING([--enable-ndpi], [Enable nDPI support]),[enable_ndpi=$enableval],[enable_ndpi=no]) + +NDPI_HOME= +AC_ARG_WITH([ndpi], [ --with-ndpi= path to nDPI source tree.], [NDPI_HOME="$withval"]) + +if ! test -z "${NDPI_HOME}" = "yes"; then + AC_MSG_CHECKING(for nDPI source) + + if test ! -z "$NDPI_HOME" ; then : + AC_MSG_RESULT(found in $NDPI_HOME) + NDPI_LIB=$NDPI_HOME/src/lib/libndpi.a + AC_MSG_CHECKING(for $NDPI_LIB) + if test -r $NDPI_LIB ; then : + AC_MSG_RESULT(found $NDPI_LIB) + else + AC_MSG_RESULT(not found $NDPI_LIB: compiling) + cd $NDPI_HOME; ./autogen.sh; ${MAKE}; cd - + fi + + CPPFLAGS="${CPPFLAGS} -I$NDPI_HOME/src/include" + LIBS="${LIBS} $NDPI_HOME/src/lib/libndpi.a" + AC_DEFINE_UNQUOTED(HAVE_NDPI, "1", [nDPI is present]) + enable_ndpi="yes ($NDPI_HOME)" + else + AC_MSG_RESULT(not found) + enable_ndpi="no" + fi +fi + # Add diagnostic filename CPPFLAGS="${CPPFLAGS} -D__SCFILENAME__=\\\"\$(*F)\\\"" @@ -2566,6 +2595,7 @@ SURICATA_BUILD_CONF="Suricata Configuration: liblz4 support: ${enable_liblz4} Landlock support: ${enable_landlock} Systemd support: ${enable_systemd} + nDPI support: ${enable_ndpi} Rust support: ${enable_rust} Rust strict mode: ${enable_rust_strict} diff --git a/doc/userguide/rules/index.rst b/doc/userguide/rules/index.rst index c8b586fecaa7..b96e21ba03b0 100644 --- a/doc/userguide/rules/index.rst +++ b/doc/userguide/rules/index.rst @@ -38,6 +38,8 @@ Suricata Rules smtp-keywords websocket-keywords app-layer + ndpi-protocol + ndpi-risk xbits noalert thresholding diff --git a/doc/userguide/rules/ndpi-protocol.rst b/doc/userguide/rules/ndpi-protocol.rst new file mode 100644 index 000000000000..d354db64ea55 --- /dev/null +++ b/doc/userguide/rules/ndpi-protocol.rst @@ -0,0 +1,35 @@ +nDPI Protocol Keyword +===================== + +ndpi-protocol +------------- + +Match on the Layer-7 protocol detected by nDPI. + +This requires Suricata to be compiled with nDPI support: + +.. code-block:: console + + ./configure --enable-ndpi --with-ndpi=/home/user/nDPI + +Syntax:: + + ndpi-protocol:[!]; + +Where protocol is one of the application protocols detected by nDPI. +Plase check ndpiReader -H for the full list. +It is possible to specify the transport protocol, the application +protocol, or both (dot-separated). + +Examples:: + + ndpi-protocol:HTTP; + ndpi-protocol:!TLS; + ndpi-protocol:TLS.YouTube; + +Here is an example of a rule matching TLS traffic on port 53: + +.. container:: example-rule + + alert tcp any any -> any 53 (msg:"TLS traffic over DNS standard port"; ndpi-protocol:TLS; sid:1;) + diff --git a/doc/userguide/rules/ndpi-risk.rst b/doc/userguide/rules/ndpi-risk.rst new file mode 100644 index 000000000000..12f78386d862 --- /dev/null +++ b/doc/userguide/rules/ndpi-risk.rst @@ -0,0 +1,41 @@ +nDPI Risk Keyword +================= + +ndpi-risk +--------- + +Match on the flow risks detected by nDPI. Risks are potential issues detected +by nDPI during the packet dissection and include: + +- Known Proto on Non Std Port +- Binary App Transfer +- Self-signed Certificate +- Susp DGA Domain name +- Malware host contacted +- and many other... + +This requires Suricata to be compiled with nDPI support: + +.. code-block:: console + + ./configure --enable-ndpi --with-ndpi=/home/user/nDPI + +Syntax:: + + ndpi-risk:[!]; + +Where risk is one (or multiple comma-separated) of the risk codes supported by +nDPI (e.g. NDPI_BINARY_APPLICATION_TRANSFER). Please check ndpiReader -H for the +full list. + +Examples:: + + ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; + ndpi-risk:NDPI_TLS_OBSOLETE_VERSION,NDPI_TLS_WEAK_CIPHER; + +Here is an example of a rule matching HTTP traffic transferring a binary application: + +.. container:: example-rule + + alert tcp any any -> any any (msg:"Binary application transfer over HTTP"; ndpi-protocol:HTTP; ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; sid:1;) + diff --git a/src/Makefile.am b/src/Makefile.am index 6970d709f35c..943a05f658c8 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -236,6 +236,8 @@ noinst_HEADERS = \ detect-mark.h \ detect-metadata.h \ detect-modbus.h \ + detect-ndpi-protocol.h \ + detect-ndpi-risk.h \ detect-quic-sni.h \ detect-quic-ua.h \ detect-quic-version.h \ @@ -811,6 +813,8 @@ libsuricata_c_a_SOURCES = \ detect-quic-cyu-hash.c \ detect-quic-cyu-string.c \ detect-msg.c \ + detect-ndpi-protocol.c \ + detect-ndpi-risk.c \ detect-nfs-procedure.c \ detect-nfs-version.c \ detect-noalert.c \ diff --git a/src/decode-ipv4.c b/src/decode-ipv4.c index 094d9ed72928..4b97ad1588ed 100644 --- a/src/decode-ipv4.c +++ b/src/decode-ipv4.c @@ -500,6 +500,10 @@ static const IPV4Hdr *DecodeIPV4Packet(Packet *p, const uint8_t *pkt, uint16_t l return NULL; } +#ifdef HAVE_NDPI + p->ip_len = len; +#endif + /* set the address struct */ SET_IPV4_SRC_ADDR(ip4h, &p->src); SET_IPV4_DST_ADDR(ip4h, &p->dst); diff --git a/src/decode-ipv6.c b/src/decode-ipv6.c index c732d7938d3b..93c2ea0b15a7 100644 --- a/src/decode-ipv6.c +++ b/src/decode-ipv6.c @@ -551,6 +551,10 @@ static const IPV6Hdr *DecodeIPV6Packet( return NULL; } +#ifdef HAVE_NDPI + p->ip_len = len; +#endif + SET_IPV6_SRC_ADDR(ip6h, &p->src); SET_IPV6_DST_ADDR(ip6h, &p->dst); diff --git a/src/decode.h b/src/decode.h index 510a7960e5d6..fe8fde6c427d 100644 --- a/src/decode.h +++ b/src/decode.h @@ -573,6 +573,10 @@ typedef struct Packet_ uint8_t *payload; uint16_t payload_len; +#ifdef HAVE_NDPI + uint16_t ip_len; +#endif + /* IPS action to take */ uint8_t action; diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 6ea3698c2c93..8d6a7c233365 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -210,6 +210,10 @@ #include "detect-quic-cyu-hash.h" #include "detect-quic-cyu-string.h" #include "detect-ja4-hash.h" +#ifdef HAVE_NDPI +#include "detect-ndpi-protocol.h" +#include "detect-ndpi-risk.h" +#endif #include "detect-bypass.h" #include "detect-ftpdata.h" @@ -695,6 +699,11 @@ void SigTableSetup(void) ScDetectSipRegister(); ScDetectTemplateRegister(); +#ifdef HAVE_NDPI + DetectnDPIProtocolRegister(); + DetectnDPIRiskRegister(); +#endif + /* close keyword registration */ DetectBufferTypeCloseRegistration(); } diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index c9134c77b83a..c8cf7da256f0 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -30,8 +30,12 @@ enum DetectKeywordId { DETECT_REV, DETECT_CLASSTYPE, - /* sorted by prefilter priority. Higher in this list means it will be - * picked over ones lower in the list */ +/* sorted by prefilter priority. Higher in this list means it will be + * picked over ones lower in the list */ +#ifdef HAVE_NDPI + DETECT_NDPI_PROTOCOL, + DETECT_NDPI_RISK, +#endif DETECT_AL_APP_LAYER_PROTOCOL, DETECT_ACK, DETECT_SEQ, diff --git a/src/detect-ndpi-protocol.c b/src/detect-ndpi-protocol.c new file mode 100644 index 000000000000..dcb5b82779ca --- /dev/null +++ b/src/detect-ndpi-protocol.c @@ -0,0 +1,334 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#include "suricata-common.h" +#include "detect-engine.h" +#include "detect-engine-build.h" +#include "detect-engine-prefilter.h" +#include "detect-engine-prefilter-common.h" +#include "detect-parse.h" +#include "detect-ndpi-protocol.h" +#include "util-debug.h" +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#ifdef HAVE_NDPI + +#ifdef UNITTESTS +static void DetectnDPIProtocolRegisterTests(void); +#endif + +typedef struct DetectnDPIProtocolData_ { + ndpi_master_app_protocol l7_protocol; + uint8_t negated; +} DetectnDPIProtocolData; + +static int DetectnDPIProtocolPacketMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) +{ + SCEnter(); + + bool r; + const DetectnDPIProtocolData *data = (const DetectnDPIProtocolData *)ctx; + + /* if the sig is PD-only we only match when PD packet flags are set */ + /* + if (s->type == SIG_TYPE_PDONLY && + (p->flags & (PKT_PROTO_DETECT_TS_DONE | PKT_PROTO_DETECT_TC_DONE)) == 0) { + SCLogDebug("packet %"PRIu64": flags not set", p->pcap_cnt); + SCReturnInt(0); + } + */ + + if (!p->flow->detection_completed) { + SCLogDebug("packet %" PRIu64 ": ndpi protocol not yet detected", p->pcap_cnt); + SCReturnInt(0); + } + + const Flow *f = p->flow; + if (f == NULL) { + SCLogDebug("packet %" PRIu64 ": no flow", p->pcap_cnt); + SCReturnInt(0); + } + + r = ndpi_is_proto_equals(f->detected_l7_protocol.proto, data->l7_protocol, false); + r = r ^ data->negated; + + if (r) { + SCLogDebug("ndpi protocol match on protocol = %u.%u (match %u)", + f->detected_l7_protocol.app_protocol, f->detected_l7_protocol.master_protocol, + data->l7_protocol); + SCReturnInt(1); + } + SCReturnInt(0); +} + +static DetectnDPIProtocolData *DetectnDPIProtocolParse(const char *arg, bool negate) +{ + DetectnDPIProtocolData *data; + struct ndpi_detection_module_struct *ndpi_struct; + ndpi_master_app_protocol l7_protocol; + char *l7_protocol_name = (char *)arg; + NDPI_PROTOCOL_BITMASK all; + + /* convert protocol name (string) to ID */ + ndpi_struct = ndpi_init_detection_module(NULL); + if (unlikely(ndpi_struct == NULL)) + return NULL; + + ndpi_struct = ndpi_init_detection_module(NULL); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); + ndpi_finalize_initialization(ndpi_struct); + + l7_protocol = ndpi_get_protocol_by_name(ndpi_struct, l7_protocol_name); + ndpi_exit_detection_module(ndpi_struct); + + if (ndpi_is_proto_unknown(l7_protocol)) { + SCLogError("failure parsing nDPI protocol '%s'", l7_protocol_name); + return NULL; + } + + data = SCMalloc(sizeof(DetectnDPIProtocolData)); + if (unlikely(data == NULL)) + return NULL; + + memcpy(&data->l7_protocol, &l7_protocol, sizeof(ndpi_master_app_protocol)); + data->negated = negate; + + return data; +} + +static bool HasConflicts(const DetectnDPIProtocolData *us, const DetectnDPIProtocolData *them) +{ + /* check for mix of negated and non negated */ + if (them->negated ^ us->negated) + return true; + + /* check for multiple non-negated */ + if (!us->negated) + return true; + + /* check for duplicate */ + if (ndpi_is_proto_equals(us->l7_protocol, them->l7_protocol, true)) + return true; + + return false; +} + +static int DetectnDPIProtocolSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg) +{ + DetectnDPIProtocolData *data = NULL; + + data = DetectnDPIProtocolParse(arg, s->init_data->negated); + if (data == NULL) + goto error; + + SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for (; tsm != NULL; tsm = tsm->next) { + if (tsm->type == DETECT_NDPI_PROTOCOL) { + const DetectnDPIProtocolData *them = (const DetectnDPIProtocolData *)tsm->ctx; + + if (HasConflicts(data, them)) { + SCLogError("can't mix " + "positive ndpi-protocol match with negated"); + goto error; + } + } + } + + if (SigMatchAppendSMToList(de_ctx, s, DETECT_NDPI_PROTOCOL, (SigMatchCtx *)data, + DETECT_SM_LIST_MATCH) == NULL) { + goto error; + } + return 0; + +error: + if (data != NULL) + SCFree(data); + return -1; +} + +static void DetectnDPIProtocolFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +/** \internal + * \brief prefilter function for protocol detect matching + */ +static void PrefilterPacketnDPIProtocolMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + const PrefilterPacketHeaderCtx *ctx = pectx; + + if (p->flow == NULL || !p->flow->detection_completed) { + SCLogDebug("packet %" PRIu64 ": no flow, no ndpi detection", p->pcap_cnt); + SCReturn; + } + + Flow *f = p->flow; + bool negated = (bool)ctx->v1.u8[4]; + + if (!ndpi_is_proto_unknown(f->detected_l7_protocol.proto)) { + ndpi_master_app_protocol p = { ctx->v1.u16[0], ctx->v1.u16[1] }; + + if (ndpi_is_proto_equals(f->detected_l7_protocol.proto, p, false) ^ negated) { + PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); + } + } +} + +static void PrefilterPacketnDPIProtocolSet(PrefilterPacketHeaderValue *v, void *smctx) +{ + const DetectnDPIProtocolData *a = smctx; + + v->u16[0] = a->l7_protocol.master_protocol; + v->u16[1] = a->l7_protocol.app_protocol; + v->u8[4] = (uint8_t)a->negated; +} + +static bool PrefilterPacketnDPIProtocolCompare(PrefilterPacketHeaderValue v, void *smctx) +{ + const DetectnDPIProtocolData *a = smctx; + ndpi_master_app_protocol p = { v.u16[0], v.u16[1] }; + bool negated = (bool)v.u8[4]; + + return (ndpi_is_proto_equals(a->l7_protocol, p, false) ^ negated); +} + +static int PrefilterSetupnDPIProtocol(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_PROTOCOL, SIG_MASK_REQUIRE_FLOW, + PrefilterPacketnDPIProtocolSet, PrefilterPacketnDPIProtocolCompare, + PrefilterPacketnDPIProtocolMatch); +} + +static bool PrefilternDPIProtocolIsPrefilterable(const Signature *s) +{ + if (s->type == SIG_TYPE_PDONLY) { + SCLogDebug("prefilter on PD %u", s->id); + return true; + } + return false; +} + +void DetectnDPIProtocolRegister(void) +{ + sigmatch_table[DETECT_NDPI_PROTOCOL].name = "ndpi-protocol"; + sigmatch_table[DETECT_NDPI_PROTOCOL].desc = "match on the detected nDPI protocol"; + sigmatch_table[DETECT_NDPI_PROTOCOL].url = "/rules/ndpi-protocol.html"; + sigmatch_table[DETECT_NDPI_PROTOCOL].Match = DetectnDPIProtocolPacketMatch; + sigmatch_table[DETECT_NDPI_PROTOCOL].Setup = DetectnDPIProtocolSetup; + sigmatch_table[DETECT_NDPI_PROTOCOL].Free = DetectnDPIProtocolFree; +#ifdef UNITTESTS + sigmatch_table[DETECT_NDPI_PROTOCOL].RegisterTests = DetectnDPIProtocolRegisterTests; +#endif + sigmatch_table[DETECT_NDPI_PROTOCOL].flags = + (SIGMATCH_QUOTES_OPTIONAL | SIGMATCH_HANDLE_NEGATION); + + sigmatch_table[DETECT_NDPI_PROTOCOL].SetupPrefilter = PrefilterSetupnDPIProtocol; + sigmatch_table[DETECT_NDPI_PROTOCOL].SupportsPrefilter = PrefilternDPIProtocolIsPrefilterable; +} + +/**********************************Unittests***********************************/ + +#ifdef UNITTESTS + +static int DetectnDPIProtocolTest01(void) +{ + DetectnDPIProtocolData *data = DetectnDPIProtocolParse("HTTP", false); + FAIL_IF_NULL(data); + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated != 0); + DetectnDPIProtocolFree(NULL, data); + PASS; +} + +static int DetectnDPIProtocolTest02(void) +{ + DetectnDPIProtocolData *data = DetectnDPIProtocolParse("HTTP", true); + FAIL_IF_NULL(data); + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated == 0); + DetectnDPIProtocolFree(NULL, data); + PASS; +} + +static int DetectnDPIProtocolTest03(void) +{ + Signature *s = NULL; + DetectnDPIProtocolData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-protocol:HTTP; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIProtocolData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated); + DetectEngineCtxFree(de_ctx); + PASS; +} + +static int DetectnDPIProtocolTest04(void) +{ + Signature *s = NULL; + DetectnDPIProtocolData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-protocol:!HTTP; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIProtocolData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF_NULL(data); + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated == 0); + + DetectEngineCtxFree(de_ctx); + PASS; +} + +static void DetectnDPIProtocolRegisterTests(void) +{ + UtRegisterTest("DetectnDPIProtocolTest01", DetectnDPIProtocolTest01); + UtRegisterTest("DetectnDPIProtocolTest02", DetectnDPIProtocolTest02); + UtRegisterTest("DetectnDPIProtocolTest03", DetectnDPIProtocolTest03); + UtRegisterTest("DetectnDPIProtocolTest04", DetectnDPIProtocolTest04); +} +#endif /* UNITTESTS */ + +#endif /* HAVE_NDPI */ diff --git a/src/detect-ndpi-protocol.h b/src/detect-ndpi-protocol.h new file mode 100644 index 000000000000..2f61ca5c57eb --- /dev/null +++ b/src/detect-ndpi-protocol.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#ifndef SURICATA_NDPI_PROTOCOL_H +#define SURICATA_NDPI_PROTOCOL_H + +void DetectnDPIProtocolRegister(void); + +#endif /* SURICATA_NDPI_PROTOCOL_H */ diff --git a/src/detect-ndpi-risk.c b/src/detect-ndpi-risk.c new file mode 100644 index 000000000000..5f9a809f762d --- /dev/null +++ b/src/detect-ndpi-risk.c @@ -0,0 +1,334 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#include "suricata-common.h" +#include "detect-engine.h" +#include "detect-engine-build.h" +#include "detect-engine-prefilter.h" +#include "detect-engine-prefilter-common.h" +#include "detect-parse.h" +#include "detect-ndpi-risk.h" +#include "util-debug.h" +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#ifdef HAVE_NDPI + +#ifdef UNITTESTS +static void DetectnDPIRiskRegisterTests(void); +#endif + +typedef struct DetectnDPIRiskData_ { + ndpi_risk risk_mask; /* uint64 */ + uint8_t negated; +} DetectnDPIRiskData; + +static int DetectnDPIRiskPacketMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) +{ + SCEnter(); + + bool r; + const DetectnDPIRiskData *data = (const DetectnDPIRiskData *)ctx; + + if (!p->flow->detection_completed) { + SCLogDebug("packet %" PRIu64 ": ndpi risks not yet detected", p->pcap_cnt); + SCReturnInt(0); + } + + const Flow *f = p->flow; + if (f == NULL) { + SCLogDebug("packet %" PRIu64 ": no flow", p->pcap_cnt); + SCReturnInt(0); + } + + r = ((f->ndpi_flow->risk & data->risk_mask) == data->risk_mask); + r = r ^ data->negated; + + if (r) { + SCLogDebug("ndpi risks match on risk bitmap = %" PRIu64 " (matching bitmap %" PRIu64 ")", + f->ndpi_flow->risk, data->risk_mask); + SCReturnInt(1); + } + + SCReturnInt(0); +} + +static DetectnDPIRiskData *DetectnDPIRiskParse(const char *arg, bool negate) +{ + DetectnDPIRiskData *data; + struct ndpi_detection_module_struct *ndpi_struct; + ndpi_risk risk_mask; + NDPI_PROTOCOL_BITMASK all; + + /* convert list of risk names (string) to mask */ + ndpi_struct = ndpi_init_detection_module(NULL); + if (unlikely(ndpi_struct == NULL)) + return NULL; + + ndpi_struct = ndpi_init_detection_module(NULL); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); + ndpi_finalize_initialization(ndpi_struct); + + if (isdigit(arg[0])) + risk_mask = atoll(arg); + else { + char *dup = SCStrdup(arg), *tmp, *token; + + NDPI_ZERO_BIT(risk_mask); + + if (dup != NULL) { + token = strtok_r(dup, ",", &tmp); + + while (token != NULL) { + ndpi_risk_enum risk_id = ndpi_code2risk(token); + if (risk_id >= NDPI_MAX_RISK) { + SCLogError("unrecognized risk '%s', " + "please check ndpiReader -H for valid risk codes", + token); + return NULL; + } + NDPI_SET_BIT(risk_mask, risk_id); + token = strtok_r(NULL, ",", &tmp); + } + + SCFree(dup); + } + } + + data = SCMalloc(sizeof(DetectnDPIRiskData)); + if (unlikely(data == NULL)) + return NULL; + + data->risk_mask = risk_mask; + data->negated = negate; + + return data; +} + +static bool HasConflicts(const DetectnDPIRiskData *us, const DetectnDPIRiskData *them) +{ + /* check for duplicate */ + if (us->risk_mask == them->risk_mask) + return true; + + return false; +} + +static int DetectnDPIRiskSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg) +{ + DetectnDPIRiskData *data = NULL; + + data = DetectnDPIRiskParse(arg, s->init_data->negated); + if (data == NULL) + goto error; + + SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for (; tsm != NULL; tsm = tsm->next) { + if (tsm->type == DETECT_NDPI_RISK) { + const DetectnDPIRiskData *them = (const DetectnDPIRiskData *)tsm->ctx; + + if (HasConflicts(data, them)) { + SCLogError("can't mix " + "positive ndpi-risk match with negated"); + goto error; + } + } + } + + if (SigMatchAppendSMToList( + de_ctx, s, DETECT_NDPI_RISK, (SigMatchCtx *)data, DETECT_SM_LIST_MATCH) == NULL) { + goto error; + } + return 0; + +error: + if (data != NULL) + SCFree(data); + return -1; +} + +static void DetectnDPIRiskFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +/** \internal + * \brief prefilter function for risk matching + */ +static void PrefilterPacketnDPIRiskMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + const PrefilterPacketHeaderCtx *ctx = pectx; + + if (p->flow == NULL || !p->flow->detection_completed) { + SCLogDebug("packet %" PRIu64 ": no flow, no ndpi detection", p->pcap_cnt); + SCReturn; + } + + Flow *f = p->flow; + bool negated = (bool)ctx->v1.u8[9]; + ndpi_risk risk_mask = ctx->v1.u64[0]; + bool ret = ((f->ndpi_flow->risk & risk_mask) == risk_mask) ? true : false; + + if (ret ^ negated) { + PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); + } +} + +static void PrefilterPacketnDPIRiskSet(PrefilterPacketHeaderValue *v, void *smctx) +{ + const DetectnDPIRiskData *a = smctx; + + v->u64[0] = a->risk_mask; + v->u8[9] = (uint8_t)a->negated; +} + +static bool PrefilterPacketnDPIRiskCompare(PrefilterPacketHeaderValue v, void *smctx) +{ + const DetectnDPIRiskData *a = smctx; + ndpi_risk p = v.u64[0]; + bool negated = (bool)v.u8[9]; + bool ret; + + ret = ((a->risk_mask & p) == p) ? true : false; + return (ret ^ negated); +} + +static int PrefilterSetupnDPIRisk(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_RISK, SIG_MASK_REQUIRE_FLOW, + PrefilterPacketnDPIRiskSet, PrefilterPacketnDPIRiskCompare, + PrefilterPacketnDPIRiskMatch); +} + +static bool PrefilternDPIRiskIsPrefilterable(const Signature *s) +{ + if (s->type == SIG_TYPE_PDONLY) { + SCLogDebug("prefilter on PD %u", s->id); + return true; + } + return false; +} + +void DetectnDPIRiskRegister(void) +{ + sigmatch_table[DETECT_NDPI_RISK].name = "ndpi-risk"; + sigmatch_table[DETECT_NDPI_RISK].desc = "match on the detected nDPI risk"; + sigmatch_table[DETECT_NDPI_RISK].url = "/rules/ndpi-risk.html"; + sigmatch_table[DETECT_NDPI_RISK].Match = DetectnDPIRiskPacketMatch; + sigmatch_table[DETECT_NDPI_RISK].Setup = DetectnDPIRiskSetup; + sigmatch_table[DETECT_NDPI_RISK].Free = DetectnDPIRiskFree; +#ifdef UNITTESTS + sigmatch_table[DETECT_NDPI_RISK].RegisterTests = DetectnDPIRiskRegisterTests; +#endif + sigmatch_table[DETECT_NDPI_RISK].flags = (SIGMATCH_QUOTES_OPTIONAL | SIGMATCH_HANDLE_NEGATION); + + sigmatch_table[DETECT_NDPI_RISK].SetupPrefilter = PrefilterSetupnDPIRisk; + sigmatch_table[DETECT_NDPI_RISK].SupportsPrefilter = PrefilternDPIRiskIsPrefilterable; +} + +/**********************************Unittests***********************************/ + +#ifdef UNITTESTS + +static int DetectnDPIRiskTest01(void) +{ + DetectnDPIRiskData *data = DetectnDPIRiskParse("NDPI_PROBING_ATTEMPT", false); + FAIL_IF_NULL(data); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); + FAIL_IF(data->negated != 0); + DetectnDPIRiskFree(NULL, data); + PASS; +} + +static int DetectnDPIRiskTest02(void) +{ + DetectnDPIRiskData *data = DetectnDPIRiskParse("NDPI_PROBING_ATTEMPT", true); + FAIL_IF_NULL(data); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); + FAIL_IF(data->negated == 0); + DetectnDPIRiskFree(NULL, data); + PASS; +} + +static int DetectnDPIRiskTest03(void) +{ + Signature *s = NULL; + DetectnDPIRiskData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-risk:NDPI_PROBING_ATTEMPT; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIRiskData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); + FAIL_IF(data->negated); + DetectEngineCtxFree(de_ctx); + PASS; +} + +static int DetectnDPIRiskTest04(void) +{ + Signature *s = NULL; + DetectnDPIRiskData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-risk:!NDPI_PROBING_ATTEMPT; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIRiskData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF_NULL(data); + + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); + FAIL_IF(data->negated == 0); + + DetectEngineCtxFree(de_ctx); + PASS; +} + +static void DetectnDPIRiskRegisterTests(void) +{ + UtRegisterTest("DetectnDPIRiskTest01", DetectnDPIRiskTest01); + UtRegisterTest("DetectnDPIRiskTest02", DetectnDPIRiskTest02); + UtRegisterTest("DetectnDPIRiskTest03", DetectnDPIRiskTest03); + UtRegisterTest("DetectnDPIRiskTest04", DetectnDPIRiskTest04); +} + +#endif /* UNITTESTS */ + +#endif /* HAVE_NDPI */ diff --git a/src/detect-ndpi-risk.h b/src/detect-ndpi-risk.h new file mode 100644 index 000000000000..cb1729670d84 --- /dev/null +++ b/src/detect-ndpi-risk.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#ifndef SURICATA_NDPI_RISK_H +#define SURICATA_NDPI_RISK_H + +void DetectnDPIRiskRegister(void); + +#endif /* SURICATA_NDPI_RISK_H */ diff --git a/src/flow-util.c b/src/flow-util.c index 7e11da41f527..f788642a8f3f 100644 --- a/src/flow-util.c +++ b/src/flow-util.c @@ -70,6 +70,16 @@ Flow *FlowAlloc(void) /* coverity[missing_lock] */ FLOW_INITIALIZE(f); + +#ifdef HAVE_NDPI + f->ndpi_flow = (struct ndpi_flow_struct *)ndpi_flow_malloc(SIZEOF_FLOW_STRUCT); + if (f->ndpi_flow != NULL) { + memset(f->ndpi_flow, 0, SIZEOF_FLOW_STRUCT); + f->detection_completed = 0; + /* printf("%s - Allocated flow\n", __FUNCTION__); */ + } +#endif + return f; } @@ -81,6 +91,13 @@ Flow *FlowAlloc(void) */ void FlowFree(Flow *f) { +#ifdef HAVE_NDPI + if (f->ndpi_flow) { + ndpi_flow_free(f->ndpi_flow); + /* printf("%s - Freed flow\n", __FUNCTION__); */ + } +#endif + FLOW_DESTROY(f); SCFree(f); diff --git a/src/flow.c b/src/flow.c index 7bfa80ea0a9b..5ca552a16bc8 100644 --- a/src/flow.c +++ b/src/flow.c @@ -387,6 +387,41 @@ void FlowHandlePacketUpdate(Flow *f, Packet *p, ThreadVars *tv, DecodeThreadVars { SCLogDebug("packet %"PRIu64" -- flow %p", p->pcap_cnt, f); +#ifdef HAVE_NDPI + if (tv->ndpi_struct && f->ndpi_flow && (!f->detection_completed) && (p->ip_len > 0)) { + uint64_t time_ms = ((uint64_t)p->ts.secs) * 1000 /* TICK_RESOLUTION */ + + p->ts.usecs / (1000000 / 1000 /* TICK_RESOLUTION */); + + f->detected_l7_protocol = ndpi_detection_process_packet(tv->ndpi_struct, f->ndpi_flow, + PacketIsIPv4(p) ? (void *)PacketGetIPv4(p) : (void *)PacketGetIPv6(p), p->ip_len, + time_ms, NULL); + + if (ndpi_is_protocol_detected(f->detected_l7_protocol) != 0) { + if (!ndpi_is_proto_unknown(f->detected_l7_protocol.proto)) { + if (!ndpi_extra_dissection_possible(tv->ndpi_struct, f->ndpi_flow)) + f->detection_completed = 1; + } + } else { + u_int16_t max_num_pkts = (f->proto == IPPROTO_UDP) ? 8 : 24; + + if ((f->todstpktcnt + f->tosrcpktcnt) > max_num_pkts) { + u_int8_t proto_guessed; + + f->detected_l7_protocol = + ndpi_detection_giveup(tv->ndpi_struct, f->ndpi_flow, &proto_guessed); + f->detection_completed = 1; + } + } + + if (f->detection_completed) { + SCLogDebug("Detected protocol: %s | app protocol: %s | category: %s", + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.master_protocol), + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.app_protocol), + ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); + } + } +#endif + const int pkt_dir = FlowGetPacketDirection(f, p); #ifdef CAPTURE_OFFLOAD int state = f->flow_state; diff --git a/src/flow.h b/src/flow.h index 554f9fca4a32..2a3d485cf50c 100644 --- a/src/flow.h +++ b/src/flow.h @@ -499,6 +499,12 @@ typedef struct Flow_ uint64_t todstbytecnt; uint64_t tosrcbytecnt; +#ifdef HAVE_NDPI + struct ndpi_flow_struct *ndpi_flow; + ndpi_protocol detected_l7_protocol; + uint8_t detection_completed; +#endif + Storage storage[]; } Flow; diff --git a/src/output-filestore.c b/src/output-filestore.c index f9a660f56e72..fdbc0dc06bcd 100644 --- a/src/output-filestore.c +++ b/src/output-filestore.c @@ -167,7 +167,12 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv, const OutputFilestoreLo WARN_ONCE(WOT_SNPRINTF, "Failed to write file info record. Output filename truncated."); } else { JsonBuilder *js_fileinfo = - JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL); + JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL +#ifdef HAVE_NDPI + , + tv +#endif + ); if (likely(js_fileinfo != NULL)) { jb_close(js_fileinfo); FILE *out = fopen(js_metadata_filename, "w"); diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 7822cc798045..5191531600fd 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -684,6 +684,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) } } +#ifdef HAVE_NDPI + ndpiJsonBuilder(p->flow, jb, tv); +#endif EveAddAppProto(p->flow, jb); if (p->flowflags & FLOW_PKT_TOSERVER) { diff --git a/src/output-json-file.c b/src/output-json-file.c index 509ae488bbee..9d1e4c1cd314 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -79,9 +79,370 @@ typedef struct JsonFileLogThread_ { OutputJsonThreadCtx *ctx; } JsonFileLogThread; -JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, - OutputJsonCtx *eve_ctx) +#ifdef HAVE_NDPI + +static void ndpiJsonBuilderTLSQUIC(Flow *f, JsonBuilder *js, ThreadVars *tv) +{ + char buf[64]; + char notBefore[32], notAfter[32]; + struct tm a, b, *before = NULL, *after = NULL; + u_int i, off; + u_int8_t unknown_tls_version; + char version[16], unknown_cipher[8]; + + if (!f->ndpi_flow->protos.tls_quic.ssl_version) { + return; + } + + ndpi_ssl_version2str(version, sizeof(version), f->ndpi_flow->protos.tls_quic.ssl_version, + &unknown_tls_version); + + if (f->ndpi_flow->protos.tls_quic.notBefore) + before = ndpi_gmtime_r((const time_t *)&f->ndpi_flow->protos.tls_quic.notBefore, &a); + + if (f->ndpi_flow->protos.tls_quic.notAfter) + after = ndpi_gmtime_r((const time_t *)&f->ndpi_flow->protos.tls_quic.notAfter, &b); + + if (!unknown_tls_version) { + jb_open_object(js, "tls"); + jb_set_string(js, "version", version); + + if (f->ndpi_flow->protos.tls_quic.server_names) + jb_set_string(js, "server_names", f->ndpi_flow->protos.tls_quic.server_names); + + if (before) { + strftime(notBefore, sizeof(notBefore), "%Y-%m-%d %H:%M:%S", before); + jb_set_string(js, "notbefore", notBefore); + } + + if (after) { + strftime(notAfter, sizeof(notAfter), "%Y-%m-%d %H:%M:%S", after); + jb_set_string(js, "notafter", notAfter); + } + + /* Note: ja3, ja3s, ja4 are not serialized as as Suricata already supports them */ + + jb_set_uint(js, "unsafe_cipher", f->ndpi_flow->protos.tls_quic.server_unsafe_cipher); + jb_set_string(js, "cipher", + ndpi_cipher2str(f->ndpi_flow->protos.tls_quic.server_cipher, unknown_cipher)); + + if (f->ndpi_flow->protos.tls_quic.issuerDN) + jb_set_string(js, "issuerDN", f->ndpi_flow->protos.tls_quic.issuerDN); + + if (f->ndpi_flow->protos.tls_quic.subjectDN) + jb_set_string(js, "subjectDN", f->ndpi_flow->protos.tls_quic.subjectDN); + + if (f->ndpi_flow->protos.tls_quic.advertised_alpns) + jb_set_string(js, "advertised_alpns", f->ndpi_flow->protos.tls_quic.advertised_alpns); + + if (f->ndpi_flow->protos.tls_quic.negotiated_alpn) + jb_set_string(js, "negotiated_alpn", f->ndpi_flow->protos.tls_quic.negotiated_alpn); + + if (f->ndpi_flow->protos.tls_quic.tls_supported_versions) + jb_set_string(js, "tls_supported_versions", + f->ndpi_flow->protos.tls_quic.tls_supported_versions); + + if (f->ndpi_flow->protos.tls_quic.sha1_certificate_fingerprint[0] != '\0') { + for (i = 0, off = 0; i < 20; i++) { + int rc = ndpi_snprintf(&buf[off], sizeof(buf) - off, "%s%02X", (i > 0) ? ":" : "", + f->ndpi_flow->protos.tls_quic.sha1_certificate_fingerprint[i] & 0xFF); + if (rc <= 0) + break; + else + off += rc; + } + jb_set_string(js, "fingerprint", buf); + } + + jb_set_uint(js, "blocks", f->ndpi_flow->l4.tcp.tls.num_tls_blocks); + + jb_close(js); + } +} + +/* Suricata backport of ndpi_dpi2json */ +void ndpiJsonBuilder(Flow *f, JsonBuilder *js, ThreadVars *tv) +{ + char buf[64]; + char const *host_server_name; + char quic_version[16]; + ndpi_protocol l7_protocol; + + if (f == NULL) + return; + + jb_open_object(js, "ndpi"); + jb_set_string(js, "app_protocol", + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.app_protocol)); + jb_set_string(js, "master_protocol", + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.master_protocol)); + jb_set_string(js, "category", + ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); + + if (f->ndpi_flow->risk) { + u_int risk_id; + + jb_open_array(js, "risks"); + + for (risk_id = 0; risk_id < NDPI_MAX_RISK; risk_id++) + if (NDPI_ISSET_BIT(f->ndpi_flow->risk, risk_id)) { + char str[256]; + snprintf(str, sizeof(str), "%s (%s)", ndpi_risk2code(risk_id), + ndpi_risk2str(risk_id)); + jb_append_string(js, str); + } + + jb_close(js); + } /* risk */ + + host_server_name = ndpi_get_flow_info(f->ndpi_flow, &l7_protocol); + if (host_server_name != NULL) { + jb_set_string(js, "hostname", host_server_name); + } + + switch (l7_protocol.proto.master_protocol ? l7_protocol.proto.master_protocol + : l7_protocol.proto.app_protocol) { + case NDPI_PROTOCOL_IP_ICMP: + if (f->ndpi_flow->entropy > 0.0f) { + char buf[64]; + snprintf(buf, sizeof(buf), "%.6f", f->ndpi_flow->entropy); + jb_set_string(js, "entropy", buf); + } + break; + case NDPI_PROTOCOL_DHCP: + jb_open_object(js, "dhcp"); + jb_set_string(js, "fingerprint", f->ndpi_flow->protos.dhcp.fingerprint); + jb_set_string(js, "class_ident", f->ndpi_flow->protos.dhcp.class_ident); + jb_close(js); + break; + case NDPI_PROTOCOL_BITTORRENT: { + u_int i, j, n = 0; + char bittorent_hash[sizeof(f->ndpi_flow->protos.bittorrent.hash) * 2 + 1]; + for (i = 0, j = 0; j < sizeof(bittorent_hash) - 1; i++) { + snprintf(&bittorent_hash[j], sizeof(bittorent_hash) - j, "%02x", + f->ndpi_flow->protos.bittorrent.hash[i]); + + j += 2; + n += f->ndpi_flow->protos.bittorrent.hash[i]; + } + if (n == 0) + bittorent_hash[0] = '\0'; + jb_open_object(js, "bittorrent"); + jb_set_string(js, "hash", bittorent_hash); + jb_close(js); + } break; + case NDPI_PROTOCOL_COLLECTD: + jb_open_object(js, "collectd"); + jb_set_string(js, "client_username", f->ndpi_flow->protos.collectd.client_username); + jb_close(js); + break; + case NDPI_PROTOCOL_DNS: + jb_open_object(js, "dns"); + jb_set_uint(js, "num_queries", f->ndpi_flow->protos.dns.num_queries); + jb_set_uint(js, "num_answers", f->ndpi_flow->protos.dns.num_answers); + jb_set_uint(js, "reply_code", f->ndpi_flow->protos.dns.reply_code); + jb_set_uint(js, "query_type", f->ndpi_flow->protos.dns.query_type); + jb_set_uint(js, "rsp_type", f->ndpi_flow->protos.dns.rsp_type); + inet_ntop(AF_INET, &f->ndpi_flow->protos.dns.rsp_addr, buf, sizeof(buf)); + jb_set_string(js, "rsp_addr", buf); + jb_close(js); + break; + case NDPI_PROTOCOL_NTP: + jb_open_object(js, "ntp"); + jb_set_uint(js, "request_code", f->ndpi_flow->protos.ntp.request_code); + jb_set_uint(js, "version", f->ndpi_flow->protos.ntp.request_code); + jb_close(js); + break; + case NDPI_PROTOCOL_MDNS: + jb_open_object(js, "mdns"); + jb_close(js); + break; + case NDPI_PROTOCOL_UBNTAC2: + jb_open_object(js, "ubntac2"); + jb_set_string(js, "version", f->ndpi_flow->protos.ubntac2.version); + jb_close(js); + break; + case NDPI_PROTOCOL_KERBEROS: + jb_open_object(js, "kerberos"); + jb_set_string(js, "hostname", f->ndpi_flow->protos.kerberos.hostname); + jb_set_string(js, "domain", f->ndpi_flow->protos.kerberos.domain); + jb_set_string(js, "username", f->ndpi_flow->protos.kerberos.username); + jb_close(js); + break; + case NDPI_PROTOCOL_SOFTETHER: + jb_open_object(js, "softether"); + jb_set_string(js, "client_ip", f->ndpi_flow->protos.softether.ip); + jb_set_string(js, "client_port", f->ndpi_flow->protos.softether.port); + jb_set_string(js, "hostname", f->ndpi_flow->protos.softether.hostname); + jb_set_string(js, "fqdn", f->ndpi_flow->protos.softether.fqdn); + jb_close(js); + break; + case NDPI_PROTOCOL_NATPMP: + jb_open_object(js, "natpmp"); + jb_set_uint(js, "result", f->ndpi_flow->protos.natpmp.result_code); + jb_set_uint(js, "internal_port", f->ndpi_flow->protos.natpmp.internal_port); + jb_set_uint(js, "external_port", f->ndpi_flow->protos.natpmp.external_port); + inet_ntop( + AF_INET, &f->ndpi_flow->protos.natpmp.external_address.ipv4, buf, sizeof(buf)); + jb_set_string(js, "external_address", buf); + jb_close(js); + break; + case NDPI_PROTOCOL_RSH: + jb_open_object(js, "rsh"); + jb_set_string(js, "client_username", f->ndpi_flow->protos.rsh.client_username); + jb_set_string(js, "server_username", f->ndpi_flow->protos.rsh.server_username); + jb_set_string(js, "command", f->ndpi_flow->protos.rsh.command); + jb_close(js); + break; + case NDPI_PROTOCOL_SNMP: + jb_open_object(js, "snmp"); + jb_set_uint(js, "version", f->ndpi_flow->protos.snmp.version); + jb_set_uint(js, "primitive", f->ndpi_flow->protos.snmp.primitive); + jb_set_uint(js, "error_status", f->ndpi_flow->protos.snmp.error_status); + jb_close(js); + break; + case NDPI_PROTOCOL_TELNET: + jb_open_object(js, "telnet"); + jb_set_string(js, "username", f->ndpi_flow->protos.telnet.username); + jb_set_string(js, "password", f->ndpi_flow->protos.telnet.password); + jb_close(js); + break; + case NDPI_PROTOCOL_TFTP: + jb_open_object(js, "tftp"); + jb_set_string(js, "filename", f->ndpi_flow->protos.tftp.filename); + jb_close(js); + break; + case NDPI_PROTOCOL_TIVOCONNECT: + jb_open_object(js, "tivoconnect"); + jb_set_string(js, "identity_uuid", f->ndpi_flow->protos.tivoconnect.identity_uuid); + jb_set_string(js, "machine", f->ndpi_flow->protos.tivoconnect.machine); + jb_set_string(js, "platform", f->ndpi_flow->protos.tivoconnect.platform); + jb_set_string(js, "services", f->ndpi_flow->protos.tivoconnect.services); + jb_close(js); + break; + case NDPI_PROTOCOL_HTTP: + case NDPI_PROTOCOL_HTTP_CONNECT: + case NDPI_PROTOCOL_HTTP_PROXY: + jb_open_object(js, "http"); + if (f->ndpi_flow->http.url != NULL) { + jb_set_string(js, "url", f->ndpi_flow->http.url); + jb_set_uint(js, "code", f->ndpi_flow->http.response_status_code); + jb_set_string(js, "content_type", f->ndpi_flow->http.content_type); + jb_set_string(js, "user_agent", f->ndpi_flow->http.user_agent); + } + if (f->ndpi_flow->http.request_content_type != NULL) { + jb_set_string(js, "request_content_type", f->ndpi_flow->http.request_content_type); + } + if (f->ndpi_flow->http.detected_os != NULL) { + jb_set_string(js, "detected_os", f->ndpi_flow->http.detected_os); + } + if (f->ndpi_flow->http.nat_ip != NULL) { + jb_set_string(js, "nat_ip", f->ndpi_flow->http.nat_ip); + } + jb_close(js); + break; + case NDPI_PROTOCOL_QUIC: + jb_open_object(js, "quic"); + if (f->ndpi_flow->http.user_agent) { + jb_set_string(js, "user_agent", f->ndpi_flow->http.user_agent); + } + ndpi_quic_version2str( + quic_version, sizeof(quic_version), f->ndpi_flow->protos.tls_quic.quic_version); + jb_set_string(js, "quic_version", quic_version); + ndpiJsonBuilderTLSQUIC(f, js, tv); + jb_close(js); + break; + case NDPI_PROTOCOL_MAIL_IMAP: + jb_open_object(js, "imap"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_MAIL_POP: + jb_open_object(js, "pop"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_MAIL_SMTP: + jb_open_object(js, "smtp"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_FTP_CONTROL: + jb_open_object(js, "ftp"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_DISCORD: + if (l7_protocol.proto.master_protocol != NDPI_PROTOCOL_TLS) { + jb_open_object(js, "discord"); + jb_set_string(js, "client_ip", f->ndpi_flow->protos.discord.client_ip); + jb_close(js); + } + break; + case NDPI_PROTOCOL_SSH: + jb_open_object(js, "ssh"); + jb_set_string(js, "client_signature", f->ndpi_flow->protos.ssh.client_signature); + jb_set_string(js, "server_signature", f->ndpi_flow->protos.ssh.server_signature); + jb_set_string(js, "hassh_client", f->ndpi_flow->protos.ssh.hassh_client); + jb_set_string(js, "hassh_server", f->ndpi_flow->protos.ssh.hassh_server); + jb_close(js); + break; + case NDPI_PROTOCOL_STUN: + jb_open_object(js, "stun"); + if (f->ndpi_flow->stun.mapped_address.port) { + jb_set_string(js, "mapped_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.mapped_address, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.peer_address.port) { + jb_set_string(js, "peer_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.peer_address, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.relayed_address.port) { + jb_set_string(js, "relayed_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.relayed_address, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.response_origin.port) { + jb_set_string(js, "response_origin", + print_ndpi_address_port( + &f->ndpi_flow->stun.response_origin, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.other_address.port) { + jb_set_string(js, "other_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.other_address, buf, sizeof(buf))); + } + jb_close(js); + break; + case NDPI_PROTOCOL_TLS: + case NDPI_PROTOCOL_DTLS: + ndpiJsonBuilderTLSQUIC(f, js, tv); + break; + } /* switch */ + + jb_close(js); // "ndpi" +} + +#endif + +JsonBuilder *JsonBuildFileInfoRecord( + const Packet *p, const File *ff, void *tx, const uint64_t tx_id, const bool stored, + uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx +#ifdef HAVE_NDPI + , + ThreadVars *tv +#endif +) { enum OutputJsonLogDirection fdir = LOG_DIR_FLOW; @@ -191,6 +552,10 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, jb_set_string(js, "app_proto", AppProtoToString(p->flow->alproto)); +#ifdef HAVE_NDPI + ndpiJsonBuilder(p->flow, js, tv); +#endif + jb_open_object(js, "fileinfo"); if (stored) { // the file has just been stored on disk cf OUTPUT_FILEDATA_FLAG_CLOSE @@ -214,11 +579,22 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, * \brief Write meta data on a single line json record */ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) + const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx +#ifdef HAVE_NDPI + , + ThreadVars *tv +#endif +) { HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg; - JsonBuilder *js = JsonBuildFileInfoRecord(p, ff, tx, tx_id, false, dir, xff_cfg, eve_ctx); + JsonBuilder *js = JsonBuildFileInfoRecord(p, ff, tx, tx_id, false, dir, xff_cfg, eve_ctx +#ifdef HAVE_NDPI + , + tv +#endif + ); + if (unlikely(js == NULL)) { return; } @@ -237,7 +613,13 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, co SCLogDebug("ff %p", ff); - FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); + FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx +#ifdef HAVE_NDPI + , + tv +#endif + ); + return 0; } diff --git a/src/output-json-file.h b/src/output-json-file.h index e2eeeccb2ffc..0603666124a9 100644 --- a/src/output-json-file.h +++ b/src/output-json-file.h @@ -29,8 +29,13 @@ typedef struct OutputJsonCtx_ OutputJsonCtx; void JsonFileLogRegister(void); -JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, - OutputJsonCtx *eve_ctx); +JsonBuilder *JsonBuildFileInfoRecord( + const Packet *p, const File *ff, void *tx, const uint64_t tx_id, const bool stored, + uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx +#ifdef HAVE_NDPI + , + ThreadVars *tv +#endif +); #endif /* SURICATA_OUTPUT_JSON_FILE_H */ diff --git a/src/output-json-flow.c b/src/output-json-flow.c index f7826734f0cb..4de3f1ef944c 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -215,8 +215,16 @@ void EveAddFlow(Flow *f, JsonBuilder *js) } /* Eve format logging */ -static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f) +static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f +#ifdef HAVE_NDPI + , + ThreadVars *tv +#endif +) { +#ifdef HAVE_NDPI + ndpiJsonBuilder(f, jb, tv); +#endif EveAddAppProto(f, jb); jb_open_object(jb, "flow"); EveAddFlow(f, jb); @@ -338,7 +346,12 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) SCReturnInt(TM_ECODE_OK); } - EveFlowLogJSON(thread, jb, f); + EveFlowLogJSON(thread, jb, f +#ifdef HAVE_NDPI + , + tv +#endif + ); OutputJsonBuilderBuffer(jb, thread); jb_free(jb); diff --git a/src/output-json-flow.h b/src/output-json-flow.h index e334315e56fc..a2777e14abeb 100644 --- a/src/output-json-flow.h +++ b/src/output-json-flow.h @@ -27,5 +27,8 @@ void JsonFlowLogRegister(void); void EveAddFlow(Flow *f, JsonBuilder *js); void EveAddAppProto(Flow *f, JsonBuilder *js); +#ifdef HAVE_NDPI +void EveAddnDPIProto(Flow *f, JsonBuilder *js, ThreadVars *tv); +#endif #endif /* SURICATA_OUTPUT_JSON_FLOW_H */ diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index 2e359bb909c5..d6ded0da9483 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -175,8 +175,17 @@ static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir) } /* JSON format logging */ -static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f) +static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f +#ifdef HAVE_NDPI + , + ThreadVars *tv +#endif +) { +#ifdef HAVE_NDPI + ndpiJsonBuilder(f, js, tv); +#endif + jb_set_string(js, "app_proto", AppProtoToString(f->alproto_ts ? f->alproto_ts : f->alproto)); @@ -219,8 +228,17 @@ static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f) } } -static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f) +static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f +#ifdef HAVE_NDPI + , + ThreadVars *tv +#endif +) { +#ifdef HAVE_NDPI + ndpiJsonBuilder(f, js, tv); +#endif + jb_set_string(js, "app_proto", AppProtoToString(f->alproto_tc ? f->alproto_tc : f->alproto)); @@ -274,7 +292,12 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) JsonBuilder *jb = CreateEveHeaderFromNetFlow(f, 0); if (unlikely(jb == NULL)) return TM_ECODE_OK; - NetFlowLogEveToServer(jb, f); + NetFlowLogEveToServer(jb, f +#ifdef HAVE_NDPI + , + tv +#endif + ); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER); OutputJsonBuilderBuffer(jb, jhl); jb_free(jb); @@ -284,7 +307,12 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) jb = CreateEveHeaderFromNetFlow(f, 1); if (unlikely(jb == NULL)) return TM_ECODE_OK; - NetFlowLogEveToClient(jb, f); + NetFlowLogEveToClient(jb, f +#ifdef HAVE_NDPI + , + tv +#endif + ); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT); OutputJsonBuilderBuffer(jb, jhl); jb_free(jb); diff --git a/src/output-json.h b/src/output-json.h index 761064f7e10a..27c35a2b252f 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -120,4 +120,7 @@ OutputJsonThreadCtx *CreateEveThreadCtx(ThreadVars *t, OutputJsonCtx *ctx); void FreeEveThreadCtx(OutputJsonThreadCtx *ctx); void JSONFormatAndAddMACAddr(JsonBuilder *js, const char *key, const uint8_t *val, bool is_array); +#ifdef HAVE_NDPI +void ndpiJsonBuilder(Flow *f, JsonBuilder *js, ThreadVars *tv); +#endif #endif /* SURICATA_OUTPUT_JSON_H */ diff --git a/src/suricata-common.h b/src/suricata-common.h index 51af99a023e8..a0822b79a333 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -517,6 +517,10 @@ typedef struct lua_State lua_State; #include "queue.h" #include "tree.h" +#ifdef HAVE_NDPI +#include "ndpi_api.h" +#endif + #ifndef HAVE_STRLCAT size_t strlcat(char *, const char *src, size_t siz); #endif diff --git a/src/threadvars.h b/src/threadvars.h index cebcdb4e3ac1..b2cdca2cad4c 100644 --- a/src/threadvars.h +++ b/src/threadvars.h @@ -135,6 +135,10 @@ typedef struct ThreadVars_ { struct FlowQueue_ *flow_queue; bool break_loop; +#ifdef HAVE_NDPI + struct ndpi_detection_module_struct *ndpi_struct; +#endif + } ThreadVars; /** Thread setup flags: */ diff --git a/src/tm-threads.c b/src/tm-threads.c index b0d0f8686ba0..7310ade016c6 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -1011,6 +1011,19 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i if (mucond != 0) TmThreadInitMC(tv); +#ifdef HAVE_NDPI + tv->ndpi_struct = ndpi_init_detection_module(NULL); + if (tv->ndpi_struct != NULL) { + NDPI_PROTOCOL_BITMASK protos; + + NDPI_BITMASK_SET_ALL(protos); + ndpi_set_protocol_detection_bitmask2(tv->ndpi_struct, &protos); + ndpi_finalize_initialization(tv->ndpi_struct); + + /* printf("%s - ndpi_init_detection_module()\n", __FUNCTION__); */ + } +#endif + return tv; error: @@ -1607,6 +1620,13 @@ static void TmThreadFree(ThreadVars *tv) SCFree(ps); } +#ifdef HAVE_NDPI + if (tv->ndpi_struct != NULL) { + ndpi_exit_detection_module(tv->ndpi_struct); + /* printf("%s - ndpi_exit_detection_module()\n", __FUNCTION__); */ + } +#endif + TmThreadsUnregisterThread(tv->id); SCFree(tv); }