From c8d659f6bcfafecfa33aca341003ec269b8ae1fb Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 21 Aug 2024 07:39:01 +0200 Subject: [PATCH 1/4] Add initial nDPI integration Add configure --enable-ndpi and --with-ndpi params Add nDPI metadata export in alerts/netflow (Eve JSON) Add ndpi-protocol signature match Add ndpi-risk signature match --- configure.ac | 30 +++ src/Makefile.am | 4 + src/decode-ipv4.c | 4 + src/decode-ipv6.c | 5 + src/decode.h | 4 + src/detect-engine-register.c | 9 + src/detect-engine-register.h | 4 + src/detect-ndpi-protocol.c | 348 ++++++++++++++++++++++++++++++++ src/detect-ndpi-protocol.h | 30 +++ src/detect-ndpi-risk.c | 343 ++++++++++++++++++++++++++++++++ src/detect-ndpi-risk.h | 30 +++ src/flow-util.c | 17 ++ src/flow.c | 36 ++++ src/flow.h | 6 + src/output-filestore.c | 6 +- src/output-json-alert.c | 3 + src/output-json-file.c | 372 ++++++++++++++++++++++++++++++++++- src/output-json-file.h | 6 +- src/output-json-flow.c | 15 +- src/output-json-flow.h | 3 + src/output-json-netflow.c | 32 ++- src/output-json.h | 3 + src/suricata-common.h | 4 + src/threadvars.h | 4 + src/tm-threads.c | 20 ++ 25 files changed, 1326 insertions(+), 12 deletions(-) create mode 100644 src/detect-ndpi-protocol.c create mode 100644 src/detect-ndpi-protocol.h create mode 100644 src/detect-ndpi-risk.c create mode 100644 src/detect-ndpi-risk.h diff --git a/configure.ac b/configure.ac index d19680e091d3..ea52aaa324c4 100644 --- a/configure.ac +++ b/configure.ac @@ -2395,6 +2395,35 @@ if test "${enable_ebpf}" = "yes" || test "${enable_nfqueue}" = "yes" || test "${ AC_DEFINE([CAPTURE_OFFLOAD], [1],[Building flow capture bypass code]) fi +AC_ARG_ENABLE(ndpi, AS_HELP_STRING([--enable-ndpi], [Enable nDPI support]),[enable_ndpi=$enableval],[enable_ndpi=no]) + +NDPI_HOME= +AC_ARG_WITH([ndpi], [ --with-ndpi= path to nDPI source tree.], [NDPI_HOME="$withval"]) + +if ! test -z "${NDPI_HOME}" = "yes"; then + AC_MSG_CHECKING(for nDPI source) + + if test ! -z "$NDPI_HOME" ; then : + AC_MSG_RESULT(found in $NDPI_HOME) + NDPI_LIB=$NDPI_HOME/src/lib/libndpi.a + AC_MSG_CHECKING(for $NDPI_LIB) + if test -r $NDPI_LIB ; then : + AC_MSG_RESULT(found $NDPI_LIB) + else + AC_MSG_RESULT(not found $NDPI_LIB: compiling) + cd $NDPI_HOME; ./autogen.sh; ${MAKE}; cd - + fi + + CPPFLAGS="${CPPFLAGS} -I$NDPI_HOME/src/include" + LIBS="${LIBS} $NDPI_HOME/src/lib/libndpi.a" + AC_DEFINE_UNQUOTED(HAVE_NDPI, "1", [nDPI is present]) + enable_ndpi="yes ($NDPI_HOME)" + else + AC_MSG_RESULT(not found) + enable_ndpi="no" + fi +fi + # Add diagnostic filename CPPFLAGS="${CPPFLAGS} -D__SCFILENAME__=\\\"\$(*F)\\\"" @@ -2544,6 +2573,7 @@ SURICATA_BUILD_CONF="Suricata Configuration: liblz4 support: ${enable_liblz4} Landlock support: ${enable_landlock} Systemd support: ${enable_systemd} + nDPI support: ${enable_ndpi} Rust support: ${enable_rust} Rust strict mode: ${enable_rust_strict} diff --git a/src/Makefile.am b/src/Makefile.am index 148821ea2b69..cfd66ac7d039 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -236,6 +236,8 @@ noinst_HEADERS = \ detect-mark.h \ detect-metadata.h \ detect-modbus.h \ + detect-ndpi-protocol.h \ + detect-ndpi-risk.h \ detect-quic-sni.h \ detect-quic-ua.h \ detect-quic-version.h \ @@ -821,6 +823,8 @@ libsuricata_c_a_SOURCES = \ detect-quic-cyu-hash.c \ detect-quic-cyu-string.c \ detect-msg.c \ + detect-ndpi-protocol.c \ + detect-ndpi-risk.c \ detect-nfs-procedure.c \ detect-nfs-version.c \ detect-noalert.c \ diff --git a/src/decode-ipv4.c b/src/decode-ipv4.c index 094d9ed72928..98980f1b15d1 100644 --- a/src/decode-ipv4.c +++ b/src/decode-ipv4.c @@ -500,6 +500,10 @@ static const IPV4Hdr *DecodeIPV4Packet(Packet *p, const uint8_t *pkt, uint16_t l return NULL; } +#ifdef HAVE_NDPI + p->ip_len = len; +#endif + /* set the address struct */ SET_IPV4_SRC_ADDR(ip4h, &p->src); SET_IPV4_DST_ADDR(ip4h, &p->dst); diff --git a/src/decode-ipv6.c b/src/decode-ipv6.c index c732d7938d3b..483f74cb50ba 100644 --- a/src/decode-ipv6.c +++ b/src/decode-ipv6.c @@ -550,6 +550,11 @@ static const IPV6Hdr *DecodeIPV6Packet( ENGINE_SET_INVALID_EVENT(p, IPV6_TRUNC_PKT); return NULL; } + +#ifdef HAVE_NDPI + p->ip_len = len; +#endif + SET_IPV6_SRC_ADDR(ip6h, &p->src); SET_IPV6_DST_ADDR(ip6h, &p->dst); diff --git a/src/decode.h b/src/decode.h index b7398bc2a0d6..11ee1849b06a 100644 --- a/src/decode.h +++ b/src/decode.h @@ -580,6 +580,10 @@ typedef struct Packet_ uint8_t *payload; uint16_t payload_len; +#ifdef HAVE_NDPI + uint16_t ip_len; +#endif + /* IPS action to take */ uint8_t action; diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index bbf8b916ae93..0e63c5561b03 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -216,6 +216,10 @@ #include "detect-quic-cyu-hash.h" #include "detect-quic-cyu-string.h" #include "detect-ja4-hash.h" +#ifdef HAVE_NDPI +#include "detect-ndpi-protocol.h" +#include "detect-ndpi-risk.h" +#endif #include "detect-bypass.h" #include "detect-ftpdata.h" @@ -705,6 +709,11 @@ void SigTableSetup(void) ScDetectMqttRegister(); ScDetectRfbRegister(); +#ifdef HAVE_NDPI + DetectnDPIProtocolRegister(); + DetectnDPIRiskRegister(); +#endif + /* close keyword registration */ DetectBufferTypeCloseRegistration(); } diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 7c3b5b4514b0..ee9631997092 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -32,6 +32,10 @@ enum DetectKeywordId { /* sorted by prefilter priority. Higher in this list means it will be * picked over ones lower in the list */ +#ifdef HAVE_NDPI + DETECT_NDPI_PROTOCOL, + DETECT_NDPI_RISK, +#endif DETECT_AL_APP_LAYER_PROTOCOL, DETECT_ACK, DETECT_SEQ, diff --git a/src/detect-ndpi-protocol.c b/src/detect-ndpi-protocol.c new file mode 100644 index 000000000000..f340ec9fb917 --- /dev/null +++ b/src/detect-ndpi-protocol.c @@ -0,0 +1,348 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#include "suricata-common.h" +#include "detect-engine.h" +#include "detect-engine-build.h" +#include "detect-engine-prefilter.h" +#include "detect-engine-prefilter-common.h" +#include "detect-parse.h" +#include "detect-ndpi-protocol.h" +#include "util-debug.h" +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#ifdef HAVE_NDPI + +#ifdef UNITTESTS +static void DetectnDPIProtocolRegisterTests(void); +#endif + +typedef struct DetectnDPIProtocolData_ { + ndpi_master_app_protocol l7_protocol; + uint8_t negated; +} DetectnDPIProtocolData; + +static int DetectnDPIProtocolPacketMatch( + DetectEngineThreadCtx *det_ctx, + Packet *p, const Signature *s, const SigMatchCtx *ctx) +{ + SCEnter(); + + bool r; + const DetectnDPIProtocolData *data = (const DetectnDPIProtocolData *)ctx; + + /* if the sig is PD-only we only match when PD packet flags are set */ + /* + if (s->type == SIG_TYPE_PDONLY && + (p->flags & (PKT_PROTO_DETECT_TS_DONE | PKT_PROTO_DETECT_TC_DONE)) == 0) { + SCLogDebug("packet %"PRIu64": flags not set", p->pcap_cnt); + SCReturnInt(0); + } + */ + + if (!p->flow->detection_completed) { + SCLogDebug("packet %"PRIu64": ndpi protocol not yet detected", p->pcap_cnt); + SCReturnInt(0); + } + + const Flow *f = p->flow; + if (f == NULL) { + SCLogDebug("packet %"PRIu64": no flow", p->pcap_cnt); + SCReturnInt(0); + } + + r = ndpi_is_proto_equals(f->detected_l7_protocol.proto, data->l7_protocol, false); + r = r ^ data->negated; + + if (r) { + SCLogDebug("ndpi protocol match on protocol = %u.%u (match %u)", + f->detected_l7_protocol.app_protocol, + f->detected_l7_protocol.master_protocol, + data->l7_protocol); + SCReturnInt(1); + } + SCReturnInt(0); +} + +static DetectnDPIProtocolData *DetectnDPIProtocolParse(const char *arg, bool negate) +{ + DetectnDPIProtocolData *data; + struct ndpi_detection_module_struct *ndpi_struct; + ndpi_master_app_protocol l7_protocol; + char *l7_protocol_name = (char *)arg; + NDPI_PROTOCOL_BITMASK all; + + /* convert protocol name (string) to ID */ + ndpi_struct = ndpi_init_detection_module(NULL); + if (unlikely(ndpi_struct == NULL)) + return NULL; + + ndpi_struct = ndpi_init_detection_module(NULL); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); + ndpi_finalize_initialization(ndpi_struct); + + l7_protocol = ndpi_get_protocol_by_name(ndpi_struct, l7_protocol_name); + ndpi_exit_detection_module(ndpi_struct); + + if (ndpi_is_proto_unknown(l7_protocol)) { + SCLogError("failure parsing nDPI protocol '%s'", l7_protocol_name); + return NULL; + } + + data = SCMalloc(sizeof(DetectnDPIProtocolData)); + if (unlikely(data == NULL)) + return NULL; + + memcpy(&data->l7_protocol, &l7_protocol, sizeof(ndpi_master_app_protocol)); + data->negated = negate; + + return data; +} + +static bool HasConflicts(const DetectnDPIProtocolData *us, const DetectnDPIProtocolData *them) +{ + /* check for mix of negated and non negated */ + if (them->negated ^ us->negated) + return true; + + /* check for multiple non-negated */ + if (!us->negated) + return true; + + /* check for duplicate */ + if (ndpi_is_proto_equals(us->l7_protocol, them->l7_protocol, true)) + return true; + + return false; +} + +static int DetectnDPIProtocolSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg) +{ + DetectnDPIProtocolData *data = NULL; + + data = DetectnDPIProtocolParse(arg, s->init_data->negated); + if (data == NULL) + goto error; + + SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for ( ; tsm != NULL; tsm = tsm->next) { + if (tsm->type == DETECT_NDPI_PROTOCOL) { + const DetectnDPIProtocolData *them = (const DetectnDPIProtocolData *)tsm->ctx; + + if (HasConflicts(data, them)) { + SCLogError("can't mix " + "positive ndpi-protocol match with negated"); + goto error; + } + } + } + + if (SigMatchAppendSMToList(de_ctx, s, DETECT_NDPI_PROTOCOL, (SigMatchCtx *)data, + DETECT_SM_LIST_MATCH) == NULL) { + goto error; + } + return 0; + +error: + if (data != NULL) + SCFree(data); + return -1; +} + +static void DetectnDPIProtocolFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +/** \internal + * \brief prefilter function for protocol detect matching + */ +static void +PrefilterPacketnDPIProtocolMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + const PrefilterPacketHeaderCtx *ctx = pectx; + + if (p->flow == NULL || !p->flow->detection_completed) { + SCLogDebug("packet %"PRIu64": no flow, no ndpi detection", p->pcap_cnt); + SCReturn; + } + + Flow *f = p->flow; + bool negated = (bool)ctx->v1.u8[4]; + + if (!ndpi_is_proto_unknown(f->detected_l7_protocol.proto)) { + ndpi_master_app_protocol p = { ctx->v1.u16[0], ctx->v1.u16[1] }; + + if (ndpi_is_proto_equals(f->detected_l7_protocol.proto, p, false) ^ negated) { + PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); + } + } +} + +static void +PrefilterPacketnDPIProtocolSet(PrefilterPacketHeaderValue *v, void *smctx) +{ + const DetectnDPIProtocolData *a = smctx; + + v->u16[0] = a->l7_protocol.master_protocol; + v->u16[1] = a->l7_protocol.app_protocol; + v->u8[4] = (uint8_t)a->negated; +} + +static bool +PrefilterPacketnDPIProtocolCompare(PrefilterPacketHeaderValue v, void *smctx) +{ + const DetectnDPIProtocolData *a = smctx; + ndpi_master_app_protocol p = { v.u16[0], v.u16[1] }; + bool negated = (bool)v.u8[4]; + + return (ndpi_is_proto_equals(a->l7_protocol, p, false) ^ negated); +} + +static int PrefilterSetupnDPIProtocol(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_PROTOCOL, + SIG_MASK_REQUIRE_FLOW, PrefilterPacketnDPIProtocolSet, PrefilterPacketnDPIProtocolCompare, + PrefilterPacketnDPIProtocolMatch); +} + +static bool PrefilternDPIProtocolIsPrefilterable(const Signature *s) +{ + if (s->type == SIG_TYPE_PDONLY) { + SCLogDebug("prefilter on PD %u", s->id); + return true; + } + return false; +} + +void DetectnDPIProtocolRegister(void) +{ + sigmatch_table[DETECT_NDPI_PROTOCOL].name = "ndpi-protocol"; + sigmatch_table[DETECT_NDPI_PROTOCOL].desc = "match on the detected nDPI protocol"; + sigmatch_table[DETECT_NDPI_PROTOCOL].url = "/rules/index.html"; + sigmatch_table[DETECT_NDPI_PROTOCOL].Match = + DetectnDPIProtocolPacketMatch; + sigmatch_table[DETECT_NDPI_PROTOCOL].Setup = + DetectnDPIProtocolSetup; + sigmatch_table[DETECT_NDPI_PROTOCOL].Free = + DetectnDPIProtocolFree; +#ifdef UNITTESTS + sigmatch_table[DETECT_NDPI_PROTOCOL].RegisterTests = + DetectnDPIProtocolRegisterTests; +#endif + sigmatch_table[DETECT_NDPI_PROTOCOL].flags = + (SIGMATCH_QUOTES_OPTIONAL|SIGMATCH_HANDLE_NEGATION); + + sigmatch_table[DETECT_NDPI_PROTOCOL].SetupPrefilter = + PrefilterSetupnDPIProtocol; + sigmatch_table[DETECT_NDPI_PROTOCOL].SupportsPrefilter = + PrefilternDPIProtocolIsPrefilterable; +} + +/**********************************Unittests***********************************/ + +#ifdef UNITTESTS + +static int DetectnDPIProtocolTest01(void) +{ + DetectnDPIProtocolData *data = DetectnDPIProtocolParse("HTTP", false); + FAIL_IF_NULL(data); + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated != 0); + DetectnDPIProtocolFree(NULL, data); + PASS; +} + +static int DetectnDPIProtocolTest02(void) +{ + DetectnDPIProtocolData *data = DetectnDPIProtocolParse("HTTP", true); + FAIL_IF_NULL(data); + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated == 0); + DetectnDPIProtocolFree(NULL, data); + PASS; +} + +static int DetectnDPIProtocolTest03(void) +{ + Signature *s = NULL; + DetectnDPIProtocolData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-protocol:HTTP; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIProtocolData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated); + DetectEngineCtxFree(de_ctx); + PASS; +} + +static int DetectnDPIProtocolTest04(void) +{ + Signature *s = NULL; + DetectnDPIProtocolData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-protocol:!HTTP; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIProtocolData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF_NULL(data); + FAIL_IF(data->l7_protocol.master_protocol != NDPI_PROTOCOL_HTTP); + FAIL_IF(data->negated == 0); + + DetectEngineCtxFree(de_ctx); + PASS; +} + +static void DetectnDPIProtocolRegisterTests(void) +{ + UtRegisterTest("DetectnDPIProtocolTest01", + DetectnDPIProtocolTest01); + UtRegisterTest("DetectnDPIProtocolTest02", + DetectnDPIProtocolTest02); + UtRegisterTest("DetectnDPIProtocolTest03", + DetectnDPIProtocolTest03); + UtRegisterTest("DetectnDPIProtocolTest04", + DetectnDPIProtocolTest04); +} +#endif /* UNITTESTS */ + +#endif /* HAVE_NDPI */ diff --git a/src/detect-ndpi-protocol.h b/src/detect-ndpi-protocol.h new file mode 100644 index 000000000000..2f61ca5c57eb --- /dev/null +++ b/src/detect-ndpi-protocol.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#ifndef SURICATA_NDPI_PROTOCOL_H +#define SURICATA_NDPI_PROTOCOL_H + +void DetectnDPIProtocolRegister(void); + +#endif /* SURICATA_NDPI_PROTOCOL_H */ diff --git a/src/detect-ndpi-risk.c b/src/detect-ndpi-risk.c new file mode 100644 index 000000000000..d2b81e572a18 --- /dev/null +++ b/src/detect-ndpi-risk.c @@ -0,0 +1,343 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#include "suricata-common.h" +#include "detect-engine.h" +#include "detect-engine-build.h" +#include "detect-engine-prefilter.h" +#include "detect-engine-prefilter-common.h" +#include "detect-parse.h" +#include "detect-ndpi-risk.h" +#include "util-debug.h" +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#ifdef HAVE_NDPI + +#ifdef UNITTESTS +static void DetectnDPIRiskRegisterTests(void); +#endif + +typedef struct DetectnDPIRiskData_ { + ndpi_risk risk_mask; /* uint64 */ + uint8_t negated; +} DetectnDPIRiskData; + +static int DetectnDPIRiskPacketMatch(DetectEngineThreadCtx *det_ctx, + Packet *p, const Signature *s, const SigMatchCtx *ctx) +{ + SCEnter(); + + bool r; + const DetectnDPIRiskData *data = (const DetectnDPIRiskData *)ctx; + + if (!p->flow->detection_completed) { + SCLogDebug("packet %"PRIu64": ndpi risks not yet detected", p->pcap_cnt); + SCReturnInt(0); + } + + const Flow *f = p->flow; + if (f == NULL) { + SCLogDebug("packet %"PRIu64": no flow", p->pcap_cnt); + SCReturnInt(0); + } + + r = ((f->ndpi_flow->risk & data->risk_mask) == data->risk_mask); + r = r ^ data->negated; + + if (r) { + SCLogDebug("ndpi risks match on risk bitmap = %"PRIu64" (matching bitmap %"PRIu64")", + f->ndpi_flow->risk, + data->risk_mask); + SCReturnInt(1); + } + + SCReturnInt(0); +} + +static DetectnDPIRiskData *DetectnDPIRiskParse(const char *arg, bool negate) +{ + DetectnDPIRiskData *data; + struct ndpi_detection_module_struct *ndpi_struct; + ndpi_risk risk_mask; + NDPI_PROTOCOL_BITMASK all; + + /* convert list of risk names (string) to mask */ + ndpi_struct = ndpi_init_detection_module(NULL); + if (unlikely(ndpi_struct == NULL)) + return NULL; + + ndpi_struct = ndpi_init_detection_module(NULL); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); + ndpi_finalize_initialization(ndpi_struct); + + if (isdigit(arg[0])) + risk_mask = atoll(arg); + else { + char *dup = SCStrdup(arg), *tmp, *token; + + NDPI_ZERO_BIT(risk_mask); + + if (dup != NULL) { + token = strtok_r(dup, ",", &tmp); + + while (token != NULL) { + ndpi_risk_enum risk_id = ndpi_code2risk(token); + if (risk_id >= NDPI_MAX_RISK) { + SCLogError("unrecognized risk '%s', " + "please check ndpiReader -H for valid risk codes", + token); + return NULL; + } + NDPI_SET_BIT(risk_mask, risk_id); + token = strtok_r(NULL, ",", &tmp); + } + + SCFree(dup); + } + } + + data = SCMalloc(sizeof(DetectnDPIRiskData)); + if (unlikely(data == NULL)) + return NULL; + + data->risk_mask = risk_mask; + data->negated = negate; + + return data; +} + +static bool HasConflicts(const DetectnDPIRiskData *us, const DetectnDPIRiskData *them) +{ + /* check for duplicate */ + if (us->risk_mask == them->risk_mask) + return true; + + return false; +} + +static int DetectnDPIRiskSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg) +{ + DetectnDPIRiskData *data = NULL; + + data = DetectnDPIRiskParse(arg, s->init_data->negated); + if (data == NULL) + goto error; + + SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for ( ; tsm != NULL; tsm = tsm->next) { + if (tsm->type == DETECT_NDPI_RISK) { + const DetectnDPIRiskData *them = (const DetectnDPIRiskData *)tsm->ctx; + + if (HasConflicts(data, them)) { + SCLogError("can't mix " + "positive ndpi-risk match with negated"); + goto error; + } + } + } + + if (SigMatchAppendSMToList(de_ctx, s, DETECT_NDPI_RISK, (SigMatchCtx *)data, + DETECT_SM_LIST_MATCH) == NULL) { + goto error; + } + return 0; + + error: + if (data != NULL) + SCFree(data); + return -1; +} + +static void DetectnDPIRiskFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +/** \internal + * \brief prefilter function for risk matching + */ +static void +PrefilterPacketnDPIRiskMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + const PrefilterPacketHeaderCtx *ctx = pectx; + + if (p->flow == NULL || !p->flow->detection_completed) { + SCLogDebug("packet %"PRIu64": no flow, no ndpi detection", p->pcap_cnt); + SCReturn; + } + + Flow *f = p->flow; + bool negated = (bool)ctx->v1.u8[9]; + ndpi_risk risk_mask = ctx->v1.u64[0]; + bool ret = ((f->ndpi_flow->risk & risk_mask) == risk_mask) ? true: false; + + if (ret ^ negated) { + PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); + } +} + +static void +PrefilterPacketnDPIRiskSet(PrefilterPacketHeaderValue *v, void *smctx) +{ + const DetectnDPIRiskData *a = smctx; + + v->u64[0] = a->risk_mask; + v->u8[9] = (uint8_t)a->negated; +} + +static bool +PrefilterPacketnDPIRiskCompare(PrefilterPacketHeaderValue v, void *smctx) +{ + const DetectnDPIRiskData *a = smctx; + ndpi_risk p = v.u64[0]; + bool negated = (bool)v.u8[9]; + bool ret; + + ret = ((a->risk_mask & p) == p) ? true: false; + return (ret ^ negated); +} + +static int PrefilterSetupnDPIRisk(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_RISK, + SIG_MASK_REQUIRE_FLOW, + PrefilterPacketnDPIRiskSet, + PrefilterPacketnDPIRiskCompare, + PrefilterPacketnDPIRiskMatch); +} + +static bool PrefilternDPIRiskIsPrefilterable(const Signature *s) +{ + if (s->type == SIG_TYPE_PDONLY) { + SCLogDebug("prefilter on PD %u", s->id); + return true; + } + return false; +} + +void DetectnDPIRiskRegister(void) +{ + sigmatch_table[DETECT_NDPI_RISK].name = "ndpi-risk"; + sigmatch_table[DETECT_NDPI_RISK].desc = "match on the detected nDPI risk"; + sigmatch_table[DETECT_NDPI_RISK].url = "/rules/index.html"; + sigmatch_table[DETECT_NDPI_RISK].Match = DetectnDPIRiskPacketMatch; + sigmatch_table[DETECT_NDPI_RISK].Setup = DetectnDPIRiskSetup; + sigmatch_table[DETECT_NDPI_RISK].Free = DetectnDPIRiskFree; +#ifdef UNITTESTS + sigmatch_table[DETECT_NDPI_RISK].RegisterTests = + DetectnDPIRiskRegisterTests; +#endif + sigmatch_table[DETECT_NDPI_RISK].flags = + (SIGMATCH_QUOTES_OPTIONAL|SIGMATCH_HANDLE_NEGATION); + + sigmatch_table[DETECT_NDPI_RISK].SetupPrefilter = + PrefilterSetupnDPIRisk; + sigmatch_table[DETECT_NDPI_RISK].SupportsPrefilter = + PrefilternDPIRiskIsPrefilterable; +} + +/**********************************Unittests***********************************/ + +#ifdef UNITTESTS + +static int DetectnDPIRiskTest01(void) +{ + DetectnDPIRiskData *data = DetectnDPIRiskParse("NDPI_PROBING_ATTEMPT", false); + FAIL_IF_NULL(data); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(data->negated != 0); + DetectnDPIRiskFree(NULL, data); + PASS; +} + +static int DetectnDPIRiskTest02(void) +{ + DetectnDPIRiskData *data = DetectnDPIRiskParse("NDPI_PROBING_ATTEMPT", true); + FAIL_IF_NULL(data); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(data->negated == 0); + DetectnDPIRiskFree(NULL, data); + PASS; +} + +static int DetectnDPIRiskTest03(void) +{ + Signature *s = NULL; + DetectnDPIRiskData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-risk:NDPI_PROBING_ATTEMPT; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIRiskData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(data->negated); + DetectEngineCtxFree(de_ctx); + PASS; +} + +static int DetectnDPIRiskTest04(void) +{ + Signature *s = NULL; + DetectnDPIRiskData *data = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " + "(ndpi-risk:!NDPI_PROBING_ATTEMPT; sid:1;)"); + FAIL_IF_NULL(s); + + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); + + data = (DetectnDPIRiskData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; + FAIL_IF_NULL(data); + + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(data->negated == 0); + + DetectEngineCtxFree(de_ctx); + PASS; +} + +static void DetectnDPIRiskRegisterTests(void) +{ + UtRegisterTest("DetectnDPIRiskTest01", DetectnDPIRiskTest01); + UtRegisterTest("DetectnDPIRiskTest02", DetectnDPIRiskTest02); + UtRegisterTest("DetectnDPIRiskTest03", DetectnDPIRiskTest03); + UtRegisterTest("DetectnDPIRiskTest04", DetectnDPIRiskTest04); +} + +#endif /* UNITTESTS */ + +#endif /* HAVE_NDPI */ diff --git a/src/detect-ndpi-risk.h b/src/detect-ndpi-risk.h new file mode 100644 index 000000000000..cb1729670d84 --- /dev/null +++ b/src/detect-ndpi-risk.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2007-2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Luca Deri + * \author Alfredo Cardigliano + */ + +#ifndef SURICATA_NDPI_RISK_H +#define SURICATA_NDPI_RISK_H + +void DetectnDPIRiskRegister(void); + +#endif /* SURICATA_NDPI_RISK_H */ diff --git a/src/flow-util.c b/src/flow-util.c index 7e11da41f527..f788642a8f3f 100644 --- a/src/flow-util.c +++ b/src/flow-util.c @@ -70,6 +70,16 @@ Flow *FlowAlloc(void) /* coverity[missing_lock] */ FLOW_INITIALIZE(f); + +#ifdef HAVE_NDPI + f->ndpi_flow = (struct ndpi_flow_struct *)ndpi_flow_malloc(SIZEOF_FLOW_STRUCT); + if (f->ndpi_flow != NULL) { + memset(f->ndpi_flow, 0, SIZEOF_FLOW_STRUCT); + f->detection_completed = 0; + /* printf("%s - Allocated flow\n", __FUNCTION__); */ + } +#endif + return f; } @@ -81,6 +91,13 @@ Flow *FlowAlloc(void) */ void FlowFree(Flow *f) { +#ifdef HAVE_NDPI + if (f->ndpi_flow) { + ndpi_flow_free(f->ndpi_flow); + /* printf("%s - Freed flow\n", __FUNCTION__); */ + } +#endif + FLOW_DESTROY(f); SCFree(f); diff --git a/src/flow.c b/src/flow.c index 7bfa80ea0a9b..4cf995df06ab 100644 --- a/src/flow.c +++ b/src/flow.c @@ -387,6 +387,42 @@ void FlowHandlePacketUpdate(Flow *f, Packet *p, ThreadVars *tv, DecodeThreadVars { SCLogDebug("packet %"PRIu64" -- flow %p", p->pcap_cnt, f); +#ifdef HAVE_NDPI + if (tv->ndpi_struct && + f->ndpi_flow && + (!f->detection_completed) && + (p->ip_len > 0)) { + uint64_t time_ms = ((uint64_t) p->ts.secs) * 1000 /* TICK_RESOLUTION */ + p->ts.usecs / (1000000 / 1000 /* TICK_RESOLUTION */); + + f->detected_l7_protocol = ndpi_detection_process_packet(tv->ndpi_struct, f->ndpi_flow, + PacketIsIPv4(p) ? (void*)PacketGetIPv4(p) : (void*)PacketGetIPv6(p), + p->ip_len, time_ms, NULL); + + if (ndpi_is_protocol_detected(f->detected_l7_protocol) != 0) { + if(!ndpi_is_proto_unknown(f->detected_l7_protocol.proto)) { + if(!ndpi_extra_dissection_possible(tv->ndpi_struct, f->ndpi_flow)) + f->detection_completed = 1; + } + } else { + u_int16_t max_num_pkts = (f->proto == IPPROTO_UDP) ? 8 : 24; + + if ((f->todstpktcnt+f->tosrcpktcnt) > max_num_pkts) { + u_int8_t proto_guessed; + + f->detected_l7_protocol = ndpi_detection_giveup(tv->ndpi_struct, f->ndpi_flow, &proto_guessed); + f->detection_completed = 1; + } + } + + if (f->detection_completed) { + SCLogDebug("Detected protocol: %s | app protocol: %s | category: %s", + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.master_protocol), + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.app_protocol), + ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); + } + } +#endif + const int pkt_dir = FlowGetPacketDirection(f, p); #ifdef CAPTURE_OFFLOAD int state = f->flow_state; diff --git a/src/flow.h b/src/flow.h index bf28d02a5812..da730f299a07 100644 --- a/src/flow.h +++ b/src/flow.h @@ -503,6 +503,12 @@ typedef struct Flow_ uint64_t todstbytecnt; uint64_t tosrcbytecnt; +#ifdef HAVE_NDPI + struct ndpi_flow_struct *ndpi_flow; + ndpi_protocol detected_l7_protocol; + uint8_t detection_completed; +#endif + Storage storage[]; } Flow; diff --git a/src/output-filestore.c b/src/output-filestore.c index 5960f4ea473c..ff23accd09a8 100644 --- a/src/output-filestore.c +++ b/src/output-filestore.c @@ -167,7 +167,11 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv, const OutputFilestoreLo WARN_ONCE(WOT_SNPRINTF, "Failed to write file info record. Output filename truncated."); } else { JsonBuilder *js_fileinfo = - JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL); + JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL +#ifdef HAVE_NDPI + , tv +#endif + ); if (likely(js_fileinfo != NULL)) { jb_close(js_fileinfo); FILE *out = fopen(js_metadata_filename, "w"); diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 4126e8feaddc..d51777f277e0 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -659,6 +659,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) } } +#ifdef HAVE_NDPI + ndpiJsonBuilder(p->flow, jb, tv); +#endif EveAddAppProto(p->flow, jb); if (p->flowflags & FLOW_PKT_TOSERVER) { diff --git a/src/output-json-file.c b/src/output-json-file.c index ae8400a18d53..f17d698593cd 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -79,9 +79,355 @@ typedef struct JsonFileLogThread_ { OutputJsonThreadCtx *ctx; } JsonFileLogThread; +#ifdef HAVE_NDPI + +static void ndpiJsonBuilderTLSQUIC(Flow *f, JsonBuilder *js, ThreadVars *tv) +{ + char buf[64]; + char notBefore[32], notAfter[32]; + struct tm a, b, *before = NULL, *after = NULL; + u_int i, off; + u_int8_t unknown_tls_version; + char version[16], unknown_cipher[8]; + + if (!f->ndpi_flow->protos.tls_quic.ssl_version) { + return; + } + + ndpi_ssl_version2str(version, sizeof(version), f->ndpi_flow->protos.tls_quic.ssl_version, &unknown_tls_version); + + if (f->ndpi_flow->protos.tls_quic.notBefore) + before = ndpi_gmtime_r((const time_t *)&f->ndpi_flow->protos.tls_quic.notBefore, &a); + + if (f->ndpi_flow->protos.tls_quic.notAfter) + after = ndpi_gmtime_r((const time_t *)&f->ndpi_flow->protos.tls_quic.notAfter, &b); + + if (!unknown_tls_version) { + jb_open_object(js, "tls"); + jb_set_string(js, "version", version); + + if (f->ndpi_flow->protos.tls_quic.server_names) + jb_set_string(js, "server_names", f->ndpi_flow->protos.tls_quic.server_names); + + if (before) { + strftime(notBefore, sizeof(notBefore), "%Y-%m-%d %H:%M:%S", before); + jb_set_string(js, "notbefore", notBefore); + } + + if (after) { + strftime(notAfter, sizeof(notAfter), "%Y-%m-%d %H:%M:%S", after); + jb_set_string(js, "notafter", notAfter); + } + + /* Note: ja3, ja3s, ja4 are not serialized as as Suricata already supports them */ + + jb_set_uint(js,"unsafe_cipher", f->ndpi_flow->protos.tls_quic.server_unsafe_cipher); + jb_set_string(js, "cipher", ndpi_cipher2str(f->ndpi_flow->protos.tls_quic.server_cipher, unknown_cipher)); + + if (f->ndpi_flow->protos.tls_quic.issuerDN) + jb_set_string(js, "issuerDN", f->ndpi_flow->protos.tls_quic.issuerDN); + + if (f->ndpi_flow->protos.tls_quic.subjectDN) + jb_set_string(js, "subjectDN", f->ndpi_flow->protos.tls_quic.subjectDN); + + if (f->ndpi_flow->protos.tls_quic.advertised_alpns) + jb_set_string(js, "advertised_alpns", f->ndpi_flow->protos.tls_quic.advertised_alpns); + + if (f->ndpi_flow->protos.tls_quic.negotiated_alpn) + jb_set_string(js, "negotiated_alpn", f->ndpi_flow->protos.tls_quic.negotiated_alpn); + + if (f->ndpi_flow->protos.tls_quic.tls_supported_versions) + jb_set_string(js, "tls_supported_versions", f->ndpi_flow->protos.tls_quic.tls_supported_versions); + + if (f->ndpi_flow->protos.tls_quic.sha1_certificate_fingerprint[0] != '\0') { + for(i=0, off=0; i<20; i++) { + int rc = ndpi_snprintf(&buf[off], sizeof(buf)-off,"%s%02X", (i > 0) ? ":" : "", + f->ndpi_flow->protos.tls_quic.sha1_certificate_fingerprint[i] & 0xFF); + if (rc <= 0) break; else off += rc; + } + jb_set_string(js, "fingerprint", buf); + } + + jb_set_uint(js,"blocks", f->ndpi_flow->l4.tcp.tls.num_tls_blocks); + + jb_close(js); + } +} + +/* Suricata backport of ndpi_dpi2json */ +void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) +{ + char buf[64]; + char const *host_server_name; + char quic_version[16]; + ndpi_protocol l7_protocol; + + if (f == NULL) return; + + jb_open_object(js, "ndpi"); + jb_set_string(js, "app_protocol", ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.app_protocol)); + jb_set_string(js, "master_protocol", ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.master_protocol)); + jb_set_string(js, "category", ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); + + if (f->ndpi_flow->risk) { + u_int risk_id; + + jb_open_array(js, "risks"); + + for (risk_id = 0; risk_id < NDPI_MAX_RISK; risk_id++) + if (NDPI_ISSET_BIT(f->ndpi_flow->risk, risk_id)) { + char str[256]; + snprintf(str, sizeof(str), "%s (%s)", ndpi_risk2code(risk_id), ndpi_risk2str(risk_id)); + jb_append_string(js, str); + } + + jb_close(js); + } /* risk */ + + host_server_name = ndpi_get_flow_info(f->ndpi_flow, &l7_protocol); + if (host_server_name != NULL) { + jb_set_string(js, "hostname", host_server_name); + } + + switch (l7_protocol.proto.master_protocol ? l7_protocol.proto.master_protocol : l7_protocol.proto.app_protocol) { + case NDPI_PROTOCOL_IP_ICMP: + if (f->ndpi_flow->entropy > 0.0f) { + char buf[64]; + snprintf(buf, sizeof(buf), "%.6f", f->ndpi_flow->entropy); + jb_set_string(js, "entropy", buf); + } + break; + case NDPI_PROTOCOL_DHCP: + jb_open_object(js,"dhcp"); + jb_set_string(js, "fingerprint", f->ndpi_flow->protos.dhcp.fingerprint); + jb_set_string(js, "class_ident", f->ndpi_flow->protos.dhcp.class_ident); + jb_close(js); + break; + case NDPI_PROTOCOL_BITTORRENT: + { + u_int i, j, n = 0; + char bittorent_hash[sizeof(f->ndpi_flow->protos.bittorrent.hash)*2+1]; + for (i = 0, j = 0; j < sizeof(bittorent_hash)-1; i++) { + snprintf(&bittorent_hash[j], + sizeof(bittorent_hash) - j, + "%02x", + f->ndpi_flow->protos.bittorrent.hash[i]); + + j += 2; + n += f->ndpi_flow->protos.bittorrent.hash[i]; + } + if (n == 0) bittorent_hash[0] = '\0'; + jb_open_object(js,"bittorrent"); + jb_set_string(js, "hash", bittorent_hash); + jb_close(js); + } + break; + case NDPI_PROTOCOL_COLLECTD: + jb_open_object(js,"collectd"); + jb_set_string(js, "client_username", f->ndpi_flow->protos.collectd.client_username); + jb_close(js); + break; + case NDPI_PROTOCOL_DNS: + jb_open_object(js,"dns"); + jb_set_uint(js, "num_queries", f->ndpi_flow->protos.dns.num_queries); + jb_set_uint(js, "num_answers", f->ndpi_flow->protos.dns.num_answers); + jb_set_uint(js, "reply_code", f->ndpi_flow->protos.dns.reply_code); + jb_set_uint(js, "query_type", f->ndpi_flow->protos.dns.query_type); + jb_set_uint(js, "rsp_type", f->ndpi_flow->protos.dns.rsp_type); + inet_ntop(AF_INET, &f->ndpi_flow->protos.dns.rsp_addr, buf, sizeof(buf)); + jb_set_string(js, "rsp_addr", buf); + jb_close(js); + break; + case NDPI_PROTOCOL_NTP: + jb_open_object(js,"ntp"); + jb_set_uint(js, "request_code", f->ndpi_flow->protos.ntp.request_code); + jb_set_uint(js, "version", f->ndpi_flow->protos.ntp.request_code); + jb_close(js); + break; + case NDPI_PROTOCOL_MDNS: + jb_open_object(js,"mdns"); + jb_close(js); + break; + case NDPI_PROTOCOL_UBNTAC2: + jb_open_object(js,"ubntac2"); + jb_set_string(js, "version", f->ndpi_flow->protos.ubntac2.version); + jb_close(js); + break; + case NDPI_PROTOCOL_KERBEROS: + jb_open_object(js,"kerberos"); + jb_set_string(js, "hostname", f->ndpi_flow->protos.kerberos.hostname); + jb_set_string(js, "domain", f->ndpi_flow->protos.kerberos.domain); + jb_set_string(js, "username", f->ndpi_flow->protos.kerberos.username); + jb_close(js); + break; + case NDPI_PROTOCOL_SOFTETHER: + jb_open_object(js,"softether"); + jb_set_string(js, "client_ip", f->ndpi_flow->protos.softether.ip); + jb_set_string(js, "client_port", f->ndpi_flow->protos.softether.port); + jb_set_string(js, "hostname", f->ndpi_flow->protos.softether.hostname); + jb_set_string(js, "fqdn", f->ndpi_flow->protos.softether.fqdn); + jb_close(js); + break; + case NDPI_PROTOCOL_NATPMP: + jb_open_object(js,"natpmp"); + jb_set_uint(js, "result", f->ndpi_flow->protos.natpmp.result_code); + jb_set_uint(js, "internal_port", f->ndpi_flow->protos.natpmp.internal_port); + jb_set_uint(js, "external_port", f->ndpi_flow->protos.natpmp.external_port); + inet_ntop(AF_INET, &f->ndpi_flow->protos.natpmp.external_address.ipv4, buf, sizeof(buf)); + jb_set_string(js, "external_address", buf); + jb_close(js); + break; + case NDPI_PROTOCOL_RSH: + jb_open_object(js,"rsh"); + jb_set_string(js, "client_username", f->ndpi_flow->protos.rsh.client_username); + jb_set_string(js, "server_username", f->ndpi_flow->protos.rsh.server_username); + jb_set_string(js, "command", f->ndpi_flow->protos.rsh.command); + jb_close(js); + break; + case NDPI_PROTOCOL_SNMP: + jb_open_object(js,"snmp"); + jb_set_uint(js, "version", f->ndpi_flow->protos.snmp.version); + jb_set_uint(js, "primitive", f->ndpi_flow->protos.snmp.primitive); + jb_set_uint(js, "error_status", f->ndpi_flow->protos.snmp.error_status); + jb_close(js); + break; + case NDPI_PROTOCOL_TELNET: + jb_open_object(js,"telnet"); + jb_set_string(js, "username", f->ndpi_flow->protos.telnet.username); + jb_set_string(js, "password", f->ndpi_flow->protos.telnet.password); + jb_close(js); + break; + case NDPI_PROTOCOL_TFTP: + jb_open_object(js,"tftp"); + jb_set_string(js, "filename", f->ndpi_flow->protos.tftp.filename); + jb_close(js); + break; + case NDPI_PROTOCOL_TIVOCONNECT: + jb_open_object(js,"tivoconnect"); + jb_set_string(js, "identity_uuid", f->ndpi_flow->protos.tivoconnect.identity_uuid); + jb_set_string(js, "machine", f->ndpi_flow->protos.tivoconnect.machine); + jb_set_string(js, "platform", f->ndpi_flow->protos.tivoconnect.platform); + jb_set_string(js, "services", f->ndpi_flow->protos.tivoconnect.services); + jb_close(js); + break; + case NDPI_PROTOCOL_HTTP: + case NDPI_PROTOCOL_HTTP_CONNECT: + case NDPI_PROTOCOL_HTTP_PROXY: + jb_open_object(js,"http"); + if (f->ndpi_flow->http.url != NULL) { + jb_set_string(js, "url", f->ndpi_flow->http.url); + jb_set_uint(js, "code", f->ndpi_flow->http.response_status_code); + jb_set_string(js, "content_type", f->ndpi_flow->http.content_type); + jb_set_string(js, "user_agent", f->ndpi_flow->http.user_agent); + } + if (f->ndpi_flow->http.request_content_type != NULL) { + jb_set_string(js, "request_content_type", + f->ndpi_flow->http.request_content_type); + } + if (f->ndpi_flow->http.detected_os != NULL) { + jb_set_string(js, "detected_os", + f->ndpi_flow->http.detected_os); + } + if (f->ndpi_flow->http.nat_ip != NULL) { + jb_set_string(js, "nat_ip", + f->ndpi_flow->http.nat_ip); + } + jb_close(js); + break; + case NDPI_PROTOCOL_QUIC: + jb_open_object(js,"quic"); + if (f->ndpi_flow->http.user_agent) { + jb_set_string(js, "user_agent", f->ndpi_flow->http.user_agent); + } + ndpi_quic_version2str(quic_version, sizeof(quic_version), + f->ndpi_flow->protos.tls_quic.quic_version); + jb_set_string(js, "quic_version", quic_version); + ndpiJsonBuilderTLSQUIC(f, js, tv); + jb_close(js); + break; + case NDPI_PROTOCOL_MAIL_IMAP: + jb_open_object(js,"imap"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", + f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_MAIL_POP: + jb_open_object(js,"pop"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", + f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_MAIL_SMTP: + jb_open_object(js,"smtp"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", + f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_FTP_CONTROL: + jb_open_object(js,"ftp"); + jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); + jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_close(js); + break; + case NDPI_PROTOCOL_DISCORD: + if (l7_protocol.proto.master_protocol != NDPI_PROTOCOL_TLS) { + jb_open_object(js,"discord"); + jb_set_string(js, "client_ip", f->ndpi_flow->protos.discord.client_ip); + jb_close(js); + } + break; + case NDPI_PROTOCOL_SSH: + jb_open_object(js, "ssh"); + jb_set_string(js, "client_signature", f->ndpi_flow->protos.ssh.client_signature); + jb_set_string(js, "server_signature", f->ndpi_flow->protos.ssh.server_signature); + jb_set_string(js, "hassh_client", f->ndpi_flow->protos.ssh.hassh_client); + jb_set_string(js, "hassh_server", f->ndpi_flow->protos.ssh.hassh_server); + jb_close(js); + break; + case NDPI_PROTOCOL_STUN: + jb_open_object(js,"stun"); + if (f->ndpi_flow->stun.mapped_address.port) { + jb_set_string(js, "mapped_address", print_ndpi_address_port(&f->ndpi_flow->stun.mapped_address, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.peer_address.port) { + jb_set_string(js, "peer_address", print_ndpi_address_port(&f->ndpi_flow->stun.peer_address, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.relayed_address.port) { + jb_set_string(js, "relayed_address", print_ndpi_address_port(&f->ndpi_flow->stun.relayed_address, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.response_origin.port) { + jb_set_string(js, "response_origin", print_ndpi_address_port(&f->ndpi_flow->stun.response_origin, buf, sizeof(buf))); + } + if (f->ndpi_flow->stun.other_address.port) { + jb_set_string(js, "other_address", print_ndpi_address_port(&f->ndpi_flow->stun.other_address, buf, sizeof(buf))); + } + jb_close(js); + break; + case NDPI_PROTOCOL_TLS: + case NDPI_PROTOCOL_DTLS: + ndpiJsonBuilderTLSQUIC(f, js, tv); + break; + } /* switch */ + + jb_close(js); // "ndpi" +} + +#endif + JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, - OutputJsonCtx *eve_ctx) + OutputJsonCtx *eve_ctx +#ifdef HAVE_NDPI + , ThreadVars *tv +#endif +) { enum OutputJsonLogDirection fdir = LOG_DIR_FLOW; @@ -191,6 +537,10 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, jb_set_string(js, "app_proto", AppProtoToString(p->flow->alproto)); +#ifdef HAVE_NDPI + ndpiJsonBuilder(p->flow, js, tv); +#endif + jb_open_object(js, "fileinfo"); if (stored) { // the file has just been stored on disk cf OUTPUT_FILEDATA_FLAG_CLOSE @@ -214,11 +564,20 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, * \brief Write meta data on a single line json record */ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) + const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx +#ifdef HAVE_NDPI + , ThreadVars *tv +#endif +) { HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg; - JsonBuilder *js = JsonBuildFileInfoRecord(p, ff, tx, tx_id, false, dir, xff_cfg, eve_ctx); + JsonBuilder *js = JsonBuildFileInfoRecord(p, ff, tx, tx_id, false, dir, xff_cfg, eve_ctx +#ifdef HAVE_NDPI + , tv +#endif + ); + if (unlikely(js == NULL)) { return; } @@ -237,7 +596,12 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, co SCLogDebug("ff %p", ff); - FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); + FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx +#ifdef HAVE_NDPI + , tv +#endif + ); + return 0; } diff --git a/src/output-json-file.h b/src/output-json-file.h index e2eeeccb2ffc..f51d919ca139 100644 --- a/src/output-json-file.h +++ b/src/output-json-file.h @@ -31,6 +31,10 @@ typedef struct OutputJsonCtx_ OutputJsonCtx; void JsonFileLogRegister(void); JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, - OutputJsonCtx *eve_ctx); + OutputJsonCtx *eve_ctx +#ifdef HAVE_NDPI + , ThreadVars *tv +#endif +); #endif /* SURICATA_OUTPUT_JSON_FILE_H */ diff --git a/src/output-json-flow.c b/src/output-json-flow.c index 487185f2ed5b..d5d2f1ab255f 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -215,8 +215,15 @@ void EveAddFlow(Flow *f, JsonBuilder *js) } /* Eve format logging */ -static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f) +static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f +#ifdef HAVE_NDPI + , ThreadVars *tv +#endif +) { +#ifdef HAVE_NDPI + ndpiJsonBuilder(f, jb, tv); +#endif EveAddAppProto(f, jb); jb_open_object(jb, "flow"); EveAddFlow(f, jb); @@ -337,7 +344,11 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) SCReturnInt(TM_ECODE_OK); } - EveFlowLogJSON(thread, jb, f); + EveFlowLogJSON(thread, jb, f +#ifdef HAVE_NDPI + , tv +#endif + ); OutputJsonBuilderBuffer(jb, thread); jb_free(jb); diff --git a/src/output-json-flow.h b/src/output-json-flow.h index e334315e56fc..a2777e14abeb 100644 --- a/src/output-json-flow.h +++ b/src/output-json-flow.h @@ -27,5 +27,8 @@ void JsonFlowLogRegister(void); void EveAddFlow(Flow *f, JsonBuilder *js); void EveAddAppProto(Flow *f, JsonBuilder *js); +#ifdef HAVE_NDPI +void EveAddnDPIProto(Flow *f, JsonBuilder *js, ThreadVars *tv); +#endif #endif /* SURICATA_OUTPUT_JSON_FLOW_H */ diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index 98873e5f063f..a5d6a4261741 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -175,8 +175,16 @@ static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir) } /* JSON format logging */ -static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f) +static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f +#ifdef HAVE_NDPI + , ThreadVars *tv +#endif +) { +#ifdef HAVE_NDPI + ndpiJsonBuilder(f, js, tv); +#endif + jb_set_string(js, "app_proto", AppProtoToString(f->alproto_ts ? f->alproto_ts : f->alproto)); @@ -219,8 +227,16 @@ static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f) } } -static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f) +static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f +#ifdef HAVE_NDPI + , ThreadVars *tv +#endif +) { +#ifdef HAVE_NDPI + ndpiJsonBuilder(f, js, tv); +#endif + jb_set_string(js, "app_proto", AppProtoToString(f->alproto_tc ? f->alproto_tc : f->alproto)); @@ -274,7 +290,11 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) JsonBuilder *jb = CreateEveHeaderFromNetFlow(f, 0); if (unlikely(jb == NULL)) return TM_ECODE_OK; - NetFlowLogEveToServer(jb, f); + NetFlowLogEveToServer(jb, f +#ifdef HAVE_NDPI + , tv +#endif + ); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER); OutputJsonBuilderBuffer(jb, jhl); jb_free(jb); @@ -284,7 +304,11 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) jb = CreateEveHeaderFromNetFlow(f, 1); if (unlikely(jb == NULL)) return TM_ECODE_OK; - NetFlowLogEveToClient(jb, f); + NetFlowLogEveToClient(jb, f +#ifdef HAVE_NDPI + , tv +#endif + ); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT); OutputJsonBuilderBuffer(jb, jhl); jb_free(jb); diff --git a/src/output-json.h b/src/output-json.h index defc86d0374a..283fc752badd 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -120,4 +120,7 @@ OutputJsonThreadCtx *CreateEveThreadCtx(ThreadVars *t, OutputJsonCtx *ctx); void FreeEveThreadCtx(OutputJsonThreadCtx *ctx); void JSONFormatAndAddMACAddr(JsonBuilder *js, const char *key, const uint8_t *val, bool is_array); +#ifdef HAVE_NDPI +void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv); +#endif #endif /* SURICATA_OUTPUT_JSON_H */ diff --git a/src/suricata-common.h b/src/suricata-common.h index 2091250e49c5..d4e01665b0ed 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -511,6 +511,10 @@ typedef struct lua_State lua_State; #include "queue.h" #include "tree.h" +#ifdef HAVE_NDPI +#include "ndpi_api.h" +#endif + #ifndef HAVE_STRLCAT size_t strlcat(char *, const char *src, size_t siz); #endif diff --git a/src/threadvars.h b/src/threadvars.h index cebcdb4e3ac1..252ef0d2b26d 100644 --- a/src/threadvars.h +++ b/src/threadvars.h @@ -135,6 +135,10 @@ typedef struct ThreadVars_ { struct FlowQueue_ *flow_queue; bool break_loop; +#ifdef HAVE_NDPI + struct ndpi_detection_module_struct * ndpi_struct; +#endif + } ThreadVars; /** Thread setup flags: */ diff --git a/src/tm-threads.c b/src/tm-threads.c index cd9bf6df1fa7..2bddb95c8094 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -1004,6 +1004,19 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i if (mucond != 0) TmThreadInitMC(tv); +#ifdef HAVE_NDPI + tv->ndpi_struct = ndpi_init_detection_module(NULL); + if (tv->ndpi_struct != NULL) { + NDPI_PROTOCOL_BITMASK protos; + + NDPI_BITMASK_SET_ALL(protos); + ndpi_set_protocol_detection_bitmask2(tv->ndpi_struct, &protos); + ndpi_finalize_initialization(tv->ndpi_struct); + + /* printf("%s - ndpi_init_detection_module()\n", __FUNCTION__); */ + } +#endif + return tv; error: @@ -1600,6 +1613,13 @@ static void TmThreadFree(ThreadVars *tv) SCFree(ps); } +#ifdef HAVE_NDPI + if(tv->ndpi_struct != NULL) { + ndpi_exit_detection_module(tv->ndpi_struct); + /* printf("%s - ndpi_exit_detection_module()\n", __FUNCTION__); */ + } +#endif + TmThreadsUnregisterThread(tv->id); SCFree(tv); } From c9262c4e36a14ae7abcc0298977d5f42253a4552 Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Wed, 28 Aug 2024 17:24:03 +0200 Subject: [PATCH 2/4] Format ndpi patch with clang-format --- src/decode-ipv4.c | 2 +- src/decode-ipv6.c | 3 +- src/detect-engine-register.h | 4 +- src/detect-ndpi-protocol.c | 88 ++++++------- src/detect-ndpi-risk.c | 75 +++++------ src/flow.c | 31 +++-- src/output-filestore.c | 5 +- src/output-json-file.c | 236 +++++++++++++++++++---------------- src/output-json-file.h | 9 +- src/output-json-flow.c | 8 +- src/output-json-netflow.c | 16 ++- src/output-json.h | 2 +- src/threadvars.h | 2 +- src/tm-threads.c | 6 +- 14 files changed, 244 insertions(+), 243 deletions(-) diff --git a/src/decode-ipv4.c b/src/decode-ipv4.c index 98980f1b15d1..4b97ad1588ed 100644 --- a/src/decode-ipv4.c +++ b/src/decode-ipv4.c @@ -503,7 +503,7 @@ static const IPV4Hdr *DecodeIPV4Packet(Packet *p, const uint8_t *pkt, uint16_t l #ifdef HAVE_NDPI p->ip_len = len; #endif - + /* set the address struct */ SET_IPV4_SRC_ADDR(ip4h, &p->src); SET_IPV4_DST_ADDR(ip4h, &p->dst); diff --git a/src/decode-ipv6.c b/src/decode-ipv6.c index 483f74cb50ba..93c2ea0b15a7 100644 --- a/src/decode-ipv6.c +++ b/src/decode-ipv6.c @@ -550,11 +550,10 @@ static const IPV6Hdr *DecodeIPV6Packet( ENGINE_SET_INVALID_EVENT(p, IPV6_TRUNC_PKT); return NULL; } - + #ifdef HAVE_NDPI p->ip_len = len; #endif - SET_IPV6_SRC_ADDR(ip6h, &p->src); SET_IPV6_DST_ADDR(ip6h, &p->dst); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index ee9631997092..43e0b8574f8d 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -30,8 +30,8 @@ enum DetectKeywordId { DETECT_REV, DETECT_CLASSTYPE, - /* sorted by prefilter priority. Higher in this list means it will be - * picked over ones lower in the list */ +/* sorted by prefilter priority. Higher in this list means it will be + * picked over ones lower in the list */ #ifdef HAVE_NDPI DETECT_NDPI_PROTOCOL, DETECT_NDPI_RISK, diff --git a/src/detect-ndpi-protocol.c b/src/detect-ndpi-protocol.c index f340ec9fb917..f77385327b12 100644 --- a/src/detect-ndpi-protocol.c +++ b/src/detect-ndpi-protocol.c @@ -40,13 +40,12 @@ static void DetectnDPIProtocolRegisterTests(void); #endif typedef struct DetectnDPIProtocolData_ { - ndpi_master_app_protocol l7_protocol; - uint8_t negated; + ndpi_master_app_protocol l7_protocol; + uint8_t negated; } DetectnDPIProtocolData; static int DetectnDPIProtocolPacketMatch( - DetectEngineThreadCtx *det_ctx, - Packet *p, const Signature *s, const SigMatchCtx *ctx) + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) { SCEnter(); @@ -63,24 +62,23 @@ static int DetectnDPIProtocolPacketMatch( */ if (!p->flow->detection_completed) { - SCLogDebug("packet %"PRIu64": ndpi protocol not yet detected", p->pcap_cnt); + SCLogDebug("packet %" PRIu64 ": ndpi protocol not yet detected", p->pcap_cnt); SCReturnInt(0); } const Flow *f = p->flow; if (f == NULL) { - SCLogDebug("packet %"PRIu64": no flow", p->pcap_cnt); + SCLogDebug("packet %" PRIu64 ": no flow", p->pcap_cnt); SCReturnInt(0); } r = ndpi_is_proto_equals(f->detected_l7_protocol.proto, data->l7_protocol, false); r = r ^ data->negated; - + if (r) { SCLogDebug("ndpi protocol match on protocol = %u.%u (match %u)", - f->detected_l7_protocol.app_protocol, - f->detected_l7_protocol.master_protocol, - data->l7_protocol); + f->detected_l7_protocol.app_protocol, f->detected_l7_protocol.master_protocol, + data->l7_protocol); SCReturnInt(1); } SCReturnInt(0); @@ -108,8 +106,8 @@ static DetectnDPIProtocolData *DetectnDPIProtocolParse(const char *arg, bool neg ndpi_exit_detection_module(ndpi_struct); if (ndpi_is_proto_unknown(l7_protocol)) { - SCLogError("failure parsing nDPI protocol '%s'", l7_protocol_name); - return NULL; + SCLogError("failure parsing nDPI protocol '%s'", l7_protocol_name); + return NULL; } data = SCMalloc(sizeof(DetectnDPIProtocolData)); @@ -148,7 +146,7 @@ static int DetectnDPIProtocolSetup(DetectEngineCtx *de_ctx, Signature *s, const goto error; SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; - for ( ; tsm != NULL; tsm = tsm->next) { + for (; tsm != NULL; tsm = tsm->next) { if (tsm->type == DETECT_NDPI_PROTOCOL) { const DetectnDPIProtocolData *them = (const DetectnDPIProtocolData *)tsm->ctx; @@ -180,13 +178,13 @@ static void DetectnDPIProtocolFree(DetectEngineCtx *de_ctx, void *ptr) /** \internal * \brief prefilter function for protocol detect matching */ -static void -PrefilterPacketnDPIProtocolMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +static void PrefilterPacketnDPIProtocolMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) { const PrefilterPacketHeaderCtx *ctx = pectx; if (p->flow == NULL || !p->flow->detection_completed) { - SCLogDebug("packet %"PRIu64": no flow, no ndpi detection", p->pcap_cnt); + SCLogDebug("packet %" PRIu64 ": no flow, no ndpi detection", p->pcap_cnt); SCReturn; } @@ -194,16 +192,15 @@ PrefilterPacketnDPIProtocolMatch(DetectEngineThreadCtx *det_ctx, Packet *p, cons bool negated = (bool)ctx->v1.u8[4]; if (!ndpi_is_proto_unknown(f->detected_l7_protocol.proto)) { - ndpi_master_app_protocol p = { ctx->v1.u16[0], ctx->v1.u16[1] }; - - if (ndpi_is_proto_equals(f->detected_l7_protocol.proto, p, false) ^ negated) { + ndpi_master_app_protocol p = { ctx->v1.u16[0], ctx->v1.u16[1] }; + + if (ndpi_is_proto_equals(f->detected_l7_protocol.proto, p, false) ^ negated) { PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); } } } -static void -PrefilterPacketnDPIProtocolSet(PrefilterPacketHeaderValue *v, void *smctx) +static void PrefilterPacketnDPIProtocolSet(PrefilterPacketHeaderValue *v, void *smctx) { const DetectnDPIProtocolData *a = smctx; @@ -212,20 +209,19 @@ PrefilterPacketnDPIProtocolSet(PrefilterPacketHeaderValue *v, void *smctx) v->u8[4] = (uint8_t)a->negated; } -static bool -PrefilterPacketnDPIProtocolCompare(PrefilterPacketHeaderValue v, void *smctx) +static bool PrefilterPacketnDPIProtocolCompare(PrefilterPacketHeaderValue v, void *smctx) { const DetectnDPIProtocolData *a = smctx; - ndpi_master_app_protocol p = { v.u16[0], v.u16[1] }; - bool negated = (bool)v.u8[4]; - + ndpi_master_app_protocol p = { v.u16[0], v.u16[1] }; + bool negated = (bool)v.u8[4]; + return (ndpi_is_proto_equals(a->l7_protocol, p, false) ^ negated); } static int PrefilterSetupnDPIProtocol(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { - return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_PROTOCOL, - SIG_MASK_REQUIRE_FLOW, PrefilterPacketnDPIProtocolSet, PrefilterPacketnDPIProtocolCompare, + return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_PROTOCOL, SIG_MASK_REQUIRE_FLOW, + PrefilterPacketnDPIProtocolSet, PrefilterPacketnDPIProtocolCompare, PrefilterPacketnDPIProtocolMatch); } @@ -243,23 +239,17 @@ void DetectnDPIProtocolRegister(void) sigmatch_table[DETECT_NDPI_PROTOCOL].name = "ndpi-protocol"; sigmatch_table[DETECT_NDPI_PROTOCOL].desc = "match on the detected nDPI protocol"; sigmatch_table[DETECT_NDPI_PROTOCOL].url = "/rules/index.html"; - sigmatch_table[DETECT_NDPI_PROTOCOL].Match = - DetectnDPIProtocolPacketMatch; - sigmatch_table[DETECT_NDPI_PROTOCOL].Setup = - DetectnDPIProtocolSetup; - sigmatch_table[DETECT_NDPI_PROTOCOL].Free = - DetectnDPIProtocolFree; + sigmatch_table[DETECT_NDPI_PROTOCOL].Match = DetectnDPIProtocolPacketMatch; + sigmatch_table[DETECT_NDPI_PROTOCOL].Setup = DetectnDPIProtocolSetup; + sigmatch_table[DETECT_NDPI_PROTOCOL].Free = DetectnDPIProtocolFree; #ifdef UNITTESTS - sigmatch_table[DETECT_NDPI_PROTOCOL].RegisterTests = - DetectnDPIProtocolRegisterTests; + sigmatch_table[DETECT_NDPI_PROTOCOL].RegisterTests = DetectnDPIProtocolRegisterTests; #endif sigmatch_table[DETECT_NDPI_PROTOCOL].flags = - (SIGMATCH_QUOTES_OPTIONAL|SIGMATCH_HANDLE_NEGATION); + (SIGMATCH_QUOTES_OPTIONAL | SIGMATCH_HANDLE_NEGATION); - sigmatch_table[DETECT_NDPI_PROTOCOL].SetupPrefilter = - PrefilterSetupnDPIProtocol; - sigmatch_table[DETECT_NDPI_PROTOCOL].SupportsPrefilter = - PrefilternDPIProtocolIsPrefilterable; + sigmatch_table[DETECT_NDPI_PROTOCOL].SetupPrefilter = PrefilterSetupnDPIProtocol; + sigmatch_table[DETECT_NDPI_PROTOCOL].SupportsPrefilter = PrefilternDPIProtocolIsPrefilterable; } /**********************************Unittests***********************************/ @@ -295,7 +285,7 @@ static int DetectnDPIProtocolTest03(void) de_ctx->flags |= DE_QUIET; s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " - "(ndpi-protocol:HTTP; sid:1;)"); + "(ndpi-protocol:HTTP; sid:1;)"); FAIL_IF_NULL(s); FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); @@ -317,7 +307,7 @@ static int DetectnDPIProtocolTest04(void) de_ctx->flags |= DE_QUIET; s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " - "(ndpi-protocol:!HTTP; sid:1;)"); + "(ndpi-protocol:!HTTP; sid:1;)"); FAIL_IF_NULL(s); FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); @@ -334,14 +324,10 @@ static int DetectnDPIProtocolTest04(void) static void DetectnDPIProtocolRegisterTests(void) { - UtRegisterTest("DetectnDPIProtocolTest01", - DetectnDPIProtocolTest01); - UtRegisterTest("DetectnDPIProtocolTest02", - DetectnDPIProtocolTest02); - UtRegisterTest("DetectnDPIProtocolTest03", - DetectnDPIProtocolTest03); - UtRegisterTest("DetectnDPIProtocolTest04", - DetectnDPIProtocolTest04); + UtRegisterTest("DetectnDPIProtocolTest01", DetectnDPIProtocolTest01); + UtRegisterTest("DetectnDPIProtocolTest02", DetectnDPIProtocolTest02); + UtRegisterTest("DetectnDPIProtocolTest03", DetectnDPIProtocolTest03); + UtRegisterTest("DetectnDPIProtocolTest04", DetectnDPIProtocolTest04); } #endif /* UNITTESTS */ diff --git a/src/detect-ndpi-risk.c b/src/detect-ndpi-risk.c index d2b81e572a18..bcbf6caa925d 100644 --- a/src/detect-ndpi-risk.c +++ b/src/detect-ndpi-risk.c @@ -44,8 +44,8 @@ typedef struct DetectnDPIRiskData_ { uint8_t negated; } DetectnDPIRiskData; -static int DetectnDPIRiskPacketMatch(DetectEngineThreadCtx *det_ctx, - Packet *p, const Signature *s, const SigMatchCtx *ctx) +static int DetectnDPIRiskPacketMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) { SCEnter(); @@ -53,13 +53,13 @@ static int DetectnDPIRiskPacketMatch(DetectEngineThreadCtx *det_ctx, const DetectnDPIRiskData *data = (const DetectnDPIRiskData *)ctx; if (!p->flow->detection_completed) { - SCLogDebug("packet %"PRIu64": ndpi risks not yet detected", p->pcap_cnt); + SCLogDebug("packet %" PRIu64 ": ndpi risks not yet detected", p->pcap_cnt); SCReturnInt(0); } const Flow *f = p->flow; if (f == NULL) { - SCLogDebug("packet %"PRIu64": no flow", p->pcap_cnt); + SCLogDebug("packet %" PRIu64 ": no flow", p->pcap_cnt); SCReturnInt(0); } @@ -67,9 +67,8 @@ static int DetectnDPIRiskPacketMatch(DetectEngineThreadCtx *det_ctx, r = r ^ data->negated; if (r) { - SCLogDebug("ndpi risks match on risk bitmap = %"PRIu64" (matching bitmap %"PRIu64")", - f->ndpi_flow->risk, - data->risk_mask); + SCLogDebug("ndpi risks match on risk bitmap = %" PRIu64 " (matching bitmap %" PRIu64 ")", + f->ndpi_flow->risk, data->risk_mask); SCReturnInt(1); } @@ -82,7 +81,7 @@ static DetectnDPIRiskData *DetectnDPIRiskParse(const char *arg, bool negate) struct ndpi_detection_module_struct *ndpi_struct; ndpi_risk risk_mask; NDPI_PROTOCOL_BITMASK all; - + /* convert list of risk names (string) to mask */ ndpi_struct = ndpi_init_detection_module(NULL); if (unlikely(ndpi_struct == NULL)) @@ -102,19 +101,19 @@ static DetectnDPIRiskData *DetectnDPIRiskParse(const char *arg, bool negate) if (dup != NULL) { token = strtok_r(dup, ",", &tmp); - + while (token != NULL) { ndpi_risk_enum risk_id = ndpi_code2risk(token); if (risk_id >= NDPI_MAX_RISK) { SCLogError("unrecognized risk '%s', " "please check ndpiReader -H for valid risk codes", - token); + token); return NULL; } NDPI_SET_BIT(risk_mask, risk_id); token = strtok_r(NULL, ",", &tmp); } - + SCFree(dup); } } @@ -147,7 +146,7 @@ static int DetectnDPIRiskSetup(DetectEngineCtx *de_ctx, Signature *s, const char goto error; SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; - for ( ; tsm != NULL; tsm = tsm->next) { + for (; tsm != NULL; tsm = tsm->next) { if (tsm->type == DETECT_NDPI_RISK) { const DetectnDPIRiskData *them = (const DetectnDPIRiskData *)tsm->ctx; @@ -159,13 +158,13 @@ static int DetectnDPIRiskSetup(DetectEngineCtx *de_ctx, Signature *s, const char } } - if (SigMatchAppendSMToList(de_ctx, s, DETECT_NDPI_RISK, (SigMatchCtx *)data, - DETECT_SM_LIST_MATCH) == NULL) { + if (SigMatchAppendSMToList( + de_ctx, s, DETECT_NDPI_RISK, (SigMatchCtx *)data, DETECT_SM_LIST_MATCH) == NULL) { goto error; } return 0; - error: +error: if (data != NULL) SCFree(data); return -1; @@ -179,28 +178,27 @@ static void DetectnDPIRiskFree(DetectEngineCtx *de_ctx, void *ptr) /** \internal * \brief prefilter function for risk matching */ -static void -PrefilterPacketnDPIRiskMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +static void PrefilterPacketnDPIRiskMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) { const PrefilterPacketHeaderCtx *ctx = pectx; if (p->flow == NULL || !p->flow->detection_completed) { - SCLogDebug("packet %"PRIu64": no flow, no ndpi detection", p->pcap_cnt); + SCLogDebug("packet %" PRIu64 ": no flow, no ndpi detection", p->pcap_cnt); SCReturn; } Flow *f = p->flow; bool negated = (bool)ctx->v1.u8[9]; ndpi_risk risk_mask = ctx->v1.u64[0]; - bool ret = ((f->ndpi_flow->risk & risk_mask) == risk_mask) ? true: false; - + bool ret = ((f->ndpi_flow->risk & risk_mask) == risk_mask) ? true : false; + if (ret ^ negated) { PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); - } + } } -static void -PrefilterPacketnDPIRiskSet(PrefilterPacketHeaderValue *v, void *smctx) +static void PrefilterPacketnDPIRiskSet(PrefilterPacketHeaderValue *v, void *smctx) { const DetectnDPIRiskData *a = smctx; @@ -208,25 +206,22 @@ PrefilterPacketnDPIRiskSet(PrefilterPacketHeaderValue *v, void *smctx) v->u8[9] = (uint8_t)a->negated; } -static bool -PrefilterPacketnDPIRiskCompare(PrefilterPacketHeaderValue v, void *smctx) +static bool PrefilterPacketnDPIRiskCompare(PrefilterPacketHeaderValue v, void *smctx) { const DetectnDPIRiskData *a = smctx; ndpi_risk p = v.u64[0]; bool negated = (bool)v.u8[9]; bool ret; - - ret = ((a->risk_mask & p) == p) ? true: false; + + ret = ((a->risk_mask & p) == p) ? true : false; return (ret ^ negated); } static int PrefilterSetupnDPIRisk(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { - return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_RISK, - SIG_MASK_REQUIRE_FLOW, - PrefilterPacketnDPIRiskSet, - PrefilterPacketnDPIRiskCompare, - PrefilterPacketnDPIRiskMatch); + return PrefilterSetupPacketHeader(de_ctx, sgh, DETECT_NDPI_RISK, SIG_MASK_REQUIRE_FLOW, + PrefilterPacketnDPIRiskSet, PrefilterPacketnDPIRiskCompare, + PrefilterPacketnDPIRiskMatch); } static bool PrefilternDPIRiskIsPrefilterable(const Signature *s) @@ -247,16 +242,12 @@ void DetectnDPIRiskRegister(void) sigmatch_table[DETECT_NDPI_RISK].Setup = DetectnDPIRiskSetup; sigmatch_table[DETECT_NDPI_RISK].Free = DetectnDPIRiskFree; #ifdef UNITTESTS - sigmatch_table[DETECT_NDPI_RISK].RegisterTests = - DetectnDPIRiskRegisterTests; + sigmatch_table[DETECT_NDPI_RISK].RegisterTests = DetectnDPIRiskRegisterTests; #endif - sigmatch_table[DETECT_NDPI_RISK].flags = - (SIGMATCH_QUOTES_OPTIONAL|SIGMATCH_HANDLE_NEGATION); + sigmatch_table[DETECT_NDPI_RISK].flags = (SIGMATCH_QUOTES_OPTIONAL | SIGMATCH_HANDLE_NEGATION); - sigmatch_table[DETECT_NDPI_RISK].SetupPrefilter = - PrefilterSetupnDPIRisk; - sigmatch_table[DETECT_NDPI_RISK].SupportsPrefilter = - PrefilternDPIRiskIsPrefilterable; + sigmatch_table[DETECT_NDPI_RISK].SetupPrefilter = PrefilterSetupnDPIRisk; + sigmatch_table[DETECT_NDPI_RISK].SupportsPrefilter = PrefilternDPIRiskIsPrefilterable; } /**********************************Unittests***********************************/ @@ -292,7 +283,7 @@ static int DetectnDPIRiskTest03(void) de_ctx->flags |= DE_QUIET; s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " - "(ndpi-risk:NDPI_PROBING_ATTEMPT; sid:1;)"); + "(ndpi-risk:NDPI_PROBING_ATTEMPT; sid:1;)"); FAIL_IF_NULL(s); FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); @@ -314,7 +305,7 @@ static int DetectnDPIRiskTest04(void) de_ctx->flags |= DE_QUIET; s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " - "(ndpi-risk:!NDPI_PROBING_ATTEMPT; sid:1;)"); + "(ndpi-risk:!NDPI_PROBING_ATTEMPT; sid:1;)"); FAIL_IF_NULL(s); FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]); diff --git a/src/flow.c b/src/flow.c index 4cf995df06ab..5ca552a16bc8 100644 --- a/src/flow.c +++ b/src/flow.c @@ -388,37 +388,36 @@ void FlowHandlePacketUpdate(Flow *f, Packet *p, ThreadVars *tv, DecodeThreadVars SCLogDebug("packet %"PRIu64" -- flow %p", p->pcap_cnt, f); #ifdef HAVE_NDPI - if (tv->ndpi_struct && - f->ndpi_flow && - (!f->detection_completed) && - (p->ip_len > 0)) { - uint64_t time_ms = ((uint64_t) p->ts.secs) * 1000 /* TICK_RESOLUTION */ + p->ts.usecs / (1000000 / 1000 /* TICK_RESOLUTION */); + if (tv->ndpi_struct && f->ndpi_flow && (!f->detection_completed) && (p->ip_len > 0)) { + uint64_t time_ms = ((uint64_t)p->ts.secs) * 1000 /* TICK_RESOLUTION */ + + p->ts.usecs / (1000000 / 1000 /* TICK_RESOLUTION */); f->detected_l7_protocol = ndpi_detection_process_packet(tv->ndpi_struct, f->ndpi_flow, - PacketIsIPv4(p) ? (void*)PacketGetIPv4(p) : (void*)PacketGetIPv6(p), - p->ip_len, time_ms, NULL); + PacketIsIPv4(p) ? (void *)PacketGetIPv4(p) : (void *)PacketGetIPv6(p), p->ip_len, + time_ms, NULL); if (ndpi_is_protocol_detected(f->detected_l7_protocol) != 0) { - if(!ndpi_is_proto_unknown(f->detected_l7_protocol.proto)) { - if(!ndpi_extra_dissection_possible(tv->ndpi_struct, f->ndpi_flow)) - f->detection_completed = 1; - } + if (!ndpi_is_proto_unknown(f->detected_l7_protocol.proto)) { + if (!ndpi_extra_dissection_possible(tv->ndpi_struct, f->ndpi_flow)) + f->detection_completed = 1; + } } else { u_int16_t max_num_pkts = (f->proto == IPPROTO_UDP) ? 8 : 24; - if ((f->todstpktcnt+f->tosrcpktcnt) > max_num_pkts) { + if ((f->todstpktcnt + f->tosrcpktcnt) > max_num_pkts) { u_int8_t proto_guessed; - f->detected_l7_protocol = ndpi_detection_giveup(tv->ndpi_struct, f->ndpi_flow, &proto_guessed); + f->detected_l7_protocol = + ndpi_detection_giveup(tv->ndpi_struct, f->ndpi_flow, &proto_guessed); f->detection_completed = 1; } } if (f->detection_completed) { SCLogDebug("Detected protocol: %s | app protocol: %s | category: %s", - ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.master_protocol), - ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.app_protocol), - ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.master_protocol), + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.app_protocol), + ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); } } #endif diff --git a/src/output-filestore.c b/src/output-filestore.c index ff23accd09a8..0c8263f9786a 100644 --- a/src/output-filestore.c +++ b/src/output-filestore.c @@ -169,9 +169,10 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv, const OutputFilestoreLo JsonBuilder *js_fileinfo = JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL #ifdef HAVE_NDPI - , tv + , + tv #endif - ); + ); if (likely(js_fileinfo != NULL)) { jb_close(js_fileinfo); FILE *out = fopen(js_metadata_filename, "w"); diff --git a/src/output-json-file.c b/src/output-json-file.c index f17d698593cd..07abf20d748a 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -94,26 +94,27 @@ static void ndpiJsonBuilderTLSQUIC(Flow *f, JsonBuilder *js, ThreadVars *tv) return; } - ndpi_ssl_version2str(version, sizeof(version), f->ndpi_flow->protos.tls_quic.ssl_version, &unknown_tls_version); + ndpi_ssl_version2str(version, sizeof(version), f->ndpi_flow->protos.tls_quic.ssl_version, + &unknown_tls_version); if (f->ndpi_flow->protos.tls_quic.notBefore) before = ndpi_gmtime_r((const time_t *)&f->ndpi_flow->protos.tls_quic.notBefore, &a); - + if (f->ndpi_flow->protos.tls_quic.notAfter) after = ndpi_gmtime_r((const time_t *)&f->ndpi_flow->protos.tls_quic.notAfter, &b); - + if (!unknown_tls_version) { jb_open_object(js, "tls"); jb_set_string(js, "version", version); if (f->ndpi_flow->protos.tls_quic.server_names) jb_set_string(js, "server_names", f->ndpi_flow->protos.tls_quic.server_names); - + if (before) { strftime(notBefore, sizeof(notBefore), "%Y-%m-%d %H:%M:%S", before); jb_set_string(js, "notbefore", notBefore); } - + if (after) { strftime(notAfter, sizeof(notAfter), "%Y-%m-%d %H:%M:%S", after); jb_set_string(js, "notafter", notAfter); @@ -121,147 +122,155 @@ static void ndpiJsonBuilderTLSQUIC(Flow *f, JsonBuilder *js, ThreadVars *tv) /* Note: ja3, ja3s, ja4 are not serialized as as Suricata already supports them */ - jb_set_uint(js,"unsafe_cipher", f->ndpi_flow->protos.tls_quic.server_unsafe_cipher); - jb_set_string(js, "cipher", ndpi_cipher2str(f->ndpi_flow->protos.tls_quic.server_cipher, unknown_cipher)); + jb_set_uint(js, "unsafe_cipher", f->ndpi_flow->protos.tls_quic.server_unsafe_cipher); + jb_set_string(js, "cipher", + ndpi_cipher2str(f->ndpi_flow->protos.tls_quic.server_cipher, unknown_cipher)); if (f->ndpi_flow->protos.tls_quic.issuerDN) jb_set_string(js, "issuerDN", f->ndpi_flow->protos.tls_quic.issuerDN); - + if (f->ndpi_flow->protos.tls_quic.subjectDN) jb_set_string(js, "subjectDN", f->ndpi_flow->protos.tls_quic.subjectDN); - + if (f->ndpi_flow->protos.tls_quic.advertised_alpns) jb_set_string(js, "advertised_alpns", f->ndpi_flow->protos.tls_quic.advertised_alpns); - + if (f->ndpi_flow->protos.tls_quic.negotiated_alpn) jb_set_string(js, "negotiated_alpn", f->ndpi_flow->protos.tls_quic.negotiated_alpn); if (f->ndpi_flow->protos.tls_quic.tls_supported_versions) - jb_set_string(js, "tls_supported_versions", f->ndpi_flow->protos.tls_quic.tls_supported_versions); + jb_set_string(js, "tls_supported_versions", + f->ndpi_flow->protos.tls_quic.tls_supported_versions); if (f->ndpi_flow->protos.tls_quic.sha1_certificate_fingerprint[0] != '\0') { - for(i=0, off=0; i<20; i++) { - int rc = ndpi_snprintf(&buf[off], sizeof(buf)-off,"%s%02X", (i > 0) ? ":" : "", - f->ndpi_flow->protos.tls_quic.sha1_certificate_fingerprint[i] & 0xFF); - if (rc <= 0) break; else off += rc; + for (i = 0, off = 0; i < 20; i++) { + int rc = ndpi_snprintf(&buf[off], sizeof(buf) - off, "%s%02X", (i > 0) ? ":" : "", + f->ndpi_flow->protos.tls_quic.sha1_certificate_fingerprint[i] & 0xFF); + if (rc <= 0) + break; + else + off += rc; } jb_set_string(js, "fingerprint", buf); } - jb_set_uint(js,"blocks", f->ndpi_flow->l4.tcp.tls.num_tls_blocks); + jb_set_uint(js, "blocks", f->ndpi_flow->l4.tcp.tls.num_tls_blocks); jb_close(js); } } /* Suricata backport of ndpi_dpi2json */ -void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) +void ndpiJsonBuilder(Flow *f, JsonBuilder *js, ThreadVars *tv) { char buf[64]; char const *host_server_name; char quic_version[16]; ndpi_protocol l7_protocol; - - if (f == NULL) return; - + + if (f == NULL) + return; + jb_open_object(js, "ndpi"); - jb_set_string(js, "app_protocol", ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.app_protocol)); - jb_set_string(js, "master_protocol", ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.master_protocol)); - jb_set_string(js, "category", ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); - + jb_set_string(js, "app_protocol", + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.app_protocol)); + jb_set_string(js, "master_protocol", + ndpi_get_proto_name(tv->ndpi_struct, f->detected_l7_protocol.proto.master_protocol)); + jb_set_string(js, "category", + ndpi_category_get_name(tv->ndpi_struct, f->detected_l7_protocol.category)); + if (f->ndpi_flow->risk) { u_int risk_id; - + jb_open_array(js, "risks"); - + for (risk_id = 0; risk_id < NDPI_MAX_RISK; risk_id++) if (NDPI_ISSET_BIT(f->ndpi_flow->risk, risk_id)) { - char str[256]; - snprintf(str, sizeof(str), "%s (%s)", ndpi_risk2code(risk_id), ndpi_risk2str(risk_id)); - jb_append_string(js, str); - } - + char str[256]; + snprintf(str, sizeof(str), "%s (%s)", ndpi_risk2code(risk_id), + ndpi_risk2str(risk_id)); + jb_append_string(js, str); + } + jb_close(js); } /* risk */ - + host_server_name = ndpi_get_flow_info(f->ndpi_flow, &l7_protocol); if (host_server_name != NULL) { jb_set_string(js, "hostname", host_server_name); } - - switch (l7_protocol.proto.master_protocol ? l7_protocol.proto.master_protocol : l7_protocol.proto.app_protocol) { + + switch (l7_protocol.proto.master_protocol ? l7_protocol.proto.master_protocol + : l7_protocol.proto.app_protocol) { case NDPI_PROTOCOL_IP_ICMP: if (f->ndpi_flow->entropy > 0.0f) { char buf[64]; - snprintf(buf, sizeof(buf), "%.6f", f->ndpi_flow->entropy); + snprintf(buf, sizeof(buf), "%.6f", f->ndpi_flow->entropy); jb_set_string(js, "entropy", buf); } break; case NDPI_PROTOCOL_DHCP: - jb_open_object(js,"dhcp"); + jb_open_object(js, "dhcp"); jb_set_string(js, "fingerprint", f->ndpi_flow->protos.dhcp.fingerprint); jb_set_string(js, "class_ident", f->ndpi_flow->protos.dhcp.class_ident); jb_close(js); break; - case NDPI_PROTOCOL_BITTORRENT: - { - u_int i, j, n = 0; - char bittorent_hash[sizeof(f->ndpi_flow->protos.bittorrent.hash)*2+1]; - for (i = 0, j = 0; j < sizeof(bittorent_hash)-1; i++) { - snprintf(&bittorent_hash[j], - sizeof(bittorent_hash) - j, - "%02x", - f->ndpi_flow->protos.bittorrent.hash[i]); - - j += 2; - n += f->ndpi_flow->protos.bittorrent.hash[i]; - } - if (n == 0) bittorent_hash[0] = '\0'; - jb_open_object(js,"bittorrent"); - jb_set_string(js, "hash", bittorent_hash); - jb_close(js); + case NDPI_PROTOCOL_BITTORRENT: { + u_int i, j, n = 0; + char bittorent_hash[sizeof(f->ndpi_flow->protos.bittorrent.hash) * 2 + 1]; + for (i = 0, j = 0; j < sizeof(bittorent_hash) - 1; i++) { + snprintf(&bittorent_hash[j], sizeof(bittorent_hash) - j, "%02x", + f->ndpi_flow->protos.bittorrent.hash[i]); + + j += 2; + n += f->ndpi_flow->protos.bittorrent.hash[i]; } - break; + if (n == 0) + bittorent_hash[0] = '\0'; + jb_open_object(js, "bittorrent"); + jb_set_string(js, "hash", bittorent_hash); + jb_close(js); + } break; case NDPI_PROTOCOL_COLLECTD: - jb_open_object(js,"collectd"); + jb_open_object(js, "collectd"); jb_set_string(js, "client_username", f->ndpi_flow->protos.collectd.client_username); jb_close(js); break; case NDPI_PROTOCOL_DNS: - jb_open_object(js,"dns"); + jb_open_object(js, "dns"); jb_set_uint(js, "num_queries", f->ndpi_flow->protos.dns.num_queries); jb_set_uint(js, "num_answers", f->ndpi_flow->protos.dns.num_answers); - jb_set_uint(js, "reply_code", f->ndpi_flow->protos.dns.reply_code); - jb_set_uint(js, "query_type", f->ndpi_flow->protos.dns.query_type); - jb_set_uint(js, "rsp_type", f->ndpi_flow->protos.dns.rsp_type); + jb_set_uint(js, "reply_code", f->ndpi_flow->protos.dns.reply_code); + jb_set_uint(js, "query_type", f->ndpi_flow->protos.dns.query_type); + jb_set_uint(js, "rsp_type", f->ndpi_flow->protos.dns.rsp_type); inet_ntop(AF_INET, &f->ndpi_flow->protos.dns.rsp_addr, buf, sizeof(buf)); jb_set_string(js, "rsp_addr", buf); jb_close(js); break; case NDPI_PROTOCOL_NTP: - jb_open_object(js,"ntp"); + jb_open_object(js, "ntp"); jb_set_uint(js, "request_code", f->ndpi_flow->protos.ntp.request_code); jb_set_uint(js, "version", f->ndpi_flow->protos.ntp.request_code); jb_close(js); break; - case NDPI_PROTOCOL_MDNS: - jb_open_object(js,"mdns"); + case NDPI_PROTOCOL_MDNS: + jb_open_object(js, "mdns"); jb_close(js); break; - case NDPI_PROTOCOL_UBNTAC2: - jb_open_object(js,"ubntac2"); + case NDPI_PROTOCOL_UBNTAC2: + jb_open_object(js, "ubntac2"); jb_set_string(js, "version", f->ndpi_flow->protos.ubntac2.version); jb_close(js); break; case NDPI_PROTOCOL_KERBEROS: - jb_open_object(js,"kerberos"); + jb_open_object(js, "kerberos"); jb_set_string(js, "hostname", f->ndpi_flow->protos.kerberos.hostname); jb_set_string(js, "domain", f->ndpi_flow->protos.kerberos.domain); jb_set_string(js, "username", f->ndpi_flow->protos.kerberos.username); jb_close(js); break; case NDPI_PROTOCOL_SOFTETHER: - jb_open_object(js,"softether"); + jb_open_object(js, "softether"); jb_set_string(js, "client_ip", f->ndpi_flow->protos.softether.ip); jb_set_string(js, "client_port", f->ndpi_flow->protos.softether.port); jb_set_string(js, "hostname", f->ndpi_flow->protos.softether.hostname); @@ -269,41 +278,42 @@ void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) jb_close(js); break; case NDPI_PROTOCOL_NATPMP: - jb_open_object(js,"natpmp"); + jb_open_object(js, "natpmp"); jb_set_uint(js, "result", f->ndpi_flow->protos.natpmp.result_code); jb_set_uint(js, "internal_port", f->ndpi_flow->protos.natpmp.internal_port); jb_set_uint(js, "external_port", f->ndpi_flow->protos.natpmp.external_port); - inet_ntop(AF_INET, &f->ndpi_flow->protos.natpmp.external_address.ipv4, buf, sizeof(buf)); + inet_ntop( + AF_INET, &f->ndpi_flow->protos.natpmp.external_address.ipv4, buf, sizeof(buf)); jb_set_string(js, "external_address", buf); jb_close(js); break; case NDPI_PROTOCOL_RSH: - jb_open_object(js,"rsh"); + jb_open_object(js, "rsh"); jb_set_string(js, "client_username", f->ndpi_flow->protos.rsh.client_username); jb_set_string(js, "server_username", f->ndpi_flow->protos.rsh.server_username); jb_set_string(js, "command", f->ndpi_flow->protos.rsh.command); jb_close(js); break; case NDPI_PROTOCOL_SNMP: - jb_open_object(js,"snmp"); + jb_open_object(js, "snmp"); jb_set_uint(js, "version", f->ndpi_flow->protos.snmp.version); jb_set_uint(js, "primitive", f->ndpi_flow->protos.snmp.primitive); jb_set_uint(js, "error_status", f->ndpi_flow->protos.snmp.error_status); jb_close(js); break; case NDPI_PROTOCOL_TELNET: - jb_open_object(js,"telnet"); + jb_open_object(js, "telnet"); jb_set_string(js, "username", f->ndpi_flow->protos.telnet.username); jb_set_string(js, "password", f->ndpi_flow->protos.telnet.password); jb_close(js); break; case NDPI_PROTOCOL_TFTP: - jb_open_object(js,"tftp"); + jb_open_object(js, "tftp"); jb_set_string(js, "filename", f->ndpi_flow->protos.tftp.filename); jb_close(js); break; case NDPI_PROTOCOL_TIVOCONNECT: - jb_open_object(js,"tivoconnect"); + jb_open_object(js, "tivoconnect"); jb_set_string(js, "identity_uuid", f->ndpi_flow->protos.tivoconnect.identity_uuid); jb_set_string(js, "machine", f->ndpi_flow->protos.tivoconnect.machine); jb_set_string(js, "platform", f->ndpi_flow->protos.tivoconnect.platform); @@ -313,7 +323,7 @@ void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) case NDPI_PROTOCOL_HTTP: case NDPI_PROTOCOL_HTTP_CONNECT: case NDPI_PROTOCOL_HTTP_PROXY: - jb_open_object(js,"http"); + jb_open_object(js, "http"); if (f->ndpi_flow->http.url != NULL) { jb_set_string(js, "url", f->ndpi_flow->http.url); jb_set_uint(js, "code", f->ndpi_flow->http.response_status_code); @@ -321,56 +331,50 @@ void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) jb_set_string(js, "user_agent", f->ndpi_flow->http.user_agent); } if (f->ndpi_flow->http.request_content_type != NULL) { - jb_set_string(js, "request_content_type", - f->ndpi_flow->http.request_content_type); + jb_set_string(js, "request_content_type", f->ndpi_flow->http.request_content_type); } if (f->ndpi_flow->http.detected_os != NULL) { - jb_set_string(js, "detected_os", - f->ndpi_flow->http.detected_os); + jb_set_string(js, "detected_os", f->ndpi_flow->http.detected_os); } if (f->ndpi_flow->http.nat_ip != NULL) { - jb_set_string(js, "nat_ip", - f->ndpi_flow->http.nat_ip); + jb_set_string(js, "nat_ip", f->ndpi_flow->http.nat_ip); } jb_close(js); break; case NDPI_PROTOCOL_QUIC: - jb_open_object(js,"quic"); + jb_open_object(js, "quic"); if (f->ndpi_flow->http.user_agent) { jb_set_string(js, "user_agent", f->ndpi_flow->http.user_agent); } - ndpi_quic_version2str(quic_version, sizeof(quic_version), - f->ndpi_flow->protos.tls_quic.quic_version); + ndpi_quic_version2str( + quic_version, sizeof(quic_version), f->ndpi_flow->protos.tls_quic.quic_version); jb_set_string(js, "quic_version", quic_version); ndpiJsonBuilderTLSQUIC(f, js, tv); jb_close(js); break; case NDPI_PROTOCOL_MAIL_IMAP: - jb_open_object(js,"imap"); + jb_open_object(js, "imap"); jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); - jb_set_uint(js, "auth_failed", - f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); jb_close(js); break; case NDPI_PROTOCOL_MAIL_POP: - jb_open_object(js,"pop"); + jb_open_object(js, "pop"); jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); - jb_set_uint(js, "auth_failed", - f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); jb_close(js); break; case NDPI_PROTOCOL_MAIL_SMTP: - jb_open_object(js,"smtp"); + jb_open_object(js, "smtp"); jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); - jb_set_uint(js, "auth_failed", - f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); + jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); jb_close(js); break; case NDPI_PROTOCOL_FTP_CONTROL: - jb_open_object(js,"ftp"); + jb_open_object(js, "ftp"); jb_set_string(js, "user", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username); jb_set_string(js, "password", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password); jb_set_uint(js, "auth_failed", f->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); @@ -378,9 +382,9 @@ void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) break; case NDPI_PROTOCOL_DISCORD: if (l7_protocol.proto.master_protocol != NDPI_PROTOCOL_TLS) { - jb_open_object(js,"discord"); - jb_set_string(js, "client_ip", f->ndpi_flow->protos.discord.client_ip); - jb_close(js); + jb_open_object(js, "discord"); + jb_set_string(js, "client_ip", f->ndpi_flow->protos.discord.client_ip); + jb_close(js); } break; case NDPI_PROTOCOL_SSH: @@ -392,21 +396,31 @@ void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) jb_close(js); break; case NDPI_PROTOCOL_STUN: - jb_open_object(js,"stun"); + jb_open_object(js, "stun"); if (f->ndpi_flow->stun.mapped_address.port) { - jb_set_string(js, "mapped_address", print_ndpi_address_port(&f->ndpi_flow->stun.mapped_address, buf, sizeof(buf))); + jb_set_string(js, "mapped_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.mapped_address, buf, sizeof(buf))); } if (f->ndpi_flow->stun.peer_address.port) { - jb_set_string(js, "peer_address", print_ndpi_address_port(&f->ndpi_flow->stun.peer_address, buf, sizeof(buf))); + jb_set_string(js, "peer_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.peer_address, buf, sizeof(buf))); } if (f->ndpi_flow->stun.relayed_address.port) { - jb_set_string(js, "relayed_address", print_ndpi_address_port(&f->ndpi_flow->stun.relayed_address, buf, sizeof(buf))); + jb_set_string(js, "relayed_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.relayed_address, buf, sizeof(buf))); } if (f->ndpi_flow->stun.response_origin.port) { - jb_set_string(js, "response_origin", print_ndpi_address_port(&f->ndpi_flow->stun.response_origin, buf, sizeof(buf))); + jb_set_string(js, "response_origin", + print_ndpi_address_port( + &f->ndpi_flow->stun.response_origin, buf, sizeof(buf))); } if (f->ndpi_flow->stun.other_address.port) { - jb_set_string(js, "other_address", print_ndpi_address_port(&f->ndpi_flow->stun.other_address, buf, sizeof(buf))); + jb_set_string(js, "other_address", + print_ndpi_address_port( + &f->ndpi_flow->stun.other_address, buf, sizeof(buf))); } jb_close(js); break; @@ -421,11 +435,12 @@ void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv) #endif -JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, - OutputJsonCtx *eve_ctx +JsonBuilder *JsonBuildFileInfoRecord( + const Packet *p, const File *ff, void *tx, const uint64_t tx_id, const bool stored, + uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx #ifdef HAVE_NDPI - , ThreadVars *tv + , + ThreadVars *tv #endif ) { @@ -566,7 +581,8 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx, const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx #ifdef HAVE_NDPI - , ThreadVars *tv + , + ThreadVars *tv #endif ) { @@ -574,7 +590,8 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F : aft->filelog_ctx->parent_xff_cfg; JsonBuilder *js = JsonBuildFileInfoRecord(p, ff, tx, tx_id, false, dir, xff_cfg, eve_ctx #ifdef HAVE_NDPI - , tv + , + tv #endif ); @@ -598,7 +615,8 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, co FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx #ifdef HAVE_NDPI - , tv + , + tv #endif ); diff --git a/src/output-json-file.h b/src/output-json-file.h index f51d919ca139..0603666124a9 100644 --- a/src/output-json-file.h +++ b/src/output-json-file.h @@ -29,11 +29,12 @@ typedef struct OutputJsonCtx_ OutputJsonCtx; void JsonFileLogRegister(void); -JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, - OutputJsonCtx *eve_ctx +JsonBuilder *JsonBuildFileInfoRecord( + const Packet *p, const File *ff, void *tx, const uint64_t tx_id, const bool stored, + uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx #ifdef HAVE_NDPI - , ThreadVars *tv + , + ThreadVars *tv #endif ); diff --git a/src/output-json-flow.c b/src/output-json-flow.c index d5d2f1ab255f..1d469e9a1687 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -217,12 +217,13 @@ void EveAddFlow(Flow *f, JsonBuilder *js) /* Eve format logging */ static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f #ifdef HAVE_NDPI - , ThreadVars *tv + , + ThreadVars *tv #endif ) { #ifdef HAVE_NDPI - ndpiJsonBuilder(f, jb, tv); + ndpiJsonBuilder(f, jb, tv); #endif EveAddAppProto(f, jb); jb_open_object(jb, "flow"); @@ -346,7 +347,8 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) EveFlowLogJSON(thread, jb, f #ifdef HAVE_NDPI - , tv + , + tv #endif ); diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index a5d6a4261741..ade0fbdd4de8 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -177,12 +177,13 @@ static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir) /* JSON format logging */ static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f #ifdef HAVE_NDPI - , ThreadVars *tv + , + ThreadVars *tv #endif ) { #ifdef HAVE_NDPI - ndpiJsonBuilder(f, js, tv); + ndpiJsonBuilder(f, js, tv); #endif jb_set_string(js, "app_proto", @@ -229,12 +230,13 @@ static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f #ifdef HAVE_NDPI - , ThreadVars *tv + , + ThreadVars *tv #endif ) { #ifdef HAVE_NDPI - ndpiJsonBuilder(f, js, tv); + ndpiJsonBuilder(f, js, tv); #endif jb_set_string(js, "app_proto", @@ -292,7 +294,8 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToServer(jb, f #ifdef HAVE_NDPI - , tv + , + tv #endif ); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER); @@ -306,7 +309,8 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToClient(jb, f #ifdef HAVE_NDPI - , tv + , + tv #endif ); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT); diff --git a/src/output-json.h b/src/output-json.h index 283fc752badd..5430882d69c2 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -121,6 +121,6 @@ void FreeEveThreadCtx(OutputJsonThreadCtx *ctx); void JSONFormatAndAddMACAddr(JsonBuilder *js, const char *key, const uint8_t *val, bool is_array); #ifdef HAVE_NDPI -void ndpiJsonBuilder (Flow *f, JsonBuilder *js, ThreadVars *tv); +void ndpiJsonBuilder(Flow *f, JsonBuilder *js, ThreadVars *tv); #endif #endif /* SURICATA_OUTPUT_JSON_H */ diff --git a/src/threadvars.h b/src/threadvars.h index 252ef0d2b26d..b2cdca2cad4c 100644 --- a/src/threadvars.h +++ b/src/threadvars.h @@ -136,7 +136,7 @@ typedef struct ThreadVars_ { bool break_loop; #ifdef HAVE_NDPI - struct ndpi_detection_module_struct * ndpi_struct; + struct ndpi_detection_module_struct *ndpi_struct; #endif } ThreadVars; diff --git a/src/tm-threads.c b/src/tm-threads.c index 2bddb95c8094..3d7d1b4a0bd4 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -1614,9 +1614,9 @@ static void TmThreadFree(ThreadVars *tv) } #ifdef HAVE_NDPI - if(tv->ndpi_struct != NULL) { - ndpi_exit_detection_module(tv->ndpi_struct); - /* printf("%s - ndpi_exit_detection_module()\n", __FUNCTION__); */ + if (tv->ndpi_struct != NULL) { + ndpi_exit_detection_module(tv->ndpi_struct); + /* printf("%s - ndpi_exit_detection_module()\n", __FUNCTION__); */ } #endif From d8e237d7c135ba1c96ff8425f24dcbceb0710840 Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Wed, 28 Aug 2024 18:21:53 +0200 Subject: [PATCH 3/4] Add ndpi-protocol and ndpi-risk keywords documentation --- doc/userguide/rules/index.rst | 2 ++ doc/userguide/rules/ndpi-protocol.rst | 35 +++++++++++++++++++++++ doc/userguide/rules/ndpi-risk.rst | 41 +++++++++++++++++++++++++++ src/detect-ndpi-protocol.c | 2 +- src/detect-ndpi-risk.c | 2 +- 5 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 doc/userguide/rules/ndpi-protocol.rst create mode 100644 doc/userguide/rules/ndpi-risk.rst diff --git a/doc/userguide/rules/index.rst b/doc/userguide/rules/index.rst index c8b586fecaa7..b96e21ba03b0 100644 --- a/doc/userguide/rules/index.rst +++ b/doc/userguide/rules/index.rst @@ -38,6 +38,8 @@ Suricata Rules smtp-keywords websocket-keywords app-layer + ndpi-protocol + ndpi-risk xbits noalert thresholding diff --git a/doc/userguide/rules/ndpi-protocol.rst b/doc/userguide/rules/ndpi-protocol.rst new file mode 100644 index 000000000000..d354db64ea55 --- /dev/null +++ b/doc/userguide/rules/ndpi-protocol.rst @@ -0,0 +1,35 @@ +nDPI Protocol Keyword +===================== + +ndpi-protocol +------------- + +Match on the Layer-7 protocol detected by nDPI. + +This requires Suricata to be compiled with nDPI support: + +.. code-block:: console + + ./configure --enable-ndpi --with-ndpi=/home/user/nDPI + +Syntax:: + + ndpi-protocol:[!]; + +Where protocol is one of the application protocols detected by nDPI. +Plase check ndpiReader -H for the full list. +It is possible to specify the transport protocol, the application +protocol, or both (dot-separated). + +Examples:: + + ndpi-protocol:HTTP; + ndpi-protocol:!TLS; + ndpi-protocol:TLS.YouTube; + +Here is an example of a rule matching TLS traffic on port 53: + +.. container:: example-rule + + alert tcp any any -> any 53 (msg:"TLS traffic over DNS standard port"; ndpi-protocol:TLS; sid:1;) + diff --git a/doc/userguide/rules/ndpi-risk.rst b/doc/userguide/rules/ndpi-risk.rst new file mode 100644 index 000000000000..12f78386d862 --- /dev/null +++ b/doc/userguide/rules/ndpi-risk.rst @@ -0,0 +1,41 @@ +nDPI Risk Keyword +================= + +ndpi-risk +--------- + +Match on the flow risks detected by nDPI. Risks are potential issues detected +by nDPI during the packet dissection and include: + +- Known Proto on Non Std Port +- Binary App Transfer +- Self-signed Certificate +- Susp DGA Domain name +- Malware host contacted +- and many other... + +This requires Suricata to be compiled with nDPI support: + +.. code-block:: console + + ./configure --enable-ndpi --with-ndpi=/home/user/nDPI + +Syntax:: + + ndpi-risk:[!]; + +Where risk is one (or multiple comma-separated) of the risk codes supported by +nDPI (e.g. NDPI_BINARY_APPLICATION_TRANSFER). Please check ndpiReader -H for the +full list. + +Examples:: + + ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; + ndpi-risk:NDPI_TLS_OBSOLETE_VERSION,NDPI_TLS_WEAK_CIPHER; + +Here is an example of a rule matching HTTP traffic transferring a binary application: + +.. container:: example-rule + + alert tcp any any -> any any (msg:"Binary application transfer over HTTP"; ndpi-protocol:HTTP; ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; sid:1;) + diff --git a/src/detect-ndpi-protocol.c b/src/detect-ndpi-protocol.c index f77385327b12..dcb5b82779ca 100644 --- a/src/detect-ndpi-protocol.c +++ b/src/detect-ndpi-protocol.c @@ -238,7 +238,7 @@ void DetectnDPIProtocolRegister(void) { sigmatch_table[DETECT_NDPI_PROTOCOL].name = "ndpi-protocol"; sigmatch_table[DETECT_NDPI_PROTOCOL].desc = "match on the detected nDPI protocol"; - sigmatch_table[DETECT_NDPI_PROTOCOL].url = "/rules/index.html"; + sigmatch_table[DETECT_NDPI_PROTOCOL].url = "/rules/ndpi-protocol.html"; sigmatch_table[DETECT_NDPI_PROTOCOL].Match = DetectnDPIProtocolPacketMatch; sigmatch_table[DETECT_NDPI_PROTOCOL].Setup = DetectnDPIProtocolSetup; sigmatch_table[DETECT_NDPI_PROTOCOL].Free = DetectnDPIProtocolFree; diff --git a/src/detect-ndpi-risk.c b/src/detect-ndpi-risk.c index bcbf6caa925d..3de81b15e3cf 100644 --- a/src/detect-ndpi-risk.c +++ b/src/detect-ndpi-risk.c @@ -237,7 +237,7 @@ void DetectnDPIRiskRegister(void) { sigmatch_table[DETECT_NDPI_RISK].name = "ndpi-risk"; sigmatch_table[DETECT_NDPI_RISK].desc = "match on the detected nDPI risk"; - sigmatch_table[DETECT_NDPI_RISK].url = "/rules/index.html"; + sigmatch_table[DETECT_NDPI_RISK].url = "/rules/ndpi-risk.html"; sigmatch_table[DETECT_NDPI_RISK].Match = DetectnDPIRiskPacketMatch; sigmatch_table[DETECT_NDPI_RISK].Setup = DetectnDPIRiskSetup; sigmatch_table[DETECT_NDPI_RISK].Free = DetectnDPIRiskFree; From de33b1100591dcb444b82492cf1651bd7d9faabc Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Fri, 11 Oct 2024 17:57:28 +0200 Subject: [PATCH 4/4] Fix unit tests in detect-ndpi-risk --- src/detect-ndpi-risk.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/detect-ndpi-risk.c b/src/detect-ndpi-risk.c index 3de81b15e3cf..5f9a809f762d 100644 --- a/src/detect-ndpi-risk.c +++ b/src/detect-ndpi-risk.c @@ -258,7 +258,7 @@ static int DetectnDPIRiskTest01(void) { DetectnDPIRiskData *data = DetectnDPIRiskParse("NDPI_PROBING_ATTEMPT", false); FAIL_IF_NULL(data); - FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); FAIL_IF(data->negated != 0); DetectnDPIRiskFree(NULL, data); PASS; @@ -268,7 +268,7 @@ static int DetectnDPIRiskTest02(void) { DetectnDPIRiskData *data = DetectnDPIRiskParse("NDPI_PROBING_ATTEMPT", true); FAIL_IF_NULL(data); - FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); FAIL_IF(data->negated == 0); DetectnDPIRiskFree(NULL, data); PASS; @@ -290,7 +290,7 @@ static int DetectnDPIRiskTest03(void) FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx); data = (DetectnDPIRiskData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; - FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); FAIL_IF(data->negated); DetectEngineCtxFree(de_ctx); PASS; @@ -314,7 +314,7 @@ static int DetectnDPIRiskTest04(void) data = (DetectnDPIRiskData *)s->init_data->smlists[DETECT_SM_LIST_MATCH]->ctx; FAIL_IF_NULL(data); - FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT)); + FAIL_IF(!(NDPI_ISSET_BIT(data->risk_mask, NDPI_PROBING_ATTEMPT))); FAIL_IF(data->negated == 0); DetectEngineCtxFree(de_ctx);