From 995996076e1362e8606ca068cd2518fec50a8f8f Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Wed, 28 Dec 2022 18:16:54 +0100 Subject: [PATCH 01/12] dns proxy enable/disable config --- hogg-common/src/config.rs | 2 ++ hogg-daemon/src/dnsproxy.rs | 5 +++++ resources/config-template/hogg.toml | 1 + 3 files changed, 8 insertions(+) diff --git a/hogg-common/src/config.rs b/hogg-common/src/config.rs index 2b357db..714285d 100644 --- a/hogg-common/src/config.rs +++ b/hogg-common/src/config.rs @@ -13,6 +13,8 @@ pub struct HoggConfig { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct DnsProxyConfig { + #[serde(default)] + pub enabled: bool, pub bind: String, pub upstreams: Vec, } diff --git a/hogg-daemon/src/dnsproxy.rs b/hogg-daemon/src/dnsproxy.rs index b9f8f8a..5396f93 100644 --- a/hogg-daemon/src/dnsproxy.rs +++ b/hogg-daemon/src/dnsproxy.rs @@ -19,6 +19,11 @@ pub async fn dns_proxy_task( config: &HoggConfig, scan_function: FA + Send + 'static>, ) { + if !config.dnsproxy.enabled { + logs::warn!("DNS Proxy is disabled."); + return; + } + let socket = match UdpSocket::bind(config.dnsproxy.bind.clone()).await { Ok(socket) => socket, Err(e) => { diff --git a/resources/config-template/hogg.toml b/resources/config-template/hogg.toml index d7c0739..7adfeab 100644 --- a/resources/config-template/hogg.toml +++ b/resources/config-template/hogg.toml @@ -1,4 +1,5 @@ [dnsproxy] +enabled = true bind = "127.0.0.1:53" upstreams = ["1.1.1.1:53", "8.8.8.8:53"] From 52bad1dad7f33e8bcdc2b05047cf351402a91c45 Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Wed, 28 Dec 2022 18:21:48 +0100 Subject: [PATCH 02/12] grpc server (poc v1) --- Cargo.lock | 435 ++++++++++++++++++- Cargo.toml | 1 + hogg-daemon/Cargo.toml | 1 + hogg-daemon/src/main.rs | 17 +- hogg-grpc/Cargo.toml | 16 + hogg-grpc/build.rs | 6 + hogg-grpc/proto/daemon.proto | 14 + hogg-grpc/src/grpc/mod.rs | 28 ++ hogg-grpc/src/grpc/services/daemon_health.rs | 22 + hogg-grpc/src/grpc/services/mod.rs | 3 + hogg-grpc/src/lib.rs | 1 + 11 files changed, 540 insertions(+), 4 deletions(-) create mode 100644 hogg-grpc/Cargo.toml create mode 100644 hogg-grpc/build.rs create mode 100644 hogg-grpc/proto/daemon.proto create mode 100644 hogg-grpc/src/grpc/mod.rs create mode 100644 hogg-grpc/src/grpc/services/daemon_health.rs create mode 100644 hogg-grpc/src/grpc/services/mod.rs create mode 100644 hogg-grpc/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ae9fe79..cc3faf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,6 +103,27 @@ dependencies = [ "syn", ] +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-task" version = "4.3.0" @@ -126,6 +147,52 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08b108ad2665fa3f6e6a517c3d80ec3e77d224c47d605167aefaa5d7ef97fa48" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b8558f5a0581152dc94dcd289132a1d377494bdeafcd41869b3258e3e2ad92" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "base64" version = "0.13.1" @@ -376,6 +443,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -406,6 +479,20 @@ dependencies = [ "syn", ] +[[package]] +name = "err-derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34a887c8df3ed90498c1c437ce21f211c8e27672921a8ffa293cb8d6d4caa9e" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "rustversion", + "syn", + "synstructure", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -421,6 +508,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fnv" version = "1.0.7" @@ -574,6 +667,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.2.6" @@ -612,6 +711,7 @@ dependencies = [ "async-trait", "chrono", "hogg-common", + "hogg-grpc", "include_dir", "lazy_static", "logs", @@ -622,6 +722,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "hogg-grpc" +version = "0.1.0" +dependencies = [ + "err-derive", + "logs", + "prost", + "tokio", + "tonic", + "tonic-build", +] + [[package]] name = "http" version = "0.2.8" @@ -644,6 +756,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + [[package]] name = "httparse" version = "1.8.0" @@ -680,6 +798,18 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -772,6 +902,15 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.5" @@ -859,6 +998,12 @@ dependencies = [ "libc", ] +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "memchr" version = "2.5.0" @@ -892,6 +1037,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "native-tls" version = "0.2.11" @@ -1092,6 +1243,36 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1130,6 +1311,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8992a85d8e93a28bdf76137db888d3874e3b230dee5ed8bebac4c9f7617773" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "1.2.1" @@ -1141,6 +1332,30 @@ dependencies = [ "toml", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.49" @@ -1150,6 +1365,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c01db6702aa05baa3f57dec92b8eeeeb4cb19e894e73996b32a4093289e54592" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb5320c680de74ba083512704acb90fe00f28f79207286a848e730c45dd73ed6" +dependencies = [ + "bytes", + "heck 0.4.0", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8842bad1a5419bca14eac663ba798f6bc19c413c2fdceb5f3ba3b0932d96720" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "017f79637768cde62820bc2d4fe0e45daaa027755c323ad077767c6c5f173091" +dependencies = [ + "bytes", + "prost", +] + [[package]] name = "quick-xml" version = "0.23.1" @@ -1281,6 +1551,12 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustversion" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" + [[package]] name = "ryu" version = "1.0.12" @@ -1452,7 +1728,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "339f799d8b549e3744c7ac7feb216383e4005d94bdb22561b3ab8f3b808ae9fb" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro2", "quote", "syn", @@ -1469,6 +1745,24 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "tauri-winrt-notification" version = "0.1.0" @@ -1595,6 +1889,16 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "1.8.2" @@ -1616,6 +1920,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.4" @@ -1639,6 +1954,96 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -1652,6 +2057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1677,6 +2083,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "try-lock" version = "0.2.3" @@ -1732,6 +2148,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "url" version = "2.3.1" @@ -1868,6 +2290,17 @@ dependencies = [ "cc", ] +[[package]] +name = "which" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +dependencies = [ + "either", + "libc", + "once_cell", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index f9073d7..eef0013 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ "hogg-daemon", + "hogg-grpc", "hogg-common" ] diff --git a/hogg-daemon/Cargo.toml b/hogg-daemon/Cargo.toml index 15349be..0c385b2 100644 --- a/hogg-daemon/Cargo.toml +++ b/hogg-daemon/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] hogg-common = { path = "../hogg-common" } +hogg-grpc = { path = "../hogg-grpc" } anyhow = "1.0.68" include_dir = { version = "0.7.3", features = ["glob"] } diff --git a/hogg-daemon/src/main.rs b/hogg-daemon/src/main.rs index baf4857..e84b356 100644 --- a/hogg-daemon/src/main.rs +++ b/hogg-daemon/src/main.rs @@ -1,7 +1,8 @@ -use std::path::Path; +use std::{path::Path, sync::Arc}; use anyhow::Result; use dnsproxy::dns_proxy_task; +use hogg_grpc::grpc; use include_dir::{include_dir, Dir}; use hogg_common::{ @@ -43,6 +44,16 @@ async fn main() -> Result<()> { CONFIG = Some(config.clone()); } - dns_proxy_task(&config, scan_function).await; - Ok(()) + let config = Arc::new(config); + + { + let config = config.clone(); + tokio::spawn(async move { + dns_proxy_task(&config, scan_function).await; + }); + } + + grpc::tokio_serve_hogg_grpc()?; + + loop { } } diff --git a/hogg-grpc/Cargo.toml b/hogg-grpc/Cargo.toml new file mode 100644 index 0000000..fc28320 --- /dev/null +++ b/hogg-grpc/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "hogg-grpc" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +err-derive = "0.3.1" +logs = "0.7.1" +prost = "0.11.5" +tokio = { version = "1.23.0", features = ["macros", "rt-multi-thread"] } +tonic = "0.8.3" + +[build-dependencies] +tonic-build = "0.8" diff --git a/hogg-grpc/build.rs b/hogg-grpc/build.rs new file mode 100644 index 0000000..cfb2ab1 --- /dev/null +++ b/hogg-grpc/build.rs @@ -0,0 +1,6 @@ +fn main() { + const PROTO_FILE: &str = "proto/daemon.proto"; + + tonic_build::compile_protos(PROTO_FILE).unwrap(); + println!("cargo:rerun-if-changed={}", PROTO_FILE); +} diff --git a/hogg-grpc/proto/daemon.proto b/hogg-grpc/proto/daemon.proto new file mode 100644 index 0000000..22d0aee --- /dev/null +++ b/hogg-grpc/proto/daemon.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +package daemon; + +service DaemonHealth { + rpc Ping(PingRequest) returns (PingResponse) {} +} + +message PingRequest { + string message = 1; +} + +message PingResponse { + string message = 1; +} \ No newline at end of file diff --git a/hogg-grpc/src/grpc/mod.rs b/hogg-grpc/src/grpc/mod.rs new file mode 100644 index 0000000..c2d7886 --- /dev/null +++ b/hogg-grpc/src/grpc/mod.rs @@ -0,0 +1,28 @@ +pub mod services; + +pub mod daemon_proto { + tonic::include_proto!("daemon"); +} + +pub use daemon_proto::daemon_health_server::{DaemonHealth, DaemonHealthServer}; +pub use daemon_proto::{PingRequest, PingResponse}; + + +#[derive(Debug, err_derive::Error)] +pub enum Error { + #[error(display = "GRPC Transport Error")] + GrpcTransportError(#[error(source)] tonic::transport::Error), +} + +pub fn tokio_serve_hogg_grpc() -> Result>, Error> { + Ok(tokio::spawn(async move { + let addr = "[::1]:1396".parse().unwrap(); + let daemon_health = services::DaemonHealthService::default(); + + tonic::transport::Server::builder() + .add_service(DaemonHealthServer::new(daemon_health)) + .serve(addr) + .await + .map_err(Error::GrpcTransportError) + })) +} diff --git a/hogg-grpc/src/grpc/services/daemon_health.rs b/hogg-grpc/src/grpc/services/daemon_health.rs new file mode 100644 index 0000000..50e2170 --- /dev/null +++ b/hogg-grpc/src/grpc/services/daemon_health.rs @@ -0,0 +1,22 @@ +use tonic::{Request, Response, Status}; + +use crate::grpc::{DaemonHealth, PingRequest, PingResponse}; + +#[derive(Debug, Default)] +pub struct DaemonHealthService {} + +#[tonic::async_trait] +impl DaemonHealth for DaemonHealthService { + async fn ping( + &self, + request: Request, + ) -> Result, Status> { + logs::trace!("Got ping request: {:?}", request); + + let reply = PingResponse { + message: "Hello from the other side!".into(), + }; + + Ok(Response::new(reply)) + } +} \ No newline at end of file diff --git a/hogg-grpc/src/grpc/services/mod.rs b/hogg-grpc/src/grpc/services/mod.rs new file mode 100644 index 0000000..bd9e447 --- /dev/null +++ b/hogg-grpc/src/grpc/services/mod.rs @@ -0,0 +1,3 @@ +pub mod daemon_health; + +pub use daemon_health::DaemonHealthService; \ No newline at end of file diff --git a/hogg-grpc/src/lib.rs b/hogg-grpc/src/lib.rs new file mode 100644 index 0000000..31e18c8 --- /dev/null +++ b/hogg-grpc/src/lib.rs @@ -0,0 +1 @@ +pub mod grpc; \ No newline at end of file From 22f7889df6e6f28f4f1a0f7e244607fdb63f9533 Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Wed, 28 Dec 2022 18:25:37 +0100 Subject: [PATCH 03/12] rename daemonhealth to daemon (protobuf) --- hogg-grpc/proto/daemon.proto | 2 +- hogg-grpc/src/grpc/mod.rs | 6 +++--- hogg-grpc/src/grpc/services/{daemon_health.rs => daemon.rs} | 6 +++--- hogg-grpc/src/grpc/services/mod.rs | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) rename hogg-grpc/src/grpc/services/{daemon_health.rs => daemon.rs} (75%) diff --git a/hogg-grpc/proto/daemon.proto b/hogg-grpc/proto/daemon.proto index 22d0aee..4bfe0ca 100644 --- a/hogg-grpc/proto/daemon.proto +++ b/hogg-grpc/proto/daemon.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package daemon; -service DaemonHealth { +service Daemon { rpc Ping(PingRequest) returns (PingResponse) {} } diff --git a/hogg-grpc/src/grpc/mod.rs b/hogg-grpc/src/grpc/mod.rs index c2d7886..7b0b912 100644 --- a/hogg-grpc/src/grpc/mod.rs +++ b/hogg-grpc/src/grpc/mod.rs @@ -4,7 +4,7 @@ pub mod daemon_proto { tonic::include_proto!("daemon"); } -pub use daemon_proto::daemon_health_server::{DaemonHealth, DaemonHealthServer}; +pub use daemon_proto::daemon_server::{Daemon, DaemonServer}; pub use daemon_proto::{PingRequest, PingResponse}; @@ -17,10 +17,10 @@ pub enum Error { pub fn tokio_serve_hogg_grpc() -> Result>, Error> { Ok(tokio::spawn(async move { let addr = "[::1]:1396".parse().unwrap(); - let daemon_health = services::DaemonHealthService::default(); + let daemon_health = services::DaemonService::default(); tonic::transport::Server::builder() - .add_service(DaemonHealthServer::new(daemon_health)) + .add_service(DaemonServer::new(daemon_health)) .serve(addr) .await .map_err(Error::GrpcTransportError) diff --git a/hogg-grpc/src/grpc/services/daemon_health.rs b/hogg-grpc/src/grpc/services/daemon.rs similarity index 75% rename from hogg-grpc/src/grpc/services/daemon_health.rs rename to hogg-grpc/src/grpc/services/daemon.rs index 50e2170..a74c943 100644 --- a/hogg-grpc/src/grpc/services/daemon_health.rs +++ b/hogg-grpc/src/grpc/services/daemon.rs @@ -1,12 +1,12 @@ use tonic::{Request, Response, Status}; -use crate::grpc::{DaemonHealth, PingRequest, PingResponse}; +use crate::grpc::{Daemon, PingRequest, PingResponse}; #[derive(Debug, Default)] -pub struct DaemonHealthService {} +pub struct DaemonService {} #[tonic::async_trait] -impl DaemonHealth for DaemonHealthService { +impl Daemon for DaemonService { async fn ping( &self, request: Request, diff --git a/hogg-grpc/src/grpc/services/mod.rs b/hogg-grpc/src/grpc/services/mod.rs index bd9e447..86b216c 100644 --- a/hogg-grpc/src/grpc/services/mod.rs +++ b/hogg-grpc/src/grpc/services/mod.rs @@ -1,3 +1,3 @@ -pub mod daemon_health; +pub mod daemon; -pub use daemon_health::DaemonHealthService; \ No newline at end of file +pub use daemon::DaemonService; \ No newline at end of file From 18f8c06305e7b20f1d58ded24e39266a36235833 Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Wed, 28 Dec 2022 21:39:18 +0100 Subject: [PATCH 04/12] ping grpc implementation --- Cargo.lock | 124 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + hogg-cli/.gitignore | 1 + hogg-cli/Cargo.toml | 15 +++++ hogg-cli/src/cmds/mod.rs | 2 + hogg-cli/src/cmds/ping.rs | 16 +++++ hogg-cli/src/main.rs | 37 ++++++++++++ hogg-grpc/src/grpc/mod.rs | 8 +++ 8 files changed, 204 insertions(+) create mode 100644 hogg-cli/.gitignore create mode 100644 hogg-cli/Cargo.toml create mode 100644 hogg-cli/src/cmds/mod.rs create mode 100644 hogg-cli/src/cmds/ping.rs create mode 100644 hogg-cli/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index cc3faf8..0ce7517 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,6 +265,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "clap" +version = "4.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +dependencies = [ + "bitflags", + "clap_derive", + "clap_lex", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -493,6 +530,27 @@ dependencies = [ "synstructure", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -688,6 +746,18 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hogg-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "hogg-grpc", + "logs", + "tokio", +] + [[package]] name = "hogg-common" version = "0.1.0" @@ -896,12 +966,34 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-lifetimes" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + [[package]] name = "ipnet" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +[[package]] +name = "is-terminal" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys 0.42.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -947,6 +1039,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "lock_api" version = "0.4.9" @@ -1208,6 +1306,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + [[package]] name = "parking" version = "2.0.0" @@ -1551,6 +1655,20 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustix" +version = "0.36.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.42.0", +] + [[package]] name = "rustversion" version = "1.0.11" @@ -1713,6 +1831,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strum" version = "0.22.0" diff --git a/Cargo.toml b/Cargo.toml index eef0013..e938488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,5 +2,6 @@ members = [ "hogg-daemon", "hogg-grpc", + "hogg-cli", "hogg-common" ] diff --git a/hogg-cli/.gitignore b/hogg-cli/.gitignore new file mode 100644 index 0000000..a6f89c2 --- /dev/null +++ b/hogg-cli/.gitignore @@ -0,0 +1 @@ +/target/ \ No newline at end of file diff --git a/hogg-cli/Cargo.toml b/hogg-cli/Cargo.toml new file mode 100644 index 0000000..ba67647 --- /dev/null +++ b/hogg-cli/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "hogg-cli" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +hogg-grpc = { path = "../hogg-grpc" } + +anyhow = "1.0.68" +async-trait = "0.1.60" +clap = { version = "4.0.32", features = ["derive", "cargo"] } +logs = "0.7.1" +tokio = "1.23.0" diff --git a/hogg-cli/src/cmds/mod.rs b/hogg-cli/src/cmds/mod.rs new file mode 100644 index 0000000..04c04e8 --- /dev/null +++ b/hogg-cli/src/cmds/mod.rs @@ -0,0 +1,2 @@ +pub mod ping; +pub use ping::run as ping_command; \ No newline at end of file diff --git a/hogg-cli/src/cmds/ping.rs b/hogg-cli/src/cmds/ping.rs new file mode 100644 index 0000000..bab4147 --- /dev/null +++ b/hogg-cli/src/cmds/ping.rs @@ -0,0 +1,16 @@ +use hogg_grpc::grpc; +use anyhow::Result; + +pub async fn run() -> Result<()> { + let mut grpc = grpc::connect_grpc_client().await?; + + let ping_request = grpc::PingRequest { + message: "Hello from hogg-cli".to_string(), + }; + + let ping_response = grpc.ping(ping_request).await?; + + logs::info!("Received pong: {:?}", ping_response); + + Ok(()) +} diff --git a/hogg-cli/src/main.rs b/hogg-cli/src/main.rs new file mode 100644 index 0000000..59bf103 --- /dev/null +++ b/hogg-cli/src/main.rs @@ -0,0 +1,37 @@ +use clap::{Parser, Subcommand}; +use logs::LevelFilter; +use anyhow::Result; + +mod cmds; + +#[derive(Parser)] +#[command()] +struct CliArgs { + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Commands { + Ping, +} + +#[tokio::main] +async fn main() -> Result<()> { + let logger = logs::Logs::new().color(true); + match logger.level_from_env("HOGG_CLI_LOG") { + Ok(logger) => { + logger.init(); + } + Err(_) => { + logs::Logs::new().color(true).level(LevelFilter::Info).init(); + } + }; + + match CliArgs::parse().command { + Some(Commands::Ping) => cmds::ping_command().await?, + None => logs::error!("No command given"), + } + + Ok(()) +} diff --git a/hogg-grpc/src/grpc/mod.rs b/hogg-grpc/src/grpc/mod.rs index 7b0b912..3588dd1 100644 --- a/hogg-grpc/src/grpc/mod.rs +++ b/hogg-grpc/src/grpc/mod.rs @@ -6,6 +6,9 @@ pub mod daemon_proto { pub use daemon_proto::daemon_server::{Daemon, DaemonServer}; pub use daemon_proto::{PingRequest, PingResponse}; +use tonic::transport::Channel; + +use self::daemon_proto::daemon_client::DaemonClient; #[derive(Debug, err_derive::Error)] @@ -26,3 +29,8 @@ pub fn tokio_serve_hogg_grpc() -> Result Result, Error> { + Ok(DaemonClient::connect("http://[::1]:1396").await?) +} From 1e9da14163be925122b2e8da7ad35572b5fefaef Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Thu, 29 Dec 2022 14:48:26 +0100 Subject: [PATCH 05/12] wip hogg database hogg database will be used in future within daemon and grpc. the database stores all the detections, which user can easily access with cli (in future). --- Cargo.lock | 2 + hogg-common/Cargo.toml | 2 + hogg-common/src/config.rs | 10 +++- hogg-common/src/db.rs | 71 +++++++++++++++++++++++++++++ hogg-common/src/lib.rs | 1 + hogg-daemon/src/main.rs | 2 + hogg-daemon/src/nuclei.rs | 27 +++++++++-- resources/config-template/hogg.toml | 5 ++ 8 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 hogg-common/src/db.rs diff --git a/Cargo.lock b/Cargo.lock index 0ce7517..28fcb99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,11 +765,13 @@ dependencies = [ "anyhow", "async-trait", "directories", + "err-derive", "include_dir", "logs", "reqwest", "serde", "serde_derive", + "serde_json", "toml", ] diff --git a/hogg-common/Cargo.toml b/hogg-common/Cargo.toml index 8df53e1..e5fe560 100644 --- a/hogg-common/Cargo.toml +++ b/hogg-common/Cargo.toml @@ -9,9 +9,11 @@ edition = "2021" anyhow = "1.0.68" async-trait = "0.1.60" directories = "4.0.1" +err-derive = "0.3.1" include_dir = { version = "0.7.3", features = ["glob"] } logs = "0.7.1" reqwest = "0.11.13" serde = { version = "1.0.151", features = ["derive"] } serde_derive = "1.0.151" +serde_json = "1.0.91" toml = "0.5.10" diff --git a/hogg-common/src/config.rs b/hogg-common/src/config.rs index 714285d..5b7a312 100644 --- a/hogg-common/src/config.rs +++ b/hogg-common/src/config.rs @@ -6,9 +6,10 @@ pub struct HoggConfig { pub dnsproxy: DnsProxyConfig, pub daemon: DaemonConfig, pub scanner: ScannerConfig, + pub database: DatabaseConfig, #[serde(skip)] - _file: String, + pub _file: String, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -46,6 +47,13 @@ pub struct ScannerNucleiConfig { pub using_community_templates: Vec, } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct DatabaseConfig { + pub save_detections: bool, + pub detections_limiter_enabled: bool, + pub max_detections: usize +} + impl HoggConfig { pub fn from_file(path: &str) -> Result { let config = std::fs::read_to_string(path)?; diff --git a/hogg-common/src/db.rs b/hogg-common/src/db.rs new file mode 100644 index 0000000..514a891 --- /dev/null +++ b/hogg-common/src/db.rs @@ -0,0 +1,71 @@ +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::config::{HoggConfig, DatabaseConfig}; + + +pub const DB_VERSION: &str = "1.0.0"; + +#[derive(Debug, err_derive::Error)] +pub enum Error { + #[error(display = "IO Error")] + IoError(#[error(source)] std::io::Error), + #[error(display = "Serde Json Error")] + JsonError(#[error(source)] serde_json::Error), +} + +#[derive(Serialize, Deserialize)] +pub struct Detection { + pub viewed: bool, + pub data: T +} + +#[derive(Serialize, Deserialize, Default)] +pub struct DbStruct { + pub version: String, + pub detections: Vec> +} + +pub struct HoggDatabase Deserialize<'a>> { + pub path: String, + pub structure: DbStruct, + + pub config: DatabaseConfig, +} + +impl Deserialize<'a>> HoggDatabase { + pub fn from_file(path: String, config: HoggConfig) -> Result { + if !Path::new(&path).exists() { + std::fs::write(&path, serde_json::to_string(&DbStruct:: { version: DB_VERSION.to_string(), detections: Vec::new() })?)?; + } + let structure = serde_json::from_str(&std::fs::read_to_string(path.clone())?)?; + Ok(Self { path, structure, config: config.database }) + } + + pub fn save(&self) -> Result<(), Error> { + std::fs::write(self.path.clone(), serde_json::to_string(&self.structure)?)?; + Ok(()) + } + + pub fn add_detection(&mut self, detection: T) { + self.structure.detections.push(Detection { viewed: false, data: detection }); + } + + pub fn get_unviewed_detections(&mut self, mark_as_viewed: bool) -> Vec<&mut Detection> { + let mut detections = Vec::new(); + for detection in self.structure.detections.iter_mut() { + if !detection.viewed { + if mark_as_viewed { + detection.viewed = true; + } + detections.push(detection); + } + } + detections + } + + pub fn get_detections(&self, offset: usize, limit: usize) -> Vec<&Detection> { + self.structure.detections[offset..limit+offset].iter().collect() + } +} diff --git a/hogg-common/src/lib.rs b/hogg-common/src/lib.rs index 263d05f..e05b682 100644 --- a/hogg-common/src/lib.rs +++ b/hogg-common/src/lib.rs @@ -2,3 +2,4 @@ pub mod config; pub mod dnslib; pub mod env; pub mod ssladapter; +pub mod db; diff --git a/hogg-daemon/src/main.rs b/hogg-daemon/src/main.rs index e84b356..4c23e7c 100644 --- a/hogg-daemon/src/main.rs +++ b/hogg-daemon/src/main.rs @@ -44,6 +44,8 @@ async fn main() -> Result<()> { CONFIG = Some(config.clone()); } + nuclei::prepare_database(&config); + let config = Arc::new(config); { diff --git a/hogg-daemon/src/nuclei.rs b/hogg-daemon/src/nuclei.rs index 6c6bc20..cab0d92 100644 --- a/hogg-daemon/src/nuclei.rs +++ b/hogg-daemon/src/nuclei.rs @@ -1,7 +1,7 @@ -use std::vec; +use std::{vec, path::Path}; use anyhow::Result; -use hogg_common::{config::HoggConfig, ssladapter}; +use hogg_common::{config::HoggConfig, ssladapter, db::HoggDatabase}; use tokio::{ io::{AsyncBufReadExt, BufReader}, process::Command, @@ -21,7 +21,7 @@ pub struct NucleiTreeInfo { pub reference: Option>, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct NucleiJsonOutput { pub template: String, #[serde(rename = "template-id")] @@ -44,6 +44,20 @@ pub struct NucleiJsonOutput { pub matched_line: Option, } +static mut DATABASE: Option> = None; +pub const DATABASE_FILENAME: &'static str = ".hoggdb.json"; + +pub fn prepare_database(config: &HoggConfig) { + logs::debug!("Preparing nuclei database"); + unsafe { + DATABASE = Some(HoggDatabase::from_file( + Path::new(&config._file).parent().unwrap().join(DATABASE_FILENAME).as_path().to_str().unwrap().to_string(), + config.clone(), + ) + .unwrap()); + } +} + pub async fn scan_with_nuclei( domain: String, config: &HoggConfig, @@ -83,8 +97,13 @@ pub async fn scan_with_nuclei( }; logs::debug!("New nuclei profits: {:#?}", json); - notifications::show_detections_notification(&domain); + unsafe { if config.database.save_detections { // WHY RUST CAN'T JUST HAVE UNSAFE IF + let db = DATABASE.as_mut().unwrap(); + db.add_detection(json.clone()); + db.save()?; + } } answers.push(json); + notifications::show_detections_notification(&domain); } Ok(answers) diff --git a/resources/config-template/hogg.toml b/resources/config-template/hogg.toml index 7adfeab..054821e 100644 --- a/resources/config-template/hogg.toml +++ b/resources/config-template/hogg.toml @@ -23,3 +23,8 @@ using-community-templates = [ "exposures/files/ds-store-file.yaml", "misconfiguration/server-status-localhost.yaml", ] + +[database] +save_detections = true # better do not turn it off, detections will be printed only to daemon stdout +detections_limiter_enabled = true +max_detections = 1000 From 10b840d2f4096cf7f876528dae9834d28e4b2f18 Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Thu, 29 Dec 2022 14:49:21 +0100 Subject: [PATCH 06/12] Create .gitignore --- hogg-grpc/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 hogg-grpc/.gitignore diff --git a/hogg-grpc/.gitignore b/hogg-grpc/.gitignore new file mode 100644 index 0000000..a6f89c2 --- /dev/null +++ b/hogg-grpc/.gitignore @@ -0,0 +1 @@ +/target/ \ No newline at end of file From 0012c872987c2b8cae6760e41c98c0b9aa7ef800 Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Thu, 29 Dec 2022 15:55:01 +0100 Subject: [PATCH 07/12] CLI - unviewed detections commands [WIP] --- Cargo.lock | 2 + hogg-cli/Cargo.toml | 2 + hogg-cli/src/cmds/detections.rs | 61 +++++++++++++++++++++++ hogg-cli/src/cmds/mod.rs | 5 +- hogg-cli/src/cmds/ping.rs | 2 +- hogg-cli/src/main.rs | 9 +++- hogg-common/src/config.rs | 4 +- hogg-common/src/db.rs | 72 +++++++++++++++++++++------ hogg-common/src/lib.rs | 2 +- hogg-daemon/Cargo.toml | 4 ++ hogg-daemon/src/dnsproxy.rs | 8 +-- hogg-daemon/src/lib.rs | 2 + hogg-daemon/src/main.rs | 9 ++-- hogg-daemon/src/notifications.rs | 8 ++- hogg-daemon/src/nuclei.rs | 36 +++++++++----- hogg-grpc/build.rs | 2 +- hogg-grpc/src/grpc/mod.rs | 2 - hogg-grpc/src/grpc/services/daemon.rs | 7 +-- hogg-grpc/src/grpc/services/mod.rs | 2 +- hogg-grpc/src/lib.rs | 2 +- 20 files changed, 188 insertions(+), 53 deletions(-) create mode 100644 hogg-cli/src/cmds/detections.rs create mode 100644 hogg-daemon/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 28fcb99..e77e2ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -753,6 +753,8 @@ dependencies = [ "anyhow", "async-trait", "clap", + "hogg-common", + "hogg-daemon", "hogg-grpc", "logs", "tokio", diff --git a/hogg-cli/Cargo.toml b/hogg-cli/Cargo.toml index ba67647..5b7dfd2 100644 --- a/hogg-cli/Cargo.toml +++ b/hogg-cli/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" [dependencies] hogg-grpc = { path = "../hogg-grpc" } +hogg-common = { path = "../hogg-common" } +hogg-daemon = { path = "../hogg-daemon" } anyhow = "1.0.68" async-trait = "0.1.60" diff --git a/hogg-cli/src/cmds/detections.rs b/hogg-cli/src/cmds/detections.rs new file mode 100644 index 0000000..1e18d6c --- /dev/null +++ b/hogg-cli/src/cmds/detections.rs @@ -0,0 +1,61 @@ +use std::borrow::Borrow; +use std::path::Path; + +use anyhow::Result; +use hogg_common::db::HoggDatabase; +use hogg_common::env; +use hogg_daemon::nuclei::NucleiJsonOutput; + +fn get_database_dir() -> String { + Path::new(&env::get_hogg_dir()) + .join(".hoggdb.json") + .to_str() + .unwrap() + .to_string() +} + +pub async fn run() -> Result<()> { + let mut db = HoggDatabase::::from_file_unconfigured(get_database_dir())?; + + let detections = db.get_unviewed_detections(true)?; + + println!(); + for d in detections.iter() { + println!( + "Vulnerability {} - {}", + d.data.info.name, d.data.info.severity + ); + println!( + " - Host: {}", + d.data.host + ); + println!( + " - Matched at: {}", + d.data.matched_at.clone().unwrap_or("".to_string()) + ); + println!( + " - Description: {}", + d.data + .info + .description + .clone() + .unwrap_or("No description".to_string()) + .trim() + ); + println!( + " - References: {}", + d.data + .info + .reference + .clone() + .unwrap_or(vec!["None".to_string()]) + .join(", ") + ); + + println!(" - Timestamp: {}", d.data.timestamp); + println!(); + println!(); + } + logs::info!("There were {} unviewed detections", detections.len()); + Ok(()) +} diff --git a/hogg-cli/src/cmds/mod.rs b/hogg-cli/src/cmds/mod.rs index 04c04e8..bc8852b 100644 --- a/hogg-cli/src/cmds/mod.rs +++ b/hogg-cli/src/cmds/mod.rs @@ -1,2 +1,5 @@ pub mod ping; -pub use ping::run as ping_command; \ No newline at end of file +pub use ping::run as ping_command; + +pub mod detections; +pub use detections::run as unviewed_detections_command; diff --git a/hogg-cli/src/cmds/ping.rs b/hogg-cli/src/cmds/ping.rs index bab4147..133c2bd 100644 --- a/hogg-cli/src/cmds/ping.rs +++ b/hogg-cli/src/cmds/ping.rs @@ -1,5 +1,5 @@ -use hogg_grpc::grpc; use anyhow::Result; +use hogg_grpc::grpc; pub async fn run() -> Result<()> { let mut grpc = grpc::connect_grpc_client().await?; diff --git a/hogg-cli/src/main.rs b/hogg-cli/src/main.rs index 59bf103..a0ca871 100644 --- a/hogg-cli/src/main.rs +++ b/hogg-cli/src/main.rs @@ -1,6 +1,6 @@ +use anyhow::Result; use clap::{Parser, Subcommand}; use logs::LevelFilter; -use anyhow::Result; mod cmds; @@ -14,6 +14,7 @@ struct CliArgs { #[derive(Subcommand)] enum Commands { Ping, + UnviewedDetections, } #[tokio::main] @@ -24,12 +25,16 @@ async fn main() -> Result<()> { logger.init(); } Err(_) => { - logs::Logs::new().color(true).level(LevelFilter::Info).init(); + logs::Logs::new() + .color(true) + .level(LevelFilter::Info) + .init(); } }; match CliArgs::parse().command { Some(Commands::Ping) => cmds::ping_command().await?, + Some(Commands::UnviewedDetections) => cmds::unviewed_detections_command().await?, None => logs::error!("No command given"), } diff --git a/hogg-common/src/config.rs b/hogg-common/src/config.rs index 5b7a312..1fc2d3b 100644 --- a/hogg-common/src/config.rs +++ b/hogg-common/src/config.rs @@ -47,11 +47,11 @@ pub struct ScannerNucleiConfig { pub using_community_templates: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct DatabaseConfig { pub save_detections: bool, pub detections_limiter_enabled: bool, - pub max_detections: usize + pub max_detections: usize, } impl HoggConfig { diff --git a/hogg-common/src/db.rs b/hogg-common/src/db.rs index 514a891..c45bc41 100644 --- a/hogg-common/src/db.rs +++ b/hogg-common/src/db.rs @@ -1,9 +1,8 @@ -use std::path::Path; +use std::{borrow::BorrowMut, path::Path}; use serde::{Deserialize, Serialize}; -use crate::config::{HoggConfig, DatabaseConfig}; - +use crate::config::{DatabaseConfig, HoggConfig}; pub const DB_VERSION: &str = "1.0.0"; @@ -15,32 +14,63 @@ pub enum Error { JsonError(#[error(source)] serde_json::Error), } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct Detection { pub viewed: bool, - pub data: T + pub data: T, } -#[derive(Serialize, Deserialize, Default)] -pub struct DbStruct { +#[derive(Serialize, Deserialize, Default, Clone)] +pub struct DbStruct +where + T: Clone, +{ pub version: String, - pub detections: Vec> + pub detections: Vec>, } -pub struct HoggDatabase Deserialize<'a>> { +pub struct HoggDatabase Deserialize<'a>> { pub path: String, pub structure: DbStruct, pub config: DatabaseConfig, } -impl Deserialize<'a>> HoggDatabase { +impl Deserialize<'a>> HoggDatabase { pub fn from_file(path: String, config: HoggConfig) -> Result { if !Path::new(&path).exists() { - std::fs::write(&path, serde_json::to_string(&DbStruct:: { version: DB_VERSION.to_string(), detections: Vec::new() })?)?; + std::fs::write( + &path, + serde_json::to_string(&DbStruct:: { + version: DB_VERSION.to_string(), + detections: Vec::new(), + })?, + )?; } let structure = serde_json::from_str(&std::fs::read_to_string(path.clone())?)?; - Ok(Self { path, structure, config: config.database }) + Ok(Self { + path, + structure, + config: config.database, + }) + } + + pub fn from_file_unconfigured(path: String) -> Result { + if !Path::new(&path).exists() { + std::fs::write( + &path, + serde_json::to_string(&DbStruct:: { + version: DB_VERSION.to_string(), + detections: Vec::new(), + })?, + )?; + } + let structure = serde_json::from_str(&std::fs::read_to_string(path.clone())?)?; + Ok(Self { + path, + structure, + config: DatabaseConfig::default(), + }) } pub fn save(&self) -> Result<(), Error> { @@ -49,10 +79,17 @@ impl Deserialize<'a>> HoggDatabase { } pub fn add_detection(&mut self, detection: T) { - self.structure.detections.push(Detection { viewed: false, data: detection }); + self.structure.detections.push(Detection { + viewed: false, + data: detection, + }); } - pub fn get_unviewed_detections(&mut self, mark_as_viewed: bool) -> Vec<&mut Detection> { + pub fn get_unviewed_detections( + &mut self, + mark_as_viewed: bool, + ) -> Result>, Error> { + // TODO: fix saving of database let mut detections = Vec::new(); for detection in self.structure.detections.iter_mut() { if !detection.viewed { @@ -62,10 +99,13 @@ impl Deserialize<'a>> HoggDatabase { detections.push(detection); } } - detections + + Ok(detections) } pub fn get_detections(&self, offset: usize, limit: usize) -> Vec<&Detection> { - self.structure.detections[offset..limit+offset].iter().collect() + self.structure.detections[offset..limit + offset] + .iter() + .collect() } } diff --git a/hogg-common/src/lib.rs b/hogg-common/src/lib.rs index e05b682..f1c2fce 100644 --- a/hogg-common/src/lib.rs +++ b/hogg-common/src/lib.rs @@ -1,5 +1,5 @@ pub mod config; +pub mod db; pub mod dnslib; pub mod env; pub mod ssladapter; -pub mod db; diff --git a/hogg-daemon/Cargo.toml b/hogg-daemon/Cargo.toml index 0c385b2..cbd3cff 100644 --- a/hogg-daemon/Cargo.toml +++ b/hogg-daemon/Cargo.toml @@ -3,6 +3,10 @@ name = "hogg-daemon" version = "0.1.0" edition = "2021" +[[bin]] +name = "hogg-daemon" +path = "src/main.rs" + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/hogg-daemon/src/dnsproxy.rs b/hogg-daemon/src/dnsproxy.rs index 5396f93..df70fc9 100644 --- a/hogg-daemon/src/dnsproxy.rs +++ b/hogg-daemon/src/dnsproxy.rs @@ -1,14 +1,14 @@ -use std::{future::Future, time::Duration, collections::HashMap}; +use std::{collections::HashMap, future::Future, time::Duration}; use anyhow::{anyhow, Result}; +use chrono; use hogg_common::{ config::HoggConfig, dnslib::{BytePacketBuffer, DnsPacket}, }; use lazy_static::lazy_static; -use tokio::{net::UdpSocket, time::timeout}; use std::sync::Mutex; -use chrono; +use tokio::{net::UdpSocket, time::timeout}; type FA = fn(String) -> R; lazy_static! { @@ -61,7 +61,7 @@ pub async fn dns_proxy_task( let domain = q.name.to_string(); let now = chrono::Utc::now().timestamp() as u64; let mut scan_cache = SCAN_CACHE.lock().unwrap(); - + if let Some(last_scan) = scan_cache.get(&domain) { // Domain has been scanned before, check if TTL has expired if now - last_scan > config.scanner.cache_ttl.into() { diff --git a/hogg-daemon/src/lib.rs b/hogg-daemon/src/lib.rs new file mode 100644 index 0000000..764b63b --- /dev/null +++ b/hogg-daemon/src/lib.rs @@ -0,0 +1,2 @@ +pub mod notifications; +pub mod nuclei; // noqa diff --git a/hogg-daemon/src/main.rs b/hogg-daemon/src/main.rs index 4c23e7c..6860d54 100644 --- a/hogg-daemon/src/main.rs +++ b/hogg-daemon/src/main.rs @@ -12,7 +12,7 @@ use hogg_common::{ mod dnsproxy; mod notifications; -mod nuclei; +pub mod nuclei; static CONFIG_TEMPLATE: Dir<'_> = include_dir!("resources/config-template"); static mut CONFIG: Option = None; @@ -28,7 +28,10 @@ async fn scan_function(domain: String) { async fn main() -> Result<()> { match logs::Logs::new().level_from_default_env() { Ok(logs) => logs.color(true).init(), - Err(_) => logs::Logs::new().level(logs::LevelFilter::Info).color(true).init(), + Err(_) => logs::Logs::new() + .level(logs::LevelFilter::Info) + .color(true) + .init(), } let config_path = env::get_hogg_dir(); @@ -57,5 +60,5 @@ async fn main() -> Result<()> { grpc::tokio_serve_hogg_grpc()?; - loop { } + loop {} } diff --git a/hogg-daemon/src/notifications.rs b/hogg-daemon/src/notifications.rs index e5e3f16..cacf045 100644 --- a/hogg-daemon/src/notifications.rs +++ b/hogg-daemon/src/notifications.rs @@ -3,7 +3,13 @@ use notify_rust::Notification; pub fn show_detections_notification(domain: &String) { Notification::new() .summary("Hogg") - .body(format!("New vulnerability detected @ {}\nCheck CLI for more information", domain).as_str()) + .body( + format!( + "New vulnerability detected @ {}\nCheck CLI for more information", + domain + ) + .as_str(), + ) .show() .unwrap(); } diff --git a/hogg-daemon/src/nuclei.rs b/hogg-daemon/src/nuclei.rs index cab0d92..00b0778 100644 --- a/hogg-daemon/src/nuclei.rs +++ b/hogg-daemon/src/nuclei.rs @@ -1,7 +1,7 @@ -use std::{vec, path::Path}; +use std::{path::Path, vec}; use anyhow::Result; -use hogg_common::{config::HoggConfig, ssladapter, db::HoggDatabase}; +use hogg_common::{config::HoggConfig, db::HoggDatabase, ssladapter}; use tokio::{ io::{AsyncBufReadExt, BufReader}, process::Command, @@ -50,11 +50,20 @@ pub const DATABASE_FILENAME: &'static str = ".hoggdb.json"; pub fn prepare_database(config: &HoggConfig) { logs::debug!("Preparing nuclei database"); unsafe { - DATABASE = Some(HoggDatabase::from_file( - Path::new(&config._file).parent().unwrap().join(DATABASE_FILENAME).as_path().to_str().unwrap().to_string(), - config.clone(), - ) - .unwrap()); + DATABASE = Some( + HoggDatabase::from_file( + Path::new(&config._file) + .parent() + .unwrap() + .join(DATABASE_FILENAME) + .as_path() + .to_str() + .unwrap() + .to_string(), + config.clone(), + ) + .unwrap(), + ); } } @@ -97,11 +106,14 @@ pub async fn scan_with_nuclei( }; logs::debug!("New nuclei profits: {:#?}", json); - unsafe { if config.database.save_detections { // WHY RUST CAN'T JUST HAVE UNSAFE IF - let db = DATABASE.as_mut().unwrap(); - db.add_detection(json.clone()); - db.save()?; - } } + unsafe { + if config.database.save_detections { + // WHY RUST CAN'T JUST HAVE UNSAFE IF + let db = DATABASE.as_mut().unwrap(); + db.add_detection(json.clone()); + db.save()?; + } + } answers.push(json); notifications::show_detections_notification(&domain); } diff --git a/hogg-grpc/build.rs b/hogg-grpc/build.rs index cfb2ab1..f269867 100644 --- a/hogg-grpc/build.rs +++ b/hogg-grpc/build.rs @@ -1,6 +1,6 @@ fn main() { const PROTO_FILE: &str = "proto/daemon.proto"; - + tonic_build::compile_protos(PROTO_FILE).unwrap(); println!("cargo:rerun-if-changed={}", PROTO_FILE); } diff --git a/hogg-grpc/src/grpc/mod.rs b/hogg-grpc/src/grpc/mod.rs index 3588dd1..448391b 100644 --- a/hogg-grpc/src/grpc/mod.rs +++ b/hogg-grpc/src/grpc/mod.rs @@ -10,7 +10,6 @@ use tonic::transport::Channel; use self::daemon_proto::daemon_client::DaemonClient; - #[derive(Debug, err_derive::Error)] pub enum Error { #[error(display = "GRPC Transport Error")] @@ -30,7 +29,6 @@ pub fn tokio_serve_hogg_grpc() -> Result Result, Error> { Ok(DaemonClient::connect("http://[::1]:1396").await?) } diff --git a/hogg-grpc/src/grpc/services/daemon.rs b/hogg-grpc/src/grpc/services/daemon.rs index a74c943..d3018d1 100644 --- a/hogg-grpc/src/grpc/services/daemon.rs +++ b/hogg-grpc/src/grpc/services/daemon.rs @@ -7,10 +7,7 @@ pub struct DaemonService {} #[tonic::async_trait] impl Daemon for DaemonService { - async fn ping( - &self, - request: Request, - ) -> Result, Status> { + async fn ping(&self, request: Request) -> Result, Status> { logs::trace!("Got ping request: {:?}", request); let reply = PingResponse { @@ -19,4 +16,4 @@ impl Daemon for DaemonService { Ok(Response::new(reply)) } -} \ No newline at end of file +} diff --git a/hogg-grpc/src/grpc/services/mod.rs b/hogg-grpc/src/grpc/services/mod.rs index 86b216c..f8e16bb 100644 --- a/hogg-grpc/src/grpc/services/mod.rs +++ b/hogg-grpc/src/grpc/services/mod.rs @@ -1,3 +1,3 @@ pub mod daemon; -pub use daemon::DaemonService; \ No newline at end of file +pub use daemon::DaemonService; diff --git a/hogg-grpc/src/lib.rs b/hogg-grpc/src/lib.rs index 31e18c8..773d491 100644 --- a/hogg-grpc/src/lib.rs +++ b/hogg-grpc/src/lib.rs @@ -1 +1 @@ -pub mod grpc; \ No newline at end of file +pub mod grpc; From aa1b8f698a031311d0b4b0a5a3ab7c08dee92bfe Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Thu, 29 Dec 2022 18:38:32 +0100 Subject: [PATCH 08/12] detections flush command for cli --- hogg-cli/src/cmds/detections.rs | 9 +++++++-- hogg-cli/src/cmds/mod.rs | 3 ++- hogg-cli/src/main.rs | 8 +++++++- hogg-common/src/db.rs | 12 ++++++++++-- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/hogg-cli/src/cmds/detections.rs b/hogg-cli/src/cmds/detections.rs index 1e18d6c..3d82579 100644 --- a/hogg-cli/src/cmds/detections.rs +++ b/hogg-cli/src/cmds/detections.rs @@ -1,4 +1,3 @@ -use std::borrow::Borrow; use std::path::Path; use anyhow::Result; @@ -14,7 +13,7 @@ fn get_database_dir() -> String { .to_string() } -pub async fn run() -> Result<()> { +pub async fn get_unviewed() -> Result<()> { let mut db = HoggDatabase::::from_file_unconfigured(get_database_dir())?; let detections = db.get_unviewed_detections(true)?; @@ -59,3 +58,9 @@ pub async fn run() -> Result<()> { logs::info!("There were {} unviewed detections", detections.len()); Ok(()) } + +pub async fn flush_detections() -> Result<()> { + let mut db = HoggDatabase::::from_file_unconfigured(get_database_dir())?; + db.flush_detections()?; + Ok(()) +} diff --git a/hogg-cli/src/cmds/mod.rs b/hogg-cli/src/cmds/mod.rs index bc8852b..d8dee4c 100644 --- a/hogg-cli/src/cmds/mod.rs +++ b/hogg-cli/src/cmds/mod.rs @@ -2,4 +2,5 @@ pub mod ping; pub use ping::run as ping_command; pub mod detections; -pub use detections::run as unviewed_detections_command; +pub use detections::get_unviewed as unviewed_detections_command; +pub use detections::flush_detections as flush_command; \ No newline at end of file diff --git a/hogg-cli/src/main.rs b/hogg-cli/src/main.rs index a0ca871..b977613 100644 --- a/hogg-cli/src/main.rs +++ b/hogg-cli/src/main.rs @@ -5,7 +5,8 @@ use logs::LevelFilter; mod cmds; #[derive(Parser)] -#[command()] +#[command(name = "hogg-cli", version = "0.1.0", author = "yallxe")] +#[command(about = "A CLI for Hogg")] struct CliArgs { #[command(subcommand)] command: Option, @@ -13,8 +14,12 @@ struct CliArgs { #[derive(Subcommand)] enum Commands { + #[command(about = "Ping the Hogg Daemon to check if it's running")] Ping, + #[command(about = "Get all unviewed detections from hogg database")] UnviewedDetections, + #[command(about = "Delete all the detections from hogg database")] + Flush } #[tokio::main] @@ -35,6 +40,7 @@ async fn main() -> Result<()> { match CliArgs::parse().command { Some(Commands::Ping) => cmds::ping_command().await?, Some(Commands::UnviewedDetections) => cmds::unviewed_detections_command().await?, + Some(Commands::Flush) => cmds::flush_command().await?, None => logs::error!("No command given"), } diff --git a/hogg-common/src/db.rs b/hogg-common/src/db.rs index c45bc41..dabd4bc 100644 --- a/hogg-common/src/db.rs +++ b/hogg-common/src/db.rs @@ -1,4 +1,4 @@ -use std::{borrow::BorrowMut, path::Path}; +use std::path::Path; use serde::{Deserialize, Serialize}; @@ -91,6 +91,7 @@ impl Deserialize<'a>> HoggDatabase { ) -> Result>, Error> { // TODO: fix saving of database let mut detections = Vec::new(); + for detection in self.structure.detections.iter_mut() { if !detection.viewed { if mark_as_viewed { @@ -99,10 +100,17 @@ impl Deserialize<'a>> HoggDatabase { detections.push(detection); } } - + + // self.save()?; Ok(detections) } + pub fn flush_detections(&mut self) -> Result<(), Error> { + self.structure.detections = Vec::new(); + self.save()?; + Ok(()) + } + pub fn get_detections(&self, offset: usize, limit: usize) -> Vec<&Detection> { self.structure.detections[offset..limit + offset] .iter() From 7044975cb889f253531c95f4c431ea0b54143c3d Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Thu, 29 Dec 2022 18:57:46 +0100 Subject: [PATCH 09/12] Eq & PartialEq comparing before adding to database --- hogg-common/src/db.rs | 29 ++++++++++++++++++++++------- hogg-daemon/src/nuclei.rs | 28 +++++++++++++++++++++------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/hogg-common/src/db.rs b/hogg-common/src/db.rs index dabd4bc..ecbe92f 100644 --- a/hogg-common/src/db.rs +++ b/hogg-common/src/db.rs @@ -15,28 +15,36 @@ pub enum Error { } #[derive(Serialize, Deserialize, Clone)] -pub struct Detection { +pub struct Detection { pub viewed: bool, pub data: T, } +impl PartialEq for Detection { + fn eq(&self, other: &Self) -> bool { + self.data == other.data + } +} + +impl Eq for Detection { } + #[derive(Serialize, Deserialize, Default, Clone)] pub struct DbStruct where - T: Clone, + T: Clone + Eq + PartialEq, { pub version: String, pub detections: Vec>, } -pub struct HoggDatabase Deserialize<'a>> { +pub struct HoggDatabase Deserialize<'a>> { pub path: String, pub structure: DbStruct, pub config: DatabaseConfig, } -impl Deserialize<'a>> HoggDatabase { +impl Deserialize<'a>> HoggDatabase { pub fn from_file(path: String, config: HoggConfig) -> Result { if !Path::new(&path).exists() { std::fs::write( @@ -78,11 +86,18 @@ impl Deserialize<'a>> HoggDatabase { Ok(()) } - pub fn add_detection(&mut self, detection: T) { - self.structure.detections.push(Detection { + pub fn add_detection(&mut self, detection: T) -> bool { + // return true if detection was added, false if it already existed + let detection = Detection { viewed: false, data: detection, - }); + }; + + if !self.structure.detections.contains(&detection) { + self.structure.detections.push(detection); + return true; + } + false } pub fn get_unviewed_detections( diff --git a/hogg-daemon/src/nuclei.rs b/hogg-daemon/src/nuclei.rs index 00b0778..f381ebe 100644 --- a/hogg-daemon/src/nuclei.rs +++ b/hogg-daemon/src/nuclei.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::notifications; -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct NucleiTreeInfo { pub name: String, pub author: Vec, @@ -44,6 +44,17 @@ pub struct NucleiJsonOutput { pub matched_line: Option, } +impl PartialEq for NucleiJsonOutput { + fn eq(&self, other: &Self) -> bool { + self.template_id == other.template_id && + self.matched_at == other.matched_at && + self.host == other.host && + self.info == other.info + } +} + +impl Eq for NucleiJsonOutput {} + static mut DATABASE: Option> = None; pub const DATABASE_FILENAME: &'static str = ".hoggdb.json"; @@ -104,18 +115,21 @@ pub async fn scan_with_nuclei( continue; } }; - - logs::debug!("New nuclei profits: {:#?}", json); + unsafe { if config.database.save_detections { // WHY RUST CAN'T JUST HAVE UNSAFE IF let db = DATABASE.as_mut().unwrap(); - db.add_detection(json.clone()); - db.save()?; + if db.add_detection(json.clone()) { + db.save()?; + + logs::debug!("New nuclei profits: {:#?}", json); + notifications::show_detections_notification(&domain); + answers.push(json); + } } } - answers.push(json); - notifications::show_detections_notification(&domain); + } Ok(answers) From 68443a806753010e668a85fbb00ddc172ed678d9 Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Thu, 29 Dec 2022 19:49:18 +0100 Subject: [PATCH 10/12] reload database on flush (noqa) --- Cargo.lock | 8 +++---- hogg-cli/src/cmds/detections.rs | 7 +++++++ hogg-daemon/src/main.rs | 8 ++++--- hogg-daemon/src/nuclei.rs | 4 ++-- hogg-grpc/proto/daemon.proto | 8 +++++++ hogg-grpc/src/grpc/mod.rs | 6 +++--- hogg-grpc/src/grpc/services/daemon.rs | 30 +++++++++++++++++++++++---- 7 files changed, 55 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e77e2ed..96db060 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1732,18 +1732,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", diff --git a/hogg-cli/src/cmds/detections.rs b/hogg-cli/src/cmds/detections.rs index 3d82579..0268712 100644 --- a/hogg-cli/src/cmds/detections.rs +++ b/hogg-cli/src/cmds/detections.rs @@ -4,6 +4,7 @@ use anyhow::Result; use hogg_common::db::HoggDatabase; use hogg_common::env; use hogg_daemon::nuclei::NucleiJsonOutput; +use hogg_grpc::grpc; fn get_database_dir() -> String { Path::new(&env::get_hogg_dir()) @@ -60,7 +61,13 @@ pub async fn get_unviewed() -> Result<()> { } pub async fn flush_detections() -> Result<()> { + logs::info!("Flushing detections from database..."); let mut db = HoggDatabase::::from_file_unconfigured(get_database_dir())?; db.flush_detections()?; + + logs::info!("Sending database forced reload to hogg-daemon..."); + let mut grpc = grpc::connect_grpc_client().await?; + grpc.reload_database(grpc::ReloadDatabaseRequest {}).await?; + Ok(()) } diff --git a/hogg-daemon/src/main.rs b/hogg-daemon/src/main.rs index 6860d54..ac46873 100644 --- a/hogg-daemon/src/main.rs +++ b/hogg-daemon/src/main.rs @@ -47,7 +47,7 @@ async fn main() -> Result<()> { CONFIG = Some(config.clone()); } - nuclei::prepare_database(&config); + nuclei::load_database(&config); let config = Arc::new(config); @@ -57,8 +57,10 @@ async fn main() -> Result<()> { dns_proxy_task(&config, scan_function).await; }); } - - grpc::tokio_serve_hogg_grpc()?; + + grpc::tokio_serve_hogg_grpc(move || { + nuclei::load_database(&config); + })?; loop {} } diff --git a/hogg-daemon/src/nuclei.rs b/hogg-daemon/src/nuclei.rs index f381ebe..56f6935 100644 --- a/hogg-daemon/src/nuclei.rs +++ b/hogg-daemon/src/nuclei.rs @@ -58,8 +58,8 @@ impl Eq for NucleiJsonOutput {} static mut DATABASE: Option> = None; pub const DATABASE_FILENAME: &'static str = ".hoggdb.json"; -pub fn prepare_database(config: &HoggConfig) { - logs::debug!("Preparing nuclei database"); +pub fn load_database(config: &HoggConfig) { + logs::debug!("(Re)loading nuclei database"); unsafe { DATABASE = Some( HoggDatabase::from_file( diff --git a/hogg-grpc/proto/daemon.proto b/hogg-grpc/proto/daemon.proto index 4bfe0ca..84c3f3d 100644 --- a/hogg-grpc/proto/daemon.proto +++ b/hogg-grpc/proto/daemon.proto @@ -3,6 +3,14 @@ package daemon; service Daemon { rpc Ping(PingRequest) returns (PingResponse) {} + rpc ReloadDatabase(ReloadDatabaseRequest) returns (ReloadDatabaseResponse) {} +} + +message ReloadDatabaseRequest { } + +message ReloadDatabaseResponse { + bool success = 1; + optional string error = 2; } message PingRequest { diff --git a/hogg-grpc/src/grpc/mod.rs b/hogg-grpc/src/grpc/mod.rs index 448391b..198cf9b 100644 --- a/hogg-grpc/src/grpc/mod.rs +++ b/hogg-grpc/src/grpc/mod.rs @@ -5,7 +5,7 @@ pub mod daemon_proto { } pub use daemon_proto::daemon_server::{Daemon, DaemonServer}; -pub use daemon_proto::{PingRequest, PingResponse}; +pub use daemon_proto::{PingRequest, PingResponse, ReloadDatabaseRequest, ReloadDatabaseResponse}; use tonic::transport::Channel; use self::daemon_proto::daemon_client::DaemonClient; @@ -16,10 +16,10 @@ pub enum Error { GrpcTransportError(#[error(source)] tonic::transport::Error), } -pub fn tokio_serve_hogg_grpc() -> Result>, Error> { +pub fn tokio_serve_hogg_grpc ()>(db_reload: F) -> Result>, Error> { Ok(tokio::spawn(async move { let addr = "[::1]:1396".parse().unwrap(); - let daemon_health = services::DaemonService::default(); + let daemon_health = services::DaemonService::new(db_reload); tonic::transport::Server::builder() .add_service(DaemonServer::new(daemon_health)) diff --git a/hogg-grpc/src/grpc/services/daemon.rs b/hogg-grpc/src/grpc/services/daemon.rs index d3018d1..651beae 100644 --- a/hogg-grpc/src/grpc/services/daemon.rs +++ b/hogg-grpc/src/grpc/services/daemon.rs @@ -1,12 +1,22 @@ use tonic::{Request, Response, Status}; -use crate::grpc::{Daemon, PingRequest, PingResponse}; +use crate::grpc::{Daemon, PingRequest, PingResponse, ReloadDatabaseRequest, ReloadDatabaseResponse}; -#[derive(Debug, Default)] -pub struct DaemonService {} +#[derive(Debug)] +pub struct DaemonService ()> { + pub db_reload: F, +} + +impl ()> DaemonService { + pub fn new(db_reload: F) -> Self { + Self { + db_reload, + } + } +} #[tonic::async_trait] -impl Daemon for DaemonService { +impl ()> Daemon for DaemonService { async fn ping(&self, request: Request) -> Result, Status> { logs::trace!("Got ping request: {:?}", request); @@ -16,4 +26,16 @@ impl Daemon for DaemonService { Ok(Response::new(reply)) } + + async fn reload_database( + &self, + _request: Request, + ) -> Result, Status> { + (self.db_reload)(); + + Ok(Response::new(ReloadDatabaseResponse { + success: true, + error: None + })) + } } From 9f79a9a41639e12b991bf5b851d8c06f4ccde4bd Mon Sep 17 00:00:00 2001 From: yallxe <82591945+yallxe@users.noreply.github.com> Date: Thu, 29 Dec 2022 23:53:10 +0100 Subject: [PATCH 11/12] fix cpu overload --- hogg-daemon/src/main.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hogg-daemon/src/main.rs b/hogg-daemon/src/main.rs index ac46873..106c1be 100644 --- a/hogg-daemon/src/main.rs +++ b/hogg-daemon/src/main.rs @@ -57,10 +57,12 @@ async fn main() -> Result<()> { dns_proxy_task(&config, scan_function).await; }); } - + grpc::tokio_serve_hogg_grpc(move || { nuclei::load_database(&config); })?; - loop {} + loop { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } } From d7ad43ebba720ca2ea03ed908299e7a2539f4e5b Mon Sep 17 00:00:00 2001 From: Yallxe <82591945+yallxe@users.noreply.github.com> Date: Fri, 30 Dec 2022 01:18:12 +0100 Subject: [PATCH 12/12] Update README.md --- README.md | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 890a1f9..c5c669c 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,45 @@ # Hogg 🐽 + An experimental passive website scanner. Hogg acts as a proxy between you and your DNS server and scans every website you visit for common vulnerabilities. -## Installation & Usage -Currently you can only run the daemon. In future, you will be able to use a special CLI or GUI to interact with daemon. The daemon is responsible of sending notifications. -```shell -git clone https://github.com/yallxe/hogg -cd hogg -cargo run -p hogg-deamon # requires root on linux & macos -``` -After you run the daemon, you can set you DNS server to `localhost:53`, so all the DNS requests will be sent to hogg. +## Building + +0. Make sure you have Rust installed. If not, follow the instructions [here](https://www.rust-lang.org/tools/install). +1. Install dependencies for [tonic](https://github.com/hyperium/tonic) from [here](https://github.com/hyperium/tonic#dependencies). +2. Install [Nuclei](https://github.com/projectdiscovery/nuclei) and make sure it's in your `$PATH`. +3. Clone the repo and `cd` into it. +4. Run `cargo build --release` to build the binary. + +## Using it + +To make hogg work, you need to run the daemon, which will serve the DNS proxy and scan the websites you visit. You will get a notification when a vulnerability is found. To view the vulnerabilities, you can use the `hogg` CLI. Use `hogg-cli -h` to see the available commands. +To run the daemon, use `hogg-daemon` binary. ## Configuration + Checkout your configuration path, which is printed when you start the daemon, or use `echo $HOGG_CONFIG_DIR` ## How does it work? + 1. Your browser or a desktop app resolves a domain name via DNS. 2. Hogg requests the data from your upstream DNS provider (Cloudflare by default) and sends it back to the app. 3. Hogg scans the website using [Nuclei](https://github.com/projectdiscovery/nuclei). ## How is it different? + Hogg will help you scan almost every website you visit (not limited to your browser) without causing any disruption to the app's functionality. ## Anything besides DNS? + Not yet. Stay tuned for future updates that may include other solutions (like an HTTP proxy). ## Limitations + - Hogg doesn't yet support DNS over HTTPS, DNS over TLS etc. - Some apps may bypass your system's default DNS resolver. In this case, Hogg will not intercept the app's requests. ## Progress + - [x] Working DNS proxy and Nuclei scanner - [x] Notifications (OS notifications for now) - [ ] Automatic request redirection to DNS Proxy @@ -36,4 +47,5 @@ Not yet. Stay tuned for future updates that may include other solutions (like an - [ ] DNS over HTTPS ## Credits + - Inspired by [Trufflehog-Chrome-Extension](https://github.com/trufflesecurity/Trufflehog-Chrome-Extension) ❤️