根据空间搜索引擎语法自动生成指纹

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change Log
 
 <!-- next-header -->
+## [2024.7.31] - 2024.7.31
+
+### Fixes
+
+- 添加空间搜索引擎数据
+- 添加unix-socket接口 socket over http
+- 根据空间搜索引擎语法自动生成指纹规则
 
 ## [2024.7.25] - 2024.7.25
 

engine/src/info.rs

@@ -1,4 +1,4 @@
-use crate::matchers::{Favicon, MRegex, MatcherType, Word};
+use crate::matchers::{Favicon, MRegex, Matcher, MatcherType, Word};
 use crate::serde_format::{is_default, string_vec_serde, Value};
 use fancy_regex::Captures;
 use serde::{Deserialize, Serialize};
@@ -185,56 +185,66 @@ impl Info {
     }
   }
   pub fn set_cse(&mut self, cse: CSE) {
-    self.metadata.insert(
-      "zoomeye-query".to_string(),
-      Value::List(
-        cse
-          .zoomeye_query
-          .iter()
-          .map(|x| Value::String(x.to_string()))
-          .collect(),
-      ),
-    );
-    self.metadata.insert(
-      "fofa-query".to_string(),
-      Value::List(
-        cse
-          .fofa_query
-          .iter()
-          .map(|x| Value::String(x.to_string()))
-          .collect(),
-      ),
-    );
-    self.metadata.insert(
-      "hunter-query".to_string(),
-      Value::List(
-        cse
-          .hunter_query
-          .iter()
-          .map(|x| Value::String(x.to_string()))
-          .collect(),
-      ),
-    );
-    self.metadata.insert(
-      "shodan-query".to_string(),
-      Value::List(
-        cse
-          .shodan_query
-          .iter()
-          .map(|x| Value::String(x.to_string()))
-          .collect(),
-      ),
-    );
-    self.metadata.insert(
-      "google-query".to_string(),
-      Value::List(
-        cse
-          .google_query
-          .iter()
-          .map(|x| Value::String(x.to_string()))
-          .collect(),
-      ),
-    );
+    if !cse.zoomeye_query.is_empty() {
+      self.metadata.insert(
+        "zoomeye-query".to_string(),
+        Value::List(
+          cse
+            .zoomeye_query
+            .iter()
+            .map(|x| Value::String(x.to_string()))
+            .collect(),
+        ),
+      );
+    }
+    if !cse.fofa_query.is_empty() {
+      self.metadata.insert(
+        "fofa-query".to_string(),
+        Value::List(
+          cse
+            .fofa_query
+            .iter()
+            .map(|x| Value::String(x.to_string()))
+            .collect(),
+        ),
+      );
+    }
+    if !cse.hunter_query.is_empty() {
+      self.metadata.insert(
+        "hunter-query".to_string(),
+        Value::List(
+          cse
+            .hunter_query
+            .iter()
+            .map(|x| Value::String(x.to_string()))
+            .collect(),
+        ),
+      );
+    }
+    if !cse.shodan_query.is_empty() {
+      self.metadata.insert(
+        "shodan-query".to_string(),
+        Value::List(
+          cse
+            .shodan_query
+            .iter()
+            .map(|x| Value::String(x.to_string()))
+            .collect(),
+        ),
+      );
+    }
+    if !cse.google_query.is_empty() {
+      self.metadata.insert(
+        "google-query".to_string(),
+        Value::List(
+          cse
+            .google_query
+            .iter()
+            .map(|x| Value::String(x.to_string()))
+            .collect(),
+        ),
+      );
+    }
   }
 }
 // 空间搜索引擎查询语法CyberspaceSearchEngineQuery
@@ -315,41 +325,55 @@ impl CSE {
     parts
   }
 }
-impl Into<Vec<MatcherType>> for CSE {
-  fn into(self) -> Vec<MatcherType> {
+impl From<CSE> for Vec<Matcher> {
+  fn from(val: CSE) -> Self {
     let mut mt = Vec::new();
     let mut keyword = HashSet::new();
     let mut title = HashSet::new();
     let mut hash = HashSet::new();
-    let trim = &['"', '\''];
-    for query in &self.shodan_query {
-      if let Some((k, v)) = query.split_once(":") {
-        let v = v.to_lowercase().trim_matches(trim).to_string();
-        match k {
-          "title" | "http.title" => {
-            title.insert(format!("<\\btitle\\b.*?>{}<\\/\\btitle\\b>", v));
-          }
-          "http.html" | "html" => {
-            keyword.insert(v);
-          }
-          "http.favicon.hash" => {
-            hash.insert(v);
+    let trim = &['"', '\'', '\\'];
+    for query in &val.shodan_query {
+      if let Some((k, v)) = query.split_once(':') {
+        let v = v
+          .to_lowercase()
+          .trim_matches(trim)
+          .replace("\\\"", "")
+          .to_string();
+        for vv in val.or_and_split(&v) {
+          match k {
+            "title" | "http.title" => {
+              title.insert(vv);
+            }
+            "http.html" | "html" => {
+              keyword.insert(vv);
+            }
+            "http.favicon.hash" => {
+              hash.extend(
+                vv.split(',')
+                  .map(|x| x.to_string())
+                  .collect::<Vec<String>>(),
+              );
+            }
+            _ => {}
           }
-          _ => {}
         }
       } else {
         // 都归关键词
         keyword.insert(query.to_lowercase().trim_matches(trim).to_string());
       }
     }
-    for query in &self.fofa_query {
+    for query in &val.fofa_query {
       let query = query.trim_matches(trim);
-      if let Some((k, v)) = query.split_once("=") {
-        for vv in self.or_and_split(&v) {
-          let vv = vv.to_lowercase().trim_matches(trim).to_string();
+      if let Some((k, v)) = query.split_once('=') {
+        for vv in val.or_and_split(v) {
+          let vv = vv
+            .to_lowercase()
+            .trim_matches(trim)
+            .replace("\\\"", "")
+            .to_string();
           match k {
             "title" => {
-              title.insert(format!("<\\btitle\\b.*?>{}<\\/\\btitle\\b>", vv));
+              title.insert(vv);
             }
             "body" => {
               keyword.insert(vv);
@@ -362,28 +386,46 @@ impl Into<Vec<MatcherType>> for CSE {
         }
       } else {
         // 都归关键词
-        for vv in self.or_and_split(&query) {
+        for vv in val.or_and_split(query) {
           keyword.insert(vv.to_lowercase().trim_matches(trim).to_string());
         }
       }
     }
     if !keyword.is_empty() {
       let mut k: Vec<String> = keyword.iter().map(|x| x.to_string()).collect();
       k.sort();
-      mt.push(MatcherType::Word(Word { words: k }));
+      let m = Matcher {
+        matcher_type: MatcherType::Word(Word { words: k }),
+        ..Matcher::default()
+      };
+      mt.push(m);
     }
     if !hash.is_empty() {
       let mut h: Vec<String> = hash.iter().map(|x| x.to_string()).collect();
       h.sort();
-      mt.push(MatcherType::Favicon(Favicon { hash: h }));
+      let m = Matcher {
+        matcher_type: MatcherType::Favicon(Favicon { hash: h }),
+        ..Matcher::default()
+      };
+      mt.push(m);
     }
     if !title.is_empty() {
-      let mut r: Vec<String> = title.iter().map(|x| x.to_string()).collect();
+      let mut r: Vec<String> = title
+        .iter()
+        .filter(|x| !keyword.contains(*x))
+        .map(|x| format!("(?mi)<title[^>]*>{}.*?</title>", x))
+        .collect();
       r.sort();
-      mt.push(MatcherType::Regex(MRegex {
-        regex: r,
-        group: None,
-      }))
+      if !r.is_empty() {
+        let m = Matcher {
+          matcher_type: MatcherType::Regex(MRegex {
+            regex: r,
+            group: None,
+          }),
+          ..Matcher::default()
+        };
+        mt.push(m);
+      }
     }
     mt
   }

engine/src/request/http/option.rs

@@ -2,7 +2,7 @@ use crate::serde_format::is_default;
 use serde::{Deserialize, Serialize};
 use slinger::http::header::HeaderValue;
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "kebab-case")]
 pub struct HttpOption {
   #[serde(default, skip_serializing_if = "is_default")]
@@ -46,21 +46,6 @@ pub struct HttpOption {
   pub read_all: bool,
 }
 
-impl Default for HttpOption {
-  fn default() -> Self {
-    Self {
-      host_redirects: false,
-      redirects: false,
-      race_count: None,
-      max_redirects: None,
-      threads: None,
-      max_size: None,
-      cookie_reuse: false,
-      read_all: false,
-    }
-  }
-}
-
 impl HttpOption {
   pub fn builder_client(&self) -> slinger::ClientBuilder {
     let redirect = if self.redirects {