自动更新指纹，添加生成json格式指纹库参数

CHANGELOG.md

@@ -3,6 +3,22 @@
 <!-- next-header -->
 
 ## [Unreleased] - ReleaseDate
+
+## [2023.7.21] - 2023.7.21
+
+### Fixes
+
+- `--gen`参数可以配合`--yaml`参数将指定yaml目录中的全部yaml指纹规则生成单个json文件，主要方便自定义指纹，生成便携单文件。
+- `/home/kali-team/IdeaProjects/FingerprintHub/web_fingerprint`是存放yaml的目录，`web_fingerprint_v3.json`是生成的文件路径。
+
+```bash
+➜  ~ ./observer_ward --yaml /home/kali-team/IdeaProjects/FingerprintHub/web_fingerprint --gen web_fingerprint_v3.json
+➜  ~ jq length web_fingerprint_v3.json
+3448
+```
+- 添加如果本地没有指纹库，会自动更新指纹。防止跑完发现没有下载指纹，白跑了目标。
+
+
 ## [2023.6.20] - 2023.6.20
 
 ### Fixes

README.md

@@ -48,6 +48,8 @@ Options:
   --stdin           read target(s) from STDIN
   --fpath           customized fingerprint file path
   --yaml            customized fingerprint yaml directory (slow)
+  --gen             generate json format fingerprint library from yaml
+                    format(requires yaml parameter)
   --path            customized nuclei template file path
   --verify          validate the specified yaml file or grep keyword
   -f, --file        read the target from the file
@@ -116,6 +118,15 @@ https://0x727.github.io/FingerprintHub/web_fingerprint_v3.json:=> /home/kali-tea
 - `--verify`指定要验证的指纹yaml文件路径，`-t`指定要识别的目标，输出请求过程和识别结果。
 - `--fpath`指定自己的`web_fingerprint_v3.json`文件。
 - `--yaml`指定`FingerprintHub`的`web_fingerprint`文件夹，加载全部yaml文件，比较慢，只适合本地测试。
+- `--gen`参数可以配合`--yaml`参数将指定yaml目录中的全部yaml指纹规则生成单个json文件，主要方便自定义指纹，生成便携单文件。
+
+```bash
+➜  ~ ./observer_ward --yaml /home/kali-team/IdeaProjects/FingerprintHub/web_fingerprint --gen web_fingerprint_v3.json
+➜  ~ jq length web_fingerprint_v3.json
+3448
+```
+- `/home/kali-team/IdeaProjects/FingerprintHub/web_fingerprint`是存放yaml的目录，`web_fingerprint_v3.json`是生成的文件路径。
+
 ```bash
 ➜  ~ ./observer_ward -t https://www.example.com --verify 0example.yaml
 Url: https://www.example.com/

src/api.rs

@@ -73,7 +73,7 @@ async fn set_config_api(
         return HttpResponse::Unauthorized().finish();
     }
     let mut helper = Helper::new(&config);
-    helper.run().await;
+    helper.run();
     helper.msg = HashMap::new();
     observer_ward_ins.write().await.reload(&config);
     let config = observer_ward_ins.read().await.config.clone();

src/cli.rs

@@ -25,6 +25,10 @@ pub struct ObserverWardConfig {
     #[argh(option)]
     #[serde(skip)]
     pub yaml: Option<PathBuf>,
+    /// generate json format fingerprint library from yaml format(requires yaml parameter)
+    #[argh(option)]
+    #[serde(skip)]
+    pub gen: Option<PathBuf>,
     /// customized nuclei template file path
     #[argh(option)]
     #[serde(skip)]

src/lib.rs

@@ -144,32 +144,29 @@ impl<'a> Helper<'a> {
             msg: Default::default(),
         }
     }
-    async fn update_fingerprint(&mut self) {
+    fn update_fingerprint(&mut self) {
         let fingerprint_path = self.config_path.join("web_fingerprint_v3.json");
         self.download_file_from_github(
             "https://0x727.github.io/FingerprintHub/web_fingerprint_v3.json",
             fingerprint_path
                 .to_str()
                 .unwrap_or("web_fingerprint_v3.json"),
-        )
-        .await;
+        );
         self.download_file_from_github(
             "https://0x727.github.io/FingerprintHub/plugins/tags.yaml",
             self.config_path
                 .join("tags.yaml")
                 .to_str()
                 .unwrap_or("tags.yaml"),
-        )
-        .await;
+        );
     }
-    async fn update_plugins(&mut self) {
+    fn update_plugins(&mut self) {
         let plugins_zip_path = self.config_path.join("plugins.zip");
         let extract_target_path = self.config_path;
         self.download_file_from_github(
             "https://github.com/0x727/FingerprintHub/releases/download/default/plugins.zip",
             plugins_zip_path.to_str().unwrap_or("plugins.zip"),
-        )
-        .await;
+        );
         match extract_plugins_zip(&plugins_zip_path, extract_target_path) {
             Ok(_) => {
                 println!("It has been extracted to the {:?}", extract_target_path);
@@ -180,15 +177,15 @@ impl<'a> Helper<'a> {
             }
         }
     }
-    pub async fn run(&mut self) -> HashMap<String, String> {
+    pub fn run(&mut self) -> HashMap<String, String> {
         if self.config.update_fingerprint {
-            self.update_fingerprint().await;
+            self.update_fingerprint();
         }
         if self.config.update_self {
-            self.update_self().await;
+            self.update_self();
         }
         if self.config.update_plugins {
-            self.update_plugins().await;
+            self.update_plugins();
         }
         if !self.msg.is_empty() {
             for (k, v) in &self.msg {
@@ -200,7 +197,7 @@ impl<'a> Helper<'a> {
 }
 
 impl<'a> Helper<'_> {
-    pub async fn update_self(&mut self) {
+    pub fn update_self(&mut self) {
         // https://doc.rust-lang.org/reference/conditional-compilation.html
         let mut base_url =
             String::from("https://github.com/0x727/ObserverWard/releases/download/default/");
@@ -216,8 +213,7 @@ impl<'a> Helper<'_> {
         };
         base_url.push_str(download_name);
         let save_filename = "update_".to_owned() + download_name;
-        self.download_file_from_github(&base_url, &save_filename)
-            .await;
+        self.download_file_from_github(&base_url, &save_filename);
         println!(
             "Please rename the file {} => {}",
             save_filename, download_name
@@ -274,14 +270,32 @@ impl<'a> Helper<'_> {
             if !config.silent {
                 println!("Load {} fingerprints.", web_fingerprint.len());
             }
+            if let Some(json_path) = &config.gen {
+                let out = File::create(json_path).expect("Failed to create file");
+                serde_json::to_writer(out, &web_fingerprint).expect("Failed to generate json file");
+                println!(
+                    "completed generating json format files, totaling {} items",
+                    web_fingerprint.len()
+                );
+            }
             return web_fingerprint;
         }
         let mut web_fingerprint_path = PathBuf::from("web_fingerprint_v3.json");
-        if !web_fingerprint_path.exists() {
-            web_fingerprint_path = self.config_path.join("web_fingerprint_v3.json");
-        }
+        // 如果有指定路径的指纹库
         if let Some(p) = &config.fpath {
             web_fingerprint_path = PathBuf::from(p);
+            if !web_fingerprint_path.exists() {
+                println!("The specified fingerprint path does not exist");
+                std::process::exit(1);
+            }
+        } else {
+            // 如果当前运行目录下没有指纹库，把路径改为config目录下的
+            if !web_fingerprint_path.exists() {
+                web_fingerprint_path = self.config_path.join("web_fingerprint_v3.json");
+            }
+            if !web_fingerprint_path.exists() {
+                self.update_fingerprint();
+            }
         }
         if let Ok(file) = File::open(web_fingerprint_path) {
             if let Ok(web_fingerprint) = serde_json::from_reader::<_, Vec<WebFingerPrint>>(&file) {
@@ -293,7 +307,7 @@ impl<'a> Helper<'_> {
             println!("The fingerprint library cannot be found in the current directory!");
             println!("Update fingerprint library with `-u` parameter!");
         }
-        Vec::new()
+        std::process::exit(1);
     }
 
     pub fn read_results_file(&self) -> Vec<WhatWebResult> {
@@ -323,14 +337,14 @@ impl<'a> Helper<'_> {
         }
         results
     }
-    async fn download_file_from_github(&mut self, update_url: &'a str, filename: &'a str) {
+    fn download_file_from_github(&mut self, update_url: &'a str, filename: &'a str) {
         let proxy = self.request_option.proxy.as_ref().cloned();
         let proxy_obj = Proxy::custom(move |_url| proxy.clone());
-        let client = reqwest::Client::builder().proxy(proxy_obj);
+        let client = reqwest::blocking::Client::builder().proxy(proxy_obj);
         if let Ok(downloading_client) = client.build() {
-            if let Ok(response) = downloading_client.get(update_url).send().await {
+            if let Ok(response) = downloading_client.get(update_url).send() {
                 let mut file = File::create(filename).unwrap();
-                let mut content = Cursor::new(response.bytes().await.unwrap_or_default());
+                let mut content = Cursor::new(response.bytes().unwrap_or_default());
                 std::io::copy(&mut content, &mut file).unwrap_or_default();
                 self.msg.insert(
                     String::from(update_url),

src/main.rs

@@ -43,7 +43,7 @@ async fn start() -> Result<(), Error> {
     if config.service {
         nmap_fingerprint = helper.read_nmap_fingerprint();
     }
-    helper.run().await;
+    helper.run();
     let observer_ward_ins = ObserverWard::new(config.clone(), web_fingerprint, nmap_fingerprint);
     let vec_results = observer_ward_ins.scan(targets).await;
     print_results_and_save(vec_results, &config);