Improve web crawler

helpers/helpers_emba_dependency_check.sh

@@ -388,7 +388,9 @@ dependency_check()
     # binwalk
     check_dep_tool "binwalk extractor" "binwalk"
     if command -v binwalk > /dev/null ; then
-      BINWALK_VER=$(binwalk 2>&1 | grep "Binwalk v" | cut -d+ -f1 | awk '{print $2}' | sed 's/^v//' || true)
+      export BINWALK_BIN=()
+      BINWALK_BIN=("python3" "-Wignore" "$(which binwalk)")
+      BINWALK_VER=$("${BINWALK_BIN[@]}" 2>&1 | grep "Binwalk v" | cut -d+ -f1 | awk '{print $2}' | sed 's/^v//' || true)
       if ! [ "$(version "$BINWALK_VER")" -ge "$(version "2.3.3")" ]; then
         echo -e "$ORANGE""    binwalk version $BINWALK_VER - not optimal""$NC"
         echo -e "$ORANGE""    Upgrade your binwalk to version 2.3.3 or higher""$NC"

modules/L25_web_checks.sh

@@ -321,6 +321,35 @@ web_access_crawler() {
     cd "${HOME_}" || exit 1
   done
 
+  # extract started processes from all our qemu logs:
+  #
+  # find all qemu logs:
+  mapfile -t FILENAME_ARR_QEMU_START < <(find "${LOG_DIR}/l10_system_emulation" -type f -name "qemu.serial.log" | sort -u || true)
+  for FILENAME_QEMU_START in "${FILENAME_ARR_QEMU_START[@]}"; do
+    # find open service names from qemu logs:
+    print_output "[*] Testing Qemu log file: ${FILENAME_QEMU_START}" "no_log"
+    mapfile -t FILENAME_STARTED_PROCESSES_ARR < <(grep -a "inet_bind" "${FILENAME_QEMU_START}" | cut -d: -f3 | awk -F[\(\)] '{print $2}' | sort -u || true)
+    for FILENAME_QEMU_STARTED in "${FILENAME_STARTED_PROCESSES_ARR[@]}"; do
+      print_output "[*] Searching for filename: ${FILENAME_QEMU_STARTED}" "no_log"
+      # find the names in the filesystem:
+      mapfile -t FILEPATH_QEMU_START_ARR < <(find "${LOG_DIR}"/firmware -name "${FILENAME_QEMU_STARTED}" -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3)
+      # check every file for further possible files to crawl:
+      for FILE_QEMU_START in "${FILEPATH_QEMU_START_ARR[@]}"; do
+        if ! file "${FILE_QEMU_START}" | grep -q "ELF"; then
+          continue
+        fi
+        print_output "[*] Identification of possible web files in: ${FILE_QEMU_START}" "no_log"
+        mapfile -t POSSIBLE_FILES_ARR < <(strings "${FILE_QEMU_START}" | grep -o -E '[-_a-zA-Z0-9]+\.[a-zA-Z0-9]{3}$' | sort -u || true)
+        # crawl all the files:
+        for FILE_QEMU_TEST in "${POSSIBLE_FILES_ARR[@]}"; do
+          print_output "[*] Testing ${ORANGE}${PROTO}://${IP_}:${PORT_}/${FILE_QEMU_TEST}${NC}" "no_log"
+          echo -e "\\n[*] Testing ${ORANGE}${PROTO}://${IP_}:${PORT_}/${FILE_QEMU_TEST}${NC}" >> "${LOG_PATH_MODULE}/crawling_${IP_}-${PORT_}.log"
+          timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "${PROTO}""://""${IP_}":"${PORT_}""/""${FILE_QEMU_TEST}" -o /dev/null >> "${LOG_PATH_MODULE}/crawling_$IP_-$PORT_.log" 2>/dev/null || true
+        done
+      done
+    done
+  done
+
   if [[ -f "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" ]]; then
     grep -A1 Testing "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" | grep -i -B1 "200 OK" | grep Testing | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g" | sed "s/.*$IP_:$PORT//" | sort -u >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_-200ok.log" || true
     grep -A1 Testing "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" | grep -i -B1 "401 Unauth" | grep Testing | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g" | sed "s/.*$IP_:$PORT//" | sort -u >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_-401Unauth.log" || true

modules/P02_firmware_bin_file_check.sh

@@ -51,11 +51,11 @@ P02_firmware_bin_file_check() {
     # Docker container and binwalk fails to save the entropy picture there
     if [[ $IN_DOCKER -eq 1 ]] ; then
       cd "$LOG_DIR" || return
-      print_output "$(binwalk -E -F -J "$FIRMWARE_PATH")"
+      print_output "$("${BINWALK_BIN[@]}" -E -F -J "$FIRMWARE_PATH")"
       mv "$(basename "$FIRMWARE_PATH".png)" "$LOG_DIR"/firmware_entropy.png 2> /dev/null || true
       cd /emba || return
     else
-      print_output "$(binwalk -E -F -J "$FIRMWARE_PATH")"
+      print_output "$("${BINWALK_BIN[@]}" -E -F -J "$FIRMWARE_PATH")"
       mv "$(basename "$FIRMWARE_PATH".png)" "$LOG_DIR"/firmware_entropy.png 2> /dev/null || true
     fi
   fi
@@ -131,11 +131,11 @@ generate_entropy_graph() {
     # Docker container and binwalk fails to save the entropy picture there
     if [[ $IN_DOCKER -eq 1 ]] ; then
       cd "$LOG_DIR" || return
-      print_output "$(binwalk -E -F -J "$FIRMWARE_PATH_BIN")"
+      print_output "$("${BINWALK_BIN[@]}" -E -F -J "$FIRMWARE_PATH_BIN")"
       mv "$(basename "$FIRMWARE_PATH_BIN".png)" "$LOG_DIR"/firmware_entropy.png 2> /dev/null || true
       cd /emba || return
     else
-      print_output "$(binwalk -E -F -J "$FIRMWARE_PATH_BIN")"
+      print_output "$("${BINWALK_BIN[@]}" -E -F -J "$FIRMWARE_PATH_BIN")"
       mv "$(basename "$FIRMWARE_PATH_BIN".png)" "$LOG_DIR"/firmware_entropy.png 2> /dev/null || true
     fi
   fi
@@ -154,9 +154,9 @@ fw_bin_detector() {
   FILE_BIN_OUT=$(file "$CHECK_FILE")
   DLINK_ENC_CHECK=$(hexdump -C "$CHECK_FILE" | head -1 || true)
   AVM_CHECK=$(strings "$CHECK_FILE" | grep -c "AVM GmbH .*. All rights reserved.\|(C) Copyright .* AVM" || true)
-  QNAP_ENC_CHECK=$(binwalk -y "qnap encrypted" "$CHECK_FILE")
+  QNAP_ENC_CHECK=$("${BINWALK_BIN[@]}" -y "qnap encrypted" "$CHECK_FILE")
   # we are running binwalk on the file to analyze the output afterwards:
-  binwalk "$CHECK_FILE" > "$TMP_DIR"/s02_binwalk_output.txt
+  "${BINWALK_BIN[@]}" "$CHECK_FILE" > "$TMP_DIR"/s02_binwalk_output.txt
   UEFI_CHECK=$(grep -c "UEFI" "$TMP_DIR"/s02_binwalk_output.txt || true)
   UEFI_CHECK=$(( "$UEFI_CHECK" + "$(grep -c "UEFI" "$CHECK_FILE" || true)" ))
 

modules/P59_binwalk_extractor.sh

@@ -131,7 +131,8 @@ binwalking() {
   sub_module_title "Analyze binary firmware blob with binwalk"
 
   print_output "[*] Basic analysis with binwalk"
-  binwalk "$FIRMWARE_PATH_" | tee -a "$LOG_FILE"
+  # just a quick fix for ignoring the warnings. As binwalk is nearly not used anymore this does not affect EMBA
+  "${BINWALK_BIN[@]}" "$FIRMWARE_PATH_" | tee -a "$LOG_FILE"
 
   print_ln "no_log"
 
@@ -182,16 +183,16 @@ binwalk_deep_extract_helper() {
 
   if [[ "$BINWALK_VER_CHECK" == 1 ]]; then
     if [[ "$MATRYOSHKA_" -eq 1 ]]; then
-      binwalk --run-as=root --preserve-symlinks --dd='.*' -e -M -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
+      "${BINWALK_BIN[@]}" --run-as=root --preserve-symlinks --dd='.*' -e -M -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
     else
       # no more Matryoshka mode ... we are doing it manually and check the files every round via MD5
-      binwalk --run-as=root --preserve-symlinks --dd='.*' -e -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
+      "${BINWALK_BIN[@]}" --run-as=root --preserve-symlinks --dd='.*' -e -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
     fi
   else
     if [[ "$MATRYOSHKA_" -eq 1 ]]; then
-      binwalk --dd='.*' -e -M -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
+      "${BINWALK_BIN[@]}" --dd='.*' -e -M -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
     else
-      binwalk --dd='.*' -e -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
+      "${BINWALK_BIN[@]}" --dd='.*' -e -C "$DEST_FILE_" "$FILE_TO_EXTRACT_" | tee -a "$LOG_FILE" || true
     fi
   fi
 }