From a10a31438cd51b5418787959cfea90813c94019b Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Thu, 16 Feb 2023 16:00:44 +0100 Subject: [PATCH 01/14] disk space monitor --- emba | 3 +++ helpers/helpers_emba_helpers.sh | 36 +++++++++++++++++++++++++++++++++ modules/S36_lighttpd.sh | 4 ++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/emba b/emba index ae02a454e..6023ec180 100755 --- a/emba +++ b/emba @@ -854,6 +854,9 @@ main() if [[ $IN_DOCKER -eq 0 ]] ; then check_cve_search_job "$EMBA_PID" & EXIT_KILL_PIDS+=("$!") + + disk_space_monitor "$EMBA_PID" & + EXIT_KILL_PIDS+=("$!") fi # if $CONTAINER_EXTRACT is set we extract the docker container with id $CONTAINER_ID outside of the diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index d31350a6d..0520353e3 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -314,3 +314,39 @@ module_wait() { sleep 1 done } + +disk_space_monitor() { + local EMBA_PID="${1:-}" + local DDISK="$LOG_DIR" + + if ! [[ "$EMBA_PID" =~ [0-9]+ ]]; then + print_output "[-] WARNING: No EMBA PID detected ... are we really running?!?" "no_log" + return + fi + + while true; do + print_output "[*] Disk space monitoring active" "no_log" + FREE_SPACE=$(df --output=avail "$DDISK" | awk 'NR==2') + if [[ "$FREE_SPACE" -lt 100000 ]]; then + print_ln "no_log" + print_output "[!] WARNING: EMBA is running out of disk space!" "no_log" + print_output "[!] WARNING: EMBA is stopping now" "no_log" + df -h || true + print_ln "no_log" + cleaner 0 + exit 1 + fi + + if [[ -f "$LOG_DIR"/emba.log ]]; then + if grep -q "Test ended\|EMBA failed" "$LOG_DIR"/emba.log 2>/dev/null; then + break + fi + fi + # shellcheck disable=SC2009 + if ! ps aux | grep -v grep | grep -q "$EMBA_PID"; then + break + fi + + sleep 10 + done +} diff --git a/modules/S36_lighttpd.sh b/modules/S36_lighttpd.sh index 5d6d0c8ff..1f38dd119 100755 --- a/modules/S36_lighttpd.sh +++ b/modules/S36_lighttpd.sh @@ -31,8 +31,8 @@ S36_lighttpd() { local LIGHTTP_BIN_ARR=() local FILE="" - readarray -t LIGHTTP_CFG_ARR < <( find "$FIRMWARE_PATH" -xdev "${EXCL_FIND[@]}" -iname '*lighttp*conf*' -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\ -f3 ) - readarray -t LIGHTTP_BIN_ARR < <( find "$FIRMWARE_PATH" -xdev "${EXCL_FIND[@]}" -type f -iname 'lighttpd' -exec file {} \; 2>/dev/null | grep "ELF" | cut -d ':' -f1 | sort -u ) + readarray -t LIGHTTP_CFG_ARR < <( find "$FIRMWARE_PATH" -xdev "${EXCL_FIND[@]}" -iname '*lighttp*conf*' -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\ -f3 || true) + readarray -t LIGHTTP_BIN_ARR < <( find "$FIRMWARE_PATH" -xdev "${EXCL_FIND[@]}" -type f -iname 'lighttpd' -exec file {} \; 2>/dev/null | grep "ELF" | cut -d ':' -f1 | sort -u || true) if [[ ${#LIGHTTP_BIN_ARR[@]} -gt 0 ]] ; then lighttpd_binary_analysis "${LIGHTTP_BIN_ARR[@]}" From c2d582e87e70e4dc17c9a3f20f876c4f48a1660b Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Thu, 16 Feb 2023 19:33:13 +0100 Subject: [PATCH 02/14] EnGenius decryptor update --- README.md | 2 +- helpers/helpers_emba_helpers.sh | 18 ++++++++++++++---- installer/I199_default_tools_github.sh | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 13d59bb19..35dfdb4bf 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ sudo ./installer.sh -d ## Quick start with default scan profile: ```console -sudo ./emba.sh -l ./log -f /firmware -p ./scan-profiles/default-scan.emba +sudo ./emba -l ./log -f /firmware -p ./scan-profiles/default-scan.emba ``` --- diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index 0520353e3..5e2a77297 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -324,8 +324,12 @@ disk_space_monitor() { return fi + while ! [[ -f "$MAIN_LOG" ]]; do + sleep 1 + done + while true; do - print_output "[*] Disk space monitoring active" "no_log" + print_output "[!] Disk space monitoring active" "no_log" FREE_SPACE=$(df --output=avail "$DDISK" | awk 'NR==2') if [[ "$FREE_SPACE" -lt 100000 ]]; then print_ln "no_log" @@ -337,14 +341,20 @@ disk_space_monitor() { exit 1 fi - if [[ -f "$LOG_DIR"/emba.log ]]; then - if grep -q "Test ended\|EMBA failed" "$LOG_DIR"/emba.log 2>/dev/null; then + if [[ -f "$MAIN_LOG" ]]; then + if grep -q "Test ended\|EMBA failed" "$MAIN_LOG" 2>/dev/null; then + print_output "[!] Disk space monitoring stopping - EMBA failed" "no_log" break fi fi # shellcheck disable=SC2009 if ! ps aux | grep -v grep | grep -q "$EMBA_PID"; then - break + print_output "[!] Disk space monitoring stopping - EMBA PID missed #1" "no_log" + sleep 5 + if ! ps aux | grep -v grep | grep -q "$EMBA_PID"; then + print_output "[!] Disk space monitoring stopping - EMBA PID missed #2" "no_log" + break + fi fi sleep 10 diff --git a/installer/I199_default_tools_github.sh b/installer/I199_default_tools_github.sh index 720fb1c2d..d9c76986c 100755 --- a/installer/I199_default_tools_github.sh +++ b/installer/I199_default_tools_github.sh @@ -27,7 +27,7 @@ I199_default_tools_github() { print_file_info "sudo-parser.pl" "Parses and tests sudoers configuration files" "https://raw.githubusercontent.com/CiscoCXSecurity/sudo-parser/master/sudo-parser.pl" "external/sudo-parser.pl" print_file_info "pixd" "pixd is a tool for visualizing binary data using a colour palette." "https://raw.githubusercontent.com/p4cx/pixd_image/master/pixd.c" "external/pixd" print_file_info "progpilot" "progpilot is a tool for static security tests on php files." "https://github.com/designsecurity/progpilot/releases/download/v0.8.0/progpilot_v0.8.0.phar" "external/progpilot" - print_file_info "EnGenius decryptor" "Decrypts EnGenius firmware files." "https://gist.githubusercontent.com/ryancdotorg/914f3ad05bfe0c359b79716f067eaa99/raw/5600956a5bba4c674a010bf27e7eaad25a496b87/decrypt.py" "external/engenius-decrypt.py" + print_file_info "EnGenius decryptor" "Decrypts EnGenius firmware files." "https://raw.githubusercontent.com/EMBA-support-repos/enfringement/main/decrypt.py" "external/engenius-decrypt.py" print_pip_info "pillow" print_git_info "jchroot" "EMBA-support-repos/jchroot" "jchroot - a chroot with more isolation" @@ -46,7 +46,7 @@ I199_default_tools_github() { download_file "sshdcc" "https://raw.githubusercontent.com/sektioneins/sshdcc/master/sshdcc" "external/sshdcc" download_file "sudo-parser.pl" "https://raw.githubusercontent.com/CiscoCXSecurity/sudo-parser/master/sudo-parser.pl" "external/sudo-parser.pl" download_file "progpilot" "https://github.com/designsecurity/progpilot/releases/download/v0.8.0/progpilot_v0.8.0.phar" "external/progpilot" - download_file "EnGenius decryptor" "https://gist.githubusercontent.com/ryancdotorg/914f3ad05bfe0c359b79716f067eaa99/raw/5600956a5bba4c674a010bf27e7eaad25a496b87/decrypt.py" "external/engenius-decrypt.py" + download_file "EnGenius decryptor" "https://raw.githubusercontent.com/EMBA-support-repos/enfringement/main/decrypt.py" "external/engenius-decrypt.py" # pixd installation pip3 install pillow 2>/dev/null From 9a0dc2bdbdab420ab2e98a61fc566880a1fe464e Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Fri, 17 Feb 2023 10:01:43 +0100 Subject: [PATCH 03/14] rpm package check --- emba | 4 +-- helpers/helpers_emba_dependency_check.sh | 3 ++ helpers/helpers_emba_helpers.sh | 21 ++----------- installer/I01_default_apps.sh | 2 ++ modules/L10_system_emulation.sh | 2 +- modules/S08_package_mgmt_extractor.sh | 40 ++++++++++++++++++++++-- 6 files changed, 48 insertions(+), 24 deletions(-) diff --git a/emba b/emba index 6023ec180..340a92516 100755 --- a/emba +++ b/emba @@ -855,7 +855,7 @@ main() check_cve_search_job "$EMBA_PID" & EXIT_KILL_PIDS+=("$!") - disk_space_monitor "$EMBA_PID" & + disk_space_monitor & EXIT_KILL_PIDS+=("$!") fi @@ -949,7 +949,7 @@ main() fi exit 0 else - print_output "[-] EMBA failed in docker mode!" "no_log" + print_output "[-] EMBA failed in docker mode!" "main" cleaner 0 write_notification "EMBA failed analysis in default mode" exit 1 diff --git a/helpers/helpers_emba_dependency_check.sh b/helpers/helpers_emba_dependency_check.sh index da2bf65aa..e248a1181 100755 --- a/helpers/helpers_emba_dependency_check.sh +++ b/helpers/helpers_emba_dependency_check.sh @@ -423,6 +423,9 @@ dependency_check() # jtr check_dep_tool "john" + # rpm for checking package management system + # check_dep_tool "rpm" + # pixd check_dep_file "pixd visualizer" "$EXT_DIR""/pixde" diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index 5e2a77297..ad8e64fae 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -316,14 +316,8 @@ module_wait() { } disk_space_monitor() { - local EMBA_PID="${1:-}" local DDISK="$LOG_DIR" - if ! [[ "$EMBA_PID" =~ [0-9]+ ]]; then - print_output "[-] WARNING: No EMBA PID detected ... are we really running?!?" "no_log" - return - fi - while ! [[ -f "$MAIN_LOG" ]]; do sleep 1 done @@ -333,8 +327,8 @@ disk_space_monitor() { FREE_SPACE=$(df --output=avail "$DDISK" | awk 'NR==2') if [[ "$FREE_SPACE" -lt 100000 ]]; then print_ln "no_log" - print_output "[!] WARNING: EMBA is running out of disk space!" "no_log" - print_output "[!] WARNING: EMBA is stopping now" "no_log" + print_output "[!] WARNING: EMBA is running out of disk space!" "main" + print_output "[!] WARNING: EMBA is stopping now" "main" df -h || true print_ln "no_log" cleaner 0 @@ -343,16 +337,7 @@ disk_space_monitor() { if [[ -f "$MAIN_LOG" ]]; then if grep -q "Test ended\|EMBA failed" "$MAIN_LOG" 2>/dev/null; then - print_output "[!] Disk space monitoring stopping - EMBA failed" "no_log" - break - fi - fi - # shellcheck disable=SC2009 - if ! ps aux | grep -v grep | grep -q "$EMBA_PID"; then - print_output "[!] Disk space monitoring stopping - EMBA PID missed #1" "no_log" - sleep 5 - if ! ps aux | grep -v grep | grep -q "$EMBA_PID"; then - print_output "[!] Disk space monitoring stopping - EMBA PID missed #2" "no_log" + print_output "[!] Disk space monitoring stopping - EMBA failed" "main" break fi fi diff --git a/installer/I01_default_apps.sh b/installer/I01_default_apps.sh index 098ab25ea..f68c1d262 100755 --- a/installer/I01_default_apps.sh +++ b/installer/I01_default_apps.sh @@ -42,6 +42,8 @@ I01_default_apps(){ print_tool_info "git" 1 print_tool_info "strace" 1 + print_tool_info "rpm" 1 + # python3.10-request print_tool_info "python3-pip" 1 print_pip_info "requests" diff --git a/modules/L10_system_emulation.sh b/modules/L10_system_emulation.sh index 59085b286..f25f31c54 100755 --- a/modules/L10_system_emulation.sh +++ b/modules/L10_system_emulation.sh @@ -2069,7 +2069,7 @@ write_results() { local ARCHIVE_PATH_="${1:-}" local TCP_SERV_CNT=0 if [[ -f "$LOG_PATH_MODULE"/"$NMAP_LOG" ]]; then - TCP_SERV_CNT="$(grep "udp.*open\ \|tcp.*open\ " "$LOG_PATH_MODULE"/"$NMAP_LOG" 2>/dev/null | awk '{print $1}' | sort -u | wc -l)" + TCP_SERV_CNT="$(grep "udp.*open\ \|tcp.*open\ " "$LOG_PATH_MODULE"/"$NMAP_LOG" 2>/dev/null | awk '{print $1}' | sort -u | wc -l || true)" fi ARCHIVE_PATH_="$(echo "$ARCHIVE_PATH_" | rev | cut -d '/' -f1 | rev)" echo "$FIRMWARE_PATH_orig;$RESULT_SOURCE;Booted $BOOTED;ICMP $ICMP;TCP-0 $TCP_0;TCP $TCP;$TCP_SERV_CNT;IP address: $IP_ADDRESS_;Network mode: $NETWORK_MODE ($NETWORK_DEVICE/$ETH_INT/$INIT_FILE);$ARCHIVE_PATH_" >> "$LOG_DIR"/emulator_online_results.log diff --git a/modules/S08_package_mgmt_extractor.sh b/modules/S08_package_mgmt_extractor.sh index f9e898f0c..451bbb736 100755 --- a/modules/S08_package_mgmt_extractor.sh +++ b/modules/S08_package_mgmt_extractor.sh @@ -26,10 +26,9 @@ S08_package_mgmt_extractor() debian_status_files_search openwrt_control_files_search - # Future work: rpm, ... - # rpm_package_files_search + rpm_package_files_search - [[ "${#DEBIAN_MGMT_STATUS[@]}" -gt 0 || "${#OPENWRT_MGMT_CONTROL[@]}" -gt 0 ]] && NEG_LOG=1 + [[ "${#DEBIAN_MGMT_STATUS[@]}" -gt 0 || "${#OPENWRT_MGMT_CONTROL[@]}" -gt 0 || "${#RPM_PACKAGES[@]}" -gt 0 ]] && NEG_LOG=1 module_end_log "${FUNCNAME[0]}" "$NEG_LOG" } @@ -112,6 +111,41 @@ openwrt_control_files_search() { fi } +rpm_package_files_search() { + sub_module_title "RPM package management identification" + + if ! command -v rpm > /dev/null; then + print_output "[-] RPM command not found ... not executing RPM test module" + return + fi + + local PACKAGING_SYSTEM="RPM" + local PACKAGE_FILE="" + local RPM_PACKAGES=() + local PACKAGE_VERSION="" + local PACKAGE_NAME="" + + mapfile -t RPM_STARTER < <(find "$FIRMWARE_PATH" "${EXCL_FIND[@]}" -xdev -path "*rpm/Packages" -type f) + + if [[ -v RPM_STARTER[@] ]] ; then + write_csv_log "Packaging system" "package dir" "package" "version" + print_output "[*] Found $ORANGE${#RPM_STARTER[@]}$NC RPM package management directories." + for PACKAGE_FILE in "${RPM_STARTER[@]}" ; do + RPM_DIR="$(dirname "$PACKAGE_FILE")" + mapfile -t RPM_PACKAGES < <(rpm -qa --dbpath "$RPM_DIR" || true) + print_ln + for PACKAGE_AND_VERSION in "${RPM_PACKAGES[@]}" ; do + PACKAGE_VERSION=$(rpm -qi --dbpath "$RPM_DIR" "$PACKAGE_AND_VERSION" | grep Version | awk '{print $3}' || true) + PACKAGE_NAME=$(rpm -qi --dbpath "$RPM_DIR" "$PACKAGE_AND_VERSION" | grep Version | awk '{print $1}' || true) + print_output "[*] RPM package details: $ORANGE$PACKAGE_NAME$NC - $ORANGE$PACKAGE_VERSION$NC" + write_csv_log "$PACKAGING_SYSTEM" "$RPM_DIR" "$PACKAGE_NAME" "$PACKAGE_VERSION" + done + done + else + print_output "[-] No RPM package management found!" + fi +} + clean_package_versions() { local VERSION_="${1:-}" export STRIPPED_VERSION="" From 4ff362bc767ad33e1c14cc0df0cfaf00b5b65b47 Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Fri, 17 Feb 2023 17:00:03 +0100 Subject: [PATCH 04/14] updates --- emba | 2 +- helpers/helpers_emba_helpers.sh | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/emba b/emba index 340a92516..0689768a9 100755 --- a/emba +++ b/emba @@ -855,7 +855,7 @@ main() check_cve_search_job "$EMBA_PID" & EXIT_KILL_PIDS+=("$!") - disk_space_monitor & + disk_space_monitor "$EMBA_PID" & EXIT_KILL_PIDS+=("$!") fi diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index ad8e64fae..bf9ce4d3e 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -83,7 +83,7 @@ max_pids_protection() { cleaner() { INTERRUPT_CLEAN="${1:-1}" if [[ "$INTERRUPT_CLEAN" -eq 1 ]]; then - print_output "[*] User interrupt detected!" "no_log" + print_output "[*] Interrupt detected!" "no_log" fi print_output "[*] Final cleanup started." "no_log" @@ -166,16 +166,16 @@ cleaner() { if [[ -d "$TMP_DIR" ]]; then rm -r "$TMP_DIR" 2>/dev/null || true fi - if [[ "$INTERRUPT_CLEAN" -eq 1 ]]; then - print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"$SECONDS" -u +%H:%M:%S)"" \\n" "no_log" - exit 1 - fi if [[ "$IN_DOCKER" -eq 0 ]]; then for KILL_PID in "${NOTIFICATION_PID[@]}"; do print_output "[*] Stopping EMBA PID $KILL_PID" "no_log" - kill "$KILL_PID" || true + kill "$KILL_PID" > /dev/null || true done fi + if [[ "$INTERRUPT_CLEAN" -eq 1 ]]; then + print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"$SECONDS" -u +%H:%M:%S)"" \\n" "no_log" + exit 1 + fi } emba_updater() { @@ -210,7 +210,6 @@ emba_updater() { # this checks if a function is available function_exists() { - FCT_TO_CHECK="${1:-}" declare -f -F "$FCT_TO_CHECK" > /dev/null return $? @@ -323,7 +322,7 @@ disk_space_monitor() { done while true; do - print_output "[!] Disk space monitoring active" "no_log" + # print_output "[*] Disk space monitoring active" "no_log" FREE_SPACE=$(df --output=avail "$DDISK" | awk 'NR==2') if [[ "$FREE_SPACE" -lt 100000 ]]; then print_ln "no_log" @@ -331,13 +330,12 @@ disk_space_monitor() { print_output "[!] WARNING: EMBA is stopping now" "main" df -h || true print_ln "no_log" - cleaner 0 - exit 1 + cleaner 1 + pkill -f emba.*"$LOG_DIR" fi if [[ -f "$MAIN_LOG" ]]; then if grep -q "Test ended\|EMBA failed" "$MAIN_LOG" 2>/dev/null; then - print_output "[!] Disk space monitoring stopping - EMBA failed" "main" break fi fi From 14439ec56b72d8e7a574dd6f6271f85a6c8a3cd7 Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Fri, 17 Feb 2023 17:17:45 +0100 Subject: [PATCH 05/14] cleanup --- modules/S08_package_mgmt_extractor.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/S08_package_mgmt_extractor.sh b/modules/S08_package_mgmt_extractor.sh index 451bbb736..6e55a213b 100755 --- a/modules/S08_package_mgmt_extractor.sh +++ b/modules/S08_package_mgmt_extractor.sh @@ -120,17 +120,19 @@ rpm_package_files_search() { fi local PACKAGING_SYSTEM="RPM" + local RPM_PACKAGE_DBS=() local PACKAGE_FILE="" local RPM_PACKAGES=() + local RPM_DIR="" local PACKAGE_VERSION="" local PACKAGE_NAME="" - mapfile -t RPM_STARTER < <(find "$FIRMWARE_PATH" "${EXCL_FIND[@]}" -xdev -path "*rpm/Packages" -type f) + mapfile -t RPM_PACKAGE_DBS < <(find "$FIRMWARE_PATH" "${EXCL_FIND[@]}" -xdev -path "*rpm/Packages" -type f) - if [[ -v RPM_STARTER[@] ]] ; then + if [[ -v RPM_PACKAGE_DBS[@] ]] ; then write_csv_log "Packaging system" "package dir" "package" "version" - print_output "[*] Found $ORANGE${#RPM_STARTER[@]}$NC RPM package management directories." - for PACKAGE_FILE in "${RPM_STARTER[@]}" ; do + print_output "[*] Found $ORANGE${#RPM_PACKAGE_DBS[@]}$NC RPM package management directories." + for PACKAGE_FILE in "${RPM_PACKAGE_DBS[@]}" ; do RPM_DIR="$(dirname "$PACKAGE_FILE")" mapfile -t RPM_PACKAGES < <(rpm -qa --dbpath "$RPM_DIR" || true) print_ln @@ -142,7 +144,7 @@ rpm_package_files_search() { done done else - print_output "[-] No RPM package management found!" + print_output "[-] No RPM package management database found!" fi } From f9ce6de92d2dd54b55fdcbc1e110b370b38955ec Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Fri, 17 Feb 2023 22:24:54 +0100 Subject: [PATCH 06/14] ram check --- installer.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/installer.sh b/installer.sh index 7ac4f67ff..d38380778 100755 --- a/installer.sh +++ b/installer.sh @@ -28,6 +28,7 @@ export DOWNLOAD_FILE_LIST=() export INSTALLER_DIR="./installer" if [[ "$STRICT_MODE" -eq 1 ]]; then + export DEBUG_SCRIPT=0 if [[ -f "./helpers/helpers_emba_load_strict_settings.sh" ]]; then # shellcheck source=/dev/null source ./helpers/helpers_emba_load_strict_settings.sh @@ -208,6 +209,13 @@ if [[ "$IN_DOCKER" -eq 0 ]]; then echo "" read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r fi + + TOTAL_MEMORY="$(grep MemTotal /proc/meminfo | awk '{print $2}' || true)" + if [[ "$TOTAL_MEMORY" -lt 4000000 ]]; then + echo -e "\\n""$ORANGE""EMBA installation in default mode needs a minimum of 4Gig of RAM""$NC" + echo "" + read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r + fi fi if [[ $LIST_DEP -eq 0 ]] ; then From a56c7103f1e023b9ed6aa73ae043bece4d9b7cd8 Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Sat, 18 Feb 2023 14:26:21 +0100 Subject: [PATCH 07/14] stacs install update, rpm dep check --- helpers/helpers_emba_dependency_check.sh | 2 +- installer/I108_stacs_password_search.sh | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/helpers/helpers_emba_dependency_check.sh b/helpers/helpers_emba_dependency_check.sh index e248a1181..e0c5bf48c 100755 --- a/helpers/helpers_emba_dependency_check.sh +++ b/helpers/helpers_emba_dependency_check.sh @@ -424,7 +424,7 @@ dependency_check() check_dep_tool "john" # rpm for checking package management system - # check_dep_tool "rpm" + check_dep_tool "rpm" # pixd check_dep_file "pixd visualizer" "$EXT_DIR""/pixde" diff --git a/installer/I108_stacs_password_search.sh b/installer/I108_stacs_password_search.sh index f011c5ec8..481fc8d8f 100755 --- a/installer/I108_stacs_password_search.sh +++ b/installer/I108_stacs_password_search.sh @@ -26,8 +26,11 @@ I108_stacs_password_search() { echo -e "\\nTo find password hashes in firmware files we install STACS and the default rules." + print_tool_info "python3-pip" 1 print_tool_info "libarchive-dev" 1 - print_pip_info "stacs" + print_tool_info "pybind11-dev" 1 + # print_pip_info "stacs" + print_git_info "stacs" "stacscan/stacs" "STACS is a fast, easy to use tool for searching of password hashes in firmware files." print_git_info "stacs-rules" "stacscan/stacs-rules" "STACS is a fast, easy to use tool for searching of password hashes in firmware files." if [[ "$LIST_DEP" -eq 1 ]] || [[ $DOCKER_SETUP -eq 1 ]] ; then @@ -39,6 +42,15 @@ I108_stacs_password_search() { case ${ANSWER:0:1} in y|Y ) apt-get install "${INSTALL_APP_LIST[@]}" -y --no-install-recommends + + if ! [[ -d external/stacs ]]; then + git clone https://github.com/stacscan/stacs.git external/stacs + fi + cd ./external/stacs || ( echo "Could not install EMBA component STACS" && exit 1 ) + pip3 install -U setuptools + python3 setup.py install + cd "$HOME_PATH" || ( echo "Could not install EMBA component STACS" && exit 1 ) + if ! [[ -d external/stacs-rules ]]; then git clone https://github.com/stacscan/stacs-rules.git external/stacs-rules fi @@ -47,7 +59,7 @@ I108_stacs_password_search() { | xargs -I{} bash -c "\ mkdir -p ./tests/fixtures/{}/{positive,negative} ; \ touch ./tests/fixtures/{}/{negative,positive}/.gitignore" || true - pip3 install stacs 2>/dev/null + #pip3 install stacs cd "$HOME_PATH" || ( echo "Could not install EMBA component STACS" && exit 1 ) if command -v stacs > /dev/null ; then echo -e "$GREEN""STACS installed successfully""$NC" From ab564221cc5cfda5ee3c2c49a21d2f41c9d68efa Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Sat, 18 Feb 2023 16:25:35 +0100 Subject: [PATCH 08/14] EMBA updater --- config/emba_updater.init | 2 +- emba | 2 +- helpers/helpers_emba_helpers.sh | 6 +++++- installer.sh | 4 ++++ installer/I108_stacs_password_search.sh | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/config/emba_updater.init b/config/emba_updater.init index 0f9491045..bd2ead9d8 100644 --- a/config/emba_updater.init +++ b/config/emba_updater.init @@ -17,7 +17,7 @@ fi echo "[*] EMBA update - main repository" | tee -a "$LOG_DIR"/emba_update.log cd EMBA_INSTALL_PATH || exit -git pull | tee -a "$LOG_DIR"/emba_update.log +git pull origin master | tee -a "$LOG_DIR"/emba_update.log cd "$BASE_PATH" || exit echo "[*] EMBA update - cve-search update" | tee -a "$LOG_DIR"/emba_update.log diff --git a/emba b/emba index 0689768a9..e731784d0 100755 --- a/emba +++ b/emba @@ -861,7 +861,7 @@ main() # if $CONTAINER_EXTRACT is set we extract the docker container with id $CONTAINER_ID outside of the # EMBA container into log directory - # we do this only outside of the EMBA container - otherwise we will not reach the docker environment + # we do this outside of the EMBA container - otherwise we will not reach the docker environment if [[ "$CONTAINER_EXTRACT" -eq 1 && "$IN_DOCKER" -eq 0 ]] ; then docker_container_extractor "$CONTAINER_ID" fi diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index bf9ce4d3e..4c92c0ada 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -181,7 +181,11 @@ cleaner() { emba_updater() { print_output "[*] EMBA update starting ..." "no_log" - git pull + if [[ -d ./.git ]]; then + git pull origin master + else + print_output "[-] Can't update EMBA without a git version" + fi EMBA="$INVOCATION_PATH" FIRMWARE="$FIRMWARE_PATH" LOG="$LOG_DIR" docker pull embeddedanalyzer/emba diff --git a/installer.sh b/installer.sh index d38380778..d90f7509c 100755 --- a/installer.sh +++ b/installer.sh @@ -143,6 +143,7 @@ fi if grep -q -i wsl /proc/version; then echo -e "\n${ORANGE}INFO: System running in WSL environment!$NC" echo -e "\n${ORANGE}INFO: WSL is currently experimental!$NC" + echo -e "\n${ORANGE}Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites$NC" echo -e "\n${ORANGE}WARNING: If you are using WSL2, disable docker integration from the docker-desktop daemon!$NC" read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r WSL=1 @@ -161,6 +162,7 @@ elif ! grep -q "kali" /etc/debian_version 2>/dev/null ; then elif grep -q "PRETTY_NAME=\"Ubuntu 20.04 LTS\"" /etc/os-release 2>/dev/null ; then echo -e "\\n""$RED""EMBA is not fully supported on Ubuntu 20.04 LTS.""$NC" echo -e "$RED""For EMBA installation you need to update docker-compose manually. See also https://github.com/e-m-b-a/emba/issues/247""$NC" + echo -e "\\n""$ORANGE""Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites""$NC" read -p "If you have updated docker-compose you can press any key to continue ..." -n1 -s -r OTHER_OS=0 # installation procedure identical to kali install UBUNTU_OS=0 # installation procedure identical to kali install @@ -204,6 +206,7 @@ if [[ "$IN_DOCKER" -eq 0 ]]; then if [[ "$FREE_SPACE" -lt 13000000 ]]; then echo -e "\\n""$ORANGE""EMBA installation in default mode needs a minimum of 13Gig for the docker image""$NC" echo -e "\\n""$ORANGE""Please free enough space on /var/lib/docker""$NC" + echo -e "\\n""$ORANGE""Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites""$NC" echo "" df -h || true echo "" @@ -213,6 +216,7 @@ if [[ "$IN_DOCKER" -eq 0 ]]; then TOTAL_MEMORY="$(grep MemTotal /proc/meminfo | awk '{print $2}' || true)" if [[ "$TOTAL_MEMORY" -lt 4000000 ]]; then echo -e "\\n""$ORANGE""EMBA installation in default mode needs a minimum of 4Gig of RAM""$NC" + echo -e "\\n""$ORANGE""Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites""$NC" echo "" read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r fi diff --git a/installer/I108_stacs_password_search.sh b/installer/I108_stacs_password_search.sh index 481fc8d8f..74b397675 100755 --- a/installer/I108_stacs_password_search.sh +++ b/installer/I108_stacs_password_search.sh @@ -59,7 +59,7 @@ I108_stacs_password_search() { | xargs -I{} bash -c "\ mkdir -p ./tests/fixtures/{}/{positive,negative} ; \ touch ./tests/fixtures/{}/{negative,positive}/.gitignore" || true - #pip3 install stacs + # pip3 install stacs cd "$HOME_PATH" || ( echo "Could not install EMBA component STACS" && exit 1 ) if command -v stacs > /dev/null ; then echo -e "$GREEN""STACS installed successfully""$NC" From 981bb21a0a05410eb75822389c7a532c34c58eb1 Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Wed, 22 Feb 2023 12:31:28 +0100 Subject: [PATCH 09/14] store pid --- emba | 34 +++++---- helpers/helpers_emba_helpers.sh | 73 ++++++++++--------- modules/P60_firmware_bin_extractor.sh | 81 ++++++++++++++++++---- modules/S02_UEFI_FwHunt.sh | 4 +- modules/S03_firmware_bin_base_analyzer.sh | 12 +++- modules/S08_package_mgmt_extractor.sh | 1 + modules/S09_firmware_base_version_check.sh | 4 +- modules/S115_usermode_emulator.sh | 4 +- modules/S116_qemu_version_detection.sh | 6 +- modules/S120_cwe_checker.sh | 4 +- modules/S13_weak_func_check.sh | 28 ++++++-- modules/S14_weak_func_radare_check.sh | 28 ++++++-- modules/S20_shell_check.sh | 4 +- modules/S21_python_check.sh | 4 +- modules/S22_php_check.sh | 4 +- modules/S25_kernel_check.sh | 4 +- modules/S99_grepit.sh | 4 +- 17 files changed, 208 insertions(+), 91 deletions(-) diff --git a/emba b/emba index e731784d0..0149aef7a 100755 --- a/emba +++ b/emba @@ -178,7 +178,9 @@ run_modules() if [[ "$MOD_FIN" -eq 0 ]]; then if [[ $THREADING_SET -eq 1 ]]; then "$MODULE_MAIN" & - WAIT_PIDS+=( "$!" ) + MOD_PID="$!" + store_kill_pids "$MOD_PID" + WAIT_PIDS+=( "$MOD_PID" ) max_pids_protection "$MAX_MODS" "${WAIT_PIDS[@]}" else "$MODULE_MAIN" @@ -224,7 +226,9 @@ run_modules() if [[ "$MOD_FIN" -eq 0 ]]; then if [[ $THREADING_SET -eq 1 ]]; then "$MODULE_MAIN" & - WAIT_PIDS+=( "$!" ) + MOD_PID="$!" + store_kill_pids "$MOD_PID" + WAIT_PIDS+=( "$MOD_PID" ) max_pids_protection "$MAX_MODS" "${WAIT_PIDS[@]}" else "$MODULE_MAIN" @@ -288,7 +292,9 @@ run_modules() if [[ "$MOD_FIN" -eq 0 ]]; then if [[ $THREADING_SET -eq 1 ]]; then "$MODULE_MAIN" & - WAIT_PIDS+=( "$!" ) + MOD_PID="$!" + store_kill_pids "$MOD_PID" + WAIT_PIDS+=( "$MOD_PID" ) max_pids_protection "$MAX_MODS" "${WAIT_PIDS[@]}" else "$MODULE_MAIN" @@ -405,7 +411,6 @@ main() if [[ -f "/etc/debian_version" ]] && grep -q kali-rolling /etc/debian_version; then export DISABLE_NOTIFICATIONS=0 # disable notifications and further desktop experience fi - export NOTIFICATION_PID="NA" export NOTIFICATION_ID=0 # initial notification id - needed for notification overlay/replacement export EMBA_ICON="" EMBA_ICON=$(realpath "$HELP_DIR"/emba.svg) @@ -422,7 +427,6 @@ main() # usually no memory limit is needed, but some modules/tools are wild and we need to protect our system export TOTAL_MEMORY=0 TOTAL_MEMORY="$(grep MemTotal /proc/meminfo | awk '{print $2}' || true)" - export EXIT_KILL_PIDS=() import_helper print_ln "no_log" @@ -713,7 +717,7 @@ main() if [[ $IN_DOCKER -eq 0 ]]; then kernel_downloader & K_DOWN_PID="$!" - EXIT_KILL_PIDS+=("$K_DOWN_PID") + # store_kill_pids "$K_DOWN_PID" print_output "[*] Started kernel downloader thread with PID $ORANGE$K_DOWN_PID$NC" "no_log" fi @@ -724,7 +728,8 @@ main() if [[ "$IN_DOCKER" -eq 0 ]]; then print_notification & NOTIFICATION_PID="$!" - EXIT_KILL_PIDS+=("$NOTIFICATION_PID") + # store_kill_pids "$NOTIFICATION_PID" + disown "$NOTIFICATION_PID" 2> /dev/null || true print_output "[*] Original user: $ORANGE${SUDO_USER:-${USER}}$NC" "no_log" print_output "[*] Notification process started with PID $ORANGE${NOTIFICATION_PID}$NC" "no_log" echo "${SUDO_USER:-${USER}}" > "$LOG_DIR"/orig_user.log @@ -853,12 +858,16 @@ main() if [[ $IN_DOCKER -eq 0 ]] ; then check_cve_search_job "$EMBA_PID" & - EXIT_KILL_PIDS+=("$!") - - disk_space_monitor "$EMBA_PID" & - EXIT_KILL_PIDS+=("$!") + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + disown "$TMP_PID" 2> /dev/null || true fi + disk_space_monitor "$EMBA_PID" & + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + disown "$TMP_PID" 2> /dev/null || true + # if $CONTAINER_EXTRACT is set we extract the docker container with id $CONTAINER_ID outside of the # EMBA container into log directory # we do this outside of the EMBA container - otherwise we will not reach the docker environment @@ -1091,9 +1100,6 @@ main() else print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"$SECONDS" -u +%H:%M:%S)"" \\n" "no_log" fi - if [[ "$NOTIFICATION_PID" != "NA" ]]; then - kill "$NOTIFICATION_PID" 2>/dev/null || true - fi write_grep_log "$(date)" "TIMESTAMP" write_grep_log "$(date -d@"$SECONDS" -u +%H:%M:%S)" "DURATION" else diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index 4c92c0ada..263d84cd0 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -103,7 +103,7 @@ cleaner() { if [[ $(grep -i -c S115 "$LOG_DIR"/"$MAIN_LOG_FILE") -eq 1 ]]; then print_output "[*] Terminating qemu processes - check it with ps" "no_log" - killall -9 --quiet -r .*qemu-.*-sta.* || true + killall -9 --quiet -r .*qemu-.*-sta.* > /dev/null || true print_output "[*] Cleaning the emulation environment\\n" "no_log" find "$FIRMWARE_PATH_CP" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null || true find "$LOG_DIR/s115_usermode_emulator" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null || true @@ -125,7 +125,7 @@ cleaner() { if [[ $(grep -i -c S120 "$LOG_DIR"/"$MAIN_LOG_FILE") -eq 1 ]]; then print_output "[*] Terminating cwe-checker processes - check it with ps" "no_log" - killall -9 --quiet -r .*cwe_checker.* || true + killall -9 --quiet -r .*cwe_checker.* > /dev/null || true fi # IF SYS_ONLINE is 1, the live system tester (system mode emulator) was able to setup the box @@ -142,35 +142,40 @@ cleaner() { if ps -p "$K_DOWN_PID" > /dev/null; then # kernel downloader is running in a thread on the host and needs to be stopped now print_output "[*] Stopping kernel downloader thread with PID $K_DOWN_PID" "no_log" - kill "$K_DOWN_PID" || true + kill "$K_DOWN_PID" > /dev/null || true fi fi - if [[ "$IN_DOCKER" -eq 0 ]] && pgrep -f "find ./external/trickest" &> /dev/null 2>&1; then - pkill -f "find ./external/trickest" 2>/dev/null || true - fi - - # just in case we have the temp trickest db left - if [[ -f "$EXT_DIR"/trickest_db-cleaned.txt ]]; then - rm "$EXT_DIR"/trickest_db-cleaned.txt || true - fi if [[ -f "$TMP_DIR"/orig_logdir ]]; then LOG_DIR_HOST=$(cat "$TMP_DIR"/orig_logdir) pkill -f "inotifywait.*$LOG_DIR_HOST" 2>/dev/null || true fi - if [[ -n "${CHECK_CVE_JOB_PID:-}" && "${CHECK_CVE_JOB_PID:-}" -ne 0 ]]; then - kill -9 "$CHECK_CVE_JOB_PID" || true + if [[ "$IN_DOCKER" -eq 1 ]]; then + fuser -k "$LOG_DIR" || true + fuser -k "$FIRMWARE_PATH" || true fi - if [[ -d "$TMP_DIR" ]]; then - rm -r "$TMP_DIR" 2>/dev/null || true + #if [[ "$IN_DOCKER" -eq 1 ]] && [[ -f "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log ]]; then + #while read -r KILL_PID; do + # if [[ -e /proc/"$KILL_PID" ]]; then + # print_output "[*] Stopping EMBA process with PID $KILL_PID" "no_log" + # kill -9 "$KILL_PID" > /dev/null || true + # fi + #done < "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log + #fi + + if [[ "$IN_DOCKER" -eq 0 ]] && [[ -f "$TMP_DIR"/EXIT_KILL_PIDS.log ]]; then + while read -r KILL_PID; do + if [[ -e /proc/"$KILL_PID" ]]; then + print_output "[*] Stopping EMBA process with PID $KILL_PID" "no_log" + kill -9 "$KILL_PID" > /dev/null || true + fi + done < "$TMP_DIR"/EXIT_KILL_PIDS.log fi - if [[ "$IN_DOCKER" -eq 0 ]]; then - for KILL_PID in "${NOTIFICATION_PID[@]}"; do - print_output "[*] Stopping EMBA PID $KILL_PID" "no_log" - kill "$KILL_PID" > /dev/null || true - done + + if [[ "$IN_DOCKER" -eq 0 ]] && [[ -d "$TMP_DIR" ]]; then + rm -r "$TMP_DIR" 2>/dev/null || true fi if [[ "$INTERRUPT_CLEAN" -eq 1 ]]; then print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"$SECONDS" -u +%H:%M:%S)"" \\n" "no_log" @@ -184,7 +189,7 @@ emba_updater() { if [[ -d ./.git ]]; then git pull origin master else - print_output "[-] Can't update EMBA without a git version" + print_output "[-] INFO: Can't update non git version of EMBA" fi EMBA="$INVOCATION_PATH" FIRMWARE="$FIRMWARE_PATH" LOG="$LOG_DIR" docker pull embeddedanalyzer/emba @@ -198,21 +203,12 @@ emba_updater() { /etc/init.d/redis-server start "$EXT_DIR"/cve-search/sbin/db_updater.py -v - print_output "[*] EMBA update - trickest PoC update" "no_log" - if [[ -d "$EXT_DIR"/trickest-cve ]]; then - BASE_PATH=$(pwd) - cd "$EXT_DIR"/trickest-cve || exit - git pull - cd "$BASE_PATH" || exit - else - git clone https://github.com/trickest/cve.git "$EXT_DIR"/trickest-cve - fi - print_output "[*] Please note that this was only a data update and no installed packages were updated." "no_log" print_output "[*] Please restart your EMBA scan to apply the updates ..." "no_log" } # this checks if a function is available +# this means the EMBA module was loaded function_exists() { FCT_TO_CHECK="${1:-}" declare -f -F "$FCT_TO_CHECK" > /dev/null @@ -318,6 +314,14 @@ module_wait() { done } +store_kill_pids() { + local PID="${1:-}" + ! [[ -d "$TMP_DIR" ]] && mkdir -p "$TMP_DIR" + [[ "$IN_DOCKER" -eq 0 ]] && echo "$PID" >> "$TMP_DIR"/EXIT_KILL_PIDS.log + [[ "$IN_DOCKER" -eq 1 ]] && echo "$PID" >> "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log + return 0 +} + disk_space_monitor() { local DDISK="$LOG_DIR" @@ -328,14 +332,15 @@ disk_space_monitor() { while true; do # print_output "[*] Disk space monitoring active" "no_log" FREE_SPACE=$(df --output=avail "$DDISK" | awk 'NR==2') - if [[ "$FREE_SPACE" -lt 100000 ]]; then + if [[ "$FREE_SPACE" -lt 10000000 ]]; then print_ln "no_log" print_output "[!] WARNING: EMBA is running out of disk space!" "main" print_output "[!] WARNING: EMBA is stopping now" "main" df -h || true print_ln "no_log" + # give the container some more seconds for the cleanup process + [[ "$IN_DOCKER" -eq 0 ]] && sleep 5 cleaner 1 - pkill -f emba.*"$LOG_DIR" fi if [[ -f "$MAIN_LOG" ]]; then @@ -344,6 +349,6 @@ disk_space_monitor() { fi fi - sleep 10 + sleep 5 done } diff --git a/modules/P60_firmware_bin_extractor.sh b/modules/P60_firmware_bin_extractor.sh index 420a08d96..122ab9ecf 100755 --- a/modules/P60_firmware_bin_extractor.sh +++ b/modules/P60_firmware_bin_extractor.sh @@ -86,15 +86,19 @@ check_disk_space() { disk_space_protection() { local SEARCHER="${1:-}" + local DDISK="$LOG_DIR" + local FREE_SPACE="" check_disk_space - if [[ "$DISK_SPACE" -gt "$MAX_EXT_SPACE" ]]; then + FREE_SPACE=$(df --output=avail "$DDISK" | awk 'NR==2') + if [[ "$FREE_SPACE" -lt 100000 ]] || [[ "$DISK_SPACE" -gt "$MAX_EXT_SPACE" ]]; then print_ln "no_log" print_output "[!] $(date) - Extractor needs too much disk space $DISK_SPACE" "main" print_output "[!] $(date) - Ending extraction processes" "main" pgrep -a -f "binwalk.*$SEARCHER.*" || true pkill -f ".*binwalk.*$SEARCHER.*" || true pkill -f ".*extract\.py.*$SEARCHER.*" || true + # PID is from wait_for_extractor kill -9 "$PID" 2>/dev/null || true DISK_SPACE_CRIT=1 fi @@ -177,84 +181,120 @@ deeper_extractor_helper() { if [[ "$VMDK_DETECTED" -eq 1 ]]; then if [[ "$THREADED" -eq 1 ]]; then vmdk_extractor "$FILE_TMP" "${FILE_TMP}_vmdk_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else vmdk_extractor "$FILE_TMP" "${FILE_TMP}_vmdk_extracted" fi elif [[ "$UBI_IMAGE" -eq 1 ]]; then if [[ "$THREADED" -eq 1 ]]; then ubi_extractor "$FILE_TMP" "${FILE_TMP}_ubi_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else ubi_extractor "$FILE_TMP" "${FILE_TMP}_ubi_extracted" fi elif [[ "$DLINK_ENC_DETECTED" -eq 1 ]]; then if [[ "$THREADED" -eq 1 ]]; then dlink_SHRS_enc_extractor "$FILE_TMP" "${FILE_TMP}_shrs_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else dlink_SHRS_enc_extractor "$FILE_TMP" "${FILE_TMP}_shrs_extracted" fi elif [[ "$DLINK_ENC_DETECTED" -eq 2 ]]; then if [[ "$THREADED" -eq 1 ]]; then dlink_enc_img_extractor "$FILE_TMP" "${FILE_TMP}_enc_img_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else dlink_enc_img_extractor "$FILE_TMP" "${FILE_TMP}_enc_img_extracted" fi elif [[ "$EXT_IMAGE" -eq 1 ]]; then if [[ "$THREADED" -eq 1 ]]; then ext_extractor "$FILE_TMP" "${FILE_TMP}_ext_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else ext_extractor "$FILE_TMP" "${FILE_TMP}_ext_extracted" fi elif [[ "$ENGENIUS_ENC_DETECTED" -ne 0 ]]; then if [[ "$THREADED" -eq 1 ]]; then engenius_enc_extractor "$FILE_TMP" "${FILE_TMP}_engenius_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else engenius_enc_extractor "$FILE_TMP" "${FILE_TMP}_engenius_extracted" fi elif [[ "$BSD_UFS" -ne 0 ]]; then if [[ "$THREADED" -eq 1 ]]; then ufs_extractor "$FILE_TMP" "${FILE_TMP}_bsd_ufs_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else ufs_extractor "$FILE_TMP" "${FILE_TMP}_bsd_ufs_extracted" fi elif [[ "$ANDROID_OTA" -ne 0 ]]; then if [[ "$THREADED" -eq 1 ]]; then android_ota_extractor "$FILE_TMP" "${FILE_TMP}_android_ota_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else android_ota_extractor "$FILE_TMP" "${FILE_TMP}_android_ota_extracted" fi elif [[ "$OPENSSL_ENC_DETECTED" -ne 0 ]]; then if [[ "$THREADED" -eq 1 ]]; then foscam_enc_extractor "$FILE_TMP" "${FILE_TMP}_foscam_enc_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else foscam_enc_extractor "$FILE_TMP" "${FILE_TMP}_foscam_enc_extracted" fi elif [[ "$BUFFALO_ENC_DETECTED" -ne 0 ]]; then if [[ "$THREADED" -eq 1 ]]; then buffalo_enc_extractor "$FILE_TMP" "${FILE_TMP}_buffalo_enc_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else buffalo_enc_extractor "$FILE_TMP" "${FILE_TMP}_buffalo_enc_extracted" fi elif [[ "$ZYXEL_ZIP" -ne 0 ]]; then if [[ "$THREADED" -eq 1 ]]; then zyxel_zip_extractor "$FILE_TMP" "${FILE_TMP}_zyxel_enc_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else zyxel_zip_extractor "$FILE_TMP" "${FILE_TMP}_zyxel_enc_extracted" fi elif [[ "$QCOW_DETECTED" -ne 0 ]]; then if [[ "$THREADED" -eq 1 ]]; then qcow_extractor "$FILE_TMP" "${FILE_TMP}_qemu_qcow_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else qcow_extractor "$FILE_TMP" "${FILE_TMP}_qemu_qcow_extracted" fi @@ -263,7 +303,10 @@ deeper_extractor_helper() { # default case to binwalk if [[ "$THREADED" -eq 1 ]]; then binwalk_deep_extract_helper "$MATRYOSHKA" "$FILE_TMP" "${FILE_TMP}_binwalk_extracted" & - WAIT_PIDS_P20+=( "$!" ) + BIN_PID="$!" + store_kill_pids "$BIN_PID" + disown "$BIN_PID" 2> /dev/null || true + WAIT_PIDS_P20+=( "$BIN_PID" ) else binwalk_deep_extract_helper "$MATRYOSHKA" "$FILE_TMP" "${FILE_TMP}_binwalk_extracted" fi @@ -275,7 +318,15 @@ deeper_extractor_helper() { check_disk_space - if [[ "$DISK_SPACE" -gt "$MAX_EXT_SPACE" ]]; then + FREE_SPACE=$(df --output=avail "$LOG_DIR" | awk 'NR==2') + if [[ "$FREE_SPACE" -lt 100000 ]]; then + # this stops the complete EMBA test + print_output "[!] $(date) - The system is running out of disk space $ORANGE$FREE_SPACE$NC" "main" + print_output "[!] $(date) - Ending EMBA firmware analysis processes" "main" + cleaner 1 + exit + elif [[ "$DISK_SPACE" -gt "$MAX_EXT_SPACE" ]]; then + # this stops the deep extractor but not EMBA print_output "[!] $(date) - Extractor needs too much disk space $DISK_SPACE" "main" print_output "[!] $(date) - Ending extraction processes" "main" DISK_SPACE_CRIT=1 diff --git a/modules/S02_UEFI_FwHunt.sh b/modules/S02_UEFI_FwHunt.sh index 90f0a336e..ecaf96c43 100755 --- a/modules/S02_UEFI_FwHunt.sh +++ b/modules/S02_UEFI_FwHunt.sh @@ -34,7 +34,9 @@ S02_UEFI_FwHunt() { for EXTRACTED_FILE in "${FILE_ARR_LIMITED[@]}"; do if [[ $THREADED -eq 1 ]]; then fwhunter "$EXTRACTED_FILE" & - WAIT_PIDS_S02+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S02+=( "$TMP_PID" ) max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_S02[@]}" else fwhunter "$EXTRACTED_FILE" diff --git a/modules/S03_firmware_bin_base_analyzer.sh b/modules/S03_firmware_bin_base_analyzer.sh index edb6733ce..6766fe0a9 100755 --- a/modules/S03_firmware_bin_base_analyzer.sh +++ b/modules/S03_firmware_bin_base_analyzer.sh @@ -33,7 +33,9 @@ S03_firmware_bin_base_analyzer() { export OUTPUT_DIR="$FIRMWARE_PATH_CP" if [[ $THREADED -eq 1 ]]; then os_identification & - WAIT_PIDS_S03+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S03+=( "$TMP_PID" ) else os_identification fi @@ -47,7 +49,9 @@ S03_firmware_bin_base_analyzer() { if [[ $RTOS -eq 1 ]] ; then if [[ $THREADED -eq 1 ]]; then binary_architecture_detection "$FIRMWARE_PATH_BAK" & - WAIT_PIDS_S03+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S03+=( "$TMP_PID" ) else binary_architecture_detection "$FIRMWARE_PATH_BAK" fi @@ -89,7 +93,9 @@ os_identification() { for OS in "${OS_SEARCHER[@]}"; do if [[ $THREADED -eq 1 ]]; then os_detection_thread_per_os "$OS" & - WAIT_PIDS_S03_1+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S03_1+=( "$TMP_PID" ) else os_detection_thread_per_os "$OS" fi diff --git a/modules/S08_package_mgmt_extractor.sh b/modules/S08_package_mgmt_extractor.sh index 6e55a213b..ab20a0607 100755 --- a/modules/S08_package_mgmt_extractor.sh +++ b/modules/S08_package_mgmt_extractor.sh @@ -134,6 +134,7 @@ rpm_package_files_search() { print_output "[*] Found $ORANGE${#RPM_PACKAGE_DBS[@]}$NC RPM package management directories." for PACKAGE_FILE in "${RPM_PACKAGE_DBS[@]}" ; do RPM_DIR="$(dirname "$PACKAGE_FILE")" + # not sure this works on an offline system - we need further tests on this: mapfile -t RPM_PACKAGES < <(rpm -qa --dbpath "$RPM_DIR" || true) print_ln for PACKAGE_AND_VERSION in "${RPM_PACKAGES[@]}" ; do diff --git a/modules/S09_firmware_base_version_check.sh b/modules/S09_firmware_base_version_check.sh index 1cffcbced..dbbc35a2b 100755 --- a/modules/S09_firmware_base_version_check.sh +++ b/modules/S09_firmware_base_version_check.sh @@ -134,7 +134,9 @@ S09_firmware_base_version_check() { if [[ "$THREADED" -eq 1 ]]; then # this will burn the CPU but in most cases the time of testing is cut into half bin_string_checker & - WAIT_PIDS_S09+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S09+=( "$TMP_PID" ) else bin_string_checker fi diff --git a/modules/S115_usermode_emulator.sh b/modules/S115_usermode_emulator.sh index b6b25b7c6..ccf4cd755 100755 --- a/modules/S115_usermode_emulator.sh +++ b/modules/S115_usermode_emulator.sh @@ -175,7 +175,9 @@ S115_usermode_emulator() { prepare_emulator "$R_PATH" "$EMULATOR" if [[ "$THREADED" -eq 1 ]]; then emulate_binary "$EMULATOR" "$R_PATH" "$BIN_" & - WAIT_PIDS_S115+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S115+=( "$TMP_PID" ) max_pids_protection "$MAX_THREADS_S115" "${WAIT_PIDS_S115[@]}" else emulate_binary "$EMULATOR" "$R_PATH" "$BIN_" diff --git a/modules/S116_qemu_version_detection.sh b/modules/S116_qemu_version_detection.sh index e106db8dd..9e40ae2a1 100755 --- a/modules/S116_qemu_version_detection.sh +++ b/modules/S116_qemu_version_detection.sh @@ -41,14 +41,16 @@ S116_qemu_version_detection() { if [[ $THREADED -eq 1 ]]; then version_detection_thread "$VERSION_LINE" & - WAIT_PIDS_F05+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S116+=( "$TMP_PID" ) else version_detection_thread "$VERSION_LINE" fi done < "$CONFIG_DIR"/bin_version_strings.cfg print_ln "no_log" - [[ $THREADED -eq 1 ]] && wait_for_pid "${WAIT_PIDS_F05[@]}" + [[ $THREADED -eq 1 ]] && wait_for_pid "${WAIT_PIDS_S116[@]}" if [[ $(wc -l "$CSV_DIR"/s116_qemu_version_detection.csv | awk '{print $1}' ) -gt 1 ]]; then NEG_LOG=1 fi diff --git a/modules/S120_cwe_checker.sh b/modules/S120_cwe_checker.sh index 39734e145..37e80d730 100755 --- a/modules/S120_cwe_checker.sh +++ b/modules/S120_cwe_checker.sh @@ -96,7 +96,9 @@ cwe_check() { fi cwe_checker_threaded "$BINARY" & - WAIT_PIDS_S120+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S120+=( "$TMP_PID" ) max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_S120[@]}" continue else diff --git a/modules/S13_weak_func_check.sh b/modules/S13_weak_func_check.sh index ab0080988..8ede4765c 100755 --- a/modules/S13_weak_func_check.sh +++ b/modules/S13_weak_func_check.sh @@ -58,21 +58,27 @@ S13_weak_func_check() if ( file "$BINARY" | grep -q "x86-64" ) ; then if [[ "$THREADED" -eq 1 ]]; then function_check_x86_64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S13+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S13+=( "$TMP_PID" ) else function_check_x86_64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi elif ( file "$BINARY" | grep -q "Intel 80386" ) ; then if [[ "$THREADED" -eq 1 ]]; then function_check_x86 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S13+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S13+=( "$TMP_PID" ) else function_check_x86 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi elif ( file "$BINARY" | grep -q "32-bit.*ARM" ) ; then if [[ "$THREADED" -eq 1 ]]; then function_check_ARM32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S13+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S13+=( "$TMP_PID" ) else function_check_ARM32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi @@ -80,7 +86,9 @@ S13_weak_func_check() # ARM 64 code is in alpha state and nearly not tested! if [[ "$THREADED" -eq 1 ]]; then function_check_ARM64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S13+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S13+=( "$TMP_PID" ) else function_check_ARM64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi @@ -88,21 +96,27 @@ S13_weak_func_check() # MIPS32 and MIPS64 if [[ "$THREADED" -eq 1 ]]; then function_check_MIPS "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S13+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S13+=( "$TMP_PID" ) else function_check_MIPS "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi elif ( file "$BINARY" | grep -q "PowerPC" ) ; then if [[ "$THREADED" -eq 1 ]]; then function_check_PPC32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S13+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S13+=( "$TMP_PID" ) else function_check_PPC32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi elif ( file "$BINARY" | grep -q "Altera Nios II" ) ; then if [[ "$THREADED" -eq 1 ]]; then function_check_NIOS2 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S13+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S13+=( "$TMP_PID" ) else function_check_NIOS2 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi diff --git a/modules/S14_weak_func_radare_check.sh b/modules/S14_weak_func_radare_check.sh index 1b0bd6359..71a756d9f 100755 --- a/modules/S14_weak_func_radare_check.sh +++ b/modules/S14_weak_func_radare_check.sh @@ -58,21 +58,27 @@ S14_weak_func_radare_check() if ( file "$BINARY" | grep -q "x86-64" ) ; then if [[ "$THREADED" -eq 1 ]]; then radare_function_check_x86_64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S14+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S14+=( "$TMP_PID" ) else radare_function_check_x86_64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi elif ( file "$BINARY" | grep -q "Intel 80386" ) ; then if [[ "$THREADED" -eq 1 ]]; then radare_function_check_x86 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S14+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S14+=( "$TMP_PID" ) else radare_function_check_x86 fi elif ( file "$BINARY" | grep -q "32-bit.*ARM" ) ; then if [[ "$THREADED" -eq 1 ]]; then radare_function_check_ARM32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S14+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S14+=( "$TMP_PID" ) else radare_function_check_ARM32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi @@ -80,7 +86,9 @@ S14_weak_func_radare_check() # ARM 64 code is in alpha state and nearly not tested! if [[ "$THREADED" -eq 1 ]]; then radare_function_check_ARM64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S14+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S14+=( "$TMP_PID" ) else radare_function_check_ARM64 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi @@ -88,21 +96,27 @@ S14_weak_func_radare_check() # MIPS32 and MIPS64 if [[ "$THREADED" -eq 1 ]]; then radare_function_check_MIPS "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S14+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S14+=( "$TMP_PID" ) else radare_function_check_MIPS "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi elif ( file "$BINARY" | grep -q "PowerPC" ) ; then if [[ "$THREADED" -eq 1 ]]; then radare_function_check_PPC32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S14+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S14+=( "$TMP_PID" ) else radare_function_check_PPC32 "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi elif ( file "$BINARY" | grep -q "QUALCOMM DSP6" ) ; then if [[ "$THREADED" -eq 1 ]]; then radare_function_check_hexagon "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" & - WAIT_PIDS_S14+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S14+=( "$TMP_PID" ) else radare_function_check_hexagon "$BINARY" "${VULNERABLE_FUNCTIONS[@]}" fi diff --git a/modules/S20_shell_check.sh b/modules/S20_shell_check.sh index 0441f0cad..b16fb76df 100755 --- a/modules/S20_shell_check.sh +++ b/modules/S20_shell_check.sh @@ -40,7 +40,9 @@ S20_shell_check() ((S20_SCRIPTS+=1)) if [[ "$THREADED" -eq 1 ]]; then s20_script_check "$SH_SCRIPT" & - WAIT_PIDS_S20+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S20+=( "$TMP_PID" ) max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_S20[@]}" continue else diff --git a/modules/S21_python_check.sh b/modules/S21_python_check.sh index 603b7bc23..ef43b398c 100755 --- a/modules/S21_python_check.sh +++ b/modules/S21_python_check.sh @@ -36,7 +36,9 @@ S21_python_check() ((S21_PY_SCRIPTS+=1)) if [[ "$THREADED" -eq 1 ]]; then s21_script_bandit "$PY_SCRIPT" & - WAIT_PIDS_S21+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S21+=( "$TMP_PID" ) max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_S21[@]}" continue else diff --git a/modules/S22_php_check.sh b/modules/S22_php_check.sh index 36ac1d787..bf9a69a30 100755 --- a/modules/S22_php_check.sh +++ b/modules/S22_php_check.sh @@ -74,7 +74,9 @@ s22_vuln_check_caller() { ((S22_PHP_SCRIPTS+=1)) if [[ "$THREADED" -eq 1 ]]; then s22_vuln_check "$PHP_SCRIPT" & - WAIT_PIDS_S22+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S22+=( "$TMP_PID" ) max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_S22[@]}" continue else diff --git a/modules/S25_kernel_check.sh b/modules/S25_kernel_check.sh index d76ac3352..7c06e5726 100755 --- a/modules/S25_kernel_check.sh +++ b/modules/S25_kernel_check.sh @@ -246,7 +246,9 @@ analyze_kernel_module() { # modinfos can run in parallel: if [[ "$THREADED" -eq 1 ]]; then module_analyzer "$KMODULE" & - WAIT_PIDS_S25+=( "$!" ) + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_S25+=( "$TMP_PID" ) else module_analyzer "$KMODULE" fi diff --git a/modules/S99_grepit.sh b/modules/S99_grepit.sh index 2253e2c7a..cd86e1460 100755 --- a/modules/S99_grepit.sh +++ b/modules/S99_grepit.sh @@ -75,7 +75,9 @@ S99_grepit() { if [[ $THREADED -eq 1 ]]; then for GREPIT_MODULE in "${GREPIT_MODULES[@]}"; do "$GREPIT_MODULE" & - WAIT_PIDS_S99+=( "$!" ) + local TMP_PID="$!" + WAIT_PIDS_S99+=( "$TMP_PID" ) + store_kill_pids "$TMP_PID" max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_S99[@]}" done else From 7ea0f69cfc56956b81576e11d5649afd577a4725 Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Wed, 22 Feb 2023 12:35:45 +0100 Subject: [PATCH 10/14] cleanup --- emba | 4 ++-- helpers/helpers_emba_dependency_check.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/emba b/emba index 0149aef7a..9f38b94a5 100755 --- a/emba +++ b/emba @@ -717,7 +717,7 @@ main() if [[ $IN_DOCKER -eq 0 ]]; then kernel_downloader & K_DOWN_PID="$!" - # store_kill_pids "$K_DOWN_PID" + store_kill_pids "$K_DOWN_PID" print_output "[*] Started kernel downloader thread with PID $ORANGE$K_DOWN_PID$NC" "no_log" fi @@ -728,7 +728,7 @@ main() if [[ "$IN_DOCKER" -eq 0 ]]; then print_notification & NOTIFICATION_PID="$!" - # store_kill_pids "$NOTIFICATION_PID" + store_kill_pids "$NOTIFICATION_PID" disown "$NOTIFICATION_PID" 2> /dev/null || true print_output "[*] Original user: $ORANGE${SUDO_USER:-${USER}}$NC" "no_log" print_output "[*] Notification process started with PID $ORANGE${NOTIFICATION_PID}$NC" "no_log" diff --git a/helpers/helpers_emba_dependency_check.sh b/helpers/helpers_emba_dependency_check.sh index e0c5bf48c..59eaeac9e 100755 --- a/helpers/helpers_emba_dependency_check.sh +++ b/helpers/helpers_emba_dependency_check.sh @@ -424,7 +424,8 @@ dependency_check() check_dep_tool "john" # rpm for checking package management system - check_dep_tool "rpm" + # module not fully tested - needs some further work + # check_dep_tool "rpm" # pixd check_dep_file "pixd visualizer" "$EXT_DIR""/pixde" From d9fa2be4eddb8ca8912729a7ffcdce21d48fd427 Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Wed, 22 Feb 2023 15:24:10 +0100 Subject: [PATCH 11/14] cleanup --- helpers/helpers_emba_helpers.sh | 8 ++++---- helpers/helpers_emba_system_emulation.sh | 3 ++- modules/S08_package_mgmt_extractor.sh | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index 263d84cd0..5e054b244 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -156,14 +156,14 @@ cleaner() { fuser -k "$FIRMWARE_PATH" || true fi - #if [[ "$IN_DOCKER" -eq 1 ]] && [[ -f "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log ]]; then - #while read -r KILL_PID; do + # if [[ "$IN_DOCKER" -eq 1 ]] && [[ -f "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log ]]; then + # while read -r KILL_PID; do # if [[ -e /proc/"$KILL_PID" ]]; then # print_output "[*] Stopping EMBA process with PID $KILL_PID" "no_log" # kill -9 "$KILL_PID" > /dev/null || true # fi - #done < "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log - #fi + # done < "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log + # fi if [[ "$IN_DOCKER" -eq 0 ]] && [[ -f "$TMP_DIR"/EXIT_KILL_PIDS.log ]]; then while read -r KILL_PID; do diff --git a/helpers/helpers_emba_system_emulation.sh b/helpers/helpers_emba_system_emulation.sh index 00683700d..2b9cf87a2 100755 --- a/helpers/helpers_emba_system_emulation.sh +++ b/helpers/helpers_emba_system_emulation.sh @@ -35,9 +35,10 @@ restart_emulation() { reset_network_emulation 2 # what an ugly hack - probably we are going to improve this later on + local HOME_PATH="$(pwd)" cd "$ARCHIVE_PATH" || (print_output "[-] Emulation archive path not found") ./run.sh & - cd "$INVOCATION_PATH" || (print_output "[-] EMBA path not available?") + cd "$HOME_PATH" || (print_output "[-] EMBA path not available?") COUNTER=0 while ! ping -c 1 "$IP_ADDRESS_" &> /dev/null; do diff --git a/modules/S08_package_mgmt_extractor.sh b/modules/S08_package_mgmt_extractor.sh index ab20a0607..88ba3c719 100755 --- a/modules/S08_package_mgmt_extractor.sh +++ b/modules/S08_package_mgmt_extractor.sh @@ -113,6 +113,7 @@ openwrt_control_files_search() { rpm_package_files_search() { sub_module_title "RPM package management identification" + export RPM_PACKAGES=() if ! command -v rpm > /dev/null; then print_output "[-] RPM command not found ... not executing RPM test module" @@ -122,7 +123,6 @@ rpm_package_files_search() { local PACKAGING_SYSTEM="RPM" local RPM_PACKAGE_DBS=() local PACKAGE_FILE="" - local RPM_PACKAGES=() local RPM_DIR="" local PACKAGE_VERSION="" local PACKAGE_NAME="" From 2c6e137ae3ddc3ccffd4fdbfda06c39b43b2cbf9 Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Wed, 22 Feb 2023 15:29:17 +0100 Subject: [PATCH 12/14] cleanup --- helpers/helpers_emba_system_emulation.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/helpers_emba_system_emulation.sh b/helpers/helpers_emba_system_emulation.sh index 2b9cf87a2..df75df896 100755 --- a/helpers/helpers_emba_system_emulation.sh +++ b/helpers/helpers_emba_system_emulation.sh @@ -35,7 +35,8 @@ restart_emulation() { reset_network_emulation 2 # what an ugly hack - probably we are going to improve this later on - local HOME_PATH="$(pwd)" + local HOME_PATH="" + HOME_PATH="$(pwd)" cd "$ARCHIVE_PATH" || (print_output "[-] Emulation archive path not found") ./run.sh & cd "$HOME_PATH" || (print_output "[-] EMBA path not available?") From 6c1c5d062333f9bbf87c45194003e770b8ccd1df Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Wed, 22 Feb 2023 19:01:43 +0100 Subject: [PATCH 13/14] error messages --- modules/S115_usermode_emulator.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/S115_usermode_emulator.sh b/modules/S115_usermode_emulator.sh index ccf4cd755..bba711eb5 100755 --- a/modules/S115_usermode_emulator.sh +++ b/modules/S115_usermode_emulator.sh @@ -651,7 +651,7 @@ emulate_binary() { # now we kill all older qemu-processes: # if we use the correct identifier $EMULATOR it will not work ... # This is very ugly and should only be used in docker environment! - pkill -9 -O "$QRUNTIME" -f .*qemu-.*-sta.* || true + pkill -9 -O "$QRUNTIME" -f .*qemu-.*-sta.* >/dev/null || true write_log "\\n-----------------------------------------------------------------\\n" "$LOG_FILE_BIN" } @@ -664,7 +664,7 @@ check_disk_space_emu() { if pgrep -f "$EMULATOR.*$KILLER" > /dev/null; then print_output "[!] Qemu processes are wasting disk space ... we try to kill it" "no_log" print_output "[*] Killing process ${ORANGE}$EMULATOR.*$KILLER.*${NC}" "no_log" - pkill -f "$EMULATOR.*$KILLER.*" || true + pkill -f "$EMULATOR.*$KILLER.*" >/dev/null|| true # rm "$LOG_DIR"/qemu_emulator/*"$KILLER"* fi done @@ -690,7 +690,7 @@ kill_qemu_threader() { # Currently this should only used in docker environment! while true; do print_output "[*] KILLING qemu processes" "no_log" - pkill -9 -O 240 -f .*qemu-.*-sta.* || true + pkill -9 -O 240 -f .*qemu-.*-sta.* >/dev/null || true sleep 20 done } @@ -742,15 +742,15 @@ s115_cleanup() { # if no emulation at all was possible the $EMULATOR variable is not defined if [[ -n "$EMULATOR" ]]; then print_output "[*] Terminating qemu processes - check it with ps" - pkill -9 -f .*qemu-.*-sta.* || true + pkill -9 -f .*qemu-.*-sta.* >/dev/null || true fi CJOBS_=$(pgrep -f qemu- || true) if [[ -n "$CJOBS_" ]] ; then print_output "[*] More emulation jobs are running ... we kill it with fire\\n" - pkill -9 -f .*"$EMULATOR".* || true + pkill -9 -f .*"$EMULATOR".* >/dev/null || true fi - kill -9 "$PID_killer" || true + kill -9 "$PID_killer" >/dev/null || true print_output "[*] Cleaning the emulation environment\\n" find "$EMULATION_PATH_BASE" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null || true From ff6ca081e17933f19d64a25b32f64121a790a88d Mon Sep 17 00:00:00 2001 From: m-1-k-3 Date: Thu, 23 Feb 2023 08:27:38 +0100 Subject: [PATCH 14/14] improve init to rdinit switch in L10 --- modules/L10_system_emulation.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/modules/L10_system_emulation.sh b/modules/L10_system_emulation.sh index f25f31c54..20981d3c6 100755 --- a/modules/L10_system_emulation.sh +++ b/modules/L10_system_emulation.sh @@ -516,6 +516,7 @@ main_emulation() { elif [[ "$F_STARTUP" -eq 0 && "$NETWORK_MODE" == "None" ]] || \ [[ "$F_STARTUP" -eq 0 && "$NETWORK_MODE" == "default" ]] || [[ "$DETECTED_IP" -eq 0 ]]; then mv "$LOG_PATH_MODULE"/qemu.initial.serial.log "$LOG_PATH_MODULE"/qemu.initial.serial_"$IMAGE_NAME"_"$INIT_FNAME"_base_init.log + COUNTING_1st=$(wc -l "$LOG_PATH_MODULE"/qemu.initial.serial_"$IMAGE_NAME"_"$INIT_FNAME"_base_init.log | awk '{print $1}') if [[ "$KINIT" == "rdinit="* ]]; then print_output "[*] Warning: Unknown EMBA startup found via rdinit - testing init" # strip rd from rdinit @@ -533,17 +534,21 @@ main_emulation() { # now we need to check if something is better now or we should switch back to the original init F_STARTUP=$(grep -a -c "EMBA preInit script starting" "$LOG_PATH_MODULE"/qemu.initial.serial.log || true) F_STARTUP=$(( "$F_STARTUP" + "$(grep -a -c "Network configuration - ACTION" "$LOG_PATH_MODULE"/qemu.initial.serial.log || true)" )) + COUNTING_2nd=$(wc -l "$LOG_PATH_MODULE"/qemu.initial.serial.log | awk '{print $1}') # IPS_INT_VLAN is always at least 1 for the default configuration if [[ "${#PANICS[@]}" -gt 0 ]] || [[ "$F_STARTUP" -eq 0 && "${#IPS_INT_VLAN[@]}" -lt 2 ]] || \ [[ "$DETECTED_IP" -eq 0 ]]; then - if [[ "$KINIT" == "rdinit="* ]]; then - print_output "[*] Warning: switching back to init" - # strip rd from rdinit - KINIT="${KINIT:2}" - else - print_output "[*] Warning: switching back to rdinit" - # make rdinit from init - KINIT="rd""$KINIT" + # we only switch back if the first check has more output generated + if [[ "$COUNTING_1st" -gt "$COUNTING_2nd" ]]; then + if [[ "$KINIT" == "rdinit="* ]]; then + print_output "[*] Warning: switching back to init" + # strip rd from rdinit + KINIT="${KINIT:2}" + else + print_output "[*] Warning: switching back to rdinit" + # make rdinit from init + KINIT="rd""$KINIT" + fi fi fi