Merge pull request #485 from m-1-k-3/disk_space

disk space monitor, rpm package analysis
e-m-b-a · Feb 23, 2023 · 1500f90 · 1500f90
2 parents f124641 + ff6ca08
commit 1500f90
Show file tree

Hide file tree

Showing 27 changed files with 342 additions and 116 deletions.
diff --git a/README.md b/README.md
@@ -60,7 +60,7 @@ sudo ./installer.sh -d
 
 ## Quick start with default scan profile:
 ```console
-sudo ./emba.sh -l ./log -f /firmware -p ./scan-profiles/default-scan.emba
+sudo ./emba -l ./log -f /firmware -p ./scan-profiles/default-scan.emba
 
 ```
 ---

diff --git a/config/emba_updater.init b/config/emba_updater.init
@@ -17,7 +17,7 @@ fi
 
 echo "[*] EMBA update - main repository" | tee -a "$LOG_DIR"/emba_update.log
 cd EMBA_INSTALL_PATH || exit
-git pull | tee -a "$LOG_DIR"/emba_update.log
+git pull origin master | tee -a "$LOG_DIR"/emba_update.log
 cd "$BASE_PATH" || exit
 
 echo "[*] EMBA update - cve-search update" | tee -a "$LOG_DIR"/emba_update.log

diff --git a/emba b/emba
@@ -178,7 +178,9 @@ run_modules()
         if [[ "$MOD_FIN" -eq 0 ]]; then
           if [[ $THREADING_SET -eq 1 ]]; then
             "$MODULE_MAIN" &
-            WAIT_PIDS+=( "$!" )
+            MOD_PID="$!"
+            store_kill_pids "$MOD_PID"
+            WAIT_PIDS+=( "$MOD_PID" )
             max_pids_protection "$MAX_MODS" "${WAIT_PIDS[@]}"
           else
             "$MODULE_MAIN"
@@ -224,7 +226,9 @@ run_modules()
           if [[ "$MOD_FIN" -eq 0 ]]; then
             if [[ $THREADING_SET -eq 1 ]]; then
               "$MODULE_MAIN" &
-              WAIT_PIDS+=( "$!" )
+              MOD_PID="$!"
+              store_kill_pids "$MOD_PID"
+              WAIT_PIDS+=( "$MOD_PID" )
               max_pids_protection "$MAX_MODS" "${WAIT_PIDS[@]}"
             else
               "$MODULE_MAIN"
@@ -288,7 +292,9 @@ run_modules()
             if [[ "$MOD_FIN" -eq 0 ]]; then
               if [[ $THREADING_SET -eq 1 ]]; then
                 "$MODULE_MAIN" &
-                WAIT_PIDS+=( "$!" )
+                MOD_PID="$!"
+                store_kill_pids "$MOD_PID"
+                WAIT_PIDS+=( "$MOD_PID" )
                 max_pids_protection "$MAX_MODS" "${WAIT_PIDS[@]}"
               else
                 "$MODULE_MAIN"
@@ -405,7 +411,6 @@ main()
   if [[ -f "/etc/debian_version" ]] && grep -q kali-rolling /etc/debian_version; then
     export DISABLE_NOTIFICATIONS=0    # disable notifications and further desktop experience
   fi
-  export NOTIFICATION_PID="NA"
   export NOTIFICATION_ID=0          # initial notification id - needed for notification overlay/replacement
   export EMBA_ICON=""
   EMBA_ICON=$(realpath "$HELP_DIR"/emba.svg)
@@ -422,7 +427,6 @@ main()
   # usually no memory limit is needed, but some modules/tools are wild and we need to protect our system
   export TOTAL_MEMORY=0
   TOTAL_MEMORY="$(grep MemTotal /proc/meminfo | awk '{print $2}' || true)"
-  export EXIT_KILL_PIDS=()
 
   import_helper
   print_ln "no_log"
@@ -713,7 +717,7 @@ main()
     if [[ $IN_DOCKER -eq 0 ]]; then
       kernel_downloader &
       K_DOWN_PID="$!"
-      EXIT_KILL_PIDS+=("$K_DOWN_PID")
+      store_kill_pids "$K_DOWN_PID"
       print_output "[*] Started kernel downloader thread with PID $ORANGE$K_DOWN_PID$NC" "no_log"
     fi
 
@@ -724,7 +728,8 @@ main()
     if [[ "$IN_DOCKER" -eq 0 ]]; then
       print_notification &
       NOTIFICATION_PID="$!"
-      EXIT_KILL_PIDS+=("$NOTIFICATION_PID")
+      store_kill_pids "$NOTIFICATION_PID"
+      disown "$NOTIFICATION_PID" 2> /dev/null || true
       print_output "[*] Original user: $ORANGE${SUDO_USER:-${USER}}$NC" "no_log"
       print_output "[*] Notification process started with PID $ORANGE${NOTIFICATION_PID}$NC" "no_log"
       echo "${SUDO_USER:-${USER}}" > "$LOG_DIR"/orig_user.log
@@ -853,12 +858,19 @@ main()
 
     if [[ $IN_DOCKER -eq 0 ]] ; then
       check_cve_search_job "$EMBA_PID" &
-      EXIT_KILL_PIDS+=("$!")
+      local TMP_PID="$!"
+      store_kill_pids "$TMP_PID"
+      disown "$TMP_PID" 2> /dev/null || true
     fi
 
+    disk_space_monitor "$EMBA_PID" &
+    local TMP_PID="$!"
+    store_kill_pids "$TMP_PID"
+    disown "$TMP_PID" 2> /dev/null || true
+
     # if $CONTAINER_EXTRACT is set we extract the docker container with id $CONTAINER_ID outside of the
     # EMBA container into log directory
-    # we do this only outside of the EMBA container - otherwise we will not reach the docker environment
+    # we do this outside of the EMBA container - otherwise we will not reach the docker environment
     if [[ "$CONTAINER_EXTRACT" -eq 1 && "$IN_DOCKER" -eq 0 ]] ; then
       docker_container_extractor "$CONTAINER_ID"
     fi
@@ -946,7 +958,7 @@ main()
       fi
       exit 0
     else
-      print_output "[-] EMBA failed in docker mode!" "no_log"
+      print_output "[-] EMBA failed in docker mode!" "main"
       cleaner 0
       write_notification "EMBA failed analysis in default mode"
       exit 1
@@ -1088,9 +1100,6 @@ main()
     else
       print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"$SECONDS" -u +%H:%M:%S)"" \\n" "no_log"
     fi
-    if [[ "$NOTIFICATION_PID" != "NA" ]]; then
-      kill "$NOTIFICATION_PID" 2>/dev/null || true
-    fi
     write_grep_log "$(date)" "TIMESTAMP"
     write_grep_log "$(date -d@"$SECONDS" -u +%H:%M:%S)" "DURATION"
   else

diff --git a/helpers/helpers_emba_dependency_check.sh b/helpers/helpers_emba_dependency_check.sh
@@ -423,6 +423,10 @@ dependency_check()
     # jtr
     check_dep_tool "john"
 
+    # rpm for checking package management system
+    # module not fully tested - needs some further work
+    # check_dep_tool "rpm"
+
     # pixd
     check_dep_file "pixd visualizer" "$EXT_DIR""/pixde"
 

diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh
@@ -83,7 +83,7 @@ max_pids_protection() {
 cleaner() {
   INTERRUPT_CLEAN="${1:-1}"
   if [[ "$INTERRUPT_CLEAN" -eq 1 ]]; then
-    print_output "[*] User interrupt detected!" "no_log"
+    print_output "[*] Interrupt detected!" "no_log"
   fi
   print_output "[*] Final cleanup started." "no_log"
 
@@ -103,7 +103,7 @@ cleaner() {
     if [[ $(grep -i -c S115 "$LOG_DIR"/"$MAIN_LOG_FILE") -eq 1 ]]; then
 
       print_output "[*] Terminating qemu processes - check it with ps" "no_log"
-      killall -9 --quiet -r .*qemu-.*-sta.* || true
+      killall -9 --quiet -r .*qemu-.*-sta.* > /dev/null || true
       print_output "[*] Cleaning the emulation environment\\n" "no_log"
       find "$FIRMWARE_PATH_CP" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null || true
       find "$LOG_DIR/s115_usermode_emulator" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null || true
@@ -125,7 +125,7 @@ cleaner() {
 
     if [[ $(grep -i -c S120 "$LOG_DIR"/"$MAIN_LOG_FILE") -eq 1 ]]; then
       print_output "[*] Terminating cwe-checker processes - check it with ps" "no_log"
-      killall -9 --quiet -r .*cwe_checker.* || true
+      killall -9 --quiet -r .*cwe_checker.* > /dev/null || true
     fi
 
     # IF SYS_ONLINE is 1, the live system tester (system mode emulator) was able to setup the box
@@ -142,46 +142,55 @@ cleaner() {
     if ps -p "$K_DOWN_PID" > /dev/null; then
       # kernel downloader is running in a thread on the host and needs to be stopped now
       print_output "[*] Stopping kernel downloader thread with PID $K_DOWN_PID" "no_log"
-      kill "$K_DOWN_PID" || true
+      kill "$K_DOWN_PID" > /dev/null || true
     fi
   fi
-  if [[ "$IN_DOCKER" -eq 0 ]] && pgrep -f "find ./external/trickest" &> /dev/null 2>&1; then
-    pkill -f "find ./external/trickest" 2>/dev/null || true
-  fi
-
-  # just in case we have the temp trickest db left
-  if [[ -f "$EXT_DIR"/trickest_db-cleaned.txt ]]; then
-    rm "$EXT_DIR"/trickest_db-cleaned.txt || true
-  fi
 
   if [[ -f "$TMP_DIR"/orig_logdir ]]; then
     LOG_DIR_HOST=$(cat "$TMP_DIR"/orig_logdir)
     pkill -f "inotifywait.*$LOG_DIR_HOST" 2>/dev/null || true
   fi
 
-  if [[ -n "${CHECK_CVE_JOB_PID:-}" && "${CHECK_CVE_JOB_PID:-}" -ne 0 ]]; then
-    kill -9 "$CHECK_CVE_JOB_PID" || true
+  if [[ "$IN_DOCKER" -eq 1 ]]; then
+    fuser -k "$LOG_DIR" || true
+    fuser -k "$FIRMWARE_PATH" || true
+  fi
+
+  # if [[ "$IN_DOCKER" -eq 1 ]] && [[ -f "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log ]]; then
+    # while read -r KILL_PID; do
+    #  if [[ -e /proc/"$KILL_PID" ]]; then
+    #    print_output "[*] Stopping EMBA process with PID $KILL_PID" "no_log"
+    #    kill -9 "$KILL_PID" > /dev/null || true
+    #  fi
+    # done < "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log
+  # fi
+
+  if [[ "$IN_DOCKER" -eq 0 ]] && [[ -f "$TMP_DIR"/EXIT_KILL_PIDS.log ]]; then
+    while read -r KILL_PID; do
+      if [[ -e /proc/"$KILL_PID" ]]; then
+        print_output "[*] Stopping EMBA process with PID $KILL_PID" "no_log"
+        kill -9 "$KILL_PID" > /dev/null || true
+      fi
+    done < "$TMP_DIR"/EXIT_KILL_PIDS.log
   fi
 
-  if [[ -d "$TMP_DIR" ]]; then
+  if [[ "$IN_DOCKER" -eq 0 ]] && [[ -d "$TMP_DIR" ]]; then
     rm -r "$TMP_DIR" 2>/dev/null || true
   fi
   if [[ "$INTERRUPT_CLEAN" -eq 1 ]]; then
     print_output "[!] Test ended on ""$(date)"" and took about ""$(date -d@"$SECONDS" -u +%H:%M:%S)"" \\n" "no_log"
     exit 1
   fi
-  if [[ "$IN_DOCKER" -eq 0 ]]; then
-    for KILL_PID in "${NOTIFICATION_PID[@]}"; do
-      print_output "[*] Stopping EMBA PID $KILL_PID" "no_log"
-      kill "$KILL_PID" || true
-    done
-  fi
 }
 
 emba_updater() {
   print_output "[*] EMBA update starting ..." "no_log"
 
-  git pull
+  if [[ -d ./.git ]]; then
+    git pull origin master
+  else
+    print_output "[-] INFO: Can't update non git version of EMBA"
+  fi
 
   EMBA="$INVOCATION_PATH" FIRMWARE="$FIRMWARE_PATH" LOG="$LOG_DIR" docker pull embeddedanalyzer/emba
 
@@ -194,23 +203,13 @@ emba_updater() {
   /etc/init.d/redis-server start
   "$EXT_DIR"/cve-search/sbin/db_updater.py -v
 
-  print_output "[*] EMBA update - trickest PoC update" "no_log"
-  if [[ -d "$EXT_DIR"/trickest-cve ]]; then
-    BASE_PATH=$(pwd)
-    cd "$EXT_DIR"/trickest-cve || exit
-    git pull
-    cd "$BASE_PATH" || exit
-  else
-    git clone https://github.com/trickest/cve.git "$EXT_DIR"/trickest-cve
-  fi
-
   print_output "[*] Please note that this was only a data update and no installed packages were updated." "no_log"
   print_output "[*] Please restart your EMBA scan to apply the updates ..." "no_log"
 }
 
 # this checks if a function is available
+# this means the EMBA module was loaded
 function_exists() {
-
   FCT_TO_CHECK="${1:-}"
   declare -f -F "$FCT_TO_CHECK" > /dev/null
   return $?
@@ -314,3 +313,42 @@ module_wait() {
     sleep 1
   done
 }
+
+store_kill_pids() {
+  local PID="${1:-}"
+  ! [[ -d "$TMP_DIR" ]] && mkdir -p "$TMP_DIR"
+  [[ "$IN_DOCKER" -eq 0 ]] && echo "$PID" >> "$TMP_DIR"/EXIT_KILL_PIDS.log
+  [[ "$IN_DOCKER" -eq 1 ]] && echo "$PID" >> "$TMP_DIR"/EXIT_KILL_PIDS_DOCKER.log
+  return 0
+}
+
+disk_space_monitor() {
+  local DDISK="$LOG_DIR"
+
+  while ! [[ -f "$MAIN_LOG" ]]; do
+    sleep 1
+  done
+
+  while true; do
+    # print_output "[*] Disk space monitoring active" "no_log"
+    FREE_SPACE=$(df --output=avail "$DDISK" | awk 'NR==2')
+    if [[ "$FREE_SPACE" -lt 10000000 ]]; then
+      print_ln "no_log"
+      print_output "[!] WARNING: EMBA is running out of disk space!" "main"
+      print_output "[!] WARNING: EMBA is stopping now" "main"
+      df -h || true
+      print_ln "no_log"
+      # give the container some more seconds for the cleanup process
+      [[ "$IN_DOCKER" -eq 0 ]] && sleep 5
+      cleaner 1
+    fi
+
+    if [[ -f "$MAIN_LOG" ]]; then
+      if grep -q "Test ended\|EMBA failed" "$MAIN_LOG" 2>/dev/null; then
+        break
+      fi
+    fi
+
+    sleep 5
+  done
+}
diff --git a/helpers/helpers_emba_system_emulation.sh b/helpers/helpers_emba_system_emulation.sh
@@ -35,9 +35,11 @@ restart_emulation() {
   reset_network_emulation 2
 
   # what an ugly hack - probably we are going to improve this later on
+  local HOME_PATH=""
+  HOME_PATH="$(pwd)"
   cd "$ARCHIVE_PATH" || (print_output "[-] Emulation archive path not found")
   ./run.sh &
-  cd "$INVOCATION_PATH" || (print_output "[-] EMBA path not available?")
+  cd "$HOME_PATH" || (print_output "[-] EMBA path not available?")
 
   COUNTER=0
   while ! ping -c 1 "$IP_ADDRESS_" &> /dev/null; do

diff --git a/installer.sh b/installer.sh
@@ -28,6 +28,7 @@ export DOWNLOAD_FILE_LIST=()
 export INSTALLER_DIR="./installer"
 
 if [[ "$STRICT_MODE" -eq 1 ]]; then
+  export DEBUG_SCRIPT=0
   if [[ -f "./helpers/helpers_emba_load_strict_settings.sh" ]]; then
     # shellcheck source=/dev/null
     source ./helpers/helpers_emba_load_strict_settings.sh
@@ -142,6 +143,7 @@ fi
 if grep -q -i wsl /proc/version; then
   echo -e "\n${ORANGE}INFO: System running in WSL environment!$NC"
   echo -e "\n${ORANGE}INFO: WSL is currently experimental!$NC"
+  echo -e "\n${ORANGE}Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites$NC"
   echo -e "\n${ORANGE}WARNING: If you are using WSL2, disable docker integration from the docker-desktop daemon!$NC"
   read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r
   WSL=1
@@ -160,6 +162,7 @@ elif ! grep -q "kali" /etc/debian_version 2>/dev/null ; then
   elif grep -q "PRETTY_NAME=\"Ubuntu 20.04 LTS\"" /etc/os-release 2>/dev/null ; then
     echo -e "\\n""$RED""EMBA is not fully supported on Ubuntu 20.04 LTS.""$NC"
     echo -e "$RED""For EMBA installation you need to update docker-compose manually. See also https://github.com/e-m-b-a/emba/issues/247""$NC"
+    echo -e "\\n""$ORANGE""Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites""$NC"
     read -p "If you have updated docker-compose you can press any key to continue ..." -n1 -s -r
     OTHER_OS=0  # installation procedure identical to kali install
     UBUNTU_OS=0 # installation procedure identical to kali install
@@ -203,11 +206,20 @@ if [[ "$IN_DOCKER" -eq 0 ]]; then
   if [[ "$FREE_SPACE" -lt 13000000 ]]; then
     echo -e "\\n""$ORANGE""EMBA installation in default mode needs a minimum of 13Gig for the docker image""$NC"
     echo -e "\\n""$ORANGE""Please free enough space on /var/lib/docker""$NC"
+    echo -e "\\n""$ORANGE""Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites""$NC"
     echo ""
     df -h || true
     echo ""
     read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r
   fi
+
+  TOTAL_MEMORY="$(grep MemTotal /proc/meminfo | awk '{print $2}' || true)"
+  if [[ "$TOTAL_MEMORY" -lt 4000000 ]]; then
+    echo -e "\\n""$ORANGE""EMBA installation in default mode needs a minimum of 4Gig of RAM""$NC"
+    echo -e "\\n""$ORANGE""Please check the documentation https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites""$NC"
+    echo ""
+    read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r
+  fi
 fi
 
 if [[ $LIST_DEP -eq 0 ]] ; then

diff --git a/installer/I01_default_apps.sh b/installer/I01_default_apps.sh
@@ -42,6 +42,8 @@ I01_default_apps(){
     print_tool_info "git" 1
     print_tool_info "strace" 1
 
+    print_tool_info "rpm" 1
+
     # python3.10-request
     print_tool_info "python3-pip" 1
     print_pip_info "requests"