Merge pull request #446 from m-1-k-3/comments

Code cleanup - comments

check_project.sh

@@ -46,6 +46,7 @@ MODULES_TO_CHECK_ARR_TAB=()
 MODULES_TO_CHECK_ARR_SEMGREP=()
 MODULES_TO_CHECK_ARR_DOCKER=()
 MODULES_TO_CHECK_ARR_PERM=()
+MODULES_TO_CHECK_ARR_COMMENT=()
 
 import_config_scripts() {
   mapfile -t HELPERS < <(find "$CONF_DIR" -iname "*.sh" 2>/dev/null)
@@ -153,6 +154,19 @@ check() {
     fi
   done
 
+  echo -e "\\n""$GREEN""Check all source for correct comment usage:""$NC""\\n"
+  for SOURCE in "${SOURCES[@]}"; do
+    echo -e "\\n""$GREEN""Run ${ORANGE}comment check$GREEN on $ORANGE$SOURCE""$NC""\\n"
+    if [[ $(grep -E -R "^( )+?#" "$SOURCE" | grep -v "#\ \|bash\|/bin/sh\|shellcheck" | grep -v -E -c "#$") -eq 0 ]]; then
+      echo -e "$GREEN""$BOLD""==> SUCCESS""$NC""\\n"
+    else
+      grep -E -R -n "^( )+?#" "$SOURCE" | grep -v "#\ \|bash\|shellcheck" | grep -v -E "#$"
+      echo -e "\\n""$ORANGE""$BOLD""==> FIX ERRORS""$NC""\\n"
+      MODULES_TO_CHECK_ARR_COMMENT+=("$SOURCE")
+    fi
+  done
+
+
   echo -e "\\n""$GREEN""Run shellcheck and semgrep:""$NC""\\n"
   for SOURCE in "${SOURCES[@]}"; do
     echo -e "\\n""$GREEN""Run ${ORANGE}shellcheck$GREEN on $ORANGE$SOURCE""$NC""\\n"
@@ -199,6 +213,15 @@ summary() {
     echo -e "$ORANGE""WARNING: Fix the errors before pushing to the EMBA repository!"
   fi
 
+  if [[ "${#MODULES_TO_CHECK_ARR_COMMENT[@]}" -gt 0 ]]; then
+    echo -e "\\n\\n""$GREEN$BOLD""SUMMARY:$NC\\n"
+    echo -e "Modules to check (space after # sign): ${#MODULES_TO_CHECK_ARR_COMMENT[@]}\\n"
+    for MODULE in "${MODULES_TO_CHECK_ARR_COMMENT[@]}"; do
+      echo -e "$ORANGE$BOLD==> FIX MODULE: ""$MODULE""$NC"
+    done
+    echo -e "$ORANGE""WARNING: Fix the errors before pushing to the EMBA repository!"
+  fi
+
   if [[ "${#MODULES_TO_CHECK_ARR[@]}" -gt 0 ]]; then
     echo -e "\\n\\n""$GREEN$BOLD""SUMMARY:$NC\\n"
     echo -e "Modules to check (shellcheck): ${#MODULES_TO_CHECK_ARR[@]}\\n"
@@ -258,6 +281,7 @@ dockerchecker
 summary
 
 if [[ "${#MODULES_TO_CHECK_ARR_TAB[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR[@]}" -gt 0 ]] || \
-  [[ "${#MODULES_TO_CHECK_ARR_SEMGREP[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_DOCKER[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_PERM[@]}" -gt 0 ]]; then
+  [[ "${#MODULES_TO_CHECK_ARR_SEMGREP[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_DOCKER[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_PERM[@]}" -gt 0 ]] || \
+  [[ "${#MODULES_TO_CHECK_ARR_COMMENT[@]}" -gt 0 ]]; then
   exit 1
 fi

helpers/helpers_emba_dependency_check.sh

@@ -637,10 +637,10 @@ architecture_dep_check() {
   elif [[ "$ARCH" == "x86" ]] ; then
     ARCH_STR="i386"
   elif [[ "$ARCH" == "x64" ]] ; then
-    #ARCH_STR="i386:x86-64"
+    # ARCH_STR="i386:x86-64"
     ARCH_STR="x86-64"
   elif [[ "$ARCH" == "PPC" ]] ; then
-    #ARCH_STR="powerpc:common"
+    # ARCH_STR="powerpc:common"
     ARCH_STR="powerpc"
   elif [[ "$ARCH" == "PPC64" ]] ; then
     ARCH_STR="powerpc64"

helpers/helpers_emba_helpers.sh

@@ -30,15 +30,15 @@ run_web_reporter_mod_name() {
 wait_for_pid() {
   local WAIT_PIDS=("$@")
   local PID
-  #print_output "[*] wait pid protection: ${#WAIT_PIDS[@]}"
+  # print_output "[*] wait pid protection: ${#WAIT_PIDS[@]}"
   for PID in "${WAIT_PIDS[@]}"; do
-    #print_output "[*] wait pid protection: $PID"
+    # print_output "[*] wait pid protection: $PID"
     print_dot
     if ! [[ -e /proc/"$PID" ]]; then
       continue
     fi
     while [[ -e /proc/"$PID" ]]; do
-      #print_output "[*] wait pid protection - running pid: $PID"
+      # print_output "[*] wait pid protection - running pid: $PID"
       print_dot
       # if S115 is running we have to kill old qemu processes
       if [[ -f "$LOG_DIR"/"$MAIN_LOG_FILE" ]] && [[ $(grep -c S115_ "$LOG_DIR"/"$MAIN_LOG_FILE") -eq 1 && -n "$QRUNTIME" ]]; then
@@ -146,9 +146,9 @@ cleaner() {
   fi
 
   # what a quick fix - need to come back to this!
-  #if [[ "$NOTIFICATION_PID" != "NA" ]]; then
+  # if [[ "$NOTIFICATION_PID" != "NA" ]]; then
   #  kill "$NOTIFICATION_PID" 2>/dev/null || true
-  #fi
+  # fi
   if [[ -f "$TMP_DIR"/orig_logdir ]]; then
     LOG_DIR_HOST=$(cat "$TMP_DIR"/orig_logdir)
     pkill -f "inotifywait.*$LOG_DIR_HOST" 2>/dev/null || true
@@ -190,7 +190,7 @@ emba_updater() {
     git pull
     cd "$BASE_PATH" || exit
   else
-    #git clone https://github.com/trickest/cve.git "$EXT_DIR"/trickest-cve
+    # git clone https://github.com/trickest/cve.git "$EXT_DIR"/trickest-cve
     git clone https://github.com/EMBA-support-repos/trickest-cve.git "$EXT_DIR"/trickest-cve
   fi
 

helpers/helpers_emba_prepare.sh

@@ -381,7 +381,7 @@ prepare_file_arr()
 
   # xdev will do the trick for us:
   # remove ./proc/* executables (for live testing)
-  #rm_proc_binary "${FILE_ARR[@]}"
+  # rm_proc_binary "${FILE_ARR[@]}"
 }
 
 prepare_binary_arr()
@@ -392,7 +392,7 @@ prepare_binary_arr()
   # lets try to get an unique binary array
   # Necessary for providing BINARIES array (usable in every module)
   export BINARIES=()
-  #readarray -t BINARIES < <( find "$FIRMWARE_PATH" "${EXCL_FIND[@]}" -type f -executable -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3 )
+  # readarray -t BINARIES < <( find "$FIRMWARE_PATH" "${EXCL_FIND[@]}" -type f -executable -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3 )
 
   # In some firmwares we miss the exec permissions in the complete firmware. In such a case we try to find ELF files and unique it
   readarray -t BINARIES_TMP < <(find "$FIRMWARE_PATH" "${EXCL_FIND[@]}" -type f -exec file {} \; 2>/dev/null | grep ELF | cut -d: -f1 || true)
@@ -410,7 +410,7 @@ prepare_binary_arr()
   fi
 
   # remove ./proc/* executables (for live testing)
-  #rm_proc_binary "${BINARIES[@]}"
+  # rm_proc_binary "${BINARIES[@]}"
 }
 
 prepare_file_arr_limited() {

helpers/helpers_emba_print.sh

@@ -45,7 +45,7 @@ welcome()
 
 module_log_init()
 {
-  #local LOG_FILE_NAME
+  # local LOG_FILE_NAME
   LOG_FILE_NAME="${1:-}"
   local FILE_NAME
   MODULE_NUMBER="$(echo "$LOG_FILE_NAME" | cut -d "_" -f1 | cut -c2- )"
@@ -591,7 +591,7 @@ module_start_log() {
   print_output "[*] $(date) - $MODULE_MAIN_NAME starting" "main"
   export LOG_PATH_MODULE
   if [[ "${LOG_DIR: -1}" == "/" ]]; then
-    #strip final slash from log dir
+    # strip final slash from log dir
     LOG_DIR="${LOG_DIR:: -1}"
   fi
   LOG_PATH_MODULE=$(abs_path "$LOG_DIR""/""$(echo "$MODULE_MAIN_NAME" | tr '[:upper:]' '[:lower:]')")

helpers/snyk_crawler.sh

@@ -68,7 +68,7 @@ echo ""
 echo -e "[*] The following advisories have PoC code included:"
 PoC_CNT=0
 # removed exploit-db as we already have it in EMBA
-#echo "CVE;advisory name;advisory URL;unknown PoC;Github PoC;exploit-db;Curl PoC;XML PoC;" > "$SAVE_PATH"/Snyk_PoC_results.csv
+# echo "CVE;advisory name;advisory URL;unknown PoC;Github PoC;exploit-db;Curl PoC;XML PoC;" > "$SAVE_PATH"/Snyk_PoC_results.csv
 echo "CVE;advisory name;advisory URL;unknown PoC;Github PoC;Curl PoC;XML PoC;" > "$SAVE_PATH"/Snyk_PoC_results.csv
 
 while IFS= read -r -d '' ADV; do
@@ -99,13 +99,13 @@ while IFS= read -r -d '' ADV; do
   fi
   # removed exploit-db as we already have it in EMBA
   # exploit-db references:
-  #PoC_EDB=$(grep -a -c -E "https://www.exploit-db.com/exploits/[0-9]+" "$ADV")
-  #((PoC+="$PoC_EDB"))
-  #if [[ "$PoC_EDB" -gt 0 ]]; then
+  # PoC_EDB=$(grep -a -c -E "https://www.exploit-db.com/exploits/[0-9]+" "$ADV")
+  # ((PoC+="$PoC_EDB"))
+  # if [[ "$PoC_EDB" -gt 0 ]]; then
   #  PoC_EDB="yes"
-  #else
+  # else
   #  PoC_EDB="no"
-  #fi
+  # fi
   # curl http exploits:
   PoC_CURL=$(grep -a -c "curl http" "$ADV")
   ((PoC+="$PoC_CURL"))
@@ -129,7 +129,7 @@ while IFS= read -r -d '' ADV; do
     for CVE in "${CVEs[@]}"; do
       echo -e "[+] Found PoC for $ORANGE$CVE$NC in advisory $ORANGE$ADV_NAME$NC (unknown PoC: $ORANGE$PoC_PoC$NC / Github: $ORANGE$PoC_GH$NC / exploit-db: $ORANGE$PoC_EDB$NC / Curl: $ORANGE$PoC_CURL$NC / XML: $ORANGE$PoC_XML$NC)"
       # removed exploit-db as we already have it in EMBA
-      #echo "$CVE;$ADV_NAME;$ADV_URL;$PoC_PoC;$PoC_GH;$PoC_EDB;$PoC_CURL;$PoC_XML;" >> "$SAVE_PATH"/Snyk_PoC_results.csv
+      # echo "$CVE;$ADV_NAME;$ADV_URL;$PoC_PoC;$PoC_GH;$PoC_EDB;$PoC_CURL;$PoC_XML;" >> "$SAVE_PATH"/Snyk_PoC_results.csv
       echo "$CVE;$ADV_NAME;$ADV_URL;$PoC_PoC;$PoC_GH;$PoC_CURL;$PoC_XML;" >> "$SAVE_PATH"/Snyk_PoC_results.csv
       ((PoC_CNT+=1))
     done

installer/I02_UEFI_fwhunt.sh

@@ -76,9 +76,9 @@ I02_UEFI_fwhunt() {
         cd external/fwhunt-scan || ( echo "Could not install EMBA component fwhunt-scan" && exit 1 )
         git clone https://github.com/EMBA-support-repos/FwHunt.git rules
         echo "Installed $(find rules/ -iname "BRLY-*" | wc -l) fwhunt rules"
-        #ldconfig
-        #export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64/
-        #python3 setup.py install
+        # ldconfig
+        # export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64/
+        # python3 setup.py install
         pip3 install fwhunt-scan
         cd "$HOME_PATH" || ( echo "Could not install EMBA component fwhunt-scan" && exit 1 )
       ;;

installer/ID1_ubuntu_os.sh

@@ -38,7 +38,7 @@ ID1_ubuntu_os() {
     if ! dpkg -l libssl1.1 &>/dev/null; then
         # libssl1.1 missing
         echo -e "\\n""$BOLD""Installing libssl1.1 for mongodb!""$NC"
-        #echo "deb http://security.ubuntu.com/ubuntu impish-security main" | tee /etc/apt/sources.list.d/impish-security.list
+        # echo "deb http://security.ubuntu.com/ubuntu impish-security main" | tee /etc/apt/sources.list.d/impish-security.list
         wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1-1ubuntu2.1~18.04.20_amd64.deb -O external/libssl-dev_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
         wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb -O external/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
         dpkg -i external/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb

installer/IF20_cve_search.sh

@@ -49,22 +49,22 @@ IF20_cve_search() {
       done < requirements.system
 
       # we do not need to install the Flask web environment - we do it manually
-      #while read -r TOOL_NAME; do
+      # while read -r TOOL_NAME; do
       #  PIP_NAME=$(echo "$TOOL_NAME" | cut -d= -f1)
       #  TOOL_VERSION=$(echo "$TOOL_NAME" | cut -d= -f3)
       #  print_pip_info "$PIP_NAME" "$TOOL_VERSION"
-      #done < requirements.txt
+      # done < requirements.txt
 
-      #xargs sudo apt-get install -y < requirements.system
+      # xargs sudo apt-get install -y < requirements.system
       while read -r TOOL_NAME; do
         apt-get install -y "$TOOL_NAME" --no-install-recommends
       done < requirements.system
 
       # this is a temp solution - Currently needed to fulfill broken deps:
-      #python3 -m pip install -Iv crackmapexec==5.1.7.dev0
+      # python3 -m pip install -Iv crackmapexec==5.1.7.dev0
 
       # we do not need to install the Flask web environment - we do it manually
-      #python3 -m pip install -r requirements.txt
+      # python3 -m pip install -r requirements.txt
       pip3 install requests==2.28.1
       pip3 install Whoosh==2.7.4
       pip3 install tqdm==4.64.0

installer/IL15_emulated_checks_init.sh

@@ -41,13 +41,13 @@ IL15_emulated_checks_init() {
 
     # needed for cutycapt
     print_tool_info "xvfb" 1
-    #print_tool_info "libqt5webkit5" 1
-    #print_tool_info "xfonts-100dpi" 1
-    #print_tool_info "xfonts-75dpi" 1
-    #print_tool_info "xfonts-cyrillic" 1
-    #print_tool_info "xorg" 1
-    #print_tool_info "dbus-x11" 1
-    #print_tool_info "g++" 1
+    # print_tool_info "libqt5webkit5" 1
+    # print_tool_info "xfonts-100dpi" 1
+    # print_tool_info "xfonts-75dpi" 1
+    # print_tool_info "xfonts-cyrillic" 1
+    # print_tool_info "xorg" 1
+    # print_tool_info "dbus-x11" 1
+    # print_tool_info "g++" 1
     # needed for cutycapt
 
     if [[ "$LIST_DEP" -eq 1 ]] || [[ $DOCKER_SETUP -eq 1 ]] ; then

installer/IP00_extractors.sh

@@ -26,7 +26,7 @@ IP00_extractors(){
     print_pip_info "bsdiff4"
     print_git_info "payload_dumper" "EMBA-support-repos/payload_dumper" "Android OTA payload.bin extractor"
     # ubireader:
-    #print_tool_info "python3-lzo" 1
+    # print_tool_info "python3-lzo" 1
     print_tool_info "liblzo2-dev" 1
     print_pip_info "python-lzo"
     # vmdk extractor:

installer/IP99_binwalk_default.sh

@@ -63,7 +63,7 @@ IP99_binwalk_default() {
     print_tool_info "python3-pyqt5.qtopengl" 1
     print_tool_info "python3-numpy" 1
     print_tool_info "python3-scipy" 1
-    #print_tool_info "python3-lzo" 1
+    # print_tool_info "python3-lzo" 1
     print_pip_info "python-lzo"
     # python-setuptools is needed for ubireader installation
     print_tool_info "python-setuptools" 1
@@ -111,7 +111,7 @@ IP99_binwalk_default() {
         pip3 install "python-lzo>=1.14"
 
         if ! [[ -d external/binwalk ]]; then
-          #git clone https://github.com/ReFirmLabs/binwalk.git external/binwalk
+          # git clone https://github.com/ReFirmLabs/binwalk.git external/binwalk
           git clone https://github.com/EMBA-support-repos/binwalk.git external/binwalk
         fi
 

modules/F20_vul_aggregator.sh

@@ -211,7 +211,7 @@ aggregate_versions() {
       # we ensure that we search for the correct kernel version by adding a : at the end of the search string
       VERSION=${VERSION/%/:}
       VERSIONS_KERNEL+=( "$VERSION" )
-      #print_output "[+] Added modfied Kernel Version details (${ORANGE}kernel$GREEN): ""$ORANGE$VERSION$NC"
+      # print_output "[+] Added modfied Kernel Version details (${ORANGE}kernel$GREEN): ""$ORANGE$VERSION$NC"
     done
 
     for CVE_ENTRY in "${CVE_S02_DETAILS[@]}"; do
@@ -649,13 +649,13 @@ cve_extractor() {
   # On the other hand, do not forget that we are also using the s25 results if we can find the
   # same CVE here via version detection.
 
-  #if [[ "$BINARY" == *kernel* ]]; then
+  # if [[ "$BINARY" == *kernel* ]]; then
   #  if [[ -f "$S25_LOG" ]]; then
   #    for KERNEL_CVE_EXPLOIT in "${KERNEL_CVE_EXPLOITS[@]}"; do
   #      KCVE_VALUE=$(echo "$KERNEL_CVE_EXPLOIT" | cut -d\; -f3)
   #    done
   #  fi
-  #fi
+  # fi
 
   if [[ -f "$LOG_PATH_MODULE"/"$AGG_LOG_FILE" ]]; then
     for CVE_OUTPUT in "${CVEs_OUTPUT[@]}"; do

modules/F50_base_aggregator.sh

@@ -49,7 +49,7 @@ F50_base_aggregator() {
   S109_LOG="s109_jtr_local_pw_cracking.txt"
   S110_LOG="s110_yara_check.txt"
   S120_LOG="s120_cwe_checker.txt"
-  #L10_LOG="l10_system_emulator.txt"
+  # L10_LOG="l10_system_emulator.txt"
   L15_LOG="l15_emulated_checks_init.txt"
   L20_LOG="l20_snmp_checks.txt"
   L25_LOG="l25_web_checks.txt"
@@ -458,7 +458,7 @@ output_binaries() {
     readarray -t RESULTS_STRCPY < <( find "$LOG_DIR"/s1[34]*/ -xdev -iname "vul_func_*_strcpy-*.txt" 2> /dev/null | sed "s/.*vul_func_//" | sort -g -r | head -10 | sed "s/_strcpy-/ strcpy /" | sed "s/\.txt//" 2> /dev/null || true)
     readarray -t RESULTS_SYSTEM < <( find "$LOG_DIR"/s1[34]*/ -xdev -iname "vul_func_*_system-*.txt" 2> /dev/null | sed "s/.*vul_func_//" | sort -g -r | head -10 | sed "s/_system-/ system /" | sed "s/\.txt//" 2> /dev/null || true)
 
-    #strcpy:
+    # strcpy:
     if [[ "${#RESULTS_STRCPY[@]}" -gt 0 ]]; then
       print_ln
       print_output "[+] STRCPY - top 10 results:"
@@ -475,7 +475,7 @@ output_binaries() {
       print_output "$NC"
     fi
 
-    #system:
+    # system:
     if [[ "${#RESULTS_SYSTEM[@]}" -gt 0 ]]; then
       print_ln
       print_output "[+] SYSTEM - top 10 results:"
@@ -848,7 +848,7 @@ get_data() {
     MODE=$(grep -e "Booted yes;\|ICMP ok;\|TCP-0 ok;\|TCP ok" "$SYS_EMU_RESULTS" | cut -d\; -f8 | sed 's/Network mode: //g'| tr -d '[:blank:]' | cut -d\( -f1 | sort -u | tr '\n' '-' | sed 's/-$//g' || true)
   fi
   if [[ -f "$LOG_DIR"/"$L15_LOG" ]]; then
-    #NMAP_UP=$(grep -a "\[\*\]\ Statistics:" "$LOG_DIR"/"$L15_LOG" | cut -d: -f2 || true)
+    # NMAP_UP=$(grep -a "\[\*\]\ Statistics:" "$LOG_DIR"/"$L15_LOG" | cut -d: -f2 || true)
     SNMP_UP=$(grep -a "\[\*\]\ Statistics:" "$LOG_DIR"/"$L20_LOG" | cut -d: -f2 || true)
     WEB_UP=$(grep -a "\[\*\]\ Statistics:" "$LOG_DIR"/"$L25_LOG" | cut -d: -f2 || true)
     ROUTERSPLOIT_VULN=$(grep -a "\[\*\]\ Statistics:" "$LOG_DIR"/"$L30_LOG" | cut -d: -f2 || true)
@@ -881,7 +881,7 @@ get_data() {
     KNOWN_EXPLOITED_COUNTER=$(cat "$TMP_DIR"/KNOWN_EXPLOITED_COUNTER.tmp)
   fi
   if [[ -f "$F20_EXPLOITS_LOG" ]]; then
-    #EXPLOIT_COUNTER="$(grep -c -E "Exploit\ .*" "$F20_EXPLOITS_LOG" || true)"
+    # EXPLOIT_COUNTER="$(grep -c -E "Exploit\ .*" "$F20_EXPLOITS_LOG" || true)"
     EXPLOIT_COUNTER="$(grep -E "Exploit\ .*" "$F20_EXPLOITS_LOG" | grep -cv "Exploit summary" || true)"
     MSF_MODULE_CNT="$(grep -c -E "Exploit\ .*MSF" "$F20_EXPLOITS_LOG" || true)"
     REMOTE_EXPLOIT_CNT="$(grep -c -E "Exploit\ .*\ \(R\)" "$F20_EXPLOITS_LOG" || true)"