idaholab#285, allow customizing Arkime's freeSpaceG setting (for PCAP…

… deletion) in an environment variable

arkime/scripts/docker_entrypoint.sh

@@ -10,6 +10,7 @@ function urlencodeall() {
 
 ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"}
 ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"}
+ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"}
 
 MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"}
 OPENSEARCH_URL_FINAL=${OPENSEARCH_URL:-"http://opensearch:9200"}
@@ -48,6 +49,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then
     cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_DIR}"/etc/config.ini
     sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_DIR}"/etc/config.ini
     sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_DIR}"/etc/config.ini
+    sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_DIR}"/etc/config.ini
     if [[ "$MALCOLM_PROFILE" == "hedgehog" ]]; then
         sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini
         sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini

config/arkime.env.example

@@ -1,6 +1,7 @@
 # Whether or not Arkime is allowed to delete uploaded/captured PCAP (see
 #   https://arkime.com/faq#pcap-deletion)
 MANAGE_PCAP_FILES=false
+ARKIME_FREESPACEG=10%
 # The number of Arkime capture processes allowed to run concurrently
 ARKIME_ANALYZE_PCAP_THREADS=1
 

docs/kubernetes.md

@@ -379,6 +379,8 @@ Determine oldest indices by name (instead of creation time)? (Y / n): y
 
 Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)? (y / N): y
 
+Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.): 10%
+
 Automatically analyze all PCAP files with Suricata? (Y / n): y
 
 Download updated Suricata signatures periodically? (y / N): y

docs/malcolm-hedgehog-e2e-iso-install.md

@@ -179,6 +179,8 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest
         - Most of the configuration around OpenSearch [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) and [Snapshot Management](https://opensearch.org/docs/latest/opensearch/snapshots/sm-dashboards/) can be done in OpenSearch Dashboards. In addition to (or instead of) the OpenSearch index state management operations, Malcolm can also be configured to delete the oldest network session metadata indices when the database exceeds a certain size to prevent filling up all available storage with OpenSearch indices.
     - **Should Arkime delete PCAP files based on available storage?**
         - Answering **Y** allows Arkime to prune (delete) old PCAP files based on available disk space (see https://arkime.com/faq#pcap-deletion).
+    - **Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)**
+        - If [Arkime PCAP-deletion](https://arkime.com/faq#pcap-deletion) is enabled, Arkime will delete PCAP files when **free space** is lower than this value, specified as integer gigabytes (e.g., `500`) or a percentage (e.g., `10%`)
 * **Automatically analyze all PCAP files with Suricata?**
     - This option is used to enable [Suricata](https://suricata.io/) (an IDS and threat detection engine) to analyze PCAP files uploaded to Malcolm via its upload web interface.
 * **Download updated Suricata signatures periodically?**

scripts/install.py

@@ -993,6 +993,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
         indexPruneSizeLimit = '0'
         indexPruneNameSort = False
         arkimeManagePCAP = False
+        arkimeFreeSpaceG = '10%'
 
         if InstallerYesOrNo(
             'Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?'
@@ -1032,6 +1033,16 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                     default=args.arkimeManagePCAP,
                 )
             )
+            if arkimeManagePCAP:
+                arkimeFreeSpaceGTmp = ''
+                loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid PCAP deletion threshold')
+                while (not re.match(r'^\d+%?$', arkimeFreeSpaceGTmp, flags=re.IGNORECASE)) and loopBreaker.increment():
+                    arkimeFreeSpaceGTmp = InstallerAskForString(
+                        'Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)',
+                        default=args.arkimeFreeSpaceG,
+                    )
+                if arkimeFreeSpaceGTmp:
+                    arkimeFreeSpaceG = arkimeFreeSpaceGTmp
 
         autoSuricata = InstallerYesOrNo(
             'Automatically analyze all PCAP files with Suricata?', default=args.autoSuricata
@@ -1376,6 +1387,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                 'MANAGE_PCAP_FILES',
                 TrueOrFalseNoQuote(arkimeManagePCAP),
             ),
+            # Threshold for Arkime PCAP deletion
+            EnvValue(
+                os.path.join(args.configDir, 'arkime.env'),
+                'ARKIME_FREESPACEG',
+                arkimeFreeSpaceG,
+            ),
             # authentication method: basic (true), ldap (false) or no_authentication
             EnvValue(
                 os.path.join(args.configDir, 'auth-common.env'),
@@ -3471,6 +3488,15 @@ def main():
         default=False,
         help="Arkime should delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)",
     )
+    storageArgGroup.add_argument(
+        '--delete-pcap-threshold',
+        dest='arkimeFreeSpaceG',
+        required=False,
+        metavar='<string>',
+        type=str,
+        default='',
+        help=f'Threshold for Arkime PCAP deletion (see https://arkime.com/faq#pcap-deletion)',
+    )
     storageArgGroup.add_argument(
         '--delete-index-threshold',
         dest='indexPruneSizeLimit',

sensor-iso/interface/sensor_ctl/control_vars.conf

@@ -21,7 +21,8 @@ export ARKIME_COMPRESSION_LEVEL=0
 export ARKIME_VIEWER_CERT=viewer.crt
 export ARKIME_VIEWER_KEY=viewer.key
 # Password hash secret for Arkime viewer cluster (see https://arkime.com/settings)
-ARKIME_PASSWORD_SECRET=Malcolm
+export ARKIME_PASSWORD_SECRET=Malcolm
+export ARKIME_FREESPACEG=7%
 
 export DOCUMENTATION_PORT=8420
 export MISCBEAT_PORT=9516

sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh

@@ -60,6 +60,11 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t
     sed -r -i "s/(maxFileTimeM)\s*=\s*.*/\1=$PCAP_ROTATE_MINUTES/" "$ARKIME_CONFIG_FILE"
   fi
 
+  # pcap deletion threshold
+  if [[ -n $ARKIME_FREESPACEG ]]; then
+    sed -r -i "s/(freeSpaceG)\s*=\s*.*/\1=$ARKIME_FREESPACEG/" "$ARKIME_CONFIG_FILE"
+  fi
+
   # pcap compression
   COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}"
   COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}"