diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 16a1ca30a..a7b2fe542 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -10,6 +10,7 @@ function urlencodeall() { ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"} ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"} +ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"} MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} OPENSEARCH_URL_FINAL=${OPENSEARCH_URL:-"http://opensearch:9200"} @@ -48,6 +49,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_DIR}"/etc/config.ini sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_DIR}"/etc/config.ini + sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_DIR}"/etc/config.ini if [[ "$MALCOLM_PROFILE" == "hedgehog" ]]; then sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini diff --git a/config/arkime.env.example b/config/arkime.env.example index 183e970e3..8248a636d 100644 --- a/config/arkime.env.example +++ b/config/arkime.env.example @@ -1,6 +1,7 @@ # Whether or not Arkime is allowed to delete uploaded/captured PCAP (see # https://arkime.com/faq#pcap-deletion) MANAGE_PCAP_FILES=false +ARKIME_FREESPACEG=10% # The number of Arkime capture processes allowed to run concurrently ARKIME_ANALYZE_PCAP_THREADS=1 diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 28b631d65..02c62b0d3 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -379,6 +379,8 @@ Determine oldest indices by name (instead of creation time)? (Y / n): y Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)? (y / N): y +Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.): 10% + Automatically analyze all PCAP files with Suricata? (Y / n): y Download updated Suricata signatures periodically? (y / N): y diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index 280698e90..a537f9f65 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -179,6 +179,8 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest - Most of the configuration around OpenSearch [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) and [Snapshot Management](https://opensearch.org/docs/latest/opensearch/snapshots/sm-dashboards/) can be done in OpenSearch Dashboards. In addition to (or instead of) the OpenSearch index state management operations, Malcolm can also be configured to delete the oldest network session metadata indices when the database exceeds a certain size to prevent filling up all available storage with OpenSearch indices. - **Should Arkime delete PCAP files based on available storage?** - Answering **Y** allows Arkime to prune (delete) old PCAP files based on available disk space (see https://arkime.com/faq#pcap-deletion). + - **Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)** + - If [Arkime PCAP-deletion](https://arkime.com/faq#pcap-deletion) is enabled, Arkime will delete PCAP files when **free space** is lower than this value, specified as integer gigabytes (e.g., `500`) or a percentage (e.g., `10%`) * **Automatically analyze all PCAP files with Suricata?** - This option is used to enable [Suricata](https://suricata.io/) (an IDS and threat detection engine) to analyze PCAP files uploaded to Malcolm via its upload web interface. * **Download updated Suricata signatures periodically?** diff --git a/scripts/install.py b/scripts/install.py index 6a6479923..8408144ed 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -993,6 +993,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): indexPruneSizeLimit = '0' indexPruneNameSort = False arkimeManagePCAP = False + arkimeFreeSpaceG = '10%' if InstallerYesOrNo( 'Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?' @@ -1032,6 +1033,16 @@ def tweak_malcolm_runtime(self, malcolm_install_path): default=args.arkimeManagePCAP, ) ) + if arkimeManagePCAP: + arkimeFreeSpaceGTmp = '' + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid PCAP deletion threshold') + while (not re.match(r'^\d+%?$', arkimeFreeSpaceGTmp, flags=re.IGNORECASE)) and loopBreaker.increment(): + arkimeFreeSpaceGTmp = InstallerAskForString( + 'Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)', + default=args.arkimeFreeSpaceG, + ) + if arkimeFreeSpaceGTmp: + arkimeFreeSpaceG = arkimeFreeSpaceGTmp autoSuricata = InstallerYesOrNo( 'Automatically analyze all PCAP files with Suricata?', default=args.autoSuricata @@ -1376,6 +1387,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'MANAGE_PCAP_FILES', TrueOrFalseNoQuote(arkimeManagePCAP), ), + # Threshold for Arkime PCAP deletion + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'ARKIME_FREESPACEG', + arkimeFreeSpaceG, + ), # authentication method: basic (true), ldap (false) or no_authentication EnvValue( os.path.join(args.configDir, 'auth-common.env'), @@ -3471,6 +3488,15 @@ def main(): default=False, help="Arkime should delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)", ) + storageArgGroup.add_argument( + '--delete-pcap-threshold', + dest='arkimeFreeSpaceG', + required=False, + metavar='', + type=str, + default='', + help=f'Threshold for Arkime PCAP deletion (see https://arkime.com/faq#pcap-deletion)', + ) storageArgGroup.add_argument( '--delete-index-threshold', dest='indexPruneSizeLimit', diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index 255a2d22d..097c2a186 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -21,7 +21,8 @@ export ARKIME_COMPRESSION_LEVEL=0 export ARKIME_VIEWER_CERT=viewer.crt export ARKIME_VIEWER_KEY=viewer.key # Password hash secret for Arkime viewer cluster (see https://arkime.com/settings) -ARKIME_PASSWORD_SECRET=Malcolm +export ARKIME_PASSWORD_SECRET=Malcolm +export ARKIME_FREESPACEG=7% export DOCUMENTATION_PORT=8420 export MISCBEAT_PORT=9516 diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index a47b80795..048e2944e 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -60,6 +60,11 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t sed -r -i "s/(maxFileTimeM)\s*=\s*.*/\1=$PCAP_ROTATE_MINUTES/" "$ARKIME_CONFIG_FILE" fi + # pcap deletion threshold + if [[ -n $ARKIME_FREESPACEG ]]; then + sed -r -i "s/(freeSpaceG)\s*=\s*.*/\1=$ARKIME_FREESPACEG/" "$ARKIME_CONFIG_FILE" + fi + # pcap compression COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}"