diff --git a/config/arkime-secret.env.example b/config/arkime-secret.env.example new file mode 100644 index 000000000..7718ec2ec --- /dev/null +++ b/config/arkime-secret.env.example @@ -0,0 +1,4 @@ +# MaxMind GeoIP database update API key (see +# https://support.maxmind.com/hc/en-us/articles/4407116112539-Using-License-Keys) +MAXMIND_GEOIP_DB_LICENSE_KEY=0 +K8S_SECRET=True \ No newline at end of file diff --git a/config/arkime.env.example b/config/arkime.env.example index 6edb657a9..183e970e3 100644 --- a/config/arkime.env.example +++ b/config/arkime.env.example @@ -3,8 +3,5 @@ MANAGE_PCAP_FILES=false # The number of Arkime capture processes allowed to run concurrently ARKIME_ANALYZE_PCAP_THREADS=1 -# MaxMind GeoIP database update API key (see -# https://support.maxmind.com/hc/en-us/articles/4407116112539-Using-License-Keys) -MAXMIND_GEOIP_DB_LICENSE_KEY=0 OPENSEARCH_MAX_SHARDS_PER_NODE=2500 \ No newline at end of file diff --git a/config/auth.env.example b/config/auth.env.example index 60a2bf032..6ffb97a65 100644 --- a/config/auth.env.example +++ b/config/auth.env.example @@ -1,3 +1,4 @@ # Malcolm Administrator username and encrypted password for nginx reverse proxy (and upload server's SFTP access) MALCOLM_USERNAME=admin MALCOLM_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX== +K8S_SECRET=True \ No newline at end of file diff --git a/config/netbox-postgres.env.example b/config/netbox-postgres.env.example index 8e4861b2e..6814396b9 100644 --- a/config/netbox-postgres.env.example +++ b/config/netbox-postgres.env.example @@ -1,3 +1,4 @@ POSTGRES_DB=netbox POSTGRES_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXX POSTGRES_USER=netbox +K8S_SECRET=True \ No newline at end of file diff --git a/config/netbox-redis-cache.env.example b/config/netbox-redis-cache.env.example index 7883e1460..b5e80205d 100644 --- a/config/netbox-redis-cache.env.example +++ b/config/netbox-redis-cache.env.example @@ -1 +1,2 @@ REDIS_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXX +K8S_SECRET=True \ No newline at end of file diff --git a/config/netbox-redis.env.example b/config/netbox-redis.env.example index 7883e1460..b5e80205d 100644 --- a/config/netbox-redis.env.example +++ b/config/netbox-redis.env.example @@ -1 +1,2 @@ REDIS_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXX +K8S_SECRET=True \ No newline at end of file diff --git a/config/netbox-secret.env.example b/config/netbox-secret.env.example new file mode 100644 index 000000000..d629b84df --- /dev/null +++ b/config/netbox-secret.env.example @@ -0,0 +1,14 @@ +DB_PASSWORD=xxxxxxxxxxxxxxxx +DB_USER=netbox +EMAIL_PASSWORD= +EMAIL_USERNAME=netbox +NAPALM_PASSWORD= +NAPALM_USERNAME= +REDIS_CACHE_PASSWORD=xxxxxxxxxxxxxxxx +REDIS_PASSWORD=xxxxxxxxxxxxxxxx +SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +SUPERUSER_API_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +SUPERUSER_NAME=admin +SUPERUSER_PASSWORD=admin + +K8S_SECRET=True \ No newline at end of file diff --git a/config/netbox.env.example b/config/netbox.env.example index 1bd31f5b3..ddafba06f 100644 --- a/config/netbox.env.example +++ b/config/netbox.env.example @@ -14,16 +14,12 @@ REMOTE_AUTH_SUPERUSERS= EXEMPT_VIEW_PERMISSIONS=* DB_HOST=netbox-postgres DB_NAME=netbox -DB_PASSWORD=xxxxxxxxxxxxxxxx -DB_USER=netbox EMAIL_FROM=netbox@bar.com -EMAIL_PASSWORD= EMAIL_PORT=25 EMAIL_SERVER=localhost EMAIL_SSL_CERTFILE= EMAIL_SSL_KEYFILE= EMAIL_TIMEOUT=5 -EMAIL_USERNAME=netbox # EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`! EMAIL_USE_SSL=false EMAIL_USE_TLS=false @@ -32,25 +28,17 @@ HOUSEKEEPING_INTERVAL=86400 MAX_PAGE_SIZE=1000 MEDIA_ROOT=/opt/netbox/netbox/media METRICS_ENABLED=false -NAPALM_PASSWORD= NAPALM_TIMEOUT=10 -NAPALM_USERNAME= REDIS_CACHE_DATABASE=1 REDIS_CACHE_HOST=netbox-redis-cache REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY=false -REDIS_CACHE_PASSWORD=xxxxxxxxxxxxxxxx REDIS_CACHE_SSL=false REDIS_DATABASE=0 REDIS_HOST=netbox-redis REDIS_INSECURE_SKIP_TLS_VERIFY=false -REDIS_PASSWORD=xxxxxxxxxxxxxxxx REDIS_SSL=false RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases -SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx SKIP_STARTUP_SCRIPTS=true SKIP_SUPERUSER=false -SUPERUSER_API_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx SUPERUSER_EMAIL=admin@example.com -SUPERUSER_NAME=admin -SUPERUSER_PASSWORD=admin -WEBHOOKS_ENABLED=true +WEBHOOKS_ENABLED=true \ No newline at end of file diff --git a/config/process.env.example b/config/process.env.example index ca0a1923f..9cd2b8f71 100644 --- a/config/process.env.example +++ b/config/process.env.example @@ -3,5 +3,5 @@ PUID=1000 PGID=1000 # for debugging container init via tini (https://github.com/krallin/tini) TINI_VERBOSITY=1 -# for handling configmap files/directories -CONFIG_MAP_DIR=configmap \ No newline at end of file +# for handling configmap/secrets files/directories +CONFIG_MAP_DIR=configmap;secretmap \ No newline at end of file diff --git a/config/zeek-secret.env.example b/config/zeek-secret.env.example new file mode 100644 index 000000000..8ce2739c6 --- /dev/null +++ b/config/zeek-secret.env.example @@ -0,0 +1,5 @@ +# A VirusTotal Public API v.20 used to submit hashes of Zeek-extracted files +VTOT_API2_KEY=0 +# Specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files served over HTTP +EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined +K8S_SECRET=True \ No newline at end of file diff --git a/config/zeek.env.example b/config/zeek.env.example index 3563a08c3..060fe1290 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -21,8 +21,6 @@ EXTRACTED_FILE_PRESERVATION=quarantined EXTRACTED_FILE_MIN_BYTES=64 # The maximum size (in bytes) for files to be extracted by Zeek EXTRACTED_FILE_MAX_BYTES=134217728 -# A VirusTotal Public API v.20 used to submit hashes of Zeek-extracted files -VTOT_API2_KEY=0 # Rate limiting for VirusTotal, ClamAV, YARA and capa with Zeek-extracted files VTOT_REQUESTS_PER_MINUTE=4 CLAMD_MAX_REQUESTS=8 @@ -46,8 +44,6 @@ EXTRACTED_FILE_PIPELINE_VERBOSITY= EXTRACTED_FILE_HTTP_SERVER_ENABLE=false # Whether or not Zeek-extracted files served over HTTP will be AES-256-CBC-encrypted EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=true -# Specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files served over HTTP -EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined # Environment variables for tweaking Zeek at runtime (see local.zeek) # Set to any non-blank value to disable the corresponding feature ZEEK_DISABLE_HASH_ALL_FILES= diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 4066ea3ee..7c9367c9b 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -109,6 +109,7 @@ services: - ./config/opensearch.env - ./config/netbox-common.env - ./config/netbox.env + - ./config/netbox-secret.env - ./config/beats-common.env - ./config/lookup-common.env - ./config/logstash.env @@ -180,6 +181,7 @@ services: - ./config/upload-common.env - ./config/auth.env - ./config/arkime.env + - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' ulimits: @@ -222,6 +224,7 @@ services: - ./config/ssl.env - ./config/upload-common.env - ./config/zeek.env + - ./config/zeek-secret.env - ./config/zeek-offline.env depends_on: - opensearch @@ -258,6 +261,7 @@ services: - ./config/upload-common.env - ./config/pcap-capture.env - ./config/zeek.env + - ./config/zeek-secret.env - ./config/zeek-live.env volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro @@ -338,6 +342,7 @@ services: - ./config/process.env - ./config/ssl.env - ./config/zeek.env + - ./config/zeek-secret.env environment: VIRTUAL_HOST : 'file-monitor.malcolm.local' volumes: @@ -487,6 +492,7 @@ services: - ./config/ssl.env - ./config/netbox-common.env - ./config/netbox.env + - ./config/netbox-secret.env environment: VIRTUAL_HOST : 'netbox.malcolm.local' depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index f47da7a76..3338d0424 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,7 @@ services: - ./config/opensearch.env - ./config/netbox-common.env - ./config/netbox.env + - ./config/netbox-secret.env - ./config/beats-common.env - ./config/lookup-common.env - ./config/logstash.env @@ -202,6 +203,7 @@ services: - ./config/upload-common.env - ./config/auth.env - ./config/arkime.env + - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' ulimits: @@ -250,6 +252,7 @@ services: - ./config/ssl.env - ./config/upload-common.env - ./config/zeek.env + - ./config/zeek-secret.env - ./config/zeek-offline.env depends_on: - opensearch @@ -290,6 +293,7 @@ services: - ./config/upload-common.env - ./config/pcap-capture.env - ./config/zeek.env + - ./config/zeek-secret.env - ./config/zeek-live.env volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro @@ -380,6 +384,7 @@ services: - ./config/process.env - ./config/ssl.env - ./config/zeek.env + - ./config/zeek-secret.env environment: VIRTUAL_HOST : 'file-monitor.malcolm.local' volumes: @@ -547,6 +552,7 @@ services: - ./config/ssl.env - ./config/netbox-common.env - ./config/netbox.env + - ./config/netbox-secret.env environment: VIRTUAL_HOST : 'netbox.malcolm.local' depends_on: diff --git a/docs/asset-interaction-analysis.md b/docs/asset-interaction-analysis.md index a72884e26..90637e1d2 100644 --- a/docs/asset-interaction-analysis.md +++ b/docs/asset-interaction-analysis.md @@ -97,4 +97,4 @@ To clear the existing NetBox database and restore a previous backup, run the fol ``` -Note that some of the data in the NetBox database is cryptographically signed with the value of the `SECRET_KEY` environment variable in the `./netbox/env/netbox.env` environment file. A restored NetBox backup **will not work** if this value is different from when it was created. +Note that some of the data in the NetBox database is cryptographically signed with the value of the `SECRET_KEY` environment variable in the `./netbox/env/netbox-secret.env` environment file. A restored NetBox backup **will not work** if this value is different from when it was created. diff --git a/docs/contributing-local-modifications.md b/docs/contributing-local-modifications.md index afec0a841..289513aa3 100644 --- a/docs/contributing-local-modifications.md +++ b/docs/contributing-local-modifications.md @@ -9,9 +9,6 @@ Some configuration changes can be put in place by modifying local copies of conf ``` $ grep -P "^( - ./| [\w-]+:)" docker-compose-standalone.yml opensearch: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro @@ -19,28 +16,13 @@ opensearch: - ./opensearch-backup:/opt/opensearch/backup:delegated - ./opensearch/opensearch.keystore:/usr/share/opensearch/config/persist/opensearch.keystore:rw dashboards-helper: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - - ./config/dashboards-helper.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro dashboards: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro logstash: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - - ./config/netbox-common.env - - ./config/netbox.env - - ./config/beats-common.env - - ./config/lookup-common.env - - ./config/logstash.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro @@ -49,13 +31,6 @@ logstash: - ./logstash/certs/server.crt:/certs/server.crt:ro - ./logstash/certs/server.key:/certs/server.key:ro filebeat: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - - ./config/upload-common.env - - ./config/nginx.env - - ./config/beats-common.env - - ./config/filebeat.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./zeek-logs:/zeek @@ -64,107 +39,55 @@ filebeat: - ./filebeat/certs/client.crt:/certs/client.crt:ro - ./filebeat/certs/client.key:/certs/client.key:ro arkime: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - - ./config/upload-common.env - - ./config/auth.env - - ./config/arkime.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap - ./arkime-logs:/opt/arkime/logs - ./arkime-raw:/opt/arkime/raw zeek: - - ./config/process.env - - ./config/ssl.env - - ./config/upload-common.env - - ./config/zeek.env - - ./config/zeek-offline.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap:/pcap - ./zeek-logs/upload:/zeek/upload - ./zeek-logs/extract_files:/zeek/extract_files - ./zeek/intel:/opt/zeek/share/zeek/site/intel zeek-live: - - ./config/process.env - - ./config/ssl.env - - ./config/upload-common.env - - ./config/pcap-capture.env - - ./config/zeek.env - - ./config/zeek-live.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./zeek-logs/live:/zeek/live - ./zeek-logs/extract_files:/zeek/extract_files - ./zeek/intel:/opt/zeek/share/zeek/site/intel suricata: - - ./config/process.env - - ./config/ssl.env - - ./config/upload-common.env - - ./config/suricata.env - - ./config/suricata-offline.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./suricata-logs:/var/log/suricata - ./pcap:/data/pcap - ./suricata/rules:/opt/suricata/rules:ro suricata-live: - - ./config/process.env - - ./config/ssl.env - - ./config/upload-common.env - - ./config/pcap-capture.env - - ./config/suricata.env - - ./config/suricata-live.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./suricata-logs:/var/log/suricata - ./suricata/rules:/opt/suricata/rules:ro file-monitor: - - ./config/process.env - - ./config/ssl.env - - ./config/zeek.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./zeek-logs/extract_files:/zeek/extract_files - ./zeek-logs/current:/zeek/logs - ./yara/rules:/yara-rules/custom:ro pcap-capture: - - ./config/process.env - - ./config/ssl.env - - ./config/pcap-capture.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - - ./config/upload-common.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./zeek-logs:/zeek - ./pcap:/pcap upload: - - ./config/process.env - - ./config/ssl.env - - ./config/auth.env - - ./config/upload.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/var/www/upload/server/php/chroot/files htadmin: - - ./config/process.env - - ./config/ssl.env - - ./config/auth-common.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./htadmin/config.ini:/var/www/htadmin/config/config.ini:rw - ./htadmin/metadata:/var/www/htadmin/config/metadata:rw - ./nginx/htpasswd:/var/www/htadmin/auth/htpasswd:rw freq: - - ./config/process.env - - ./config/ssl.env - - ./config/lookup-common.env - ./nginx/ca-trust:/var/local/ca-trust:ro netbox: - - ./config/process.env - - ./config/ssl.env - - ./config/netbox-common.env - - ./config/netbox.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./netbox/config/configuration:/etc/netbox/config:ro - ./netbox/config/reports:/etc/netbox/reports:ro @@ -172,36 +95,17 @@ netbox: - ./netbox/media:/opt/netbox/netbox/media:rw - ./net-map.json:/usr/local/share/net-map.json:ro netbox-postgres: - - ./config/process.env - - ./config/ssl.env - - ./config/netbox-common.env - - ./config/netbox-postgres.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./netbox/postgres:/var/lib/postgresql/data:rw netbox-redis: - - ./config/process.env - - ./config/ssl.env - - ./config/netbox-common.env - - ./config/netbox-redis.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./netbox/redis:/data netbox-redis-cache: - - ./config/process.env - - ./config/ssl.env - - ./config/netbox-common.env - - ./config/netbox-redis-cache.env - ./nginx/ca-trust:/var/local/ca-trust:ro api: - - ./config/process.env - - ./config/ssl.env - - ./config/opensearch.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro nginx-proxy: - - ./config/process.env - - ./config/ssl.env - - ./config/auth-common.env - - ./config/nginx.env - ./nginx/ca-trust:/var/local/ca-trust:ro - ./nginx/nginx_ldap.conf:/etc/nginx/nginx_ldap.conf:ro - ./nginx/htpasswd:/etc/nginx/auth/htpasswd:ro diff --git a/docs/file-scanning.md b/docs/file-scanning.md index 7847bb144..57c9560f8 100644 --- a/docs/file-scanning.md +++ b/docs/file-scanning.md @@ -12,7 +12,7 @@ To specify which files should be extracted, the following values are acceptable Extracted files can be examined through any of the following methods: -* submitting file hashes to [**VirusTotal**](https://www.virustotal.com/en/#search); to enable this method, specify the `VTOT_API2_KEY` [environment variable in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) +* submitting file hashes to [**VirusTotal**](https://www.virustotal.com/en/#search); to enable this method, specify the `VTOT_API2_KEY` [environment variable in `zeek-secret.env`](malcolm-config.md#MalcolmConfigEnvVars) * scanning files with [**ClamAV**](https://www.clamav.net/); to enable this method, set the `EXTRACTED_FILE_ENABLE_CLAMAV` [environment variable in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) to `true` * scanning files with [**Yara**](https://github.com/VirusTotal/yara); to enable this method, set the `EXTRACTED_FILE_ENABLE_YARA` [environment variable in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) to `true` * scanning PE (portable executable) files with [**Capa**](https://github.com/fireeye/capa); to enable this method, set the `EXTRACTED_FILE_ENABLE_CAPA` [environment variable in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) to `true` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index c7b6e2c2e..bea11a4dc 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -247,8 +247,8 @@ $ ./scripts/start -f /path/to/kubeconfig.yml The Kubernetes resources under the `malcolm` namespace (its pods, storage volumes, containers, etc.) will be initialized and started using the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/), including: -* creating [ConfigMap objects](https://kubernetes.io/docs/concepts/configuration/configmap/) from Malcolm's [environment variable files](malcolm-config.md#MalcolmConfigEnvVars) -* creating [ConfigMap objects](https://kubernetes.io/docs/concepts/configuration/configmap/) from other configuration files stored locally below the Malcolm directory +* creating [ConfigMap objects](https://kubernetes.io/docs/concepts/configuration/configmap/) and [Secret objects](https://kubernetes.io/docs/concepts/configuration/secret/) from Malcolm's [environment variable files](malcolm-config.md#MalcolmConfigEnvVars) +* creating [ConfigMap objects](https://kubernetes.io/docs/concepts/configuration/configmap/) and [Secret objects](https://kubernetes.io/docs/concepts/configuration/secret/) from other configuration files stored locally below the Malcolm directory * deploying the objects defined in the [Kubernetes manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/) in `./kubernetes` After a few moments you can check the status of the deployment: diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 0fa5d4789..ce8317c62 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -8,7 +8,7 @@ Run `./scripts/configure` and answer the questions to configure Malcolm. For an Although the configuration script automates many of the following configuration and tuning parameters, some environment variables of particular interest are listed here for reference. -* **`arkime.env`** - settings for [Arkime](https://arkime.com/) +* **`arkime.env`** and **`arkime-secret.env`** - settings for [Arkime](https://arkime.com/) - `ARKIME_ANALYZE_PCAP_THREADS` – the number of threads available to Arkime for analyzing PCAP files (default `1`) - `MANAGE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will be marked as available for deletion by Arkime if available storage space becomes too low (default `false`) - `MAXMIND_GEOIP_DB_LICENSE_KEY` - Malcolm uses MaxMind's free GeoLite2 databases for GeoIP lookups. As of December 30, 2019, these databases are [no longer available](https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/) for download via a public URL. Instead, they must be downloaded using a MaxMind license key (available without charge [from MaxMind](https://www.maxmind.com/en/geolite2/signup)). The license key can be specified here for GeoIP database downloads during build- and run-time. @@ -33,7 +33,7 @@ Although the configuration script automates many of the following configuration - `FREQ_SEVERITY_THRESHOLD` - when [severity scoring](severity.md#Severity) is enabled, this variable indicates the entropy threshold for assigning severity to events with entropy scores calculated by [`freq`](https://github.com/MarkBaggett/freq); a lower value will only assign severity scores to fewer domain names with higher entropy (e.g., `2.0` for `NQZHTFHRMYMTVBQJE.COM`), while a higher value will assign severity scores to more domain names with lower entropy (e.g., `7.5` for `naturallanguagedomain.example.org`) (default `2.0`) - `SENSITIVE_COUNTRY_CODES` - when [severity scoring](severity.md#Severity) is enabled, this variable defines a comma-separated list of sensitive countries (using [ISO 3166-1 alpha-2 codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Current_codes)) (default `'AM,AZ,BY,CN,CU,DZ,GE,HK,IL,IN,IQ,IR,KG,KP,KZ,LY,MD,MO,PK,RU,SD,SS,SY,TJ,TM,TW,UA,UZ'`, taken from the U.S. Department of Energy Sensitive Country List) - `TOTAL_MEGABYTES_SEVERITY_THRESHOLD` - when [severity scoring](severity.md#Severity) is enabled, this variable indicates the size threshold (in megabytes) for assigning severity to large connections or file transfers (default `1000`) -* **`netbox-common.env`**, `netbox.env`, `netbox-postgres.env`, `netbox-redis-cache.env` and `netbox-redis.env` - settings related to [NetBox](https://netbox.dev/) and [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis) +* **`netbox-common.env`**, `netbox.env`, `netbox-secret.env`, `netbox-postgres.env`, `netbox-redis-cache.env` and `netbox-redis.env` - settings related to [NetBox](https://netbox.dev/) and [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis) - `NETBOX_DISABLED` - if set to `true`, Malcolm will **not** start and manage a [NetBox](asset-interaction-analysis.md#AssetInteractionAnalysis) instance (default `true`) * **`nginx.env`** - settings specific to Malcolm's nginx reverse proxy - `NGINX_LOG_ACCESS_AND_ERRORS` - if set to `true`, all access to Malcolm via its [web interfaces](quickstart.md#UserInterfaceURLs) will be logged to OpenSearch (default `false`) @@ -67,7 +67,7 @@ Although the configuration script automates many of the following configuration - `SURICATA_…` - the [`suricata` container entrypoint script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/suricata_config_populate.py) can use **many** more environment variables to tweak [suricata.yaml](https://github.com/OISF/suricata/blob/master/suricata.yaml.in); in that script, `DEFAULT_VARS` defines those variables (albeit without the `SURICATA_` prefix you must add to each for use) * **`upload-common.env`** and **`upload.env`** - settings for dealing with PCAP files [uploaded](upload.md#Upload) to Malcolm for analysis - `AUTO_TAG` – if set to `true`, Malcolm will automatically create Arkime sessions and Zeek logs with tags based on the filename, as described in [Tagging](upload.md#Tagging) (default `true`) -* **`zeek.env`**, **`zeek-live.env`** and **`zeek-offline.env`** - settings for [Zeek](https://www.zeek.org/index.html) and for scanning [extracted files](file-scanning.md#ZeekFileExtraction) Zeek observes in network traffic +* **`zeek.env`**, **`zeek-secret.env`**, **`zeek-live.env`** and **`zeek-offline.env`** - settings for [Zeek](https://www.zeek.org/index.html) and for scanning [extracted files](file-scanning.md#ZeekFileExtraction) Zeek observes in network traffic - `EXTRACTED_FILE_CAPA_VERBOSE` – if set to `true`, all Capa rule hits will be logged; otherwise (`false`) only [MITRE ATT&CK® technique](https://attack.mitre.org/techniques) classifications will be logged - `EXTRACTED_FILE_ENABLE_CAPA` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) that are determined to be PE (portable executable) files will be scanned with [Capa](https://github.com/fireeye/capa) - `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) will be scanned with [ClamAV](https://www.clamav.net/) diff --git a/kubernetes/02-opensearch.yml b/kubernetes/02-opensearch.yml index 16899ed91..976a6988c 100644 --- a/kubernetes/02-opensearch.yml +++ b/kubernetes/02-opensearch.yml @@ -56,14 +56,14 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: opensearch-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: opensearch-opensearch-curlrc-volume + - mountPath: /var/local/curlrc/secretmap + name: opensearch-opensearch-curlrc-secret-volume - mountPath: "/usr/share/opensearch/data" name: opensearch-data-volume - mountPath: "/opt/opensearch/backup" name: opensearch-backup-volume - - name: opensearch-keystore-default-volume - mountPath: /usr/share/opensearch/config/bootstrap/configmap + - name: opensearch-keystore-default-secret-volume + mountPath: /usr/share/opensearch/config/bootstrap/secretmap - name: opensearch-config-persist-volume mountPath: /usr/share/opensearch/config/persist subPath: "opensearch" @@ -71,18 +71,18 @@ spec: - name: opensearch-var-local-catrust-volume configMap: name: var-local-catrust - - name: opensearch-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc + - name: opensearch-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc - name: opensearch-data-volume persistentVolumeClaim: claimName: opensearch-claim - name: opensearch-backup-volume persistentVolumeClaim: claimName: opensearch-backup-claim - - name: opensearch-keystore-default-volume - configMap: - name: opensearch-keystore + - name: opensearch-keystore-default-secret-volume + secret: + secretName: opensearch-keystore - name: opensearch-config-persist-volume persistentVolumeClaim: claimName: config-claim \ No newline at end of file diff --git a/kubernetes/03-dashboards.yml b/kubernetes/03-dashboards.yml index 4f62d5fc7..4a2043177 100644 --- a/kubernetes/03-dashboards.yml +++ b/kubernetes/03-dashboards.yml @@ -61,12 +61,12 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: dashboards-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: dashboards-opensearch-curlrc-volume + - mountPath: /var/local/curlrc/secretmap + name: dashboards-opensearch-curlrc-secret-volume volumes: - name: dashboards-var-local-catrust-volume configMap: name: var-local-catrust - - name: dashboards-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc + - name: dashboards-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc diff --git a/kubernetes/04-upload.yml b/kubernetes/04-upload.yml index 1c15e0d3c..3e47c9d1b 100644 --- a/kubernetes/04-upload.yml +++ b/kubernetes/04-upload.yml @@ -50,7 +50,7 @@ spec: name: process-env - configMapRef: name: ssl-env - - configMapRef: + - secretRef: name: auth-env - configMapRef: name: upload-env diff --git a/kubernetes/05-pcap-monitor.yml b/kubernetes/05-pcap-monitor.yml index e19938fc4..eb649186e 100644 --- a/kubernetes/05-pcap-monitor.yml +++ b/kubernetes/05-pcap-monitor.yml @@ -62,8 +62,8 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: pcap-monitor-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: pcap-monitor-opensearch-curlrc-volume + - mountPath: /var/local/curlrc/secretmap + name: pcap-monitor-opensearch-curlrc-secret-volume - mountPath: "/pcap" name: pcap-monitor-pcap-volume - mountPath: "/zeek" @@ -72,9 +72,9 @@ spec: - name: pcap-monitor-var-local-catrust-volume configMap: name: var-local-catrust - - name: pcap-monitor-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc + - name: pcap-monitor-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc - name: pcap-monitor-pcap-volume persistentVolumeClaim: claimName: pcap-claim diff --git a/kubernetes/06-arkime.yml b/kubernetes/06-arkime.yml index cc373ff35..0d5a3e70c 100644 --- a/kubernetes/06-arkime.yml +++ b/kubernetes/06-arkime.yml @@ -50,12 +50,14 @@ spec: name: ssl-env - configMapRef: name: opensearch-env - - configMapRef: + - secretRef: name: auth-env - configMapRef: name: upload-common-env - configMapRef: name: arkime-env + - secretRef: + name: arkime-secret-env env: - name: VIRTUAL_HOST value: "arkime.malcolm.local" @@ -72,8 +74,8 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: arkime-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: arkime-opensearch-curlrc-volume + - mountPath: /var/local/curlrc/secretmap + name: arkime-opensearch-curlrc-secret-volume - mountPath: "/data/pcap" name: arkime-pcap-volume - name: arkime-runtime-logs-volume @@ -83,9 +85,9 @@ spec: - name: arkime-var-local-catrust-volume configMap: name: var-local-catrust - - name: arkime-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc + - name: arkime-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc - name: arkime-pcap-volume persistentVolumeClaim: claimName: pcap-claim diff --git a/kubernetes/07-api.yml b/kubernetes/07-api.yml index 42cf3c3e1..a93795ec5 100644 --- a/kubernetes/07-api.yml +++ b/kubernetes/07-api.yml @@ -63,12 +63,12 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: api-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: api-opensearch-curlrc-volume + - mountPath: /var/local/curlrc/secretmap + name: api-opensearch-curlrc-secret-volume volumes: - name: api-var-local-catrust-volume configMap: name: var-local-catrust - - name: api-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc + - name: api-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc diff --git a/kubernetes/08-dashboards-helper.yml b/kubernetes/08-dashboards-helper.yml index 4ce938b93..a0b26a4e0 100644 --- a/kubernetes/08-dashboards-helper.yml +++ b/kubernetes/08-dashboards-helper.yml @@ -65,12 +65,12 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: dashboards-helper-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: dashboards-helper-opensearch-curlrc-volume + - mountPath: /var/local/curlrc/secretmap + name: dashboards-helper-opensearch-curlrc-secret-volume volumes: - name: dashboards-helper-var-local-catrust-volume configMap: name: var-local-catrust - - name: dashboards-helper-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc + - name: dashboards-helper-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc diff --git a/kubernetes/09-zeek.yml b/kubernetes/09-zeek.yml index c99d3e21b..4c505c03e 100644 --- a/kubernetes/09-zeek.yml +++ b/kubernetes/09-zeek.yml @@ -37,6 +37,8 @@ spec: name: upload-common-env - configMapRef: name: zeek-env + - secretRef: + name: zeek-secret-env - configMapRef: name: zeek-offline-env livenessProbe: diff --git a/kubernetes/11-file-monitor.yml b/kubernetes/11-file-monitor.yml index 9606cc927..503302dc5 100644 --- a/kubernetes/11-file-monitor.yml +++ b/kubernetes/11-file-monitor.yml @@ -51,6 +51,8 @@ spec: name: ssl-env - configMapRef: name: zeek-env + - secretRef: + name: zeek-secret-env env: - name: VIRTUAL_HOST value: "file-monitor.malcolm.local" diff --git a/kubernetes/12-filebeat.yml b/kubernetes/12-filebeat.yml index 74447c240..73bef36aa 100644 --- a/kubernetes/12-filebeat.yml +++ b/kubernetes/12-filebeat.yml @@ -68,10 +68,10 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: filebeat-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: filebeat-opensearch-curlrc-volume - - mountPath: /certs/configmap - name: filebeat-certs-volume + - mountPath: /var/local/curlrc/secretmap + name: filebeat-opensearch-curlrc-secret-volume + - mountPath: /certs/secretmap + name: filebeat-certs-secret-volume - mountPath: "/zeek" name: filebeat-zeek-volume - mountPath: "/suricata" @@ -83,12 +83,12 @@ spec: - name: filebeat-var-local-catrust-volume configMap: name: var-local-catrust - - name: filebeat-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc - - name: filebeat-certs-volume - configMap: - name: filebeat-certs + - name: filebeat-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc + - name: filebeat-certs-secret-volume + secret: + secretName: filebeat-certs - name: filebeat-zeek-volume persistentVolumeClaim: claimName: zeek-claim diff --git a/kubernetes/13-logstash.yml b/kubernetes/13-logstash.yml index 255eb4d78..0a003523b 100644 --- a/kubernetes/13-logstash.yml +++ b/kubernetes/13-logstash.yml @@ -77,6 +77,8 @@ spec: name: netbox-common-env - configMapRef: name: netbox-env + - secretRef: + name: netbox-secret-env - configMapRef: name: beats-common-env - configMapRef: @@ -96,14 +98,14 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: logstash-var-local-catrust-volume - - mountPath: /var/local/curlrc/configmap - name: logstash-opensearch-curlrc-volume - - mountPath: /certs/configmap - name: logstash-certs-volume + - mountPath: /var/local/curlrc/secretmap + name: logstash-opensearch-curlrc-secret-volume + - mountPath: /certs/secretmap + name: logstash-certs-secret-volume - mountPath: /etc/configmap name: logstash-maps-volume - - name: logstash-keystore-default-volume - mountPath: /usr/share/logstash/config/bootstrap/configmap + - name: logstash-keystore-default-secret-volume + mountPath: /usr/share/logstash/config/bootstrap/secretmap - name: logstash-config-persist-volume mountPath: /usr/share/logstash/config/persist subPath: "logstash" @@ -111,18 +113,18 @@ spec: - name: logstash-var-local-catrust-volume configMap: name: var-local-catrust - - name: logstash-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc - - name: logstash-certs-volume - configMap: - name: logstash-certs + - name: logstash-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc + - name: logstash-certs-secret-volume + secret: + secretName: logstash-certs - name: logstash-maps-volume configMap: name: logstash-maps - - name: logstash-keystore-default-volume - configMap: - name: logstash-keystore + - name: logstash-keystore-default-secret-volume + secret: + secretName: logstash-keystore - name: logstash-config-persist-volume persistentVolumeClaim: claimName: config-claim \ No newline at end of file diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 173005739..575a9f0f9 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -47,7 +47,7 @@ spec: name: ssl-env - configMapRef: name: netbox-common-env - - configMapRef: + - secretRef: name: netbox-redis-env env: - name: VIRTUAL_HOST diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 31e5b21b3..959aae61b 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -47,7 +47,7 @@ spec: name: ssl-env - configMapRef: name: netbox-common-env - - configMapRef: + - secretRef: name: netbox-redis-cache-env env: - name: VIRTUAL_HOST diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index e719d1f5a..7f86bb314 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -45,7 +45,7 @@ spec: name: ssl-env - configMapRef: name: netbox-common-env - - configMapRef: + - secretRef: name: netbox-postgres-env env: - name: VIRTUAL_HOST diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 67abd4a09..a12fbf112 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -59,6 +59,8 @@ spec: name: netbox-common-env - configMapRef: name: netbox-env + - secretRef: + name: netbox-secret-env env: - name: VIRTUAL_HOST value: "netbox.malcolm.local" diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index 0a87e3872..2fa785ebf 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -56,6 +56,8 @@ spec: subPath: "auth" - mountPath: /var/www/htadmin/default/configmap name: htadmin-config-default-volume + - mountPath: /var/www/htadmin/default/secretmap + name: htadmin-config-default-secret-volume - mountPath: /var/www/htadmin/config name: htadmin-config-volume subPath: "htadmin" @@ -69,4 +71,7 @@ spec: - name: htadmin-config-default-volume configMap: name: htadmin-config + - name: htadmin-config-default-secret-volume + secret: + secretName: htadmin-config diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 2f8c6b03f..23f0c7a2d 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -37,6 +37,8 @@ spec: name: upload-common-env - configMapRef: name: zeek-env + - secretRef: + name: zeek-secret-env - configMapRef: name: zeek-live-env - configMapRef: diff --git a/kubernetes/99-nginx-proxy.yml b/kubernetes/99-nginx-proxy.yml index 72ebc014f..4b0b544d3 100644 --- a/kubernetes/99-nginx-proxy.yml +++ b/kubernetes/99-nginx-proxy.yml @@ -73,19 +73,21 @@ spec: volumeMounts: - name: nginx-etc-nginx-volume mountPath: /etc/nginx/configmap + - name: nginx-etc-nginx-secret-volume + mountPath: /etc/nginx/secretmap - name: nginx-var-local-catrust-volume mountPath: /var/local/ca-trust/configmap - - name: nginx-etc-nginx-certs-volume - mountPath: /etc/nginx/certs/configmap + - name: nginx-etc-nginx-certs-secret-volume + mountPath: /etc/nginx/certs/secretmap - name: nginx-etc-nginx-certs-pem-volume mountPath: /etc/nginx/dhparam/configmap - - name: nginx-opensearch-curlrc-volume - mountPath: /var/local/curlrc/configmap + - name: nginx-opensearch-curlrc-secret-volume + mountPath: /var/local/curlrc/secretmap - name: nginx-etc-auth-volume mountPath: /etc/nginx/auth subPath: "auth" - - name: nginx-etc-auth-default-volume - mountPath: /tmp/auth/default/configmap + - name: nginx-etc-auth-default-secret-volume + mountPath: /tmp/auth/default/secretmap - name: nginx-runtime-logs-volume mountPath: /var/log/nginx subPath: "nginx" @@ -93,24 +95,27 @@ spec: - name: nginx-etc-nginx-volume configMap: name: etc-nginx + - name: nginx-etc-nginx-secret-volume + secret: + secretName: etc-nginx - name: nginx-var-local-catrust-volume configMap: name: var-local-catrust - - name: nginx-etc-nginx-certs-volume - configMap: - name: etc-nginx-certs + - name: nginx-etc-nginx-certs-secret-volume + secret: + secretName: etc-nginx-certs - name: nginx-etc-nginx-certs-pem-volume configMap: name: etc-nginx-certs-pem - - name: nginx-opensearch-curlrc-volume - configMap: - name: opensearch-curlrc + - name: nginx-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc - name: nginx-etc-auth-volume persistentVolumeClaim: claimName: config-claim - - name: nginx-etc-auth-default-volume - configMap: - name: etc-nginx-auth + - name: nginx-etc-auth-default-secret-volume + secret: + secretName: etc-nginx-auth - name: nginx-runtime-logs-volume persistentVolumeClaim: claimName: runtime-logs-claim \ No newline at end of file diff --git a/scripts/control.py b/scripts/control.py index 6a9129d3b..d82df1738 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1119,7 +1119,9 @@ def authSetup(wipe=False): 'netbox', "(Re)generate internal passwords for NetBox", False, - not os.path.isfile(os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'netbox.env')))), + not os.path.isfile( + os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'netbox-secret.env'))) + ), ), ( 'txfwcerts', @@ -1190,6 +1192,7 @@ def authSetup(wipe=False): ) f.write(f'MALCOLM_USERNAME={username}\n') f.write(f'MALCOLM_PASSWORD={b64encode(passwordEncrypted.encode()).decode("ascii")}\n') + f.write('K8S_SECRET=True\n') os.chmod(authEnvFile, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) # create or update the htpasswd file @@ -1627,20 +1630,23 @@ def authSetup(wipe=False): f.write('POSTGRES_DB=netbox\n') f.write(f'POSTGRES_PASSWORD={netboxPostGresPassword}\n') f.write('POSTGRES_USER=netbox\n') + f.write('K8S_SECRET=True\n') os.chmod('netbox-postgres.env', stat.S_IRUSR | stat.S_IWUSR) with open('netbox-redis-cache.env', 'w') as f: f.write(f'REDIS_PASSWORD={netboxRedisCachePassword}\n') + f.write('K8S_SECRET=True\n') os.chmod('netbox-redis-cache.env', stat.S_IRUSR | stat.S_IWUSR) with open('netbox-redis.env', 'w') as f: f.write(f'REDIS_PASSWORD={netboxRedisPassword}\n') + f.write('K8S_SECRET=True\n') os.chmod('netbox-redis.env', stat.S_IRUSR | stat.S_IWUSR) - if (not os.path.isfile('netbox.env')) and (os.path.isfile('netbox.env.example')): - shutil.copy2('netbox.env.example', 'netbox.env') + if (not os.path.isfile('netbox-secret.env')) and (os.path.isfile('netbox-secret.env.example')): + shutil.copy2('netbox-secret.env.example', 'netbox-secret.env') - with fileinput.FileInput('netbox.env', inplace=True, backup=None) as envFile: + with fileinput.FileInput('netbox-secret.env', inplace=True, backup=None) as envFile: for line in envFile: line = line.rstrip("\n") @@ -1680,10 +1686,16 @@ def authSetup(wipe=False): fr"\g<1>{netboxSuToken}", line, ) + elif line.startswith('K8S_SECRET'): + line = re.sub( + r'(SUPERUSER_API_TOKEN\s*=\s*)(\S+)', + fr"\g<1>True", + line, + ) print(line) - os.chmod('netbox.env', stat.S_IRUSR | stat.S_IWUSR) + os.chmod('netbox-secret.env', stat.S_IRUSR | stat.S_IWUSR) elif authItem[0] == 'txfwcerts': DisplayMessage( diff --git a/scripts/install.py b/scripts/install.py index e5e9ab45a..8afec8a11 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1269,13 +1269,13 @@ def tweak_malcolm_runtime( ), # key for encrypted HTTP-served extracted files (' -> '' for escaping in YAML) EnvValue( - os.path.join(args.configDir, 'zeek.env'), + os.path.join(args.configDir, 'zeek-secret.env'), 'EXTRACTED_FILE_HTTP_SERVER_KEY', fileCarveHttpServeEncryptKey, ), # virustotal API key EnvValue( - os.path.join(args.configDir, 'zeek.env'), + os.path.join(args.configDir, 'zeek-secret.env'), 'VTOT_API2_KEY', vtotApiKey, ), diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 94e3c9c55..8dfb84a3b 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -601,7 +601,7 @@ def MalcolmAuthFilesExist(configDir=None): and os.path.isfile(os.path.join(MalcolmPath, os.path.join('nginx', os.path.join('certs', 'cert.pem')))) and os.path.isfile(os.path.join(MalcolmPath, os.path.join('nginx', os.path.join('certs', 'key.pem')))) and os.path.isfile(os.path.join(MalcolmPath, os.path.join('htadmin', 'config.ini'))) - and os.path.isfile(os.path.join(configDirToCheck, 'netbox.env')) + and os.path.isfile(os.path.join(configDirToCheck, 'netbox-secret.env')) and os.path.isfile(os.path.join(configDirToCheck, 'netbox-postgres.env')) and os.path.isfile(os.path.join(configDirToCheck, 'netbox-redis-cache.env')) and os.path.isfile(os.path.join(configDirToCheck, 'netbox-redis.env')) diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index 73dc83158..9dc88a517 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -34,63 +34,128 @@ ################################################################################################### MALCOLM_IMAGE_PREFIX = 'ghcr.io/idaholab/malcolm/' +MALCOLM_DOTFILE_SECRET_KEY = 'K8S_SECRET' + MALCOLM_CONFIGMAPS = { 'etc-nginx': [ - os.path.join(MalcolmPath, os.path.join('nginx', 'nginx_ldap.conf')), - os.path.join(MalcolmPath, os.path.join('nginx', 'nginx.conf')), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('nginx', 'nginx_ldap.conf')), + }, + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('nginx', 'nginx.conf')), + }, ], 'var-local-catrust': [ - os.path.join(MalcolmPath, os.path.join('nginx', 'ca-trust')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('nginx', 'ca-trust')), + }, ], 'etc-nginx-certs': [ - os.path.join(MalcolmPath, os.path.join('nginx', 'certs')), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('nginx', 'certs')), + }, ], 'etc-nginx-certs-pem': [ - os.path.join(MalcolmPath, os.path.join(os.path.join('nginx', 'certs'), 'dhparam.pem')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join(os.path.join('nginx', 'certs'), 'dhparam.pem')), + }, ], 'etc-nginx-auth': [ - os.path.join(MalcolmPath, os.path.join('nginx', 'htpasswd')), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('nginx', 'htpasswd')), + }, ], 'opensearch-curlrc': [ - os.path.join(MalcolmPath, '.opensearch.primary.curlrc'), - os.path.join(MalcolmPath, '.opensearch.secondary.curlrc'), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, '.opensearch.primary.curlrc'), + }, + { + 'secret': True, + 'path': os.path.join(MalcolmPath, '.opensearch.secondary.curlrc'), + }, ], 'opensearch-keystore': [ - os.path.join(MalcolmPath, os.path.join('opensearch', 'opensearch.keystore')), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('opensearch', 'opensearch.keystore')), + }, ], 'logstash-certs': [ - os.path.join(MalcolmPath, os.path.join('logstash', 'certs')), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('logstash', 'certs')), + }, ], 'logstash-maps': [ - os.path.join(MalcolmPath, os.path.join('logstash', 'maps')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('logstash', 'maps')), + }, ], 'logstash-keystore': [ - os.path.join(MalcolmPath, os.path.join('logstash', 'logstash.keystore')), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('logstash', 'logstash.keystore')), + }, ], 'yara-rules': [ - os.path.join(MalcolmPath, os.path.join('yara', 'rules')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('yara', 'rules')), + }, ], 'suricata-rules': [ - os.path.join(MalcolmPath, os.path.join('suricata', 'rules')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('suricata', 'rules')), + }, ], 'filebeat-certs': [ - os.path.join(MalcolmPath, os.path.join('filebeat', 'certs')), + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('filebeat', 'certs')), + }, ], 'netbox-netmap-json': [ - 'net-map.json', + { + 'secret': False, + 'path': os.path.join(MalcolmPath, 'net-map.json'), + }, ], 'netbox-config': [ - os.path.join(MalcolmPath, os.path.join(os.path.join('netbox', 'config'), 'configuration')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join(os.path.join('netbox', 'config'), 'configuration')), + }, ], 'netbox-reports': [ - os.path.join(MalcolmPath, os.path.join(os.path.join('netbox', 'config'), 'reports')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join(os.path.join('netbox', 'config'), 'reports')), + }, ], 'netbox-scripts': [ - os.path.join(MalcolmPath, os.path.join(os.path.join('netbox', 'config'), 'scripts')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join(os.path.join('netbox', 'config'), 'scripts')), + }, ], 'htadmin-config': [ - os.path.join(MalcolmPath, os.path.join('htadmin', 'config.ini')), - os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')), + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('htadmin', 'config.ini')), + }, + { + 'secret': True, + 'path': os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')), + }, ], } @@ -584,85 +649,107 @@ def StartMalcolm(namespace, malcolmPath, configPath): # create configmaps from files results_dict['create_namespaced_config_map']['result'] = dict() + results_dict['create_namespaced_secret']['result'] = dict() for configMapName, configMapFiles in MALCOLM_CONFIGMAPS.items(): - try: - dataMap = {} - binaryDataMap = {} - for fname in configMapFiles: - if os.path.isfile(fname): - contents = file_contents( - fname, - binary_fallback=True, - ) - if hasattr(contents, 'decode'): - binaryDataMap[os.path.basename(fname)] = base64.b64encode(contents).decode('utf-8') - else: - dataMap[os.path.basename(fname)] = contents - elif os.path.isdir(fname): - for subfname in glob.iglob(os.path.join(os.path.join(fname, '**'), '*'), recursive=True): - if os.path.isfile(subfname): + for isSecret in (True, False): + resultsEntry = 'create_namespaced_secret' if isSecret else 'create_namespaced_config_map' + mapFiles = [x['path'] for x in configMapFiles if (x.get('secret', False) is isSecret)] + if mapFiles: + try: + dataMap = {} + binaryDataMap = {} + for fname in mapFiles: + if os.path.isfile(fname): contents = file_contents( - subfname, + fname, binary_fallback=True, ) if hasattr(contents, 'decode'): - binaryDataMap[os.path.basename(subfname)] = base64.b64encode(contents).decode( - 'utf-8' - ) + binaryDataMap[os.path.basename(fname)] = base64.b64encode(contents).decode('utf-8') else: - dataMap[os.path.basename(subfname)] = contents - results_dict['create_namespaced_config_map']['result'][ - configMapName - ] = client.create_namespaced_config_map( - namespace=namespace, - body=kubeImported.client.V1ConfigMap( - metadata=kubeImported.client.V1ObjectMeta( + dataMap[os.path.basename(fname)] = contents + elif os.path.isdir(fname): + for subfname in glob.iglob( + os.path.join(os.path.join(fname, '**'), '*'), recursive=True + ): + if os.path.isfile(subfname): + contents = file_contents( + subfname, + binary_fallback=True, + ) + if hasattr(contents, 'decode'): + binaryDataMap[os.path.basename(subfname)] = base64.b64encode( + contents + ).decode('utf-8') + else: + dataMap[os.path.basename(subfname)] = contents + metadata = kubeImported.client.V1ObjectMeta( name=configMapName, namespace=namespace, - ), - data=dataMap if dataMap else {}, - binary_data=binaryDataMap if binaryDataMap else {}, - ), - ).metadata - except kubeImported.client.rest.ApiException as x: - if x.status != 409: - if 'error' not in results_dict['create_namespaced_config_map']: - results_dict['create_namespaced_config_map']['error'] = dict() - results_dict['create_namespaced_config_map']['error'][ - os.path.basename(configMapName) - ] = LoadStrIfJson(str(x)) - if not results_dict['create_namespaced_config_map']['error'][os.path.basename(configMapName)]: - results_dict['create_namespaced_config_map']['error'][os.path.basename(configMapName)] = str(x) - - # create configmaps from .env files + ) + if isSecret: + results_dict[resultsEntry]['result'][configMapName] = client.create_namespaced_secret( + namespace=namespace, + body=kubeImported.client.V1Secret( + metadata=metadata, + string_data=dataMap if dataMap else {}, + data=binaryDataMap if binaryDataMap else {}, + ), + ).metadata + else: + results_dict[resultsEntry]['result'][configMapName] = client.create_namespaced_config_map( + namespace=namespace, + body=kubeImported.client.V1ConfigMap( + metadata=metadata, + data=dataMap if dataMap else {}, + binary_data=binaryDataMap if binaryDataMap else {}, + ), + ).metadata + except kubeImported.client.rest.ApiException as x: + if x.status != 409: + if 'error' not in results_dict[resultsEntry]: + results_dict[resultsEntry]['error'] = dict() + results_dict[resultsEntry]['error'][os.path.basename(configMapName)] = LoadStrIfJson(str(x)) + if not results_dict[resultsEntry]['error'][os.path.basename(configMapName)]: + results_dict[resultsEntry]['error'][os.path.basename(configMapName)] = str(x) + + # create configmaps (or secrets, given a K8S_SECRET key) from .env files results_dict['create_namespaced_config_map_from_env_file']['result'] = dict() + results_dict['create_namespaced_secret_from_env_file']['result'] = dict() for envFileName in glob.iglob(os.path.join(configPath, '*.env'), recursive=False): if os.path.isfile(envFileName): try: - configMap = kubeImported.client.V1ConfigMap() - configMap.metadata = kubeImported.client.V1ObjectMeta( + values = dotenvImported.dotenv_values(envFileName) + isSecret = val2bool(values.pop(MALCOLM_DOTFILE_SECRET_KEY, False)) + metadata = kubeImported.client.V1ObjectMeta( name=remove_suffix(os.path.basename(envFileName), '.env') + '-env' ) - configMap.data = dotenvImported.dotenv_values(envFileName) - results_dict['create_namespaced_config_map_from_env_file']['result'][ - configMap.metadata.name - ] = client.create_namespaced_config_map( - namespace=namespace, - body=configMap, - ).metadata + if isSecret: + resultsEntry = 'create_namespaced_secret_from_env_file' + results_dict[resultsEntry]['result'][metadata.name] = client.create_namespaced_secret( + namespace=namespace, + body=kubeImported.client.V1Secret( + metadata=metadata, + string_data=values if values else {}, + ), + ).metadata + else: + resultsEntry = 'create_namespaced_config_map_from_env_file' + results_dict[resultsEntry]['result'][metadata.name] = client.create_namespaced_config_map( + namespace=namespace, + body=kubeImported.client.V1ConfigMap( + metadata=metadata, + data=values if values else {}, + ), + ).metadata + except kubeImported.client.rest.ApiException as x: if x.status != 409: - if 'error' not in results_dict['create_namespaced_config_map_from_env_file']: - results_dict['create_namespaced_config_map_from_env_file']['error'] = dict() - results_dict['create_namespaced_config_map_from_env_file']['error'][ - os.path.basename(envFileName) - ] = LoadStrIfJson(str(x)) - if not results_dict['create_namespaced_config_map_from_env_file']['error'][ - os.path.basename(envFileName) - ]: - results_dict['create_namespaced_config_map_from_env_file']['error'][ - os.path.basename(envFileName) - ] = str(x) + if 'error' not in results_dict[resultsEntry]: + results_dict[resultsEntry]['error'] = dict() + results_dict[resultsEntry]['error'][os.path.basename(envFileName)] = LoadStrIfJson(str(x)) + if not results_dict[resultsEntry]['error'][os.path.basename(envFileName)]: + results_dict[resultsEntry]['error'][os.path.basename(envFileName)] = str(x) # apply manifests results_dict['create_from_yaml']['result'] = dict()