allow overriding destination index for Zeek and Suricata logs, work i…

…n progress, idaholab#313

Dockerfiles/api.Dockerfile

@@ -44,29 +44,13 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
 ARG FLASK_ENV=production
-ARG ARKIME_FIELDS_INDEX="arkime_fields"
-ARG ARKIME_INDEX_PATTERN="arkime_sessions3-*"
-ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
-ARG BEATS_INDEX_PATTERN="malcolm_beats_*"
-ARG BEATS_INDEX_TIME_FIELD="@timestamp"
-ARG DASHBOARDS_URL="http://dashboards:5601/dashboards"
-ARG OPENSEARCH_URL="http://opensearch:9200"
-ARG OPENSEARCH_PRIMARY="opensearch-local"
 ARG RESULT_SET_LIMIT="500"
 
 ENV HOME=/malcolm
 ENV APP_HOME="${HOME}"/api
 ENV APP_FOLDER="${APP_HOME}"
 ENV FLASK_APP=project/__init__.py
 ENV FLASK_ENV $FLASK_ENV
-ENV ARKIME_FIELDS_INDEX $ARKIME_FIELDS_INDEX
-ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
-ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
-ENV BEATS_INDEX_PATTERN $BEATS_INDEX_PATTERN
-ENV BEATS_INDEX_TIME_FIELD $BEATS_INDEX_TIME_FIELD
-ENV DASHBOARDS_URL $DASHBOARDS_URL
-ENV OPENSEARCH_URL $OPENSEARCH_URL
-ENV OPENSEARCH_PRIMARY $OPENSEARCH_PRIMARY
 ENV RESULT_SET_LIMIT $RESULT_SET_LIMIT
 
 WORKDIR "${APP_HOME}"

Dockerfiles/arkime.Dockerfile

@@ -106,8 +106,6 @@ ENV TERM xterm
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-ARG OPENSEARCH_URL="http://opensearch:9200"
-ARG OPENSEARCH_PRIMARY="opensearch-local"
 ARG MALCOLM_USERNAME=admin
 ARG ARKIME_ECS_PROVIDER=arkime
 ARG ARKIME_ECS_DATASET=session
@@ -135,8 +133,6 @@ ARG PCAP_NODE_NAME=malcolm
 ARG MAXMIND_GEOIP_DB_LICENSE_KEY=""
 
 # Declare envs vars for each arg
-ENV OPENSEARCH_URL $OPENSEARCH_URL
-ENV OPENSEARCH_PRIMARY $OPENSEARCH_PRIMARY
 ENV ARKIME_INTERFACE $ARKIME_INTERFACE
 ENV MALCOLM_USERNAME $MALCOLM_USERNAME
 # this needs to be present, but is unused as nginx is going to handle auth for us

Dockerfiles/dashboards-helper.Dockerfile

@@ -20,30 +20,18 @@ ENV PUSER_PRIV_DROP true
 
 ENV TERM xterm
 
-ARG ARKIME_INDEX_PATTERN="arkime_sessions3-*"
-ARG ARKIME_INDEX_PATTERN_ID="arkime_sessions3-*"
-ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
 ARG CREATE_OS_ARKIME_SESSION_INDEX="true"
-ARG OPENSEARCH_URL="http://opensearch:9200"
-ARG OPENSEARCH_PRIMARY="opensearch-local"
 ARG ISM_SNAPSHOT_COMPRESSED=false
 ARG ISM_SNAPSHOT_REPO=logs
 ARG OFFLINE_REGION_MAPS_PORT="28991"
 ARG OPENSEARCH_DEFAULT_DASHBOARD="0ad3d7c2-3441-485e-9dfe-dbb22e84e576"
-ARG DASHBOARDS_URL="http://dashboards:5601/dashboards"
 ARG DASHBOARDS_DARKMODE="true"
 
-ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
-ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
-ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
 ENV CREATE_OS_ARKIME_SESSION_INDEX $CREATE_OS_ARKIME_SESSION_INDEX
-ENV OPENSEARCH_URL $OPENSEARCH_URL
-ENV OPENSEARCH_PRIMARY $OPENSEARCH_PRIMARY
 ENV ISM_SNAPSHOT_COMPRESSED $ISM_SNAPSHOT_COMPRESSED
 ENV ISM_SNAPSHOT_REPO $ISM_SNAPSHOT_REPO
 ENV OFFLINE_REGION_MAPS_PORT $OFFLINE_REGION_MAPS_PORT
 ENV OPENSEARCH_DEFAULT_DASHBOARD $OPENSEARCH_DEFAULT_DASHBOARD
-ENV DASHBOARDS_URL $DASHBOARDS_URL
 ENV DASHBOARDS_DARKMODE $DASHBOARDS_DARKMODE
 ENV PATH="/data:${PATH}"
 

Dockerfiles/dashboards.Dockerfile

@@ -22,23 +22,10 @@ ENV TERM xterm
 ENV TINI_VERSION v0.19.0
 ENV OSD_TRANSFORM_VIS_VERSION 2.11.0
 
-ARG OPENSEARCH_URL="http://opensearch:9200"
-ARG OPENSEARCH_PRIMARY="opensearch-local"
-ARG CREATE_OS_ARKIME_SESSION_INDEX="true"
-ARG ARKIME_INDEX_PATTERN="arkime_sessions3-*"
-ARG ARKIME_INDEX_PATTERN_ID="arkime_sessions3-*"
-ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
 ARG NODE_OPTIONS="--max_old_space_size=4096"
+ENV NODE_OPTIONS $NODE_OPTIONS
 
-ENV CREATE_OS_ARKIME_SESSION_INDEX $CREATE_OS_ARKIME_SESSION_INDEX
-ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
-ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
-ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
-ENV OPENSEARCH_DEFAULT_DASHBOARD $OPENSEARCH_DEFAULT_DASHBOARD
 ENV PATH="/data:${PATH}"
-ENV OPENSEARCH_URL $OPENSEARCH_URL
-ENV OPENSEARCH_PRIMARY $OPENSEARCH_PRIMARY
-ENV NODE_OPTIONS $NODE_OPTIONS
 
 USER root
 

Dockerfiles/filebeat.Dockerfile

@@ -44,8 +44,6 @@ ARG FILEBEAT_WATCHER_POLLING_ASSUME_CLOSED_SEC=10
 ARG LOG_CLEANUP_MINUTES=0
 ARG ZIP_CLEANUP_MINUTES=0
 ARG NGINX_LOG_ACCESS_AND_ERRORS=false
-ARG OPENSEARCH_URL="http://opensearch:9200"
-ARG OPENSEARCH_PRIMARY="opensearch-local"
 ARG FILEBEAT_TCP_LISTEN=false
 ARG FILEBEAT_TCP_PORT=5045
 ARG FILEBEAT_TCP_LOG_FORMAT="raw"
@@ -146,8 +144,6 @@ ENV FILEBEAT_ZEEK_LOG_LIVE_PATH $FILEBEAT_ZEEK_LOG_LIVE_PATH
 ENV FILEBEAT_SURICATA_LOG_PATH $FILEBEAT_SURICATA_LOG_PATH
 ENV FILEBEAT_NGINX_LOG_PATH $FILEBEAT_NGINX_LOG_PATH
 ENV NGINX_LOG_ACCESS_AND_ERRORS $NGINX_LOG_ACCESS_AND_ERRORS
-ENV OPENSEARCH_URL $OPENSEARCH_URL
-ENV OPENSEARCH_PRIMARY $OPENSEARCH_PRIMARY
 ENV FILEBEAT_TCP_LISTEN $FILEBEAT_TCP_LISTEN
 ENV FILEBEAT_TCP_PORT $FILEBEAT_TCP_PORT
 ENV FILEBEAT_TCP_LOG_FORMAT $FILEBEAT_TCP_LOG_FORMAT

Dockerfiles/opensearch.Dockerfile

@@ -24,12 +24,6 @@ ENV TERM xterm
 
 ENV TINI_VERSION v0.19.0
 
-ARG OPENSEARCH_PRIMARY="opensearch-local"
-ENV OPENSEARCH_PRIMARY $OPENSEARCH_PRIMARY
-
-ARG MALCOLM_API_URL="http://api:5000/mapi/event"
-ENV MALCOLM_API_URL $MALCOLM_API_URL
-
 ARG DISABLE_INSTALL_DEMO_CONFIG=true
 ARG DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true
 ENV DISABLE_INSTALL_DEMO_CONFIG $DISABLE_INSTALL_DEMO_CONFIG

Dockerfiles/pcap-monitor.Dockerfile

@@ -26,8 +26,6 @@ ENV TERM xterm
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-ARG OPENSEARCH_URL="http://opensearch:9200"
-ARG OPENSEARCH_PRIMARY="opensearch-local"
 ARG PCAP_PATH=/pcap
 ARG PCAP_PIPELINE_VERBOSITY=""
 ARG PCAP_PIPELINE_IGNORE_PREEXISTING=false
@@ -36,8 +34,6 @@ ARG PCAP_PIPELINE_POLLING_ASSUME_CLOSED_SEC=10
 ARG PCAP_NODE_NAME=malcolm
 ARG ZEEK_PATH=/zeek
 
-ENV OPENSEARCH_URL $OPENSEARCH_URL
-ENV OPENSEARCH_PRIMARY $OPENSEARCH_PRIMARY
 ENV PCAP_PATH $PCAP_PATH
 ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY
 ENV PCAP_PIPELINE_IGNORE_PREEXISTING $PCAP_PIPELINE_IGNORE_PREEXISTING

api/project/__init__.py

@@ -381,16 +381,24 @@ def index_from_args(args):
     ----------
     args : dict
         The dictionary which should contain 'doctype' value. Missing
-        key returns value of app.config["ARKIME_INDEX_PATTERN"]
+        key returns value of app.config["MALCOLM_NETWORK_INDEX_PATTERN"]
 
     Returns
     -------
     return index
-        app.config["ARKIME_INDEX_PATTERN"] or app.config["BEATS_INDEX_PATTERN"]
+        app.config["MALCOLM_OTHER_INDEX_PATTERN"],
+        app.config["ARKIME_NETWORK_INDEX_PATTERN"],
+        app.config["MALCOLM_NETWORK_INDEX_PATTERN"],
     """
-    return (
-        app.config["BEATS_INDEX_PATTERN"] if doctype_from_args(args) == 'host' else app.config["ARKIME_INDEX_PATTERN"]
-    )
+    index = None
+    if dtype := str(doctype_from_args(args)).lower():
+        if dtype.startswith('host') or dtype.startswith('beat') or dtype.startswith('miscbeat'):
+            index = app.config["MALCOLM_OTHER_INDEX_PATTERN"]
+        elif dtype.startswith('arkime') or dtype.startswith('session'):
+            index = app.config["ARKIME_NETWORK_INDEX_PATTERN"]
+        else:
+            index = app.config["MALCOLM_NETWORK_INDEX_PATTERN"]
+    return index
 
 
 def timefield_from_args(args):
@@ -401,18 +409,24 @@ def timefield_from_args(args):
     ----------
     args : dict
         The dictionary which should contain 'doctype' value. Missing
-        key returns value of app.config["ARKIME_INDEX_TIME_FIELD"]
+        key returns value of app.config["MALCOLM_NETWORK_INDEX_PATTERN"]
 
     Returns
     -------
-    return index
-        app.config["ARKIME_INDEX_TIME_FIELD"] or app.config["BEATS_INDEX_TIME_FIELD"]
+    timefield index
+        app.config["MALCOLM_OTHER_INDEX_TIME_FIELD"],
+        app.config["ARKIME_NETWORK_INDEX_TIME_FIELD"],
+        app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"],
     """
-    return (
-        app.config["BEATS_INDEX_TIME_FIELD"]
-        if doctype_from_args(args) == 'host'
-        else app.config["ARKIME_INDEX_TIME_FIELD"]
-    )
+    timefield = None
+    if dtype := str(doctype_from_args(args)).lower():
+        if dtype.startswith('host') or dtype.startswith('beat') or dtype.startswith('miscbeat'):
+            timefield = app.config["MALCOLM_OTHER_INDEX_TIME_FIELD"]
+        elif dtype.startswith('arkime') or dtype.startswith('session'):
+            timefield = app.config["ARKIME_NETWORK_INDEX_TIME_FIELD"]
+        else:
+            timefield = app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]
+    return timefield
 
 
 def filtertime(search, args, default_from="1 day ago", default_to="now"):
@@ -860,8 +874,7 @@ def ping():
 )
 def event():
     """Webhook that accepts alert data (like that from the OpenSearch Alerting API) to be
-    reindexed into OpenSearch as session records (e.g., arkime_sessions3-*) for viewing
-    in Malcolm's default visualizations.
+    reindexed into OpenSearch as session records for viewing in Malcolm's default visualizations.
 
     See Malcolm's malcolm_api_loopback_monitor.json and malcolm_api_loopback_destination.json
     for formatting template examples.
@@ -921,7 +934,7 @@ def event():
     data = get_request_arguments(request)
     nowTimeStr = datetime.now().astimezone(pytz.utc).isoformat().replace('+00:00', 'Z')
     if 'alert' in data:
-        alert['@timestamp'] = deep_get(
+        alert[app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]] = deep_get(
             data,
             [
                 'alert',
@@ -930,7 +943,7 @@ def event():
             ],
             nowTimeStr,
         )
-        alert['firstPacket'] = alert['@timestamp']
+        alert['firstPacket'] = alert[app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]]
         alert['lastPacket'] = deep_get(
             data,
             [
@@ -1012,9 +1025,9 @@ def event():
                 if hitCount := deep_get(alertResults[0], ['hits', 'total', 'value'], 0):
                     alert['event']['hits'] = hitCount
 
-        docDateStr = dateparser.parse(alert['@timestamp']).strftime('%y%m%d')
+        docDateStr = dateparser.parse(alert[app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]]).strftime('%y%m%d')
         idxResponse = databaseClient.index(
-            index=f"{app.config['ARKIME_INDEX_PATTERN'].rstrip('*')}{docDateStr}",
+            index=f"{app.config['MALCOLM_NETWORK_INDEX_PATTERN'].rstrip('*')}{docDateStr}",
             id=f"{docDateStr}-{alert['event']['id']}",
             body=alert,
         )

api/project/config.py

@@ -6,10 +6,13 @@
 
 class Config(object):
     ARKIME_FIELDS_INDEX = f"{os.getenv('ARKIME_FIELDS_INDEX', 'arkime_fields')}"
-    ARKIME_INDEX_PATTERN = f"{os.getenv('ARKIME_INDEX_PATTERN', 'arkime_sessions3-*')}"
-    ARKIME_INDEX_TIME_FIELD = f"{os.getenv('ARKIME_INDEX_TIME_FIELD', 'firstPacket')}"
-    BEATS_INDEX_PATTERN = f"{os.getenv('BEATS_INDEX_PATTERN', 'malcolm_beats_*')}"
-    BEATS_INDEX_TIME_FIELD = f"{os.getenv('BEATS_INDEX_TIME_FIELD', '@timestamp')}"
+    MALCOLM_NETWORK_INDEX_PATTERN = f"{os.getenv('MALCOLM_NETWORK_INDEX_PATTERN', 'arkime_sessions3-*')}"
+    MALCOLM_NETWORK_INDEX_TIME_FIELD = f"{os.getenv('MALCOLM_NETWORK_INDEX_TIME_FIELD', 'firstPacket')}"
+    MALCOLM_OTHER_INDEX_PATTERN = f"{os.getenv('MALCOLM_OTHER_INDEX_PATTERN', 'malcolm_beats_*')}"
+    MALCOLM_OTHER_INDEX_TIME_FIELD = f"{os.getenv('MALCOLM_OTHER_INDEX_TIME_FIELD', '@timestamp')}"
+    ARKIME_NETWORK_INDEX_PATTERN = f"{os.getenv('ARKIME_NETWORK_INDEX_PATTERN', 'arkime_sessions3-*')}"
+    ARKIME_NETWORK_INDEX_TIME_FIELD = f"{os.getenv('ARKIME_NETWORK_INDEX_TIME_FIELD', 'firstPacket')}"
+
     DOCTYPE_DEFAULT = f"{os.getenv('DOCTYPE_DEFAULT', 'network')}"
     BUILD_DATE = f"{os.getenv('BUILD_DATE', 'unknown')}"
     DASHBOARDS_URL = f"{os.getenv('DASHBOARDS_URL', 'http://dashboards:5601/dashboards')}"

config/dashboards-helper.env.example

@@ -1,8 +1,7 @@
 # Whether or not to set OpenSearch Dashboards to dark mode
 DASHBOARDS_DARKMODE=true
 # The maximum cumulative size of OpenSearch indices containing network traffic metadata
-#   (arkime_sessions3-*) before which the oldest indices will be deleted ('' to disable
-#   storage-based index pruning).
+#   before which the oldest indices will be deleted ('' to disable storage-based index pruning).
 OPENSEARCH_INDEX_SIZE_PRUNE_LIMIT=0
 # Whether to determine the "oldest" indices for storage-based index pruning by creation
 #   date/time ('true') or index name ('false')

config/opensearch.env.example

@@ -41,6 +41,15 @@ OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION=false
 # OpenSearch memory allowance and other Java options
 OPENSEARCH_JAVA_OPTS=-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true
 
+# OpenSearch index patterns and timestamp fields
+MALCOLM_NETWORK_INDEX_PATTERN=arkime_sessions3-*
+MALCOLM_NETWORK_INDEX_TIME_FIELD=firstPacket
+MALCOLM_OTHER_INDEX_PATTERN=malcolm_beats_*
+MALCOLM_OTHER_INDEX_TIME_FIELD=@timestamp
+ARKIME_NETWORK_INDEX_PATTERN=arkime_sessions3-*
+ARKIME_NETWORK_INDEX_TIME_FIELD=firstPacket
+
+# Miscellaneous
 logger.level=WARN
 bootstrap.memory_lock=true
 MAX_LOCKED_MEMORY=unlimited

dashboards/alerting/monitors/malcolm_api_loopback_monitor.json

@@ -13,7 +13,7 @@
     {
       "search": {
         "indices": [
-          "arkime_sessions3-*"
+          "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
         ],
         "query": {
           "size": 0,
@@ -22,7 +22,7 @@
               "filter": [
                 {
                   "range": {
-                    "firstPacket": {
+                    "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER": {
                       "from": "{{period_end}}||-10m",
                       "to": "{{period_end}}",
                       "include_lower": true,

dashboards/anomaly_detectors/action_result_user.json

@@ -1,9 +1,9 @@
 {
   "name": "action_result_user",
   "description": "Detect anomalies in action (event.action), result (event.result) and user (related.user) within application protocols (network.protocol)",
-  "time_field": "firstPacket",
+  "time_field": "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER",
   "indices": [
-    "arkime_sessions3-*"
+    "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
   ],
   "category_field": [
     "network.protocol"

dashboards/anomaly_detectors/file_mime_type.json

@@ -1,9 +1,9 @@
 {
   "name": "file_mime_type",
   "description": "Detect anomalies based on transferred file type (file.mime_type)",
-  "time_field": "firstPacket",
+  "time_field": "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER",
   "indices": [
-    "arkime_sessions3-*"
+    "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
   ],
   "feature_attributes": [
     {

dashboards/anomaly_detectors/malcolm_init_dummy.json

@@ -1,9 +1,9 @@
 {
   "name": "malcolm_init_dummy",
   "description": "A dummy detector to force opensearch anomaly detection index creation",
-  "time_field": "firstPacket",
+  "time_field": "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER",
   "indices": [
-    "arkime_sessions3-*"
+    "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
   ],
   "feature_attributes": [
     {

dashboards/anomaly_detectors/network_protocol.json

@@ -1,9 +1,9 @@
 {
   "name": "network_protocol",
   "description": "Detect anomalies based on application protocol (network.protocol)",
-  "time_field": "firstPacket",
+  "time_field": "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER",
   "indices": [
-    "arkime_sessions3-*"
+    "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
   ],
   "feature_attributes": [
     {

dashboards/anomaly_detectors/total_bytes.json

@@ -1,9 +1,9 @@
 {
   "name": "total_bytes",
   "description": "Detect anomalies based on traffic size (sum of network.bytes)",
-  "time_field": "firstPacket",
+  "time_field": "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER",
   "indices": [
-    "arkime_sessions3-*"
+    "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER"
   ],
   "feature_attributes": [
     {