Work in progress of idaholab#281, arkime-live container; done in arki…

…me-capture branch
mmguero-dev · Nov 28, 2023 · 3fdfd98 · 3fdfd98
1 parent a74ad8b
commit 3fdfd98
Show file tree

Hide file tree

Showing 3 changed files with 123 additions and 67 deletions.
diff --git a/arkime/scripts/initarkime.sh b/arkime/scripts/initarkime.sh
@@ -3,7 +3,6 @@
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 
 MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"}
-ARKIME_LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-"false"}
 OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"}
 OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"}
 OPENSEARCH_SSL_CERTIFICATE_VERIFICATION=${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"}
@@ -36,81 +35,76 @@ fi
 
 if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then
 
-  echo "Giving $OPENSEARCH_PRIMARY time to start..."
-  /opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!"
-
   # download and/or update geo updates
   $ARKIME_DIR/bin/arkime_update_geo.sh
 
-  # don't do database initialization or run wise in arkime-live mode
-  if [[ "$ARKIME_LIVE_CAPTURE" == "false" ]]; then
-
-    # start and wait patiently for WISE
-    if [[ "$WISE" = "on" ]] ; then
-      touch /var/run/arkime/runwise
-      echo "Giving WISE time to start..."
-      sleep 5
-      until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1"
-      do
-          echo "Waiting for WISE to start"
-          sleep 1
-      done
-      echo "WISE is running!"
-      echo
-    fi
+  echo "Giving $OPENSEARCH_PRIMARY time to start..."
+  /opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!"
 
-    # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created)
-    if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 ]]; then
+  # start and wait patiently for WISE
+  if [[ "$WISE" = "on" ]] ; then
+    touch /var/run/arkime/runwise
+    echo "Giving WISE time to start..."
+    sleep 5
+    until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1"
+    do
+        echo "Waiting for WISE to start"
+        sleep 1
+    done
+    echo "WISE is running!"
+    echo
+  fi
 
-      echo "Initializing $OPENSEARCH_PRIMARY database..."
+  # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created)
+  if (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 )); then
 
-    	$ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt
+    echo "Initializing $OPENSEARCH_PRIMARY database..."
 
-      echo "Creating default user..."
+  	$ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt
 
-    	# this password isn't going to be used by Arkime, nginx will do the auth instead
-    	$ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG
+    echo "Creating default user..."
 
-      echo "Initializing fields..."
+  	# this password isn't going to be used by Arkime, nginx will do the auth instead
+  	$ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG
 
-      # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S.
-      touch /tmp/not_a_packet.pcap
-      $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1
-      rm -f /tmp/not_a_packet.pcap
+    echo "Initializing fields..."
 
-      echo "Initializing views..."
+    # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S.
+    touch /tmp/not_a_packet.pcap
+    $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1
+    rm -f /tmp/not_a_packet.pcap
 
-      for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do
-        TEMP_JSON=$(mktemp --suffix=.json)
-        RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')"
-        echo "Creating view $(jq '.name' < "${VIEW_FILE}")"
-        jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}"
-        curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}"
-        rm -f "${TEMP_JSON}"
-      done
+    echo "Initializing views..."
 
-      echo "Setting defaults..."
+    for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do
+      TEMP_JSON=$(mktemp --suffix=.json)
+      RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')"
+      echo "Creating view $(jq '.name' < "${VIEW_FILE}")"
+      jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}"
+      curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}"
+      rm -f "${TEMP_JSON}"
+    done
 
-      curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json"
+    echo "Setting defaults..."
 
-      echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n"
+    curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json"
 
-    else
-      echo "$OPENSEARCH_PRIMARY database previously initialized!"
-      echo
+    echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n"
 
-      $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded
-      echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!"
+  else
+    echo "$OPENSEARCH_PRIMARY database previously initialized!"
+    echo
 
-    fi # if/else OpenSearch database initialized
+    $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded
+    echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!"
 
-    # increase OpenSearch max shards per node from default if desired
-    if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then
-      # see https://github.com/elastic/elasticsearch/issues/40803
-      curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }"
-    fi
+  fi # if/else OpenSearch database initialized
 
-  fi # "$ARKIME_LIVE_CAPTURE" == "false"
+  # increase OpenSearch max shards per node from default if desired
+  if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then
+    # see https://github.com/elastic/elasticsearch/issues/40803
+    curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }"
+  fi
 
   # before running viewer, call _refresh to make sure everything is available for search first
   curl "${CURL_CONFIG_PARAMS[@]}" -sS -XPOST "${OPENSEARCH_URL}/_refresh"

diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
+
+ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"}
+CERT_FILE="${ARKIME_DIR}"/etc/viewer.crt
+KEY_FILE="${ARKIME_DIR}"/etc/viewer.key
+ARKIME_PACKET_THREADS=${ARKIME_PACKET_THREADS:-1}
+PUSER=${PUSER:-"arkime"}
+PGROUP=${PGROUP:-"arkime"}
+ARKIME_LIVE_NODE_NAME=${ARKIME_LIVE_NODE_NAME:-"malcolm"}
+ARKIME_LIVE_NODE_HOST=${ARKIME_LIVE_NODE_HOST:-""}
+
+OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"}
+OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"}
+OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"}
+OPENSEARCH_SSL_CERTIFICATE_VERIFICATION=${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"}
+OPENSEARCH_CREDS_CONFIG_FILE=${OPENSEARCH_CREDS_CONFIG_FILE:-"/var/local/curlrc/.opensearch.primary.curlrc"}
+if ( [[ "$OPENSEARCH_PRIMARY" == "opensearch-remote" ]] || [[ "$OPENSEARCH_PRIMARY" == "elasticsearch-remote" ]] ) && [[ -r "$OPENSEARCH_CREDS_CONFIG_FILE" ]]; then
+  CURL_CONFIG_PARAMS=(
+    --config
+    "$OPENSEARCH_CREDS_CONFIG_FILE"
+    )
+else
+  CURL_CONFIG_PARAMS=()
+fi
+
+rm -f /var/run/arkime/initialized /var/run/arkime/runwise
+
+# make sure TLS certificates exist prior to starting up
+CERT_FILE=$ARKIME_DIR/etc/viewer.crt
+KEY_FILE=$ARKIME_DIR/etc/viewer.key
+if ( [[ ! -f "$CERT_FILE" ]] || [[ ! -f "$KEY_FILE" ]] ) && [[ -x /usr/local/bin/self_signed_key_gen.sh ]]; then
+  rm -f "$CERT_FILE" "$KEY_FILE" ./newcerts
+  pushd $ARKIME_DIR/etc/ >/dev/null 2>&1
+  /usr/local/bin/self_signed_key_gen.sh -n -o ./newcerts >/dev/null 2>&1
+  mv ./newcerts/server.crt "$CERT_FILE"
+  mv ./newcerts/server.key "$KEY_FILE"
+  rm -rf ./newcerts
+  popd >/dev/null 2>&1
+fi
+
+# download and/or update geo updates
+$ARKIME_DIR/bin/arkime_update_geo.sh
+
+# wait patiently for the non-live Arkime to initialize the database
+
+echo "Giving $OPENSEARCH_PRIMARY time to start..."
+/opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!"
+echo "Giving Arkime time to initialize..."
+sleep 5
+until (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) >= 1 )); do
+    sleep 1
+done
+
+# this will also allow viewer to kick off
+touch /var/run/arkime/initialized
+
+echo "Arkime is initialized!"
+echo
+
+"${ARKIME_DIR}"/bin/capture --insecure \
+  -c "${ARKIME_DIR}"/etc/config.ini \
+  -o pcapDir=/data/pcap/processed \
+  -o packetThreads=${ARKIME_PACKET_THREADS} \
+  -o dropUser=${PUSER} \
+  -o dropGroup=${PGROUP} \
+  -o ecsEventProvider=arkime \
+  -o ecsEventDataset=session \
+  --node "${ARKIME_LIVE_NODE_NAME}" \
+  --host "${ARKIME_LIVE_NODE_HOST}" | tee -a "${ARKIME_DIR}"/logs/capture.log 2>&1
diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf
@@ -21,7 +21,7 @@ serverurl=unix:///tmp/supervisor.sock
 command=/opt/initarkime.sh
 startsecs=0
 startretries=0
-autostart=true
+autostart=%(ENV_ARKIME_PCAP_PROCESSOR)s
 autorestart=false
 stopasgroup=true
 killasgroup=true
@@ -84,16 +84,7 @@ redirect_stderr=true
 user=%(ENV_PUSER)s
 
 [program:live-arkime]
-command="%(ENV_ARKIME_DIR)s"/bin/capture --insecure
-  -c "%(ENV_ARKIME_DIR)s"/etc/config.ini
-  -o pcapDir=/data/pcap/processed
-  -o packetThreads=%(ENV_ARKIME_PACKET_THREADS)s
-  -o dropUser=%(ENV_PUSER)s
-  -o dropGroup=%(ENV_PGROUP)s
-  -o ecsEventProvider=arkime
-  -o ecsEventDataset=session
-  --node "%(ENV_ARKIME_LIVE_NODE_NAME)s"
-  --host "%(ENV_ARKIME_LIVE_NODE_HOST)s"
+command=/opt/live_capture.sh
 autostart=%(ENV_ARKIME_LIVE_CAPTURE)s
 autorestart=%(ENV_ARKIME_LIVE_CAPTURE)s
 startsecs=30