Fixes for creating Views in Arkime v4.0

Dockerfiles/arkime.Dockerfile

@@ -140,6 +140,7 @@ RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.l
       file \
       geoip-bin \
       gettext \
+      jq \
       libcap2-bin \
       libjson-perl \
       libkrb5-3 \

arkime/etc/config.ini

@@ -34,7 +34,7 @@ dropGroup=arkime
 # The userAutoCreateTmpl should more or less match what's in /etc/user_settings.json
 #    which is what's used when creating the default admin user.
 userNameHeader=http_auth_http_user
-userAutoCreateTmpl={"userId": "${this.http_auth_http_user}", "userName": "${this.http_auth_http_user}", "enabled": true, "createEnabled": false, "webEnabled": true, "headerAuthEnabled": true, "emailSearch": true, "removeEnabled": false, "packetSearch": true, "hideStats": false, "hideFiles": false, "hidePcap": false, "disablePcapDownload": false, "settings": { "timezone": "local", "detailFormat": "last", "showTimestamps": "last", "sortColumn": "start", "sortDirection": "desc", "spiGraph": "protocol", "connSrcField": "source.ip", "connDstField": "destination.ip", "numPackets": "last", "theme" : "custom1: #222222,#E2E2E2,#FFFFFF,#00789E,#004A79,#017D73,#092B40,#42b7c5,#2A7580,#ecb30a,#333333,#89ADCC,#6D6D6D,#FFE7E7,#ECFEFF", "manualQuery": false }, "views": { "Public IP Addresses": { "expression": "(country.dst == EXISTS!) || (country.src == EXISTS!) || (ip.dst == EXISTS! && ip.dst != 0.0.0.0/8 && ip.dst != 10.0.0.0/8 && ip.dst != 100.64.0.0/10 && ip.dst != 127.0.0.0/8 && ip.dst != 169.254.0.0/16 && ip.dst != 172.16.0.0/12 && ip.dst != 192.0.0.0/24  && ip.dst != 192.0.2.0/24 && ip.dst != 192.88.99.0/24 && ip.dst != 192.168.0.0/16 && ip.dst != 198.18.0.0/15 && ip.dst != 198.51.100.0/24 && ip.dst != 203.0.113.0/24 && ip.dst != 224.0.0.0/4 && ip.dst != 232.0.0.0/8 && ip.dst != 233.0.0.0/8 && ip.dst != 234.0.0.0/8 && ip.dst != 239.0.0.0/8 && ip.dst != 240.0.0.0/4 && ip.dst != 255.255.255.255 && ip.dst != :: && ip.dst != ::1 && ip.dst != ff00::/8 && ip.dst != fe80::/10 && ip.dst != fc00::/7 && ip.dst != fd00::/8) || (ip.src == EXISTS! && ip.src != 0.0.0.0/8 && ip.src != 10.0.0.0/8 && ip.src != 100.64.0.0/10 && ip.src != 127.0.0.0/8 && ip.src != 169.254.0.0/16 && ip.src != 172.16.0.0/12 && ip.src != 192.0.0.0/24  && ip.src != 192.0.2.0/24 && ip.src != 192.88.99.0/24 && ip.src != 192.168.0.0/16 && ip.src != 198.18.0.0/15 && ip.src != 198.51.100.0/24 && ip.src != 203.0.113.0/24 && ip.src != 224.0.0.0/4 && ip.src != 232.0.0.0/8 && ip.src != 233.0.0.0/8 && ip.src != 234.0.0.0/8 && ip.src != 239.0.0.0/8 && ip.src != 240.0.0.0/4 && ip.src != 255.255.255.255 && ip.src != :: && ip.src != ::1 && ip.src != ff00::/8 && ip.src != fe80::/10 && ip.src != fc00::/7 && ip.src != fd00::/8)" }, "Arkime Sessions": { "expression": "event.provider == arkime" }, "Suricata Logs": { "expression": "event.provider == suricata" }, "Suricata Alerts": { "expression": "event.provider == suricata && event.dataset == alert" }, "Zeek Logs": { "expression": "event.provider == zeek" }, "Zeek conn.log": { "expression": "event.provider == zeek && event.dataset == conn" }, "Zeek Exclude conn.log": { "expression": "event.provider == zeek && event.dataset != conn" } }, "tableStates": { "sessionsNew": { "order": [ [ "firstPacket", "desc" ] ], "visibleHeaders": [ "protocol", "event.provider", "event.dataset", "firstPacket", "lastPacket", "src", "source.port", "dst", "destination.port", "network.packets", "dbby", "tags", "info" ] } } }
+userAutoCreateTmpl={"userId": "${this.http_auth_http_user}", "userName": "${this.http_auth_http_user}", "enabled": true, "createEnabled": false, "webEnabled": true, "headerAuthEnabled": true, "emailSearch": true, "removeEnabled": false, "packetSearch": true, "hideStats": false, "hideFiles": false, "hidePcap": false, "disablePcapDownload": false, "settings": { "timezone": "local", "detailFormat": "last", "showTimestamps": "last", "sortColumn": "firstPacket", "sortDirection": "desc", "spiGraph": "protocol", "connSrcField": "source.ip", "connDstField": "destination.ip", "numPackets": "last", "theme" : "custom1: #222222,#E2E2E2,#FFFFFF,#00789E,#004A79,#017D73,#092B40,#42b7c5,#2A7580,#ecb30a,#333333,#89ADCC,#6D6D6D,#FFE7E7,#ECFEFF", "manualQuery": false }, "tableStates": { "sessionsNew": { "order": [ [ "firstPacket", "desc" ] ], "visibleHeaders": [ "protocol", "event.provider", "event.dataset", "firstPacket", "lastPacket", "src", "source.port", "dst", "destination.port", "network.packets", "dbby", "tags", "info" ] } } }
 parseSMTP=true
 parseSMB=true
 parseQSValue=false

arkime/etc/user_settings.json

@@ -15,7 +15,7 @@
       "timezone": "local",
       "detailFormat": "last",
       "showTimestamps": "last",
-      "sortColumn": "start",
+      "sortColumn": "firstPacket",
       "sortDirection": "desc",
       "spiGraph": "protocol",
       "connSrcField": "source.ip",
@@ -24,29 +24,6 @@
       "theme" : "custom1: #222222,#E2E2E2,#FFFFFF,#00789E,#004A79,#017D73,#092B40,#42b7c5,#2A7580,#ecb30a,#333333,#89ADCC,#6D6D6D,#FFE7E7,#ECFEFF",
       "manualQuery": false
     },
-    "views": {
-      "Public IP Addresses": {
-        "expression": "(country.dst == EXISTS!) || (country.src == EXISTS!) || (ip.dst == EXISTS! && ip.dst != 0.0.0.0/8 && ip.dst != 10.0.0.0/8 && ip.dst != 100.64.0.0/10 && ip.dst != 127.0.0.0/8 && ip.dst != 169.254.0.0/16 && ip.dst != 172.16.0.0/12 && ip.dst != 192.0.0.0/24  && ip.dst != 192.0.2.0/24 && ip.dst != 192.88.99.0/24 && ip.dst != 192.168.0.0/16 && ip.dst != 198.18.0.0/15 && ip.dst != 198.51.100.0/24 && ip.dst != 203.0.113.0/24 && ip.dst != 224.0.0.0/4 && ip.dst != 232.0.0.0/8 && ip.dst != 233.0.0.0/8 && ip.dst != 234.0.0.0/8 && ip.dst != 239.0.0.0/8 && ip.dst != 240.0.0.0/4 && ip.dst != 255.255.255.255 && ip.dst != :: && ip.dst != ::1 && ip.dst != ff00::/8 && ip.dst != fe80::/10 && ip.dst != fc00::/7 && ip.dst != fd00::/8) || (ip.src == EXISTS! && ip.src != 0.0.0.0/8 && ip.src != 10.0.0.0/8 && ip.src != 100.64.0.0/10 && ip.src != 127.0.0.0/8 && ip.src != 169.254.0.0/16 && ip.src != 172.16.0.0/12 && ip.src != 192.0.0.0/24  && ip.src != 192.0.2.0/24 && ip.src != 192.88.99.0/24 && ip.src != 192.168.0.0/16 && ip.src != 198.18.0.0/15 && ip.src != 198.51.100.0/24 && ip.src != 203.0.113.0/24 && ip.src != 224.0.0.0/4 && ip.src != 232.0.0.0/8 && ip.src != 233.0.0.0/8 && ip.src != 234.0.0.0/8 && ip.src != 239.0.0.0/8 && ip.src != 240.0.0.0/4 && ip.src != 255.255.255.255 && ip.src != :: && ip.src != ::1 && ip.src != ff00::/8 && ip.src != fe80::/10 && ip.src != fc00::/7 && ip.src != fd00::/8)"
-      },
-      "Arkime Sessions": {
-        "expression": "event.provider == arkime"
-      },
-      "Suricata Logs": {
-        "expression": "event.provider == suricata"
-      },
-      "Suricata Alerts": {
-        "expression": "event.provider == suricata && event.dataset == alert"
-      },
-      "Zeek Logs": {
-        "expression": "event.provider == zeek"
-      },
-      "Zeek conn.log": {
-        "expression": "event.provider == zeek && event.dataset == conn"
-      },
-      "Zeek Exclude conn.log": {
-        "expression": "event.provider == zeek && event.dataset != conn"
-      }
-    },
     "tableStates": {
       "sessionsNew": {
         "order": [

arkime/etc/views/arkime_sessions.json

@@ -0,0 +1,6 @@
+{
+  "name": "Arkime Sessions",
+  "expression": "event.provider == arkime",
+  "roles": ["arkimeUser"],
+  "users" : [ ]
+}

arkime/etc/views/public_ip_addresses.json

@@ -0,0 +1,6 @@
+{
+  "name": "Public IP Addresses",
+  "expression": "(country.dst == EXISTS!) || (country.src == EXISTS!) || (ip.dst == EXISTS! && ip.dst != 0.0.0.0/8 && ip.dst != 10.0.0.0/8 && ip.dst != 100.64.0.0/10 && ip.dst != 127.0.0.0/8 && ip.dst != 169.254.0.0/16 && ip.dst != 172.16.0.0/12 && ip.dst != 192.0.0.0/24  && ip.dst != 192.0.2.0/24 && ip.dst != 192.88.99.0/24 && ip.dst != 192.168.0.0/16 && ip.dst != 198.18.0.0/15 && ip.dst != 198.51.100.0/24 && ip.dst != 203.0.113.0/24 && ip.dst != 224.0.0.0/4 && ip.dst != 232.0.0.0/8 && ip.dst != 233.0.0.0/8 && ip.dst != 234.0.0.0/8 && ip.dst != 239.0.0.0/8 && ip.dst != 240.0.0.0/4 && ip.dst != 255.255.255.255 && ip.dst != :: && ip.dst != ::1 && ip.dst != ff00::/8 && ip.dst != fe80::/10 && ip.dst != fc00::/7 && ip.dst != fd00::/8) || (ip.src == EXISTS! && ip.src != 0.0.0.0/8 && ip.src != 10.0.0.0/8 && ip.src != 100.64.0.0/10 && ip.src != 127.0.0.0/8 && ip.src != 169.254.0.0/16 && ip.src != 172.16.0.0/12 && ip.src != 192.0.0.0/24  && ip.src != 192.0.2.0/24 && ip.src != 192.88.99.0/24 && ip.src != 192.168.0.0/16 && ip.src != 198.18.0.0/15 && ip.src != 198.51.100.0/24 && ip.src != 203.0.113.0/24 && ip.src != 224.0.0.0/4 && ip.src != 232.0.0.0/8 && ip.src != 233.0.0.0/8 && ip.src != 234.0.0.0/8 && ip.src != 239.0.0.0/8 && ip.src != 240.0.0.0/4 && ip.src != 255.255.255.255 && ip.src != :: && ip.src != ::1 && ip.src != ff00::/8 && ip.src != fe80::/10 && ip.src != fc00::/7 && ip.src != fd00::/8)",
+  "roles": ["arkimeUser"],
+  "users" : [ ]
+}

arkime/etc/views/suricata_alerts.json

@@ -0,0 +1,6 @@
+{
+  "name": "Suricata Alerts",
+  "expression": "event.provider == suricata && event.dataset == alert",
+  "roles": ["arkimeUser"],
+  "users" : [ ]
+}

arkime/etc/views/suricata_logs.json

@@ -0,0 +1,6 @@
+{
+  "name": "Suricata Logs",
+  "expression": "event.provider == suricata",
+  "roles": ["arkimeUser"],
+  "users" : [ ]
+}

arkime/etc/views/zeek_conn.json

@@ -0,0 +1,6 @@
+{
+  "name": "Zeek conn.log",
+  "expression": "event.provider == zeek && event.dataset == conn",
+  "roles": ["arkimeUser"],
+  "users" : [ ]
+}

arkime/etc/views/zeek_exclude_conn.json

@@ -0,0 +1,6 @@
+{
+  "name": "Zeek Exclude conn.log",
+  "expression": "event.provider == zeek && event.dataset != conn",
+  "roles": ["arkimeUser"],
+  "users" : [ ]
+}

arkime/etc/views/zeek_logs.json

@@ -0,0 +1,6 @@
+{
+  "name": "Zeek Logs",
+  "expression": "event.provider == zeek",
+  "roles": ["arkimeUser"],
+  "users" : [ ]
+}

arkime/scripts/initarkime.sh

@@ -58,9 +58,20 @@ if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/j
   $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1
   rm -f /tmp/not_a_packet.pcap
 
+  echo "Initializing views..."
+
+  for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do
+    TEMP_JSON=$(mktemp --suffix=.json)
+    RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')"
+    echo "Creating view $(jq '.name' < "${VIEW_FILE}")"
+    jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}"
+    curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}"
+    rm -f "${TEMP_JSON}"
+  done
+
   echo "Setting defaults..."
 
-  curl "${CURL_CONFIG_PARAMS[@]}" -sS -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json"
+  curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json"
 
   echo -e "\nOpenSearch database initialized!\n"
 
@@ -84,7 +95,7 @@ fi # if/else OpenSearch database initialized
 # increase OpenSearch max shards per node from default if desired
 if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then
   # see https://github.com/elastic/elasticsearch/issues/40803
-  curl "${CURL_CONFIG_PARAMS[@]}" -sS -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }"
+  curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }"
 fi
 
 # before running viewer, call _refresh to make sure everything is available for search first
@@ -93,3 +104,4 @@ curl "${CURL_CONFIG_PARAMS[@]}" -sS -XPOST "${OPENSEARCH_URL}/_refresh"
 touch /var/run/arkime/initialized
 
 # the (viewer|wise)_service.sh scripts will start/restart those processes
+