fix idaholab#335, pivot links from Arkime to Kibana in external elast…

…icsearch are not working

Here's a list of what's been fixed/modified:

* `opensearch.env` and `dashboards.env` environment variables are now provided to `nginx-proxy` container so that the decisions for these links can be made based on them
* `envsubst` is used to expand variables into NGINX conf files in `/etc/nginx/conf.d` based on templates found in `/etc/nginx/templates`
* some custom logic in the `docker_entrpoint.sh` script for NGINX to do the environment variable substitution and some variable massaging prior to starting up NGINX
* two new environment variables in `dashboards.env.example` (blank by default but they could be manually overridden if there's more going on than we can automatically figure out based on the DASHBOARDS_URL variable:
```
NGINX_DASHBOARDS_PREFIX=
NGINX_DASHBOARDS_PROXY_PASS=
```
* not 100% related, but the landing page now shows "Kibana" with an elastic icon rather than "Dashboards" if we're in elasticsearch/kibana mode
* fixed issue with arkime to dashboards links not using correct index and time field names if those are not the defaults (see idaholab#313)
* removed some stuff left over from base image that wasn't ever really used

Dockerfiles/nginx.Dockerfile

@@ -4,7 +4,6 @@
 # thanks to:  nginx                       -  https://github.com/nginxinc/docker-nginx/blob/master/mainline/alpine/Dockerfile
 #             kvspb/nginx-auth-ldap       -  https://github.com/kvspb/nginx-auth-ldap
 #             tiredofit/docker-nginx-ldap -  https://github.com/tiredofit/docker-nginx-ldap/blob/master/Dockerfile
-#             jwilder/nginx-proxy         -  https://github.com/jwilder/nginx-proxy/blob/master/Dockerfile.alpine
 
 ####################################################################################
 
@@ -201,7 +200,7 @@ RUN set -x ; \
   make -j$(getconf _NPROCESSORS_ONLN) ; \
   make install ; \
   rm -rf /etc/nginx/html/ ; \
-  mkdir -p /etc/nginx/conf.d/ /etc/nginx/auth/ /usr/share/nginx/html/ ; \
+  mkdir -p /etc/nginx/conf.d/ /etc/nginx/templates/ /etc/nginx/auth/ /usr/share/nginx/html/ ; \
   install -m644 html/50x.html /usr/share/nginx/html/ ; \
   install -m755 objs/nginx-debug /usr/sbin/nginx-debug ; \
   install -m755 objs/ngx_http_xslt_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_xslt_filter_module-debug.so ; \
@@ -227,7 +226,7 @@ RUN set -x ; \
       | xargs -r apk info --installed \
       | sort -u \
   )" ; \
-  apk add --no-cache --virtual .nginx-rundeps $runDeps ca-certificates bash wget openssl apache2-utils openldap shadow stunnel supervisor tini tzdata; \
+  apk add --no-cache --virtual .nginx-rundeps $runDeps ca-certificates bash jq wget openssl apache2-utils openldap shadow stunnel supervisor tini tzdata; \
   update-ca-certificates; \
   apk del .nginx-build-deps ; \
   apk del .gettext ; \
@@ -237,15 +236,13 @@ RUN set -x ; \
   find /usr/share/nginx/html/ -type d -exec chmod 755 "{}" \; && \
   find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \;
 
-COPY --from=jwilder/nginx-proxy:alpine /app/nginx.tmpl /etc/nginx/
-COPY --from=jwilder/nginx-proxy:alpine /etc/nginx/network_internal.conf /etc/nginx/
-COPY --from=jwilder/nginx-proxy:alpine /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/
 COPY --from=docbuild /site/_site /usr/share/nginx/html/readme
 
 ADD nginx/landingpage /usr/share/nginx/html
 COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD nginx/scripts /usr/local/bin/
 ADD nginx/*.conf /etc/nginx/
+ADD nginx/templates /etc/nginx/templates/
 ADD nginx/supervisord.conf /etc/
 COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/assets/favicon.ico
 COPY --chmod=644 docs/images/logo/Malcolm_background.png /usr/share/nginx/html/assets/img/bg-masthead.png

config/dashboards.env.example

@@ -4,3 +4,8 @@
 #   'http://dashboards:5601/dashboards', otherwise specify the Dashboards URL
 #   in the format 'protocol://host:port/uri'.
 DASHBOARDS_URL=http://dashboards:5601/dashboards
+# These values are used to handle the Arkime value actions to pivot from Arkime
+#   to Dashboards. The nginx-proxy container's entrypoint will try to formulate
+#   them automatically, but they may be specified explicitly here.
+NGINX_DASHBOARDS_PREFIX=
+NGINX_DASHBOARDS_PROXY_PASS=

docker-compose-standalone.yml

@@ -700,6 +700,8 @@ services:
     env_file:
       - ./config/process.env
       - ./config/ssl.env
+      - ./config/opensearch.env
+      - ./config/dashboards.env
       - ./config/auth-common.env
       - ./config/nginx.env
     depends_on:

docker-compose.yml

@@ -780,6 +780,8 @@ services:
     env_file:
       - ./config/process.env
       - ./config/ssl.env
+      - ./config/opensearch.env
+      - ./config/dashboards.env
       - ./config/auth-common.env
       - ./config/nginx.env
     depends_on:

kubernetes/98-nginx-proxy.yml

@@ -58,6 +58,10 @@ spec:
               name: process-env
           - configMapRef:
               name: ssl-env
+          - configMapRef:
+              name: opensearch-env
+          - configMapRef:
+              name: dashboards-env
           - configMapRef:
               name: auth-common-env
           - configMapRef:

nginx/landingpage/index.html

@@ -40,8 +40,8 @@ <h1 class="mb-5"/>
                 <div class="row">
                     <div class="col-lg-3">
                         <div class="features-icons-item mx-auto mb-0 mb-lg-3">
-                            <a href="/dashboards/"><img style='max-width: 35%; height: auto; object-fit: contain' src="assets/img/opensearch_mark_default.svg"/>
-                            <h3>Dashboards</h3></a>
+                            <a href="MALCOLM_DASHBOARDS_URL_REPLACER"><img style='max-width: 35%; height: auto; object-fit: contain' src="assets/img/MALCOLM_DASHBOARDS_ICON_REPLACER"/>
+                            <h3>MALCOLM_DASHBOARDS_NAME_REPLACER</h3></a>
                             <p class="lead mb-0">Visualize traffic or track down security concerns with dozens of <a href="/readme/docs/dashboards.html#PrebuiltVisualizations">pre-built dashboards</a>, or <a href="/readme/docs/dashboards.html#BuildDashboard">create your own</a></p>
                         </div>
                     </div>

nginx/nginx.conf

@@ -8,6 +8,7 @@ events { worker_connections 1024; }
 
 http {
 
+  include /etc/nginx/conf.d/*.conf;
   include /etc/nginx/mime.types;
   sendfile on;
 
@@ -139,7 +140,7 @@ http {
     }
 
     # Arkime -> Dashboards shortcut
-    location ~* ^/idark2dash(.*) {
+    location ~* /idark2dash(.*) {
       include /etc/nginx/nginx_auth_rt.conf;
       set $filter_start_time now-1d;
       if ($arg_start != '') {
@@ -161,11 +162,7 @@ http {
         set $filter_value $arg_value;
       }
 
-      # TODO: index and time field could be specified by environment variables
-      rewrite ^/idark2dash/(.*) /dashboards/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'arkime_sessions3-*',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'arkime_sessions3-*',interval:auto,query:(language:lucene,query:''),sort:!(firstPacket,desc)) redirect;
-      proxy_pass http://dashboards;
-      proxy_redirect off;
-      proxy_set_header Host dashboards.malcolm.local;
+      include /etc/nginx/nginx_idark2dash_rewrite_rt.conf;
     }
 
     # Dashboards -> Arkime shortcut

nginx/nginx_idark2dash_rewrite_dashboards.conf

@@ -0,0 +1,4 @@
+rewrite ^.*/idark2dash/(.*) $dashboards_prefix/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'$sessions_index',interval:auto,query:(language:lucene,query:''),sort:!($time_field,desc)) redirect;
+proxy_pass $dashboards_proxy_pass;
+proxy_redirect off;
+proxy_set_header Host dashboards.malcolm.local;

nginx/nginx_idark2dash_rewrite_kibana.conf

@@ -0,0 +1 @@
+rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'$sessions_index',interval:auto,query:(language:lucene,query:''),sort:!($time_field,desc)) redirect;

nginx/nginx_readonly.conf

@@ -8,6 +8,7 @@ events { worker_connections 1024; }
 
 http {
 
+  include /etc/nginx/conf.d/*.conf;
   include /etc/nginx/mime.types;
   sendfile on;
 
@@ -73,7 +74,7 @@ http {
     include /etc/nginx/nginx_auth_rt.conf;
 
     # Arkime -> Dashboards shortcut
-    location ~* ^/idark2dash(.*) {
+    location ~* /idark2dash(.*) {
 
       set $filter_start_time now-1d;
       if ($arg_start != '') {
@@ -95,11 +96,7 @@ http {
         set $filter_value $arg_value;
       }
 
-      # TODO: index and time field could be specified by environment variables
-      rewrite ^/idark2dash/(.*) /dashboards/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'arkime_sessions3-*',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'arkime_sessions3-*',interval:auto,query:(language:lucene,query:''),sort:!(firstPacket,desc)) redirect;
-      proxy_pass http://dashboards;
-      proxy_redirect off;
-      proxy_set_header Host dashboards.malcolm.local;
+      include /etc/nginx/nginx_idark2dash_rewrite_rt.conf;
     }
 
     # Dashboards -> Arkime shortcut

nginx/scripts/docker_entrypoint.sh

@@ -1,33 +1,11 @@
 #!/bin/bash
 set -e
 
-# Warn if the DOCKER_HOST socket does not exist
-if [[ $DOCKER_HOST = unix://* ]]; then
-  socket_file=${DOCKER_HOST#unix://}
-  if ! [ -S $socket_file ]; then
-    cat >&2 <<-EOT
-  ERROR: you need to share your Docker host socket with a volume at $socket_file
-  Typically you should run your container with: \`-v /var/run/docker.sock:$socket_file:ro\`
-  See the jwilder/nginx-proxy documentation at http://git.io/vZaGJ
-EOT
-    socketMissing=1
-  fi
-fi
-
-# Compute the DNS resolvers for use in the templates - if the IP contains ":", it's IPv6 and must be enclosed in []
-export RESOLVERS=$(awk '$1 == "nameserver" {print ($2 ~ ":")? "["$2"]": $2}' ORS=' ' /etc/resolv.conf | sed 's/ *$//g')
-if [ "x$RESOLVERS" = "x" ]; then
-    echo "Warning: unable to determine DNS resolvers for nginx" >&2
-    unset RESOLVERS
-fi
-
-# If the user has run the default command and the socket doesn't exist, fail
-if [ "$socketMissing" = 1 -a "$1" = 'supervisord' -a "$2" = '-c' -a "$3" = '/etc/supervisord.conf' ]; then
-  exit 1
-fi
-
 NGINX_LANDING_INDEX_HTML=/usr/share/nginx/html/index.html
 
+NGINX_TEMPLATES_DIR=/etc/nginx/templates
+NGINX_CONFD_DIR=/etc/nginx/conf.d
+
 # set up for HTTPS/HTTP and NGINX HTTP basic vs. LDAP/LDAPS/LDAP+StartTLS auth
 
 # "include" file that sets 'ssl on' and indicates the locations of the PEM files
@@ -58,6 +36,11 @@ NGINX_RUNTIME_AUTH_CONF=/etc/nginx/nginx_auth_rt.conf
 # runtime "include" file for ldap config (link to either NGINX_BLANK_CONF or (possibly modified) NGINX_LDAP_USER_CONF)
 NGINX_RUNTIME_LDAP_CONF=/etc/nginx/nginx_ldap_rt.conf
 
+# "include" files for idark2dash rewrite using opensearch dashboards, kibana, and runtime copy, respectively
+NGINX_DASHBOARDS_IDARK2DASH_REWRITE_CONF=/etc/nginx/nginx_idark2dash_rewrite_dashboards.conf
+NGINX_KIBANA_IDARK2DASH_REWRITE_CONF=/etc/nginx/nginx_idark2dash_rewrite_kibana.conf
+NGINX_RUNTIME_IDARK2DASH_REWRITE_CONF=/etc/nginx/nginx_idark2dash_rewrite_rt.conf
+
 # config file for stunnel if using stunnel to issue LDAP StartTLS function
 STUNNEL_CONF=/etc/stunnel/stunnel.conf
 
@@ -239,15 +222,83 @@ EOF
 
 fi # basic vs. ldap
 
+# if the runtime htpasswd file doesn't exist but the "preseed" does, copy the preseed over for runtime
 if [[ ! -f /etc/nginx/auth/htpasswd ]] && [[ -f /tmp/auth/default/htpasswd ]]; then
   cp /tmp/auth/default/htpasswd /etc/nginx/auth/htpasswd
   [[ -n ${PUID} ]] && chown -f ${PUID} /etc/nginx/auth/htpasswd
   [[ -n ${PGID} ]] && chown -f :${PGID} /etc/nginx/auth/htpasswd
   rm -rf /tmp/auth/* || true
 fi
 
-[[ -f "${NGINX_LANDING_INDEX_HTML}" ]] && sed -i "s/MALCOLM_VERSION_REPLACER/v${MALCOLM_VERSION:-unknown} (${VCS_REVISION:-} @ ${BUILD_DATE:-})/g" "${NGINX_LANDING_INDEX_HTML}"
+# do environment variable substitutions from $NGINX_TEMPLATES_DIR to $NGINX_CONFD_DIR
+# NGINX_DASHBOARDS_... are a special case as they have to be crafted a bit based on a few variables
+set +e
+
+[[ "${OPENSEARCH_PRIMARY:-opensearch-local}" == "elasticsearch-remote" ]] && \
+  ln -sf "$NGINX_KIBANA_IDARK2DASH_REWRITE_CONF" "$NGINX_RUNTIME_IDARK2DASH_REWRITE_CONF" || \
+  ln -sf "$NGINX_DASHBOARDS_IDARK2DASH_REWRITE_CONF" "$NGINX_RUNTIME_IDARK2DASH_REWRITE_CONF"
+
+# first parse DASHBOARDS_URL and assign the resultant urlsplit named tuple to an associative array
+#   going to use Python to do so as urllib will do a better job at parsing DASHBOARDS_URL than bash
+DASHBOARDS_URL_PARSED="$( ( /usr/bin/env python3 -c "import sys; import json; from urllib.parse import urlsplit; [ sys.stdout.write(json.dumps(urlsplit(line)._asdict()) + '\n') for line in sys.stdin ]" 2>/dev/null <<< "${DASHBOARDS_URL:-http://dashboards:5601/dashboards}" ) | head -n 1 )"
+declare -A DASHBOARDS_URL_DICT
+for KEY in $(jq -r 'keys[]' 2>/dev/null <<< $DASHBOARDS_URL_PARSED); do
+  DASHBOARDS_URL_DICT["$KEY"]=$(jq -r ".$KEY" 2>/dev/null <<< $DASHBOARDS_URL_PARSED)
+done
+
+# the "path" from the parsed URL is the dashboards prefix
+[[ -z "${NGINX_DASHBOARDS_PREFIX:-}" ]] && \
+  [[ -v DASHBOARDS_URL_DICT[path] ]] && \
+  NGINX_DASHBOARDS_PREFIX="${DASHBOARDS_URL_DICT[path]}"
+# if we failed to get it, use the default
+[[ -z "${NGINX_DASHBOARDS_PREFIX:-}" ]] && \
+  [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" != "elasticsearch-remote" ]] && \
+  NGINX_DASHBOARDS_PREFIX=/dashboards
+
+# the "path" from the parsed URL is the dashboards prefix
+if [[ -z "${NGINX_DASHBOARDS_PROXY_PASS:-}" ]]; then
+  # if Malcolm is running in anything other than "elasticsearch-remote" mode, then
+  #   the dashboards service is already defined in the upstream
+  if [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" == "elasticsearch-remote" ]] && [[ -v DASHBOARDS_URL_DICT[scheme] ]] && [[ -v DASHBOARDS_URL_DICT[netloc] ]]; then
+    NGINX_DASHBOARDS_PROXY_PASS="${DASHBOARDS_URL_DICT[scheme]}://${DASHBOARDS_URL_DICT[netloc]}"
+  else
+    NGINX_DASHBOARDS_PROXY_PASS=http://dashboards
+  fi
+fi
+# if we failed to get it, use the default
+[[ -z "${NGINX_DASHBOARDS_PROXY_PASS:-}" ]] && \
+  [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" != "elasticsearch-remote" ]] && \
+  NGINX_DASHBOARDS_PROXY_PASS=http://dashboards
+
+export NGINX_DASHBOARDS_PREFIX
+export NGINX_DASHBOARDS_PROXY_PASS
+export NGINX_DASHBOARDS_PROXY_URL="$(echo "$(echo "$NGINX_DASHBOARDS_PROXY_PASS" | sed 's@/$@@')/$(echo "$NGINX_DASHBOARDS_PREFIX" | sed 's@^/@@')" | sed 's@/$@@')"
+
+# now process the environment variable substitutions
+for TEMPLATE in "$NGINX_TEMPLATES_DIR"/*.conf.template; do
+  DOLLAR=$ envsubst < "$TEMPLATE" > "$NGINX_CONFD_DIR/$(basename "$TEMPLATE"| sed 's/\.template$//')"
+done
+
+set -e
+
+# insert some build and runtime information into the landing page
+if [[ -f "${NGINX_LANDING_INDEX_HTML}" ]]; then
+  if [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" == "elasticsearch-remote" ]]; then
+    MALCOLM_DASHBOARDS_NAME=Kibana
+    MALCOLM_DASHBOARDS_URL="$NGINX_DASHBOARDS_PROXY_URL"
+    MALCOLM_DASHBOARDS_ICON=elastic.svg
+  else
+    MALCOLM_DASHBOARDS_NAME=Dashboards
+    MALCOLM_DASHBOARDS_URL="$(echo "$NGINX_DASHBOARDS_PREFIX" | sed 's@/$@@')/"
+    MALCOLM_DASHBOARDS_ICON=opensearch_mark_default.svg
+  fi
+  sed -i "s@MALCOLM_DASHBOARDS_NAME_REPLACER@${MALCOLM_DASHBOARDS_NAME}@g" "${NGINX_LANDING_INDEX_HTML}"
+  sed -i "s@MALCOLM_DASHBOARDS_URL_REPLACER@${MALCOLM_DASHBOARDS_URL}@g" "${NGINX_LANDING_INDEX_HTML}"
+  sed -i "s@MALCOLM_DASHBOARDS_ICON_REPLACER@${MALCOLM_DASHBOARDS_ICON}@g" "${NGINX_LANDING_INDEX_HTML}"
+  sed -i "s/MALCOLM_VERSION_REPLACER/v${MALCOLM_VERSION:-unknown} (${VCS_REVISION:-} @ ${BUILD_DATE:-})/g" "${NGINX_LANDING_INDEX_HTML}"
+fi
 
+# some cleanup, if necessary
 rm -rf /var/log/nginx/* || true
 
 # start supervisor (which will spawn nginx, stunnel, etc.) or whatever the default command is

nginx/templates/01_template_variables.conf.template

@@ -0,0 +1,19 @@
+map ${DOLLAR}host ${DOLLAR}sessions_index {
+  default "$MALCOLM_NETWORK_INDEX_PATTERN";
+}
+
+map ${DOLLAR}host ${DOLLAR}time_field {
+  default "$MALCOLM_NETWORK_INDEX_TIME_FIELD";
+}
+
+map ${DOLLAR}host ${DOLLAR}dashboards_prefix {
+  default "$NGINX_DASHBOARDS_PREFIX";
+}
+
+map ${DOLLAR}host ${DOLLAR}dashboards_proxy_pass {
+  default "$NGINX_DASHBOARDS_PROXY_PASS";
+}
+
+map ${DOLLAR}host ${DOLLAR}dashboards_proxy_url {
+  default "$NGINX_DASHBOARDS_PROXY_URL";
+}