Docker: Collect JVM heap dump in case server run into error (#2546)

Signed-off-by: Viet Nguyen Duc <[email protected]>

Base/Dockerfile

@@ -63,7 +63,7 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
     supervisor \
     gnupg2 \
     libnss3-tools \
-    openjdk-${JRE_VERSION}-jre-headless \
+    openjdk-${JRE_VERSION}-jdk-headless \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 
 RUN --mount=type=secret,id=SEL_PASSWD \
@@ -96,7 +96,7 @@ RUN --mount=type=secret,id=SEL_PASSWD \
 #==========
 # Selenium & relaxing permissions for OpenShift and other non-sudo environments
 #==========
-  && mkdir -p /opt/selenium /opt/selenium/assets /opt/selenium/secrets /var/run/supervisor /var/log/supervisor ${SEL_DOWNLOAD_DIR} \
+  && mkdir -p /opt/selenium /opt/selenium/assets /opt/selenium/secrets /opt/selenium/logs /var/run/supervisor /var/log/supervisor ${SEL_DOWNLOAD_DIR} \
     ${HOME}/.mozilla ${HOME}/.vnc ${HOME}/.pki/nssdb ${VIDEO_FOLDER} \
   # NSSDB initialization with an empty password
   && certutil -d sql:${HOME}/.pki/nssdb -N --empty-password \
@@ -142,8 +142,9 @@ RUN --mount=type=secret,id=SEL_PASSWD \
 #======================================
 # Add Grid check script
 #======================================
-COPY --chown="${SEL_UID}:${SEL_GID}" check-grid.sh entry_point.sh configs/node/nodeGridUrl.sh configs/node/nodePreStop.sh /opt/bin/
+COPY --chown="${SEL_UID}:${SEL_GID}" check-grid.sh entry_point.sh configs/node/nodeGridUrl.sh configs/node/nodePreStop.sh handle_heap_dump.sh /opt/bin/
 COPY --chown="${SEL_UID}:${SEL_GID}" mask /usr/local/bin/
+RUN chmod +x /opt/bin/*.sh /usr/local/bin/mask
 
 #======================================
 # Add Supervisor configuration file
@@ -187,6 +188,8 @@ ENV SE_BIND_HOST=false \
     SE_STRUCTURED_LOGS=false \
     SE_ENABLE_TRACING=true \
     SE_ENABLE_TLS=false \
+    SE_JAVA_OPTS_DEFAULT="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/selenium/logs" \
+    SE_JAVA_HEAP_DUMP="false" \
     SE_JAVA_HTTPCLIENT_VERSION="HTTP_1_1" \
     SE_JAVA_SSL_TRUST_STORE="/opt/selenium/secrets/server.jks" \
     SE_JAVA_SSL_TRUST_STORE_PASSWORD="/opt/selenium/secrets/server.pass" \

Base/handle_heap_dump.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+SELENIUM_SERVER_PID=$1
+LOG_DIR=$2
+TIMESTAMP=$(date +%s)
+
+filename="$LOG_DIR/dump_pid${SELENIUM_SERVER_PID}_${TIMESTAMP}.hprof"
+if ps -p "${SELENIUM_SERVER_PID}" >/dev/null; then
+  echo "Server process is still running. Create heap dump by using jmap"
+  jmap -dump:live,format=b,file="${filename}" "${SELENIUM_SERVER_PID}"
+else
+  filename_source="$LOG_DIR/java_pid${SELENIUM_SERVER_PID}.hprof"
+  if [ -f "$filename_source" ]; then
+    echo "Server is not running. Check HeapDumpOnOutOfMemoryError created"
+    mv "$filename_source" "$filename"
+  else
+    echo "Server is not running. No heap dump is created"
+  fi
+fi

Distributor/start-selenium-grid-distributor.sh

@@ -183,9 +183,26 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} \
   distributor \
   --bind-bus false \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

EventBus/start-selenium-grid-eventbus.sh

@@ -113,10 +113,27 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} event-bus \
   --bind-host ${SE_BIND_HOST} \
   ${HOST_CONFIG} \
   ${PORT_CONFIG} \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

Hub/start-selenium-grid-hub.sh

@@ -163,8 +163,25 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} \
   hub \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

NodeBase/start-selenium-node.sh

@@ -12,6 +12,9 @@ pacmd set-default-source v1.monitor
 
 rm -f /tmp/.X*lock
 
+# set -e: exit asap if a command exits with a non-zero status
+set -e
+
 function append_se_opts() {
   local option="${1}"
   local value="${2:-""}"
@@ -175,11 +178,28 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   ${CHROME_DRIVER_PATH_PROPERTY} \
   ${EDGE_DRIVER_PATH_PROPERTY} \
   ${GECKO_DRIVER_PATH_PROPERTY} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} \
   node \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

NodeDocker/start-selenium-grid-docker.sh

@@ -123,6 +123,19 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} node \
@@ -131,4 +144,8 @@ java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   --bind-host ${SE_BIND_HOST} \
   --detect-drivers false \
   --config /opt/selenium/${SE_NODE_DOCKER_CONFIG_FILENAME:-"config.toml"} \
-  ${SE_GRID_URL} ${SE_OPTS}
+  ${SE_GRID_URL} ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

Router/start-selenium-grid-router.sh

@@ -164,6 +164,19 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} router \
@@ -177,4 +190,8 @@ java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   ${HOST_CONFIG} \
   ${PORT_CONFIG} \
   ${SUB_PATH_CONFIG} \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

SessionQueue/start-selenium-grid-session-queue.sh

@@ -117,6 +117,19 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} sessionqueue \
@@ -125,4 +138,8 @@ java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   --bind-host ${SE_BIND_HOST} \
   ${HOST_CONFIG} \
   ${PORT_CONFIG} \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

Sessions/start-selenium-grid-sessions.sh

@@ -152,6 +152,19 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} sessions \
@@ -160,4 +173,8 @@ java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   --bind-host ${SE_BIND_HOST} \
   ${HOST_CONFIG} \
   ${PORT_CONFIG} \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

Standalone/start-selenium-standalone.sh

@@ -166,6 +166,19 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   ${CHROME_DRIVER_PATH_PROPERTY} \
   ${EDGE_DRIVER_PATH_PROPERTY} \
@@ -180,4 +193,8 @@ java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   --bind-host ${SE_BIND_HOST} \
   --config /opt/selenium/config.toml \
   ${SUB_PATH_CONFIG} \
-  ${SE_OPTS}
+  ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

StandaloneDocker/start-selenium-grid-docker.sh

@@ -128,6 +128,19 @@ if [ -n "${SE_JAVA_HTTPCLIENT_VERSION}" ]; then
   SE_JAVA_OPTS="$SE_JAVA_OPTS -Dwebdriver.httpclient.version=${SE_JAVA_HTTPCLIENT_VERSION}"
 fi
 
+if [ -n "${SE_JAVA_OPTS_DEFAULT}" ]; then
+  SE_JAVA_OPTS="${SE_JAVA_OPTS_DEFAULT} $SE_JAVA_OPTS"
+fi
+
+function handle_heap_dump() {
+  /opt/bin/handle_heap_dump.sh $SELENIUM_SERVER_PID /opt/selenium/logs
+}
+if [ "${SE_JAVA_HEAP_DUMP}" = "true" ]; then
+  trap handle_heap_dump ERR SIGTERM SIGINT
+else
+  trap handle_heap_dump ERR
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   -jar /opt/selenium/selenium-server.jar \
   ${EXTRA_LIBS} standalone \
@@ -137,4 +150,8 @@ java ${JAVA_OPTS:-$SE_JAVA_OPTS} \
   --detect-drivers false \
   --bind-host ${SE_BIND_HOST} \
   --config /opt/selenium/config.toml \
-  ${SE_GRID_URL} ${SE_OPTS}
+  ${SE_GRID_URL} ${SE_OPTS} &
+
+SELENIUM_SERVER_PID=$!
+
+wait $SELENIUM_SERVER_PID

charts/selenium-grid/CONFIGURATION.md

@@ -289,6 +289,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | components.sessionQueue.priorityClassName | string | `""` | Priority class name for Session Queue pods |
 | components.extraEnvironmentVariables | list | `[]` | Custom environment variables for all components |
 | components.extraEnvFrom | list | `[]` | Custom environment variables by sourcing entire configMap, Secret, etc. for all components |
+| components.extraVolumeMounts | list | `[]` | Extra volume mounts for component container |
+| components.extraVolumes | list | `[]` | Extra volumes for component pod |
 | hub.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) |
 | hub.imageName | string | `"hub"` | Selenium Hub image name |
 | hub.imageTag | string | `nil` | Selenium Hub image tag (this overwrites global.seleniumGrid.imageTag parameter) |
@@ -312,8 +314,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | hub.subPath | string | `""` | Custom sub path for the hub deployment |
 | hub.extraEnvironmentVariables | list | `[]` | Custom environment variables for selenium-hub |
 | hub.extraEnvFrom | list | `[]` | Custom environment variables by sourcing entire configMap, Secret, etc. for selenium-hub |
-| hub.extraVolumeMounts | list | `[]` | Extra volume mounts for selenium-hub container |
-| hub.extraVolumes | list | `[]` | Extra volumes for selenium-hub pod |
+| hub.extraVolumeMounts | list | `[]` | Extra volume mounts for Hub container |
+| hub.extraVolumes | list | `[]` | Extra volumes for Hub pod |
 | hub.resources | object | `{}` | Resources for selenium-hub container |
 | hub.securityContext | object | `{}` | SecurityContext for selenium-hub container |
 | hub.serviceType | string | `"ClusterIP"` | Kubernetes service type (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) |