diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index fb5fdd1f9..61574c38c 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -81,7 +81,11 @@ jobs: - name: Update tag in docs and files run: ./update_tag_in_docs_and_files.sh ${LATEST_TAG} ${NEXT_TAG} - name: Setup environment to build chart - run: make chart_setup_env + uses: nick-invision/retry@master + with: + timeout_minutes: 10 + max_attempts: 3 + command: CLUSTER=${CLUSTER} HELM_VERSION=${HELM_VERSION} make chart_setup_env - name: Build and lint charts run: | make chart_build_nightly diff --git a/Makefile b/Makefile index 937643bcf..862f09f40 100644 --- a/Makefile +++ b/Makefile @@ -461,16 +461,18 @@ chart_test_edge: chart_test_autoscaling_deployment_https: CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \ + SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling chart_test_autoscaling_deployment: - CHART_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_HOST=$$(hostname -i) \ + CHART_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_HOST=$$(hostname -i) RELEASE_NAME=selenium \ + SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling chart_test_autoscaling_job_https: - SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_BASIC_AUTH=true RELEASE_NAME=selenium SELENIUM_GRID_PORT=443 \ + SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_BASIC_AUTH=true RELEASE_NAME=selenium SELENIUM_GRID_PORT=443 SUB_PATH=/ \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh JobAutoscaling @@ -480,7 +482,7 @@ chart_test_autoscaling_job_hostname: ./tests/charts/make/chart_test.sh JobAutoscaling chart_test_autoscaling_job: - CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_HOST=selenium-grid.local RELEASE_NAME=selenium \ + CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_HOST=selenium-grid.local RELEASE_NAME=selenium SUB_PATH=/ \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh JobAutoscaling diff --git a/charts/selenium-grid/configs/node/nodePreStop.sh b/charts/selenium-grid/configs/node/nodePreStop.sh index 7f0e2285f..db4ccbdb2 100644 --- a/charts/selenium-grid/configs/node/nodePreStop.sh +++ b/charts/selenium-grid/configs/node/nodePreStop.sh @@ -1,13 +1,24 @@ #!/bin/bash +probe_name="lifecycle.${1:-"preStop"}" + +max_time=3 + +ID=$(echo $RANDOM) +tmp_node_file="/tmp/nodeProbe${ID}" + function on_exit() { - rm -rf /tmp/preStopOutput + rm -rf ${tmp_node_file} } trap on_exit EXIT +function init_file() { + echo "{}" > ${tmp_node_file} +} +init_file + # Set headers if Node Registration Secret is set -if [ ! -z "${SE_REGISTRATION_SECRET}" ]; -then +if [ ! -z "${SE_REGISTRATION_SECRET}" ]; then HEADERS="X-REGISTRATION-SECRET: ${SE_REGISTRATION_SECRET}" else HEADERS="X-REGISTRATION-SECRET;" @@ -16,66 +27,81 @@ fi function is_full_distributed_mode() { if [ -n "${SE_DISTRIBUTOR_HOST}" ] && [ -n "${SE_DISTRIBUTOR_PORT}" ]; then DISTRIBUTED_MODE=true - echo "Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap" + echo "$(date +%FT%T%Z) [${probe_name}] - Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap" else DISTRIBUTED_MODE=false - echo "Detected full distributed mode: ${DISTRIBUTED_MODE}" + echo "$(date +%FT%T%Z) [${probe_name}] - Detected full distributed mode: ${DISTRIBUTED_MODE}" fi } is_full_distributed_mode +function get_grid_url() { + if [ -z "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] || [ -z "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - There is no configured HUB or ROUTER host. preStop ignores to send drain request to upstream." + grid_url="" + fi + if [ -n "${SE_BASIC_AUTH}" ] && [ "${SE_BASIC_AUTH}" != "*@" ]; then + SE_BASIC_AUTH="${SE_BASIC_AUTH}@" + fi + if [ "${SE_SUB_PATH}" = "/" ]; then + SE_SUB_PATH="" + fi + grid_url=${SE_SERVER_PROTOCOL}://${SE_BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH} + grid_url_checks=$(curl -m ${max_time} -s -o /dev/null -w "%{http_code}" ${grid_url}) + if [ "${grid_url_checks}" = "401" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - Host requires Basic Auth. Please add the credentials to the SE_BASIC_AUTH variable (e.g: user:password). preStop ignores to send drain request to upstream." + grid_url="" + fi + if [ "${grid_url_checks}" = "404" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - The Grid is not available or it might have /subPath configured. Please wait a moment or check the SE_SUB_PATH variable if needed." + fi +} + function signal_distributor_to_drain_node() { if [ "${DISTRIBUTED_MODE}" = true ]; then - echo "Signaling Distributor to drain node" - set -x - curl -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" - set +x + echo "$(date +%FT%T%Z) [${probe_name}] - Signaling Distributor to drain node" + curl -m ${max_time} -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" fi } function signal_hub_to_drain_node() { if [ "${DISTRIBUTED_MODE}" = false ]; then - echo "Signaling Hub to drain node" - curl -k -X POST ${SE_GRID_URL}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" + get_grid_url + if [ -n "${grid_url}" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - Signaling Hub to drain node" + curl -m ${max_time} -k -X POST ${grid_url}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" + fi fi } function signal_node_to_drain() { - echo "Signaling Node to drain itself" - curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}" + echo "$(date +%FT%T%Z) [${probe_name}] - Signaling Node to drain itself" + curl -m ${max_time} -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}" } -function replace_localhost_by_service_name() { - internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}" - echo "SE_NODE_GRID_URL: ${SE_NODE_GRID_URL}" - if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then - SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}} - elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then - SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}} - elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then - SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}} - else - SE_GRID_URL=${SE_NODE_GRID_URL} - fi - echo "Set SE_GRID_URL internally: ${SE_GRID_URL}" -} -replace_localhost_by_service_name - -if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > /tmp/preStopOutput; then - NODE_ID=$(jq -r '.value.node.nodeId' /tmp/preStopOutput) +if curl -m ${max_time} -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > ${tmp_node_file}; then + NODE_ID=$(jq -r '.value.node.nodeId' ${tmp_node_file} || "") if [ -n "${NODE_ID}" ]; then - echo "Current Node ID is: ${NODE_ID}" - signal_hub_to_drain_node + echo "$(date +%FT%T%Z) [${probe_name}] - Current Node ID is: ${NODE_ID}" signal_distributor_to_drain_node + signal_hub_to_drain_node echo fi signal_node_to_drain # Wait for the current session to be finished if any - while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/preStopOutput; + while curl -m ${max_time} -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o ${tmp_node_file}; do - echo "Node preStop is waiting for current session to be finished if any. Node details: message: $(jq -r '.value.message' /tmp/preStopOutput || "unknown"), availability: $(jq -r '.value.node.availability' /tmp/preStopOutput || "unknown")" - sleep 1; + SLOT_HAS_SESSION=$(jq -e ".value.node.slots[]|select(.session != null).id.id" ${tmp_node_file} | tr -d '"' || "") + if [ -z "${SLOT_HAS_SESSION}" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - There is no session running. Node is ready to be terminated." + echo "$(date +%FT%T%Z) [${probe_name}] - $(cat ${tmp_node_file} || "")" + echo + exit 0 + else + echo "$(date +%FT%T%Z) [${probe_name}] - Node preStop is waiting for current session on slot ${SLOT_HAS_SESSION} to be finished. Node details: message: $(jq -r '.value.message' ${tmp_node_file} || "unknown"), availability: $(jq -r '.value.node.availability' ${tmp_node_file} || "unknown")" + sleep 1; + fi done else - echo "Node is already drained. Shutting down gracefully!" + echo "$(date +%FT%T%Z) [${probe_name}] - Node is already drained. Shutting down gracefully!" fi diff --git a/charts/selenium-grid/configs/node/nodeProbe.sh b/charts/selenium-grid/configs/node/nodeProbe.sh index 13e1538de..466f428ef 100644 --- a/charts/selenium-grid/configs/node/nodeProbe.sh +++ b/charts/selenium-grid/configs/node/nodeProbe.sh @@ -1,53 +1,79 @@ #!/bin/bash +max_time=3 +probe_name="Probe.${1:-"Startup"}" + +ID=$(echo $RANDOM) +tmp_node_file="/tmp/nodeProbe${ID}" +tmp_grid_file="/tmp/gridProbe${ID}" + function on_exit() { - rm -rf /tmp/nodeProbe${ID} - rm -rf /tmp/gridProbe${ID} + rm -rf ${tmp_node_file} + rm -rf ${tmp_grid_file} } trap on_exit EXIT -ID=$(echo $RANDOM) +function init_file() { + echo "{}" > ${tmp_node_file} + echo "{}" > ${tmp_grid_file} +} +init_file -function replace_localhost_by_service_name() { - internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}" - echo "SE_NODE_GRID_URL: ${SE_NODE_GRID_URL}" - if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then - SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}} - elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then - SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}} - elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then - SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}} - else - SE_GRID_URL=${SE_NODE_GRID_URL} +function help_message() { + echo "$(date +%FT%T%Z) [${probe_name}] - If you believe Node is registered successfully but probe still report this message and fail for a long time. Workaround by set 'global.seleniumGrid.defaultNodeStartupProbe' to 'httpGet' and report us an issue for Chart improvement with your scenario." +} + +function get_grid_url() { + if [ -z "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] || [ -z "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - There is no configured HUB or ROUTER host. Probe ignores the registration checks on upstream." + exit 0 + fi + if [[ -n "${SE_BASIC_AUTH}" && "${SE_BASIC_AUTH}" != *@ ]]; then + SE_BASIC_AUTH="${SE_BASIC_AUTH}@" + fi + if [ "${SE_SUB_PATH}" = "/" ]; then + SE_SUB_PATH="" + fi + grid_url=${SE_SERVER_PROTOCOL}://${SE_BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH} + grid_url_checks=$(curl -m ${max_time} -skf -o /dev/null -w "%{http_code}" ${grid_url}) + if [ "${grid_url_checks}" = "401" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - Host requires Basic Auth. Please add the credentials to the SE_BASIC_AUTH variable (e.g: user:password)." + help_message + exit 1 + fi + if [ "${grid_url_checks}" = "404" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - The Grid is not available or it might have /subPath configured. Please wait a moment or check the SE_SUB_PATH variable if needed." + help_message + exit 1 fi - echo "Set SE_GRID_URL internally: ${SE_GRID_URL}" } -replace_localhost_by_service_name -if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/nodeProbe${ID}; then - NODE_ID=$(jq -r '.value.node.nodeId' /tmp/nodeProbe${ID}) - NODE_STATUS=$(jq -r '.value.node.availability' /tmp/nodeProbe${ID}) +if curl -m ${max_time} -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o ${tmp_node_file}; then + NODE_ID=$(jq -r '.value.node.nodeId' ${tmp_node_file} || "") + NODE_STATUS=$(jq -r '.value.node.availability' ${tmp_node_file} || "") if [ -n "${NODE_ID}" ]; then - echo "Node responds the ID: ${NODE_ID} with status: ${NODE_STATUS}" + echo "$(date +%FT%T%Z) [${probe_name}] - Node responds the ID: ${NODE_ID} with status: ${NODE_STATUS}" else - echo "Wait for the Node to report its status" + echo "$(date +%FT%T%Z) [${probe_name}] - Wait for the Node to report its status" exit 1 fi - curl -sfk "${SE_GRID_URL}/status" -o /tmp/gridProbe${ID} - GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" /tmp/gridProbe${ID} | tr -d '"' || true) + get_grid_url + + curl -m ${max_time} -sfk "${grid_url}/status" -o ${tmp_grid_file} + GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" ${tmp_grid_file} | tr -d '"' || "") if [ -n "${GRID_NODE_ID}" ]; then - echo "Grid responds a matched Node ID: ${GRID_NODE_ID}" + echo "$(date +%FT%T%Z) [${probe_name}] - Grid responds a matched Node ID: ${GRID_NODE_ID}" fi - if [ "${NODE_STATUS}" = "UP" ] && [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then - echo "Node ID: ${NODE_ID} is found in the Grid. The registration is successful." + if [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then + echo "$(date +%FT%T%Z) [${probe_name}] - Node ID: ${NODE_ID} is found in the Grid. Node is ready." exit 0 else - echo "Node ID: ${NODE_ID} is not found in the Grid. The registration could be in progress." + echo "$(date +%FT%T%Z) [${probe_name}] - Node ID: ${NODE_ID} is not found in the Grid. Node is not ready." exit 1 fi else - echo "Wait for the Node to report its status" + echo "$(date +%FT%T%Z) [${probe_name}] - Wait for the Node to report its status" exit 1 fi diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index cdcd06498..c8479aaca 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -35,6 +35,14 @@ Check user define custom probe method {{- $overrideProbe | toYaml -}} {{- end -}} +{{- define "seleniumGrid.probe.stdout" -}} +{{- $stdout := "" -}} +{{- if .Values.global.seleniumGrid.stdoutProbeLog -}} + {{- $stdout = ">> /proc/1/fd/1" -}} +{{- end -}} +{{- $stdout -}} +{{- end -}} + {{/* Get probe settings */}} @@ -155,8 +163,10 @@ Common autoscaling spec template {{- if not .Values.autoscaling.scaledOptions.triggers }} triggers: - type: selenium-grid + metadata: + triggerIndex: '{{ default $.Values.autoscaling.scaledOptions.minReplicaCount (.node.scaledOptions).minReplicaCount }}' {{- with .node.hpa }} - metadata: {{- tpl (toYaml .) $ | nindent 6 }} + {{- tpl (toYaml .) $ | nindent 6 }} {{- end }} {{- end }} {{- end -}} @@ -207,6 +217,14 @@ template: value: {{ .name | quote }} - name: SE_NODE_PORT value: {{ .node.port | quote }} + {{- with .node.startupProbe.timeoutSeconds }} + - name: SE_NODE_REGISTER_PERIOD + value: {{ . | quote }} + {{- end }} + {{- with .node.startupProbe.periodSeconds }} + - name: SE_NODE_REGISTER_CYCLE + value: {{ . | quote }} + {{- end }} {{- with .node.extraEnvironmentVariables }} {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} @@ -268,7 +286,7 @@ template: {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else if eq $.Values.global.seleniumGrid.defaultNodeStartupProbe "exec" }} exec: - command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh >> /proc/1/fd/1"] + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh Startup {{ include "seleniumGrid.probe.stdout" $ }}"] {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -301,6 +319,9 @@ template: livenessProbe: {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} + {{- else if eq $.Values.global.seleniumGrid.defaultNodeLivenessProbe "exec" }} + exec: + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh Liveness {{ include "seleniumGrid.probe.stdout" $ }}"] {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -547,7 +568,7 @@ Define preStop hook for the node pod. Node preStop script is stored in a ConfigM {{- define "seleniumGrid.node.deregisterLifecycle" -}} preStop: exec: - command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodePreStop.sh >> /proc/1/fd/1"] + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodePreStop.sh {{ include "seleniumGrid.probe.stdout" $ }}"] {{- end -}} {{/* diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml index 9374e37e7..559bb2d01 100644 --- a/charts/selenium-grid/templates/node-configmap.yaml +++ b/charts/selenium-grid/templates/node-configmap.yaml @@ -21,6 +21,8 @@ data: SE_HUB_HOST: '{{ include "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}' SE_HUB_PORT: '{{ .Values.hub.port }}' {{- end }} + SE_BASIC_AUTH: '{{ template "seleniumGrid.url.basicAuth" $ }}' + SE_SUB_PATH: '{{ template "seleniumGrid.url.subPath" $ }}' SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}' SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}' SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" . }}' diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 39ea9be65..63bf74ba8 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -19,6 +19,8 @@ global: # Set default startup probe for all nodes (supplied values: httpGet, exec). If not set, the default is httpGet # startup probe method `exec.command` is using a script is mounted from `nodeConfigMap.extraScripts.nodeProbe.sh` defaultNodeStartupProbe: exec + defaultNodeLivenessProbe: exec + stdoutProbeLog: true tls: enabled: false @@ -665,8 +667,8 @@ chromeNode: path: /status initialDelaySeconds: 0 periodSeconds: 5 - timeoutSeconds: 125 - failureThreshold: 25 + timeoutSeconds: 60 + failureThreshold: 12 successThreshold: 1 # Readiness probe settings @@ -683,9 +685,9 @@ chromeNode: livenessProbe: enabled: false path: /status - initialDelaySeconds: 15 - failureThreshold: 10 - timeoutSeconds: 10 + initialDelaySeconds: 30 + failureThreshold: 6 + timeoutSeconds: 60 periodSeconds: 10 successThreshold: 1 @@ -714,6 +716,7 @@ chromeNode: url: '{{ template "seleniumGrid.graphqlURL" . }}' browserName: 'chrome' sessionBrowserName: 'chrome' + platformName: 'Linux' # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid. unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional @@ -825,8 +828,8 @@ firefoxNode: path: /status initialDelaySeconds: 0 periodSeconds: 5 - timeoutSeconds: 125 - failureThreshold: 25 + timeoutSeconds: 60 + failureThreshold: 12 successThreshold: 1 # Readiness probe settings @@ -843,9 +846,9 @@ firefoxNode: livenessProbe: enabled: false path: /status - initialDelaySeconds: 15 - failureThreshold: 10 - timeoutSeconds: 10 + initialDelaySeconds: 30 + failureThreshold: 6 + timeoutSeconds: 60 periodSeconds: 10 successThreshold: 1 @@ -874,6 +877,7 @@ firefoxNode: url: '{{ template "seleniumGrid.graphqlURL" . }}' browserName: 'firefox' sessionBrowserName: 'firefox' + platformName: 'Linux' unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add initContainers in the same pod of the browser node. @@ -983,8 +987,8 @@ edgeNode: path: /status initialDelaySeconds: 0 periodSeconds: 5 - timeoutSeconds: 125 - failureThreshold: 25 + timeoutSeconds: 60 + failureThreshold: 12 successThreshold: 1 # Readiness probe settings @@ -1001,9 +1005,9 @@ edgeNode: livenessProbe: enabled: false path: /status - initialDelaySeconds: 15 - failureThreshold: 10 - timeoutSeconds: 10 + initialDelaySeconds: 30 + failureThreshold: 6 + timeoutSeconds: 60 periodSeconds: 10 successThreshold: 1 @@ -1031,7 +1035,8 @@ edgeNode: hpa: url: '{{ template "seleniumGrid.graphqlURL" . }}' browserName: 'MicrosoftEdge' - sessionBrowserName: 'MicrosoftEdge' + sessionBrowserName: 'msedge' + platformName: 'Linux' unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add initContainers in the same pod of the browser node. @@ -1166,13 +1171,13 @@ customLabels: {} # Configuration for dependency chart keda keda: additionalAnnotations: - "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook": pre-install podAnnotations: keda: - "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook": pre-install crds: additionalAnnotations: - "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook": pre-install http: timeout: 60000 webhooks: diff --git a/tests/charts/ci/DeploymentAutoscaling-values.yaml b/tests/charts/ci/DeploymentAutoscaling-values.yaml index fe385700f..26da41501 100644 --- a/tests/charts/ci/DeploymentAutoscaling-values.yaml +++ b/tests/charts/ci/DeploymentAutoscaling-values.yaml @@ -1,9 +1,8 @@ autoscaling: - enabled: true scalingType: deployment scaledOptions: minReplicaCount: 0 - maxReplicaCount: 2 + maxReplicaCount: 3 pollingInterval: 20 scaledObjectOptions: cooldownPeriod: 30 @@ -45,10 +44,6 @@ chromeNode: value: "1080" - name: TZ value: "Asia/Saigon" - - name: SE_NODE_REGISTER_PERIOD - value: "800" - - name: SE_NODE_REGISTER_CYCLE - value: "3" readinessProbe: enabled: &readinessProbe true livenessProbe: diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index 25920dca9..04c3dc6ff 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -1,11 +1,10 @@ autoscaling: - enabled: true scalingType: job scaledJobOptions: successfulJobsHistoryLimit: 0 failedJobsHistoryLimit: 0 scalingStrategy: - strategy: accurate + strategy: default scaledOptions: minReplicaCount: 0 maxReplicaCount: 30 @@ -24,14 +23,10 @@ chromeNode: value: "1080" - name: TZ value: "Asia/Saigon" - - name: SE_NODE_REGISTER_PERIOD - value: "800" - - name: SE_NODE_REGISTER_CYCLE - value: "3" readinessProbe: enabled: &readinessProbe false livenessProbe: - enabled: &livenessProbe false + enabled: &livenessProbe true # Configuration for edge nodes edgeNode: nameOverride: my-edge-name diff --git a/tests/charts/ci/base-auth-ingress-values.yaml b/tests/charts/ci/base-auth-ingress-values.yaml index 9e02237c1..e55c649b0 100644 --- a/tests/charts/ci/base-auth-ingress-values.yaml +++ b/tests/charts/ci/base-auth-ingress-values.yaml @@ -3,20 +3,8 @@ global: logLevel: INFO ingress: - annotations: - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$2 - nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" className: nginx hostname: "" - paths: - - path: /selenium(/|$)(.*) - pathType: ImplementationSpecific - backend: - service: - name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' - port: - number: 4444 basicAuth: enabled: false @@ -24,7 +12,6 @@ basicAuth: isolateComponents: true hub: - subPath: *gridAppRoot extraEnvironmentVariables: &extraEnvironmentVariables - name: SE_SESSION_REQUEST_TIMEOUT value: "800" @@ -36,7 +23,6 @@ hub: value: "false" components: - subPath: *gridAppRoot extraEnvironmentVariables: *extraEnvironmentVariables ingress-nginx: diff --git a/tests/charts/ci/base-subPath-values.yaml b/tests/charts/ci/base-subPath-values.yaml new file mode 100644 index 000000000..785ebf1c1 --- /dev/null +++ b/tests/charts/ci/base-subPath-values.yaml @@ -0,0 +1,26 @@ +ingress: + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" + paths: + - path: /selenium(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' + port: + number: 4444 + - path: /(/?)(session/.*/element) + pathType: ImplementationSpecific + backend: + service: + name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' + port: + number: 4444 + +hub: + subPath: *gridAppRoot + +components: + subPath: *gridAppRoot diff --git a/tests/charts/make/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh index 76dcbefcb..6b401a112 100755 --- a/tests/charts/make/chart_cluster_setup.sh +++ b/tests/charts/make/chart_cluster_setup.sh @@ -18,6 +18,7 @@ SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the clust KUBERNETES_VERSION=${KUBERNETES_VERSION:-$(curl -L -s https://dl.k8s.io/release/stable.txt)} CNI=${CNI:-"calico"} # auto, calico, cilium CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"docker"} # docker, containerd, cri-o +TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA:-"true"} # Function to clean up for retry step on workflow cleanup() { @@ -60,6 +61,11 @@ elif [ "${CLUSTER}" = "minikube" ]; then sudo chown -R $USER $HOME/.kube $HOME/.minikube fi +if [ "${TEST_EXISTING_KEDA}" = "true" ]; then + echo "Install KEDA core on kind kubernetes cluster" + helm upgrade -i ${KEDA_NAMESPACE} -n ${KEDA_NAMESPACE} --create-namespace --set webhooks.enabled=false kedacore/keda +fi + if [ "${CLUSTER}" = "kind" ]; then echo "Load built local Docker Images into Kind Cluster" image_list=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep ${NAMESPACE} | grep ${BUILD_DATE:-$VERSION}) @@ -67,3 +73,8 @@ if [ "${CLUSTER}" = "kind" ]; then kind load docker-image --name ${CLUSTER_NAME} "$image" done fi + +if [ "${TEST_EXISTING_INGRESS}" = "true" ]; then + echo "Wait for KEDA core to be ready" + kubectl -n ${KEDA_NAMESPACE} wait --for=condition=ready pod -l app.kubernetes.io/instance=${KEDA_NAMESPACE} --timeout 180s +fi diff --git a/tests/charts/make/chart_setup_env.sh b/tests/charts/make/chart_setup_env.sh index ac1d2b8c0..be4d77a79 100755 --- a/tests/charts/make/chart_setup_env.sh +++ b/tests/charts/make/chart_setup_env.sh @@ -24,6 +24,7 @@ if [ "$(uname -m)" = "x86_64" ]; then "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update -qq sudo apt-get install -yq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin sudo docker version echo "===============================" diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 25d4bdbee..7737cf498 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -16,13 +16,13 @@ SELENIUM_GRID_HOST=${SELENIUM_GRID_HOST:-"localhost"} SELENIUM_GRID_PORT=${SELENIUM_GRID_PORT:-"80"} MATRIX_BROWSER=${1:-"NodeChrome"} SELENIUM_GRID_AUTOSCALING=${2:-"true"} -SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=${3:-"0"} +SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=${SELENIUM_GRID_AUTOSCALING_MIN_REPLICA:-"0"} WAIT_TIMEOUT=${WAIT_TIMEOUT:-"90s"} HUB_CHECKS_INTERVAL=${HUB_CHECKS_INTERVAL:-45} HUB_CHECKS_MAX_ATTEMPTS=${HUB_CHECKS_MAX_ATTEMPTS:-6} WEB_DRIVER_WAIT_TIMEOUT=${WEB_DRIVER_WAIT_TIMEOUT:-120} AUTOSCALING_POLL_INTERVAL=${AUTOSCALING_POLL_INTERVAL:-20} -SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the cluster after the test run +SKIP_CLEANUP=${SKIP_CLEANUP:-"true"} # For debugging purposes, retain the cluster after the test run CHART_CERT_PATH=${CHART_CERT_PATH:-"${CHART_PATH}/certs/selenium.pem"} SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} VIDEO_TAG=${VIDEO_TAG:-"latest"} @@ -33,7 +33,8 @@ CHART_ENABLE_INGRESS_HOSTNAME=${CHART_ENABLE_INGRESS_HOSTNAME:-"false"} CHART_ENABLE_BASIC_AUTH=${CHART_ENABLE_BASIC_AUTH:-"false"} BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME:-"sysAdminUser"} BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD:-"myStrongPassword"} -LOG_LEVEL=${LOG_LEVEL:-"FINE"} +LOG_LEVEL=${LOG_LEVEL:-"INFO"} +TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA:-"true"} cleanup() { # Get the list of pods @@ -95,6 +96,18 @@ HELM_COMMAND_SET_IMAGES=" \ --set global.seleniumGrid.logLevel=${LOG_LEVEL} \ " +if [ "${TEST_EXISTING_KEDA}" = "true" ]; then + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set autoscaling.enabled=false \ + --set autoscaling.enableWithExistingKEDA=true \ + " +else + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set autoscaling.enabled=true \ + --set autoscaling.enableWithExistingKEDA=false \ + " +fi + if [ -n "${SET_MAX_REPLICAS}" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set autoscaling.scaledOptions.maxReplicaCount=${SET_MAX_REPLICAS} \ @@ -114,18 +127,15 @@ else HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set global.K8S_PUBLIC_IP=${SELENIUM_GRID_HOST} \ " -fi - -if [[ ! $(cat /etc/hosts) == *"alertmanager.selenium-grid.prod"* ]]; then - sudo -- sh -c -e "echo \"$(hostname -i) alertmanager.selenium-grid.prod\" >> /etc/hosts" -fi - -if [[ ! $(cat /etc/hosts) == *"grafana.selenium-grid.prod"* ]]; then - sudo -- sh -c -e "echo \"$(hostname -i) grafana.selenium-grid.prod\" >> /etc/hosts" -fi - -if [[ ! $(cat /etc/hosts) == *"pts.selenium-grid.prod"* ]]; then - sudo -- sh -c -e "echo \"$(hostname -i) pts.selenium-grid.prod\" >> /etc/hosts" + if [[ ! $(cat /etc/hosts) == *"alertmanager.selenium-grid.prod"* ]]; then + sudo -- sh -c -e "echo \"$(hostname -i) alertmanager.selenium-grid.prod\" >> /etc/hosts" + fi + if [[ ! $(cat /etc/hosts) == *"grafana.selenium-grid.prod"* ]]; then + sudo -- sh -c -e "echo \"$(hostname -i) grafana.selenium-grid.prod\" >> /etc/hosts" + fi + if [[ ! $(cat /etc/hosts) == *"pts.selenium-grid.prod"* ]]; then + sudo -- sh -c -e "echo \"$(hostname -i) pts.selenium-grid.prod\" >> /etc/hosts" + fi fi if [ "${CHART_ENABLE_BASIC_AUTH}" = "true" ]; then @@ -151,6 +161,16 @@ HELM_COMMAND_SET_BASE_VALUES=" \ --values ${TEST_VALUES_PATH}/base-resources-values.yaml \ " +if [ "${SUB_PATH}" = "/selenium" ]; then + HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ + --values ${TEST_VALUES_PATH}/base-subPath-values.yaml \ + " +fi + +if [ "${SUB_PATH}" = "/" ]; then + SUB_PATH="" +fi + if [ "${SELENIUM_GRID_PROTOCOL}" = "https" ]; then HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ --values ${TEST_VALUES_PATH}/base-tls-values.yaml \