diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index 4f0a7097c..013bf6ca0 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -499,6 +499,57 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | edgeNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option | | edgeNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option | | edgeNode.videoRecorder | object | `{}` | Override specific video recording settings for edge node | +| relayNode.enabled | bool | `false` | Enable relay nodes | +| relayNode.deploymentEnabled | bool | `true` | NOTE: Only used when autoscaling.enabled is false Enable creation of Deployment true (default) - if you want long-living pods false - for provisioning your own custom type such as Jobs | +| relayNode.updateStrategy | object | `{"type":"RollingUpdate"}` | Global update strategy will be overwritten by individual component | +| relayNode.replicas | int | `1` | Number of relay nodes | +| relayNode.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) | +| relayNode.imageName | string | `"node-base"` | Image of relay nodes | +| relayNode.imageTag | string | `nil` | Image of relay nodes (this overwrites global.seleniumGrid.nodesImageTag) | +| relayNode.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) | +| relayNode.imagePullSecret | string | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) | +| relayNode.ports | list | `[]` | Extra ports list to enable on the node container (e.g. SSH, VNC, NoVNC, etc.) | +| relayNode.port | int | `5555` | Node component port | +| relayNode.nodePort | string | `nil` | Node component expose NodePort | +| relayNode.affinity | object | `{}` | Specify affinity for relay-node pods, this overwrites global.seleniumGrid.affinity parameter | +| relayNode.topologySpreadConstraints | list | `[]` | Specify topologySpreadConstraints for relay-node pods, this overwrites global.seleniumGrid.topologySpreadConstraints parameter | +| relayNode.annotations | object | `{}` | Annotations for relay-node pods | +| relayNode.labels | object | `{}` | Labels for relay-node pods | +| relayNode.shareProcessNamespace | bool | `true` | Shared process namespace for relay-node pods | +| relayNode.resources.requests | object | `{"cpu":"1","memory":"1Gi"}` | Request resources for relay-node pods | +| relayNode.resources.limits | object | `{"cpu":"1","memory":"2Gi"}` | Limit resources for relay-node pods | +| relayNode.securityContext | object | `{}` | SecurityContext for relay-node container | +| relayNode.tolerations | list | `[]` | Tolerations for relay-node pods | +| relayNode.nodeSelector | object | `{}` | Node selector for relay-node pods | +| relayNode.hostAliases | string | `nil` | Custom host aliases for relay nodes | +| relayNode.extraEnvironmentVariables | string | `nil` | Custom environment variables for relay nodes | +| relayNode.extraEnvFrom | string | `nil` | Custom environment variables by sourcing entire configMap, Secret, etc. for relay nodes | +| relayNode.service.enabled | bool | `false` | Create a service for node | +| relayNode.service.type | string | `"ClusterIP"` | Service type | +| relayNode.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) | +| relayNode.service.ports | string | `nil` | Extra ports exposed in node service | +| relayNode.service.annotations | object | `{}` | Custom annotations for service | +| relayNode.dshmVolumeSizeLimit | string | `""` | Size limit for DSH volume mounted in container (if not set, default is disabled, e.g "1Gi") | +| relayNode.priorityClassName | string | `""` | Priority class name for relay-node pods | +| relayNode.startupProbe | object | `{"enabled":true,"failureThreshold":12,"initialDelaySeconds":0,"path":"/status","periodSeconds":5,"successThreshold":1,"timeoutSeconds":60}` | Startup probe settings | +| relayNode.readinessProbe | object | `{"enabled":false,"failureThreshold":10,"initialDelaySeconds":10,"path":"/status","periodSeconds":10,"successThreshold":1,"timeoutSeconds":10}` | Readiness probe settings | +| relayNode.livenessProbe | object | `{"enabled":false,"failureThreshold":6,"initialDelaySeconds":30,"path":"/status","periodSeconds":10,"successThreshold":1,"timeoutSeconds":60}` | Liveness probe settings | +| relayNode.terminationGracePeriodSeconds | int | `30` | Time to wait for pod termination | +| relayNode.deregisterLifecycle | string | `nil` | Define preStop command to shut down the relay node gracefully. This overwrites autoscaling.deregisterLifecycle | +| relayNode.lifecycle | object | `{}` | Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any | +| relayNode.extraVolumeMounts | list | `[]` | Extra volume mounts for relay-node container | +| relayNode.extraVolumes | list | `[]` | Extra volumes for relay-node pod | +| relayNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node | +| relayNode.scaledOptions | string | `nil` | Override the scaled options for relay nodes | +| relayNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for relay nodes | +| relayNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for relay nodes | +| relayNode.hpa.browserName | string | `"chrome"` | browserName from the capability | +| relayNode.hpa.sessionBrowserName | string | `""` | sessionBrowserName if the browserName is different from the sessionBrowserName | +| relayNode.hpa.platformName | string | `"Android"` | platformName from the capability | +| relayNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint | +| relayNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option | +| relayNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option | +| relayNode.videoRecorder | object | `{}` | Override specific video recording settings for edge node | | videoRecorder.enabled | bool | `false` | Enable video recording in all browser nodes | | videoRecorder.name | string | `"video"` | Container name is set to resource specs | | videoRecorder.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) | diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl index 31a4e9414..162e7d2ec 100644 --- a/charts/selenium-grid/templates/_nameHelpers.tpl +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -140,6 +140,13 @@ Edge node fullname {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-edge-node" $)) .Values.edgeNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Relay node fullname +*/}} +{{- define "seleniumGrid.relayNode.fullname" -}} +{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-relay-node" $)) .Values.relayNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Ingress fullname */}} diff --git a/charts/selenium-grid/templates/relay-node-deployment.yaml b/charts/selenium-grid/templates/relay-node-deployment.yaml new file mode 100644 index 000000000..9255e204c --- /dev/null +++ b/charts/selenium-grid/templates/relay-node-deployment.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.relayNode.enabled ((eq (include "seleniumGrid.useKEDA" .) "true") | ternary (eq .Values.autoscaling.scalingType "deployment") .Values.relayNode.deploymentEnabled) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "seleniumGrid.relayNode.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "seleniumGrid.relayNode.fullname" . }} + app.kubernetes.io/name: {{ template "seleniumGrid.relayNode.fullname" . }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.relayNode.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + strategy: + {{- template "seleniumGrid.updateStrategy" (list $.Values.relayNode $.Values.global.seleniumGrid) }} + {{- if not (eq (include "seleniumGrid.useKEDA" $) "true") }} + replicas: {{ .Values.relayNode.replicas }} + {{- else }} + replicas: {{ default $.Values.autoscaling.scaledOptions.minReplicaCount ($.Values.relayNode.scaledOptions).minReplicaCount }} + {{- end }} + revisionHistoryLimit: {{ .Values.global.seleniumGrid.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "seleniumGrid.relayNode.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- $podScope := deepCopy . -}} +{{- $_ := set $podScope "name" (include "seleniumGrid.relayNode.fullname" .) -}} +{{- $_ = set $podScope "node" .Values.relayNode -}} +{{- $_ = set $podScope "recorder" (mergeOverwrite .Values.videoRecorder .Values.relayNode.videoRecorder) -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} +{{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} +{{- end }} diff --git a/charts/selenium-grid/templates/relay-node-hpa.yaml b/charts/selenium-grid/templates/relay-node-hpa.yaml new file mode 100644 index 000000000..20d1a054b --- /dev/null +++ b/charts/selenium-grid/templates/relay-node-hpa.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.relayNode.enabled (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "deployment") }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ template "seleniumGrid.relayNode.fullname" . }} + namespace: {{ .Release.Namespace }} + annotations: + {{- with .Values.autoscaling.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + deploymentName: {{ template "seleniumGrid.relayNode.fullname" . }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} + {{- with .Values.relayNode.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.relayNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.relayNode -}} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} +{{- end }} diff --git a/charts/selenium-grid/templates/relay-node-scaledjobs.yaml b/charts/selenium-grid/templates/relay-node-scaledjobs.yaml new file mode 100644 index 000000000..dbdce04ff --- /dev/null +++ b/charts/selenium-grid/templates/relay-node-scaledjobs.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.relayNode.enabled (include "seleniumGrid.useKEDA" .) (eq .Values.autoscaling.scalingType "job") }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: {{ template "seleniumGrid.relayNode.fullname" . }} + namespace: {{ .Release.Namespace }} + annotations: + {{- with .Values.autoscaling.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "seleniumGrid.relayNode.fullname" . }} + app.kubernetes.io/name: {{ template "seleniumGrid.relayNode.fullname" . }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} + {{- with .Values.relayNode.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- $podScope := deepCopy . -}} + {{- $_ := set $podScope "name" (include "seleniumGrid.relayNode.fullname" .) -}} + {{- $_ = set $podScope "node" .Values.relayNode -}} + {{- $_ = set $podScope "recorder" (mergeOverwrite .Values.videoRecorder .Values.relayNode.videoRecorder) -}} + {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} + {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} + {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} +{{- end }} diff --git a/charts/selenium-grid/templates/relay-node-service.yaml b/charts/selenium-grid/templates/relay-node-service.yaml new file mode 100644 index 000000000..661cbc0af --- /dev/null +++ b/charts/selenium-grid/templates/relay-node-service.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.relayNode.enabled .Values.relayNode.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "seleniumGrid.relayNode.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + name: {{ template "seleniumGrid.relayNode.fullname" . }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.relayNode.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.relayNode.service.type }} + selector: + app: {{ template "seleniumGrid.relayNode.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if and (eq .Values.relayNode.service.type "LoadBalancer") (.Values.relayNode.service.loadBalancerIP) }} + loadBalancerIP: {{ .Values.relayNode.service.loadBalancerIP }} + {{- end }} + ports: + - name: tcp-chrome + protocol: TCP + port: {{ .Values.relayNode.port }} + targetPort: {{ .Values.relayNode.port }} + {{- if and (eq $.Values.relayNode.service.type "NodePort") .Values.relayNode.nodePort }} + nodePort: {{ .Values.relayNode.nodePort }} + {{- end }} + {{- with .Values.relayNode.service.ports }} + {{- range . }} + - name: {{ .name }} + port: {{ .port }} + targetPort: {{ .targetPort }} + {{- if .protocol }} + protocol: {{ .protocol }} + {{- end }} + {{- if and (eq $.Values.relayNode.service.type "NodePort") .nodePort }} + nodePort: {{ .nodePort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index de916f199..501489251 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1449,6 +1449,189 @@ edgeNode: # -- Override specific video recording settings for edge node videoRecorder: {} +# Configuration for relay nodes +relayNode: + # -- Enable relay nodes + enabled: false + + # -- NOTE: Only used when autoscaling.enabled is false + # Enable creation of Deployment + # true (default) - if you want long-living pods + # false - for provisioning your own custom type such as Jobs + deploymentEnabled: true + # -- Global update strategy will be overwritten by individual component + updateStrategy: + type: RollingUpdate + # -- Number of relay nodes + replicas: 1 + # -- Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) + imageRegistry: + # -- Image of relay nodes + imageName: node-base + # -- Image of relay nodes (this overwrites global.seleniumGrid.nodesImageTag) + imageTag: + # -- Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) + imagePullPolicy: IfNotPresent + # -- Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) + imagePullSecret: "" + + # -- Extra ports list to enable on the node container (e.g. SSH, VNC, NoVNC, etc.) + ports: [] + # - 5900 + # - 7900 + # -- Node component port + port: 5555 + # -- Node component expose NodePort + nodePort: + # -- Specify affinity for relay-node pods, this overwrites global.seleniumGrid.affinity parameter + affinity: {} + # -- Specify topologySpreadConstraints for relay-node pods, this overwrites global.seleniumGrid.topologySpreadConstraints parameter + topologySpreadConstraints: [] + # -- Annotations for relay-node pods + annotations: {} + # -- Labels for relay-node pods + labels: {} + # -- Shared process namespace for relay-node pods + shareProcessNamespace: true + # Resources for relay-node container + resources: + # -- Request resources for relay-node pods + requests: + memory: "1Gi" + cpu: "1" + # -- Limit resources for relay-node pods + limits: + memory: "2Gi" + cpu: "1" + # -- SecurityContext for relay-node container + securityContext: {} + # -- Tolerations for relay-node pods + tolerations: [] + # -- Node selector for relay-node pods + nodeSelector: {} + # -- Custom host aliases for relay nodes + hostAliases: + # - ip: "198.51.100.0" + # hostnames: + # - "example.com" + # - "example.net" + # - ip: "203.0.113.0" + # hostnames: + # - "example.org" + # -- Custom environment variables for relay nodes + extraEnvironmentVariables: + # - name: SE_JAVA_OPTS + # value: "-Xmx512m" + # - name: + # valueFrom: + # secretKeyRef: + # name: secret-name + # key: secret-key + # -- Custom environment variables by sourcing entire configMap, Secret, etc. for relay nodes + extraEnvFrom: + # - configMapRef: + # name: proxy-settings + # - secretRef: + # name: mysecret + # Service configuration + service: + # -- Create a service for node + enabled: false + # -- Service type + type: ClusterIP + # -- Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) + loadBalancerIP: "" + # -- Extra ports exposed in node service + ports: + # - name: vnc-port + # port: 5900 + # targetPort: 5900 + # -- Custom annotations for service + annotations: {} + # -- Size limit for DSH volume mounted in container (if not set, default is disabled, e.g "1Gi") + dshmVolumeSizeLimit: "" + # -- Priority class name for relay-node pods + priorityClassName: "" + + # -- Startup probe settings + startupProbe: + enabled: true + path: /status + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 60 + failureThreshold: 12 + successThreshold: 1 + + # -- Readiness probe settings + readinessProbe: + enabled: false + path: /status + initialDelaySeconds: 10 + failureThreshold: 10 + timeoutSeconds: 10 + periodSeconds: 10 + successThreshold: 1 + + # -- Liveness probe settings + livenessProbe: + enabled: false + path: /status + initialDelaySeconds: 30 + failureThreshold: 6 + timeoutSeconds: 60 + periodSeconds: 10 + successThreshold: 1 + + # -- Time to wait for pod termination + terminationGracePeriodSeconds: 30 + # -- Define preStop command to shut down the relay node gracefully. This overwrites autoscaling.deregisterLifecycle + deregisterLifecycle: + # -- Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any + lifecycle: {} + # -- Extra volume mounts for relay-node container + extraVolumeMounts: [] + # - name: my-extra-volume + # mountPath: /home/seluser/Downloads + + # -- Extra volumes for relay-node pod + extraVolumes: [] + # - name: my-extra-volume + # emptyDir: {} + # - name: my-extra-volume-from-pvc + # persistentVolumeClaim: + # claimName: my-pv-claim + + # -- Override the number of max sessions per node + nodeMaxSessions: + # -- Override the scaled options for relay nodes + scaledOptions: + # -- Override the scaledJobOptions for relay nodes + scaledJobOptions: + # -- Override the scaledObjectOptions for relay nodes + scaledObjectOptions: + hpa: + # -- browserName from the capability + browserName: "chrome" + # -- sessionBrowserName if the browserName is different from the sessionBrowserName + sessionBrowserName: "" + # -- platformName from the capability + platformName: "Android" + # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid. + # -- Skip check SSL when connecting to the Graphql endpoint + unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional + + # -- It is used to add initContainers in the same pod of the browser node. + # It should be set using the --set-json option + initContainers: [] + + # -- It is used to add sidecars proxy in the same pod of the browser node. + # It means it will add a new container to the deployment itself. + # It should be set using the --set-json option + sidecars: [] + # -- Override specific video recording settings for edge node + videoRecorder: {} + # Video recording configuration for all browser nodes. Can be overridden by each browser node videoRecorder: # -- Enable video recording in all browser nodes diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index 7db079d53..bb8a8ffcf 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -3,6 +3,7 @@ import os import traceback import time +import random from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait @@ -264,12 +265,15 @@ def run(self, test_classes): futures = [] tests = [] start_times = {} + mixed_tests = [] for test_class in test_classes: suite = unittest.TestLoader().loadTestsFromTestCase(test_class) - for test in suite: - start_times[test] = time.time() - futures.append(executor.submit(test)) - tests.append(test) + mixed_tests.extend(suite) + random.shuffle(mixed_tests) + for test in mixed_tests: + start_times[test] = time.time() + futures.append(executor.submit(test)) + tests.append(test) print(f"Number of tests were added to worker: {len(tests)}") failed_tests = [] for future, test in zip(concurrent.futures.as_completed(futures), tests): diff --git a/tests/docker-compose-v3-test-node-relay.yml b/tests/docker-compose-v3-test-node-relay.yml index 0e736a23e..d0c32a080 100644 --- a/tests/docker-compose-v3-test-node-relay.yml +++ b/tests/docker-compose-v3-test-node-relay.yml @@ -8,6 +8,7 @@ services: volumes: - ./videos/relay_config.toml:/opt/selenium/config.toml environment: + - SE_ENABLE_TRACING=false - SE_EVENT_BUS_HOST=selenium-hub - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 @@ -18,12 +19,14 @@ services: image: ${NAMESPACE}/standalone-${BROWSER}:${TAG} shm_size: 2gb environment: + - SE_ENABLE_TRACING=false - SE_OPTS=--enable-cdp true - SE_NODE_ENABLE_CDP=true selenium-hub: image: ${NAMESPACE}/hub:${TAG} environment: + - SE_ENABLE_TRACING=false - SE_LOG_LEVEL=${LOG_LEVEL} - SE_SESSION_REQUEST_TIMEOUT=${REQUEST_TIMEOUT} ports: @@ -57,6 +60,7 @@ services: - selenium-hub - emulator environment: + - SE_ENABLE_TRACING=false - SE_EVENT_BUS_HOST=selenium-hub - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443