From 450f91cab82175794694536a1f94909a33a5d672 Mon Sep 17 00:00:00 2001 From: Chaoyi Sun Date: Wed, 16 Oct 2024 15:18:48 -0700 Subject: [PATCH] #403 pipeline-invocation-service java17 upgrade --- DRAFT_RELEASE_NOTES.md | 16 + build-parent/pom.xml | 1 + .../src/main/resources/docker/Dockerfile | 2 +- ...parkoperator.k8s.io_sparkapplications.yaml | 4427 ----------------- .../aissemble-spark-operator-chart/README.md | 15 - .../values.template.yaml | 12 +- .../README.md | 23 +- .../templates/.gitkeep | 0 .../templates/rbac.yaml | 24 +- .../templates/serviceaccount.yaml | 5 + .../tests/rbac_test.yaml | 74 + .../tests/serviceaccount_test.yaml | 19 + .../values.yaml | 4 +- .../pom.xml | 2 +- extensions/extensions-helm/pom.xml | 3 +- .../pom.xml | 14 +- .../service/PipelineInvocationAgent.java | 2 +- .../service/endpoint/HttpEndpoint.java | 2 +- .../service/endpoint/MessageEndpoint.java | 32 +- .../util/exec/HelmCommandExecutor.java | 2 +- .../java-pipeline-debug-values.yaml | 2 +- .../artifacts-maven-plugin/pom.xml | 1 - .../mda-maven-plugin/pom.xml | 1 - foundation/foundation-mda/pom.xml | 2 +- .../deployment/argocd/spark-operator.yaml.vm | 20 +- .../pom.xml | 4 +- test/test-mda-models/pom.xml | 2 +- .../pom.xml | 2 +- .../test-data-delivery-spark-model/pom.xml | 2 +- 29 files changed, 199 insertions(+), 4516 deletions(-) delete mode 100644 extensions/extensions-helm/aissemble-spark-application-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml delete mode 100644 extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/.gitkeep rename extensions/extensions-helm/{aissemble-spark-operator-chart => extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart}/templates/rbac.yaml (74%) create mode 100644 extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/serviceaccount.yaml create mode 100644 extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/rbac_test.yaml create mode 100644 extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/serviceaccount_test.yaml diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md index 3d60110b6..289ad16ed 100644 --- a/DRAFT_RELEASE_NOTES.md +++ b/DRAFT_RELEASE_NOTES.md @@ -71,8 +71,24 @@ To start your aiSSEMBLE upgrade, update your project's pom.xml to use the 1.10.0 ``` +### Split Data Records for the Spark Pipeline +If your spark pipeline is using `aissemble-data-records-separate-module` profile for your data records, you must add the `` tag for +the `jackson-mapper-asl` dependency artifact in the root pom.xml file to enable the build. +```xml + + org.codehaus.jackson + jackson-mapper-asl + + ${version.jackson.mapper.asl} + +``` + + ## Conditional Steps +### For projects that have customized the Spark Operator Service Account permissions +The service account for the pipeline invocation service is now separated from spark operator and configured solely for the service. +If you added any custom configurations to the `sparkoperator` service account pertaining to the pipeline invocation service, you will need to migrate the related changes to the new `pipeline-invocation-service-sa`. Refer to Pipeline Invocation Helm Chart [README](https://github.com/boozallen/aissemble/blob/dev/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md) for detail. + ## Final Steps - Required for All Projects ### Finalizing the Upgrade 1. Run `./mvnw org.technologybrewery.baton:baton-maven-plugin:baton-migrate` to apply the automatic migrations diff --git a/build-parent/pom.xml b/build-parent/pom.xml index 180281ff3..4a73a8df7 100644 --- a/build-parent/pom.xml +++ b/build-parent/pom.xml @@ -67,6 +67,7 @@ 4.0.3 3.5.1 1.9.3 + 3.4.1 4.0.1 diff --git a/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile index 09eac4ecf..5d99ae2bd 100644 --- a/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi9/openjdk-11-runtime:1.20 AS builder +FROM registry.access.redhat.com/ubi9/openjdk-17-runtime:1.20 AS builder USER root RUN microdnf install -y openssl gzip && \ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash diff --git a/extensions/extensions-helm/aissemble-spark-application-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml b/extensions/extensions-helm/aissemble-spark-application-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml deleted file mode 100644 index d34353275..000000000 --- a/extensions/extensions-helm/aissemble-spark-application-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml +++ /dev/null @@ -1,4427 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - api-approved.kubernetes.io: https://github.com/kubeflow/spark-operator/pull/1298 - "helm.sh/hook": pre-install - name: sparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: SparkApplication - listKind: SparkApplicationList - plural: sparkapplications - shortNames: - - sparkapp - singular: sparkapplication - scope: Namespaced - versions: - - name: v1beta2 - served: true - storage: true - subresources: - status: { } - additionalPrinterColumns: - - jsonPath: .status.applicationState.state - name: Status - type: string - - jsonPath: .status.executionAttempts - name: Attempts - type: string - - jsonPath: .status.lastSubmissionAttemptTime - name: Start - type: string - - jsonPath: .status.terminationTime - name: Finish - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - deps: - properties: - excludePackages: - items: - type: string - type: array - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - packages: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - repositories: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostAliases: - items: - properties: - hostnames: - items: - type: string - type: array - ip: - type: string - type: object - type: array - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - kubernetesMaster: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - podSecurityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - serviceAccount: - type: string - serviceAnnotations: - additionalProperties: - type: string - type: object - shareProcessNamespace: - type: boolean - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - dynamicAllocation: - properties: - enabled: - type: boolean - initialExecutors: - format: int32 - type: integer - maxExecutors: - format: int32 - type: integer - minExecutors: - format: int32 - type: integer - shuffleTrackingTimeout: - format: int64 - type: integer - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostAliases: - items: - properties: - hostnames: - items: - type: string - type: array - ip: - type: string - type: object - type: array - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podSecurityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - serviceAccount: - type: string - shareProcessNamespace: - type: boolean - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - portName: - type: string - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - proxyUser: - type: string - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkUIOptions: - properties: - serviceAnnotations: - additionalProperties: - type: string - type: object - ingressAnnotations: - additionalProperties: - type: string - type: object - ingressTLS: - items: - properties: - hosts: - items: - type: string - type: array - secretName: - type: string - type: object - type: array - servicePort: - format: int32 - type: integer - servicePortName: - type: string - serviceType: - type: string - type: object - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - csi: - properties: - driver: - type: string - fsType: - type: string - nodePublishSecretRef: - properties: - name: - type: string - type: object - readOnly: - type: boolean - volumeAttributes: - additionalProperties: - type: string - type: object - required: - - driver - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - spec: - properties: - accessModes: - items: - type: string - type: array - resources: - properties: - requests: - properties: - storage: - type: string - type: object - type: object - storageClassName: - type: string - type: object - type: object - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - tenant: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - status: - properties: - applicationState: - properties: - errorMessage: - type: string - state: - type: string - required: - - state - type: object - driverInfo: - properties: - podName: - type: string - webUIAddress: - type: string - webUIIngressAddress: - type: string - webUIIngressName: - type: string - webUIPort: - format: int32 - type: integer - webUIServiceName: - type: string - type: object - executionAttempts: - format: int32 - type: integer - executorState: - additionalProperties: - type: string - type: object - lastSubmissionAttemptTime: - format: date-time - nullable: true - type: string - sparkApplicationId: - type: string - submissionAttempts: - format: int32 - type: integer - submissionID: - type: string - terminationTime: - format: date-time - nullable: true - type: string - required: - - driverInfo - type: object - required: - - metadata - - spec - type: object -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/README.md b/extensions/extensions-helm/aissemble-spark-operator-chart/README.md index 96dd29a76..adcdd7fb5 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/README.md +++ b/extensions/extensions-helm/aissemble-spark-operator-chart/README.md @@ -45,25 +45,10 @@ aissemble-spark-operator-chart: | volumes | Volumes for the pod | No | `spark-logging=/tmp/spark-logging`, `ivy-cache=/home/spark/.ivy2` | | volumeMounts | Volume Mounts for the pod | No | `spark-logging=/tmp/spark-logging`, `ivy-cache=/home/spark/.ivy2` | | fullnameOverride | String to override release name | No | spark-operator | -| rbac.createClusterRole | See `Migrated Properties` | No | false | | serviceAccounts.spark.name | Name for the spark service account | No | spark | | serviceAccounts.sparkoperator.name | Name for the spark service account | No | sparkoperator | | podSecurityContext | Pod security context | No | runAsUser: 185
runAsGroup: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch" | -## Migrated Properties -The following properties have been migrated from the `spark-operator` subchart to the `aissemble-spark-operator-chart` chart. -Any required overrides should be cognisant of the alternate path. For example: - -```yaml -aissemble-spark-operator-chart: - rbac: - createClusterRole: false -``` - -| Property | Description | Default | -|------------------------|-------------------------------------------------------------------------------|---------| -| rbac.createClusterRole | Create and use RBAC `ClusterRole` resources. Migrated to use modified rules. | true | - # Shared Ivy Cache Spark uses [Ivy](https://ant.apache.org/ivy/) to resolve and download dependencies for Spark applications. By default, diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml b/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml index b56db396d..17a671f76 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml +++ b/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml @@ -29,12 +29,6 @@ spark-operator: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: "OnRootMismatch" - - rbac: - # -- Create and use RBAC `ClusterRole` resources - # -- Set to false in order to enable overriding with our own RBAC template - createClusterRole: false - # volumes - Operator volumes volumes: - name: spark-logging @@ -61,8 +55,4 @@ spark-operator: sparkoperator: # -- Optional name for the operator service account - name: "sparkoperator" - -rbac: - # -- Set to True in order to enable overriding with our own RBAC template - createClusterRole: True \ No newline at end of file + name: "sparkoperator" \ No newline at end of file diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md index 443a7850c..92a32b3a4 100644 --- a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md @@ -10,16 +10,17 @@ helm install pipeline-invocation-service oci://ghcr.io/boozallen/aissemble-pipel **Note**: *the version should match the aiSSEMBLE project version.* # Properties -| Property | Description | Required Override | Default | -|------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------| -| ingress.apiVersion | k8s API version to use | No | networking.k8s.io/v1 | -| ingress.enabled | k8s Whether to enable ingress | No | false | -| ingress.kind | Type of kubernetes entity | No | Ingress | -| ingress.metadata.name | Name of the ingress | No | pipeline-invocation-service-web | -| ingress.metadata.annotations.kubernetes.io/ingress.class | Ingress class name | No | nginx | -| ingress.metadata.annotations.ingress.metadata.annotations.nginx.ingress.kubernetes.io/server-snippet | Custom configurations for the nginx ingress class | No | gunzip on; gzip on; gzip_proxied any; gzip_types *; | -| ingress.spec.rules.hosts | A list of hosts for ingress to support, each with their own path definition | No | | -| ingress.status | Load balancer IP if required | No | None | +| Property | Description | Required Override | Default | +|-------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------| +| ingress.apiVersion | k8s API version to use | No | networking.k8s.io/v1 | +| ingress.enabled | k8s Whether to enable ingress | No | false | +| ingress.kind | Type of kubernetes entity | No | Ingress | +| ingress.metadata.name | Name of the ingress | No | pipeline-invocation-service-web | +| ingress.metadata.annotations.kubernetes.io/ingress.class | Ingress class name | No | nginx | +| ingress.metadata.annotations.nginx.ingress.kubernetes.io/server-snippet | Custom configurations for the nginx ingress class | No | gunzip on; gzip on; gzip_proxied any; gzip_types *; | +| ingress.spec.rules.hosts | A list of hosts for ingress to support, each with their own path definition | No | | +| ingress.status | Load balancer IP if required | No | None | +| rbac.createClusterRole | Create and use RBAC `ClusterRole` resources. | No | true | # Quarkus Configuration @@ -30,3 +31,5 @@ The following configuration of the service is provided. Additional configuratio |---------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------| | kafka.bootstrap.servers | Specifies the kafka bootstrap server when using kafka for messaging | Any valid URI | | mp.messaging.incoming.pipeline-invocation.* | Specifies and configures the smallrye connector to use. Supported connectors are `smallrye-amqp`, `smallrye-kafka`, `smallrye-mqtt`, and `smallrye-rabbitmq` | See xref:messaging-details.adoc[the Messaging documentation] for more details | + + diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/.gitkeep b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/rbac.yaml similarity index 74% rename from extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml rename to extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/rbac.yaml index 14d8d2746..6dc9bd9c2 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/rbac.yaml @@ -1,24 +1,22 @@ {{- /* aiSSEMBLE Custom rbac.yaml -Required custom rbac.yaml file that grants the sparkoperator service account +Required custom rbac.yaml file that grants the pipeline-invocation-service service account create, delete, and update access to the apigroup apiextensions.k8s.io. This is necessary for the pipeline-invocation-service to create instances of the SparkApplication CRD to submit pipelines to the Spark Operator for execution. */}} - -{{- if or .Values.rbac.create .Values.rbac.createClusterRole }} +{{ if .Values.rbac.createClusterRole }} +{{- $serviceAccountName := (index .Values "aissemble-quarkus-chart" "deployment" "serviceAccountName") -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "spark-operator.fullname" (index .Subcharts "spark-operator") }} + name: {{ $serviceAccountName | default "pipeline-invocation-service" }}-clusterrole annotations: "helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-delete-policy": hook-failed, before-hook-creation "helm.sh/hook-weight": "-10" - labels: - {{- include "spark-operator.labels" (index .Subcharts "spark-operator") | nindent 4 }} rules: - apiGroups: - "" @@ -34,6 +32,7 @@ rules: - configmaps - secrets verbs: + - list - create - get - delete @@ -75,6 +74,7 @@ rules: resources: - customresourcedefinitions verbs: + - create - get - apiGroups: - admissionregistration.k8s.io @@ -97,7 +97,6 @@ rules: - scheduledsparkapplications/finalizers verbs: - "*" - {{- if .Values.batchScheduler.enable }} # required for the `volcano` batch scheduler - apiGroups: - scheduling.incubator.k8s.io @@ -107,34 +106,29 @@ rules: - podgroups verbs: - "*" - {{- end }} - {{ if .Values.webhook.enable }} - apiGroups: - batch resources: - jobs verbs: - delete - {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "spark-operator.fullname" (index .Subcharts "spark-operator") }} + name: {{ $serviceAccountName | default "pipeline-invocation-service" }}-clusterrole-binding annotations: "helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-delete-policy": hook-failed, before-hook-creation "helm.sh/hook-weight": "-10" - labels: - {{- include "spark-operator.labels" (index .Subcharts "spark-operator") | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ include "spark-operator.serviceAccountName" (index .Subcharts "spark-operator") }} + name: {{ $serviceAccountName | default "pipeline-invocation-service" }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: {{ include "spark-operator.fullname" (index .Subcharts "spark-operator") }} + name: {{ $serviceAccountName | default "pipeline-invocation-service" }}-clusterrole apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/serviceaccount.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/serviceaccount.yaml new file mode 100644 index 000000000..a3c66e8bf --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/serviceaccount.yaml @@ -0,0 +1,5 @@ +{{- $serviceAccountName := (index .Values "aissemble-quarkus-chart" "deployment" "serviceAccountName") -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $serviceAccountName | default "pipeline-invocation-service" }} \ No newline at end of file diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/rbac_test.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/rbac_test.yaml new file mode 100644 index 000000000..fea9cc44f --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/rbac_test.yaml @@ -0,0 +1,74 @@ +suite: Pipeline Invocation Service RBAC +templates: + - rbac.yaml +tests: + - it: Should contain ClusterRole document + documentIndex: 0 + asserts: + - containsDocument: + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + - it: Should contain ClusterRoleBinding document + documentIndex: 1 + asserts: + - containsDocument: + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + - it: Should be 2 documents in total + asserts: + - hasDocuments: + count: 2 + - it: Do not contain any documents if options are disabled + set: + rbac: + createClusterRole: false + asserts: + - hasDocuments: + count: 0 + - it: ClusterRole should include appropriate default values + documentIndex: 0 + asserts: + - equal: + path: metadata.name + value: pipeline-invocation-service-sa-clusterrole + - it: ClusterRoleBinding should include appropriate default values + documentIndex: 1 + release: + namespace: default + asserts: + - equal: + path: metadata.name + value: pipeline-invocation-service-sa-clusterrole-binding + - contains: + path: subjects + content: + kind: ServiceAccount + name: pipeline-invocation-service-sa + namespace: default + - equal: + path: roleRef.kind + value: ClusterRole + - equal: + path: roleRef.name + value: pipeline-invocation-service-sa-clusterrole + - equal: + path: roleRef.apiGroup + value: rbac.authorization.k8s.io + - it: Should set values appropriately for the cluster role binding + set: + aissemble-quarkus-chart: + deployment: + serviceAccountName: test + release: + namespace: default + documentIndex: 1 + asserts: + - contains: + path: subjects + content: + kind: ServiceAccount + name: test + namespace: default + - equal: + path: metadata.name + value: test-clusterrole-binding \ No newline at end of file diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/serviceaccount_test.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/serviceaccount_test.yaml new file mode 100644 index 000000000..6a62aab5f --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/serviceaccount_test.yaml @@ -0,0 +1,19 @@ +suite: Pipeline Invocation Service Account +templates: + - serviceaccount.yaml +tests: + - it: ServiceAccount should include appropriate default values + asserts: + - equal: + path: metadata.name + value: pipeline-invocation-service-sa + + - it: Should set values appropriately for the service account + set: + aissemble-quarkus-chart: + deployment: + serviceAccountName: test + asserts: + - equal: + path: metadata.name + value: test diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml index 370b2eafc..7969dd6f2 100644 --- a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml @@ -12,7 +12,7 @@ aissemble-quarkus-chart: containerPort: 9000 protocol: TCP restartPolicy: Always - serviceAccountName: sparkoperator + serviceAccountName: pipeline-invocation-service-sa automountServiceAccountToken: true supplementalVolumeMounts: @@ -60,3 +60,5 @@ aissemble-quarkus-chart: name: pipeline-invocation-service port: number: 8080 +rbac: + createClusterRole: true diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml index 060812070..6f2f706b5 100644 --- a/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml @@ -23,7 +23,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} run tests diff --git a/extensions/extensions-helm/pom.xml b/extensions/extensions-helm/pom.xml index d9a437361..286d51b8e 100644 --- a/extensions/extensions-helm/pom.xml +++ b/extensions/extensions-helm/pom.xml @@ -199,7 +199,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} run tests @@ -250,7 +250,6 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 run tests diff --git a/extensions/extensions-pipeline-invocation-service/pom.xml b/extensions/extensions-pipeline-invocation-service/pom.xml index b4d84ddc8..5e5e0eeaa 100644 --- a/extensions/extensions-pipeline-invocation-service/pom.xml +++ b/extensions/extensions-pipeline-invocation-service/pom.xml @@ -41,18 +41,6 @@ io.quarkus quarkus-kubernetes-client - - io.quarkus - quarkus-smallrye-fault-tolerance - - - io.quarkus - quarkus-smallrye-metrics - - - io.quarkus - quarkus-smallrye-health - io.quarkus quarkus-smallrye-reactive-messaging-kafka @@ -83,7 +71,7 @@ io.quarkus - quarkus-rest-client-reactive-jackson + quarkus-resteasy-reactive-jackson org.apache.commons diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java index dd9a2b406..60799cbad 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java @@ -101,7 +101,7 @@ protected List buildFinalValuesOverrides(PipelineInvocationRequest reque List args = new ArrayList<>(); args.add("--set"); - args.add("spec.serviceEnabled=false"); + args.add("service.enabled=false"); for(Map.Entry entry : request.getOverrideValues().entrySet()) { args.add("--set"); diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java index ab7a933a5..01e7a0cea 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java @@ -30,7 +30,7 @@ @Path("/invoke-pipeline") @ApplicationScoped public class HttpEndpoint { - private static final Logger logger = Logger.getLogger(MessageEndpoint.class); + private static final Logger logger = Logger.getLogger(HttpEndpoint.class); @Inject private PipelineInvocationAgent pipelineInvocationAgent; diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java index 95dfce20d..1e8f51991 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java @@ -18,6 +18,10 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import io.quarkus.runtime.Shutdown; /** * Entrypoint for message-based requests for SparkApplication submission. @@ -27,6 +31,7 @@ public class MessageEndpoint { public static final String CHANNEL_NAME = "pipeline-invocation"; private static final Logger logger = Logger.getLogger(MessageEndpoint.class); + private final ExecutorService executorService = Executors.newFixedThreadPool(16); @Inject PipelineInvocationAgent pipelineInvocationAgent; @@ -38,13 +43,28 @@ public class MessageEndpoint { @Incoming(CHANNEL_NAME) @Acknowledgment(Acknowledgment.Strategy.PRE_PROCESSING) public void receivePipelineRequest(String rawRequest) { + // Running the helm command asynchronously in a separated thread to avoid vertx event loop thread block issue + executorService.execute(() -> { + try { + PipelineInvocationRequest pipelineInvocationRequest = PipelineInvocationRequest.fromString(rawRequest); + logger.info("Received message request to submit " + pipelineInvocationRequest.getApplicationName() + "."); + pipelineInvocationAgent.submitSparkApplication(pipelineInvocationRequest); + logger.info("Submitted " + pipelineInvocationRequest.getApplicationName() + " for processing."); + } catch (Exception e) { + throw new RuntimeException("Fail executing pipeline invocation command", e); + } + }); + } + + @Shutdown + public void shutdown() { + executorService.shutdown(); try { - PipelineInvocationRequest pipelineInvocationRequest = PipelineInvocationRequest.fromString(rawRequest); - logger.info("Received message request to submit " + pipelineInvocationRequest.getApplicationName() + "."); - pipelineInvocationAgent.submitSparkApplication(pipelineInvocationRequest); - logger.info("Submitted " + pipelineInvocationRequest.getApplicationName() + " for processing."); - } catch (Exception failsafe) { - throw new RuntimeException(failsafe); + if (!executorService.awaitTermination(1500, TimeUnit.MILLISECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); } } } diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java index 959b2582f..4e06c6d60 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java @@ -68,7 +68,7 @@ private HelmCommandExecutor( * @throws ShellExecutionException */ public int executeAndLogOutput(List arguments) throws ShellExecutionException { - logger.error("Executing Helm command: {} {}", HELM_COMMAND, StringUtils.join(arguments, " ")); + logger.info("Executing Helm command: {} {}", HELM_COMMAND, StringUtils.join(arguments, " ")); ProcessExecutor executor = createHelmExecutor(arguments); return executor.executeAndRedirectOutput(logger); diff --git a/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml b/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml index 46ef25e3c..06890e14f 100644 --- a/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml +++ b/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml @@ -1,5 +1,4 @@ spec: - serviceEnabled: "true" driver: javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:4747" executor: @@ -7,6 +6,7 @@ spec: service: + enabled: true spec: ports: - name: "debug" diff --git a/foundation/foundation-maven-plugins/artifacts-maven-plugin/pom.xml b/foundation/foundation-maven-plugins/artifacts-maven-plugin/pom.xml index d89733ead..59785ec63 100644 --- a/foundation/foundation-maven-plugins/artifacts-maven-plugin/pom.xml +++ b/foundation/foundation-maven-plugins/artifacts-maven-plugin/pom.xml @@ -15,7 +15,6 @@ 3.3.0 - 3.1.0 7.17.0 1.1.0 diff --git a/foundation/foundation-maven-plugins/mda-maven-plugin/pom.xml b/foundation/foundation-maven-plugins/mda-maven-plugin/pom.xml index f25ab75f1..e9af50a79 100644 --- a/foundation/foundation-maven-plugins/mda-maven-plugin/pom.xml +++ b/foundation/foundation-maven-plugins/mda-maven-plugin/pom.xml @@ -20,7 +20,6 @@ 2.4.0 3.3.0 - 3.1.0 1.1.0 diff --git a/foundation/foundation-mda/pom.xml b/foundation/foundation-mda/pom.xml index 3bf9e0e72..5d06bc2b1 100644 --- a/foundation/foundation-mda/pom.xml +++ b/foundation/foundation-mda/pom.xml @@ -47,7 +47,7 @@ org.codehaus.mojo exec-maven-plugin - 3.4.1 + ${version.exec.maven.plugin} create-generator-usage-comments diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm index 32f4fc8be..b4f67a00a 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm @@ -23,5 +23,21 @@ spec: {{ end }} {{- with .Values.spec.syncPolicy }} syncPolicy: - {{- toYaml . | nindent 4 }} - {{- end }} \ No newline at end of file + {{- with .Values.spec.syncPolicy }} + {{- if .automated }} + automated: + {{- toYaml .automated | nindent 6}} + {{- else if hasKey . "automated" }} + automated: {} + {{- end }} + {{- end }} + {{/* This template comment explains how the logic below works*/}} + {{- /* This one trims the newlines so it doesn't affect output */ -}} + #these options are important + syncOptions: + - ServerSideApply=true # https://github.com/argoproj/argo-cd/issues/820#issuecomment-135463693 + {{- if .Values.spec.syncPolicy }} + {{- with .Values.spec.syncPolicy.syncOptions }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml index 662353df3..d4e4ed2a6 100644 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml +++ b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml @@ -96,7 +96,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart @@ -196,7 +196,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} ensure-docker-compose-installed diff --git a/test/test-mda-models/pom.xml b/test/test-mda-models/pom.xml index de7a6cbc9..ce56bf6b7 100644 --- a/test/test-mda-models/pom.xml +++ b/test/test-mda-models/pom.xml @@ -97,7 +97,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart diff --git a/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml b/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml index 061bb011e..e27ffb2c9 100644 --- a/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml +++ b/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml @@ -68,7 +68,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart diff --git a/test/test-mda-models/test-data-delivery-spark-model/pom.xml b/test/test-mda-models/test-data-delivery-spark-model/pom.xml index c66f09c98..fe205866e 100644 --- a/test/test-mda-models/test-data-delivery-spark-model/pom.xml +++ b/test/test-mda-models/test-data-delivery-spark-model/pom.xml @@ -77,7 +77,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart