diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/README.md b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/README.md index 79eb97b10..491bb5a83 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/README.md +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/README.md @@ -4,7 +4,7 @@ Baseline Helm chart for deploying the Spark History Server. Chart is built and m # Basic usage with Helm CLI To use the module, perform [extension-helm setup](../README.md#leveraging-extensions-helm) and override the chart version with the desired aiSSEMBLE version. For example: ```bash -helm install spark-operator ghcr.io/boozallen/aissemble-spark-history-chart --version +helm install spark-history ghcr.io/boozallen/aissemble-spark-history-chart --version ``` _**NOTE:**_ *the version should match the aiSSEMBLE project version.* diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/pom.xml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/pom.xml index 40361e655..df03ea8c4 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/pom.xml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/pom.xml @@ -18,14 +18,6 @@ ${group.helm.plugin} helm-maven-plugin - - - - - ${project.version} - - - org.codehaus.mojo @@ -38,4 +30,4 @@ - \ No newline at end of file + diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/configmap.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/configmap.yaml index 06f231717..89b0567f3 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/configmap.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Release.Name }}-spark-conf + name: {{ .Release.Name }}-shs-spark-conf data: sparkConf: {{- toYaml .Values.sparkConf | nindent 8 }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/deployment.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/deployment.yaml index 7bdbb31c7..18ea6e8d4 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/deployment.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Release.Name }} + name: {{ .Release.Name }}-shs {{- if .Values.deployment.annotations }} annotations: {{ toYaml .Values.deployment.annotations | default "" }} @@ -112,7 +112,7 @@ spec: {{- if not (eq .Values.sparkConf "") }} - name: spark-config configMap: - name: {{ .Release.Name }}-spark-conf + name: {{ .Release.Name }}-shs-spark-conf items: - key: sparkConf path: spark-defaults.conf diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/ingress.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/ingress.yaml index 11f7b583c..25fae4657 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/ingress.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/ingress.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ .Release.Name }}-ingress + name: {{ .Release.Name }}-shs-ingress {{- with .Values.ingress.metadata.annotations }} annotations: {{- toYaml . | nindent 8 }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/service.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/service.yaml index 348163c86..f2b52cf56 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/service.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/templates/service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ .Release.Name }}-service + name: {{ .Release.Name }}-shs-service {{- if .Values.service.annotations }} annotations: {{ .Values.service.annotations }} {{- end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/ingress_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/ingress_test.yaml index 3f7f9bd2a..01665d861 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/ingress_test.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/ingress_test.yaml @@ -39,7 +39,7 @@ tests: asserts: - equal: path: metadata.name - value: RELEASE-NAME-ingress + value: RELEASE-NAME-shs-ingress - notExists: path: metadata.annotations diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/service_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/service_test.yaml index 403b25150..780e0143f 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/service_test.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/service_test.yaml @@ -19,7 +19,7 @@ tests: count: 1 - equal: path: metadata.name - value: RELEASE-NAME-service + value: RELEASE-NAME-shs-service - notExists: path: metadata.annotations - equal: diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/spark_conf_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/spark_conf_test.yaml index 2efc58d7a..e7c78f691 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/spark_conf_test.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/tests/spark_conf_test.yaml @@ -25,7 +25,7 @@ tests: - notFailedTemplate: {} - equal: path: metadata.name - value: "RELEASE-NAME-spark-conf" + value: "RELEASE-NAME-shs-spark-conf" - equal: path: data.sparkConf value: "spark.executor.instances: 2" @@ -41,7 +41,7 @@ tests: content: name: spark-config configMap: - name: RELEASE-NAME-spark-conf + name: RELEASE-NAME-shs-spark-conf items: - key: sparkConf path: spark-defaults.conf diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/.helmignore b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/.helmignore new file mode 100644 index 000000000..66f9f8d3f --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/.helmignore @@ -0,0 +1,3 @@ +target +pom.xml +examples/* diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/Chart.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/Chart.yaml new file mode 100644 index 000000000..b8bd4a4d9 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/Chart.yaml @@ -0,0 +1,11 @@ +--- +name: aissemble-thrift-server-chart +# Version is automatically set by the plugin so this is only a placeholder +version: 1.0.0 +apiVersion: v2 +description: Baseline Spark History chart -- see the extensions-helm directory in aiSSEMBLE for more details. +# App Version is automatically set by the plugin so this is only a placeholder +appVersion: 1.0.0 +sources: + - https://github.com/boozallen/aissemble +dependencies: [] diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/LICENSE.txt b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/LICENSE.txt new file mode 100644 index 000000000..a9bd80cda --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/LICENSE.txt @@ -0,0 +1,49 @@ +Booz Allen Public License v1.0 +------------------------------ + +INTRODUCTION +The Booz Allen Public License allows government, non-profit academic, other non-profit, and commercial entities access to distinctive, disruptive, and robust code with the goal of Empowering People to Change the World(SM). Products licensed under the Booz Allen Public License are founded on the basis that collective ingenuity can make the largest impact in the community. + +DEFINITIONS +* **Commercial Entity.** “Commercial Entity” means any individual or entity other than a government, non-profit academic, or other non-profit entity. +* **Derivative.** “Derivative” means any work of authorship in Source Code or Object Code form that results from an addition to, deletion from, or modification of the Source Code of the Product. +* **License.** “License” means this Booz Allen Public License. +* **Object Code.** “Object Code” means the form resulting from transformation or translation of Source Code into machine readable code, including but not limited to, compiled object code. +* **Originator.** “Originator” means each individual or legal entity that creates, contributes to the creation of, or owns the Product. +* **Patent Claims.** “Patent Claims” means any patent claim(s) in any patent to which Originator has a right to grant a license that would be infringed by Your making, using, selling, offering for sale, having made, or importing of the Product, but for the grant of this License. +* **Product.** “Product” means the Source Code of the software which the initial Originator made available under this License, and any Derivative of such Source Code. +* **Source Code.** “Source Code” means software in human-readable form. +* **You.** “You” means either an individual or an entity (if you are taking this license on behalf of an entity) that exercises the rights granted under this License. + +LICENSE +**Government/Non-Profit Academic/Other Non-Profit.** +This Section applies if You are not a Commercial Entity. + +* **License.** Subject to the terms and conditions of this License, each Originator hereby grants You a perpetual, worldwide, non-exclusive, royalty-free license to reproduce, display, perform, modify, distribute and otherwise use the Product and Derivatives, in Source Code and Object Code form, in accordance with the terms and conditions of this License in order to support the general public good and for your internal business purposes. +* **Distribution.** You may distribute to third parties copies of the Product, including any Derivative that You create, in Source Code or Object Code form. If You distribute copies of the Product, including any Derivative that You create, in Source Code form, such distribution must be under the terms of this License and You must inform recipients of the Source Code that the Product is governed under this License and how they can obtain a copy of this License. You may distribute to third parties copies of the Product, including any Derivative that You create, in Object Code form, or allow third parties to access or use the Product, including any Derivative that You create, under a license of Your choice. +* **Commercial Sales.** You may not distribute, or allow third parties to access or use, the Product or any Derivative for a fee, unless You first obtain permission from the Originator. If Booz Allen Hamilton is the Originator, please contact Booz Allen Hamilton at . + +**Commercial Entities**. +This Section applies if You are a Commercial Entity. + +* **License.** Subject to the terms and conditions of this License, each Originator hereby grants You a perpetual, worldwide, non-exclusive, royalty-free license to reproduce, display, perform, modify, distribute and otherwise use the Product and Derivatives, in Source Code and Object Code form, in accordance with the terms and conditions of this License for the sole purpose of Your internal business purposes and the provision of services to government, non-profit academic, and other non-profit entities. +* **Distribution and Derivatives.** You may distribute to third parties copies of the Product, including any Derivative that You create, in Source Code or Object Code form. If You distribute copies of the Product, including any Derivative that You create, in Source Code form, such distribution must be under the terms of this License and You must inform recipients of the Source Code that the Product is governed under this License and how they can obtain a copy of this License. You may distribute to third parties copies of the Product, including any Derivative that You create, in Object Code form, or allow third parties to access or use the Product, including any Derivative that You create, under a license of Your choice, provided that You make available, and inform the recipient of such distribution how they can obtain, a copy of the Source Code thereof, at no charge, and inform the recipient of the Source Code that the Product is governed under this License and how they can obtain a copy of this License. +* **Commercial Sales.** You may not distribute, or allow third parties to access or use, the Product or any Derivative for a fee, unless You first obtain permission from the Originator. If Booz Allen Hamilton, please contact Booz Allen Hamilton at . + + +**Patent Claim(s)**. +This Section applies regardless of whether You are a government, non-profit academic, or other non-profit entity or a Commercial Entity. + +* **Patent License.** Subject to the limitations in the Sections above, each Originator hereby grants You a perpetual, worldwide, non-exclusive, royalty-free license under Patent Claims of such Originator to make, use, sell, offer for sale, have made, and import the Product. The foregoing patent license does not apply (a) to any code that an Originator has removed from the Product, or (b) for infringement caused by Your modifications of the Product or the combination of any Derivative created by You or on Your behalf with other software. + +GENERAL TERMS +This Section applies regardless of whether You are a government, non-profit academic, or other non-profit entity or a Commercial Entity. + +* **Required Notices.** If You distribute the Product or a Derivative, in Object Code or Source Code form, You shall not remove or otherwise modify any proprietary markings or notices contained within or placed upon the Product or any Derivative. Any distribution of the Product or a Derivative, in Object Code or Source Code form, shall contain a clear and conspicuous Originator copyright and license reference in accordance with the below: + * *Unmodified Product Notice*: “This software package is licensed under the Booz Allen Public License. Copyright © 20__ [Copyright Holder Name]. All Rights Reserved.” + * *Derivative Notice*: “This software package is licensed under the Booz Allen Public License. Portions of this code are Copyright © 20__ [Copyright Holder Name]. All Rights Reserved.” +* **Compliance with Laws.** You agree that You shall not reproduce, display, perform, modify, distribute and otherwise use the Product in any way that violates applicable law or regulation or infringes or violates the rights of others, including, but not limited to, third party intellectual property, privacy, and publicity rights. +* **Disclaimer.** You understand that the Product is licensed to You, and not sold. The Product is provided on an “As Is” basis, without any warranties, representations, and guarantees, whether oral or written, express, implied or statutory, with regard to the Product, including without limitation, warranties of merchantability, fitness for a particular purpose, title, non-infringement, non-interference, and warranties arising from course of dealing or usage of trade, to the maximum extent permitted by applicable law. Originator does not warrant that (i) the Product will meet your needs; (ii) the Product will be error-free or accessible at all times; or (iii) the use or the results of the use of the Product will be correct, accurate, timely, or otherwise reliable. You acknowledge that the Product has not been prepared to meet Your individual requirements, whether or not such requirements have been communicated to Originator. You assume all responsibility for use of the Product. +* **Limitation of Liability.** Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Originator, or anyone who distributes the Product in accordance with this License, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if informed of the possibility of such damages. +* **Export Control.** The Product is subject to U.S. export control laws and may be subject to export or import regulations in other countries. You agree to strictly comply with all such laws and regulations and acknowledges that You are responsible for obtaining such licenses to export, re-export, or import as may be required. +* **Severability.** If the application of any provision of this License to any particular facts or circumstances shall be held to be invalid or unenforceable, then the validity and enforceability of other provisions of this License shall not in any way be affected or impaired thereby. diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/README.md b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/README.md new file mode 100644 index 000000000..6d7638433 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/README.md @@ -0,0 +1,37 @@ +# aiSSEMBLE™ Spark Thrift Helm Chart +Baseline Helm chart for deploying the Spark Thrift Server. Chart is built and managed during the normal Maven build lifecycle and placed in the **target/helm/repo** directory. See https://github.com/kokuwaio/helm-maven-plugin for more details. + +# Basic usage with Helm CLI +To use the module, perform [extension-helm setup](../README.md#leveraging-extensions-helm) and override the chart version with the desired aiSSEMBLE version. For example: +```bash +helm install thriftserver ghcr.io/boozallen/aissemble-thrift-server-chart --version +``` +_**NOTE:**_ *the version should match the aiSSEMBLE project version.* + +# Properties + +| Property | Description | Default | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------| +| enable | Enable or disable the entirety of the spark-thrift-server deployment. When false, equivalent to not installing the chart. | true | +| deployment.annotations | Annotations to apply to the Spark Thrift Server Deployment. | {} | +| deployment.labels | Labels to apply to the Spark Thrift Server Deployment. | {} | +| deployment.replicas | Number of replicas for the Spark Thrift Server Deployment. | 1 | +| deployment.image.repository | Repository for the Spark Thrift Server image. | "apache/spark" | +| deployment.image.tag | Tag for the Spark Thrift Server image. | "3.5.1" | +| deployment.image.imagePullPolicy | Image pull policy for the Spark Thrift Server image. | "IfNotPresent" | +| deployment.command | Command to run in the container. | `["/opt/spark/sbin/start-thriftserver.sh"]` | +| deployment.env | Environment variables to set in the Spark Thrift Server Deployment. | `SPARK_NO_DAEMONIZE: "true"` | +| deployment.envFromSecret | Environment variables to pull from a Secret. Format:
`ENV_VAR.secretName: k8s_secret_name`
`ENV_VAR.key: k8s_secret_key` | {} | +| deployment.volumes | Volumes to attach to the Spark Thrift Server Deployment. | [] | +| deployment.volumeMounts | Volume mounts to attach to the Spark Thrift Server Deployment. | [] | +| dependencies.packages | List of packages to install in the Spark Thrift Server Deployment. | [] | +| dependencies.jars | List of jars to install in the Spark Thrift Server Deployment. | [] | +| ingress.enabled | Enable or disable the Spark Thrift Server Ingress. | false | +| ingress.metadata.annotations | Annotations to apply to the Spark Thrift Server Ingress. | {} | +| ingress.ingressClassName | Ingress class to use for the Spark Thrift Server Ingress. | "nginx" | +| ingress.hosts | Hosts to apply to the Spark Thrift Server Ingress. | `[paths: []]` | +| service.annotations | Annotations to apply to the Spark Thrift Server Service. | {} | +| service.type | Type of service to create for the Spark Thrift Server. | "ClusterIP" | +| service.ports | Name of the service port. | `[{name: "thrift", port: 10000}, {name: "thrift-http", port: 10001}]` | +| sparkConf | Configuration for the Spark Thrift Server. | "" | +| hiveSite | Configuration for the Hive Site. | "" | diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/pom.xml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/pom.xml new file mode 100644 index 000000000..81f3ca635 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + + com.boozallen.aissemble + extensions-helm-spark-infrastructure + 1.7.0-SNAPSHOT + + + aissemble-thrift-server-chart + helm + + aiSSEMBLE::Extensions::Helm::Spark Thrift Server + Contains Helm Spark Thrift Server Chart + + + + + ${group.helm.plugin} + helm-maven-plugin + + + org.codehaus.mojo + exec-maven-plugin + + + org.apache.maven.plugins + maven-clean-plugin + + + + + diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/NOTES.txt b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/NOTES.txt new file mode 100644 index 000000000..87fad52f6 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/NOTES.txt @@ -0,0 +1,9 @@ +************************ +******* Services ******* +************************ + +{{- range .Values.service.ports }} +{{ .name }}: +- To access the {{ $.Release.Name }}-{{ .name }} service within the cluster, use the following URL: + {{ $.Release.Name }}-service.{{ $.Release.Namespace }}.svc.cluster.local:{{ .port }} +{{- end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/_helpers.tpl b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/_helpers.tpl new file mode 100644 index 000000000..ac3225b31 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/_helpers.tpl @@ -0,0 +1,25 @@ +{{/* +Create the list of ivy dependencies to be installed on the spark history server +*/}} +{{- define "deps.packages.ivy" -}} +{{- if .Values.enable -}} + {{- range .Values.dependencies.packages }} + {{- $dep := printf "-dependency, %s," . | replace ":" ", " -}} + {{ default $dep -}} + {{- end }} +{{- else -}} + {{ default "" }} +{{- end -}} +{{- end -}} + +{{/* +Create the list of jars to be downloaded directly to the dependencies volume +*/}} +{{- define "deps.jars" -}} +{{- if .Values.enable -}} + {{- range .Values.dependencies.jars }} + {{- $jar := printf "%s, " . -}} + {{ default $jar -}} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/configmap.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/configmap.yaml new file mode 100644 index 000000000..559ae98a4 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if not (and (eq .Values.sparkConf "") (eq .Values.hiveSite "")) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-spark-conf +data: +{{- if not (eq .Values.sparkConf "") }} + sparkConf: + {{- toYaml .Values.sparkConf | nindent 8 }} +{{- end }} +{{- if not (eq .Values.hiveSite "") }} + hiveSite: + {{- toYaml .Values.hiveSite | nindent 8 }} +{{- end }} +{{- end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/deployment.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/deployment.yaml new file mode 100644 index 000000000..00801c389 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/deployment.yaml @@ -0,0 +1,128 @@ +{{- if .Values.enable }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-sts + {{- if .Values.deployment.annotations }} + annotations: + {{ toYaml .Values.deployment.annotations | default "" }} + {{- end }} + labels: + app: {{ .Release.Name }} + {{- if .Values.deployment.labels }} + {{ toYaml .Values.deployment.labels }} + {{- end }} +spec: + replicas: {{ .Values.deployment.replicas }} + selector: + matchLabels: + app: {{ .Release.Name }} + {{- if .Values.deployment.labels }} + {{ toYaml .Values.deployment.labels }} + {{- end }} + template: + metadata: + {{- if .Values.deployment.annotations }} + annotations: + {{ toYaml .Values.deployment.annotations | default "" }} + {{- end }} + labels: + app: {{ .Release.Name }} + {{- if .Values.deployment.labels }} + {{ toYaml .Values.deployment.labels }} + {{- end }} + spec: + {{- if or (not (empty .Values.dependencies.packages)) (not (empty .Values.dependencies.jars)) }} + initContainers: + - name: "populate-sts-jar-volume" + image: {{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }} + imagePullPolicy: {{ .Values.deployment.image.imagePullPolicy | default "IfNotPresent" }} + command: ["sh", "-c", "cp /opt/spark/jars/* /tmp/jars/"] + volumeMounts: + - mountPath: /tmp/jars + name: sts-jars + {{- if not (empty .Values.dependencies.packages) }} + - name: "install-thrift-packages" + image: {{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }} + imagePullPolicy: {{ .Values.deployment.image.imagePullPolicy | default "IfNotPresent" }} + # Uses the Ivy jar packaged with spark to download dependency packages (and their transitive dependencies) to /tmp/jars/... + # Specifically downloads the binaries-- Does not download supplemental classifiers, ie sources + command: ["java", "-Divy.cache.dir=/tmp/", "-Divy.home=/tmp/", "-jar", "/opt/spark/jars/ivy-2.5.1.jar", "-confs", "default", "-retrieve", "/tmp/jars/[artifact].[ext]", {{ include "deps.packages.ivy" . }}] + volumeMounts: + - mountPath: /tmp/jars + name: sts-jars + {{- end }} + {{- if not (empty .Values.dependencies.jars) }} + - name: "install-thrift-jars" + image: {{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }} + imagePullPolicy: {{ .Values.deployment.image.imagePullPolicy | default "IfNotPresent" }} + command: ["wget", "-P", "/tmp/jars/", {{ include "deps.jars" . }}] + volumeMounts: + - mountPath: /tmp/jars + name: sts-jars + {{- end }} + {{- end }} + containers: + - name: {{ .Release.Name }} + image: "{{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }}" + imagePullPolicy: {{ .Values.deployment.image.imagePullPolicy }} + command: {{ .Values.deployment.command }} + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "/opt/spark/sbin/stop-thriftserver.sh"] + env: + {{- range $k,$v:= .Values.deployment.env }} + - name: {{ $k }} + value: "{{ $v }}" + {{- end }} + {{- range $k,$v:= .Values.deployment.envFromSecret }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: {{ $v.secretName }} + key: {{ $v.key }} + {{- end }} + ports: + {{- range .Values.service.ports }} + - containerPort: {{ .targetPort | default .port }} + {{- if .name }} + name: {{ .name }} + {{- end }} + {{- end }} + volumeMounts: + {{- if or (not (empty .Values.dependencies.packages)) (not (empty .Values.dependencies.jars)) }} + - mountPath: /opt/spark/jars/ + name: sts-jars + {{- end }} + {{- if or (not (eq .Values.sparkConf "")) (not ( eq .Values.hiveSite "")) }} + - name: spark-config + mountPath: /opt/spark/conf/ + readOnly: true + {{- end }} + {{- if .Values.deployment.volumeMounts }} + {{- toYaml .Values.deployment.volumeMounts }} + {{- end }} + volumes: + {{- if or (not (empty .Values.dependencies.packages)) (not (empty .Values.dependencies.jars)) }} + - name: sts-jars + emptyDir: {} + {{- end }} + {{- if or (not (eq .Values.sparkConf "")) (not ( eq .Values.hiveSite "")) }} + - name: spark-config + configMap: + name: {{ .Release.Name }}-spark-conf + items: + {{- if not (eq .Values.sparkConf "") }} + - key: sparkConf + path: spark-defaults.conf + {{- end }} + {{- if not (eq .Values.hiveSite "") }} + - key: hiveSite + path: hive-site.xml + {{- end }} + {{- end }} + {{- if .Values.deployment.volumes }} + {{- toYaml .Values.deployment.volumes }} + {{- end }} +{{- end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/ingress.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/ingress.yaml new file mode 100644 index 000000000..948f2ac98 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/ingress.yaml @@ -0,0 +1,28 @@ +{{ if and .Values.enable .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }}-sts-ingress + {{- with .Values.ingress.metadata.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.ingressClassName }} + rules: + {{- $releasename := printf "%s-service" .Release.Name }} + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ .backend.service.name | default $releasename }} + port: + number: {{ .backend.service.port.number }} + {{- end }} + {{- end }} +{{ end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/service.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/service.yaml new file mode 100644 index 000000000..32c7bf724 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/templates/service.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enable }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-sts-service + {{- if .Values.service.annotations }} + annotations: {{ .Values.service.annotations }} + {{- end }} +spec: + type: {{.Values.service.type }} + selector: + app: {{ .Release.Name }} + ports: + {{ .Values.service.ports | toYaml | nindent 4}} +{{- end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/deployment_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/deployment_test.yaml new file mode 100644 index 000000000..f52871f2c --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/deployment_test.yaml @@ -0,0 +1,367 @@ +Suite: Deployment + +templates: + - deployment.yaml + +tests: + - it: Should not produce deployment document if disabled + set: + enable: false + asserts: + - hasDocuments: + count: 0 + + - it: By default, produces a deployment document + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Deployment + - notFailedTemplate: {} + + - it: Should apply requested labels + set: + deployment.labels.label1: label2 + asserts: + - equal: + path: metadata.labels.label1 + value: label2 + - equal: + path: spec.template.metadata.labels.label1 + value: label2 + - equal: + path: spec.selector.matchLabels.label1 + value: label2 + + - it: Should retain default label even when requesting supplemental labels + set: + deployment.labels.label1: label2 + asserts: + - equal: + path: metadata.labels.app + value: RELEASE-NAME + - equal: + path: spec.template.metadata.labels.app + value: RELEASE-NAME + - equal: + path: spec.selector.matchLabels.app + value: RELEASE-NAME + + - it: Should apply requested annotations + set: + deployment.annotations.annotation1: annotation2 + asserts: + - equal: + path: metadata.annotations.annotation1 + value: annotation2 + - equal: + path: spec.template.metadata.annotations.annotation1 + value: annotation2 + + - it: Should set the requested number of replicas + set: + deployment.replicas: 5 + asserts: + - equal: + path: spec.replicas + value: 5 + + - it: Should have the expected default number of replicas + asserts: + - equal: + path: spec.replicas + value: 1 + + - it: If packages are specified, the appropriate initcontainers should be requested + set: + dependencies.packages: + - somePackage + asserts: + - contains: + path: spec.template.spec.initContainers + any: true + content: + name: "populate-sts-jar-volume" + - contains: + path: spec.template.spec.initContainers + any: true + content: + name: "install-thrift-packages" + - notContains: + path: spec.template.spec.initContainers + any: true + content: + name: "install-thrift-jars" + + - it: If jars are specified, the appropriate initcontainers should be requested + set: + dependencies.jars: + - someJar + asserts: + - contains: + path: spec.template.spec.initContainers + any: true + content: + name: "populate-sts-jar-volume" + - contains: + path: spec.template.spec.initContainers + any: true + content: + name: "install-thrift-jars" + - notContains: + path: spec.template.spec.initContainers + any: true + content: + name: "install-thrift-packages" + + - it: If both jars and packages are specified, the appropriate initcontainers should be requested + set: + dependencies.packages: + - somePackage + dependencies.jars: + - someJar + asserts: + - contains: + path: spec.template.spec.initContainers + any: true + content: + name: "populate-sts-jar-volume" + - contains: + path: spec.template.spec.initContainers + any: true + content: + name: "install-thrift-jars" + - contains: + path: spec.template.spec.initContainers + any: true + content: + name: "install-thrift-packages" + + - it: If neither jars nor packages are specified, then no initContainers should be requested + asserts: + - notExists: + path: spec.template.spec.initContainers + + - it: The same spark image is used in all locations + set: + dependencies.packages: + - somePackage + dependencies.jars: + - someJar + deployment.image.repository: "some_image" + deployment.image.tag: "some_tag" + asserts: + - equal: + path: spec.template.spec.initContainers[0].image + value: "some_image:some_tag" + - equal: + path: spec.template.spec.initContainers[1].image + value: "some_image:some_tag" + - equal: + path: spec.template.spec.initContainers[2].image + value: "some_image:some_tag" + - equal: + path: spec.template.spec.containers[0].image + value: "some_image:some_tag" + + - it: The same imagePullPolicy is used in all locations + set: + dependencies.packages: + - somePackage + dependencies.jars: + - someJar + deployment.image.imagePullPolicy: Always + asserts: + - equal: + path: spec.template.spec.initContainers[0].imagePullPolicy + value: Always + - equal: + path: spec.template.spec.initContainers[1].imagePullPolicy + value: Always + - equal: + path: spec.template.spec.initContainers[2].imagePullPolicy + value: Always + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: The jar volume population initContainer command appropriately references the mount point. + set: + dependencies.packages: + - somePackage + asserts: + - equal: + path: spec.template.spec.initContainers[0].volumeMounts[0].mountPath + value: "/tmp/jars" + - matchRegex: + path: spec.template.spec.initContainers[0].command[-1] + pattern: .*/tmp/jars/$ + + - it: The package installation initContainer command appropriately references the mount point. + set: + dependencies.packages: + - somePackage + asserts: + - equal: + path: spec.template.spec.initContainers[1].volumeMounts[0].mountPath + value: "/tmp/jars" + - matchRegex: + path: spec.template.spec.initContainers[1].command[8] + pattern: /tmp/jars/.* + + - it: The jar installation initContainer command appropriately references the mount point. + set: + dependencies.jars: + - someJar + asserts: + - equal: + path: spec.template.spec.initContainers[1].volumeMounts[0].mountPath + value: "/tmp/jars" + - matchRegex: + path: spec.template.spec.initContainers[1].command[2] + pattern: /tmp/jars/ + + - it: When specifying jar dependencies, the appropriate volumes are created and mounted + set: + dependencies.jars: + - someJar + asserts: + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + any: true + content: + name: "sts-jars" + - contains: + path: spec.template.spec.initContainers[1].volumeMounts + any: true + content: + name: "sts-jars" + - contains: + path: spec.template.spec.containers[0].volumeMounts + any: true + content: + name: "sts-jars" + - contains: + path: spec.template.spec.volumes + any: true + content: + name: "sts-jars" + + - it: When specifying package dependencies, the appropriate volumes are created and mounted + set: + dependencies.packages: + - somePackage + asserts: + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + any: true + content: + name: "sts-jars" + - contains: + path: spec.template.spec.initContainers[1].volumeMounts + any: true + content: + name: "sts-jars" + - contains: + path: spec.template.spec.containers[0].volumeMounts + any: true + content: + name: "sts-jars" + - contains: + path: spec.template.spec.volumes + any: true + content: + name: "sts-jars" + + - it: When specifying jar dependencies, each dependency is appropriately listed for download + set: + dependencies.jars: + - some_jar.jar + - another_jar.jar + asserts: + - equal: + path: spec.template.spec.initContainers[1].command[-2] + value: some_jar.jar + - equal: + path: spec.template.spec.initContainers[1].command[-1] + value: another_jar.jar + + - it: When specifying package dependencies, each dependency is properly listed for download + set: + dependencies.packages: + - group:artifact:version + asserts: + - equal: + path: spec.template.spec.initContainers[1].command[-3] + value: group + - equal: + path: spec.template.spec.initContainers[1].command[-2] + value: artifact + - equal: + path: spec.template.spec.initContainers[1].command[-1] + value: version + + - it: When a command is unspecified, the default is applied + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: ["/opt/spark/sbin/start-thriftserver.sh"] + + - it: When a command specified, it is applied to the main container + set: + deployment.command: ["test"] + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: ["test"] + + - it: Environment variables are appropriately converted from a mapping to an array in the necessary format + set: + deployment.env: + myKey: myValue + asserts: + - contains: + path: spec.template.spec.containers[0].env + any: true + content: + name: myKey + value: myValue + + - it: Environment variables can be specified from secrets + set: + deployment.envFromSecret: + myVar: + secretName: secret_name + key: secret_key + asserts: + - contains: + path: spec.template.spec.containers[0].env + any: true + content: + name: myVar + valueFrom: + secretKeyRef: + name: secret_name + key: secret_key + + - it: The appropriate containerPorts are exposed to reach the Thrift server (10000, 10001) + asserts: + - contains: + path: spec.template.spec.containers[0].ports + any: true + content: + containerPort: 10000 + - contains: + path: spec.template.spec.containers[0].ports + any: true + content: + containerPort: 10001 + + - it: The appropriate default port name is applied for the STS port + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].name + value: "thrift" + - equal: + path: spec.template.spec.containers[0].ports[1].name + value: "thrift-http" diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/ingress_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/ingress_test.yaml new file mode 100644 index 000000000..39660ae0e --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/ingress_test.yaml @@ -0,0 +1,91 @@ +Suite: Ingress testing + +templates: + - ingress.yaml + +tests: + - it: No ingress should be created when the chart is disabled + set: + enable: false + asserts: + - hasDocuments: + count: 0 + + - it: No ingress should be created when ingress is disabled + set: + enable: true + ingress: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: Ingress should be created when the chart is enabled and ingress is enabled + set: + enable: true + ingress: + enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Ingress + - notFailedTemplate: {} + + - it: Ingress should have the correct default metadata + set: + ingress: + enabled: true + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-sts-ingress + - notExists: + path: metadata.annotations + + - it: Ingress annotations can be specified + set: + ingress: + enabled: true + metadata: + annotations: + someAnnotation: someValue + asserts: + - equal: + path: metadata.annotations.someAnnotation + value: someValue + + - it: Ingress class name can be specified + set: + ingress: + enabled: true + ingressClassName: some-ingress-class + asserts: + - equal: + path: spec.ingressClassName + value: some-ingress-class + + - it: Ingress hosts can be specified + set: + ingress: + enabled: true + hosts: + - paths: + - path: /some-path + pathType: Prefix + backend: + service: + name: some-service + port: + number: 8080 + asserts: + - contains: + path: spec.rules[0].http.paths + content: + path: /some-path + pathType: Prefix + backend: + service: + name: some-service + port: + number: 8080 diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/service_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/service_test.yaml new file mode 100644 index 000000000..8fd3ad170 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/service_test.yaml @@ -0,0 +1,63 @@ +Suite: Service + +templates: + - service.yaml + +tests: + - it: Should not produce service if enable is false + set: + enable: false + asserts: + - hasDocuments: + count: 0 + + - it: Should produce service with default values + set: + enable: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: RELEASE-NAME-sts-service + - notExists: + path: metadata.annotations + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.selector.app + value: RELEASE-NAME + - contains: + path: spec.ports + any: true + content: + name: thrift + port: 10000 + - contains: + path: spec.ports + any: true + content: + name: thrift-http + port: 10001 + + - it: Should be able to specify service type + set: + enable: true + service: + type: LoadBalancer + asserts: + - equal: + path: spec.type + value: LoadBalancer + + - it: Should be able to specify service port + set: + enable: true + service: + ports: + - port: 8080 + asserts: + - equal: + path: spec.ports[0].port + value: 8080 diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/spark_conf_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/spark_conf_test.yaml new file mode 100644 index 000000000..fe31ff7ba --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/tests/spark_conf_test.yaml @@ -0,0 +1,108 @@ +suite: Spark Conf + +templates: + - configmap.yaml + - deployment.yaml + +tests: + - it: If no spark conf is specified, the ConfigMap should not be created + templates: + - configmap.yaml + asserts: + - hasDocuments: + count: 0 + + - it: If spark conf is specified, the ConfigMap should be created with the provided values + set: + sparkConf: "spark.executor.instances: 2" + templates: + - configmap.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - notFailedTemplate: {} + - equal: + path: metadata.name + value: "RELEASE-NAME-spark-conf" + - equal: + path: data.sparkConf + value: "spark.executor.instances: 2" + + - it: If spark conf is specified, a volume should be created from the created configmap + set: + sparkConf: "spark.executor.instances: 2" + templates: + - deployment.yaml + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: spark-config + configMap: + name: RELEASE-NAME-spark-conf + items: + - key: sparkConf + path: spark-defaults.conf + + - it: If spark conf is specified, the volume should be mounted in the correct path + set: + sparkConf: "spark.executor.instances: 2" + templates: + - deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: spark-config + mountPath: /opt/spark/conf/ + readOnly: true + + + - it: If hive site is specified, the ConfigMap should be created with the provided values + set: + hiveSite: "spark.executor.instances: 2" + templates: + - configmap.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - notFailedTemplate: {} + - equal: + path: metadata.name + value: "RELEASE-NAME-spark-conf" + - equal: + path: data.hiveSite + value: "spark.executor.instances: 2" + + - it: If hive site is specified, a volume should be created from the created configmap + set: + hiveSite: "spark.executor.instances: 2" + templates: + - deployment.yaml + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: spark-config + configMap: + name: RELEASE-NAME-spark-conf + items: + - key: hiveSite + path: hive-site.xml + + - it: If hive site is specified, the volume should be mounted in the correct path + set: + hiveSite: "spark.executor.instances: 2" + templates: + - deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: spark-config + mountPath: /opt/spark/conf/ + readOnly: true diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.schema.json b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.schema.json new file mode 100644 index 000000000..1c599b5e8 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.schema.json @@ -0,0 +1,146 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "required": [ + "deployment", + "dependencies", + "enable", + "ingress", + "service" + ], + "properties": { + "ingress": { + "type": "object", + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "metadata": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + } + } + }, + "ingressClassName": { + "type": "string" + }, + "hosts": { + "type": "array" + } + }, + "allOf": [ + { + "if": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + }, + "then": { + "required": [ + "metadata", + "ingressClassName", + "hosts" + ] + } + } + ] + }, + "dependencies": { + "type": "object", + "properties": { + "jars": { + "type": "array" + }, + "packages": { + "type": "array" + } + } + }, + "service": { + "type": "object", + "required": [ + "ports", + "type" + ], + "properties": { + "type": { + "enum": [ + "ClusterIP", + "LoadBalancer", + "NodePort" + ] + }, + "port": { + "type": "array" + } + } + }, + "sparkConf": { + "type": "string" + }, + "hiveSite": { + "type": "string" + }, + "deployment": { + "type": "object", + "required": [ + "image", + "command" + ], + "properties": { + "annotations": { + "type": "object" + }, + "replicas": { + "type": "integer" + }, + "labels": { + "type": "object" + }, + "env": { + "type": "object" + }, + "envFromSecret": { + "type": "object" + }, + "volumeMounts": { + "type": "array" + }, + "volumes": { + "type": "array" + }, + "command": { + "type": "array" + }, + "image": { + "type": "object", + "required": [ + "repository", + "tag", + "imagePullPolicy" + ], + "properties": { + "imagePullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + } + } + } + } +} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.yaml new file mode 100644 index 000000000..06bfb7bc6 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.yaml @@ -0,0 +1,43 @@ +# This file contains the default values for the aiSSEMBLE Spark History Helm chart. + +enable: true + +deployment: + annotations: {} + labels: {} + replicas: 1 + image: + repository: "apache/spark" + imagePullPolicy: IfNotPresent + tag: "3.5.1" + command: ["/opt/spark/sbin/start-thriftserver.sh"] + env: + SPARK_NO_DAEMONIZE: "true" + envFromSecret: {} + volumes: [] + volumeMounts: [] + +dependencies: + packages: [] + jars: [] + +service: + annotations: {} + type: ClusterIP + ports: + - name: "thrift" + port: 10000 + - name: "thrift-http" + port: 10001 + +ingress: + enabled: false + metadata: + annotations: {} + ingressClassName: nginx + hosts: + - paths: [] + +sparkConf: |- + +hiveSite: |- diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml index 44bc022ef..48b1f8cff 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml @@ -15,6 +15,7 @@ aissemble-spark-history-chart + aissemble-thrift-server-chart @@ -34,4 +35,4 @@ - \ No newline at end of file + diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm index 0d37be6c6..98bf7b602 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm @@ -32,4 +32,7 @@ dependencies: - name: aissemble-spark-history-chart version: ${versionTag} repository: oci://ghcr.io/boozallen + - name: aissemble-thrift-server-chart + version: ${versionTag} + repository: oci://ghcr.io/boozallen diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.dev.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.dev.yaml.vm index 20cbfda3f..92ce3ae63 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.dev.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.dev.yaml.vm @@ -1,8 +1,8 @@ -# This file contains the default values for the aiSSEMBLE Spark History Helm chart. +# This file contains the default values for the aiSSEMBLE Spark Infrastructure Helm chart. aissemble-spark-history-chart: service: type: LoadBalancer ingress: - enabled: false \ No newline at end of file + enabled: false diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm index 0987d9e7e..553245d01 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm @@ -1,5 +1,42 @@ -# This file contains the default values for the aiSSEMBLE Spark History Helm chart. +# This file contains the default values for the aiSSEMBLE Spark Infrastructure Helm chart. aissemble-spark-history-chart: - eventVolume: - enabled: true \ No newline at end of file + eventVolume: + enabled: true + +aissemble-thrift-server-chart: + dependencies: + packages: + - org.apache.hadoop:hadoop-aws:3.3.4 + + deployment: + envFromSecret: + AWS_ACCESS_KEY_ID: + secretName: aws-s3-credentials + key: aws_access_key_id + AWS_SECRET_ACCESS_KEY: + secretName: aws-s3-credentials + key: aws_secret_access_key + + sparkConf: | + spark.hadoop.fs.s3a.endpoint=http://s3-local:4566 + spark.hadoop.fs.s3a.access.key=#[[${env:AWS_ACCESS_KEY_ID}]]# + spark.hadoop.fs.s3a.secret.key=#[[${env:AWS_SECRET_ACCESS_KEY}]]# + spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem + spark.hive.server2.thrift.port=10000 + spark.hive.server2.thrift.http.port=10001 + spark.hive.server2.transport.mode=http + spark.hive.metastore.warehouse.dir=s3a://spark-infrastructure/warehouse + spark.hadoop.fs.s3a.path.style.access=true + spark.hive.server2.thrift.http.path=cliservice + spark.hive.metastore.schema.verification=false + spark.hive.metastore.uris=thrift://hive-metastore-service:9083/default + + hiveSite: | + + + datanucleus.schema.autoCreateAll + true + Creates necessary schema on a startup if one does not exist + +