cluster-with-config.yaml

apiVersion: radanalytics.io/v1
kind: SparkCluster
metadata:
  name: sparky-cluster                                  # compulsory
spec:
  mavenDependencies:                                    # optional: array of Maven resources identified by GAVs (groupId:artifactId:version)
  - com.amazonaws:aws-java-sdk-pom:1.10.34
  - org.apache.hadoop:hadoop-aws:2.7.3
  worker:
    instances: "2"                                      # optional defaults to 1
    memory: "1Gi"                                       # optional no defaults
    cpu: 0.2                                            # optional no defaults
  master:
    instances: "1"                                      # optional defaults to 1
    memory: "1Gi"                                       # optional no defaults
    cpu: 0.2                                            # optional no defaults
  customImage: quay.io/radanalyticsio/openshift-spark:2.4-latest   # optional defaults to quay.io/radanalyticsio/openshift-spark:2.4-latest
  metrics: false                                        # on each pod expose the metrics endpoint on port 7777 for prometheus, defautls to false
  sparkWebUI: true                                      # create a service with the Spark UI, defaults to true
  env:                                                  # optional
  - name: SPARK_WORKER_CORES
    value: 2
  - name: FOO
    value: bar
  sparkConfigurationMap: my-config                    # optional defaults to ${name}-config
                                                      # kubectl create configmap my-config --from-file=example/config.conf
  sparkConfiguration:                                 # optional, it overrides the config defined above
  - name: spark.executor.memory
    value: 500m
  - name: spark.sql.conf.autoBroadcastJoinThreshold
    value: 20971520
  downloadData:                                       # optional, it downloads these files to each node
  - url: https://raw.githubusercontent.com/radanalyticsio/spark-operator/master/README.md
    to: /tmp/