Add support for graphite-exporter and set Spark cluster defaults (#10)

SketchBench · Aug 1, 2021 · a979f31 · a979f31
1 parent 80d12a1
commit a979f31
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 3 deletions.
diff --git a/charts/bullet/Chart.yaml b/charts/bullet/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: bullet
 description: A real-time query engine for very large data streams
 type: application
-version: 0.4.0
+version: 0.4.1
 appVersion: "1.0.4"
 home: https://github.com/SketchBench/bullet-k8s
 # icon:

diff --git a/charts/bullet/templates/configmap.yaml b/charts/bullet/templates/configmap.yaml
@@ -322,5 +322,19 @@ data:
     *.sink.prometheusServlet.path=/metrics/prometheus
     master.sink.prometheusServlet.path=/metrics/master/prometheus
     applications.sink.prometheusServlet.path=/metrics/applications/prometheus
+    {{- if and .Values.sparkBackend.metrics.enabled .Values.sparkBackend.metrics.graphiteExporter.enabled }}
+    # Example configuration for Graphite sink
+    *.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink
+    *.sink.graphite.host={{ .Values.sparkBackend.metrics.graphiteExporter.host }}
+    *.sink.graphite.port={{ .Values.sparkBackend.metrics.graphiteExporter.port }}
+    *.sink.graphite.period={{ .Values.sparkBackend.metrics.graphiteExporter.periodSeconds }}
+    *.sink.graphite.unit=seconds
+    #*.sink.graphite.prefix=<optional_value>
+    # Enable JvmSource
+    master.source.jvm.class=org.apache.spark.metrics.source.JvmSource
+    worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource
+    driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource
+    executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource
+    {{- end }}
   dsl_schema.json: |-
 {{ .Values.sparkBackend.dsl.schema | indent 4 }}
diff --git a/charts/bullet/values.yaml b/charts/bullet/values.yaml
@@ -75,6 +75,9 @@ sparkBackend:
       namespace: ""
       interval: 30s
       scrapeTimeout:
+    graphiteExporter:
+      enabled: false
+      periodSeconds: 30
   config:
     # See https://github.com/bullet-db/bullet-spark/blob/master/src/main/resources/bullet_spark_defaults.yaml
     ### Bullet ###
@@ -255,6 +258,13 @@ sparkBackend:
     port: 4041
     blockManagerPort: 4042
     confs:
+      - --conf
+      - spark.cores.max=6
+      - --conf
+      - spark.executor.cores=1
+      - --conf
+      - spark.executor.memory=2G
+
 ### External Dependencies ###
 spark:
   enabled: true
@@ -264,9 +274,25 @@ spark:
     pullPolicy: IfNotPresent
   worker:
     replicaCount: 3
-    coreLimit: 3
+    coreLimit: 2
+    resources:
+      requests:
+        memory: "5Gi"
+        cpu: "2.25"
+      limits:
+        memory: "5Gi"
+        cpu: "2.25"
+  master:
+    resources:
+      requests:
+        memory: "5Gi"
+        cpu: "2.25"
+      limits:
+        memory: "5Gi"
+        cpu: "2.25"
+
 hdfs:
-  enabled: true
+  enabled: false
   config:
     hdfsSite:
       # https://github.com/gchq/gaffer-docker/pull/152