Add OpenTelemetry and Tempo for AI Telemetry in obs (OCP-on-NERC#627)

* feat: add OTel and Tempo to obs cluster Signed-off-by: Dheeraj<[email protected]> * Adding additional OpenTelemetry resources for obs cluster --------- Signed-off-by: Dheeraj<[email protected]> Co-authored-by: Dheeraj <[email protected]>
dystewart · Dec 16, 2024 · 56c7b15 · 56c7b15
1 parent a602534
commit 56c7b15
Show file tree

Hide file tree

Showing 12 changed files with 210 additions and 0 deletions.
diff --git a/cluster-scope/base/core/namespaces/opentelemetry/kustomization.yaml b/cluster-scope/base/core/namespaces/opentelemetry/kustomization.yaml
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
diff --git a/cluster-scope/base/core/namespaces/opentelemetry/namespace.yaml b/cluster-scope/base/core/namespaces/opentelemetry/namespace.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: opentelemetry
+spec: {}
diff --git a/cluster-scope/bundles/opentelemetry/kustomization.yaml b/cluster-scope/bundles/opentelemetry/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+commonLabels:
+  nerc.mghpcc.org/bundle: opentelemetry
+resources:
+- ../../base/core/namespaces/opentelemetry
diff --git a/cluster-scope/overlays/nerc-ocp-obs/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-obs/kustomization.yaml
@@ -16,6 +16,7 @@ resources:
 - ../../bundles/zookeeper
 - ../../bundles/solr
 - ../../bundles/minio
+- ../../bundles/opentelemetry
 - ../../base/core/namespaces/openshift-gitops
 - ../../base/core/namespaces/dex
 - ../../base/rbac.authorization.k8s.io/clusterroles/allow-edit-rbac

diff --git a/opentelemetry/base/externalsecret-opentelemetry.yaml b/opentelemetry/base/externalsecret-opentelemetry.yaml
@@ -0,0 +1,15 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: opentelemetry
+  namespace: opentelemetry
+spec:
+  refreshInterval: "1h"
+  secretStoreRef:
+    name: nerc-secret-store
+    kind: SecretStore
+  target:
+    name: opentelemetry
+  dataFrom:
+    - extract:
+        key: $ENV/$CLUSTER/ai-telemetry/opentelemetry
diff --git a/opentelemetry/base/kustomization.yaml b/opentelemetry/base/kustomization.yaml
@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: opentelemetry
+commonLabels:
+  app: opentelemetry
+
+resources:
+- externalsecret-opentelemetry.yaml
+- opentelemetry-rbac.yaml
+- opentelemetry-tempostack.yaml
+- opentelemetry-collector.yaml
+- opentelemetry-instrumentation.yaml
diff --git a/opentelemetry/base/opentelemetry-collector.yaml b/opentelemetry/base/opentelemetry-collector.yaml
@@ -0,0 +1,64 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: opentelemetry
+  namespace: opentelemetry
+spec:
+  mode: deployment
+  observability:
+    metrics:
+      enableMetrics: true
+  config: |
+    connectors:
+      spanmetrics:
+        metrics_flush_interval: 15s
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+      jaeger:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:14250
+          thrift_http:
+            endpoint: 0.0.0.0:14268
+          thrift_compact:
+            endpoint: 0.0.0.0:6831
+          thrift_binary:
+            endpoint: 0.0.0.0:6832
+    processors:
+      batch:
+        timeout: 5s
+        send_batch_max_size: 10000
+      memory_limiter:
+        check_interval: 1s
+        limit_percentage: 50
+        spike_limit_percentage: 30
+    exporters:
+      prometheus:
+        endpoint: 0.0.0.0:8889
+        add_metric_suffixes: false
+        resource_to_telemetry_conversion:
+          enabled: true
+      otlp:
+        endpoint: tempo-opentelemetry-distributor.opentelemetry.svc:4317
+        tls:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+        headers:
+          X-Scope-OrgID: "dev"
+    service:
+      telemetry:
+        metrics:
+          address: ":8888"
+      pipelines:
+        traces:
+          receivers: [otlp, jaeger]
+          processors: [memory_limiter, batch]
+          exporters: [otlp, spanmetrics]
+        metrics:
+          receivers: [otlp, spanmetrics]
+          processors: [memory_limiter, batch]
+          exporters: [otlp, prometheus]
diff --git a/opentelemetry/base/opentelemetry-instrumentation.yaml b/opentelemetry/base/opentelemetry-instrumentation.yaml
@@ -0,0 +1,19 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: Instrumentation
+metadata:
+  name: java-instrumentation
+spec:
+  env:
+    - name: OTEL_EXPORTER_OTLP_TIMEOUT
+      value: "20"
+  exporter:
+    endpoint: http://opentelemetry-collector.opentelemetry.svc.cluster.local:4317
+  propagators:
+    - ottrace
+  sampler:
+    type: parentbased_traceidratio
+    argument: "0.25"
+  java:
+    env:
+    - name: OTEL_JAVAAGENT_DEBUG
+      value: "true"
diff --git a/opentelemetry/base/opentelemetry-rbac.yaml b/opentelemetry/base/opentelemetry-rbac.yaml
@@ -0,0 +1,31 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: generate-processors-rbac
+rules:
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  - clusterroles
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: generate-processors-rbac
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: generate-processors-rbac
+subjects:
+- kind: ServiceAccount
+  name: opentelemetry-operator-controller-manager
+  namespace: openshift-opentelemetry-operator
diff --git a/opentelemetry/base/opentelemetry-tempostack.yaml b/opentelemetry/base/opentelemetry-tempostack.yaml
@@ -0,0 +1,34 @@
+apiVersion: tempo.grafana.com/v1alpha1
+kind: TempoStack
+metadata:
+  name: opentelemetry
+  namespace: opentelemetry
+spec:
+  storageSize: 100Gi
+  storage:
+    secret:
+      name: opentelemetry
+      type: s3
+    tls:
+      enabled: true
+  template:
+    gateway:
+      enabled: false
+    distributor:
+      tls:
+        enabled: true
+    queryFrontend:
+      jaegerQuery:
+        enabled: true
+        monitorTab:
+          enabled: true
+          prometheusEndpoint: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
+        ingress:
+          route:
+            termination: edge
+          type: route
+  resources:
+    total:
+      limits:
+        memory: 4Gi
+        cpu: 4
diff --git a/opentelemetry/overlays/nerc-ocp-obs/externalsecrets/patch-opentelemetry.yaml b/opentelemetry/overlays/nerc-ocp-obs/externalsecrets/patch-opentelemetry.yaml
@@ -0,0 +1,12 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: opentelemetry
+  namespace: opentelemetry
+spec:
+  secretStoreRef:
+    name: nerc-cluster-secrets
+    kind: ClusterSecretStore
+  dataFrom:
+    - extract:
+        key: nerc/nerc-ocp-obs/ai-telemetry/opentelemetry
diff --git a/opentelemetry/overlays/nerc-ocp-obs/kustomization.yaml b/opentelemetry/overlays/nerc-ocp-obs/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+
+patches:
+  - path: externalsecrets/patch-opentelemetry.yaml