Add observability using Loki/Tempo and Grafana Agent

Makefile

@@ -307,11 +307,11 @@ helm/check: $(BIN)/kubeval $(BIN)/helm
 
 .PHONY: deploy
 deploy: $(BIN)/kind $(BIN)/helm docker-image/phlare/build
-	$(call deploy,phlare-dev,)
+	$(call deploy,phlare-dev,--set=phlare.extraEnvVars.JAEGER_AGENT_HOST=jaeger.monitoring.svc.cluster.local.)
 
 .PHONY: deploy-micro-services
 deploy-micro-services: $(BIN)/kind $(BIN)/helm docker-image/phlare/build
-	$(call deploy,phlare-micro-services,--values=operations/phlare/helm/phlare/values-micro-services.yaml)
+	$(call deploy,phlare-micro-services,--values=operations/phlare/helm/phlare/values-micro-services.yaml --set=phlare.extraEnvVars.JAEGER_AGENT_HOST=jaeger.monitoring.svc.cluster.local.)
 
 .PHONY: deploy-monitoring
 deploy-monitoring: $(BIN)/tk $(BIN)/kind tools/monitoring/environments/default/spec.json

tools/monitoring/environments/default/loki.libsonnet

@@ -0,0 +1,128 @@
+local agent = import 'github.com/grafana/agent/production/tanka/grafana-agent/v2/main.libsonnet';
+local loki = import 'github.com/grafana/loki/production/ksonnet/loki-simple-scalable/loki.libsonnet';
+local k = import 'ksonnet-util/kausal.libsonnet';
+local prom_k_grafana = import 'prometheus-ksonnet/grafana/grafana.libsonnet';
+
+
+{
+  local namespace = if std.objectHas($._config, 'namespace') then $._config.namespace else 'mynamespace',
+
+  local podsLogPath = '/var/log/pods',
+
+  _config+: {},
+
+  agent+: {
+    _config+: { agent_config+: {
+      logs+: {
+        configs: [{
+          name: 'kubernetes-logs',
+          clients: [{
+            url: 'http://%s:3100/loki/api/v1/push' % $.loki.fqdn,
+          }],
+          positions: {
+            filename: podsLogPath + '/grafana-agent-positions.yaml',
+          },
+          scrape_configs: agent.newKubernetesLogs({}),
+        }],
+      },
+    } },
+
+    controller+:
+      local sts = k.apps.v1.statefulSet;
+      sts.spec.template.spec.withVolumesMixin(
+        local volume = k.core.v1.volume;
+        [
+          volume.fromHostPath('logs', podsLogPath),
+        ]
+      ),
+
+    container+:
+      {
+        volumeMounts+:
+          local volumeMount = k.core.v1.volumeMount;
+          [
+            volumeMount.new('logs', podsLogPath),
+          ],
+
+      },
+  },
+
+  loki:
+    local upstream = loki {
+      _config+:: {
+        headless_service_name: 'loki',
+        http_listen_port: 3100,
+        read_replicas: 1,
+        write_replicas: 1,
+        loki: {
+          auth_enabled: false,
+        },
+      },
+
+
+      write_args: {
+        'config.file': '/etc/loki/local-config.yaml',
+      },
+
+      write_statefulset+:
+        {
+          spec+: {
+            volumeClaimTemplates:: [],
+            template+: {
+              spec+: {
+                volumes:
+                  local volume = k.core.v1.volume;
+                  [
+                    volume.fromEmptyDir('data'),
+                  ],
+                containers: [
+                  c {
+                    name: 'loki',
+                    volumeMounts: [
+                      x {
+                        name: 'data',
+                      }
+                      for x in super.volumeMounts
+                      if x.name == 'write-data'
+                    ],
+                  }
+                  for c in super.containers
+                ],
+              },
+            },
+          },
+        },
+    };
+
+    {
+      fqdn:: 'loki.%s.svc.cluster.local.' % namespace,
+
+      stateful_set:
+        local sts = k.apps.v1.statefulSet;
+
+        upstream.write_statefulset +
+        sts.metadata.withName('loki') +
+        sts.metadata.withLabelsMixin({ name: 'loki' }) +
+        sts.spec.template.metadata.withLabelsMixin({ name: 'loki' }) +
+        sts.spec.selector.withMatchLabelsMixin({ name: 'loki' }) +
+        { spec+: { template+: { spec+: { affinity:: {} } } } },
+
+      service:
+        local svc = k.core.v1.service;
+        svc.new('loki', { name: 'loki' }, [{ port: 3100 }]),
+
+    },
+
+  grafana_datasource_config_map+: k.core.v1.configMap.withDataMixin({
+    'loki-datasource.yml': k.util.manifestYaml({
+      apiVersion: 1,
+      datasources: [
+        prom_k_grafana.grafana_datasource(
+          'Loki',
+          'http://%s:3100' % $.loki.fqdn,
+          type='loki'
+        ),
+      ],
+    }),
+  }),
+}

tools/monitoring/environments/default/monitoring.libsonnet

@@ -4,12 +4,17 @@ local k = import 'ksonnet-util/kausal.libsonnet';
 local prom_k_grafana = import 'prometheus-ksonnet/grafana/grafana.libsonnet';
 local prometheus = import 'prometheus-ksonnet/prometheus-ksonnet.libsonnet';
 
+local namespace = 'monitoring';
+local fqdn = {
+  phlare: 'phlare-micro-services-query-frontend.default.svc.cluster.local.',
+};
+
 
 prometheus {
   local cluster_name = 'dev',
   _config+:: {
     cluster_name: cluster_name,
-    namespace: 'monitoring',
+    namespace: namespace,
     grafana_ini+: {
       sections+: {
         feature_toggles+: {
@@ -42,18 +47,105 @@ prometheus {
   mixins+:: {
     phlare: phlare {},
   },
+
+  agent:
+    local pvc = k.core.v1.persistentVolumeClaim;
+    local volumeMount = k.core.v1.volumeMount;
+    local containerPort = k.core.v1.containerPort;
+    local extraPorts = [
+      {
+        name: 'thrift-compact',
+        port: 6831,
+        protocol: 'UDP',
+      },
+      {
+        name: 'sampling',
+        port: 5778,
+        protocol: 'TCP',
+      },
+    ];
+    local agent = import 'github.com/grafana/agent/production/tanka/grafana-agent/v2/main.libsonnet';
+    agent.new(name='grafana-agent', namespace=$._config.namespace) +
+    agent.withStatefulSetController(
+      replicas=1,
+    ) +
+    // problems with remote sampling in anything later than v0.28.1
+    // see
+    // https://github.com/grafana/agent/issues/2911
+    agent.withImagesMixin({ agent: 'grafana/agent:v0.28.1' }) +
+    agent.withArgsMixin({
+      'enable-features': 'integrations-next',
+    },) +
+    // add dummy config or else will fail
+    agent.withAgentConfig({
+      server: { log_level: 'debug' },
+    }) +
+    agent.withVolumeMountsMixin([volumeMount.new('agent-wal', '/var/lib/agent')]) +
+    // headless svc needed by statefulset
+    agent.withService() +
+    {
+      controller_service+: {
+        spec+: {
+          clusterIP: 'None',
+        },
+      },
+      controller+: {
+        spec+: {
+          template+: { spec+: {
+            volumes+:
+              local volume = k.core.v1.volume;
+              [
+                volume.fromEmptyDir('agent-wal'),
+              ],
+          } },
+        },
+      },
+
+
+      container+: {
+        ports+: [
+          std.prune(p {
+            containerPort: p.port,
+            port::: null,
+          })
+          for p in extraPorts
+        ],
+      },
+
+      configMap+: {
+        data+: {
+          'strategy.json': std.manifestJsonMinified({
+            default_strategy: { param: 1.0, type: 'probabilistic' },
+          }),
+        },
+      },
+
+      jaeger_service:
+        local svc = k.core.v1.service;
+
+        super.controller_service +
+        svc.metadata.withName('jaeger') +
+        svc.metadata.withLabelsMixin({ name: 'jaeger' }) +
+        svc.spec.withPorts([
+          p {
+            targetPort: p.port,
+          }
+          for p in extraPorts
+        ]),
+    },
+
   grafana_datasource_config_map+: k.core.v1.configMap.withDataMixin({
     'phlare-datasource.yml': k.util.manifestYaml({
       apiVersion: 1,
       datasources: [
         prom_k_grafana.grafana_datasource(
-          'phlare',
-          'http://phlare-micro-services-query-frontend.default.svc.cluster.local.:4100',
+          'Phlare',
+          'http://%s:4100' % fqdn.phlare,
           type='phlare'
-        ) + grafana.datasource.withJsonData({
-          path: 'http://phlare-micro-services-query-frontend.default.svc.cluster.local.:4100/',
-        },),
+        ),
       ],
     }),
   }),
-}
+} +
+(import 'tempo.libsonnet') +
+(import 'loki.libsonnet')

tools/monitoring/environments/default/tempo.libsonnet

@@ -0,0 +1,109 @@
+local agent = import 'github.com/grafana/agent/production/tanka/grafana-agent/v2/main.libsonnet';
+local tempo = import 'github.com/grafana/tempo/operations/jsonnet/single-binary/tempo.libsonnet';
+local k = import 'ksonnet-util/kausal.libsonnet';
+local prom_k_grafana = import 'prometheus-ksonnet/grafana/grafana.libsonnet';
+
+{
+  local namespace = if std.objectHas($._config, 'namespace') then $._config.namespace else 'mynamespace',
+
+  _config+: {},
+
+  agent+: {
+    _config+: { agent_config+: {
+      traces+: {
+        configs: [{
+          name: 'kubernetes-traces',
+          receivers: {
+            jaeger: {
+              protocols: {
+                grpc: null,
+                thrift_binary: null,
+                thrift_compact: null,
+                thrift_http: null,
+              },
+              remote_sampling: {
+                strategy_file: '/etc/agent/strategy.json',
+                tls: {
+                  insecure: true,
+                },
+              },
+            },
+          },
+          remote_write: [{
+            endpoint: '%s:4317' % $.tempo.fqdn,
+            insecure: true,
+            retry_on_failure: {
+              enabled: true,
+            },
+          }],
+          scrape_configs: agent.newKubernetesTraces({}),
+        }],
+      },
+    } },
+  },
+
+
+  tempo:
+    tempo {
+      local cluster_name = 'dev',
+      _config+:: {
+        cluster_name: cluster_name,
+        namespace: 'monitoring',
+        receivers: {
+          jaeger: {
+            protocols: {
+              grpc: null,
+              thrift_http: null,
+              thrift_binary: null,
+              thrift_compact: null,
+            },
+          },
+          zipkin: null,
+          otlp: {
+            protocols: {
+              http: null,
+              grpc: null,
+            },
+          },
+          opencensus: null,
+        },
+
+        // need to set something, but will use empty dir anyhow
+        pvc_size: null,
+        pvc_storage_class: null,
+      },
+
+      tempo_container+:
+        k.util.resourcesRequests('25m', '128Mi') +
+        k.util.resourcesLimits(null, null),
+
+      tempo_statefulset+: {
+        spec+: {
+          volumeClaimTemplates: [
+          ],
+          template+: { spec+: {
+            volumes+:
+              local volume = k.core.v1.volume;
+              [
+                volume.fromEmptyDir('tempo-data'),
+              ],
+          } },
+        },
+      },
+    } {
+      fqdn:: 'tempo.%s.svc.cluster.local.' % namespace,
+    },
+
+  grafana_datasource_config_map+: k.core.v1.configMap.withDataMixin({
+    'tempo-datasource.yml': k.util.manifestYaml({
+      apiVersion: 1,
+      datasources: [
+        prom_k_grafana.grafana_datasource(
+          'Tempo',
+          'http://%s:3200' % $.tempo.fqdn,
+          type='tempo'
+        ),
+      ],
+    }),
+  }),
+}