🌱 hack/observability: Bump Loki, Grafana and promtail charts

Tiltfile

@@ -403,11 +403,11 @@ def deploy_provider_crds():
 def deploy_observability():
     if "promtail" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/promtail.observability.yaml"), allow_duplicates = True)
-        k8s_resource(workload = "promtail", extra_pod_selectors = [{"app": "promtail"}], labels = ["observability"], resource_deps = ["loki"])
+        k8s_resource(workload = "promtail", extra_pod_selectors = [{"app": "promtail"}], labels = ["observability"], resource_deps = ["loki"], objects = ["promtail:serviceaccount"])
 
     if "loki" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/loki.observability.yaml"), allow_duplicates = True)
-        k8s_resource(workload = "loki", port_forwards = "3100", extra_pod_selectors = [{"app": "loki"}], labels = ["observability"])
+        k8s_resource(workload = "loki", port_forwards = "3100", extra_pod_selectors = [{"app": "loki"}], labels = ["observability"], objects = ["loki:serviceaccount"])
 
         cmd_button(
             "loki:import logs",
@@ -426,11 +426,11 @@ def deploy_observability():
 
     if "prometheus" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/prometheus.observability.yaml"), allow_duplicates = True)
-        k8s_resource(workload = "prometheus-server", new_name = "prometheus", port_forwards = "9090", extra_pod_selectors = [{"app": "prometheus"}], labels = ["observability"])
+        k8s_resource(workload = "prometheus-server", new_name = "prometheus", port_forwards = "9090", extra_pod_selectors = [{"app": "prometheus"}], labels = ["observability"], objects = ["prometheus-server:serviceaccount"])
 
     if "kube-state-metrics" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/kube-state-metrics.observability.yaml"), allow_duplicates = True)
-        k8s_resource(workload = "kube-state-metrics", new_name = "kube-state-metrics", extra_pod_selectors = [{"app": "kube-state-metrics"}], labels = ["observability"])
+        k8s_resource(workload = "kube-state-metrics", new_name = "kube-state-metrics", extra_pod_selectors = [{"app": "kube-state-metrics"}], labels = ["observability"], objects = ["kube-state-metrics:serviceaccount"])
 
     if "visualizer" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/visualizer.observability.yaml"), allow_duplicates = True)
@@ -439,6 +439,7 @@ def deploy_observability():
             new_name = "visualizer",
             port_forwards = [port_forward(local_port = 8000, container_port = 8081, name = "View visualization")],
             labels = ["observability"],
+            objects = ["capi-visualizer:serviceaccount"],
         )
 
 def prepare_all():

docs/book/src/developer/providers/migrations/v1.4-to-v1.5.md

@@ -35,4 +35,4 @@ maintainers of providers and consumers of our Go API.
 
 ### Suggested changes for providers
 
-- 
+- 

hack/observability/grafana/kustomization.yaml

@@ -7,4 +7,4 @@ helmCharts:
     releaseName: grafana
     namespace: observability
     valuesFile: values.yaml
-    version: 6.35.0
+    version: 6.56.5

hack/observability/grafana/values.yaml

@@ -22,6 +22,12 @@ datasources:
     - name: Loki
       type: loki
       url: http://loki:3100
+      # This header has to be set as we had to set an OrgID
+      # in promtail to be able to push the logs to Loki.
+      jsonData:
+        httpHeaderName1: 'X-Scope-OrgID'
+      secureJsonData:
+        httpHeaderValue1: '1'
     - name: Prometheus
       type: prometheus
       url: http://prometheus-server

hack/observability/loki/kustomization.yaml

@@ -7,4 +7,4 @@ helmCharts:
     releaseName: loki
     namespace: observability
     valuesFile: values.yaml
-    version: 2.16.0
+    version: 5.5.2

hack/observability/loki/values.yaml

@@ -1,10 +1,45 @@
 # Placeholder for loki chart configuration, see https://github.com/grafana/helm-charts/tree/main/charts/loki
-
-# Set high ingestion limits so log-push can push logs without hitting the rate limits:
-# Push response: status: "429 Too Many Requests", body: "Ingestion rate limit exceeded for user fake (limit:
-# 4194304 bytes/sec) while attempting to ingest '1000' lines totaling '755293' bytes, reduce log volume or
-# contact your Loki administrator to see if the limit can be increased"
-config:
+# This configuration is necessary to deploy Loki in the
+# "Single Binary" mode.
+loki:
+  commonConfig:
+    replication_factor: 1
+  storage:
+    type: 'filesystem'
+  rulerConfig:
+    storage:
+      type: local
+  # Set high ingestion limits so log-push can push logs without hitting the rate limits:
+  # Push response: status: "429 Too Many Requests", body: "Ingestion rate limit exceeded for user fake (limit:
+  # 4194304 bytes/sec) while attempting to ingest '1000' lines totaling '755293' bytes, reduce log volume or
+  # contact your Loki administrator to see if the limit can be increased"
   limits_config:
     ingestion_rate_mb: 1024
     ingestion_burst_size_mb: 1024
+singleBinary:
+  replicas: 1
+
+# We are disabling basically everything because we just want
+# Loki and no additional monitoring.
+monitoring:
+  dashboards:
+    # -- If enabled, create configmap with dashboards for monitoring Loki
+    enabled: false
+  rules:
+    # -- If enabled, create PrometheusRule resource with Loki recording rules
+    enabled: false
+  alerts:
+    # -- If enabled, create PrometheusRule resource with Loki alerting rules
+    enabled: false
+  serviceMonitor:
+    # -- If enabled, ServiceMonitor resources for Prometheus Operator are created
+    enabled: false
+  selfMonitoring:
+    enabled: false
+    grafanaAgent:
+      installOperator: false
+  lokiCanary:
+    enabled: false
+
+test:
+  enabled: false

hack/observability/promtail/kustomization.yaml

@@ -7,4 +7,4 @@ helmCharts:
     releaseName: promtail
     namespace: observability
     valuesFile: values.yaml
-    version: 6.3.0
+    version: 6.11.2

hack/observability/promtail/values.yaml

@@ -4,6 +4,9 @@ config:
   # publish data to loki
   clients:
     - url: http://loki:3100/loki/api/v1/push
+      # We have to set the tenant_id/OrgID to push logs to Loki
+      # otherwise we get an auth error.
+      tenant_id: 1
 
   snippets:
     pipelineStages:

hack/tools/log-push/main.go

@@ -52,6 +52,7 @@ var (
 	logFileRegex            = flag.String("log-file-regex", "manager\\.log", "Regex used to find log files")
 	logJSONAdditionalLabels = flag.String("log-json-additional-labels", "controller,cluster,machine", "Comma-separated list of additional labels to parse from JSON logs")
 	lokiURL                 = flag.String("loki-url", "http://localhost:3100/loki/api/v1/push", "Loki URL to push the logs to")
+	lokiOrgID               = flag.String("loki-org-id", "1", "Loki Org ID used when pushing logs")
 )
 
 func main() {
@@ -67,15 +68,15 @@ func main() {
 	}
 	logJSONAdditionalLabelsArray := strings.Split(*logJSONAdditionalLabels, ",")
 
-	if err := importLogs(*logPath, logFileRegexp, logJSONAdditionalLabelsArray, *lokiURL); err != nil {
+	if err := importLogs(*logPath, logFileRegexp, logJSONAdditionalLabelsArray, *lokiURL, *lokiOrgID); err != nil {
 		fmt.Printf("Failed to import logs: %v\n", err)
 		os.Exit(1)
 	}
 
 	klog.Infof("Finished syncing logs from %s", *logPath)
 }
 
-func importLogs(logPath string, logFileRegex *regexp.Regexp, logJSONAdditionalLabels []string, lokiURL string) error {
+func importLogs(logPath string, logFileRegex *regexp.Regexp, logJSONAdditionalLabels []string, lokiURL, lokiOrgID string) error {
 	ctx := context.Background()
 
 	// Get Logs.
@@ -92,7 +93,7 @@ func importLogs(logPath string, logFileRegex *regexp.Regexp, logJSONAdditionalLa
 		}
 
 		// Push logs to Loki.
-		if err := pushLogsToLoki(ctx, lokiURL, logFile, streams); err != nil {
+		if err := pushLogsToLoki(ctx, lokiURL, lokiOrgID, logFile, streams); err != nil {
 			return errors.Wrapf(err, "failed to push logs to Loki")
 		}
 	}
@@ -378,7 +379,7 @@ func prepareLogsForLoki(ld LogData, logJSONAdditionalLabels []string) ([]LokiStr
 }
 
 // pushLogsToLoki uploads data to Loki.
-func pushLogsToLoki(ctx context.Context, lokiURL, file string, lokiLogStreamsArray []LokiStreams) error {
+func pushLogsToLoki(ctx context.Context, lokiURL, lokiOrgID, file string, lokiLogStreamsArray []LokiStreams) error {
 	klog.Infof("Pushing logs to Loki from: %s", file)
 
 	for _, streams := range lokiLogStreamsArray {
@@ -388,15 +389,15 @@ func pushLogsToLoki(ctx context.Context, lokiURL, file string, lokiLogStreamsArr
 			return errors.Wrapf(err, "failed to marshal Loki stream")
 		}
 
-		if err := pushStreamToLoki(ctx, lokiURL, body); err != nil {
+		if err := pushStreamToLoki(ctx, lokiURL, lokiOrgID, body); err != nil {
 			return errors.Wrapf(err, "failed to push Loki stream")
 		}
 	}
 
 	return nil
 }
 
-func pushStreamToLoki(ctx context.Context, lokiURL string, body []byte) error {
+func pushStreamToLoki(ctx context.Context, lokiURL, lokiOrgID string, body []byte) error {
 	// gzip JSON into buf.
 	var buf bytes.Buffer
 	gz := gzip.NewWriter(&buf)
@@ -414,6 +415,7 @@ func pushStreamToLoki(ctx context.Context, lokiURL string, body []byte) error {
 	}
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Content-Encoding", "gzip")
+	req.Header.Set("X-Scope-OrgID", lokiOrgID)
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {

hack/tools/tilt-prepare/main.go

@@ -55,6 +55,7 @@ import (
 	clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
+	logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
 	"sigs.k8s.io/cluster-api/test/infrastructure/container"
 	utilyaml "sigs.k8s.io/cluster-api/util/yaml"
 )
@@ -149,6 +150,10 @@ func init() {
 // This tool aims to speed up tilt startup time by running in parallel a set of task
 // preparing everything is required for tilt up.
 func main() {
+	// Set clusterctl logger with a log level of 5.
+	// This makes it easier to see what clusterctl is doing and to debug it.
+	logf.SetLogger(logf.NewLogger(logf.WithThreshold(pointer.Int(5))))
+
 	klog.Infof("[main] started\n")
 	start := time.Now()