Add autonode script exporter config (#1064)

* Add autonode script exporter config * Add early_exit=250 and monitoring parameters to locate
m-lab · Oct 22, 2024 · 7e3a737 · 7e3a737
1 parent 64e28ad
commit 7e3a737
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 2 deletions.
diff --git a/config/federation/prometheus/prometheus.yml.template b/config/federation/prometheus/prometheus.yml.template
@@ -696,6 +696,51 @@ scrape_configs:
         target_label: __address__
         replacement: script-exporter.default.svc.cluster.local:9172
 
+  # Scrape config for the script_exporter for autonodes.
+  - job_name: 'script-targets-autonode'
+    metrics_path: /probe
+    scrape_timeout: 50s
+
+    file_sd_configs:
+      - files:
+          - /script-targets-autonode/*.json
+        # Attempt to re-read files every five minutes.
+        refresh_interval: 5m
+
+    # The relabel config sets the script exporter address with the correct
+    # parameters for the target node.
+    relabel_configs:
+      # The value of the source label named "service" must match the name of a
+      # configured script in the script_exporter configuration. See this file:
+      # https://github.com/m-lab/prometheus-support/blob/main/config/federation/script-exporter/script_exporter.yml
+      - source_labels: [service]
+        regex: (.*)
+        target_label: __param_name
+        replacement: ${1}
+
+      # The default __address__ value is a target host from the config file.
+      # Here, we set (i.e. "replace") a request parameter "target" equal to the
+      # host value.
+      - source_labels: [__address__]
+        regex: (.*)
+        target_label: __param_target
+        replacement: ${1}
+
+      # Extract the site name from the FQDN of the node/service and drop it
+      # into a label named "site'.
+      - source_labels: [__address__]
+        regex: ^[a-z0-9^-]+-([a-z]{3}[0-9]{1,10})-.*
+        target_label: site
+        replacement: ${1}
+
+      # Since __address__ is the target that prometheus will contact,
+      # unconditionally reset the __address__ label to the script_exporter
+      # address.
+      - source_labels: []
+        regex: .*
+        target_label: __address__
+        replacement: script-exporter.default.svc.cluster.local:9172
+
   # Scrape config for App Engine Flex VMs.
   #
   # In order to scrape Prometheus metrics directly from Flex VMs, the app.yaml

diff --git a/config/federation/script-exporter/script_exporter.yml b/config/federation/script-exporter/script_exporter.yml
@@ -8,3 +8,9 @@ scripts:
       monitoring-token -machine=${TARGET} -service=wehe/replay --
       wehe-client.sh -n applemusic -t wehe-cmdline/res/ -c
     timeout: 50
+  - name: 'ndt7_client'
+    script: >
+      EXPERIMENT=ndt7 cache_exit_code.sh 3600
+      monitoring-token -machine=${TARGET} -service=ndt/ndt7 -service-url --env-name SERVICE_URL -service-url-key-name wss:///ndt/v7/download --
+      ndt7-client
+    timeout: 50
diff --git a/k8s/prometheus-federation/deployments/prometheus.yml b/k8s/prometheus-federation/deployments/prometheus.yml
@@ -80,6 +80,9 @@ spec:
         - mountPath: /script-targets-federation
           name: prometheus-storage
           subPath: script-targets-federation
+        - mountPath: /script-targets-autonode
+          name: prometheus-storage
+          subPath: script-targets-autonode
         # /bmc-targets should contain BMC target config files.
         - mountPath: /bmc-targets
           name: prometheus-storage
@@ -102,6 +105,8 @@ spec:
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/script-targets/ndt5_client.json",
                 "--http-target=/targets/script-targets-federation/wehe_client.json",
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/script-targets/wehe_client.json",
+                "--http-target=/targets/script-targets-autonode/byos-nodes.json",
+                "--http-source=https://autojoin-dot-{{AUTOJOIN_PROJECT}}.appspot.com/autojoin/v0/node/list?format=script-exporter&service=ndt7_client",
                 "--http-target=/targets/blackbox-targets/ndt_raw.json",
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/ndt_raw.json",
                 "--http-target=/targets/blackbox-targets-ipv6/ndt_raw_ipv6.json",

diff --git a/k8s/prometheus-federation/deployments/script-exporter.yml b/k8s/prometheus-federation/deployments/script-exporter.yml
@@ -32,7 +32,7 @@ spec:
 
       containers:
       - name: script-exporter
-        image: measurementlab/script-exporter-support:v0.2.10
+        image: measurementlab/script-exporter-support:v0.2.11
         args:
         - -config.file=/etc/script_exporter/script_exporter.yml
         env:
@@ -41,7 +41,7 @@ spec:
         - name: MONITORING_SIGNER_KEY
           value: /keys/monitoring-signer-key.json
         - name: LOCATE_URL
-          value: https://locate-dot-{{LOCATE_PROJECT}}.appspot.com/v2/platform/monitoring/
+          value: https://locate-dot-{{LOCATE_PROJECT}}.appspot.com/v2/platform/monitoring/?early_exit=250&monitoring=true
         ports:
         - containerPort: 9172
         volumeMounts:

diff --git a/k8s/prometheus-federation/mlab-oti.yml b/k8s/prometheus-federation/mlab-oti.yml
@@ -2,6 +2,7 @@
 
 GCLOUD_PROJECT: mlab-oti
 LOCATE_PROJECT: mlab-ns
+AUTOJOIN_PROJECT: mlab-autojoin
 PROMETHEUS_VOLUME_SIZE: 400Gi
 EXTERNAL_IP: 35.184.81.106
 GITHUB_RECEIVER_INMEMORY: false

diff --git a/k8s/prometheus-federation/mlab-sandbox.yml b/k8s/prometheus-federation/mlab-sandbox.yml
@@ -2,6 +2,7 @@
 
 GCLOUD_PROJECT: mlab-sandbox
 LOCATE_PROJECT: mlab-sandbox
+AUTOJOIN_PROJECT: mlab-sandbox
 PROMETHEUS_VOLUME_SIZE: 500Gi
 EXTERNAL_IP: 35.184.166.181
 GITHUB_RECEIVER_INMEMORY: true

diff --git a/k8s/prometheus-federation/mlab-staging.yml b/k8s/prometheus-federation/mlab-staging.yml
@@ -2,6 +2,7 @@
 
 GCLOUD_PROJECT: mlab-staging
 LOCATE_PROJECT: mlab-staging
+AUTOJOIN_PROJECT: mlab-staging
 PROMETHEUS_VOLUME_SIZE: 400Gi
 EXTERNAL_IP: 35.185.76.159
 GITHUB_RECEIVER_INMEMORY: true