diff --git a/config/federation/prometheus/prometheus.yml.template b/config/federation/prometheus/prometheus.yml.template index dc7da35e..773e7c6b 100644 --- a/config/federation/prometheus/prometheus.yml.template +++ b/config/federation/prometheus/prometheus.yml.template @@ -696,6 +696,51 @@ scrape_configs: target_label: __address__ replacement: script-exporter.default.svc.cluster.local:9172 + # Scrape config for the script_exporter for autonodes. + - job_name: 'script-targets-autonode' + metrics_path: /probe + scrape_timeout: 50s + + file_sd_configs: + - files: + - /script-targets-autonode/*.json + # Attempt to re-read files every five minutes. + refresh_interval: 5m + + # The relabel config sets the script exporter address with the correct + # parameters for the target node. + relabel_configs: + # The value of the source label named "service" must match the name of a + # configured script in the script_exporter configuration. See this file: + # https://github.com/m-lab/prometheus-support/blob/main/config/federation/script-exporter/script_exporter.yml + - source_labels: [service] + regex: (.*) + target_label: __param_name + replacement: ${1} + + # The default __address__ value is a target host from the config file. + # Here, we set (i.e. "replace") a request parameter "target" equal to the + # host value. + - source_labels: [__address__] + regex: (.*) + target_label: __param_target + replacement: ${1} + + # Extract the site name from the FQDN of the node/service and drop it + # into a label named "site'. + - source_labels: [__address__] + regex: ^[a-z0-9^-]+-([a-z]{3}[0-9]{1,10})-.* + target_label: site + replacement: ${1} + + # Since __address__ is the target that prometheus will contact, + # unconditionally reset the __address__ label to the script_exporter + # address. + - source_labels: [] + regex: .* + target_label: __address__ + replacement: script-exporter.default.svc.cluster.local:9172 + # Scrape config for App Engine Flex VMs. # # In order to scrape Prometheus metrics directly from Flex VMs, the app.yaml diff --git a/config/federation/script-exporter/script_exporter.yml b/config/federation/script-exporter/script_exporter.yml index 848a148e..af78bf5f 100644 --- a/config/federation/script-exporter/script_exporter.yml +++ b/config/federation/script-exporter/script_exporter.yml @@ -8,3 +8,9 @@ scripts: monitoring-token -machine=${TARGET} -service=wehe/replay -- wehe-client.sh -n applemusic -t wehe-cmdline/res/ -c timeout: 50 + - name: 'ndt7_client' + script: > + EXPERIMENT=ndt7 cache_exit_code.sh 3600 + monitoring-token -machine=${TARGET} -service=ndt/ndt7 -service-url --env-name SERVICE_URL -service-url-key-name wss:///ndt/v7/download -- + ndt7-client + timeout: 50 diff --git a/k8s/prometheus-federation/deployments/prometheus.yml b/k8s/prometheus-federation/deployments/prometheus.yml index 2e5c244d..d483a04c 100644 --- a/k8s/prometheus-federation/deployments/prometheus.yml +++ b/k8s/prometheus-federation/deployments/prometheus.yml @@ -80,6 +80,9 @@ spec: - mountPath: /script-targets-federation name: prometheus-storage subPath: script-targets-federation + - mountPath: /script-targets-autonode + name: prometheus-storage + subPath: script-targets-autonode # /bmc-targets should contain BMC target config files. - mountPath: /bmc-targets name: prometheus-storage @@ -102,6 +105,8 @@ spec: "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/script-targets/ndt5_client.json", "--http-target=/targets/script-targets-federation/wehe_client.json", "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/script-targets/wehe_client.json", + "--http-target=/targets/script-targets-autonode/byos-nodes.json", + "--http-source=https://autojoin-dot-{{AUTOJOIN_PROJECT}}.appspot.com/autojoin/v0/node/list?format=script-exporter&service=ndt7_client", "--http-target=/targets/blackbox-targets/ndt_raw.json", "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/ndt_raw.json", "--http-target=/targets/blackbox-targets-ipv6/ndt_raw_ipv6.json", diff --git a/k8s/prometheus-federation/deployments/script-exporter.yml b/k8s/prometheus-federation/deployments/script-exporter.yml index f092e571..3bb69a5a 100644 --- a/k8s/prometheus-federation/deployments/script-exporter.yml +++ b/k8s/prometheus-federation/deployments/script-exporter.yml @@ -32,7 +32,7 @@ spec: containers: - name: script-exporter - image: measurementlab/script-exporter-support:v0.2.10 + image: measurementlab/script-exporter-support:v0.2.11 args: - -config.file=/etc/script_exporter/script_exporter.yml env: @@ -41,7 +41,7 @@ spec: - name: MONITORING_SIGNER_KEY value: /keys/monitoring-signer-key.json - name: LOCATE_URL - value: https://locate-dot-{{LOCATE_PROJECT}}.appspot.com/v2/platform/monitoring/ + value: https://locate-dot-{{LOCATE_PROJECT}}.appspot.com/v2/platform/monitoring/?early_exit=250&monitoring=true ports: - containerPort: 9172 volumeMounts: diff --git a/k8s/prometheus-federation/mlab-oti.yml b/k8s/prometheus-federation/mlab-oti.yml index 69c5407b..7c68d35b 100644 --- a/k8s/prometheus-federation/mlab-oti.yml +++ b/k8s/prometheus-federation/mlab-oti.yml @@ -2,6 +2,7 @@ GCLOUD_PROJECT: mlab-oti LOCATE_PROJECT: mlab-ns +AUTOJOIN_PROJECT: mlab-autojoin PROMETHEUS_VOLUME_SIZE: 400Gi EXTERNAL_IP: 35.184.81.106 GITHUB_RECEIVER_INMEMORY: false diff --git a/k8s/prometheus-federation/mlab-sandbox.yml b/k8s/prometheus-federation/mlab-sandbox.yml index 23a025a9..8f27ab5a 100644 --- a/k8s/prometheus-federation/mlab-sandbox.yml +++ b/k8s/prometheus-federation/mlab-sandbox.yml @@ -2,6 +2,7 @@ GCLOUD_PROJECT: mlab-sandbox LOCATE_PROJECT: mlab-sandbox +AUTOJOIN_PROJECT: mlab-sandbox PROMETHEUS_VOLUME_SIZE: 500Gi EXTERNAL_IP: 35.184.166.181 GITHUB_RECEIVER_INMEMORY: true diff --git a/k8s/prometheus-federation/mlab-staging.yml b/k8s/prometheus-federation/mlab-staging.yml index 5e6240fb..163d04db 100644 --- a/k8s/prometheus-federation/mlab-staging.yml +++ b/k8s/prometheus-federation/mlab-staging.yml @@ -2,6 +2,7 @@ GCLOUD_PROJECT: mlab-staging LOCATE_PROJECT: mlab-staging +AUTOJOIN_PROJECT: mlab-staging PROMETHEUS_VOLUME_SIZE: 400Gi EXTERNAL_IP: 35.185.76.159 GITHUB_RECEIVER_INMEMORY: true