Initial commit

.github/workflows/default.yml

@@ -0,0 +1,40 @@
+name: default
+
+on:
+  push:
+    tags:
+      - '*'
+
+permissions:
+  # Required for Goreleaser
+  contents: write
+  packages: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        name: Checkout repository
+        with:
+          # Fetch all history for all tags and branches
+          fetch-depth: 0
+
+      - uses: docker/login-action@v2
+        name: Login to GitHub Packages
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/setup-go@v4
+        name: Setup Go
+        with:
+          go-version-file: 'go.mod'
+
+      - uses: goreleaser/goreleaser-action@v4
+        name: Run GoReleaser
+        with:
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,2 @@
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,46 @@
+project_name: nomad-service-discovery-exporter
+
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+
+archives:
+  - format: tar.gz
+    name_template: >-
+      {{ .ProjectName }}.
+      {{- .Version }}.
+      {{- .Os }}-
+      {{- .Arch }}
+
+dockers:
+  - use: buildx
+    goos: linux
+    goarch: amd64
+    image_templates:
+      - "ghcr.io/strigo/{{ .ProjectName }}:{{ .Version }}-amd64"
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/amd64"
+  - use: buildx
+    goos: linux
+    goarch: arm64
+    image_templates:
+      - "ghcr.io/strigo/{{ .ProjectName }}:{{ .Version }}-arm64"
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/arm64"
+
+docker_manifests:
+- name_template: "ghcr.io/strigo/{{ .ProjectName }}:{{ .Version }}"
+  image_templates:
+    - "ghcr.io/strigo/{{ .ProjectName }}:{{ .Version }}-amd64"
+    - "ghcr.io/strigo/{{ .ProjectName }}:{{ .Version }}-arm64"
+
+checksum:
+  name_template: "sha256sums.txt"

Dockerfile

@@ -0,0 +1,9 @@
+FROM scratch
+
+LABEL org.opencontainers.image.source=https://github.com/strigo/nomad-service-discovery-exporter
+LABEL org.opencontainers.image.description="A Prometheus exporter that reports the health status of services in Nomad's native service discovery"
+LABEL org.opencontainers.image.licenses=MIT
+
+COPY nomad-service-discovery-exporter /nomad-service-discovery-exporter
+
+ENTRYPOINT ["/nomad-service-discovery-exporter"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Strigo Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,28 @@
+Nomad Service Discovery Exporter
+================================
+
+A Prometheus exporter that reports the health status of services in [Nomad's
+native service discovery][1].
+
+This is a temporary solution until [nomad/16602][2] is fixed/implemented.
+
+
+Compatibility
+-------------
+The exporter uses an undocumented API endpoint which might change/break between
+new releases. This exporter was tested against Nomad v1.5.x.
+
+
+Command line flags
+------------------
+See `-help` for details.
+
+
+Available Metrics
+-----------------
+* `nomad_services`: The total number of services registered.
+* `nomad_services_health`: Service health status.
+
+
+[1]: https://developer.hashicorp.com/nomad/docs/networking/service-discovery
+[2]: https://github.com/hashicorp/nomad/issues/16602

exporter.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sync"
+	"time"
+
+	nomad "github.com/hashicorp/nomad/api"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	CheckSuccess = "success"
+	CheckFailure = "failure"
+	CheckPending = "pending"
+)
+
+var (
+	metricServicesTotal = metricInfo{
+		prometheus.NewDesc(
+			"nomad_services",
+			"Number of services registers to Nomad native service discovery",
+			[]string{"namespace"},
+			nil),
+		prometheus.GaugeValue,
+	}
+
+	metricServicesHealth = metricInfo{
+		prometheus.NewDesc(
+			"nomad_services_health",
+			"Health status of a service registered to Nomad Service Discovery",
+			[]string{"namespace", "job_id", "task_name", "service_name", "check_name", "check_id", "status"},
+			nil),
+		prometheus.GaugeValue,
+	}
+)
+
+type metricInfo struct {
+	Desc *prometheus.Desc
+	Type prometheus.ValueType
+}
+
+type Exporter struct {
+	Config       *ExporterConfig
+	client       *nomad.Client
+	queryOptions *nomad.QueryOptions
+
+	mutex        sync.RWMutex
+	cache        sync.Map
+	limit        chan struct{}
+
+	totalErrs prometheus.Counter
+}
+
+type ExporterConfig struct {
+	Address   string
+	Region    string
+	Namespace string
+	SecretID  string
+
+	Duration    time.Duration
+	Parallelism int
+	AllowStale  bool
+}
+
+
+func New(config *ExporterConfig) (*Exporter, error) {
+	client, err := nomad.NewClient(&nomad.Config{
+		Address:  config.Address,
+		Region:   config.Region,
+		SecretID: config.SecretID,
+		Namespace: config.Namespace,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	exporter := Exporter{
+		client:       client,
+		queryOptions: &nomad.QueryOptions{AllowStale: config.AllowStale},
+		limit:        make(chan struct{}, config.Parallelism),
+		Config:       config,
+
+		totalErrs: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "nomad_services_api_errors_total",
+			Help: "Number of scrapes that resulted with one or more API errors from Nomad",
+		}),
+	}
+
+	return &exporter, nil
+}
+
+func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
+	ch <- metricServicesTotal.Desc
+	ch <- metricServicesHealth.Desc
+	ch <- e.totalErrs.Desc()
+}
+
+func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+	e.cache = sync.Map{}
+
+	// common context for all the requests - cancel everything when deadline reached
+	ctx, cancel := context.WithTimeout(context.Background(), e.Config.Duration)
+	defer cancel()
+	e.queryOptions = e.queryOptions.WithContext(ctx)
+
+	err := e.collectServices(ch)
+	if err != nil {
+		log.Println("scrape error ", err)
+		e.totalErrs.Inc()
+	}
+
+	ch <- e.totalErrs
+}
+
+func (e *Exporter) collectServices(ch chan<- prometheus.Metric) error {
+	e.limit <- struct{}{}
+	listStub, _, err := e.client.Services().List(e.queryOptions)
+	<-e.limit
+	if err != nil {
+		return err
+	}
+
+	errg := new(errgroup.Group)
+
+	for _, stub := range listStub {
+		stub := stub
+		ch <- prometheus.MustNewConstMetric(
+			metricServicesTotal.Desc, metricServicesTotal.Type, float64(len(stub.Services)), stub.Namespace,
+		)
+
+		errg.Go(func() error {
+			return e.collectNamespace(stub.Services, ch)
+		})
+	}
+	return errg.Wait()
+}
+
+func (e *Exporter) collectNamespace(services []*nomad.ServiceRegistrationStub, ch chan<- prometheus.Metric) error {
+	errg := new(errgroup.Group)
+
+	for _, svc := range services {
+		svc := svc
+		errg.Go(func() error {
+			return e.collectService(svc.ServiceName, ch)
+		})
+	}
+
+	return errg.Wait()
+}
+
+func (e *Exporter) collectService(svc string, ch chan<- prometheus.Metric) error {
+	errg := new(errgroup.Group)
+
+	e.limit <- struct{}{}
+	registrations, _, err := e.client.Services().Get(svc, e.queryOptions)
+	<-e.limit
+	if err != nil {
+		return err
+	}
+
+	for _, r := range registrations {
+		// same allocID can exists for multiple registrations
+		// avoid scanning and reporting metrics for same allocs (prometheus will complain)
+		r := r
+		if _, exists := e.cache.LoadOrStore(r.AllocID, struct{}{}); exists {
+			continue
+		}
+
+		errg.Go(func() error {
+			return e.collectAllocation(r, ch)
+		})
+	}
+
+	return errg.Wait()
+}
+
+func (e *Exporter) collectAllocation(reg *nomad.ServiceRegistration, ch chan<- prometheus.Metric) error {
+	e.limit <- struct{}{}
+	statuses, err := e.client.Allocations().Checks(reg.AllocID, e.queryOptions)
+	<-e.limit
+	if err != nil {
+		log.Printf("unable to scrape allocation %v: %v\n", reg.AllocID, err)
+		return err
+	}
+
+	for _, status := range statuses {
+		// two types of checks exists: readiness and healthiness, we only care about the later
+		if status.Mode != "healthiness" {
+			continue
+		}
+
+		var healthy, failure, pending float64
+		switch status.Status {
+		case CheckSuccess:
+			healthy = 1
+		case CheckFailure:
+			failure = 1
+		case CheckPending:
+			pending = 1
+		}
+
+		ch <- prometheus.MustNewConstMetric(
+			metricServicesHealth.Desc, metricServicesHealth.Type, healthy, reg.Namespace, reg.JobID, status.Task, status.Service, status.Check, status.ID, CheckSuccess,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			metricServicesHealth.Desc, metricServicesHealth.Type, failure, reg.Namespace, reg.JobID, status.Task, status.Service, status.Check, status.ID, CheckFailure,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			metricServicesHealth.Desc, metricServicesHealth.Type, pending, reg.Namespace, reg.JobID, status.Task, status.Service, status.Check, status.ID, CheckPending,
+		)
+	}
+
+	return nil
+}