From c279da30067b204b52bf1c2a9e4177887ec414ea Mon Sep 17 00:00:00 2001 From: "Cel A. Skeggs" Date: Tue, 9 Jan 2018 19:12:19 -0500 Subject: [PATCH] Add package for etcd-metrics-exporter --- building/SHA512SUM.UPSTREAM | 1 + building/build-debs/.gitignore | 1 + .../.gitignore | 6 + .../build-package.sh | 8 + .../debian/changelog | 5 + .../debian/compat | 1 + .../debian/control | 14 ++ .../debian/copyright | 35 +++ .../debian/docs | 0 .../debian/install | 2 + .../debian/rules | 30 +++ .../debian/source/format | 1 + .../inner-build.sh | 30 +++ .../services/etcd-metrics-exporter.service | 12 + .../src/etcd-metrics-exporter.go | 210 ++++++++++++++++++ 15 files changed, 356 insertions(+) create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/.gitignore create mode 100755 building/build-debs/homeworld-etcd-metrics-exporter/build-package.sh create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/debian/changelog create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/debian/compat create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/debian/control create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/debian/copyright create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/debian/docs create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/debian/install create mode 100755 building/build-debs/homeworld-etcd-metrics-exporter/debian/rules create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/debian/source/format create mode 100755 building/build-debs/homeworld-etcd-metrics-exporter/inner-build.sh create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/services/etcd-metrics-exporter.service create mode 100644 building/build-debs/homeworld-etcd-metrics-exporter/src/etcd-metrics-exporter.go diff --git a/building/SHA512SUM.UPSTREAM b/building/SHA512SUM.UPSTREAM index 46d2ba599..e5835acda 100644 --- a/building/SHA512SUM.UPSTREAM +++ b/building/SHA512SUM.UPSTREAM @@ -16,3 +16,4 @@ e16d8b7f4f026b6a95b11fb59c54ec5f114f6f516294eaa95e718abdf5d37c17a9c4b5e0a0a61fca 4e5172205ebc0c7c19974af4565a9e59163ea78aa9d13367a944614e712c19a06f93986360cae6acf934bc0baab7b403d4deac292dd3715d4d479075ee068be3 prometheus-2.0.0.tar.xz af7cdf5fec21f1a90ab2e608470f3bdd85fc03b57c99bf90dc86efcf3831cc9422604ea1acc26b23cfef0e7b258cf45786097bb9e63259734ef26f89a6417838 promu-sipb-0.1.1.tar.xz 130402cfa96be87628955fac30e82c9b1c83548e2bedfb6569533f7179b3fa911a1b5d8ba67516bb36fd0cf6515cf0e279c0772dab6227d83fc8ddd38ee88e29 prometheus-node-exporter-0.15.2.tar.xz +505629881c24fb2397eb56a7c5f8e7bdca7406a7d08cd097d5ea343218895cc1df9b3318024a628ed53ee16f0bb1f64169a49c6e9642fe87fe1fe462c7bf6d7d prometheus-client_golang-0.9.0-pre1.tar.xz diff --git a/building/build-debs/.gitignore b/building/build-debs/.gitignore index aaceb8593..28d6fa4a7 100644 --- a/building/build-debs/.gitignore +++ b/building/build-debs/.gitignore @@ -19,3 +19,4 @@ homeworld-keysystem_*.tar.gz homeworld-admin-tools_*.tar.gz homeworld-prometheus_*.tar.gz homeworld-prometheus-node-exporter_*.tar.gz +homeworld-etcd-metrics-exporter_*.tar.gz diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/.gitignore b/building/build-debs/homeworld-etcd-metrics-exporter/.gitignore new file mode 100644 index 000000000..93955dc9d --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/.gitignore @@ -0,0 +1,6 @@ +go-bin-* +prometheus-*.tar.xz +/prometheus-* +/bin/ +/src/github.com +/go/ diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/build-package.sh b/building/build-debs/homeworld-etcd-metrics-exporter/build-package.sh new file mode 100755 index 000000000..9ca6f9f59 --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/build-package.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eu +source ../common/package-build-helpers.sh + +importgo +upstream "prometheus-2.0.0.tar.xz" +upstream "prometheus-client_golang-0.9.0-pre1.tar.xz" +build diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/changelog b/building/build-debs/homeworld-etcd-metrics-exporter/debian/changelog new file mode 100644 index 000000000..41d01653e --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/debian/changelog @@ -0,0 +1,5 @@ +homeworld-etcd-metrics-exporter (0.1.0) stretch; urgency=medium + + * Initial release + + -- Cel Skeggs Tue, 09 Jan 2018 17:55:05 -0500 diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/compat b/building/build-debs/homeworld-etcd-metrics-exporter/debian/compat new file mode 100644 index 000000000..ec635144f --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/debian/compat @@ -0,0 +1 @@ +9 diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/control b/building/build-debs/homeworld-etcd-metrics-exporter/debian/control new file mode 100644 index 000000000..63d0bb7f3 --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/debian/control @@ -0,0 +1,14 @@ +Source: homeworld-etcd-metrics-exporter +Section: misc +Priority: extra +Maintainer: Cel Skeggs +Build-Depends: debhelper (>= 9) +Standards-Version: 4.1.1 +Vcs-Git: https://github.com/sipb/homeworld.git +Vcs-Browser: https://github.com/sipb/homeworld/ + +Package: homeworld-etcd-metrics-exporter +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends} +Description: Homeworld etcd-metrics-exporter package. + This package is used for code deployment to Homeworld clusters. diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/copyright b/building/build-debs/homeworld-etcd-metrics-exporter/debian/copyright new file mode 100644 index 000000000..15c442c87 --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/debian/copyright @@ -0,0 +1,35 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: etcd-metrics-exporter +Source: etcd-metrics-exporter-0.1.0.tar.xz + +Files: debian/* +Copyright: 2017 Cel Skeggs +License: MIT + MIT License + . + Copyright (c) 2017 Cel Skeggs + . + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + +# Please also look if there are files or directories which have a +# different copyright/license attached and list them here. +# Please avoid to pick license terms that are more restrictive than the +# packaged work, as it may make Debian's contributions unacceptable upstream. + diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/docs b/building/build-debs/homeworld-etcd-metrics-exporter/debian/docs new file mode 100644 index 000000000..e69de29bb diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/install b/building/build-debs/homeworld-etcd-metrics-exporter/debian/install new file mode 100644 index 000000000..d1654dcd9 --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/debian/install @@ -0,0 +1,2 @@ +etcd-metrics-exporter usr/bin +services/etcd-metrics-exporter.service usr/lib/systemd/system diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/rules b/building/build-debs/homeworld-etcd-metrics-exporter/debian/rules new file mode 100755 index 000000000..4f4add386 --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/debian/rules @@ -0,0 +1,30 @@ +#!/usr/bin/make -f +# See debhelper(7) (uncomment to enable) +# output every command that modifies files on the build system. +#DH_VERBOSE = 1 + +# see EXAMPLES in dpkg-buildflags(1) and read /usr/share/dpkg/* +DPKG_EXPORT_BUILDFLAGS = 1 +include /usr/share/dpkg/default.mk + +# see FEATURE AREAS in dpkg-buildflags(1) +#export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +# see ENVIRONMENT in dpkg-buildflags(1) +# package maintainers to append CFLAGS +#export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic +# package maintainers to append LDFLAGS +#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed + +override_dh_auto_build: + VERSION="${DEB_VERSION_UPSTREAM}" ./inner-build.sh + +# main packaging script based on dh7 syntax +%: + dh $@ + +# debmake generated override targets +# This is example for Cmake (See http://bugs.debian.org/641051 ) +#override_dh_auto_configure: +# dh_auto_configure -- \ +# -DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH) diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/debian/source/format b/building/build-debs/homeworld-etcd-metrics-exporter/debian/source/format new file mode 100644 index 000000000..d3827e75a --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/debian/source/format @@ -0,0 +1 @@ +1.0 diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/inner-build.sh b/building/build-debs/homeworld-etcd-metrics-exporter/inner-build.sh new file mode 100755 index 000000000..5afba9fea --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/inner-build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e -u + +rm -rf go +tar -xf go-bin-1.8.3.tgz go/ +export GOROOT="$(pwd)/go/" +export PATH="$PATH:$GOROOT/bin" + +if [ "$(go version 2>/dev/null)" != "go version go1.8.3 linux/amd64" ] +then + echo "go version mismatch! expected 1.8.3" 1>&2 + go version 1>&2 + exit 1 +fi + +PROMETHEUS_VER="2.0.0" +PROMETHEUS_CLIENT_GOLANG_VER="0.9.0-pre1" + +rm -f src/github.com +rm -rf "prometheus-${PROMETHEUS_VER}" +tar -xf "prometheus-${PROMETHEUS_VER}.tar.xz" "prometheus-${PROMETHEUS_VER}/vendor" +tar -xf "prometheus-client_golang-${PROMETHEUS_CLIENT_GOLANG_VER}.tar.xz" "client_golang-${PROMETHEUS_CLIENT_GOLANG_VER}" +rm -rf "prometheus-${PROMETHEUS_VER}/vendor/github.com/prometheus/client_golang/" +mv "client_golang-${PROMETHEUS_CLIENT_GOLANG_VER}" -T "prometheus-${PROMETHEUS_VER}/vendor/github.com/prometheus/client_golang/" +ln -s "../prometheus-${PROMETHEUS_VER}/vendor/github.com" src/github.com + +export GOPATH="$(pwd)" +(cd src && go build -o ../etcd-metrics-exporter etcd-metrics-exporter.go) + +echo "etcd-metrics-exporter built!" diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/services/etcd-metrics-exporter.service b/building/build-debs/homeworld-etcd-metrics-exporter/services/etcd-metrics-exporter.service new file mode 100644 index 000000000..8d46ecdd0 --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/services/etcd-metrics-exporter.service @@ -0,0 +1,12 @@ +[Unit] +Description=etcd metrics exporter +Requires=network-online.target +After=network-online.target + +[Service] +Slice=machine.slice +ExecStart=/usr/bin/etcd-metrics-exporter /etc/homeworld/authorities/etcd-server.pem /etc/homeworld/keys/etcd-client.pem /etc/homeworld/keys/etcd-client.key +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/building/build-debs/homeworld-etcd-metrics-exporter/src/etcd-metrics-exporter.go b/building/build-debs/homeworld-etcd-metrics-exporter/src/etcd-metrics-exporter.go new file mode 100644 index 000000000..36a56eb13 --- /dev/null +++ b/building/build-debs/homeworld-etcd-metrics-exporter/src/etcd-metrics-exporter.go @@ -0,0 +1,210 @@ +/* +Copyright 2018 Cel Skeggs (modifications) +Copyright 2017 The Kubernetes Authors (original) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "log" + "os" + "strings" + "crypto/tls" + "crypto/x509" + "io/ioutil" + "io" +) + +// Initialize the prometheus instrumentation and client related flags. +var ( + etcdScrapeBase = "https://localhost:2379" + + httpClient *http.Client +) + +// Initialize prometheus metrics to be exported. +var ( + // Register all custom metrics with a dedicated registry to keep them separate. + customMetricRegistry = prometheus.NewRegistry() + + // Custom etcd version metric since etcd 3.2- does not export one. + // This will be replaced by https://github.com/coreos/etcd/pull/8960 in etcd 3.3. + etcdVersion = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "etcd", + Name: "version_info", + Help: "Etcd server's binary version", + }, + []string{"binary_version"}) +) + +// monitorGatherer is a custom metric gatherer for prometheus that exports custom metrics +// defined by this monitor as well as rewritten etcd metrics. +type monitorGatherer struct { +} + +func (m *monitorGatherer) Gather() ([]*dto.MetricFamily, error) { + etcdMetrics, err := scrapeMetrics() + if err != nil { + return nil, err + } + custom, err := customMetricRegistry.Gather() + if err != nil { + return nil, err + } + result := make([]*dto.MetricFamily, 0, len(etcdMetrics)+len(custom)) + for _, mf := range etcdMetrics { + result = append(result, mf) + } + result = append(result, custom...) + return result, nil +} + +// Struct for unmarshalling the json response from etcd's /version endpoint. +type EtcdVersion struct { + BinaryVersion string `json:"etcdserver"` + ClusterVersion string `json:"etcdcluster"` +} + +func requestToEtcd(endpoint string) (io.ReadCloser, error) { + if !strings.HasPrefix(endpoint, "/") { + return nil, fmt.Errorf("no '/' prefix on endpoint") + } + + resp, err := httpClient.Get(etcdScrapeBase + endpoint) + if err != nil { + return nil, fmt.Errorf("failed to receive GET response from etcd: %v", err) + } + + return resp.Body, nil +} + +// Function for fetching etcd version info and feeding it to the prometheus metric. +func getVersion(lastSeenBinaryVersion *string) error { + body, err := requestToEtcd("/version") + if err != nil { + return err + } + defer body.Close() + + // Obtain EtcdVersion from the JSON response. + var version EtcdVersion + if err := json.NewDecoder(body).Decode(&version); err != nil { + return fmt.Errorf("Failed to decode etcd version JSON: %v", err) + } + + // Return without updating the version if it stayed the same since last time. + if *lastSeenBinaryVersion == version.BinaryVersion { + return nil + } + + // Delete the metric for the previous version. + if *lastSeenBinaryVersion != "" { + deleted := etcdVersion.Delete(prometheus.Labels{"binary_version": *lastSeenBinaryVersion}) + if !deleted { + return fmt.Errorf("Failed to delete previous version's metric") + } + } + + // Record the new version in a metric. + etcdVersion.With(prometheus.Labels{ + "binary_version": version.BinaryVersion, + }).Set(1) + *lastSeenBinaryVersion = version.BinaryVersion + return nil +} + +// Periodically fetches etcd version info. +func getVersionPeriodically(stopCh <-chan struct{}) { + lastSeenBinaryVersion := "" + for { + if err := getVersion(&lastSeenBinaryVersion); err != nil { + log.Printf("Failed to fetch etcd version: %v", err) + } + select { + case <-stopCh: + break + case <-time.After(15 * time.Second): + } + } +} + +// scrapeMetrics scrapes the prometheus metrics from the etcd metrics URI. +func scrapeMetrics() (map[string]*dto.MetricFamily, error) { + body, err := requestToEtcd("/metrics") + if err != nil { + return nil, err + } + defer body.Close() + + // Parse the metrics in text format to a MetricFamily struct. + var textParser expfmt.TextParser + return textParser.TextToMetricFamilies(body) +} + +func main() { + if len(os.Args) != 4 { + log.Fatal("expected three arguments: authority, keyfile, certfile") + } + + certPool := x509.NewCertPool() + + authorities, err := ioutil.ReadFile(os.Args[1]) + if err != nil { + log.Fatal(err) + } + if !certPool.AppendCertsFromPEM(authorities) { + log.Fatal("could not parse PEM cert for CA") + } + + certCli, err := tls.LoadX509KeyPair(os.Args[3], os.Args[2]) + if err != nil { + log.Fatal(err) + } + + httpClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + Certificates: []tls.Certificate { certCli }, + }, + }, + } + + // Register the metrics we defined above with prometheus. + customMetricRegistry.MustRegister(etcdVersion) + customMetricRegistry.Unregister(prometheus.NewGoCollector()) + + // Spawn threads for periodically scraping etcd version metrics. + stopCh := make(chan struct{}) + defer close(stopCh) + go getVersionPeriodically(stopCh) + + listenAddress := ":9101" + + // Serve our metrics. + log.Printf("Listening on: %v", listenAddress) + http.Handle("/metrics", promhttp.HandlerFor(&monitorGatherer{}, promhttp.HandlerOpts{})) + log.Printf("Stopped listening/serving metrics: %v", http.ListenAndServe(listenAddress, nil)) +}