From 83d3593d6ef88a2ac193b75ea22c39bff55179f0 Mon Sep 17 00:00:00 2001 From: Xuewei Zhang Date: Mon, 9 Sep 2019 19:12:31 -0700 Subject: [PATCH] Adding stackdriver exporter --- Makefile | 2 +- .../exporterplugins/default_plugin.go | 20 +++ .../stackdriver_exporter_plugin.go | 25 +++ .../node_problem_detector.go | 16 +- cmd/options/options.go | 22 ++- cmd/options/options_test.go | 54 ++++-- config/exporter/stackdriver-exporter.json | 10 ++ .../node-problem-detector-metric-only.service | 5 +- pkg/exporters/register.go | 62 +++++++ pkg/exporters/register_test.go | 72 ++++++++ pkg/exporters/stackdriver/config/config.go | 55 ++++++ .../stackdriver/config/config_test.go | 107 ++++++++++++ pkg/exporters/stackdriver/gce/type.go | 67 ++++++++ .../stackdriver/stackdriver_exporter.go | 162 ++++++++++++++++++ .../stackdriver/stackdriver_exporter_test.go | 33 ++++ 15 files changed, 690 insertions(+), 22 deletions(-) create mode 100644 cmd/nodeproblemdetector/exporterplugins/default_plugin.go create mode 100644 cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go create mode 100644 config/exporter/stackdriver-exporter.json create mode 100644 pkg/exporters/register.go create mode 100644 pkg/exporters/register_test.go create mode 100644 pkg/exporters/stackdriver/config/config.go create mode 100644 pkg/exporters/stackdriver/config/config_test.go create mode 100644 pkg/exporters/stackdriver/gce/type.go create mode 100644 pkg/exporters/stackdriver/stackdriver_exporter.go create mode 100644 pkg/exporters/stackdriver/stackdriver_exporter_test.go diff --git a/Makefile b/Makefile index e5ff59a8d..9cdf42c6a 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ PKG:=k8s.io/node-problem-detector PKG_SOURCES:=$(shell find pkg cmd -name '*.go') # TARBALL is the name of release tar. Include binary version by default. -TARBALL:=node-problem-detector-$(VERSION).tar.gz +TARBALL?=node-problem-detector-$(VERSION).tar.gz # IMAGE is the image name of the node problem detector container image. IMAGE:=$(REGISTRY)/node-problem-detector:$(TAG) diff --git a/cmd/nodeproblemdetector/exporterplugins/default_plugin.go b/cmd/nodeproblemdetector/exporterplugins/default_plugin.go new file mode 100644 index 000000000..2e348b4e8 --- /dev/null +++ b/cmd/nodeproblemdetector/exporterplugins/default_plugin.go @@ -0,0 +1,20 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporterplugins + +// This file is necessary to make sure the exporterplugins package non-empty +// under any build tags. diff --git a/cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go b/cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go new file mode 100644 index 000000000..a580549c3 --- /dev/null +++ b/cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go @@ -0,0 +1,25 @@ +// +build !disable_stackdriver_exporter + +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporterplugins + +import ( + _ "k8s.io/node-problem-detector/pkg/exporters/stackdriver" +) + +// The stackdriver plugin takes about 6MB in the NPD binary. diff --git a/cmd/nodeproblemdetector/node_problem_detector.go b/cmd/nodeproblemdetector/node_problem_detector.go index 071853861..b551373d4 100644 --- a/cmd/nodeproblemdetector/node_problem_detector.go +++ b/cmd/nodeproblemdetector/node_problem_detector.go @@ -22,8 +22,10 @@ import ( "github.com/golang/glog" "github.com/spf13/pflag" + _ "k8s.io/node-problem-detector/cmd/nodeproblemdetector/exporterplugins" _ "k8s.io/node-problem-detector/cmd/nodeproblemdetector/problemdaemonplugins" "k8s.io/node-problem-detector/cmd/options" + "k8s.io/node-problem-detector/pkg/exporters" "k8s.io/node-problem-detector/pkg/exporters/k8sexporter" "k8s.io/node-problem-detector/pkg/exporters/prometheusexporter" "k8s.io/node-problem-detector/pkg/problemdaemon" @@ -54,21 +56,25 @@ func main() { } // Initialize exporters. - exporters := []types.Exporter{} + initializedExporters := []types.Exporter{} if ke := k8sexporter.NewExporterOrDie(npdo); ke != nil { - exporters = append(exporters, ke) + initializedExporters = append(initializedExporters, ke) glog.Info("K8s exporter started.") } if pe := prometheusexporter.NewExporterOrDie(npdo); pe != nil { - exporters = append(exporters, pe) + initializedExporters = append(initializedExporters, pe) glog.Info("Prometheus exporter started.") } - if len(exporters) == 0 { + plugableExporters := exporters.NewExporters(npdo.ExporterConfigPaths) + if len(plugableExporters) != 0 { + initializedExporters = append(initializedExporters, plugableExporters...) + } + if len(initializedExporters) == 0 { glog.Fatalf("No exporter is successfully setup") } // Initialize NPD core. - p := problemdetector.NewProblemDetector(problemDaemons, exporters) + p := problemdetector.NewProblemDetector(problemDaemons, initializedExporters) if err := p.Run(); err != nil { glog.Fatalf("Problem detector failed with error: %v", err) } diff --git a/cmd/options/options.go b/cmd/options/options.go index ab6ccdbcf..a4f9edfa8 100644 --- a/cmd/options/options.go +++ b/cmd/options/options.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/pflag" + "k8s.io/node-problem-detector/pkg/exporters" "k8s.io/node-problem-detector/pkg/problemdaemon" "k8s.io/node-problem-detector/pkg/types" ) @@ -63,6 +64,9 @@ type NodeProblemDetectorOptions struct { // PrometheusServerAddress is the address to bind the Prometheus scrape endpoint. PrometheusServerAddress string + // ExporterConfigPaths specifies the list of paths to configuration files for each exporter. + ExporterConfigPaths types.ExporterConfigPathMap + // problem daemon options // SystemLogMonitorConfigPaths specifies the list of paths to system log monitor configuration @@ -85,7 +89,14 @@ type NodeProblemDetectorOptions struct { } func NewNodeProblemDetectorOptions() *NodeProblemDetectorOptions { - npdo := &NodeProblemDetectorOptions{MonitorConfigPaths: types.ProblemDaemonConfigPathMap{}} + npdo := &NodeProblemDetectorOptions{ + ExporterConfigPaths: types.ExporterConfigPathMap{}, + MonitorConfigPaths: types.ProblemDaemonConfigPathMap{}} + + for _, exporterName := range exporters.GetExporterNames() { + var configPathHolder string + npdo.ExporterConfigPaths[exporterName] = &configPathHolder + } for _, problemDaemonName := range problemdaemon.GetProblemDaemonNames() { npdo.MonitorConfigPaths[problemDaemonName] = &[]string{} } @@ -118,6 +129,15 @@ func (npdo *NodeProblemDetectorOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&npdo.PrometheusServerAddress, "prometheus-address", "127.0.0.1", "The address to bind the Prometheus scrape endpoint.") + for _, exporterName := range exporters.GetExporterNames() { + fs.StringVar( + npdo.ExporterConfigPaths[exporterName], + "exporter."+string(exporterName), + "", + fmt.Sprintf("Configuration for %v exporter. %v", + exporterName, + exporters.GetExporterHandlerOrDie(exporterName).CmdOptionDescription)) + } for _, problemDaemonName := range problemdaemon.GetProblemDaemonNames() { fs.StringSliceVar( npdo.MonitorConfigPaths[problemDaemonName], diff --git a/cmd/options/options_test.go b/cmd/options/options_test.go index 05902db7e..68d69bca9 100644 --- a/cmd/options/options_test.go +++ b/cmd/options/options_test.go @@ -120,10 +120,13 @@ func TestSetNodeNameOrDie(t *testing.T) { } func TestValidOrDie(t *testing.T) { + emptyMonitorConfigMap := types.ProblemDaemonConfigPathMap{} fooMonitorConfigMap := types.ProblemDaemonConfigPathMap{} fooMonitorConfigMap["foo-monitor"] = &[]string{"config-a", "config-b"} - emptyMonitorConfigMap := types.ProblemDaemonConfigPathMap{} + emptyExporterConfigMap := types.ExporterConfigPathMap{} + barConfig := "config-c" + barExporterConfigMap := types.ExporterConfigPathMap{"bar-exporter": &barConfig} testCases := []struct { name string @@ -133,48 +136,54 @@ func TestValidOrDie(t *testing.T) { { name: "default k8s exporter config", npdo: NodeProblemDetectorOptions{ - MonitorConfigPaths: fooMonitorConfigMap, + MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: false, }, { name: "enables k8s exporter config", npdo: NodeProblemDetectorOptions{ - ApiServerOverride: "", - EnableK8sExporter: true, - MonitorConfigPaths: fooMonitorConfigMap, + ApiServerOverride: "", + EnableK8sExporter: true, + MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: false, }, { name: "k8s exporter config with valid ApiServerOverride", npdo: NodeProblemDetectorOptions{ - ApiServerOverride: "127.0.0.1", - EnableK8sExporter: true, - MonitorConfigPaths: fooMonitorConfigMap, + ApiServerOverride: "127.0.0.1", + EnableK8sExporter: true, + MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: false, }, { name: "k8s exporter config with invalid ApiServerOverride", npdo: NodeProblemDetectorOptions{ - ApiServerOverride: ":foo", - EnableK8sExporter: true, - MonitorConfigPaths: fooMonitorConfigMap, + ApiServerOverride: ":foo", + EnableK8sExporter: true, + MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, { name: "non-empty MonitorConfigPaths", npdo: NodeProblemDetectorOptions{ - MonitorConfigPaths: fooMonitorConfigMap, + MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: false, }, { name: "empty MonitorConfigPaths", npdo: NodeProblemDetectorOptions{ - MonitorConfigPaths: emptyMonitorConfigMap, + MonitorConfigPaths: emptyMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, @@ -188,6 +197,7 @@ func TestValidOrDie(t *testing.T) { npdo: NodeProblemDetectorOptions{ SystemLogMonitorConfigPaths: []string{"config-a"}, MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, @@ -196,6 +206,7 @@ func TestValidOrDie(t *testing.T) { npdo: NodeProblemDetectorOptions{ CustomPluginMonitorConfigPaths: []string{"config-a"}, MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, @@ -204,6 +215,7 @@ func TestValidOrDie(t *testing.T) { npdo: NodeProblemDetectorOptions{ SystemLogMonitorConfigPaths: []string{"config-a"}, MonitorConfigPaths: emptyMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, @@ -211,6 +223,7 @@ func TestValidOrDie(t *testing.T) { name: "deprecated SystemLogMonitor option with un-initialized MonitorConfigPaths", npdo: NodeProblemDetectorOptions{ SystemLogMonitorConfigPaths: []string{"config-a"}, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, @@ -219,6 +232,7 @@ func TestValidOrDie(t *testing.T) { npdo: NodeProblemDetectorOptions{ CustomPluginMonitorConfigPaths: []string{"config-b"}, MonitorConfigPaths: emptyMonitorConfigMap, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, @@ -226,9 +240,23 @@ func TestValidOrDie(t *testing.T) { name: "deprecated CustomPluginMonitor option with un-initialized MonitorConfigPaths", npdo: NodeProblemDetectorOptions{ CustomPluginMonitorConfigPaths: []string{"config-b"}, + ExporterConfigPaths: barExporterConfigMap, }, expectPanic: true, }, + { + name: "empty ExporterConfigPaths", + npdo: NodeProblemDetectorOptions{ + MonitorConfigPaths: fooMonitorConfigMap, + ExporterConfigPaths: emptyExporterConfigMap, + }, + expectPanic: false, + }, + { + name: "un-initialized ExporterConfigPaths", + npdo: NodeProblemDetectorOptions{}, + expectPanic: true, + }, } for _, test := range testCases { diff --git a/config/exporter/stackdriver-exporter.json b/config/exporter/stackdriver-exporter.json new file mode 100644 index 000000000..d948bf345 --- /dev/null +++ b/config/exporter/stackdriver-exporter.json @@ -0,0 +1,10 @@ +{ + "apiEndpoint": "staging-monitoring.sandbox.googleapis.com:443", + "exportPeriod": "60s", + "gceMetadata": { + "projectID": "xueweiz-experimental", + "zone": "us-central1-a", + "instanceID": "3133007593278616111", + "instanceName": "test-vm-2" + } +} diff --git a/config/systemd/node-problem-detector-metric-only.service b/config/systemd/node-problem-detector-metric-only.service index 142fee176..2503bb139 100644 --- a/config/systemd/node-problem-detector-metric-only.service +++ b/config/systemd/node-problem-detector-metric-only.service @@ -1,12 +1,13 @@ [Unit] Description=Node problem detector -Wants=local-fs.target -After=local-fs.target +Wants=network-online.target +After=network-online.target [Service] Restart=always RestartSec=10 ExecStart=/home/kubernetes/bin/node-problem-detector --v=2 --logtostderr --enable-k8s-exporter=false \ + --exporter.stackdriver=/home/kubernetes/node-problem-detector/config/exporter/stackdriver-exporter.json \ --config.system-log-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor.json,/home/kubernetes/node-problem-detector/config/docker-monitor.json,/home/kubernetes/node-problem-detector/config/systemd-monitor.json \ --config.custom-plugin-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor-counter.json,/home/kubernetes/node-problem-detector/config/systemd-monitor-counter.json \ --config.system-stats-monitor=/home/kubernetes/node-problem-detector/config/system-stats-monitor.json diff --git a/pkg/exporters/register.go b/pkg/exporters/register.go new file mode 100644 index 000000000..ffb71aa2b --- /dev/null +++ b/pkg/exporters/register.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporters + +import ( + "fmt" + + "k8s.io/node-problem-detector/pkg/types" +) + +var ( + handlers = make(map[types.ExporterType]types.ExporterHandler) +) + +// Register registers a exporter factory method, which will be used to create the exporter. +func Register(exporterType types.ExporterType, handler types.ExporterHandler) { + handlers[exporterType] = handler +} + +// GetExporterNames retrieves all available exporter types. +func GetExporterNames() []types.ExporterType { + exporterTypes := []types.ExporterType{} + for exporterType := range handlers { + exporterTypes = append(exporterTypes, exporterType) + } + return exporterTypes +} + +// GetExporterHandlerOrDie retrieves the ExporterHandler for a specific type of exporter, panic if error occurs.. +func GetExporterHandlerOrDie(exporterType types.ExporterType) types.ExporterHandler { + handler, ok := handlers[exporterType] + if !ok { + panic(fmt.Sprintf("Exporter handler for %v does not exist", exporterType)) + } + return handler +} + +// NewExporters creates all exporters based on the configurations provided. +func NewExporters(exporterConfigPaths types.ExporterConfigPathMap) []types.Exporter { + exporters := []types.Exporter{} + for exporterType, config := range exporterConfigPaths { + if *config == "" { + continue + } + exporters = append(exporters, handlers[exporterType].CreateExporterOrDie(*config)) + } + return exporters +} diff --git a/pkg/exporters/register_test.go b/pkg/exporters/register_test.go new file mode 100644 index 000000000..b4b69095e --- /dev/null +++ b/pkg/exporters/register_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporters + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/node-problem-detector/pkg/types" +) + +func TestRegistration(t *testing.T) { + fooExporterFactory := func(configPath string) types.Exporter { + return nil + } + fooExporterHandler := types.ExporterHandler{ + CreateExporterOrDie: fooExporterFactory, + CmdOptionDescription: "foo option", + } + + barExporterFactory := func(configPath string) types.Exporter { + return nil + } + barExporterHandler := types.ExporterHandler{ + CreateExporterOrDie: barExporterFactory, + CmdOptionDescription: "bar option", + } + + Register("foo", fooExporterHandler) + Register("bar", barExporterHandler) + + expectedExporterNames := []types.ExporterType{"foo", "bar"} + exporterNames := GetExporterNames() + + assert.ElementsMatch(t, expectedExporterNames, exporterNames) + assert.Equal(t, "foo option", GetExporterHandlerOrDie("foo").CmdOptionDescription) + assert.Equal(t, "bar option", GetExporterHandlerOrDie("bar").CmdOptionDescription) + + handlers = make(map[types.ExporterType]types.ExporterHandler) +} + +func TestGetExporterHandlerOrDie(t *testing.T) { + fooExporterFactory := func(configPath string) types.Exporter { + return nil + } + fooExporterHandler := types.ExporterHandler{ + CreateExporterOrDie: fooExporterFactory, + CmdOptionDescription: "foo option", + } + + Register("foo", fooExporterHandler) + + assert.NotPanics(t, func() { GetExporterHandlerOrDie("foo") }) + assert.Panics(t, func() { GetExporterHandlerOrDie("bar") }) + + handlers = make(map[types.ExporterType]types.ExporterHandler) +} diff --git a/pkg/exporters/stackdriver/config/config.go b/pkg/exporters/stackdriver/config/config.go new file mode 100644 index 000000000..b559d0191 --- /dev/null +++ b/pkg/exporters/stackdriver/config/config.go @@ -0,0 +1,55 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "time" + + "k8s.io/node-problem-detector/pkg/exporters/stackdriver/gce" +) + +var ( + defaultExportPeriod = (60 * time.Second).String() + defaultEndpoint = "monitoring.googleapis.com:443" + defaultMetadataFetchTimeout = (600 * time.Second).String() + defaultMetadataFetchInterval = (10 * time.Second).String() +) + +type StackdriverExporterConfig struct { + ExportPeriod string `json:"exportPeriod"` + APIEndpoint string `json:"apiEndpoint"` + Metadata gce.Metadata `json:"gceMetadata"` + MetadataFetchTimeout string `json:"metadataFetchTimeout"` + MetadataFetchInterval string `json:"metadataFetchInterval"` + PanicOnMetadataFetchFailure bool `json:"panicOnMetadataFetchFailure"` +} + +// ApplyConfiguration applies default configurations. +func (sec *StackdriverExporterConfig) ApplyConfiguration() { + if sec.ExportPeriod == "" { + sec.ExportPeriod = defaultExportPeriod + } + if sec.MetadataFetchTimeout == "" { + sec.MetadataFetchTimeout = defaultMetadataFetchTimeout + } + if sec.MetadataFetchInterval == "" { + sec.MetadataFetchInterval = defaultMetadataFetchInterval + } + if sec.APIEndpoint == "" { + sec.APIEndpoint = defaultEndpoint + } +} diff --git a/pkg/exporters/stackdriver/config/config_test.go b/pkg/exporters/stackdriver/config/config_test.go new file mode 100644 index 000000000..ae2f762b1 --- /dev/null +++ b/pkg/exporters/stackdriver/config/config_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "reflect" + "testing" + + "k8s.io/node-problem-detector/pkg/exporters/stackdriver/gce" +) + +func TestApplyConfiguration(t *testing.T) { + testCases := []struct { + name string + orignalConfig StackdriverExporterConfig + wantedConfig StackdriverExporterConfig + }{ + { + name: "normal", + orignalConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: "monitoring.googleapis.com:443", + Metadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + wantedConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: defaultEndpoint, + Metadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + }, + { + name: "staging API endpoint", + orignalConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: "staging-monitoring.sandbox.googleapis.com:443", + Metadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + wantedConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: "staging-monitoring.sandbox.googleapis.com:443", + Metadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + }, + { + name: "empty", + orignalConfig: StackdriverExporterConfig{}, + wantedConfig: StackdriverExporterConfig{ + ExportPeriod: "1m0s", + MetadataFetchTimeout: "10m0s", + MetadataFetchInterval: "10s", + APIEndpoint: "monitoring.googleapis.com:443", + Metadata: gce.Metadata{}, + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + test.orignalConfig.ApplyConfiguration() + if !reflect.DeepEqual(test.orignalConfig, test.wantedConfig) { + t.Errorf("Wanted: %+v. \nGot: %+v", test.wantedConfig, test.orignalConfig) + } + }) + } +} diff --git a/pkg/exporters/stackdriver/gce/type.go b/pkg/exporters/stackdriver/gce/type.go new file mode 100644 index 000000000..8a0f9f8bd --- /dev/null +++ b/pkg/exporters/stackdriver/gce/type.go @@ -0,0 +1,67 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "cloud.google.com/go/compute/metadata" + "github.com/golang/glog" +) + +type Metadata struct { + ProjectID string `json:"projectID"` + Zone string `json:"zone"` + InstanceID string `json:"instanceID"` + InstanceName string `json:"instanceName"` +} + +func (md *Metadata) IsValid() bool { + if md.ProjectID == "" || md.Zone == "" || md.InstanceID == "" || md.InstanceName == "" { + return false + } + return true +} + +func (md *Metadata) PopulateFromGCE() error { + var err error + glog.Info("Fetching GCE metadata from metadata server") + if md.ProjectID == "" { + md.ProjectID, err = metadata.ProjectID() + if err != nil { + return err + } + } + if md.Zone == "" { + md.Zone, err = metadata.Zone() + if err != nil { + return err + } + } + if md.InstanceID == "" { + md.InstanceID, err = metadata.InstanceID() + if err != nil { + return err + } + } + if md.InstanceName == "" { + md.InstanceName, err = metadata.InstanceName() + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/exporters/stackdriver/stackdriver_exporter.go b/pkg/exporters/stackdriver/stackdriver_exporter.go new file mode 100644 index 000000000..684b07c37 --- /dev/null +++ b/pkg/exporters/stackdriver/stackdriver_exporter.go @@ -0,0 +1,162 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stackdriverexporter + +import ( + "encoding/json" + "io/ioutil" + "time" + + "contrib.go.opencensus.io/exporter/stackdriver" + monitoredres "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "github.com/golang/glog" + "go.opencensus.io/stats/view" + "google.golang.org/api/option" + + "github.com/avast/retry-go" + "k8s.io/node-problem-detector/pkg/exporters" + seconfig "k8s.io/node-problem-detector/pkg/exporters/stackdriver/config" + "k8s.io/node-problem-detector/pkg/types" + "k8s.io/node-problem-detector/pkg/util/metrics" +) + +const exporterName = "stackdriver" + +var NPDMetricToSDMetric = map[metrics.MetricID]string{ + metrics.HostUptimeID: "compute.googleapis.com/guest/system/uptime", + metrics.ProblemCounterID: "compute.googleapis.com/guest/system/problem_count", + metrics.DiskAvgQueueLenID: "compute.googleapis.com/guest/disk/queue_length", + metrics.DiskIOTimeID: "compute.googleapis.com/guest/disk/io_time", + metrics.DiskWeightedIOID: "compute.googleapis.com/guest/disk/weighted_io_time", +} + +func init() { + exporters.Register(exporterName, types.ExporterHandler{ + CreateExporterOrDie: NewExporterOrDie, + CmdOptionDescription: "Set to the config file path."}) +} + +type stackdriverExporter struct { + configPath string + config seconfig.StackdriverExporterConfig +} + +func getMetricType(view *view.View) string { + viewName := view.Measure.Name() + // When there is no pre-defined Stackdriver Metric Type, fallback to custom metrics. + // e.g. custom.googleapis.com/npd/host/uptime + fallbackMetricType := "custom.googleapis.com/npd/" + viewName + + metricID, ok := metrics.MetricMap.ViewNameToMetricID(viewName) + if !ok { + return fallbackMetricType + } + stackdriverMetricType, ok := NPDMetricToSDMetric[metricID] + if !ok { + return fallbackMetricType + } + return stackdriverMetricType +} + +func (se *stackdriverExporter) setupOpenCensusViewExporterOrDie() { + clientOption := option.WithEndpoint(se.config.APIEndpoint) + + var globalLabels stackdriver.Labels + globalLabels.Set("instance_name", se.config.Metadata.InstanceName, "The name of the VM instance") + + viewExporter, err := stackdriver.NewExporter(stackdriver.Options{ + ProjectID: se.config.Metadata.ProjectID, + MonitoringClientOptions: []option.ClientOption{clientOption}, + MonitoredResource: &monitoredres.GCEInstance{ + ProjectID: se.config.Metadata.ProjectID, + InstanceID: se.config.Metadata.InstanceID, + Zone: se.config.Metadata.Zone, + }, + GetMetricType: getMetricType, + DefaultMonitoringLabels: &globalLabels, + }) + if err != nil { + glog.Fatalf("Failed to create Stackdriver OpenCensus view exporter: %v", err) + } + + exportPeriod, err := time.ParseDuration(se.config.ExportPeriod) + if err != nil { + glog.Fatalf("Failed to parse ExportPeriod %q: %v", se.config.ExportPeriod, err) + } + + view.SetReportingPeriod(exportPeriod) + view.RegisterExporter(viewExporter) +} + +func (se *stackdriverExporter) populateMetadataOrDie() { + if se.config.Metadata.IsValid() { + return + } + + metadataFetchTimeout, err := time.ParseDuration(se.config.MetadataFetchTimeout) + if err != nil { + glog.Fatalf("Failed to parse MetadataFetchTimeout %q: %v", se.config.MetadataFetchTimeout, err) + } + + metadataFetchInterval, err := time.ParseDuration(se.config.MetadataFetchInterval) + if err != nil { + glog.Fatalf("Failed to parse MetadataFetchInterval %q: %v", se.config.MetadataFetchInterval, err) + } + + err = retry.Do(se.config.Metadata.PopulateFromGCE, + retry.Delay(metadataFetchInterval), + retry.Attempts(uint(metadataFetchTimeout/metadataFetchInterval)), + retry.DelayType(retry.FixedDelay)) + if err == nil { + return + } + if se.config.PanicOnMetadataFetchFailure { + glog.Fatalf("Failed to populate GCE metadata: %v", err) + } else { + glog.Errorf("Failed to populate GCE metadata: %v", err) + } +} + +// NewExporterOrDie creates an exporter to export metrics to Stackdriver, panics if error occurs. +func NewExporterOrDie(configPath string) types.Exporter { + se := stackdriverExporter{configPath: configPath} + + // Apply configurations. + f, err := ioutil.ReadFile(configPath) + if err != nil { + glog.Fatalf("Failed to read configuration file %q: %v", configPath, err) + } + err = json.Unmarshal(f, &se.config) + if err != nil { + glog.Fatalf("Failed to unmarshal configuration file %q: %v", configPath, err) + } + se.config.ApplyConfiguration() + + glog.Infof("Starting Stackdriver exporter %s", configPath) + + se.populateMetadataOrDie() + glog.Infof("Using metadata: %v", se.config.Metadata) + se.setupOpenCensusViewExporterOrDie() + + return &se +} + +// ExportProblems does nothing. +// Stackdriver exporter only exports metrics. +func (se *stackdriverExporter) ExportProblems(status *types.Status) { + return +} diff --git a/pkg/exporters/stackdriver/stackdriver_exporter_test.go b/pkg/exporters/stackdriver/stackdriver_exporter_test.go new file mode 100644 index 000000000..f5f394815 --- /dev/null +++ b/pkg/exporters/stackdriver/stackdriver_exporter_test.go @@ -0,0 +1,33 @@ +// +build !disable_stackdriver_exporter + +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stackdriverexporter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/node-problem-detector/pkg/exporters" +) + +func TestRegistration(t *testing.T) { + assert.NotPanics(t, + func() { exporters.GetExporterHandlerOrDie(exporterName) }, + "Stackdriver exporter failed to register itself as an exporter.") +}