diff --git a/api/go.mod b/api/go.mod index db05221804d..ccf80d7f8fb 100644 --- a/api/go.mod +++ b/api/go.mod @@ -12,5 +12,6 @@ require ( github.com/kr/pretty v0.1.0 github.com/mitchellh/go-testing-interface v1.0.0 github.com/stretchr/testify v1.5.1 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/api/go.sum b/api/go.sum index fee1b6a7211..a4a55eba5fc 100644 --- a/api/go.sum +++ b/api/go.sum @@ -29,6 +29,8 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= diff --git a/api/operator.go b/api/operator.go index cbc5e24a190..d5bc5d061d5 100644 --- a/api/operator.go +++ b/api/operator.go @@ -304,22 +304,3 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro } return &reply, qm, nil } - -// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus -func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { - if q == nil { - q = &QueryOptions{} - } - - metricsReader, err := op.c.rawQuery("/v1/metrics", q) - if err != nil { - return nil, err - } - - metricsBytes, err := ioutil.ReadAll(metricsReader) - if err != nil { - return nil, err - } - - return metricsBytes, nil -} diff --git a/api/operator_metrics.go b/api/operator_metrics.go new file mode 100644 index 00000000000..e64198194a7 --- /dev/null +++ b/api/operator_metrics.go @@ -0,0 +1,87 @@ +package api + +import ( + "io/ioutil" + "time" +) + +// MetricsSummary holds a roll-up of metrics info for a given interval +type MetricsSummary struct { + Timestamp string + Gauges []GaugeValue + Points []PointValue + Counters []SampledValue + Samples []SampledValue +} + +type GaugeValue struct { + Name string + Hash string `json:"-"` + Value float32 + + Labels []Label `json:"-"` + DisplayLabels map[string]string `json:"Labels"` +} + +type PointValue struct { + Name string + Points []float32 +} + +type SampledValue struct { + Name string + Hash string `json:"-"` + *AggregateSample + Mean float64 + Stddev float64 + + Labels []Label `json:"-"` + DisplayLabels map[string]string `json:"Labels"` +} + +// AggregateSample is used to hold aggregate metrics +// about a sample +type AggregateSample struct { + Count int // The count of emitted pairs + Rate float64 // The values rate per time unit (usually 1 second) + Sum float64 // The sum of values + SumSq float64 `json:"-"` // The sum of squared values + Min float64 // Minimum value + Max float64 // Maximum value + LastUpdated time.Time `json:"-"` // When value was last updated +} + +type Label struct { + Name string + Value string +} + +// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus +func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { + if q == nil { + q = &QueryOptions{} + } + + metricsReader, err := op.c.rawQuery("/v1/metrics", q) + if err != nil { + return nil, err + } + + metricsBytes, err := ioutil.ReadAll(metricsReader) + if err != nil { + return nil, err + } + + return metricsBytes, nil +} + +// MetricsSummary returns a MetricsSummary struct and query metadata +func (op *Operator) MetricsSummary(q *QueryOptions) (*MetricsSummary, *QueryMeta, error) { + var resp *MetricsSummary + qm, err := op.c.query("/v1/metrics", &resp, q) + if err != nil { + return nil, nil, err + } + + return resp, qm, nil +} diff --git a/api/operator_metrics_test.go b/api/operator_metrics_test.go new file mode 100644 index 00000000000..e21a6cecc71 --- /dev/null +++ b/api/operator_metrics_test.go @@ -0,0 +1,49 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOperator_MetricsSummary(t *testing.T) { + t.Parallel() + c, s := makeClient(t, nil, nil) + defer s.Stop() + + operator := c.Operator() + qo := &QueryOptions{ + Params: map[string]string{ + "pretty": "1", + }, + } + + metrics, qm, err := operator.MetricsSummary(qo) + require.NoError(t, err) + require.NotNil(t, metrics) + require.NotNil(t, qm) + require.NotNil(t, metrics.Timestamp) // should always get a TimeStamp + require.GreaterOrEqual(t, len(metrics.Points), 0) // may not have points yet + require.GreaterOrEqual(t, len(metrics.Gauges), 1) // should have at least 1 gauge + require.GreaterOrEqual(t, len(metrics.Counters), 1) // should have at least 1 counter + require.GreaterOrEqual(t, len(metrics.Samples), 1) // should have at least 1 sample +} + +func TestOperator_Metrics_Prometheus(t *testing.T) { + t.Parallel() + c, s := makeClient(t, nil, nil) + defer s.Stop() + + operator := c.Operator() + qo := &QueryOptions{ + Params: map[string]string{ + "format": "prometheus", + }, + } + + metrics, err := operator.Metrics(qo) + require.NoError(t, err) + require.NotNil(t, metrics) + metricString := string(metrics[:]) + require.Containsf(t, metricString, "# HELP", "expected Prometheus format containing \"# HELP\", got: \n%s", metricString) +} diff --git a/command/metrics.go b/command/metrics.go index 5ceb3977866..1b7b109adb1 100644 --- a/command/metrics.go +++ b/command/metrics.go @@ -28,6 +28,13 @@ Metrics Specific Options -format Specify output format (prometheus) + + -json + Output the allocation in its JSON format. + + -t + Format and display allocation using a Go template. + ` return strings.TrimSpace(helpText) @@ -42,19 +49,23 @@ func (c *OperatorMetricsCommand) AutocompleteFlags() complete.Flags { complete.Flags{ "-pretty": complete.PredictAnything, "-format": complete.PredictAnything, + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, }) } func (c *OperatorMetricsCommand) Name() string { return "metrics" } func (c *OperatorMetricsCommand) Run(args []string) int { - var pretty bool - var format string + var pretty, json bool + var format, tmpl string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } flags.BoolVar(&pretty, "pretty", false, "") flags.StringVar(&format, "format", "", "") + flags.BoolVar(&json, "json", false, "") + flags.StringVar(&tmpl, "t", "", "") if err := flags.Parse(args); err != nil { c.Ui.Error(fmt.Sprintf("Error parsing flags: %s", err)) @@ -88,14 +99,31 @@ func (c *OperatorMetricsCommand) Run(args []string) int { Params: params, } - bs, err := client.Operator().Metrics(query) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error getting metrics: %v", err)) - return 1 + if json || len(tmpl) > 0 { + metrics, _, err := client.Operator().MetricsSummary(query) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying metrics: %v", err)) + return 1 + } + + out, err := Format(json, tmpl, metrics) + if err != nil { + c.Ui.Error(err.Error()) + return 1 + } + + c.Ui.Output(out) + return 0 + } else { + bs, err := client.Operator().Metrics(query) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error getting metrics: %v", err)) + return 1 + } + + resp := string(bs[:]) + c.Ui.Output(resp) } - resp := string(bs[:]) - c.Ui.Output(resp) - return 0 } diff --git a/command/metrics_test.go b/command/metrics_test.go index f909d4e6cca..cdf18c334d8 100644 --- a/command/metrics_test.go +++ b/command/metrics_test.go @@ -25,6 +25,20 @@ func TestCommand_Metrics_Cases(t *testing.T) { expectedOutput string expectedError string }{ + { + "gotemplate MetricsSummary", + []string{"-address=" + url, "-t", "'{{ .Timestamp }}'"}, + 0, + "UTC", + "", + }, + { + "json formatted MetricsSummary", + []string{"-address=" + url, "-json"}, + 0, + "{", + "", + }, { "pretty print json", []string{"-address=" + url, "-pretty"}, diff --git a/command/operator_debug.go b/command/operator_debug.go index b240724e276..96f7f87033b 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -74,7 +74,7 @@ Debug Options: profiles. Accepts id prefixes. -server-id=, - Comma separated list of Nomad server names, or "leader" to monitor for logs and include pprof + Comma separated list of Nomad server names, "leader", or "all" to monitor for logs and include pprof profiles. -stale= @@ -251,9 +251,25 @@ func (c *OperatorDebugCommand) Run(args []string) int { } } - // Resolve server prefixes - for _, id := range argNodes(serverIDs) { - c.serverIDs = append(c.serverIDs, id) + // Resolve servers + members, err := client.Agent().Members() + c.writeJSON("version", "members.json", members, err) + // We always write the error to the file, but don't range if no members found + if serverIDs == "all" && members != nil { + // Special case to capture from all servers + for _, member := range members.Members { + c.serverIDs = append(c.serverIDs, member.Name) + } + } else { + for _, id := range argNodes(serverIDs) { + c.serverIDs = append(c.serverIDs, id) + } + } + + // Return error if servers were specified but not found + if len(serverIDs) > 0 && len(c.serverIDs) == 0 { + c.Ui.Error(fmt.Sprintf("Failed to retrieve servers, 0 members found in list: %s", serverIDs)) + return 1 } c.manifest = make([]string, 0) @@ -267,6 +283,8 @@ func (c *OperatorDebugCommand) Run(args []string) int { stamped := "nomad-debug-" + c.timestamp c.Ui.Output("Starting debugger and capturing cluster data...") + c.Ui.Output(fmt.Sprintf("Capturing from servers: %v", c.serverIDs)) + c.Ui.Output(fmt.Sprintf("Capturing from client nodes: %v", c.nodeIDs)) c.Ui.Output(fmt.Sprintf(" Interval: '%s'", interval)) c.Ui.Output(fmt.Sprintf(" Duration: '%s'", duration)) @@ -499,6 +517,23 @@ func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client) if err == nil { c.writeBytes(path, "goroutine.prof", bs) } + + // Gather goroutine text output - debug type 1 + // debug type 1 writes the legacy text format for human readable output + opts.Debug = 1 + bs, err = client.Agent().Lookup("goroutine", opts, nil) + if err == nil { + c.writeBytes(path, "goroutine-debug1.txt", bs) + } + + // Gather goroutine text output - debug type 2 + // When printing the "goroutine" profile, debug=2 means to print the goroutine + // stacks in the same form that a Go program uses when dying due to an unrecovered panic. + opts.Debug = 2 + bs, err = client.Agent().Lookup("goroutine", opts, nil) + if err == nil { + c.writeBytes(path, "goroutine-debug2.txt", bs) + } } // collectPeriodic runs for duration, capturing the cluster state every interval. It flushes and stops @@ -576,8 +611,11 @@ func (c *OperatorDebugCommand) collectNomad(dir string, client *api.Client) erro vs, _, err := client.CSIVolumes().List(qo) c.writeJSON(dir, "volumes.json", vs, err) - metrics, err := client.Operator().Metrics(qo) - c.writeJSON(dir, "metrics.json", metrics, err) + if metricBytes, err := client.Operator().Metrics(qo); err != nil { + c.writeError(dir, "metrics.json", err) + } else { + c.writeBytes(dir, "metrics.json", metricBytes) + } return nil } @@ -628,12 +666,24 @@ func (c *OperatorDebugCommand) collectVault(dir, vault string) error { // writeBytes writes a file to the archive, recording it in the manifest func (c *OperatorDebugCommand) writeBytes(dir, file string, data []byte) error { - path := filepath.Join(dir, file) - c.manifest = append(c.manifest, path) - path = filepath.Join(c.collectDir, path) + relativePath := filepath.Join(dir, file) + c.manifest = append(c.manifest, relativePath) + dirPath := filepath.Join(c.collectDir, dir) + filePath := filepath.Join(dirPath, file) + + // Ensure parent directories exist + err := os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + // Display error immediately -- may not see this if files aren't written + c.Ui.Error(fmt.Sprintf("failed to create parent directories of \"%s\": %s", dirPath, err.Error())) + return err + } - fh, err := os.Create(path) + // Create the file + fh, err := os.Create(filePath) if err != nil { + // Display error immediately -- may not see this if files aren't written + c.Ui.Error(fmt.Sprintf("failed to create file \"%s\": %s", filePath, err.Error())) return err } defer fh.Close() diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index d329deada8d..f0806a6b892 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -6,7 +6,9 @@ import ( "testing" "time" + "github.com/hashicorp/nomad/testutil" "github.com/mitchellh/cli" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -34,7 +36,7 @@ func TestDebugUtils(t *testing.T) { func TestDebugSuccesses(t *testing.T) { t.Parallel() - srv, _, _ := testServer(t, false, nil) + srv, _, url := testServer(t, false, nil) defer srv.Shutdown() ui := cli.NewMockUi() @@ -43,21 +45,21 @@ func TestDebugSuccesses(t *testing.T) { // NOTE -- duration must be shorter than default 2m to prevent testify from timing out // Debug on the leader - code := cmd.Run([]string{"-duration", "250ms", "-server-id", "leader"}) - require.Equal(t, 0, code) + code := cmd.Run([]string{"-address", url, "-duration", "250ms", "-server-id", "leader"}) + assert.Equal(t, 0, code) // take note of failed return code, but continue to see why require.Contains(t, ui.OutputWriter.String(), "Starting debugger") ui.OutputWriter.Reset() // Debug on all servers - code = cmd.Run([]string{"-duration", "250ms", "-server-id", "all"}) - require.Equal(t, 0, code) + code = cmd.Run([]string{"-address", url, "-duration", "250ms", "-server-id", "all"}) + assert.Equal(t, 0, code) require.Contains(t, ui.OutputWriter.String(), "Starting debugger") ui.OutputWriter.Reset() } func TestDebugFails(t *testing.T) { t.Parallel() - srv, _, _ := testServer(t, false, nil) + srv, _, url := testServer(t, false, nil) defer srv.Shutdown() ui := cli.NewMockUi() @@ -91,6 +93,11 @@ func TestDebugFails(t *testing.T) { defer os.Remove(path) code = cmd.Run([]string{"-output", os.TempDir()}) require.Equal(t, 2, code) + + // Fails bad address + code = cmd.Run([]string{"-address", url + "bogus"}) + assert.Equal(t, 1, code) + ui.OutputWriter.Reset() } func TestDebugCapturedFiles(t *testing.T) { @@ -116,28 +123,32 @@ func TestDebugCapturedFiles(t *testing.T) { require.Equal(t, 0, code) ui.ErrorWriter.Reset() - // Version is always captured - require.FileExists(t, filepath.Join(path, "version", "agent-self.json")) - - // Consul and Vault contain results or errors - _, err := os.Stat(filepath.Join(path, "version", "consul-agent-self.json")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(path, "version", "vault-sys-health.json")) - require.NoError(t, err) - - // Monitor files are only created when selected - require.FileExists(t, filepath.Join(path, "server", "leader", "monitor.log")) - require.FileExists(t, filepath.Join(path, "server", "leader", "profile.prof")) - require.FileExists(t, filepath.Join(path, "server", "leader", "trace.prof")) - require.FileExists(t, filepath.Join(path, "server", "leader", "goroutine.prof")) - - // Multiple snapshots are collected, 00 is always created - require.FileExists(t, filepath.Join(path, "nomad", "0000", "jobs.json")) - require.FileExists(t, filepath.Join(path, "nomad", "0000", "nodes.json")) - require.FileExists(t, filepath.Join(path, "nomad", "0000", "metrics.json")) - - // Multiple snapshots are collected, 01 requires two intervals - require.FileExists(t, filepath.Join(path, "nomad", "0001", "jobs.json")) - require.FileExists(t, filepath.Join(path, "nomad", "0001", "nodes.json")) - require.FileExists(t, filepath.Join(path, "nomad", "0001", "metrics.json")) + serverFiles := []string{ + // Version is always captured + filepath.Join(path, "version", "agent-self.json"), + + // Consul and Vault contain results or errors + filepath.Join(path, "version", "consul-agent-self.json"), + filepath.Join(path, "version", "vault-sys-health.json"), + + // Monitor files are only created when selected + filepath.Join(path, "server", "leader", "monitor.log"), + filepath.Join(path, "server", "leader", "profile.prof"), + filepath.Join(path, "server", "leader", "trace.prof"), + filepath.Join(path, "server", "leader", "goroutine.prof"), + filepath.Join(path, "server", "leader", "goroutine-debug1.txt"), + filepath.Join(path, "server", "leader", "goroutine-debug2.txt"), + + // Multiple snapshots are collected, 00 is always created + filepath.Join(path, "nomad", "0000", "jobs.json"), + filepath.Join(path, "nomad", "0000", "nodes.json"), + filepath.Join(path, "nomad", "0000", "metrics.json"), + + // Multiple snapshots are collected, 01 requires two intervals + filepath.Join(path, "nomad", "0001", "jobs.json"), + filepath.Join(path, "nomad", "0001", "nodes.json"), + filepath.Join(path, "nomad", "0001", "metrics.json"), + } + + testutil.WaitForFiles(t, serverFiles) } diff --git a/testutil/wait.go b/testutil/wait.go index 1deaba909e1..ddc53139e68 100644 --- a/testutil/wait.go +++ b/testutil/wait.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" "github.com/kr/pretty" testing "github.com/mitchellh/go-testing-interface" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -190,3 +191,19 @@ func WaitForRunningWithToken(t testing.T, rpc rpcFn, job *structs.Job, token str func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) []*structs.AllocListStub { return WaitForRunningWithToken(t, rpc, job, "") } + +// WaitForFiles blocks until all the files in the slice are present +func WaitForFiles(t testing.T, files []string) { + assert := assert.New(t) + WaitForResult(func() (bool, error) { + for _, f := range files { + exists := assert.FileExists(f) + if !exists { + return false, fmt.Errorf("expected file to exist %s", f) + } + } + return true, nil + }, func(err error) { + t.Fatalf("missing expected files: %v", err) + }) +} diff --git a/vendor/github.com/hashicorp/nomad/api/go.mod b/vendor/github.com/hashicorp/nomad/api/go.mod index db05221804d..ccf80d7f8fb 100644 --- a/vendor/github.com/hashicorp/nomad/api/go.mod +++ b/vendor/github.com/hashicorp/nomad/api/go.mod @@ -12,5 +12,6 @@ require ( github.com/kr/pretty v0.1.0 github.com/mitchellh/go-testing-interface v1.0.0 github.com/stretchr/testify v1.5.1 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/vendor/github.com/hashicorp/nomad/api/go.sum b/vendor/github.com/hashicorp/nomad/api/go.sum index fee1b6a7211..a4a55eba5fc 100644 --- a/vendor/github.com/hashicorp/nomad/api/go.sum +++ b/vendor/github.com/hashicorp/nomad/api/go.sum @@ -29,6 +29,8 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= diff --git a/vendor/github.com/hashicorp/nomad/api/operator.go b/vendor/github.com/hashicorp/nomad/api/operator.go index cbc5e24a190..d5bc5d061d5 100644 --- a/vendor/github.com/hashicorp/nomad/api/operator.go +++ b/vendor/github.com/hashicorp/nomad/api/operator.go @@ -304,22 +304,3 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro } return &reply, qm, nil } - -// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus -func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { - if q == nil { - q = &QueryOptions{} - } - - metricsReader, err := op.c.rawQuery("/v1/metrics", q) - if err != nil { - return nil, err - } - - metricsBytes, err := ioutil.ReadAll(metricsReader) - if err != nil { - return nil, err - } - - return metricsBytes, nil -} diff --git a/vendor/github.com/hashicorp/nomad/api/operator_metrics.go b/vendor/github.com/hashicorp/nomad/api/operator_metrics.go new file mode 100644 index 00000000000..e64198194a7 --- /dev/null +++ b/vendor/github.com/hashicorp/nomad/api/operator_metrics.go @@ -0,0 +1,87 @@ +package api + +import ( + "io/ioutil" + "time" +) + +// MetricsSummary holds a roll-up of metrics info for a given interval +type MetricsSummary struct { + Timestamp string + Gauges []GaugeValue + Points []PointValue + Counters []SampledValue + Samples []SampledValue +} + +type GaugeValue struct { + Name string + Hash string `json:"-"` + Value float32 + + Labels []Label `json:"-"` + DisplayLabels map[string]string `json:"Labels"` +} + +type PointValue struct { + Name string + Points []float32 +} + +type SampledValue struct { + Name string + Hash string `json:"-"` + *AggregateSample + Mean float64 + Stddev float64 + + Labels []Label `json:"-"` + DisplayLabels map[string]string `json:"Labels"` +} + +// AggregateSample is used to hold aggregate metrics +// about a sample +type AggregateSample struct { + Count int // The count of emitted pairs + Rate float64 // The values rate per time unit (usually 1 second) + Sum float64 // The sum of values + SumSq float64 `json:"-"` // The sum of squared values + Min float64 // Minimum value + Max float64 // Maximum value + LastUpdated time.Time `json:"-"` // When value was last updated +} + +type Label struct { + Name string + Value string +} + +// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus +func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { + if q == nil { + q = &QueryOptions{} + } + + metricsReader, err := op.c.rawQuery("/v1/metrics", q) + if err != nil { + return nil, err + } + + metricsBytes, err := ioutil.ReadAll(metricsReader) + if err != nil { + return nil, err + } + + return metricsBytes, nil +} + +// MetricsSummary returns a MetricsSummary struct and query metadata +func (op *Operator) MetricsSummary(q *QueryOptions) (*MetricsSummary, *QueryMeta, error) { + var resp *MetricsSummary + qm, err := op.c.query("/v1/metrics", &resp, q) + if err != nil { + return nil, nil, err + } + + return resp, qm, nil +}