Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add endpoint label to metrics #382

Merged
merged 2 commits into from
Jul 17, 2023
Merged

Add endpoint label to metrics #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2453857
Ignore vendor/
Zhang21 Jul 11, 2023
38cbd76
feat: add endpoint label to metrics
Zhang21 Jul 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@
build/*
data/*.yaml
!.github/workflows/*.yaml
helm/easeprobe/charts/
helm/easeprobe/charts/
vendor/
22 changes: 14 additions & 8 deletions probe/base/base.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,31 +230,37 @@ func (d *DefaultProbe) ExportMetrics() {
time = d.ProbeResult.Stat.DownTime
}

// Add endpoint label according to ProbeKind(tcp/http/ping/host/...)
d.metrics.TotalCnt.With(prometheus.Labels{
"name": d.ProbeName,
"status": d.ProbeResult.Status.String(),
"name": d.ProbeName,
"status": d.ProbeResult.Status.String(),
"endpoint": d.ProbeResult.Endpoint,
}).Set(float64(cnt))

d.metrics.TotalTime.With(prometheus.Labels{
"name": d.ProbeName,
"status": d.ProbeResult.Status.String(),
"name": d.ProbeName,
"status": d.ProbeResult.Status.String(),
"endpoint": d.ProbeResult.Endpoint,
}).Set(float64(time.Seconds()))

d.metrics.Duration.With(prometheus.Labels{
"name": d.ProbeName,
"status": d.ProbeResult.Status.String(),
"name": d.ProbeName,
"status": d.ProbeResult.Status.String(),
"endpoint": d.ProbeResult.Endpoint,
}).Set(float64(d.ProbeResult.RoundTripTime.Milliseconds()))

status := ServiceUp // up
if d.ProbeResult.Status != probe.StatusUp {
status = ServiceDown // down
}
d.metrics.Status.With(prometheus.Labels{
"name": d.ProbeName,
"name": d.ProbeName,
"endpoint": d.ProbeResult.Endpoint,
}).Set(float64(status))

d.metrics.SLA.With(prometheus.Labels{
"name": d.ProbeName,
"name": d.ProbeName,
"endpoint": d.ProbeResult.Endpoint,
}).Set(float64(d.ProbeResult.SLAPercent()))
}

Expand Down
10 changes: 5 additions & 5 deletions probe/base/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,14 @@ func newMetrics(subsystem, name string) *metrics {
namespace := global.GetEaseProbe().Name
return &metrics{
TotalCnt: metric.NewGauge(namespace, subsystem, name, "total",
"Total Probed Counts", []string{"name", "status"}),
"Total Probed Counts", []string{"name", "status", "endpoint"}),
TotalTime: metric.NewGauge(namespace, subsystem, name, "total_time",
"Total Time(Seconds) of Status", []string{"name", "status"}),
"Total Time(Seconds) of Status", []string{"name", "status", "endpoint"}),
Duration: metric.NewGauge(namespace, subsystem, name, "duration",
"Probe Duration", []string{"name", "status"}),
"Probe Duration", []string{"name", "status", "endpoint"}),
Status: metric.NewGauge(namespace, subsystem, name, "status",
"Probe Status", []string{"name"}),
"Probe Status", []string{"name", "endpoint"}),
SLA: metric.NewGauge(namespace, subsystem, name, "sla",
"Probe SLA", []string{"name"}),
"Probe SLA", []string{"name", "endpoint"}),
}
}
45 changes: 27 additions & 18 deletions probe/http/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,47 +279,56 @@ func (h *HTTP) ExportMetrics(resp *http.Response) {
len = int(resp.ContentLength)
}
h.metrics.StatusCode.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Inc()

h.metrics.ContentLen.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(float64(len))

h.metrics.DNSDuration.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(toMS(h.traceStats.dnsTook))

h.metrics.ConnectDuration.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(toMS(h.traceStats.connTook))

h.metrics.TLSDuration.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(toMS(h.traceStats.tlsTook))

h.metrics.SendDuration.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(toMS(h.traceStats.sendTook))

h.metrics.WaitDuration.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(toMS(h.traceStats.waitTook))

h.metrics.TransferDuration.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(toMS(h.traceStats.transferTook))

h.metrics.TotalDuration.With(prometheus.Labels{
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"name": h.ProbeName,
"status": fmt.Sprintf("%d", code),
"endpoint": h.ProbeResult.Endpoint,
}).Set(toMS(h.traceStats.totalTook))
}
18 changes: 9 additions & 9 deletions probe/http/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,22 @@ func newMetrics(subsystem, name string) *metrics {
namespace := global.GetEaseProbe().Name
return &metrics{
StatusCode: metric.NewCounter(namespace, subsystem, name, "status_code",
"HTTP Status Code", []string{"name", "status"}),
"HTTP Status Code", []string{"name", "status", "endpoint"}),
ContentLen: metric.NewGauge(namespace, subsystem, name, "content_len",
"HTTP Content Length", []string{"name", "status"}),
"HTTP Content Length", []string{"name", "status", "endpoint"}),
DNSDuration: metric.NewGauge(namespace, subsystem, name, "dns_duration",
"DNS Duration", []string{"name", "status"}),
"DNS Duration", []string{"name", "status", "endpoint"}),
ConnectDuration: metric.NewGauge(namespace, subsystem, name, "connect_duration",
"TCP Connection Duration", []string{"name", "status"}),
"TCP Connection Duration", []string{"name", "status", "endpoint"}),
TLSDuration: metric.NewGauge(namespace, subsystem, name, "tls_duration",
"TLS Duration", []string{"name", "status"}),
"TLS Duration", []string{"name", "status", "endpoint"}),
SendDuration: metric.NewGauge(namespace, subsystem, name, "send_duration",
"Send Duration", []string{"name", "status"}),
"Send Duration", []string{"name", "status", "endpoint"}),
WaitDuration: metric.NewGauge(namespace, subsystem, name, "wait_duration",
"Wait Duration", []string{"name", "status"}),
"Wait Duration", []string{"name", "status", "endpoint"}),
TransferDuration: metric.NewGauge(namespace, subsystem, name, "transfer_duration",
"Transfer Duration", []string{"name", "status"}),
"Transfer Duration", []string{"name", "status", "endpoint"}),
TotalDuration: metric.NewGauge(namespace, subsystem, name, "total_duration",
"Total Duration", []string{"name", "status"}),
"Total Duration", []string{"name", "status", "endpoint"}),
}
}
14 changes: 7 additions & 7 deletions probe/ping/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,18 +38,18 @@ func newMetrics(subsystem, name string) *metrics {
namespace := global.GetEaseProbe().Name
return &metrics{
PacketsSent: metric.NewCounter(namespace, subsystem, name, "sent",
"Total Package Sent", []string{"name"}),
"Total Package Sent", []string{"name", "endpoint"}),
PacketsRecv: metric.NewCounter(namespace, subsystem, name, "recv",
"Total Package Received", []string{"name"}),
"Total Package Received", []string{"name", "endpoint"}),
PacketLoss: metric.NewGauge(namespace, subsystem, name, "loss",
"Package Loss Percentage", []string{"name"}),
"Package Loss Percentage", []string{"name", "endpoint"}),
MinRtt: metric.NewGauge(namespace, subsystem, name, "min_rtt",
"Minimum Round Trip Time", []string{"name"}),
"Minimum Round Trip Time", []string{"name", "endpoint"}),
MaxRtt: metric.NewGauge(namespace, subsystem, name, "max_rtt",
"Maximum Round Trip Time", []string{"name"}),
"Maximum Round Trip Time", []string{"name", "endpoint"}),
AvgRtt: metric.NewGauge(namespace, subsystem, name, "avg_rtt",
"Average Round Trip Time", []string{"name"}),
"Average Round Trip Time", []string{"name", "endpoint"}),
StdDevRtt: metric.NewGauge(namespace, subsystem, name, "stddev_rtt",
"Standard Deviation of Round Trip Time", []string{"name"}),
"Standard Deviation of Round Trip Time", []string{"name", "endpoint"}),
}
}
21 changes: 14 additions & 7 deletions probe/ping/ping.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,30 +127,37 @@ func (p *Ping) DoProbe() (bool, string) {
// ExportMetrics export Ping metrics
func (p *Ping) ExportMetrics(stats *ping.Statistics) {
p.metrics.PacketsSent.With(prometheus.Labels{
"name": p.ProbeName,
"name": p.ProbeName,
"endpoint": p.ProbeResult.Endpoint,
}).Add(float64(stats.PacketsSent))

p.metrics.PacketsRecv.With(prometheus.Labels{
"name": p.ProbeName,
"name": p.ProbeName,
"endpoint": p.ProbeResult.Endpoint,
}).Add(float64(stats.PacketsRecv))

p.metrics.PacketLoss.With(prometheus.Labels{
"name": p.ProbeName,
"name": p.ProbeName,
"endpoint": p.ProbeResult.Endpoint,
}).Set(stats.PacketLoss)

p.metrics.MaxRtt.With(prometheus.Labels{
"name": p.ProbeName,
"name": p.ProbeName,
"endpoint": p.ProbeResult.Endpoint,
}).Set(float64(stats.MaxRtt.Milliseconds()))

p.metrics.MinRtt.With(prometheus.Labels{
"name": p.ProbeName,
"name": p.ProbeName,
"endpoint": p.ProbeResult.Endpoint,
}).Set(float64(stats.MinRtt.Milliseconds()))

p.metrics.AvgRtt.With(prometheus.Labels{
"name": p.ProbeName,
"name": p.ProbeName,
"endpoint": p.ProbeResult.Endpoint,
}).Set(float64(stats.AvgRtt.Milliseconds()))

p.metrics.StdDevRtt.With(prometheus.Labels{
"name": p.ProbeName,
"name": p.ProbeName,
"endpoint": p.ProbeResult.Endpoint,
}).Set(float64(stats.StdDevRtt.Milliseconds()))
}
4 changes: 2 additions & 2 deletions probe/shell/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ func newMetrics(subsystem, name string) *metrics {
namespace := global.GetEaseProbe().Name
return &metrics{
ExitCode: metric.NewCounter(namespace, subsystem, name, "exit_code",
"Exit Code", []string{"name", "exit"}),
"Exit Code", []string{"name", "exit", "endpoint"}),
OutputLen: metric.NewGauge(namespace, subsystem, name, "output_len",
"Output Length", []string{"name", "exit"}),
"Output Length", []string{"name", "exit", "endpoint"}),
}
}
10 changes: 6 additions & 4 deletions probe/shell/shell.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,14 @@ func (s *Shell) DoProbe() (bool, string) {
// ExportMetrics export shell metrics
func (s *Shell) ExportMetrics() {
s.metrics.ExitCode.With(prometheus.Labels{
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"endpoint": s.ProbeResult.Endpoint,
}).Inc()

s.metrics.OutputLen.With(prometheus.Labels{
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"endpoint": s.ProbeResult.Endpoint,
}).Set(float64(s.outputLen))
}
4 changes: 2 additions & 2 deletions probe/ssh/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ func newMetrics(subsystem, name string) *metrics {
namespace := global.GetEaseProbe().Name
return &metrics{
ExitCode: metric.NewCounter(namespace, subsystem, name, "exit_code",
"Exit Code", []string{"name", "exit"}),
"Exit Code", []string{"name", "exit", "endpoint"}),
OutputLen: metric.NewGauge(namespace, subsystem, name, "output_len",
"Output Length", []string{"name", "exit"}),
"Output Length", []string{"name", "exit", "endpoint"}),
}
}
10 changes: 6 additions & 4 deletions probe/ssh/ssh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,12 +268,14 @@ func (s *Server) RunSSHCmd() (string, error) {
// ExportMetrics export shell metrics
func (s *Server) ExportMetrics() {
s.metrics.ExitCode.With(prometheus.Labels{
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"endpoint": s.ProbeResult.Endpoint,
}).Inc()

s.metrics.OutputLen.With(prometheus.Labels{
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"name": s.ProbeName,
"exit": fmt.Sprintf("%d", s.exitCode),
"endpoint": s.ProbeResult.Endpoint,
}).Set(float64(s.outputLen))
}
4 changes: 2 additions & 2 deletions probe/tls/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ func newMetrics(subsystem, name string) *metrics {
namespace := global.GetEaseProbe().Name
return &metrics{
EarliestCertExpiry: metric.NewGauge(namespace, subsystem, name, "earliest_cert_expiry",
"last TLS chain expiry in timestamp seconds", []string{}),
"last TLS chain expiry in timestamp seconds", []string{"endpoint"}),
LastChainExpiryTimestampSeconds: metric.NewGauge(namespace, subsystem, name, "last_chain_expiry_timestamp_seconds",
"earliest TLS cert expiry in unix time", []string{}),
"earliest TLS cert expiry in unix time", []string{"endpoint"}),
}
}

Expand Down
8 changes: 6 additions & 2 deletions probe/tls/tls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,12 @@ func (t *TLS) DoProbe() (bool, string) {

state := tconn.ConnectionState()

t.metrics.EarliestCertExpiry.With(prometheus.Labels{}).Set(float64(getEarliestCertExpiry(&state).Unix()))
t.metrics.LastChainExpiryTimestampSeconds.With(prometheus.Labels{}).Set(float64(getLastChainExpiry(&state).Unix()))
t.metrics.EarliestCertExpiry.With(prometheus.Labels{
"endpoint": t.ProbeResult.Endpoint,
}).Set(float64(getEarliestCertExpiry(&state).Unix()))
t.metrics.LastChainExpiryTimestampSeconds.With(prometheus.Labels{
"endpoint": t.ProbeResult.Endpoint,
}).Set(float64(getLastChainExpiry(&state).Unix()))

return true, "TLS Endpoint Verified Successfully!"
}