Skip to content

Commit

Permalink
add otelhttp client metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
vchitai committed Oct 27, 2020
1 parent 6fcf996 commit ccdcdd6
Show file tree
Hide file tree
Showing 6 changed files with 251 additions and 3 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ go 1.14
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/kr/pretty v0.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.13.0 // indirect
go.opentelemetry.io/otel v0.13.0
go.opentelemetry.io/otel/exporters/stdout v0.13.0 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
)
15 changes: 15 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
github.com/DataDog/sketches-go v0.0.1 h1:RtG+76WKgZuz6FIaGsjoPePmadDBkuD/KC6+ZWu78b8=
github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
Expand All @@ -14,8 +20,17 @@ github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
go.opentelemetry.io v0.1.0 h1:EANZoRCOP+A3faIlw/iN6YEWoYb1vleZRKm1EvH8T48=
go.opentelemetry.io/contrib v0.13.0/go.mod h1:HzCu6ebm0ywgNxGaEfs3izyJOMP4rZnzxycyTgpI5Sg=
go.opentelemetry.io/contrib/instrumentation/net/http v0.11.0 h1:ufewgDRmtrrdDpPgm7b4/gr4RXLS7KhDttAhyThtYS4=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.13.0 h1:dnZy1afzxEDrHybTYoJE1bQ3fphNwZF2ipSsynlITP4=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.13.0/go.mod h1:SeQm4RTCcZ2/hlMSTuHb7nwIROe5odBtgfKx+7MMqEs=
go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA=
go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY=
go.opentelemetry.io/otel/exporters/stdout v0.13.0 h1:A+XiGIPQbGoJoBOJfKAKnZyiUSjSWvL3XWETUvtom5k=
go.opentelemetry.io/otel/exporters/stdout v0.13.0/go.mod h1:JJt8RpNY6K+ft9ir3iKpceCvT/rhzJXEExGrWFCbv1o=
go.opentelemetry.io/otel/sdk v0.13.0 h1:4VCfpKamZ8GtnepXxMRurSpHpMKkcxhtO33z1S4rGDQ=
go.opentelemetry.io/otel/sdk v0.13.0/go.mod h1:dKvLH8Uu8LcEPlSAUsfW7kMGaJBhk/1NYvpPZ6wIMbU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
8 changes: 8 additions & 0 deletions instrumentation/net/http/otelhttp/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ const (
ServerLatency = "http.server.duration" // Incoming end to end duration, microseconds
)

// Client HTTP metrics
const (
ClientRequestCount = "http.client.request_count" // Incoming request count total
ClientRequestContentLength = "http.client.request_content_length" // Incoming request bytes total
ClientResponseContentLength = "http.client.response_content_length" // Incoming response bytes total
ClientRoundTripLatency = "http.client.roundtrip_latency" // Incoming end to end duration, microseconds
)

// Filter is a predicate used to determine whether a given http.request should
// be traced. A Filter must return true if the request should be traced.
type Filter func(*http.Request) bool
6 changes: 6 additions & 0 deletions instrumentation/net/http/otelhttp/go.sum
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
Expand All @@ -15,8 +18,11 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.13.0/go.mod h1:SeQm4RTCcZ2/hlMSTuHb7nwIROe5odBtgfKx+7MMqEs=
go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA=
go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY=
go.opentelemetry.io/otel/exporters/stdout v0.13.0/go.mod h1:JJt8RpNY6K+ft9ir3iKpceCvT/rhzJXEExGrWFCbv1o=
go.opentelemetry.io/otel/sdk v0.13.0/go.mod h1:dKvLH8Uu8LcEPlSAUsfW7kMGaJBhk/1NYvpPZ6wIMbU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
215 changes: 215 additions & 0 deletions instrumentation/net/http/otelhttp/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
package otelhttp

import (
"context"
"io"
"net/http"
"sync"
"time"

"go.opentelemetry.io/otel/api/metric"
"go.opentelemetry.io/otel/label"
)

type statTransport struct {
meter metric.Meter
base *Transport
labels label.Set
counters map[string]metric.Int64Counter
valueRecorders map[string]metric.Float64ValueRecorder
}

type tracker struct {
ctx context.Context
respSize int64
respContentLength int64
reqSize int64
start time.Time
body io.ReadCloser
statusCode int
endOnce sync.Once
labels label.Set

counters map[string]metric.Int64Counter
valueRecorders map[string]metric.Float64ValueRecorder
}

// The following tags are applied to stats recorded by this package. Host, Path
// and Method are applied to all measures. StatusCode is not applied to
// ClientRequestCount or ServerRequestCount, since it is recorded before the status is known.
var (
// Host is the value of the HTTP Host header.
//
// The value of this tag can be controlled by the HTTP client, so you need
// to watch out for potentially generating high-cardinality labels in your
// metrics backend if you use this tag in views.
Host = label.Key("http.host")

// StatusCode is the numeric HTTP response status code,
// or "error" if a transport error occurred and no status code was read.
StatusCode = label.Key("http.status")

// Path is the URL path (not including query string) in the request.
//
// The value of this tag can be controlled by the HTTP client, so you need
// to watch out for potentially generating high-cardinality labels in your
// metrics backend if you use this tag in views.
Path = label.Key("http.path")

// Method is the HTTP method of the request, capitalized (GET, POST, etc.).
Method = label.Key("http.method")
)

// Client tag keys.
var (
// KeyClientMethod is the HTTP method, capitalized (i.e. GET, POST, PUT, DELETE, etc.).
KeyClientMethod = label.Key("http_client_method")
// KeyClientPath is the URL path (not including query string).
KeyClientPath = label.Key("http_client_path")
// KeyClientStatus is the HTTP status code as an integer (e.g. 200, 404, 500.), or "error" if no response status line was received.
KeyClientStatus = label.Key("http_client_status")
// KeyClientHost is the value of the request Host header.
KeyClientHost = label.Key("http_client_host")
)

func (trans *statTransport) applyConfig(c *config) {
trans.base.applyConfig(c)

trans.meter = c.Meter
trans.createMeasures()
}

// RoundTrip implements http.RoundTripper, delegating to Base and recording stats for the request.
func (trans *statTransport) RoundTrip(req *http.Request) (*http.Response, error) {
trans.labels = label.NewSet(
KeyClientHost.String(req.Host),
Host.String(req.Host),
KeyClientPath.String(req.URL.Path),
Path.String(req.URL.Path),
KeyClientMethod.String(req.Method),
Method.String(req.Method),
)

ctx := req.Context()
track := &tracker{
start: time.Now(),
ctx: ctx,
counters: trans.counters,
valueRecorders: trans.valueRecorders,
}
if req.Body == nil {
// TODO: Handle cases where ContentLength is not set.
track.reqSize = -1
} else if req.ContentLength > 0 {
track.reqSize = req.ContentLength
}
trans.counters[ClientRequestCount].Add(ctx, 1, trans.labels.ToSlice()...)

// Perform request.
resp, err := trans.base.RoundTrip(req)

if err != nil {
track.statusCode = http.StatusInternalServerError
track.end()
} else {
track.statusCode = resp.StatusCode
if req.Method != "HEAD" {
track.respContentLength = resp.ContentLength
}
if resp.Body == nil {
track.end()
} else {
track.body = resp.Body
resp.Body = wrappedBodyIO(track, resp.Body)
}
}
return resp, err
}

// wrappedBodyIO returns a wrapped version of the original
// Body and only implements the same combination of additional
// interfaces as the original.
func wrappedBodyIO(wrapper io.ReadCloser, body io.ReadCloser) io.ReadCloser {
wr, i0 := body.(io.Writer)
switch {
case !i0:
return struct {
io.ReadCloser
}{wrapper}

case i0:
return struct {
io.ReadCloser
io.Writer
}{wrapper, wr}
default:
return struct {
io.ReadCloser
}{wrapper}
}
}

func (trans *statTransport) createMeasures() {
trans.counters = make(map[string]metric.Int64Counter)
trans.valueRecorders = make(map[string]metric.Float64ValueRecorder)

clientRequestCountCounter, err := trans.meter.NewInt64Counter(ClientRequestCount)
handleErr(err)

requestBytesCounter, err := trans.meter.NewInt64Counter(ClientRequestContentLength)
handleErr(err)

responseBytesCounter, err := trans.meter.NewInt64Counter(ClientResponseContentLength)
handleErr(err)

serverLatencyMeasure, err := trans.meter.NewFloat64ValueRecorder(ClientRoundTripLatency)
handleErr(err)

trans.counters[ClientRequestCount] = clientRequestCountCounter
trans.counters[ClientRequestContentLength] = requestBytesCounter
trans.counters[ClientResponseContentLength] = responseBytesCounter
trans.valueRecorders[ClientRoundTripLatency] = serverLatencyMeasure
}

var _ io.ReadCloser = (*tracker)(nil)

func (t *tracker) end() {
t.endOnce.Do(func() {
latencyMs := float64(time.Since(t.start)) / float64(time.Millisecond)
respSize := t.respSize
if t.respSize == 0 && t.respContentLength > 0 {
respSize = t.respContentLength
}
labels := label.NewSet(
append(t.labels.ToSlice(), StatusCode.Int(t.statusCode),
KeyClientStatus.Int(t.statusCode))...,
)
ls := labels.ToSlice()

t.counters[ClientResponseContentLength].Add(t.ctx, respSize, ls...)
t.valueRecorders[ClientRoundTripLatency].Record(t.ctx, latencyMs, ls...)
if t.reqSize >= 0 {
t.counters[ClientRequestContentLength].Add(t.ctx, respSize, ls...)
}
})
}

func (t *tracker) Read(b []byte) (int, error) {
n, err := t.body.Read(b)
t.respSize += int64(n)
switch err {
case nil:
return n, nil
case io.EOF:
t.end()
}
return n, err
}

func (t *tracker) Close() error {
// Invoking endSpan on Close will help catch the cases
// in which a read returned a non-nil error, we set the
// span status but didn't end the span.
t.end()
return t.body.Close()
}
8 changes: 5 additions & 3 deletions instrumentation/net/http/otelhttp/transport.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,11 @@ var _ http.RoundTripper = &Transport{}

// NewTransport wraps the provided http.RoundTripper with one that
// starts a span and injects the span context into the outbound request headers.
func NewTransport(base http.RoundTripper, opts ...Option) *Transport {
t := Transport{
rt: base,
func NewTransport(base http.RoundTripper, opts ...Option) http.RoundTripper {
t := statTransport{
base: &Transport{
rt: base,
},
}

defaultOpts := []Option{
Expand Down

0 comments on commit ccdcdd6

Please sign in to comment.