From 8702b7fdc800d2e7bee6676f043d2f4b4cde1a2b Mon Sep 17 00:00:00 2001 From: Brett Jones Date: Tue, 7 Jan 2020 18:53:03 -0600 Subject: [PATCH] sidecar: allow custom http con pool size fix #1953 Signed-off-by: Brett Jones --- CHANGELOG.md | 1 + cmd/thanos/sidecar.go | 7 ++++++- pkg/store/prometheus.go | 7 ++++++- pkg/store/prometheus_test.go | 18 ++++++++---------- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 134714a9c69..abc62fa2ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed +- [#1955](https://github.com/thanos-io/thanos/pull/1955) Sidecar: enable command line flag to set http connection pool size `--prometheus.connection-pool-size` - [#1919](https://github.com/thanos-io/thanos/issues/1919) Compactor: Fixed potential data loss when uploading older blocks, or upload taking long time while compactor is running. - [#1937](https://github.com/thanos-io/thanos/pull/1937) Compactor: Improved synchronization of meta JSON files. diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index f6961e9766b..cf8ff697684 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -45,6 +45,9 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { promReadyTimeout := cmd.Flag("prometheus.ready_timeout", "Maximum time to wait for the Prometheus instance to start up"). Default("10m").Duration() + connectionPoolSize := cmd.Flag("prometheus.connection-pool-size", "sets the MaxIdleConnsPerHost and MaxIdleConns when connecting to prometheus"). + Default("100").Int() + dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB."). Default("./data").String() @@ -95,6 +98,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) { *ignoreBlockSize, component.Sidecar, *minTime, + *connectionPoolSize, ) } } @@ -120,6 +124,7 @@ func runSidecar( ignoreBlockSize bool, comp component.Component, limitMinTime thanosmodel.TimeOrDurationValue, + connectionPoolSize int, ) error { var m = &promMetadata{ promURL: promURL, @@ -244,7 +249,7 @@ func runSidecar( { promStore, err := store.NewPrometheusStore( - logger, nil, promURL, component.Sidecar, m.Labels, m.Timestamps) + logger, nil, promURL, component.Sidecar, m.Labels, m.Timestamps, connectionPoolSize) if err != nil { return errors.Wrap(err, "create Prometheus store") } diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 4012eceb9c8..74f4eace9d7 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -73,13 +73,18 @@ func NewPrometheusStore( component component.StoreAPI, externalLabels func() labels.Labels, timestamps func() (mint int64, maxt int64), + connectionPoolSize int, ) (*PrometheusStore, error) { if logger == nil { logger = log.NewNopLogger() } if client == nil { + t := http.DefaultTransport.(*http.Transport) + t.MaxIdleConnsPerHost = connectionPoolSize + t.MaxIdleConns = connectionPoolSize + client = &http.Client{ - Transport: tracing.HTTPTripperware(logger, http.DefaultTransport), + Transport: tracing.HTTPTripperware(logger, t), } } p := &PrometheusStore{ diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 5db3ee75cc9..6102eaec33f 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "net/http" "net/url" "testing" "time" @@ -59,7 +60,7 @@ func testPrometheusStoreSeriesE2e(t *testing.T, prefix string) { limitMinT := int64(0) proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return limitMinT, -1 }) // Maxt does not matter. + func() (int64, int64) { return limitMinT, -1 }, http.DefaultMaxIdleConnsPerHost) // Maxt does not matter. testutil.Ok(t, err) // Query all three samples except for the first one. Since we round up queried data @@ -211,7 +212,7 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { limitMinT := int64(0) proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return limitMinT, -1 }) // Maxt does not matter. + func() (int64, int64) { return limitMinT, -1 }, http.DefaultMaxIdleConnsPerHost) // Maxt does not matter. testutil.Ok(t, err) { @@ -284,7 +285,7 @@ func TestPrometheusStore_LabelValues_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels, nil) + proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels, nil, http.DefaultMaxIdleConnsPerHost) testutil.Ok(t, err) resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ @@ -318,7 +319,7 @@ func TestPrometheusStore_ExternalLabelValues_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels, nil) + proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels, nil, http.DefaultMaxIdleConnsPerHost) testutil.Ok(t, err) resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{ @@ -364,8 +365,7 @@ func TestPrometheusStore_Series_MatchExternalLabel_e2e(t *testing.T) { proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 0, math.MaxInt64 }, - ) + func() (int64, int64) { return 0, math.MaxInt64 }, http.DefaultMaxIdleConnsPerHost) testutil.Ok(t, err) srv := newStoreSeriesServer(ctx) @@ -410,8 +410,7 @@ func TestPrometheusStore_Info(t *testing.T) { proxy, err := NewPrometheusStore(nil, nil, nil, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 123, 456 }, - ) + func() (int64, int64) { return 123, 456 }, http.DefaultMaxIdleConnsPerHost) testutil.Ok(t, err) resp, err := proxy.Info(ctx, &storepb.InfoRequest{}) @@ -489,8 +488,7 @@ func TestPrometheusStore_Series_SplitSamplesIntoChunksWithMaxSizeOfUint16_e2e(t proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, func() labels.Labels { return labels.FromStrings("region", "eu-west") }, - func() (int64, int64) { return 0, math.MaxInt64 }, - ) + func() (int64, int64) { return 0, math.MaxInt64 }, http.DefaultMaxIdleConnsPerHost) testutil.Ok(t, err) // We build chunks only for SAMPLES method. Make sure we ask for SAMPLES only.