diff --git a/site/content/overview/media.md b/site/content/overview/media.md index 5cc4829656..e8e732bac4 100644 --- a/site/content/overview/media.md +++ b/site/content/overview/media.md @@ -16,7 +16,7 @@ Recordings of all past meetups can be found on a [Vimeo M3 Community Meetings fo - [June 2020 Meetup](https://vimeo.com/440390957). -- [July 2020 Meetup and LinkedIn presentation](https://vimeo.com/440449118). +- [July 2020 Meetup and LinkedIn presentation](https://vimeo.com/440390957). - [August 2020 Meetup and Walmart presentation](https://vimeo.com/449883279). diff --git a/site/static/about/index.html b/site/static/about/index.html index 72de081cdd..72809ccffd 100644 --- a/site/static/about/index.html +++ b/site/static/about/index.html @@ -44,7 +44,7 @@ - + @@ -53,9 +53,8 @@ - - - + + diff --git a/site/static/community/index.html b/site/static/community/index.html index e6c4ca56ef..753b3fa53e 100644 --- a/site/static/community/index.html +++ b/site/static/community/index.html @@ -44,7 +44,7 @@ - + @@ -52,9 +52,8 @@ - - - + + diff --git a/site/static/index.html b/site/static/index.html index 648a41ee26..2827448183 100644 --- a/site/static/index.html +++ b/site/static/index.html @@ -44,7 +44,7 @@ - + @@ -53,8 +53,8 @@ - - + + diff --git a/src/cluster/placement/staged_placement.go b/src/cluster/placement/staged_placement.go index bf7b5df6a4..070562dfce 100644 --- a/src/cluster/placement/staged_placement.go +++ b/src/cluster/placement/staged_placement.go @@ -24,7 +24,8 @@ import ( "errors" "sort" "sync" - "sync/atomic" + + "go.uber.org/atomic" "github.com/m3db/m3/src/cluster/generated/proto/placementpb" "github.com/m3db/m3/src/x/clock" @@ -44,7 +45,7 @@ type activeStagedPlacement struct { onPlacementsAddedFn OnPlacementsAddedFn onPlacementsRemovedFn OnPlacementsRemovedFn - expiring int32 + expiring atomic.Int32 closed bool doneFn DoneFn } @@ -98,8 +99,6 @@ func (p *activeStagedPlacement) Close() error { } func (p *activeStagedPlacement) Version() int { - p.RLock() - defer p.RUnlock() return p.version } @@ -115,7 +114,7 @@ func (p *activeStagedPlacement) activePlacementWithLock(timeNanos int64) (Placem } placement := p.placements[idx] // If the placement that's in effect is not the first placment, expire the stale ones. - if idx > 0 && atomic.CompareAndSwapInt32(&p.expiring, 0, 1) { + if idx > 0 && p.expiring.CAS(0, 1) { go p.expire() } return placement, nil @@ -126,7 +125,7 @@ func (p *activeStagedPlacement) expire() { // because this code path is triggered very infrequently. cleanup := func() { p.Unlock() - atomic.StoreInt32(&p.expiring, 0) + p.expiring.Store(0) } p.Lock() defer cleanup() diff --git a/src/cluster/placement/staged_placement_test.go b/src/cluster/placement/staged_placement_test.go index fae5633d58..06a3ce53dd 100644 --- a/src/cluster/placement/staged_placement_test.go +++ b/src/cluster/placement/staged_placement_test.go @@ -28,6 +28,7 @@ import ( "github.com/m3db/m3/src/cluster/shard" "github.com/stretchr/testify/require" + "go.uber.org/atomic" ) var ( @@ -395,7 +396,6 @@ func TestActiveStagedPlacementExpireAlreadyClosed(t *testing.T) { p := &activeStagedPlacement{ placements: append([]Placement{}, testActivePlacements...), nowFn: func() time.Time { return time.Unix(0, 99999) }, - expiring: 1, closed: true, onPlacementsRemovedFn: func(placements []Placement) { for _, placement := range placements { @@ -403,25 +403,100 @@ func TestActiveStagedPlacementExpireAlreadyClosed(t *testing.T) { } }, } + p.expiring.Store(1) p.expire() - require.Equal(t, int32(0), p.expiring) + require.Equal(t, int32(0), p.expiring.Load()) require.Nil(t, removedInstances) } +func TestActiveStagedPlacementVersionWhileExpiring(t *testing.T) { + for i := 0; i < 100; i++ { + // test itself is fast, unless there's a deadlock + testActiveStagedPlacementVersionWhileExpiring(t) + } +} + +//nolint:gocyclo +func testActiveStagedPlacementVersionWhileExpiring(t *testing.T) { + var ( + doneCh = make(chan struct{}) + signalCh = make(chan struct{}) + version int + ranCleanup atomic.Bool + ) + + p := newActiveStagedPlacement(append([]Placement{}, testActivePlacements...), 42, nil) + p.nowFn = func() time.Time { + return time.Unix(0, testActivePlacements[len(testActivePlacements)-1].CutoverNanos()+1) + } + p.onPlacementsRemovedFn = func(_ []Placement) { + ranCleanup.Store(true) + } + + go func() { + defer close(doneCh) + for { + version = p.Version() + select { + case signalCh <- struct{}{}: + return + default: + } + } + }() + + pl, doneFn, err := p.ActivePlacement() + require.NoError(t, err) + require.NotNil(t, pl) + require.NotNil(t, doneFn) + + // active placement is not the first in the list - expiration of past + // placements must be triggered + require.Equal(t, int32(1), p.expiring.Load()) + + // make sure p.Version() call was attempted at least once + select { + case <-signalCh: + case <-time.After(time.Second): + t.Fatalf("test timed out, deadlock?") + } + + // release placement lock to unblock expiration process + doneFn() + select { + case <-doneCh: + case <-time.After(time.Second): + t.Fatalf("test timed out, deadlock?") + } + + // there's no good way to determine when expire process has been completed, + // try polling for 100ms + for i := 0; i < 100; i++ { + if ranCleanup.Load() && p.expiring.Load() == int32(0) { + break + } + time.Sleep(1 * time.Millisecond) + } + + require.Equal(t, 42, version) + require.True(t, ranCleanup.Load()) + require.Equal(t, int32(0), p.expiring.Load()) +} + func TestActiveStagedPlacementExpireAlreadyExpired(t *testing.T) { var removedInstances [][]Instance p := &activeStagedPlacement{ placements: append([]Placement{}, testActivePlacements...), nowFn: func() time.Time { return time.Unix(0, 0) }, - expiring: 1, onPlacementsRemovedFn: func(placements []Placement) { for _, placement := range placements { removedInstances = append(removedInstances, placement.Instances()) } }, } + p.expiring.Store(1) p.expire() - require.Equal(t, int32(0), p.expiring) + require.Equal(t, int32(0), p.expiring.Load()) require.Nil(t, removedInstances) } @@ -430,15 +505,15 @@ func TestActiveStagedPlacementExpireSuccess(t *testing.T) { p := &activeStagedPlacement{ placements: append([]Placement{}, testActivePlacements...), nowFn: func() time.Time { return time.Unix(0, 99999) }, - expiring: 1, onPlacementsRemovedFn: func(placements []Placement) { for _, placement := range placements { removedInstances = append(removedInstances, placement.Instances()) } }, } + p.expiring.Store(1) p.expire() - require.Equal(t, int32(0), p.expiring) + require.Equal(t, int32(0), p.expiring.Load()) require.Equal(t, [][]Instance{testActivePlacements[0].Instances()}, removedInstances) require.Equal(t, 1, len(p.placements)) validateSnapshot(t, testActivePlacements[1], p.placements[0]) diff --git a/src/cmd/services/m3coordinator/downsample/downsampler_test.go b/src/cmd/services/m3coordinator/downsample/downsampler_test.go index ddcfff7d23..25241157f3 100644 --- a/src/cmd/services/m3coordinator/downsample/downsampler_test.go +++ b/src/cmd/services/m3coordinator/downsample/downsampler_test.go @@ -601,7 +601,7 @@ func TestDownsamplerAggregationWithRulesConfigMappingRulesAggregationType(t *tes tags: map[string]string{ "__g0__": "nginx_edge", "__g1__": "health", - "__g2__": "Max", + "__g2__": "upper", }, values: []expectedValue{{value: 30}}, attributes: &storagemetadata.Attributes{ @@ -668,7 +668,7 @@ func TestDownsamplerAggregationWithRulesConfigMappingRulesMultipleAggregationTyp tags: map[string]string{ "__g0__": "nginx_edge", "__g1__": "health", - "__g2__": "Max", + "__g2__": "upper", }, values: []expectedValue{{value: 30}}, attributes: &storagemetadata.Attributes{ @@ -681,7 +681,7 @@ func TestDownsamplerAggregationWithRulesConfigMappingRulesMultipleAggregationTyp tags: map[string]string{ "__g0__": "nginx_edge", "__g1__": "health", - "__g2__": "Sum", + "__g2__": "sum", }, values: []expectedValue{{value: 60}}, attributes: &storagemetadata.Attributes{ @@ -742,7 +742,7 @@ func TestDownsamplerAggregationWithRulesConfigMappingRulesGraphitePrefixAndAggre "__g1__": "counter", "__g2__": "nginx_edge", "__g3__": "health", - "__g4__": "Max", + "__g4__": "upper", }, values: []expectedValue{{value: 30}}, attributes: &storagemetadata.Attributes{ diff --git a/src/cmd/services/m3coordinator/downsample/metrics_appender.go b/src/cmd/services/m3coordinator/downsample/metrics_appender.go index 9632e1f70d..d073e813ee 100644 --- a/src/cmd/services/m3coordinator/downsample/metrics_appender.go +++ b/src/cmd/services/m3coordinator/downsample/metrics_appender.go @@ -595,7 +595,7 @@ func (a *metricsAppender) augmentTags( var ( count = tags.countPrefix(graphite.Prefix) name = graphite.TagName(count) - value = types[0].Bytes() + value = types[0].Name() ) tags.append(name, value) } diff --git a/src/metrics/aggregation/type.go b/src/metrics/aggregation/type.go index 5c9913345e..db10e274ae 100644 --- a/src/metrics/aggregation/type.go +++ b/src/metrics/aggregation/type.go @@ -101,6 +101,31 @@ var ( } typeStringMap map[string]Type + + typeStringNames = map[Type][]byte{ + Last: []byte("last"), + Min: []byte("lower"), + Max: []byte("upper"), + Mean: []byte("mean"), + Median: []byte("median"), + Count: []byte("count"), + Sum: []byte("sum"), + SumSq: []byte("sum_sq"), + Stdev: []byte("stdev"), + P10: []byte("p10"), + P20: []byte("p20"), + P30: []byte("p30"), + P40: []byte("p40"), + P50: []byte("p50"), + P60: []byte("p60"), + P70: []byte("p70"), + P80: []byte("p80"), + P90: []byte("p90"), + P95: []byte("p95"), + P99: []byte("p99"), + P999: []byte("p999"), + P9999: []byte("p9999"), + } ) // Type defines an aggregation function. @@ -232,6 +257,15 @@ func (a *Type) UnmarshalText(data []byte) error { return nil } +// Name returns the name of the Type. +func (a Type) Name() []byte { + name, ok := typeStringNames[a] + if ok { + return name + } + return a.Bytes() +} + func validateProtoType(a aggregationpb.AggregationType) error { _, ok := aggregationpb.AggregationType_name[int32(a)] if !ok { diff --git a/src/metrics/aggregation/types_options_test.go b/src/metrics/aggregation/types_options_test.go index 00040366d5..ec7080c004 100644 --- a/src/metrics/aggregation/types_options_test.go +++ b/src/metrics/aggregation/types_options_test.go @@ -449,32 +449,8 @@ func validateQuantiles(t *testing.T, o TypesOptions) { } func typeStrings(overrides map[Type][]byte) [][]byte { - defaultTypeStrings := map[Type][]byte{ - Last: []byte("last"), - Min: []byte("lower"), - Max: []byte("upper"), - Mean: []byte("mean"), - Median: []byte("median"), - Count: []byte("count"), - Sum: []byte("sum"), - SumSq: []byte("sum_sq"), - Stdev: []byte("stdev"), - P10: []byte("p10"), - P20: []byte("p20"), - P30: []byte("p30"), - P40: []byte("p40"), - P50: []byte("p50"), - P60: []byte("p60"), - P70: []byte("p70"), - P80: []byte("p80"), - P90: []byte("p90"), - P95: []byte("p95"), - P99: []byte("p99"), - P999: []byte("p999"), - P9999: []byte("p9999"), - } res := make([][]byte, maxTypeID+1) - for t, bstr := range defaultTypeStrings { + for t, bstr := range typeStringNames { if override, exist := overrides[t]; exist { res[t.ID()] = override continue diff --git a/src/query/api/experimental/annotated/handler.go b/src/query/api/experimental/annotated/handler.go index 4243b91fd6..0625606f90 100644 --- a/src/query/api/experimental/annotated/handler.go +++ b/src/query/api/experimental/annotated/handler.go @@ -72,16 +72,18 @@ func NewHandler( func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Body == nil { - h.metrics.writeErrorsClient.Inc(1) - xhttp.WriteError(w, errEmptyBody) + err := errEmptyBody + h.metrics.incError(err) + xhttp.WriteError(w, err) return } defer r.Body.Close() req, err := parseRequest(r.Body) if err != nil { - h.metrics.writeErrorsClient.Inc(1) - xhttp.WriteError(w, xhttp.NewError(err, http.StatusBadRequest)) + resultError := xhttp.NewError(err, http.StatusBadRequest) + h.metrics.incError(resultError) + xhttp.WriteError(w, resultError) return } @@ -97,20 +99,16 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { break } - var ( - status = http.StatusBadRequest - counter = h.metrics.writeErrorsClient - ) + status := http.StatusBadRequest if foundInternalErr { status = http.StatusInternalServerError - counter = h.metrics.writeErrorsServer } - counter.Inc(1) - err = fmt.Errorf( - "unable to write metric batch, encountered %d errors: %v", len(batchErr.Errors()), batchErr.Error(), - ) - xhttp.WriteError(w, xhttp.NewError(err, status)) + err = fmt.Errorf("unable to write metric batch, encountered %d errors: %w", + len(batchErr.Errors()), batchErr) + responseError := xhttp.NewError(err, status) + h.metrics.incError(responseError) + xhttp.WriteError(w, responseError) return } @@ -150,3 +148,11 @@ func newHandlerMetrics(s tally.Scope) handlerMetrics { writeErrorsClient: s.SubScope("write").Tagged(map[string]string{"code": "4XX"}).Counter("errors"), } } + +func (m *handlerMetrics) incError(err error) { + if xhttp.IsClientError(err) { + m.writeErrorsClient.Inc(1) + } else { + m.writeErrorsServer.Inc(1) + } +} diff --git a/src/query/api/v1/handler/prometheus/native/read.go b/src/query/api/v1/handler/prometheus/native/read.go index 255f6dd69c..d1aa35437e 100644 --- a/src/query/api/v1/handler/prometheus/native/read.go +++ b/src/query/api/v1/handler/prometheus/native/read.go @@ -117,7 +117,7 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { parsedOptions, rErr := ParseRequest(ctx, r, h.instant, h.opts) if rErr != nil { - h.promReadMetrics.fetchErrorsClient.Inc(1) + h.promReadMetrics.incError(rErr) logger.Error("could not parse request", zap.Error(rErr)) xhttp.WriteError(w, rErr) return @@ -143,7 +143,7 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { logger.Error("range query error", zap.Error(err), zap.Any("parsedOptions", parsedOptions)) - h.promReadMetrics.fetchErrorsServer.Inc(1) + h.promReadMetrics.incError(err) xhttp.WriteError(w, err) return diff --git a/src/query/api/v1/handler/prometheus/native/read_common.go b/src/query/api/v1/handler/prometheus/native/read_common.go index 5420693b7b..206b1ccce0 100644 --- a/src/query/api/v1/handler/prometheus/native/read_common.go +++ b/src/query/api/v1/handler/prometheus/native/read_common.go @@ -35,6 +35,7 @@ import ( "github.com/m3db/m3/src/query/storage" "github.com/m3db/m3/src/query/ts" xerrors "github.com/m3db/m3/src/x/errors" + xhttp "github.com/m3db/m3/src/x/net/http" xopentracing "github.com/m3db/m3/src/x/opentracing" opentracinglog "github.com/opentracing/opentracing-go/log" @@ -59,6 +60,14 @@ func newPromReadMetrics(scope tally.Scope) promReadMetrics { } } +func (m *promReadMetrics) incError(err error) { + if xhttp.IsClientError(err) { + m.fetchErrorsClient.Inc(1) + } else { + m.fetchErrorsServer.Inc(1) + } +} + // ReadResponse is the response that gets returned to the user type ReadResponse struct { Results []ts.Series `json:"results,omitempty"` diff --git a/src/query/api/v1/handler/prometheus/remote/read.go b/src/query/api/v1/handler/prometheus/remote/read.go index 83f32b8f27..7bdaf99aaf 100644 --- a/src/query/api/v1/handler/prometheus/remote/read.go +++ b/src/query/api/v1/handler/prometheus/remote/read.go @@ -101,6 +101,14 @@ func newPromReadMetrics(scope tally.Scope) promReadMetrics { } } +func (m *promReadMetrics) incError(err error) { + if xhttp.IsClientError(err) { + m.fetchErrorsClient.Inc(1) + } else { + m.fetchErrorsServer.Inc(1) + } +} + func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { timer := h.promReadMetrics.fetchTimerSuccess.Start() defer timer.Stop() @@ -109,7 +117,7 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { logger := logging.WithContext(ctx, h.opts.InstrumentOpts()) req, fetchOpts, rErr := ParseRequest(ctx, r, h.opts) if rErr != nil { - h.promReadMetrics.fetchErrorsClient.Inc(1) + h.promReadMetrics.incError(rErr) logger.Error("remote read query parse error", zap.Error(rErr), zap.Any("req", req), @@ -121,7 +129,7 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { cancelWatcher := handler.NewResponseWriterCanceller(w, h.opts.InstrumentOpts()) readResult, err := Read(ctx, cancelWatcher, req, fetchOpts, h.opts) if err != nil { - h.promReadMetrics.fetchErrorsServer.Inc(1) + h.promReadMetrics.incError(err) logger.Error("remote read query error", zap.Error(err), zap.Any("req", req), @@ -183,7 +191,7 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if err != nil { - h.promReadMetrics.fetchErrorsServer.Inc(1) + h.promReadMetrics.incError(err) } else { h.promReadMetrics.fetchSuccess.Inc(1) } diff --git a/src/query/api/v1/handler/prometheus/remote/write.go b/src/query/api/v1/handler/prometheus/remote/write.go index 063670a304..e5a7ff2736 100644 --- a/src/query/api/v1/handler/prometheus/remote/write.go +++ b/src/query/api/v1/handler/prometheus/remote/write.go @@ -196,6 +196,14 @@ type promWriteMetrics struct { forwardLatency tally.Histogram } +func (m *promWriteMetrics) incError(err error) { + if xhttp.IsClientError(err) { + m.writeErrorsClient.Inc(1) + } else { + m.writeErrorsServer.Inc(1) + } +} + func newPromWriteMetrics(scope tally.Scope) (promWriteMetrics, error) { upTo1sBuckets, err := tally.LinearDurationBuckets(0, 100*time.Millisecond, 10) if err != nil { @@ -263,7 +271,7 @@ func (h *PromWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { checkedReq, err := h.checkedParseRequest(r) if err != nil { - h.metrics.writeErrorsClient.Inc(1) + h.metrics.incError(err) xhttp.WriteError(w, err) return } @@ -359,10 +367,8 @@ func (h *PromWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch { case numBadRequest == len(errs): status = http.StatusBadRequest - h.metrics.writeErrorsClient.Inc(1) default: status = http.StatusInternalServerError - h.metrics.writeErrorsServer.Inc(1) } logger := logging.WithContext(r.Context(), h.instrumentOpts) @@ -374,9 +380,9 @@ func (h *PromWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { zap.String("lastRegularError", lastRegularErr), zap.String("lastBadRequestErr", lastBadRequestErr)) - var resultErr string + var resultErrMessage string if lastRegularErr != "" { - resultErr = fmt.Sprintf("retryable_errors: count=%d, last=%s", + resultErrMessage = fmt.Sprintf("retryable_errors: count=%d, last=%s", numRegular, lastRegularErr) } if lastBadRequestErr != "" { @@ -384,10 +390,13 @@ func (h *PromWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if lastRegularErr != "" { sep = ", " } - resultErr = fmt.Sprintf("%s%sbad_request_errors: count=%d, last=%s", - resultErr, sep, numBadRequest, lastBadRequestErr) + resultErrMessage = fmt.Sprintf("%s%sbad_request_errors: count=%d, last=%s", + resultErrMessage, sep, numBadRequest, lastBadRequestErr) } - xhttp.WriteError(w, xhttp.NewError(errors.New(resultErr), status)) + + resultError := xhttp.NewError(errors.New(resultErrMessage), status) + h.metrics.incError(resultError) + xhttp.WriteError(w, resultError) return } diff --git a/src/x/net/http/errors.go b/src/x/net/http/errors.go index a0fbcde08d..331e23e4d3 100644 --- a/src/x/net/http/errors.go +++ b/src/x/net/http/errors.go @@ -116,3 +116,9 @@ func getStatusCode(err error) int { } return http.StatusInternalServerError } + +// IsClientError returns true if this error would result in 4xx status code +func IsClientError(err error) bool { + code := getStatusCode(err) + return code >= 400 && code < 500 +} diff --git a/src/x/net/http/errors_test.go b/src/x/net/http/errors_test.go new file mode 100644 index 0000000000..4bd5ecf46d --- /dev/null +++ b/src/x/net/http/errors_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package xhttp + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + xerrors "github.com/m3db/m3/src/x/errors" +) + +func TestIsClientError(t *testing.T) { + tests := []struct { + err error + expected bool + }{ + {NewError(fmt.Errorf("xhttp.Error(400)"), 400), true}, + {NewError(fmt.Errorf("xhttp.Error(499)"), 499), true}, + {xerrors.NewInvalidParamsError(fmt.Errorf("InvalidParamsError")), true}, + {xerrors.NewRetryableError(xerrors.NewInvalidParamsError( + fmt.Errorf("InvalidParamsError insde RetyrableError"))), true}, + + {NewError(fmt.Errorf("xhttp.Error(399)"), 399), false}, + {NewError(fmt.Errorf("xhttp.Error(500)"), 500), false}, + {xerrors.NewRetryableError(fmt.Errorf("any error inside RetryableError")), false}, + {fmt.Errorf("any error"), false}, + } + + for _, tt := range tests { + t.Run(tt.err.Error(), func(t *testing.T) { + require.Equal(t, tt.expected, IsClientError(tt.err)) + }) + } +}