From 4f34b58bf0bd2c2663dcd60755b3c5f004504afb Mon Sep 17 00:00:00 2001 From: bwplotka Date: Sat, 27 Aug 2022 21:34:55 +0200 Subject: [PATCH] Initial commit. Signed-off-by: bwplotka --- .bingo/.gitignore | 12 ++ .bingo/README.md | 14 ++ .bingo/Variables.mk | 55 ++++++ .bingo/bingo.mod | 5 + .bingo/faillint.mod | 5 + .bingo/go.mod | 1 + .bingo/goimports.mod | 5 + .bingo/golangci-lint.mod | 5 + .bingo/mdox.mod | 5 + .bingo/misspell.mod | 5 + .bingo/variables.env | 22 +++ .errcheck_excludes.txt | 3 + .github/workflows/go.yaml | 58 +++++++ .gitignore | 3 + COPYRIGHT | 2 + Makefile | 110 ++++++++++++ README.md | 16 +- backoff/backoff.go | 112 +++++++++++++ backoff/backoff_test.go | 108 ++++++++++++ clilog/clilog.go | 313 +++++++++++++++++++++++++++++++++++ clilog/doc.go | 17 ++ errcapture/do.go | 47 ++++++ errcapture/do_test.go | 79 +++++++++ errcapture/doc.go | 29 ++++ errors/doc.go | 19 +++ errors/errors.go | 142 ++++++++++++++++ errors/errors_test.go | 150 +++++++++++++++++ errors/stacktrace.go | 55 ++++++ errors/stacktrace_test.go | 32 ++++ go.mod | 16 ++ go.sum | 33 ++++ logerrcapture/do.go | 51 ++++++ logerrcapture/do_test.go | 66 ++++++++ logerrcapture/doc.go | 30 ++++ merrors/doc.go | 20 +++ merrors/merrors.go | 215 ++++++++++++++++++++++++ merrors/merrors_test.go | 223 +++++++++++++++++++++++++ runutil/doc.go | 28 ++++ runutil/example_test.go | 49 ++++++ runutil/runutil.go | 61 +++++++ testutil/doc.go | 6 + testutil/testorbench.go | 85 ++++++++++ testutil/testorbench_test.go | 47 ++++++ testutil/testutil.go | 181 ++++++++++++++++++++ 44 files changed, 2539 insertions(+), 1 deletion(-) create mode 100755 .bingo/.gitignore create mode 100755 .bingo/README.md create mode 100644 .bingo/Variables.mk create mode 100644 .bingo/bingo.mod create mode 100644 .bingo/faillint.mod create mode 100755 .bingo/go.mod create mode 100644 .bingo/goimports.mod create mode 100644 .bingo/golangci-lint.mod create mode 100644 .bingo/mdox.mod create mode 100644 .bingo/misspell.mod create mode 100644 .bingo/variables.env create mode 100644 .errcheck_excludes.txt create mode 100644 .github/workflows/go.yaml create mode 100644 COPYRIGHT create mode 100644 Makefile create mode 100644 backoff/backoff.go create mode 100644 backoff/backoff_test.go create mode 100644 clilog/clilog.go create mode 100644 clilog/doc.go create mode 100644 errcapture/do.go create mode 100644 errcapture/do_test.go create mode 100644 errcapture/doc.go create mode 100644 errors/doc.go create mode 100644 errors/errors.go create mode 100644 errors/errors_test.go create mode 100644 errors/stacktrace.go create mode 100644 errors/stacktrace_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 logerrcapture/do.go create mode 100644 logerrcapture/do_test.go create mode 100644 logerrcapture/doc.go create mode 100644 merrors/doc.go create mode 100644 merrors/merrors.go create mode 100644 merrors/merrors_test.go create mode 100644 runutil/doc.go create mode 100644 runutil/example_test.go create mode 100644 runutil/runutil.go create mode 100644 testutil/doc.go create mode 100644 testutil/testorbench.go create mode 100644 testutil/testorbench_test.go create mode 100644 testutil/testutil.go diff --git a/.bingo/.gitignore b/.bingo/.gitignore new file mode 100755 index 0000000..4f2055b --- /dev/null +++ b/.bingo/.gitignore @@ -0,0 +1,12 @@ + +# Ignore everything +* + +# But not these files: +!.gitignore +!*.mod +!README.md +!Variables.mk +!variables.env + +*tmp.mod diff --git a/.bingo/README.md b/.bingo/README.md new file mode 100755 index 0000000..7a5c2d4 --- /dev/null +++ b/.bingo/README.md @@ -0,0 +1,14 @@ +# Project Development Dependencies. + +This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by https://github.com/bwplotka/bingo. + +* Run `bingo get` to install all tools having each own module file in this directory. +* Run `bingo get ` to install that have own module file in this directory. +* For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use $() variable where is the .bingo/.mod. +* For shell: Run `source .bingo/variables.env` to source all environment variable for each tool. +* For go: Import `.bingo/variables.go` to for variable names. +* See https://github.com/bwplotka/bingo or -h on how to add, remove or change binaries dependencies. + +## Requirements + +* Go 1.14+ diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk new file mode 100644 index 0000000..4d28a23 --- /dev/null +++ b/.bingo/Variables.mk @@ -0,0 +1,55 @@ +# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.4.2. DO NOT EDIT. +# All tools are designed to be build inside $GOBIN. +BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +GOPATH ?= $(shell go env GOPATH) +GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin +GO ?= $(shell which go) + +# Below generated variables ensure that every time a tool under each variable is invoked, the correct version +# will be used; reinstalling only if needed. +# For example for bingo variable: +# +# In your main Makefile (for non array binaries): +# +#include .bingo/Variables.mk # Assuming -dir was set to .bingo . +# +#command: $(BINGO) +# @echo "Running bingo" +# @$(BINGO) +# +BINGO := $(GOBIN)/bingo-v0.2.4-0.20201227120526-df6eae8f6734 +$(BINGO): $(BINGO_DIR)/bingo.mod + @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. + @echo "(re)installing $(GOBIN)/bingo-v0.2.4-0.20201227120526-df6eae8f6734" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=bingo.mod -o=$(GOBIN)/bingo-v0.2.4-0.20201227120526-df6eae8f6734 "github.com/bwplotka/bingo" + +FAILLINT := $(GOBIN)/faillint-v1.5.0 +$(FAILLINT): $(BINGO_DIR)/faillint.mod + @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. + @echo "(re)installing $(GOBIN)/faillint-v1.5.0" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=faillint.mod -o=$(GOBIN)/faillint-v1.5.0 "github.com/fatih/faillint" + +GOIMPORTS := $(GOBIN)/goimports-v0.0.0-20200519204825-e64124511800 +$(GOIMPORTS): $(BINGO_DIR)/goimports.mod + @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. + @echo "(re)installing $(GOBIN)/goimports-v0.0.0-20200519204825-e64124511800" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=goimports.mod -o=$(GOBIN)/goimports-v0.0.0-20200519204825-e64124511800 "golang.org/x/tools/cmd/goimports" + +GOLANGCI_LINT := $(GOBIN)/golangci-lint-v1.26.0 +$(GOLANGCI_LINT): $(BINGO_DIR)/golangci-lint.mod + @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. + @echo "(re)installing $(GOBIN)/golangci-lint-v1.26.0" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v1.26.0 "github.com/golangci/golangci-lint/cmd/golangci-lint" + +MDOX := $(GOBIN)/mdox-v0.2.2-0.20210731105602-946757ef5f98 +$(MDOX): $(BINGO_DIR)/mdox.mod + @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. + @echo "(re)installing $(GOBIN)/mdox-v0.2.2-0.20210731105602-946757ef5f98" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.2.2-0.20210731105602-946757ef5f98 "github.com/bwplotka/mdox" + +MISSPELL := $(GOBIN)/misspell-v0.3.4 +$(MISSPELL): $(BINGO_DIR)/misspell.mod + @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. + @echo "(re)installing $(GOBIN)/misspell-v0.3.4" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=misspell.mod -o=$(GOBIN)/misspell-v0.3.4 "github.com/client9/misspell/cmd/misspell" + diff --git a/.bingo/bingo.mod b/.bingo/bingo.mod new file mode 100644 index 0000000..ec32efc --- /dev/null +++ b/.bingo/bingo.mod @@ -0,0 +1,5 @@ +module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT + +go 1.15 + +require github.com/bwplotka/bingo v0.2.4-0.20201227120526-df6eae8f6734 diff --git a/.bingo/faillint.mod b/.bingo/faillint.mod new file mode 100644 index 0000000..304af0e --- /dev/null +++ b/.bingo/faillint.mod @@ -0,0 +1,5 @@ +module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT + +go 1.14 + +require github.com/fatih/faillint v1.5.0 diff --git a/.bingo/go.mod b/.bingo/go.mod new file mode 100755 index 0000000..610249a --- /dev/null +++ b/.bingo/go.mod @@ -0,0 +1 @@ +module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. \ No newline at end of file diff --git a/.bingo/goimports.mod b/.bingo/goimports.mod new file mode 100644 index 0000000..1753f49 --- /dev/null +++ b/.bingo/goimports.mod @@ -0,0 +1,5 @@ +module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT + +go 1.14 + +require golang.org/x/tools v0.0.0-20200519204825-e64124511800 // cmd/goimports diff --git a/.bingo/golangci-lint.mod b/.bingo/golangci-lint.mod new file mode 100644 index 0000000..f6a2f50 --- /dev/null +++ b/.bingo/golangci-lint.mod @@ -0,0 +1,5 @@ +module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT + +go 1.14 + +require github.com/golangci/golangci-lint v1.26.0 // cmd/golangci-lint diff --git a/.bingo/mdox.mod b/.bingo/mdox.mod new file mode 100644 index 0000000..d6baf1b --- /dev/null +++ b/.bingo/mdox.mod @@ -0,0 +1,5 @@ +module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT + +go 1.15 + +require github.com/bwplotka/mdox v0.2.2-0.20210731105602-946757ef5f98 diff --git a/.bingo/misspell.mod b/.bingo/misspell.mod new file mode 100644 index 0000000..60c1adc --- /dev/null +++ b/.bingo/misspell.mod @@ -0,0 +1,5 @@ +module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT + +go 1.14 + +require github.com/client9/misspell v0.3.4 // cmd/misspell diff --git a/.bingo/variables.env b/.bingo/variables.env new file mode 100644 index 0000000..2872d82 --- /dev/null +++ b/.bingo/variables.env @@ -0,0 +1,22 @@ +# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.4.2. DO NOT EDIT. +# All tools are designed to be build inside $GOBIN. +# Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk. +GOBIN=${GOBIN:=$(go env GOBIN)} + +if [ -z "$GOBIN" ]; then + GOBIN="$(go env GOPATH)/bin" +fi + + +BINGO="${GOBIN}/bingo-v0.2.4-0.20201227120526-df6eae8f6734" + +FAILLINT="${GOBIN}/faillint-v1.5.0" + +GOIMPORTS="${GOBIN}/goimports-v0.0.0-20200519204825-e64124511800" + +GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.26.0" + +MDOX="${GOBIN}/mdox-v0.2.2-0.20210731105602-946757ef5f98" + +MISSPELL="${GOBIN}/misspell-v0.3.4" + diff --git a/.errcheck_excludes.txt b/.errcheck_excludes.txt new file mode 100644 index 0000000..4e175ab --- /dev/null +++ b/.errcheck_excludes.txt @@ -0,0 +1,3 @@ +(github.com/go-kit/kit/log.Logger).Log +fmt.Fprintln +fmt.Fprint diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml new file mode 100644 index 0000000..937ddef --- /dev/null +++ b/.github/workflows/go.yaml @@ -0,0 +1,58 @@ +name: go + +on: + push: + branches: + - main + tags: + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + name: Linters (Static Analysis) for Go + steps: + - name: Checkout code into the Go module directory. + uses: actions/checkout@v2 + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.15.x + + - uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + + - name: Linting & vetting. + env: + GOBIN: /tmp/.bin + run: make lint + tests: + runs-on: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + go: [ '1.14.x', '1.15.x'] + platform: [ubuntu-latest, macos-latest] + + name: Unit tests on Go ${{ matrix.go }} ${{ matrix.platform }} + steps: + - name: Checkout code into the Go module directory. + uses: actions/checkout@v2 + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + + - name: Run unit tests. + env: + GOBIN: /tmp/.bin + run: make test \ No newline at end of file diff --git a/.gitignore b/.gitignore index 66fd13c..f26f320 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ # Dependency directories (remove the comment below to include it) # vendor/ +.bin +.envrc +.idea diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..0b803fd --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,2 @@ +Copyright (c) The EfficientGo Authors. +Licensed under the Apache License 2.0. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..45016af --- /dev/null +++ b/Makefile @@ -0,0 +1,110 @@ +include .bingo/Variables.mk + +MODULES ?= $(shell find $(PWD) -name "go.mod" | grep -v ".bingo" | xargs dirname) + +GO111MODULE ?= on +export GO111MODULE + +GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin + +# Tools. +GIT ?= $(shell which git) + +# Support gsed on OSX (installed via brew), falling back to sed. On Linux +# systems gsed won't be installed, so will use sed as expected. +SED ?= $(shell which gsed 2>/dev/null || which sed) + +define require_clean_work_tree + @git update-index -q --ignore-submodules --refresh + + @if ! git diff-files --quiet --ignore-submodules --; then \ + echo >&2 "$1: you have unstaged changes."; \ + git diff-files --name-status -r --ignore-submodules -- >&2; \ + echo >&2 "Please commit or stash them."; \ + exit 1; \ + fi + + @if ! git diff-index --cached --quiet HEAD --ignore-submodules --; then \ + echo >&2 "$1: your index contains uncommitted changes."; \ + git diff-index --cached --name-status -r --ignore-submodules HEAD -- >&2; \ + echo >&2 "Please commit or stash them."; \ + exit 1; \ + fi + +endef + +help: ## Displays help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) + +.PHONY: all +all: format build + +.PHONY: build +build: ## Build all modules + @echo ">> building all modules: $(MODULES)" + for dir in $(MODULES) ; do \ + echo ">> building in $${dir}"; \ + cd $${dir} && go test -run=nope ./...; \ + done + @echo ">> building copyright" + @cd copyright && go build -o $(GOBIN)/copyright . + +.PHONY: deps +deps: ## Cleans up deps for all modules + @echo ">> running deps tidy for all modules: $(MODULES)" + for dir in $(MODULES) ; do \ + cd $${dir} && go mod tidy; \ + done + +.PHONY: docs +docs: $(MDOX) ## Generates config snippets and doc formatting. + @echo ">> generating docs $(PATH)" + @$(MDOX) fmt -l *.md + +.PHONY: format +format: ## Formats Go code. +format: $(GOIMPORTS) + @echo ">> formatting all modules Go code: $(MODULES)" + @$(GOIMPORTS) -w $(MODULES) + +.PHONY: test +test: ## Runs all Go unit tests. + @echo ">> running tests for all modules: $(MODULES)" + for dir in $(MODULES) ; do \ + cd $${dir} && go test -v -race ./...; \ + done + +.PHONY: check-git +check-git: +ifneq ($(GIT),) + @test -x $(GIT) || (echo >&2 "No git executable binary found at $(GIT)."; exit 1) +else + @echo >&2 "No git binary found."; exit 1 +endif + +# PROTIP: +# Add +# --cpu-profile-path string Path to CPU profile output file +# --mem-profile-path string Path to memory profile output file +# to debug big allocations during linting. +lint: ## Runs various static analysis against our code. +lint: $(FAILLINT) $(GOLANGCI_LINT) $(MISSPELL) build format docs check-git deps + $(call require_clean_work_tree,"detected not clean master before running lint - run make lint and commit changes.") + @echo ">> verifying imported " + for dir in $(MODULES) ; do \ + cd $${dir} && $(FAILLINT) -paths "fmt.{Print,PrintfPrintln,Sprint}" -ignore-tests ./... && \ + $(FAILLINT) -paths "github.com/stretchr/testify=github.com/efficientgo/tools/core/pkg/testutil" ./...; \ + done + @echo ">> examining all of the Go files" + for dir in $(MODULES) ; do \ + cd $${dir} && go vet -stdmethods=false ./...; \ + done + @echo ">> linting all of the Go files GOGC=${GOGC}" + for dir in $(MODULES) ; do \ + cd $${dir} && $(GOLANGCI_LINT) run; \ + done + @echo ">> detecting misspells" + @find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error + @echo ">> ensuring Copyright headers" + @$(GOBIN)/copyright $(shell go list -f "{{.Dir}}" ./... | xargs -i find "{}" -name "*.go") + $(call require_clean_work_tree,"detected files without copyright - run make lint and commit changes.") diff --git a/README.md b/README.md index 2f67afe..3fb239e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,16 @@ # core -Set of core packages every Go project needs. Minimal API, strictly versioned and with no transient dependencies. + +[![core module docs](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/efficientgo/core) + +Go module with set of core packages *every* Go project needs. Minimal API, battle-tested, strictly versioned and with no transient dependencies. + +Maintained by experienced Go developers, including author of the https://www.oreilly.com/library/view/efficient-go/9781098105709/[Efficient Go] book. + +## Packages + +* `pkg/clilog`: +* `pkg/errcapture` +* `pkg/logerrcapture` +* `pkg/merrors` +* `pkg/runutil` +* `pkg/testutil` \ No newline at end of file diff --git a/backoff/backoff.go b/backoff/backoff.go new file mode 100644 index 0000000..deab524 --- /dev/null +++ b/backoff/backoff.go @@ -0,0 +1,112 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package backoff + +// Copied from https://github.com/cortexproject/cortex/blob/0ec7b9664a01d538f1f49580b4c359a5c3cc755a/pkg/util/backoff.go + +import ( + "context" + "fmt" + "math/rand" + "time" +) + +// Config configures a Backoff. +type Config struct { + Min time.Duration `yaml:"min_period"` // Start backoff at this level + Max time.Duration `yaml:"max_period"` // Increase exponentially to this level + MaxRetries int `yaml:"max_retries"` // Give up after this many; zero means infinite retries +} + +// Backoff implements exponential backoff with randomized wait times. +type Backoff struct { + cfg Config + ctx context.Context + numRetries int + nextDelayMin time.Duration + nextDelayMax time.Duration +} + +// New creates a Backoff object. Pass a Context that can also terminate the operation. +func New(ctx context.Context, cfg Config) *Backoff { + return &Backoff{ + cfg: cfg, + ctx: ctx, + nextDelayMin: cfg.Min, + nextDelayMax: doubleDuration(cfg.Min, cfg.Max), + } +} + +// Reset the Backoff back to its initial condition. +func (b *Backoff) Reset() { + b.numRetries = 0 + b.nextDelayMin = b.cfg.Min + b.nextDelayMax = doubleDuration(b.cfg.Min, b.cfg.Max) +} + +// Ongoing returns true if caller should keep going. +func (b *Backoff) Ongoing() bool { + // Stop if Context has errored or max retry count is exceeded. + return b.ctx.Err() == nil && (b.cfg.MaxRetries == 0 || b.numRetries < b.cfg.MaxRetries) +} + +// Err returns the reason for terminating the backoff, or nil if it didn't terminate. +func (b *Backoff) Err() error { + if b.ctx.Err() != nil { + return b.ctx.Err() + } + if b.cfg.MaxRetries != 0 && b.numRetries >= b.cfg.MaxRetries { + return fmt.Errorf("terminated after %d retries", b.numRetries) + } + return nil +} + +// NumRetries returns the number of retries so far. +func (b *Backoff) NumRetries() int { + return b.numRetries +} + +// Wait sleeps for the backoff time then increases the retry count and backoff time. +// Returns immediately if Context is terminated. +func (b *Backoff) Wait() { + // Increase the number of retries and get the next delay. + sleepTime := b.NextDelay() + + if b.Ongoing() { + select { + case <-b.ctx.Done(): + case <-time.After(sleepTime): + } + } +} + +func (b *Backoff) NextDelay() time.Duration { + b.numRetries++ + + // Handle the edge case the min and max have the same value + // (or due to some misconfig max is < min). + if b.nextDelayMin >= b.nextDelayMax { + return b.nextDelayMin + } + + // Add a jitter within the next exponential backoff range. + sleepTime := b.nextDelayMin + time.Duration(rand.Int63n(int64(b.nextDelayMax-b.nextDelayMin))) + + // Apply the exponential backoff to calculate the next jitter + // range, unless we've already reached the max. + if b.nextDelayMax < b.cfg.Max { + b.nextDelayMin = doubleDuration(b.nextDelayMin, b.cfg.Max) + b.nextDelayMax = doubleDuration(b.nextDelayMax, b.cfg.Max) + } + + return sleepTime +} + +func doubleDuration(value time.Duration, max time.Duration) time.Duration { + value = value * 2 + if value <= max { + return value + } + return max +} diff --git a/backoff/backoff_test.go b/backoff/backoff_test.go new file mode 100644 index 0000000..aaf11bc --- /dev/null +++ b/backoff/backoff_test.go @@ -0,0 +1,108 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package backoff + +// Copied from https://github.com/cortexproject/cortex/blob/0ec7b9664a01d538f1f49580b4c359a5c3cc755a/pkg/util/backoff.go + +import ( + "context" + "testing" + "time" +) + +func TestBackoff_NextDelay(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + minBackoff time.Duration + maxBackoff time.Duration + expectedRanges [][]time.Duration + }{ + "exponential backoff with jitter honoring min and max": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 10 * time.Second, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + {800 * time.Millisecond, 1600 * time.Millisecond}, + {1600 * time.Millisecond, 3200 * time.Millisecond}, + {3200 * time.Millisecond, 6400 * time.Millisecond}, + {6400 * time.Millisecond, 10000 * time.Millisecond}, + {6400 * time.Millisecond, 10000 * time.Millisecond}, + }, + }, + "exponential backoff with max equal to the end of a range": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 800 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + }, + }, + "exponential backoff with max equal to the end of a range + 1": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 801 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + {800 * time.Millisecond, 801 * time.Millisecond}, + {800 * time.Millisecond, 801 * time.Millisecond}, + }, + }, + "exponential backoff with max equal to the end of a range - 1": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 799 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 799 * time.Millisecond}, + {400 * time.Millisecond, 799 * time.Millisecond}, + }, + }, + "min backoff is equal to max": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 100 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 100 * time.Millisecond}, + {100 * time.Millisecond, 100 * time.Millisecond}, + {100 * time.Millisecond, 100 * time.Millisecond}, + }, + }, + "min backoff is greater then max": { + minBackoff: 200 * time.Millisecond, + maxBackoff: 100 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {200 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 200 * time.Millisecond}, + }, + }, + } + + for testName, testData := range tests { + testData := testData + + t.Run(testName, func(t *testing.T) { + t.Parallel() + + b := New(context.Background(), Config{ + Min: testData.minBackoff, + Max: testData.maxBackoff, + MaxRetries: len(testData.expectedRanges), + }) + + for _, expectedRange := range testData.expectedRanges { + delay := b.NextDelay() + + if delay < expectedRange[0] || delay > expectedRange[1] { + t.Errorf("%d expected to be within %d and %d", delay, expectedRange[0], expectedRange[1]) + } + } + }) + } +} diff --git a/clilog/clilog.go b/clilog/clilog.go new file mode 100644 index 0000000..27267ef --- /dev/null +++ b/clilog/clilog.go @@ -0,0 +1,313 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package clilog + +import ( + "bytes" + "encoding" + "errors" + "fmt" + "io" + "reflect" + "sync" + + "github.com/efficientgo/core/merrors" +) + +// Logger interface compatible with go-kit/logger. +type Logger interface { + Log(keyvals ...interface{}) error +} + +type buf struct { + bytes.Buffer + + *Encoder +} + +func (l *buf) Reset() { + l.Encoder.Reset() + l.Buffer.Reset() +} + +var bufPool = sync.Pool{ + New: func() interface{} { + var b buf + b.Encoder = NewEncoder(&b.Buffer) + return &b + }, +} + +type logger struct { + w io.Writer +} + +// New returns a logger that encodes keyvals to the Writer in +// CLI friendly format. Each log event produces no more than one call to w.Write. +// The passed Writer must be safe for concurrent use by multiple goroutines if +// the returned Logger will be used concurrently. +func New(w io.Writer) Logger { + return &logger{w} +} + +func (l logger) Log(keyvals ...interface{}) error { + buf := bufPool.Get().(*buf) + buf.Reset() + defer bufPool.Put(buf) + + if err := buf.EncodeKeyvals(keyvals...); err != nil { + return err + } + + // Add newline to the end of the buffer + if err := buf.EndRecord(); err != nil { + return err + } + + // The Logger interface requires implementations to be safe for concurrent + // use by multiple goroutines. For this implementation that means making + // only one call to l.w.Write() for each call to Log. + if _, err := l.w.Write(buf.Bytes()); err != nil { + return err + } + return nil +} + +// MarshalKeyvals returns the clilog encoding of keyvals, a variadic sequence +// of alternating keys and values. +func MarshalKeyvals(keyvals ...interface{}) ([]byte, error) { + buf := &bytes.Buffer{} + if err := NewEncoder(buf).EncodeKeyvals(keyvals...); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// An Encoder writes clilog data to an output stream. +type Encoder struct { + w io.Writer + scratch bytes.Buffer + needSep bool + + errs []merrors.Error +} + +// NewEncoder returns a new clilog Encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +var ( + sep = []byte(": ") + newline = []byte("\n") + null = []byte("null") +) + +// EncodeKeyval writes the clilog encoding of key and value to the stream. A +// single space is written before the second and subsequent keys in a record. +// Nothing is written if a non-nil error is returned. +func (enc *Encoder) EncodeKeyval(_, value interface{}) error { + if e, ok := value.(error); ok { + if errs, ok := merrors.AsMulti(e); ok { + enc.errs = append(enc.errs, errs) + return nil + } + } + + enc.scratch.Reset() + + // Naive right now, print values only (: + if enc.needSep { + if _, err := enc.scratch.Write(sep); err != nil { + return err + } + } + if err := writeValue(&enc.scratch, value); err != nil { + return err + } + _, err := enc.w.Write(enc.scratch.Bytes()) + enc.needSep = true + return err +} + +// EncodeKeyvals writes the logfmt encoding of keyvals to the stream. Keyvals +// is a variadic sequence of alternating keys and values. Keys of unsupported +// type are skipped along with their corresponding value. Values of +// unsupported type or that cause a MarshalerError are replaced by their error +// but do not cause EncodeKeyvals to return an error. If a non-nil error is +// returned some key/value pairs may not have be written. +func (enc *Encoder) EncodeKeyvals(keyvals ...interface{}) error { + if len(keyvals) == 0 { + return nil + } + if len(keyvals)%2 == 1 { + keyvals = append(keyvals, nil) + } + for i := 0; i < len(keyvals); i += 2 { + k, v := keyvals[i], keyvals[i+1] + err := enc.EncodeKeyval(k, v) + if err == ErrUnsupportedKeyType { + continue + } + if _, ok := err.(*MarshalerError); ok || err == ErrUnsupportedValueType { + v = err + err = enc.EncodeKeyval(k, v) + } + if err != nil { + return err + } + } + return nil +} + +// EndRecord ends the log record. +func (enc *Encoder) EndRecord() error { + if len(enc.errs) > 0 { + enc.scratch.Reset() + if enc.needSep { + if _, err := enc.scratch.Write(sep); err != nil { + return err + } + } + + merr := merrors.Merge(enc.errs) + if err := merrors.PrettyPrint(&enc.scratch, merr); err != nil { + return err + } + + if _, err := enc.w.Write(enc.scratch.Bytes()); err != nil { + return err + } + } + + _, err := enc.w.Write(newline) + if err == nil { + enc.needSep = false + } + return err +} + +// Reset resets the Encoder to the beginning of a new record. +func (enc *Encoder) Reset() { + enc.needSep = false +} + +// MarshalerError represents an error encountered while marshaling a value. +type MarshalerError struct { + Type reflect.Type + Err error +} + +func (e *MarshalerError) Error() string { + return "error marshaling value of type " + e.Type.String() + ": " + e.Err.Error() +} + +// ErrUnsupportedKeyType is returned by Encoder methods if a key has an +// unsupported type. +var ErrUnsupportedKeyType = errors.New("unsupported key type") + +// ErrUnsupportedValueType is returned by Encoder methods if a value has an +// unsupported type. +var ErrUnsupportedValueType = errors.New("unsupported value type") + +func writeValue(w io.Writer, value interface{}) error { + switch v := value.(type) { + case nil: + return writeBytesValue(w, null) + case string: + return writeStringValue(w, v, true) + case []byte: + return writeBytesValue(w, v) + case encoding.TextMarshaler: + vb, err := safeMarshal(v) + if err != nil { + return err + } + if vb == nil { + vb = null + } + return writeBytesValue(w, vb) + case error: + se, ok := safeError(v) + return writeStringValue(w, se, ok) + case fmt.Stringer: + ss, ok := safeString(v) + return writeStringValue(w, ss, ok) + default: + rvalue := reflect.ValueOf(value) + switch rvalue.Kind() { + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice, reflect.Struct: + return ErrUnsupportedValueType + case reflect.Ptr: + if rvalue.IsNil() { + return writeBytesValue(w, null) + } + return writeValue(w, rvalue.Elem().Interface()) + } + return writeStringValue(w, fmt.Sprintf("%v", v), true) //nolint + } +} + +func writeStringValue(w io.Writer, value string, ok bool) error { + var err error + if ok && value == "null" { + _, err = io.WriteString(w, `"null"`) + } else { + _, err = io.WriteString(w, value) + } + return err +} + +func writeBytesValue(w io.Writer, value []byte) error { + _, err := w.Write(value) + return err +} + +func safeError(err error) (s string, ok bool) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() { + s, ok = "null", false + } else { + s, ok = fmt.Sprintf("PANIC:%v", panicVal), false + } + } + }() + s, ok = err.Error(), true + return +} + +func safeString(str fmt.Stringer) (s string, ok bool) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() { + s, ok = "null", false + } else { + s, ok = fmt.Sprintf("PANIC:%v", panicVal), true + } + } + }() + s, ok = str.String(), true + return +} + +func safeMarshal(tm encoding.TextMarshaler) (b []byte, err error) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(tm); v.Kind() == reflect.Ptr && v.IsNil() { + b, err = nil, nil + } else { + b, err = nil, fmt.Errorf("panic when marshaling: %s", panicVal) + } + } + }() + b, err = tm.MarshalText() + if err != nil { + return nil, &MarshalerError{ + Type: reflect.TypeOf(tm), + Err: err, + } + } + return +} diff --git a/clilog/doc.go b/clilog/doc.go new file mode 100644 index 0000000..ea11b50 --- /dev/null +++ b/clilog/doc.go @@ -0,0 +1,17 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package clilog + +// Logging formatter that transforms structure log entry into human readable, clean friendly entry +// suitable more for CLI tools. +// +// In details this means: +// +// * No special sign escaping. +// * No key printing. +// * Values separated with ': ' +// * Support for pretty printing multi errors (including nested ones) in format of (: ; ; ...; ) +// * TODO(bwplotka): Support for multiple multilines. +// +// Compatible with `github.com/go-kit/kit/log.Logger` diff --git a/errcapture/do.go b/errcapture/do.go new file mode 100644 index 0000000..c7176a4 --- /dev/null +++ b/errcapture/do.go @@ -0,0 +1,47 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Initially copied from Thanos +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package errcapture + +import ( + "io" + "io/ioutil" + "os" + + "github.com/efficientgo/tools/core/pkg/merrors" + "github.com/pkg/errors" +) + +type doFunc func() error + +// Do runs function and on error return error by argument including the given error (usually +// from caller function). +func Do(err *error, doer doFunc, format string, a ...interface{}) { + derr := doer() + if err == nil { + return + } + + // For os closers, it's a common case to double close. From reliability purpose this is not a problem it may only indicate + // surprising execution path. + if errors.Is(derr, os.ErrClosed) { + return + } + + *err = merrors.New(*err, errors.Wrapf(derr, format, a...)).Err() +} + +// ExhaustClose closes the io.ReadCloser with error capture but exhausts the reader before. +func ExhaustClose(err *error, r io.ReadCloser, format string, a ...interface{}) { + _, copyErr := io.Copy(ioutil.Discard, r) + + Do(err, r.Close, format, a...) + + // Prepend the io.Copy error. + *err = merrors.New(copyErr, *err).Err() +} diff --git a/errcapture/do_test.go b/errcapture/do_test.go new file mode 100644 index 0000000..916e0ff --- /dev/null +++ b/errcapture/do_test.go @@ -0,0 +1,79 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Initially copied from Thanos +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package errcapture + +import ( + "io" + "testing" + + "github.com/pkg/errors" +) + +type testCloser struct { + err error +} + +func (c testCloser) Close() error { + return c.err +} + +func TestDo(t *testing.T) { + for _, tcase := range []struct { + err error + closer io.Closer + + expectedErrStr string + }{ + { + err: nil, + closer: testCloser{err: nil}, + expectedErrStr: "", + }, + { + err: errors.New("test"), + closer: testCloser{err: nil}, + expectedErrStr: "test", + }, + { + err: nil, + closer: testCloser{err: errors.New("test")}, + expectedErrStr: "close: test", + }, + { + err: errors.New("test"), + closer: testCloser{err: errors.New("test")}, + expectedErrStr: "2 errors: test; close: test", + }, + } { + if ok := t.Run("", func(t *testing.T) { + ret := tcase.err + Do(&ret, tcase.closer.Close, "close") + + if tcase.expectedErrStr == "" { + if ret != nil { + t.Error("Expected error to be nil") + t.Fail() + } + } else { + if ret == nil { + t.Error("Expected error to be not nil") + t.Fail() + } + + if tcase.expectedErrStr != ret.Error() { + t.Errorf("%s != %s", tcase.expectedErrStr, ret.Error()) + t.Fail() + } + } + + }); !ok { + return + } + } +} diff --git a/errcapture/doc.go b/errcapture/doc.go new file mode 100644 index 0000000..96c1c22 --- /dev/null +++ b/errcapture/doc.go @@ -0,0 +1,29 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package errcapture + +// Close a `io.Closer` interface or execute any function that returns error safely while capturing error. +// It's often forgotten but it's a caller responsibility to close all implementations of `Closer`, +// such as *os.File or io.ReaderCloser. Commonly we would use: +// +// defer closer.Close() +// +// This is wrong. Close() usually return important error (e.g for os.File the actual file flush might happen and fail on `Close` method). +// It's very important to *always* check error. `errcapture` provides utility functions to capture error and add to provided one, +// still allowing to put them in a convenient `defer` statement: +// +// func <...>(...) (err error) { +// ... +// defer errcapture.Do(&err, closer.Close, "log format message") +// +// ... +// } +// +// If Close returns error, `errcapture.Do` will capture it, add to input error if not nil and return by argument. +// +// The errcapture.ExhaustClose function provide the same functionality but takes an io.ReadCloser and exhausts the whole +// reader before closing. This is useful when trying to use http keep-alive connections because for the same connection +// to be re-used the whole response body needs to be exhausted. +// +// Check https://pkg.go.dev/github.com/efficientgo/tools/pkg/logerrcapture if you want to just log an error instead. diff --git a/errors/doc.go b/errors/doc.go new file mode 100644 index 0000000..da427f9 --- /dev/null +++ b/errors/doc.go @@ -0,0 +1,19 @@ +// Initially copied from Thanos and contributed by https://github.com/bisakhmondal. +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package errors + +// Package errors provides basic utilities to manipulate errors with a useful stacktrace. It combines the +// benefits of errors.New and fmt.Errorf world into a single package. +// +// The idea of writing errors package in thanos is highly motivated from the Tast project of Chromium OS Authors. However, instead of +// copying the package, we end up writing our own simplified logic borrowing some ideas from the errors and github.com/pkg/errors. +// A big thanks to all of them. +// +// Example 1: +// +// +// Example 2: +// diff --git a/errors/errors.go b/errors/errors.go new file mode 100644 index 0000000..054e30b --- /dev/null +++ b/errors/errors.go @@ -0,0 +1,142 @@ +// Initially copied from Thanos and contributed by https://github.com/bisakhmondal. +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +// Package errors provides basic utilities to manipulate errors with a useful stacktrace. It combines the +// benefits of errors.New and fmt.Errorf world into a single package. +// +// The idea of writing errors package in thanos is highly motivated from the Tast project of Chromium OS Authors. However, instead of +// copying the package, we end up writing our own simplified logic borrowing some ideas from the errors and github.com/pkg/errors. +// A big thanks to all of them. +package errors + +import ( + "errors" + "fmt" + "strings" +) + +// base is the fundamental struct that implements the error interface and the acts as the backbone of this errors package. +type base struct { + // info contains the error message passed through calls like errors.Wrap, errors.New. + info string + // stacktrace stores information about the program counters - i.e. where this error was generated. + stack stacktrace + // err is the actual error which is being wrapped with a stacktrace and message information. + err error +} + +// Error implements the error interface. +func (b *base) Error() string { + if b.err != nil { + return fmt.Sprintf("%s: %s", b.info, b.err.Error()) + } + return b.info +} + +// Unwrap implements the error Unwrap interface. +func (b *base) Unwrap() error { + return b.err +} + +// Format implements the fmt.Formatter interface to support the formatting of an error chain with "%+v" verb. +// Whenever error is printed with %+v format verb, stacktrace info gets dumped to the output. +func (b *base) Format(s fmt.State, verb rune) { + if verb == 'v' && s.Flag('+') { + s.Write([]byte(formatErrorChain(b))) + return + } + + s.Write([]byte(b.Error())) +} + +// Newf formats according to a format specifier and returns a new error with a stacktrace +// with recent call frames. Each call to New returns a distinct error value even if the text is +// identical. An alternative of the errors.New function. +// +// If no args have been passed, it is same as `New` function without formatting. Character like +// '%' still has to be escaped in that scenario. +func Newf(format string, args ...interface{}) error { + return &base{ + info: fmt.Sprintf(format, args...), + stack: newStackTrace(), + err: nil, + } +} + +// Wrapf returns a new error by formatting the error message with the supplied format specifier +// and wrapping another error with a stacktrace containing recent call frames. +// +// If cause is nil, this is the same as fmt.Errorf. If no args have been passed, it is same as `Wrap` +// function without formatting. Character like '%' still has to be escaped in that scenario. +func Wrapf(cause error, format string, args ...interface{}) error { + return &base{ + info: fmt.Sprintf(format, args...), + stack: newStackTrace(), + err: cause, + } +} + +// Cause returns the result of repeatedly calling the Unwrap method on err, if err's +// type implements an Unwrap method. Otherwise, Cause returns the last encountered error. +// The difference between Unwrap and Cause is the first one performs unwrapping of one level +// but Cause returns the last err (whether it's nil or not) where it failed to assert +// the interface containing the Unwrap method. +// This is a replacement of errors.Cause without the causer interface from pkg/errors which +// actually can be sufficed through the errors.Is function. But considering some use cases +// where we need to peel off all the external layers applied through errors.Wrap family, +// it is useful ( where external SDK doesn't use errors.Is internally). +func Cause(err error) error { + for err != nil { + e, ok := err.(interface { + Unwrap() error + }) + if !ok { + return err + } + err = e.Unwrap() + } + return nil +} + +// formatErrorChain formats an error chain. +func formatErrorChain(err error) string { + var buf strings.Builder + for err != nil { + if e, ok := err.(*base); ok { + buf.WriteString(fmt.Sprintf("%s\n%v", e.info, e.stack)) + err = e.err + } else { + buf.WriteString(fmt.Sprintf("%s\n", err.Error())) + err = nil + } + } + return buf.String() +} + +// The functions `Is`, `As` & `Unwrap` provides a thin wrapper around the builtin errors +// package in go. Just for sake of completeness and correct autocompletion behaviors from +// IDEs they have been wrapped using functions instead of using variable to reference them +// as first class functions (eg: var Is = errros.Is ). + +// Is is a wrapper of built-in errors.Is. It reports whether any error in err's +// chain matches target. The chain consists of err itself followed by the sequence +// of errors obtained by repeatedly calling Unwrap. +func Is(err, target error) bool { + return errors.Is(err, target) +} + +// As is a wrapper of built-in errors.As. It finds the first error in err's +// chain that matches target, and if one is found, sets target to that error +// value and returns true. Otherwise, it returns false. +func As(err error, target interface{}) bool { + return errors.As(err, target) +} + +// Unwrap is a wrapper of built-in errors.Unwrap. Unwrap returns the result of +// calling the Unwrap method on err, if err's type contains an Unwrap method +// returning error. Otherwise, Unwrap returns nil. +func Unwrap(err error) error { + return errors.Unwrap(err) +} diff --git a/errors/errors_test.go b/errors/errors_test.go new file mode 100644 index 0000000..c26a3fb --- /dev/null +++ b/errors/errors_test.go @@ -0,0 +1,150 @@ +// Initially copied from Thanos and contributed by https://github.com/bisakhmondal. +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package errors_test + +import ( + //lint:ignore faillint Custom errors package tests need to import standard library errors. + stderrors "errors" + "fmt" + "regexp" + "strconv" + "testing" + + "github.com/efficientgo/core/errors" + "github.com/efficientgo/core/testutil" +) + +const msg = "test_error_message" +const wrapper = "test_wrapper" + +func TestNewf(t *testing.T) { + err := errors.Newf(msg) + testutil.Equals(t, err.Error(), msg, "the root error message must match") + + // The %+v triggers stacktrace print. + reg := regexp.MustCompile(msg + `[ \n]+> github\.com\/efficientgo\/core\/errors_test\.TestNewf .*\/errors\/errors_test\.go:\d+`) + testutil.Equals(t, reg.MatchString(fmt.Sprintf("%+v", err)), true, "matching stacktrace in errors.Newf") +} + +func TestNewfFormatted(t *testing.T) { + fmtMsg := msg + " key=%v" + expectedMsg := msg + " key=value" + + err := errors.Newf(fmtMsg, "value") + testutil.Equals(t, err.Error(), expectedMsg, "the root error message must match") + + fmt.Printf("%+v", err) + + reg := regexp.MustCompile(msg + `[ \n]+> github\.com\/efficientgo\/core\/errors_test\.TestNewfFormatted .*\/errors\/errors_test\.go:\d+`) + testutil.Equals(t, reg.MatchString(fmt.Sprintf("%+v", err)), true, "matching stacktrace in errors.TestNewfFormatted") +} + +func TestWrapf(t *testing.T) { + err := errors.Newf(msg) + err = errors.Wrapf(err, wrapper) + + expectedMsg := wrapper + ": " + msg + testutil.Equals(t, err.Error(), expectedMsg, "the root error message must match") + + reg := regexp.MustCompile(`test_wrapper[ \n]+> github\.com\/thanos-io\/thanos\/pkg\/errors\.TestWrapf .*\/pkg\/errors\/errors_test\.go:\d+ +[[:ascii:]]+test_error_message[ \n]+> github\.com\/thanos-io\/thanos\/pkg\/errors\.TestWrapf .*\/pkg\/errors\/errors_test\.go:\d+`) + + testutil.Equals(t, reg.MatchString(fmt.Sprintf("%+v", err)), true, "matching stacktrace in errors.Wrap") +} + +func TestUnwrap(t *testing.T) { + // test with base error + err := errors.Newf(msg) + + for i, tc := range []struct { + err error + expected string + isNil bool + }{ + { + // no wrapping + err: err, + isNil: true, + }, + { + err: errors.Wrapf(err, wrapper), + expected: "test_error_message", + }, + { + err: errors.Wrapf(errors.Wrapf(err, wrapper), wrapper), + expected: "test_wrapper: test_error_message", + }, + // check primitives errors + { + err: stderrors.New("std-error"), + isNil: true, + }, + { + err: errors.Wrapf(stderrors.New("std-error"), wrapper), + expected: "std-error", + }, + { + err: nil, + isNil: true, + }, + } { + t.Run("TestCase"+strconv.Itoa(i), func(t *testing.T) { + unwrapped := errors.Unwrap(tc.err) + if tc.isNil { + testutil.Equals(t, unwrapped, nil) + return + } + testutil.Equals(t, unwrapped.Error(), tc.expected, "Unwrap must match expected output") + }) + } +} + +func TestCause(t *testing.T) { + // test with base error that implements interface containing Unwrap method + err := errors.Newf(msg) + + for i, tc := range []struct { + err error + expected string + isNil bool + }{ + { + // no wrapping + err: err, + isNil: true, + }, + { + err: errors.Wrapf(err, wrapper), + isNil: true, + }, + { + err: errors.Wrapf(errors.Wrapf(err, wrapper), wrapper), + isNil: true, + }, + // check primitives errors + { + err: stderrors.New("std-error"), + expected: "std-error", + }, + { + err: errors.Wrapf(stderrors.New("std-error"), wrapper), + expected: "std-error", + }, + { + err: nil, + isNil: true, + }, + } { + t.Run("TestCase"+strconv.Itoa(i), func(t *testing.T) { + cause := errors.Cause(tc.err) + if tc.isNil { + testutil.Equals(t, cause, nil) + return + } + testutil.Equals(t, cause.Error(), tc.expected, "Cause must match expected output") + }) + } +} diff --git a/errors/stacktrace.go b/errors/stacktrace.go new file mode 100644 index 0000000..7d98219 --- /dev/null +++ b/errors/stacktrace.go @@ -0,0 +1,55 @@ +// Initially copied from Thanos and contributed by https://github.com/bisakhmondal. +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package errors + +import ( + "fmt" + "runtime" + "strings" +) + +// stacktrace holds a snapshot of program counters. +type stacktrace []uintptr + +// newStackTrace captures a stack trace. It skips first 3 frames to record the +// snapshot of the stack trace at the origin of a particular error. It tries to +// record maximum 16 frames (if available). +func newStackTrace() stacktrace { + const stackDepth = 16 // record maximum 16 frames (if available). + + pc := make([]uintptr, stackDepth) + // using skip=3 for not to count the program counter address of + // 1. the respective function from errors package (eg. errors.New) + // 2. newStacktrace itself + // 3. the function used in runtime.Callers + n := runtime.Callers(3, pc) + + // this approach is taken to reduce long term memory footprint (obtained through escape analysis). + // We are returning a new slice by re-slicing the pc with the required length and capacity (when the + // no of returned callFrames is less that stackDepth). This uses less memory compared to pc[:n] as + // the capacity of new slice is inherited from the parent slice if not specified. + return pc[:n:n] +} + +// String implements the fmt.Stringer interface to provide formatted text output. +func (s stacktrace) String() string { + var buf strings.Builder + + // CallersFrames takes the slice of Program Counter addresses returned by Callers to + // retrieve function/file/line information. + cf := runtime.CallersFrames(s) + for { + // more indicates if the next call will be successful or not. + frame, more := cf.Next() + // used formatting scheme <`>`space><:> for example: + // > testing.tRunner /home/go/go1.17.8/src/testing/testing.go:1259 + buf.WriteString(fmt.Sprintf("> %s\t%s:%d\n", frame.Func.Name(), frame.File, frame.Line)) + if !more { + break + } + } + return buf.String() +} diff --git a/errors/stacktrace_test.go b/errors/stacktrace_test.go new file mode 100644 index 0000000..6c4ae48 --- /dev/null +++ b/errors/stacktrace_test.go @@ -0,0 +1,32 @@ +// Initially copied from Thanos and contributed by https://github.com/bisakhmondal. +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package errors + +import ( + "strings" + "testing" +) + +func caller() stacktrace { + return newStackTrace() +} + +func TestStacktraceOutput(t *testing.T) { + st := caller() + expectedPhrase := "/pkg/errors/stacktrace_test.go:16" + if !strings.Contains(st.String(), expectedPhrase) { + t.Fatalf("expected %v phrase into the stacktrace, received stacktrace: \n%v", expectedPhrase, st.String()) + } +} + +func TestStacktraceProgramCounterLen(t *testing.T) { + st := caller() + output := st.String() + lines := len(strings.Split(strings.TrimSuffix(output, "\n"), "\n")) + if len(st) != lines { + t.Fatalf("output lines vs program counter size mismatch: program counter size %v, output lines %v", len(st), lines) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6ab03bd --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/efficientgo/core + +go 1.17 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/efficientgo/tools/core v0.0.0-20220817170617-6c25e3b627dd + github.com/pkg/errors v0.9.1 + github.com/pmezard/go-difflib v1.0.0 + go.uber.org/goleak v1.1.10 +) + +require ( + golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect + golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..2c501c3 --- /dev/null +++ b/go.sum @@ -0,0 +1,33 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/efficientgo/tools/core v0.0.0-20220817170617-6c25e3b627dd h1:svR6KxSP1xiPw10RN4Pd7g6BAVkEcNN628PAqZH31mM= +github.com/efficientgo/tools/core v0.0.0-20220817170617-6c25e3b627dd/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/logerrcapture/do.go b/logerrcapture/do.go new file mode 100644 index 0000000..41662b9 --- /dev/null +++ b/logerrcapture/do.go @@ -0,0 +1,51 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Initially copied from Thanos +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package logerrcapture + +import ( + "fmt" + "io" + "io/ioutil" + "os" + + "github.com/pkg/errors" +) + +// Logger interface compatible with go-kit/logger. +type Logger interface { + Log(keyvals ...interface{}) error +} + +type doFunc func() error + +// Do is making sure we log every error, even those from best effort tiny functions. +func Do(logger Logger, doer doFunc, format string, a ...interface{}) { + derr := doer() + if derr == nil { + return + } + + // For os closers, it's a common case to double close. From reliability purpose this is not a problem it may only indicate + // surprising execution path. + if errors.Is(derr, os.ErrClosed) { + return + } + + _ = logger.Log("msg", "detected do error", "err", errors.Wrap(derr, fmt.Sprintf(format, a...))) +} + +// ExhaustClose closes the io.ReadCloser with a log message on error but exhausts the reader before. +func ExhaustClose(logger Logger, r io.ReadCloser, format string, a ...interface{}) { + _, err := io.Copy(ioutil.Discard, r) + if err != nil { + _ = logger.Log("msg", "failed to exhaust reader, performance may be impeded", "err", err) + } + + Do(logger, r.Close, format, a...) +} diff --git a/logerrcapture/do_test.go b/logerrcapture/do_test.go new file mode 100644 index 0000000..feec9c3 --- /dev/null +++ b/logerrcapture/do_test.go @@ -0,0 +1,66 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Initially copied from Thanos +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package logerrcapture + +import ( + "io" + "os" + "strings" + "testing" + + "github.com/efficientgo/tools/core/pkg/testutil" + "github.com/pkg/errors" +) + +type loggerCapturer struct { + // WasCalled is true if the Log() function has been called. + WasCalled bool +} + +func (lc *loggerCapturer) Log(keyvals ...interface{}) error { + lc.WasCalled = true + return nil +} + +type emulatedCloser struct { + io.Reader + + calls int +} + +func (e *emulatedCloser) Close() error { + e.calls++ + if e.calls == 1 { + return nil + } + if e.calls == 2 { + return errors.Wrap(os.ErrClosed, "can even be a wrapped one") + } + return errors.New("something very bad happened") +} + +// newEmulatedCloser returns a ReadCloser with a Close method +// that at first returns success but then returns that +// it has been closed already. After that, it returns that +// something very bad had happened. +func newEmulatedCloser(r io.Reader) io.ReadCloser { + return &emulatedCloser{Reader: r} +} + +func TestCloseMoreThanOnce(t *testing.T) { + lc := &loggerCapturer{} + r := newEmulatedCloser(strings.NewReader("somestring")) + + Do(lc, r.Close, "should not be called") + Do(lc, r.Close, "should not be called") + testutil.Equals(t, false, lc.WasCalled) + + Do(lc, r.Close, "should be called") + testutil.Equals(t, true, lc.WasCalled) +} diff --git a/logerrcapture/doc.go b/logerrcapture/doc.go new file mode 100644 index 0000000..414efc9 --- /dev/null +++ b/logerrcapture/doc.go @@ -0,0 +1,30 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package logerrcapture + +// Close a `io.Closer` interface or execute any function that returns error safely while logging error. +// It's often forgotten but it's a caller responsibility to close all implementations of `Closer`, +// such as *os.File or io.ReaderCloser. Commonly we would use: +// +// defer closer.Close() +// +// This is wrong. Close() usually return important error (e.g for os.File the actual file flush might happen and fail on `Close` method). +// It's very important to *always* check error. `logerrcapture` provides utility functions to capture error and log it via provided +// logger, while still allowing to put them in a convenient `defer` statement: +// +// func <...>(...) (err error) { +// ... +// defer logerrcapture.Do(logger, closer.Close, "log format message") +// +// ... +// } +// +// If Close returns error, `logerrcapture.Do` will capture it, add to input error if not nil and return by argument. +// +// The logerrcapture.ExhaustClose function provide the same functionality but takes an io.ReadCloser and exhausts the whole +// reader before closing. This is useful when trying to use http keep-alive connections because for the same connection +// to be re-used the whole response body needs to be exhausted. +// +// Recommended: Check https://pkg.go.dev/github.com/efficientgo/tools/pkg/errcapture if you want to return error instead of just logging (causing +// hard error). diff --git a/merrors/doc.go b/merrors/doc.go new file mode 100644 index 0000000..d1eaf83 --- /dev/null +++ b/merrors/doc.go @@ -0,0 +1,20 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package merrors + +// Safe multi error implementation that chains errors on the same level. Supports errors.As and errors.Is functions. +// +// Example 1: +// +// return merrors.New(err1, err2).Err() +// +// Example 2: +// +// merr := merrors.New(err1) +// merr.Add(err2, errOrNil3) +// for _, err := range errs { +// merr.Add(err) +// } +// return merr.Err() +// diff --git a/merrors/merrors.go b/merrors/merrors.go new file mode 100644 index 0000000..bb0c9dc --- /dev/null +++ b/merrors/merrors.go @@ -0,0 +1,215 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package merrors + +import ( + "bytes" + stderrors "errors" + "fmt" + "io" +) + +// NilOrMultiError type allows combining multiple errors into one. +type NilOrMultiError struct { + errs []error +} + +// New returns NilOrMultiError with provided errors added if not nil. +func New(errs ...error) *NilOrMultiError { + m := &NilOrMultiError{} + m.Add(errs...) + return m +} + +// Add adds single or many errors to the error list. Each error is added only if not nil. +// If the error is a multiError type, the errors inside multiError are added to the main NilOrMultiError. +func (e *NilOrMultiError) Add(errs ...error) { + for _, err := range errs { + if err == nil { + continue + } + if merr, ok := err.(multiError); ok { + e.errs = append(e.errs, merr.errs...) + continue + } + e.errs = append(e.errs, err) + } +} + +// Err returns the error list as an Error (also implements error) or nil if it is empty. +func (e NilOrMultiError) Err() Error { + if len(e.errs) == 0 { + return nil + } + return multiError(e) +} + +// Error is extended error interface that allows to use returned read-only multi error in more advanced ways. +type Error interface { + error + + // Errors returns underlying errors. + Errors() []error + + // As finds the first error in multiError slice of error chains that matches target, and if so, sets + // target to that error value and returns true. Otherwise, it returns false. + // + // An error matches target if the error's concrete value is assignable to the value + // pointed to by target, or if the error has a method As(interface{}) bool such that + // As(target) returns true. In the latter case, the As method is responsible for + // setting target. + As(target interface{}) bool + // Is returns true if any error in multiError's slice of error chains matches the given target or + // if the target is of multiError type. + // + // An error is considered to match a target if it is equal to that target or if + // it implements a method Is(error) bool such that Is(target) returns true. + Is(target error) bool + // Count returns the number of multi error' errors that match the given target. + // Matching is defined as in Is method. + Count(target error) int +} + +// multiError implements the error and Error interfaces, and it represents NilOrMultiError (in other words []error) with at least one error inside it. +// NOTE: This type is useful to make sure that NilOrMultiError is not accidentally used for err != nil check. +type multiError struct { + errs []error +} + +// Errors returns underlying errors. +func (e multiError) Errors() []error { + return e.errs +} + +// Error returns a concatenated string of the contained errors. +func (e multiError) Error() string { + var buf bytes.Buffer + + if len(e.errs) > 1 { + fmt.Fprintf(&buf, "%d errors: ", len(e.errs)) + } + for i, err := range e.errs { + if i != 0 { + buf.WriteString("; ") + } + buf.WriteString(err.Error()) + } + + return buf.String() +} + +// As finds the first error in multiError slice of error chains that matches target, and if so, sets +// target to that error value and returns true. Otherwise, it returns false. +// +// An error matches target if the error's concrete value is assignable to the value +// pointed to by target, or if the error has a method As(interface{}) bool such that +// As(target) returns true. In the latter case, the As method is responsible for +// setting target. +func (e multiError) As(target interface{}) bool { + if t, ok := target.(*multiError); ok { + *t = e + return true + } + + for _, err := range e.errs { + if stderrors.As(err, target) { + return true + } + } + return false +} + +// Is returns true if any error in multiError's slice of error chains matches the given target or +// if the target is of multiError type. +// +// An error is considered to match a target if it is equal to that target or if +// it implements a method Is(error) bool such that Is(target) returns true. +func (e multiError) Is(target error) bool { + if m, ok := target.(multiError); ok { + if len(m.errs) != len(e.errs) { + return false + } + for i := 0; i < len(e.errs); i++ { + if !stderrors.Is(m.errs[i], e.errs[i]) { + return false + } + } + return true + } + for _, err := range e.errs { + if stderrors.Is(err, target) { + return true + } + } + return false +} + +// Count returns the number of all multi error' errors that match the given target (including nested multi errors). +// Matching is defined as in Is method. +func (e multiError) Count(target error) (count int) { + for _, err := range e.errs { + if inner, ok := AsMulti(err); ok { + count += inner.Count(target) + continue + } + + if stderrors.Is(err, target) { + count++ + } + } + return count +} + +// AsMulti casts error to multi error read only interface. It returns multi error and true if error matches multi error as +// defined by As method. If returns false if no multi error can be found. +func AsMulti(err error) (Error, bool) { + m := multiError{} + if !stderrors.As(err, &m) { + return nil, false + } + return m, true +} + +// Merge merges multiple Error to single one, but joining all errors together. +// NOTE: Nested multi errors are not merged. +func Merge(errs []Error) Error { + e := multiError{} + for _, err := range errs { + e.errs = append(e.errs, err.Errors()...) + } + return e +} + +// PrettyPrint prints the same information as multiError.Error() method but with newlines and indentation targeted +// for humans. +func PrettyPrint(w io.Writer, err Error) error { + return prettyPrint(w, "\t", err) +} + +func prettyPrint(w io.Writer, indent string, merr Error) error { + if len(merr.Errors()) > 1 { + if _, err := fmt.Fprintf(w, "%d errors:\n", len(merr.Errors())); err != nil { + return err + } + } + for i, err := range merr.Errors() { + if i != 0 { + if _, err := w.Write([]byte("\n")); err != nil { + return err + } + } + + if merr2, ok := AsMulti(err); ok { + if err := prettyPrint(w, indent+"\t", merr2); err != nil { + return nil + } + continue + } + + if _, err := w.Write([]byte(indent + err.Error())); err != nil { + return err + } + } + return nil +} diff --git a/merrors/merrors_test.go b/merrors/merrors_test.go new file mode 100644 index 0000000..54316f8 --- /dev/null +++ b/merrors/merrors_test.go @@ -0,0 +1,223 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package merrors_test + +import ( + stderrors "errors" + "testing" + + corerrors "github.com/efficientgo/core/errors" + "github.com/efficientgo/core/merrors" + "github.com/efficientgo/core/testutil" +) + +func TestNilMultiError(t *testing.T) { + testutil.Ok(t, merrors.New().Err()) + testutil.Ok(t, merrors.New(nil, nil, nil).Err()) + + e := merrors.New() + e.Add() + testutil.Ok(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add() + testutil.Ok(t, e.Err()) + + e = merrors.New() + e.Add(nil, nil, nil) + testutil.Ok(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add(nil, nil, nil) + testutil.Ok(t, e.Err()) +} + +func TestMultiError(t *testing.T) { + err := stderrors.New("test1") + testutil.NotOk(t, merrors.New(err).Err()) + testutil.NotOk(t, merrors.New(nil, err, nil).Err()) + + e := merrors.New(err) + e.Add() + testutil.NotOk(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add(err) + testutil.NotOk(t, e.Err()) + + e = merrors.New(err) + e.Add(nil, nil, nil) + testutil.NotOk(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add(nil, err, nil) + testutil.NotOk(t, e.Err()) + + testutil.NotOk(t, func() error { + return e.Err() + }()) + + testutil.Ok(t, func() error { + return merrors.New(nil, nil, nil).Err() + }()) +} + +func TestMultiError_Error(t *testing.T) { + err := stderrors.New("test1") + + testutil.Equals(t, "test1", .New(err).Err().Error()) + testutil.Equals(t, "test1", .New(err, nil).Err().Error()) + testutil.Equals(t, "4 errors: test1; test1; test2; test3", .New(err, err, stderrors.New("test2"), nil, stderrors.New("test3")).Err().Error()) +} + +type customErr struct{ error } + +type customErr2 struct{ error } + +type customErr3 struct{ error } + +func TestMultiError_As(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + + testutil.Assert(t, stderrors.As(err, &err)) + testutil.Assert(t, stderrors.As(err, &customErr{})) + + testutil.Assert(t, !stderrors.As(err, &customErr2{})) + testutil.Assert(t, !stderrors.As(err, &customErr3{})) + + // This is just to show limitation of std As. + testutil.Assert(t, !stderrors.As(&err, &err)) + testutil.Assert(t, !stderrors.As(&err, &customErr{})) + testutil.Assert(t, !stderrors.As(&err, &customErr2{})) + testutil.Assert(t, !stderrors.As(&err, &customErr3{})) + + e := .New(err).Err() + testutil.Assert(t, stderrors.As(e, &customErr{})) + same := .New(err).Err() + testutil.Assert(t, stderrors.As(e, &same)) + testutil.Assert(t, !stderrors.As(e, &customErr2{})) + testutil.Assert(t, !stderrors.As(e, &customErr3{})) + + e2 := .New(err, customErr3{error: stderrors.New("some")}).Err() + testutil.Assert(t, stderrors.As(e2, &customErr{})) + testutil.Assert(t, stderrors.As(e2, &customErr3{})) + testutil.Assert(t, !stderrors.As(e2, &customErr2{})) + + // Wrapped. + e3 := corerrors.Wrap(.New(err, customErr3{}).Err(), "wrap") + testutil.Assert(t, stderrors.As(e3, &customErr{})) + testutil.Assert(t, stderrors.As(e3, &customErr3{})) + testutil.Assert(t, !stderrors.As(e3, &customErr2{})) + + // This is just to show limitation of std As. + e4 := .New(err, &customErr3{}).Err() + testutil.Assert(t, !stderrors.As(e4, &customErr2{})) + testutil.Assert(t, !stderrors.As(e4, &customErr3{})) +} + +func TestMultiError_Is(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + + testutil.Assert(t, stderrors.Is(err, err)) + testutil.Assert(t, stderrors.Is(err, customErr{error: err.error})) + testutil.Assert(t, !stderrors.Is(err, &err)) + testutil.Assert(t, !stderrors.Is(err, customErr{})) + testutil.Assert(t, !stderrors.Is(err, customErr{error: stderrors.New("err1")})) + testutil.Assert(t, !stderrors.Is(err, customErr2{})) + testutil.Assert(t, !stderrors.Is(err, customErr3{})) + + testutil.Assert(t, stderrors.Is(&err, &err)) + testutil.Assert(t, !stderrors.Is(&err, &customErr{error: err.error})) + testutil.Assert(t, !stderrors.Is(&err, &customErr2{})) + testutil.Assert(t, !stderrors.Is(&err, &customErr3{})) + + e := .New(err).Err() + testutil.Assert(t, stderrors.Is(e, err)) + testutil.Assert(t, stderrors.Is(err, customErr{error: err.error})) + testutil.Assert(t, stderrors.Is(e, e)) + testutil.Assert(t, stderrors.Is(e, .New(err).Err())) + testutil.Assert(t, !stderrors.Is(e, &err)) + testutil.Assert(t, !stderrors.Is(err, customErr{})) + testutil.Assert(t, !stderrors.Is(e, customErr2{})) + testutil.Assert(t, !stderrors.Is(e, customErr3{})) + + e2 := .New(err, customErr3{}).Err() + testutil.Assert(t, stderrors.Is(e2, err)) + testutil.Assert(t, stderrors.Is(e2, customErr3{})) + testutil.Assert(t, stderrors.Is(e2, .New(err, customErr3{}).Err())) + testutil.Assert(t, !stderrors.Is(e2, .New(customErr3{}, err).Err())) + testutil.Assert(t, !stderrors.Is(e2, customErr{})) + testutil.Assert(t, !stderrors.Is(e2, customErr2{})) + + // Wrapped. + e3 := corerrors.Wrap(.New(err, customErr3{}).Err(), "wrap") + testutil.Assert(t, stderrors.Is(e3, err)) + testutil.Assert(t, stderrors.Is(e3, customErr3{})) + testutil.Assert(t, !stderrors.Is(e3, customErr{})) + testutil.Assert(t, !stderrors.Is(e3, customErr2{})) + + exact := &customErr3{} + e4 := .New(err, exact).Err() + testutil.Assert(t, stderrors.Is(e4, err)) + testutil.Assert(t, stderrors.Is(e4, exact)) + testutil.Assert(t, stderrors.Is(e4, .New(err, exact).Err())) + testutil.Assert(t, !stderrors.Is(e4, customErr{})) + testutil.Assert(t, !stderrors.Is(e4, customErr2{})) + testutil.Assert(t, !stderrors.Is(e4, &customErr3{})) +} + +func TestMultiError_Count(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + merr := .New() + merr.Add(customErr3{}) + + m, ok := .AsMulti(merr.Err()) + testutil.Assert(t, ok) + testutil.Equals(t, 0, m.Count(err)) + testutil.Equals(t, 1, m.Count(customErr3{})) + + merr.Add(customErr3{}) + merr.Add(customErr3{}) + + m, ok = .AsMulti(merr.Err()) + testutil.Assert(t, ok) + testutil.Equals(t, 0, m.Count(err)) + testutil.Equals(t, 3, m.Count(customErr3{})) + + // Nest multi errors with wraps. + merr2 := .New() + merr2.Add(customErr3{}) + merr2.Add(customErr3{}) + merr2.Add(customErr3{}) + + merr3 := .New() + merr3.Add(customErr3{}) + merr3.Add(customErr3{}) + + // Wrap it so Add cannot add inner errors in. + merr2.Add(corerrors.Wrap(merr3.Err(), "wrap")) + merr.Add(corerrors.Wrap(merr2.Err(), "wrap")) + + m, ok = .AsMulti(merr.Err()) + testutil.Assert(t, ok) + testutil.Equals(t, 0, m.Count(err)) + testutil.Equals(t, 8, m.Count(customErr3{})) +} + +func TestAsMulti(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + merr := .New(err, customErr3{}).Err() + wrapped := corerrors.Wrap(merr, "wrap") + + _, ok := .AsMulti(err) + testutil.Assert(t, !ok) + + m, ok := .AsMulti(merr) + testutil.Assert(t, ok) + testutil.Assert(t, stderrors.Is(m, merr)) + + m, ok = .AsMulti(wrapped) + testutil.Assert(t, ok) + testutil.Assert(t, stderrors.Is(m, merr)) +} diff --git a/runutil/doc.go b/runutil/doc.go new file mode 100644 index 0000000..a65e6cc --- /dev/null +++ b/runutil/doc.go @@ -0,0 +1,28 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package runutil + +// Helpers for advanced function scheduling control like repeat or retry. +// +// It's very often the case when you need to excutes some code every fixed intervals or have it retried automatically. +// To make it reliably with proper timeout, you need to carefully arrange some boilerplate for this. +// Below function does it for you. +// +// For repeat executes, use Repeat: +// +// err := runutil.Repeat(10*time.Second, stopc, func() error { +// // ... +// }) +// +// Retry starts executing closure function f until no error is returned from f: +// +// err := runutil.Retry(10*time.Second, stopc, func() error { +// // ... +// }) +// +// For logging an error on each f error, use RetryWithLog: +// +// err := runutil.RetryWithLog(logger, 10*time.Second, stopc, func() error { +// // ... +// }) diff --git a/runutil/example_test.go b/runutil/example_test.go new file mode 100644 index 0000000..64b4d37 --- /dev/null +++ b/runutil/example_test.go @@ -0,0 +1,49 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Initially copied from Thanos +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package runutil_test + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/efficientgo/tools/core/pkg/runutil" + "github.com/pkg/errors" +) + +func ExampleRepeat() { + // It will stop Repeat 10 seconds later. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // It will print out "Repeat" every 5 seconds. + err := runutil.Repeat(5*time.Second, ctx.Done(), func() error { + fmt.Println("Repeat") + return nil + }) + if err != nil { + log.Fatal(err) + } +} + +func ExampleRetry() { + // It will stop Retry 10 seconds later. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // It will print out "Retry" every 5 seconds. + err := runutil.Retry(5*time.Second, ctx.Done(), func() error { + fmt.Println("Retry") + return errors.New("Try to retry") + }) + if err != nil { + log.Fatal(err) + } +} diff --git a/runutil/runutil.go b/runutil/runutil.go new file mode 100644 index 0000000..fcc1e41 --- /dev/null +++ b/runutil/runutil.go @@ -0,0 +1,61 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Initially copied from Thanos +// +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. +package runutil + +import ( + "time" +) + +// Repeat executes f every interval seconds until stopc is closed or f returns an error. +// It executes f once right after being called. +func Repeat(interval time.Duration, stopc <-chan struct{}, f func() error) error { + tick := time.NewTicker(interval) + defer tick.Stop() + + for { + if err := f(); err != nil { + return err + } + select { + case <-stopc: + return nil + case <-tick.C: + } + } +} + +// Logger interface compatible with go-kit/logger. +type Logger interface { + Log(keyvals ...interface{}) error +} + +// Retry executes f every interval seconds until timeout or no error is returned from f. +func Retry(interval time.Duration, stopc <-chan struct{}, f func() error) error { + return RetryWithLog(nil, interval, stopc, f) +} + +// RetryWithLog executes f every interval seconds until timeout or no error is returned from f. It logs an error on each f error. +func RetryWithLog(logger Logger, interval time.Duration, stopc <-chan struct{}, f func() error) error { + tick := time.NewTicker(interval) + defer tick.Stop() + + var err error + for { + if err = f(); err == nil { + return nil + } + if logger != nil { + _ = logger.Log("msg", "function failed. Retrying in next tick", "err", err) + } + select { + case <-stopc: + return err + case <-tick.C: + } + } +} diff --git a/testutil/doc.go b/testutil/doc.go new file mode 100644 index 0000000..dc9f63f --- /dev/null +++ b/testutil/doc.go @@ -0,0 +1,6 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package testutil + +// Simplistic assertion helpers for testing code. TestOrBench utils for union of testing and benchmarks. diff --git a/testutil/testorbench.go b/testutil/testorbench.go new file mode 100644 index 0000000..c36b887 --- /dev/null +++ b/testutil/testorbench.go @@ -0,0 +1,85 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package testutil + +import ( + "testing" +) + +// TB represents union of test and benchmark. +// This allows the same test suite to be run by both benchmark and test, helping to reuse more code. +// The reason is that usually benchmarks are not being run on CI, especially for short tests, so you need to recreate +// usually similar tests for `Test(t *testing.T)` methods. Example of usage is presented here: +// +// func TestTestOrBench(t *testing.T) { +// tb := NewTB(t) +// tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) +// tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +// } +// +// func BenchmarkTestOrBench(b *testing.B) { +// tb := NewTB(t) +// tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) +// tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +// } +type TB interface { + testing.TB + IsBenchmark() bool + Run(name string, f func(t TB)) bool + + SetBytes(n int64) + N() int + ResetTimer() +} + +// tb implements TB as well as testing.TB interfaces. +type tb struct { + testing.TB +} + +// NewTB creates tb from testing.TB. +func NewTB(t testing.TB) TB { return &tb{TB: t} } + +// Run benchmarks/tests f as a subbenchmark/subtest with the given name. It reports +// whether there were any failures. +// +// A subbenchmark/subtest is like any other benchmark/test. +func (t *tb) Run(name string, f func(t TB)) bool { + if b, ok := t.TB.(*testing.B); ok { + return b.Run(name, func(nested *testing.B) { f(&tb{TB: nested}) }) + } + if t, ok := t.TB.(*testing.T); ok { + return t.Run(name, func(nested *testing.T) { f(&tb{TB: nested}) }) + } + panic("not a benchmark and not a test") +} + +// N returns number of iterations to do for benchmark, 1 in case of test. +func (t *tb) N() int { + if b, ok := t.TB.(*testing.B); ok { + return b.N + } + return 1 +} + +// SetBytes records the number of bytes processed in a single operation for benchmark, noop otherwise. +// If this is called, the benchmark will report ns/op and MB/s. +func (t *tb) SetBytes(n int64) { + if b, ok := t.TB.(*testing.B); ok { + b.SetBytes(n) + } +} + +// ResetTimer resets a timer, if it's a benchmark, noop otherwise. +func (t *tb) ResetTimer() { + if b, ok := t.TB.(*testing.B); ok { + b.ResetTimer() + } +} + +// IsBenchmark returns true if it's a benchmark. +func (t *tb) IsBenchmark() bool { + _, ok := t.TB.(*testing.B) + return ok +} diff --git a/testutil/testorbench_test.go b/testutil/testorbench_test.go new file mode 100644 index 0000000..02524cc --- /dev/null +++ b/testutil/testorbench_test.go @@ -0,0 +1,47 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package testutil + +import "testing" + +func TestTestOrBench(t *testing.T) { + tb := NewTB(t) + tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) + tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +} + +func BenchmarkTestOrBench(b *testing.B) { + tb := NewTB(b) + tb.Run("1", func(tb TB) { testorbenchComplexTest(tb) }) + tb.Run("2", func(tb TB) { testorbenchComplexTest(tb) }) +} + +func testorbenchComplexTest(tb TB) { + tb.Run("a", func(tb TB) { + tb.Run("aa", func(tb TB) { + tb.ResetTimer() + for i := 0; i < tb.N(); i++ { + if !tb.IsBenchmark() { + if tb.N() != 1 { + tb.FailNow() + } + } + } + }) + }) + tb.SetBytes(120220) + tb.Run("b", func(tb TB) { + tb.Run("bb", func(tb TB) { + tb.ResetTimer() + for i := 0; i < tb.N(); i++ { + if !tb.IsBenchmark() { + if tb.N() != 1 { + tb.FailNow() + } + } + } + }) + }) + +} diff --git a/testutil/testutil.go b/testutil/testutil.go new file mode 100644 index 0000000..8a46d2e --- /dev/null +++ b/testutil/testutil.go @@ -0,0 +1,181 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package testutil + +import ( + "fmt" + "path/filepath" + "reflect" + "runtime" + "runtime/debug" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/pkg/errors" + "github.com/pmezard/go-difflib/difflib" + "go.uber.org/goleak" +) + +// Assert fails the test if the condition is false. +func Assert(tb testing.TB, condition bool, v ...interface{}) { + tb.Helper() + if condition { + return + } + _, file, line, _ := runtime.Caller(1) + + var msg string + if len(v) > 0 { + msg = fmt.Sprintf(v[0].(string), v[1:]...) + } + tb.Fatalf("\033[31m%s:%d: "+msg+"\033[39m\n\n", filepath.Base(file), line) +} + +// Ok fails the test if an err is not nil. +func Ok(tb testing.TB, err error, v ...interface{}) { + tb.Helper() + if err == nil { + return + } + _, file, line, _ := runtime.Caller(1) + + var msg string + if len(v) > 0 { + msg = fmt.Sprintf(v[0].(string), v[1:]...) + } + tb.Fatalf("\033[31m%s:%d:"+msg+"\n\n unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) +} + +// NotOk fails the test if an err is nil. +func NotOk(tb testing.TB, err error, v ...interface{}) { + tb.Helper() + if err != nil { + return + } + _, file, line, _ := runtime.Caller(1) + + var msg string + if len(v) > 0 { + msg = fmt.Sprintf(v[0].(string), v[1:]...) + } + tb.Fatalf("\033[31m%s:%d:"+msg+"\n\n expected error, got nothing \033[39m\n\n", filepath.Base(file), line) +} + +// Equals fails the test if exp is not equal to act. +func Equals(tb testing.TB, exp, act interface{}, v ...interface{}) { + tb.Helper() + if reflect.DeepEqual(exp, act) { + return + } + _, file, line, _ := runtime.Caller(1) + + var msg string + if len(v) > 0 { + msg = fmt.Sprintf(v[0].(string), v[1:]...) + } + tb.Fatal(sprintfWithLimit("\033[31m%s:%d:"+msg+"\n\n\texp: %#v\n\n\tgot: %#v%s\033[39m\n\n", filepath.Base(file), line, exp, act, diff(exp, act))) +} + +func sprintfWithLimit(act string, v ...interface{}) string { + s := fmt.Sprintf(act, v...) + if len(s) > 10000 { + return s[:10000] + "...(output trimmed)" + } + return s +} + +func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { + t := reflect.TypeOf(v) + k := t.Kind() + + if k == reflect.Ptr { + t = t.Elem() + k = t.Kind() + } + return t, k +} + +// diff returns a diff of both values as long as both are of the same type and +// are a struct, map, slice, array or string. Otherwise it returns an empty string. +func diff(expected interface{}, actual interface{}) string { + if expected == nil || actual == nil { + return "" + } + + et, ek := typeAndKind(expected) + at, _ := typeAndKind(actual) + if et != at { + return "" + } + + if ek != reflect.Struct && ek != reflect.Map && ek != reflect.Slice && ek != reflect.Array && ek != reflect.String { + return "" + } + + var e, a string + c := spew.ConfigState{ + Indent: " ", + DisablePointerAddresses: true, + DisableCapacities: true, + SortKeys: true, + } + if et != reflect.TypeOf("") { + e = c.Sdump(expected) + a = c.Sdump(actual) + } else { + e = reflect.ValueOf(expected).String() + a = reflect.ValueOf(actual).String() + } + + diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(e), + B: difflib.SplitLines(a), + FromFile: "Expected", + FromDate: "", + ToFile: "Actual", + ToDate: "", + Context: 1, + }) + return "\n\nDiff:\n" + diff +} + +// TolerantVerifyLeakMain verifies go leaks but excludes the go routines that are +// launched as side effects of some of our dependencies. +func TolerantVerifyLeakMain(m *testing.M) { + goleak.VerifyTestMain(m, + // https://github.com/census-instrumentation/opencensus-go/blob/d7677d6af5953e0506ac4c08f349c62b917a443a/stats/view/worker.go#L34 + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + // https://github.com/kubernetes/klog/blob/c85d02d1c76a9ebafa81eb6d35c980734f2c4727/klog.go#L417 + goleak.IgnoreTopFunction("k8s.io/klog/v2.(*loggingT).flushDaemon"), + goleak.IgnoreTopFunction("k8s.io/klog.(*loggingT).flushDaemon"), + ) +} + +// TolerantVerifyLeak verifies go leaks but excludes the go routines that are +// launched as side effects of some of our dependencies. +func TolerantVerifyLeak(t *testing.T) { + goleak.VerifyNone(t, + // https://github.com/census-instrumentation/opencensus-go/blob/d7677d6af5953e0506ac4c08f349c62b917a443a/stats/view/worker.go#L34 + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + // https://github.com/kubernetes/klog/blob/c85d02d1c76a9ebafa81eb6d35c980734f2c4727/klog.go#L417 + goleak.IgnoreTopFunction("k8s.io/klog/v2.(*loggingT).flushDaemon"), + goleak.IgnoreTopFunction("k8s.io/klog.(*loggingT).flushDaemon"), + ) +} + +// FaultOrPanicToErr returns error if panic of fault was triggered during execution of function. +func FaultOrPanicToErr(f func()) (err error) { + // Set this go routine to panic on segfault to allow asserting on those. + debug.SetPanicOnFault(true) + defer func() { + if r := recover(); r != nil { + err = errors.Errorf("invoked function panicked or caused segmentation fault: %v", r) + } + debug.SetPanicOnFault(false) + }() + + f() + + return err +}