From 613ea74f7338778d57e6f67d47b83fa3135f6c88 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Mon, 1 Mar 2021 08:38:23 -0800 Subject: [PATCH] Add compatibility tests to CI (#1567) * Add compatibility tests to CI * Increase timeout to 60s Some of the aggregator tests are running just over 30 seconds when testing on Windows. Bump this to allow for it. * Remove cache setup jobs Having a unified job setup the cache when none exists so packages are only downloaded once is ideal, but these steps are taking too long. In some situations they are doubling the run time. Remove them and prefer to have multiple downloads when dependencies change. * Add changes to changelog * Remove mod-download target from Makefile Unused now that the cache jobs are removed. * Add compatibility testing table to changelog * Add change about precommit to changelog * Update README * Correct examples target in lint job --- .github/workflows/ci.yml | 174 +++++++++++++++++++++++++++-------- CHANGELOG.md | 17 ++++ Makefile | 191 +++++++++++++++++++-------------------- README.md | 20 ++++ 4 files changed, 265 insertions(+), 137 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95e7617114d..2fc1130e6d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,46 +5,142 @@ on: - main pull_request: env: - TEST_RESULTS: /tmp/test-results # path to where test results will be saved + # Path to where test results will be saved. + TEST_RESULTS: /tmp/test-results + # Default minimum version of Go to support. + DEFAULT_GO_VERSION: 1.14 jobs: - go: + lint: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v2.1.3 + with: + go-version: ${{ env.DEFAULT_GO_VERSION }} + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Setup Environment + run: | + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + - name: Module cache + uses: actions/cache@v2.1.4 + env: + cache-name: go-mod-cache + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }} + - name: Tools cache + uses: actions/cache@v2.1.4 + env: + cache-name: go-tools-cache + with: + path: ~/.tools + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('./internal/tools/**') }} + - name: Run linters + run: make dependabot-check license-check lint + - name: Build + run: make examples build + - name: Check clean repository + run: make check-clean-work-tree + + test-race: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v2.1.3 + with: + go-version: ${{ env.DEFAULT_GO_VERSION }} + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Setup Environment + run: | + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + - name: Module cache + uses: actions/cache@v2.1.4 + env: + cache-name: go-mod-cache + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }} + - name: Run tests with race detector + run: make test-race + + test-coverage: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v2.1.3 + with: + go-version: ${{ env.DEFAULT_GO_VERSION }} + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Setup Environment + run: | + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + - name: Module cache + uses: actions/cache@v2.1.4 + env: + cache-name: go-mod-cache + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }} + - name: Run coverage tests + run: | + make test-coverage + mkdir $TEST_RESULTS + cp coverage.out $TEST_RESULTS + cp coverage.txt $TEST_RESULTS + cp coverage.html $TEST_RESULTS + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + file: ./coverage.txt + fail_ci_if_error: true + verbose: true + - name: Store coverage test output + uses: actions/upload-artifact@v2 + with: + name: opentelemetry-go-test-output + path: ${{ env.TEST_RESULTS }} + + compatibility-test: strategy: - fail-fast: false matrix: - go_version: [1.14, 1.15] - runs-on: ubuntu-latest + go-version: [1.15, 1.14] + os: [ubuntu-latest, macos-latest, windows-latest] + # GitHub Actions does not support arm* architectures on default + # runners. It is possible to acomplish this with a self-hosted runner + # if we want to add this in the future: + # https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow + arch: ["386", amd64] + exclude: + # Not a supported Go OS/architecture. + - os: macos-latest + arch: "386" + runs-on: ${{ matrix.os }} steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - name: Setup Go - uses: actions/setup-go@v2.1.3 - with: - go-version: ${{ matrix.go_version }} - - name: Setup Environment - run: | - echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV - echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - - name: restore cache - uses: actions/cache@v2.1.4 - env: - cache-name: cache-go-mod - with: - path: /home/runner/go/pkg/mod - key: go-pkg-mod-{{ checksum "go.sum" }} - - name: Precommit and Coverage Report - run: | - make ci - mkdir $TEST_RESULTS - cp coverage.out $TEST_RESULTS - cp coverage.txt $TEST_RESULTS - cp coverage.html $TEST_RESULTS - - uses: codecov/codecov-action@v1 - with: - file: ./coverage.txt - fail_ci_if_error: true - verbose: true - - name: store test output - uses: actions/upload-artifact@v2 - with: - name: opentelemetry-go-test-output - path: ${{ env.TEST_RESULTS }} + - name: Install Go + uses: actions/setup-go@v2.1.3 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Environment + run: | + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + shell: bash + - name: Module cache + uses: actions/cache@v2.1.4 + env: + cache-name: go-mod-cache + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }} + - name: Run tests + env: + GOARCH: ${{ matrix.arch }} + run: make test-short diff --git a/CHANGELOG.md b/CHANGELOG.md index 57f65d45f96..c203216b6e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,19 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - AttributePerEventCountLimit and AttributePerLinkCountLimit for SpanLimits. (#1535) - Added `Keys()` method to `propagation.TextMapCarrier` and `propagation.HeaderCarrier` to adapt `http.Header` to this interface. (#1544) - Added `code` attributes to `go.opentelemetry.io/otel/semconv` package. (#1558) +- Compatibility testing suite in the CI system for the following systems. (#1567) + | OS | Go Version | Architecture | + | ------- | ---------- | ------------ | + | Ubuntu | 1.15 | amd64 | + | Ubuntu | 1.14 | amd64 | + | Ubuntu | 1.15 | 386 | + | Ubuntu | 1.14 | 386 | + | MacOS | 1.15 | amd64 | + | MacOS | 1.14 | amd64 | + | Windows | 1.15 | amd64 | + | Windows | 1.14 | amd64 | + | Windows | 1.15 | 386 | + | Windows | 1.14 | 386 | - Changed all examples to use `WithBatchTimeout(5 * time.Second)` rather than `WithBatchTimeout(5)` (#1621) ### Changed @@ -24,6 +37,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Rename MaxEventsPerSpan, MaxAttributesPerSpan and MaxLinksPerSpan to EventCountLimit, AttributeCountLimit and LinkCountLimit, and move these fieds into SpanLimits. (#1535) - Renamed the `otel/label` package to `otel/attribute`. (#1541) - Vendor the Jaeger exporter's dependency on Apache Thrift. (#1551) +- Parallelize the CI linting and testing. (#1567) - Stagger timestamps in exact aggregator tests. (#1569) - Prevent end-users from implementing some interfaces (#1575) ``` @@ -44,6 +58,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Removed - Removed attempt to resample spans upon changing the span name with `span.SetName()`. (#1545) +- The `test-benchmark` is no longer a dependency of the `precommit` make target. (#1567) +- The `test-386` make target. + This was replaced with a full compatibility testing suite (i.e. multi OS/arch) in the CI system. (#1567) ### Fixed diff --git a/Makefile b/Makefile index ab645da628c..fa11b5613fc 100644 --- a/Makefile +++ b/Makefile @@ -21,142 +21,127 @@ ALL_DOCS := $(shell find . -name '*.md' -type f | sort) ALL_GO_MOD_DIRS := $(filter-out $(TOOLS_MOD_DIR), $(shell find . -type f -name 'go.mod' -exec dirname {} \; | egrep -v '^./example' | sort)) $(shell find ./example -type f -name 'go.mod' -exec dirname {} \; | sort) ALL_COVERAGE_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | egrep -v '^./example|^$(TOOLS_MOD_DIR)' | sort) -# Mac OS Catalina 10.5.x doesn't support 386. Hence skip 386 test -SKIP_386_TEST = false -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Darwin) - SW_VERS := $(shell sw_vers -productVersion) - ifeq ($(shell echo $(SW_VERS) | egrep '^(10.1[5-9]|1[1-9]|[2-9])'), $(SW_VERS)) - SKIP_386_TEST = true - endif -endif - -GOTEST_MIN = go test -timeout 30s -GOTEST = $(GOTEST_MIN) -race -GOTEST_WITH_COVERAGE = $(GOTEST) -coverprofile=coverage.out -covermode=atomic -coverpkg=./... +GO = go +TIMEOUT = 60 .DEFAULT_GOAL := precommit -.PHONY: precommit +.PHONY: precommit ci +precommit: dependabot-check license-check lint build examples test-default +ci: precommit check-clean-work-tree test-with-coverage -TOOLS_DIR := $(abspath ./.tools) +# Tools -$(TOOLS_DIR)/golangci-lint: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go - cd $(TOOLS_MOD_DIR) && \ - go build -o $(TOOLS_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint +TOOLS = $(CURDIR)/.tools -$(TOOLS_DIR)/misspell: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go +$(TOOLS): + @mkdir -p $@ +$(TOOLS)/%: | $(TOOLS) cd $(TOOLS_MOD_DIR) && \ - go build -o $(TOOLS_DIR)/misspell github.com/client9/misspell/cmd/misspell + $(GO) build -o $@ $(PACKAGE) -$(TOOLS_DIR)/stringer: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go - cd $(TOOLS_MOD_DIR) && \ - go build -o $(TOOLS_DIR)/stringer golang.org/x/tools/cmd/stringer +CROSSLINK = $(TOOLS)/crosslink +$(TOOLS)/crosslink: PACKAGE=go.opentelemetry.io/otel/$(TOOLS_MOD_DIR)/crosslink -$(TOOLS_DIR)/gojq: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go - cd $(TOOLS_MOD_DIR) && \ - go build -o $(TOOLS_DIR)/gojq github.com/itchyny/gojq/cmd/gojq +GOLANGCI_LINT = $(TOOLS)/golangci-lint +$(TOOLS)/golangci-lint: PACKAGE=github.com/golangci/golangci-lint/cmd/golangci-lint -precommit: dependabot-check license-check generate build lint examples test-benchmarks test +MISSPELL = $(TOOLS)/misspell +$(TOOLS)/misspell: PACKAGE= github.com/client9/misspell/cmd/misspell -.PHONY: test-with-coverage -test-with-coverage: - set -e; \ - printf "" > coverage.txt; \ - for dir in $(ALL_COVERAGE_MOD_DIRS); do \ - echo "go test ./... + coverage in $${dir}"; \ - (cd "$${dir}" && \ - go list ./... | grep -v third_party | xargs $(GOTEST_WITH_COVERAGE) && \ - go tool cover -html=coverage.out -o coverage.html); \ - [ -f "$${dir}/coverage.out" ] && cat "$${dir}/coverage.out" >> coverage.txt; \ - done; \ - sed -i.bak -e '2,$$ { /^mode: /d; }' coverage.txt +STRINGER = $(TOOLS)/stringer +$(TOOLS)/stringer: PACKAGE=golang.org/x/tools/cmd/stringer +$(TOOLS)/gojq: PACKAGE=github.com/itchyny/gojq/cmd/gojq -.PHONY: ci -ci: precommit check-clean-work-tree test-with-coverage test-386 +.PHONY: tools +tools: $(CROSSLINK) $(GOLANGCI_LINT) $(MISSPELL) $(STRINGER) $(TOOLS)/gojq -.PHONY: check-clean-work-tree -check-clean-work-tree: - @if ! git diff --quiet; then \ - echo; \ - echo 'Working tree is not clean, did you forget to run "make precommit"?'; \ - echo; \ - git status; \ - exit 1; \ - fi -.PHONY: build -build: - # TODO: Fix this on windows. +# Build + +.PHONY: examples generate build +examples: + @set -e; for ex in $(EXAMPLES); do \ + echo "$(GO) build $${dir}/..."; \ + (cd "$${ex}" && \ + $(GO) build .); \ + done + +generate: $(STRINGER) set -e; for dir in $(ALL_GO_MOD_DIRS); do \ - echo "compiling all packages in $${dir}"; \ + echo "$(GO) generate $${dir}/..."; \ (cd "$${dir}" && \ - go build ./... && \ - go list ./... | grep -v third_party | xargs go test -vet=off -run xxxxxMatchNothingxxxxx >/dev/null); \ + PATH="$(TOOLS):$${PATH}" $(GO) generate ./...); \ done -.PHONY: test -test: +build: generate + # Build all package code including testing code. set -e; for dir in $(ALL_GO_MOD_DIRS); do \ - echo "go test ./... + race in $${dir}"; \ + echo "$(GO) build $${dir}/..."; \ (cd "$${dir}" && \ - go list ./... | grep -v third_party | xargs $(GOTEST)); \ + $(GO) build ./... && \ + $(GO) list ./... \ + | grep -v third_party \ + | xargs $(GO) test -vet=off -run xxxxxMatchNothingxxxxx >/dev/null); \ done -.PHONY: test-386 -test-386: - if [ $(SKIP_386_TEST) = true ] ; then \ - echo "skipping the test for GOARCH 386 as it is not supported on the current OS"; \ - else \ - set -e; for dir in $(ALL_GO_MOD_DIRS); do \ - echo "go test ./... GOARCH 386 in $${dir}"; \ - (cd "$${dir}" && \ - GOARCH=386 go list ./... | grep -v third_party | xargs $(GOTEST_MIN)); \ - done; \ - fi +# Tests -.PHONY: examples -examples: - @set -e; for ex in $(EXAMPLES); do \ - echo "Building $${ex}"; \ - (cd "$${ex}" && \ - go build .); \ - done - -.PHONY: test-benchmarks -test-benchmarks: +TEST_TARGETS := test-default test-bench test-short test-verbose test-race +.PHONY: $(TEST_TARGETS) test +test-default: ARGS=-v -race +test-bench: ARGS=-run=xxxxxMatchNothingxxxxx -test.benchtime=1ms -bench=. +test-short: ARGS=-short +test-verbose: ARGS=-v +test-race: ARGS=-race +$(TEST_TARGETS): test +test: @set -e; for dir in $(ALL_GO_MOD_DIRS); do \ - echo "test benchmarks in $${dir}"; \ - (cd "$${dir}" && go list ./... | grep -v third_party | xargs go test -test.benchtime=1ms -run=NONE -bench=.) > /dev/null; \ + echo "$(GO) test -timeout $(TIMEOUT)s $(ARGS) $${dir}/..."; \ + (cd "$${dir}" && \ + $(GO) list ./... \ + | grep -v third_party \ + | xargs $(GO) test -timeout $(TIMEOUT)s $(ARGS)); \ done +COVERAGE_MODE = atomic +COVERAGE_PROFILE = coverage.out +.PHONY: test-coverage +test-coverage: + @set -e; \ + printf "" > coverage.txt; \ + for dir in $(ALL_COVERAGE_MOD_DIRS); do \ + echo "$(GO) test -coverpkg=./... -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_PROFILE)" $${dir}/..."; \ + (cd "$${dir}" && \ + $(GO) test -coverpkg=./... -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_PROFILE)" ./... && \ + $(GO) tool cover -html=coverage.out -o coverage.html); \ + [ -f "$${dir}/coverage.out" ] && cat "$${dir}/coverage.out" >> coverage.txt; \ + done; \ + sed -i.bak -e '2,$$ { /^mode: /d; }' coverage.txt + .PHONY: lint -lint: $(TOOLS_DIR)/golangci-lint $(TOOLS_DIR)/misspell lint-modules +lint: misspell lint-modules | $(GOLANGCI_LINT) set -e; for dir in $(ALL_GO_MOD_DIRS); do \ echo "golangci-lint in $${dir}"; \ (cd "$${dir}" && \ - $(TOOLS_DIR)/golangci-lint run --skip-dirs-use-default --fix && \ - $(TOOLS_DIR)/golangci-lint run --skip-dirs-use-default); \ + $(GOLANGCI_LINT) run --fix && \ + $(GOLANGCI_LINT) run); \ done - $(TOOLS_DIR)/misspell -w $(ALL_DOCS) + +.PHONY: misspell +misspell: | $(MISSPELL) + $(MISSPELL) -w $(ALL_DOCS) .PHONY: lint-modules -lint-modules: +lint-modules: | $(CROSSLINK) set -e; for dir in $(ALL_GO_MOD_DIRS) $(TOOLS_MOD_DIR); do \ - echo "go mod tidy in $${dir}"; \ + echo "$(GO) mod tidy in $${dir}"; \ (cd "$${dir}" && \ - go mod tidy); \ + $(GO) mod tidy); \ done echo "cross-linking all go modules" - (cd internal/tools; go run ./crosslink) - -generate: $(TOOLS_DIR)/stringer - set -e; for dir in $(ALL_GO_MOD_DIRS); do \ - echo "running generators in $${dir}"; \ - (cd "$${dir}" && \ - PATH="$(TOOLS_DIR):$${PATH}" go generate ./...); \ - done + $(CROSSLINK) .PHONY: license-check license-check: @@ -180,3 +165,13 @@ dependabot-check: echo "missing go.mod dependabot check:"; echo "$$result"; \ exit 1; \ fi + +.PHONY: check-clean-work-tree +check-clean-work-tree: + @if ! git diff --quiet; then \ + echo; \ + echo 'Working tree is not clean, did you forget to run "make precommit"?'; \ + echo; \ + git status; \ + exit 1; \ + fi diff --git a/README.md b/README.md index f46cef70327..c841ba896e5 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,26 @@ and Project versioning information and stability guarantees can be found in the [versioning documentation](./VERSIONING.md). +### Compatibility + +This project is tested on the following systems. + +| OS | Go Version | Architecture | +| ------- | ---------- | ------------ | +| Ubuntu | 1.15 | amd64 | +| Ubuntu | 1.14 | amd64 | +| Ubuntu | 1.15 | 386 | +| Ubuntu | 1.14 | 386 | +| MacOS | 1.15 | amd64 | +| MacOS | 1.14 | amd64 | +| Windows | 1.15 | amd64 | +| Windows | 1.14 | amd64 | +| Windows | 1.15 | 386 | +| Windows | 1.14 | 386 | + +While this project should work for other systems, no compatibility guarantees +are made for those systems currently. + ## Getting Started You can find a getting started guide on [opentelemetry.io](https://opentelemetry.io/docs/go/getting-started/).