diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index c472a5c9f7a..73eb421a0ea 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -3,14 +3,14 @@ on: push: branches: [master] tags: - - v[0-9]+.[0-9]+.[0-9]+ + - 'v[0-9]+.[0-9]+.[0-9]+*' pull_request: +env: + TEST_RESULTS: testbed/tests/results/junit/results.xml jobs: windows-test: runs-on: windows-latest - env: - TEST_RESULTS: testbed/tests/results/junit/results.xml steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -18,12 +18,12 @@ jobs: uses: actions/setup-go@v2.1.3 with: go-version: 1.15 - - name: Setup env + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH shell: bash - - name: Restore module cache + - name: Cache Modules uses: actions/cache@v2 env: cache-name: cache-go-modules @@ -32,7 +32,7 @@ jobs: key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('./go.mod') }} - name: Run Unit tests run: go test ./... - - name: GitHub issue generator + - name: GitHub Issue Generator if: ${{ failure() && github.ref == 'ref/head/master' }} run: | go run cmd/issuegenerator/main.go $TEST_RESULTS @@ -47,11 +47,11 @@ jobs: uses: actions/setup-go@v2.1.3 with: go-version: 1.15 - - name: Setup env + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - - name: Restore module cache + - name: Cache Modules uses: actions/cache@v2 env: cache-name: cache-go-modules @@ -60,32 +60,18 @@ jobs: key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('./go.mod') }} - name: Install tools run: make install-tools - - name: split loadtest jobs + - name: Split Loadtest Jobs id: splitloadtest - run: | - TESTS="$(make -s testbed-list-loadtest | xargs echo|sed 's/ /|/g')" - TESTS=(${TESTS//|/ }) - MATRIX="{\"include\":[" - curr="" - for i in "${!TESTS[@]}"; do - if (( i > 0 && i % 2 == 0 )); then - curr+="|${TESTS[$i]}" - else - if [ -n "$curr" ] && (( i>1 )); then - MATRIX+=",{\"test\":\"$curr\"}" - elif [ -n "$curr" ]; then - MATRIX+="{\"test\":\"$curr\"}" - fi - curr="${TESTS[$i]}" - fi - done - MATRIX+="]}" - echo "::set-output name=matrix::$MATRIX" - - name: Upload tool binaries + run: ./.github/workflows/scripts/setup_load_tests.sh + - name: Add Execute Permissions to Binaries + run: chmod -R +x /home/runner/go/bin + - name: Create Tool Binaries Archive + run: tar -cvf tool-bin.tar /home/runner/go/bin + - name: Upload Tool Binaries uses: actions/upload-artifact@v1 with: name: tool-binaries - path: /home/runner/go/bin + path: ./tool-bin.tar lint: runs-on: ubuntu-latest needs: [setup-environment] @@ -96,24 +82,23 @@ jobs: uses: actions/setup-go@v2.1.3 with: go-version: 1.15 - - name: Setup env + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - - name: Restore module cache + - name: Cache Modules uses: actions/cache@v2 env: cache-name: cache-go-modules with: path: /home/runner/go/pkg/mod key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('./go.mod') }} - - name: Download tool binaries - uses: actions/download-artifact@v2 + - name: Download Tool Binaries + uses: actions/download-artifact@v1 with: name: tool-binaries - path: /home/runner/go/bin - - name: Add execute permissions to tool binaries - run: chmod -R +x /home/runner/go/bin + - name: Extract Tool Binaries Archive + run: tar -xvf tool-binaries/tool-bin.tar -C / - name: Lint run: make -j4 gochecklicense goimpi golint gomisspell - name: Gen Metadata @@ -130,39 +115,38 @@ jobs: uses: actions/setup-go@v2.1.3 with: go-version: 1.15 - - name: Setup env + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - - name: Restore module cache + - name: Cache Modules uses: actions/cache@v2 env: cache-name: cache-go-modules with: path: /home/runner/go/pkg/mod key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('./go.mod') }} - - name: Download tool binaries - uses: actions/download-artifact@v2 + - name: Download Tool Binaries + uses: actions/download-artifact@v1 with: name: tool-binaries - path: /home/runner/go/bin - - name: Add execute permissions to tool binaries - run: chmod -R +x /home/runner/go/bin - - name: Build collector for all archs + - name: Extract Tool Binaries Archive + run: tar -xvf tool-binaries/tool-bin.tar -C / + - name: Build Collector for All Architectures run: grep ^binaries-all-sys Makefile|fmt -w 1|tail -n +2|xargs make - - name: Upload collector binaries + - name: Create Collector Binaries Archive + run: tar -cvf bin.tar ./bin + - name: Upload Collector Binaries uses: actions/upload-artifact@v1 with: name: collector-binaries - path: ./bin + path: ./bin.tar loadtest: runs-on: ubuntu-latest needs: [setup-environment] strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup-environment.outputs.matrix) }} - env: - TEST_RESULTS: testbed/tests/results/junit/results.xml + matrix: ${{ fromJson(needs.loadtestpre.outputs.matrix) }} steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -170,24 +154,23 @@ jobs: uses: actions/setup-go@v2.1.3 with: go-version: 1.15 - - name: Download tool binaries - uses: actions/download-artifact@v2 - with: - name: tool-binaries - path: /home/runner/go/bin - - name: Add execute permissions to tool binaries - run: chmod -R +x /home/runner/go/bin - - name: Setup env + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - - name: Restore module cache + - name: Cache Modules uses: actions/cache@v2 env: cache-name: cache-go-modules with: path: /home/runner/go/pkg/mod key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('./go.mod') }} + - name: Download Tool Binaries + uses: actions/download-artifact@v1 + with: + name: tool-binaries + - name: Extract Tool Binaries Archive + run: tar -xvf tool-binaries/tool-bin.tar -C / - name: Install fluentbit if: ${{ contains(matrix.test, 'Log10kDPS') }} run: | @@ -200,26 +183,24 @@ jobs: run: make testbed-loadtest env: TEST_ARGS: "-test.run=${{ matrix.test }}" - - name: Create test result archive # some test results have invalid characters + - name: Create Test Result Archive # some test results have invalid characters if: ${{ failure() || success() }} continue-on-error: true run: tar -cvf test_results.tar testbed/tests/results - - name: Upload test results + - name: Upload Test Results if: ${{ failure() || success() }} continue-on-error: true uses: actions/upload-artifact@v1 with: name: test-results path: test_results.tar - - name: GitHub issue generator + - name: GitHub Issue Generator if: ${{ failure() && github.ref == 'ref/head/master' }} run: | go run cmd/issuegenerator/main.go $TEST_RESULTS correctness: runs-on: ubuntu-latest needs: [setup-environment] - env: - TEST_RESULTS: testbed/tests/results/junit/results.xml steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -227,27 +208,26 @@ jobs: uses: actions/setup-go@v2.1.3 with: go-version: 1.15 - - name: Download tool binaries - uses: actions/download-artifact@v2 - with: - name: tool-binaries - path: /home/runner/go/bin - - name: Setup env + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - - name: Add execute permissions to tool binaries - run: chmod -R +x /home/runner/go/bin - - name: Restore module cache + - name: Cache Modules uses: actions/cache@v2 env: cache-name: cache-go-modules with: path: /home/runner/go/pkg/mod key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('./go.mod') }} + - name: Download Tool Binaries + uses: actions/download-artifact@v1 + with: + name: tool-binaries + - name: Extract Tool Binaries Archive + run: tar -xvf tool-binaries/tool-bin.tar -C / - name: Loadtest run: make testbed-correctness - - name: GitHub issue generator + - name: GitHub Issue Generator if: ${{ failure() && github.ref == 'ref/head/master' }} run: | go run cmd/issuegenerator/main.go $TEST_RESULTS @@ -267,7 +247,7 @@ jobs: uses: actions/setup-go@v2.1.3 with: go-version: 1.15 - - name: Setup env + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH @@ -276,30 +256,23 @@ jobs: uses: actions/setup-ruby@v1 with: ruby-version: '2.6' - - name: Install fpm and dependencies + - name: Install fpm run: gem install --no-document fpm -v 1.11.0 - - name: Download tool binaries - uses: actions/download-artifact@v2 + - name: Download Tool Binaries + uses: actions/download-artifact@v1 with: name: tool-binaries - path: /home/runner/go/bin - - name: Download collector binaries - uses: actions/download-artifact@v2 + - name: Download Collector Binaries + uses: actions/download-artifact@v1 with: name: collector-binaries - path: ./bin - - name: Add execute permissions to downloaded binaries + - name: Extract Binaries Archive run: | - chmod -R +x /home/runner/go/bin - chmod -R +x ./bin + tar -xvf tool-binaries/tool-bin.tar -C / + tar -xvf collector-binaries/bin.tar - name: Set Release Tag id: github_tag - run: | - TAG="${GITHUB_REF##*/}" - if [[ $TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+.* ]] - then - echo "::set-output name=tag::$TAG" - fi + run: ./.github/workflows/scripts/set_release_tag.sh - name: Build ${{ matrix.package_type }} amd64 package run: ./internal/buildscripts/packaging/fpm/${{ matrix.package_type }}/build.sh "${{ steps.github_tag.outputs.tag }}" "amd64" "./dist/" - name: Build ${{ matrix.package_type }} arm64 package @@ -316,3 +289,177 @@ jobs: with: name: build-packages path: ./dist + windows-msi: + runs-on: windows-latest + needs: [cross-compile] + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v2.1.3 + with: + go-version: 1.15 + - name: Setup Environment + run: | + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + mkdir bin/ + - name: Cache Modules + uses: actions/cache@v2 + env: + cache-name: cache-go-modules + with: + path: \Users\runneradmin\go\pkg\mod + key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('./go.mod') }} + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: collector-binaries + - name: Extract Binaries Archive + run: tar -xvf collector-binaries/bin.tar + - name: Install Wix Toolset + run: .\internal\buildscripts\packaging\msi\make.ps1 Install-Tools + - name: Build MSI + run: | + $Version = if ($env:GITHUB_REF -match '^refs/tags/(\d+\.\d+\.\d+)') { $Matches[1] } else { "0.0.1" } + .\internal\buildscripts\packaging\msi\make.ps1 New-MSI -Version $Version + - name: Validate MSI + run: .\internal\buildscripts\packaging\msi\make.ps1 Confirm-MSI + - name: Upload MSI + uses: actions/upload-artifact@v1 + with: + name: msi-binaries + path: ./dist + publish-dev: + runs-on: ubuntu-latest + needs: [build-package, windows-msi] + if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'open-telemetry/opentelemetry-collector' + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Setup Go + uses: actions/setup-go@v2.1.3 + with: + go-version: 1.15 + - name: Setup Environment + run: | + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + mkdir bin/ dist/ + - name: Download Tool Binaries + uses: actions/download-artifact@v1 + with: + name: tool-binaries + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: collector-binaries + - name: Extract Binaries Archive + run: | + tar -xvf tool-binaries/tool-bin.tar -C / + tar -xvf collector-binaries/bin.tar + - name: Download Packages + uses: actions/download-artifact@v1 + with: + name: build-packages + path: ./dist + - name: Download MSI Packages + uses: actions/download-artifact@v1 + with: + name: msi-binaries + path: ./dist + - name: Add Execute Permissions to Binaries + run: chmod -R +x ./dist + - name: Verify Distribution Files Exist + id: check + run: ./.github/workflows/scripts/verify-dist-files-exist.sh + - name: Build Docker Image + if: steps.check.outputs.passed == 'true' + run: | + make docker-otelcol + docker tag otelcol:latest otel/opentelemetry-collector-dev:$GITHUB_SHA + docker tag otelcol:latest otel/opentelemetry-collector-dev:latest + - name: Push Docker Image + if: steps.check.outputs.passed == 'true' + run: | + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + docker push otel/opentelemetry-collector-dev:$GITHUB_SHA + docker push otel/opentelemetry-collector-dev:latest + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} + publish-stable: + runs-on: ubuntu-latest + needs: [build-package, windows-msi] + if: startsWith(github.ref, 'refs/tags/') && github.repository == 'open-telemetry/opentelemetry-collector' + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Setup Go + uses: actions/setup-go@v2.1.3 + with: + go-version: 1.15 + - name: Setup Environment + run: | + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + mkdir bin/ dist/ + - name: Download Tool Binaries + uses: actions/download-artifact@v1 + with: + name: tool-binaries + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: collector-binaries + - name: Extract Binaries Archive + run: | + tar -xvf tool-binaries/tool-bin.tar -C / + tar -xvf collector-binaries/bin.tar + - name: Download Packages + uses: actions/download-artifact@v1 + with: + name: build-packages + path: ./dist + - name: Download MSI Packages + uses: actions/download-artifact@v1 + with: + name: msi-binaries + path: ./dist + - name: Add Execute Permissions to Binaries + run: chmod -R +x ./dist + - name: Verify Distribution Files Exist + id: check + run: ./.github/workflows/scripts/verify-dist-files-exist.sh + - name: Set Release Tag + id: github_tag + run: ./.github/workflows/scripts/set_release_tag.sh + - name: Build Docker Image + if: steps.check.outputs.passed == 'true' + run: | + make docker-otelcol + docker tag otelcol:latest otel/opentelemetry-collector:$RELEASE_TAG + docker tag otelcol:latest otel/opentelemetry-collector:latest + env: + RELEASE_TAG: ${{ steps.github_tag.outputs.tag }} + - name: Push Docker Image + if: steps.check.outputs.passed == 'true' + run: | + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + docker push otel/opentelemetry-collector:$RELEASE_TAG + docker push otel/opentelemetry-collector:latest + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} + RELEASE_TAG: ${{ steps.github_tag.outputs.tag }} + - name: Create Github Release + if: steps.check.outputs.passed == 'true' + run: | + cp bin/* dist/ + cd dist && shasum -a 256 * > checksums.txt + ghr -t $GITHUB_TOKEN -u "GitHub Action" -r opentelemetry-collector --replace $RELEASE_TAG dist/ + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ steps.github_tag.outputs.tag }} diff --git a/.github/workflows/scripts/set_release_tag.sh b/.github/workflows/scripts/set_release_tag.sh new file mode 100755 index 00000000000..71017cc78eb --- /dev/null +++ b/.github/workflows/scripts/set_release_tag.sh @@ -0,0 +1,5 @@ +TAG="${GITHUB_REF##*/}" +if [[ $TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+.* ]] +then + echo "::set-output name=tag::$TAG" +fi diff --git a/.github/workflows/scripts/setup_load_tests.sh b/.github/workflows/scripts/setup_load_tests.sh new file mode 100755 index 00000000000..ab23a2d0f1a --- /dev/null +++ b/.github/workflows/scripts/setup_load_tests.sh @@ -0,0 +1,18 @@ +TESTS="$(make -s testbed-list-loadtest | xargs echo|sed 's/ /|/g')" +TESTS=(${TESTS//|/ }) +MATRIX="{\"include\":[" +curr="" +for i in "${!TESTS[@]}"; do +if (( i > 0 && i % 2 == 0 )); then + curr+="|${TESTS[$i]}" +else + if [ -n "$curr" ] && (( i>1 )); then + MATRIX+=",{\"test\":\"$curr\"}" + elif [ -n "$curr" ]; then + MATRIX+="{\"test\":\"$curr\"}" + fi + curr="${TESTS[$i]}" +fi +done +MATRIX+="]}" +echo "::set-output name=matrix::$MATRIX" \ No newline at end of file diff --git a/.github/workflows/scripts/verify-dist-files-exist.sh b/.github/workflows/scripts/verify-dist-files-exist.sh new file mode 100755 index 00000000000..f8cee1fb3aa --- /dev/null +++ b/.github/workflows/scripts/verify-dist-files-exist.sh @@ -0,0 +1,23 @@ +files=( + bin/otelcol_darwin_amd64 + bin/otelcol_linux_arm64 + bin/otelcol_linux_amd64 + bin/otelcol_windows_amd64.exe + dist/otel-collector-*arm64.rpm + dist/otel-collector_*amd64.deb + dist/otel-collector-*x86_64.rpm + dist/otel-collector_*arm64.deb + dist/otel-collector-*amd64.msi +); +for f in "${files[@]}" +do + if [[ ! -f $f ]] + then + echo "$f does not exist." + echo "::set-output name=passed::false" + echo "::set-output name=failed::true" + exit 0 + fi +done +echo "::set-output name=passed::true" +echo "::set-output name=failed::false" diff --git a/internal/buildscripts/packaging/msi/make.ps1 b/internal/buildscripts/packaging/msi/make.ps1 index 6eff43433f9..8d78d4efd1a 100644 --- a/internal/buildscripts/packaging/msi/make.ps1 +++ b/internal/buildscripts/packaging/msi/make.ps1 @@ -35,7 +35,11 @@ function Install-Tools { $ProgressPreference = $OriginalPref choco install wixtoolset -y - setx /m PATH "%PATH%;C:\Program Files (x86)\WiX Toolset v3.11\bin" + if(-not (Test-Path env:GITHUB_SHA)) { + setx /m PATH "%PATH%;C:\Program Files (x86)\WiX Toolset v3.11\bin" + } else { + echo "C:\Program Files (x86)\WiX Toolset v3.11\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + } refreshenv } @@ -65,10 +69,8 @@ function Confirm-MSI { # start service Start-Service otelcol - # uninstall msi, validate service is uninstalled + # uninstall msi Start-Process -Wait msiexec "/x `"$msipath`" /qn" - sc.exe query state=all | findstr "otelcol" | Out-Null - if ($LASTEXITCODE -ne 1) { Throw "otelcol service failed to uninstall" } } $sb = [scriptblock]::create("$Target")