From 317174aa89dcb49b4f388f8353c47912f4e6f5b7 Mon Sep 17 00:00:00 2001 From: Moritz Wiesinger Date: Wed, 24 Jul 2024 13:45:27 +0200 Subject: [PATCH] move package tests from contrib repo (#1) --- .github/workflows/base-ci-goreleaser.yaml | 9 +- .github/workflows/base-release.yaml | 2 +- .github/workflows/ci-goreleaser-contrib.yaml | 8 ++ .github/workflows/ci-goreleaser-core.yaml | 8 ++ .github/workflows/package-tests.yaml | 30 ++++++ .github/workflows/shellcheck.yml | 2 + .goreleaser.yaml | 1 + cmd/goreleaser/internal/configure.go | 5 +- .../otelcol-contrib/.goreleaser.yaml | 1 + distributions/otelcol-k8s/.goreleaser.yaml | 1 + distributions/otelcol/.goreleaser.yaml | 1 + go.mod | 4 +- go.sum | 8 +- scripts/package-tests/Dockerfile.test.deb | 14 +++ scripts/package-tests/Dockerfile.test.rpm | 9 ++ scripts/package-tests/README.md | 26 +++++ scripts/package-tests/common.sh | 44 +++++++++ scripts/package-tests/package-tests.sh | 99 +++++++++++++++++++ 18 files changed, 262 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/package-tests.yaml create mode 100644 scripts/package-tests/Dockerfile.test.deb create mode 100644 scripts/package-tests/Dockerfile.test.rpm create mode 100644 scripts/package-tests/README.md create mode 100755 scripts/package-tests/common.sh create mode 100755 scripts/package-tests/package-tests.sh diff --git a/.github/workflows/base-ci-goreleaser.yaml b/.github/workflows/base-ci-goreleaser.yaml index 3ebb8dc9..a6daa5d4 100644 --- a/.github/workflows/base-ci-goreleaser.yaml +++ b/.github/workflows/base-ci-goreleaser.yaml @@ -46,7 +46,7 @@ jobs: platforms: arm64,ppc64le,linux/arm/v7,s390x - name: Setup wixl # Required to build MSI packages for Windows - if: ${{ matrix.GOOS }} == 'windows' && (${{ matrix.GOARCH }} == '386' || ${{ matrix.GOARCH }} == 'amd64') + if: ${{ matrix.GOOS == 'windows' && ( matrix.GOARCH == '386' || matrix.GOARCH == 'amd64') }} run: | sudo apt-get update sudo apt-get install -y wixl @@ -74,3 +74,10 @@ jobs: GOOS: ${{ matrix.GOOS }} GOARCH: ${{ matrix.GOARCH }} GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + + - name: Upload linux service packages + if: ${{ matrix.GOOS == 'linux' && matrix.GOARCH == 'amd64' }} + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + with: + name: linux-packages + path: distributions/${{ inputs.distribution }}/dist/linux_amd64/* diff --git a/.github/workflows/base-release.yaml b/.github/workflows/base-release.yaml index 62affd0f..741d3374 100644 --- a/.github/workflows/base-release.yaml +++ b/.github/workflows/base-release.yaml @@ -55,7 +55,7 @@ jobs: check-latest: true - name: Setup wixl # Required to build MSI packages for Windows - if: ${{ matrix.GOOS }} == 'windows' && (${{ matrix.GOARCH }} == '386' || ${{ matrix.GOARCH }} == 'amd64') + if: ${{ matrix.GOOS == 'windows' && ( matrix.GOARCH == '386' || matrix.GOARCH == 'amd64') }} run: | sudo apt-get update sudo apt-get install -y wixl diff --git a/.github/workflows/ci-goreleaser-contrib.yaml b/.github/workflows/ci-goreleaser-contrib.yaml index 9d01d015..0a954c84 100644 --- a/.github/workflows/ci-goreleaser-contrib.yaml +++ b/.github/workflows/ci-goreleaser-contrib.yaml @@ -31,3 +31,11 @@ jobs: goos: '[ "linux", "windows", "darwin" ]' goarch: '[ "386", "amd64", "arm64", "ppc64le", "arm", "s390x" ]' secrets: inherit + + package-tests: + name: Package tests + needs: check-goreleaser + uses: ./.github/workflows/package-test.yaml + with: + distribution: otelcol-contrib + type: '[ "deb", "rpm" ]' diff --git a/.github/workflows/ci-goreleaser-core.yaml b/.github/workflows/ci-goreleaser-core.yaml index 5b447676..04b615a7 100644 --- a/.github/workflows/ci-goreleaser-core.yaml +++ b/.github/workflows/ci-goreleaser-core.yaml @@ -32,3 +32,11 @@ jobs: goos: '[ "linux", "windows", "darwin" ]' goarch: '[ "386", "amd64", "arm64", "ppc64le", "arm", "s390x" ]' secrets: inherit + + package-tests: + name: Package tests + needs: check-goreleaser + uses: ./.github/workflows/package-test.yaml + with: + distribution: otelcol + type: '[ "deb", "rpm" ]' diff --git a/.github/workflows/package-tests.yaml b/.github/workflows/package-tests.yaml new file mode 100644 index 00000000..90429064 --- /dev/null +++ b/.github/workflows/package-tests.yaml @@ -0,0 +1,30 @@ +name: Package Tests + +on: + workflow_call: + inputs: + type: + required: true + type: string + distribution: + required: true + type: string + +jobs: + package-tests: + name: Package Tests + runs-on: ubuntu-latest + strategy: + matrix: + type: ${{ fromJSON(inputs.type) }} + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Download built artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: linux-packages + + - name: Test ${{ matrix.type }} package + run: ./scripts/package-tests/package-tests.sh ./otelcol*-SNAPSHOT-*_linux_amd64.${{ matrix.type }} ${{ inputs.distribution }} diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 845cb6d7..65794f52 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -13,3 +13,5 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Run ShellCheck uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0 + env: + SHELLCHECK_OPTS: -x diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 0d0c3d3b..8a776314 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,3 +1,4 @@ +version: 2 release: header: | Check the [v{{.Version}} contrib changelog](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tags/v{{.Version}}) and the [v{{.Version}} core changelog](https://github.com/open-telemetry/opentelemetry-collector/releases/tags/v{{.Version}}) for changelogs on specific components. diff --git a/cmd/goreleaser/internal/configure.go b/cmd/goreleaser/internal/configure.go index 21cc2e46..4afdfceb 100644 --- a/cmd/goreleaser/internal/configure.go +++ b/cmd/goreleaser/internal/configure.go @@ -24,7 +24,7 @@ import ( "path" "strings" - "github.com/goreleaser/goreleaser-pro/pkg/config" + "github.com/goreleaser/goreleaser-pro/v2/pkg/config" ) const ArmArch = "arm" @@ -51,6 +51,7 @@ func Generate(dist string) config.Project { Signs: Sign(), DockerSigns: DockerSigns(), SBOMs: SBOM(), + Version: 2, } } @@ -112,7 +113,7 @@ func WinPackages(dist string) []config.MSI { // https://goreleaser.com/customization/msi/ func WinPackage(dist string) config.MSI { return config.MSI{ - ID: dist, + ID: dist, Name: fmt.Sprintf("%s_{{ .Version }}_{{ .Os }}_{{ .MsiArch }}", dist), WXS: "windows-installer.wxs", Files: []string{ diff --git a/distributions/otelcol-contrib/.goreleaser.yaml b/distributions/otelcol-contrib/.goreleaser.yaml index 477c2e64..ee792632 100644 --- a/distributions/otelcol-contrib/.goreleaser.yaml +++ b/distributions/otelcol-contrib/.goreleaser.yaml @@ -1,5 +1,6 @@ partial: by: target +version: 2 project_name: opentelemetry-collector-releases env: - COSIGN_YES=true diff --git a/distributions/otelcol-k8s/.goreleaser.yaml b/distributions/otelcol-k8s/.goreleaser.yaml index f7d91cc2..8e6aa0f0 100644 --- a/distributions/otelcol-k8s/.goreleaser.yaml +++ b/distributions/otelcol-k8s/.goreleaser.yaml @@ -1,5 +1,6 @@ partial: by: target +version: 2 project_name: opentelemetry-collector-releases env: - COSIGN_YES=true diff --git a/distributions/otelcol/.goreleaser.yaml b/distributions/otelcol/.goreleaser.yaml index fb7ff6fc..4301586c 100644 --- a/distributions/otelcol/.goreleaser.yaml +++ b/distributions/otelcol/.goreleaser.yaml @@ -1,5 +1,6 @@ partial: by: target +version: 2 project_name: opentelemetry-collector-releases env: - COSIGN_YES=true diff --git a/go.mod b/go.mod index 1d80af6b..0cb7003d 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/open-telemetry/opentelemetry-collector-releases go 1.22 require ( - github.com/goreleaser/goreleaser-pro v1.25.1-pro + github.com/goreleaser/goreleaser-pro/v2 v2.1.0-pro gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/kr/pretty v0.3.1 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 5cbf6ec3..0ff890e8 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/goreleaser/goreleaser-pro v1.25.1-pro h1:NOoSx96oAK0zNA1+hiL0p6pY1DWL101kwPmpmkiExXk= -github.com/goreleaser/goreleaser-pro v1.25.1-pro/go.mod h1:7q9HURJC4ZYBT9VyX3XlqjK0kwe5QbG/VIUAJSP3CKc= +github.com/goreleaser/goreleaser-pro/v2 v2.1.0-pro h1:9SYFXdnxoLB/8RRazytwFjKg/+394hjVplm7YLcXpQ8= +github.com/goreleaser/goreleaser-pro/v2 v2.1.0-pro/go.mod h1:GA7Uzk7qKA3efeDmgfWwcMTrDJe+V7D6H5RMqXlFvuc= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -10,8 +10,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/scripts/package-tests/Dockerfile.test.deb b/scripts/package-tests/Dockerfile.test.deb new file mode 100644 index 00000000..56d482f9 --- /dev/null +++ b/scripts/package-tests/Dockerfile.test.deb @@ -0,0 +1,14 @@ +# A debian11 image with systemd enabled. Must be run with: +# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags +FROM debian:12 + +ENV container docker +ENV DEBIAN_FRONTEND noninteractive + +# Enable systemd. +RUN apt-get update ; \ + apt-get install -y systemd systemd-sysv procps; \ + apt-get clean ; \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +CMD ["/lib/systemd/systemd"] diff --git a/scripts/package-tests/Dockerfile.test.rpm b/scripts/package-tests/Dockerfile.test.rpm new file mode 100644 index 00000000..8429e850 --- /dev/null +++ b/scripts/package-tests/Dockerfile.test.rpm @@ -0,0 +1,9 @@ +# A rockylinux8.5 image with systemd enabled. Must be run with: +# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags +FROM rockylinux:9.3 + +ENV container docker + +RUN dnf install -y initscripts + +CMD ["/usr/sbin/init"] diff --git a/scripts/package-tests/README.md b/scripts/package-tests/README.md new file mode 100644 index 00000000..fdb4719b --- /dev/null +++ b/scripts/package-tests/README.md @@ -0,0 +1,26 @@ +# Build and test deb/rpm/apk packages + +## Prerequisites + +Tools: + +- [Go](https://go.dev/) +- [GoReleaser](https://goreleaser.com/) +- [Podman](https://podman.io/) +- make + +## How to build and test + +To build the collector linux packages, a few steps are required: + +- Run `make generate` to (re-)generate sources and goreleaser files +- Go to the distribution folder that you want to build (under the `distributions` folder) +- Run `goreleaser release --snapshot --clean --skip sbom,sign,archive,docker` + - This will build a full release with all architectures and packaging types into the `dist` folder inside your + current folder. (We can skip many parts of the release build that we don't need) + - If you run into `unmarshal` errors, start to remove the parts that goreleaser complains about. This likely happens + due to a missing goreleaser pro license and therefore feature that you can't use. +- Go back to the root of the repo +- To start the package tests, + run: `./scripts/package-tests/package-tests.sh ./distributions//dist/_*-SNAPSHOT-*_linux_amd64. ` + diff --git a/scripts/package-tests/common.sh b/scripts/package-tests/common.sh new file mode 100755 index 00000000..18c859e2 --- /dev/null +++ b/scripts/package-tests/common.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +podman_cp() { + local container="$1" + local src="$2" + local dest="$3" + local dest_dir + dest_dir="$( dirname "$dest" )" + + echo "Copying $src to $container:$dest ..." + podman exec "$container" mkdir -p "$dest_dir" + podman cp "$src" "$container":"$dest" +} + +install_pkg() { + local container="$1" + local pkg_path="$2" + local pkg_base + pkg_base=$( basename "$pkg_path" ) + + echo "Installing $pkg_base ..." + podman_cp "$container" "$pkg_path" /tmp/"$pkg_base" + if [[ "${pkg_base##*.}" = "deb" ]]; then + podman exec "$container" dpkg -i /tmp/"$pkg_base" + else + podman exec "$container" rpm -ivh /tmp/"$pkg_base" + fi +} + +uninstall_pkg() { + local container="$1" + local pkg_type="$2" + local pkg_name="$3" + + echo "Uninstalling $pkg_name ..." + if [[ "$pkg_type" = "deb" ]]; then + podman exec "$container" dpkg -r "$pkg_name" + else + podman exec "$container" rpm -e "$pkg_name" + fi +} diff --git a/scripts/package-tests/package-tests.sh b/scripts/package-tests/package-tests.sh new file mode 100755 index 00000000..543f626f --- /dev/null +++ b/scripts/package-tests/package-tests.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +set -euov pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +REPO_DIR="$( cd "$SCRIPT_DIR/../../../../" && pwd )" +export REPO_DIR +PKG_PATH="${1:-}" +DISTRO="${2:-}" + +SERVICE_NAME=$DISTRO +PROCESS_NAME=$DISTRO + +# shellcheck source=scripts/package-tests/common.sh +source "$SCRIPT_DIR"/common.sh + +if [[ -z "$PKG_PATH" ]]; then + echo "usage: ${BASH_SOURCE[0]} DEB_OR_RPM_PATH" >&2 + exit 1 +fi + +if [[ ! -f "$PKG_PATH" ]]; then + echo "$PKG_PATH not found!" >&2 + exit 1 +fi + + +pkg_base="$( basename "$PKG_PATH" )" +pkg_type="${pkg_base##*.}" +if [[ ! "$pkg_type" =~ ^(deb|rpm)$ ]]; then + echo "$PKG_PATH not supported!" >&2 + exit 1 +fi +image_name="otelcontribcol-$pkg_type-test" +container_name="$image_name" +container_exec="podman exec $container_name" + +trap 'podman rm -fv $container_name >/dev/null 2>&1 || true' EXIT + +podman build -t "$image_name" -f "$SCRIPT_DIR/Dockerfile.test.$pkg_type" "$SCRIPT_DIR" +podman rm -fv "$container_name" >/dev/null 2>&1 || true + +# test install +CRUN_VER='1.14.4' +mkdir -p "${HOME}/.local/bin" +curl -L "https://github.com/containers/crun/releases/download/${CRUN_VER}/crun-${CRUN_VER}-linux-amd64" -o "${HOME}/.local/bin/crun" +chmod +x "${HOME}/.local/bin/crun" +mkdir -p "${HOME}/.config/containers" +cat << EOF > "${HOME}/.config/containers/containers.conf" +[engine.runtimes] +crun = [ + "${HOME}/.local/bin/crun", + "/usr/bin/crun" +] +EOF + +echo +podman run --name "$container_name" -d "$image_name" +install_pkg "$container_name" "$PKG_PATH" + +# ensure service has started and still running after 5 seconds +sleep 5 +echo "Checking $SERVICE_NAME service status ..." +$container_exec systemctl --no-pager status "$SERVICE_NAME" + +echo "Checking $PROCESS_NAME process ..." +if [ "$DISTRO" = "otelcol" ]; then + $container_exec pgrep -a -u otel "$PROCESS_NAME" +else + $container_exec pgrep -a -u otelcol-contrib "$PROCESS_NAME" +fi + +# test uninstall +echo +uninstall_pkg "$container_name" "$pkg_type" "$DISTRO" + +echo "Checking $SERVICE_NAME service status after uninstall ..." +if $container_exec systemctl --no-pager status "$SERVICE_NAME"; then + echo "$SERVICE_NAME service still running after uninstall" >&2 + exit 1 +fi +echo "$SERVICE_NAME service successfully stopped after uninstall" + +echo "Checking $SERVICE_NAME service existence after uninstall ..." +if $container_exec systemctl list-unit-files --all | grep "$SERVICE_NAME"; then + echo "$SERVICE_NAME service still exists after uninstall" >&2 + exit 1 +fi +echo "$SERVICE_NAME service successfully removed after uninstall" + +echo "Checking $PROCESS_NAME process after uninstall ..." +if $container_exec pgrep "$PROCESS_NAME"; then + echo "$PROCESS_NAME process still running after uninstall" + exit 1 +fi +echo "$PROCESS_NAME process successfully killed after uninstall"