Skip to content

Commit

Permalink
test: run linux package tests nightly against latest contrib main (op…
Browse files Browse the repository at this point in the history
…en-telemetry#666)

* run package tests nightly against latest main

Signed-off-by: Moritz Wiesinger <[email protected]>

* try again

Signed-off-by: Moritz Wiesinger <[email protected]>

* add mkdir for all distros

Signed-off-by: Moritz Wiesinger <[email protected]>

* start with adding goreleaser generation code

Signed-off-by: Moritz Wiesinger <[email protected]>

* generate rest of goreleaser config

Signed-off-by: Moritz Wiesinger <[email protected]>

* make generate-goreleaser - only reorders things

Signed-off-by: Moritz Wiesinger <[email protected]>

* polishing

Signed-off-by: Moritz Wiesinger <[email protected]>

* trigger build

* rename files

Signed-off-by: Moritz Wiesinger <[email protected]>

* rename agai

Signed-off-by: Moritz Wiesinger <[email protected]>

* remove push trigger

Signed-off-by: Moritz Wiesinger <[email protected]>

* rename jobs, use pinned ubuntu version

Signed-off-by: Moritz Wiesinger <[email protected]>

* remove curly braces, refactoring

Signed-off-by: Moritz Wiesinger <[email protected]>

* minor polishing and refactoring

Signed-off-by: Moritz Wiesinger <[email protected]>

* remove trigger on PR

Signed-off-by: Moritz Wiesinger <[email protected]>

* minor refactoring

Signed-off-by: Moritz Wiesinger <[email protected]>

* renaming

Signed-off-by: Moritz Wiesinger <[email protected]>

* minor refactoring

Signed-off-by: Moritz Wiesinger <[email protected]>

* revert beauty changes for separate PR

Signed-off-by: Moritz Wiesinger <[email protected]>

* rename input variable and add description

Signed-off-by: Moritz Wiesinger <[email protected]>

* rename build step flag

Signed-off-by: Moritz Wiesinger <[email protected]>

* add conditional

Signed-off-by: Moritz Wiesinger <[email protected]>

* better conditionals and namin

Signed-off-by: Moritz Wiesinger <[email protected]>

* fix workflow reference

Signed-off-by: Moritz Wiesinger <[email protected]>

* more naming fixes

Signed-off-by: Moritz Wiesinger <[email protected]>

* enable package test workflow again for debugging

Signed-off-by: Moritz Wiesinger <[email protected]>

* fix more naming

Signed-off-by: Moritz Wiesinger <[email protected]>

* refactoring

Signed-off-by: Moritz Wiesinger <[email protected]>

* use target template variable

Signed-off-by: Moritz Wiesinger <[email protected]>

* add go386 variable to filenames

Signed-off-by: Moritz Wiesinger <[email protected]>

* fix template variable

Signed-off-by: Moritz Wiesinger <[email protected]>

* add goarm64 variable

Signed-off-by: Moritz Wiesinger <[email protected]>

* adapt goreleaser generator

Signed-off-by: Moritz Wiesinger <[email protected]>

* line breaks

Signed-off-by: Moritz Wiesinger <[email protected]>

* prefix nightly steps

Signed-off-by: Moritz Wiesinger <[email protected]>

* add missing quotes

Signed-off-by: Moritz Wiesinger <[email protected]>

* try to use target template var again

Signed-off-by: Moritz Wiesinger <[email protected]>

* try to use target env var more

Signed-off-by: Moritz Wiesinger <[email protected]>

* revert name template field

Signed-off-by: Moritz Wiesinger <[email protected]>

* remove debug change

Signed-off-by: Moritz Wiesinger <[email protected]>

---------

Signed-off-by: Moritz Wiesinger <[email protected]>
  • Loading branch information
mowies authored Dec 2, 2024
1 parent b4235b4 commit 1271690
Show file tree
Hide file tree
Showing 12 changed files with 233 additions and 58 deletions.
73 changes: 72 additions & 1 deletion .github/workflows/base-ci-goreleaser.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ on:
goarch:
required: true
type: string
nightly:
required: false
type: boolean
default: false
description: "Set to true to fetch latest otelcol-contrib main branch version instead of building the version in this repo"

env:
# renovate: datasource=github-tags depName=goreleaser-pro packageName=goreleaser/goreleaser-pro
Expand Down Expand Up @@ -64,9 +69,75 @@ jobs:
go-version: '1.23'
check-latest: true

- name: Generate the sources
- name: Create artifacts directory to store build artifacts
if: inputs.distribution == 'otelcol-contrib'
run: mkdir -p distributions/otelcol-contrib/artifacts

- name: "[Nightly] Get latest finished run ID from contrib repo build-and-test"
id: get-run-id
if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
env:
GH_TOKEN: ${{ github.token }}
run: |
run_id=$(gh run list \
--branch main \
--workflow build-and-test \
--repo open-telemetry/opentelemetry-collector-contrib \
--limit 1 \
--status success \
--json databaseId \
--jq '.[0].databaseId' \
)
echo "Found run ID: $run_id"
echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
- name: "[Nightly] Create sub-directory for otelcol-contrib nightly build"
if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
run: mkdir -p distributions/otelcol-contrib/artifacts/otelcol-contrib_linux_amd64_v1

- name: "[Nightly] Download built otelcol-contrib artifact from contrib repo"
if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: collector-binaries-linux-amd64
repository: open-telemetry/opentelemetry-collector-contrib
github-token: ${{ secrets.GITHUB_TOKEN }}
run-id: ${{ steps.get-run-id.outputs.run_id }}

- name: "[Nightly] Move downloaded artifact"
if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
run: mv otelcontribcol_linux_amd64 distributions/otelcol-contrib/artifacts/otelcol-contrib_linux_amd64_v1/otelcol-contrib

- name: Generate the sources for ${{ inputs.distribution }}
if: inputs.nightly != true
env:
DISTRIBUTIONS: ${{ inputs.distribution }}
run: make generate-sources

# otelcol-contrib is built in a separate stage
- name: Build ${{ inputs.distribution }}
if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true
uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
with:
distribution: goreleaser-pro
version: ${{ env.GORELEASER_PRO_VERSION }}
workdir: distributions/otelcol-contrib
args: --snapshot --clean --timeout 2h --split --config .goreleaser-build.yaml
env:
GOOS: ${{ matrix.GOOS }}
GOARCH: ${{ matrix.GOARCH }}
GOARM: 7 # Default is 6
GOAMD64: v1
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

- name: Move built artifacts
if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true
run: mv distributions/otelcol-contrib/dist/**/* distributions/otelcol-contrib/artifacts/

- name: Show built or downloaded content
if: inputs.distribution == 'otelcol-contrib'
run: ls -laR distributions/otelcol-contrib/artifacts

- name: Run GoReleaser for ${{ inputs.distribution }}
uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
with:
Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion .github/workflows/ci-goreleaser-contrib.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
package-tests:
name: Package tests
needs: check-goreleaser
uses: ./.github/workflows/package-tests.yaml
uses: ./.github/workflows/base-package-tests.yaml
with:
distribution: otelcol-contrib
type: '[ "deb", "rpm" ]'
2 changes: 1 addition & 1 deletion .github/workflows/ci-goreleaser-core.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
package-tests:
name: Package tests
needs: check-goreleaser
uses: ./.github/workflows/package-tests.yaml
uses: ./.github/workflows/base-package-tests.yaml
with:
distribution: otelcol
type: '[ "deb", "rpm" ]'
24 changes: 24 additions & 0 deletions .github/workflows/package-test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: Package Tests - Contrib

on:
schedule:
- cron: "0 2 * * *" # every day at 2am UTC

jobs:
check-goreleaser:
name: Build - Contrib - GoReleaser
uses: ./.github/workflows/base-ci-goreleaser.yaml
with:
distribution: otelcol-contrib
goos: '[ "linux" ]'
goarch: '[ "amd64" ]'
nightly: true
secrets: inherit

package-tests:
name: Linux Package tests
needs: check-goreleaser
uses: ./.github/workflows/base-package-tests.yaml
with:
distribution: otelcol-contrib
type: '[ "deb", "rpm" ]'
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,4 @@ delete-tags:
# Used for debug only
REMOTE?[email protected]:open-telemetry/opentelemetry-collector-releases.git
.PHONY: repeat-tags
repeat-tags: delete-tags push-tags
repeat-tags: delete-tags push-tags
119 changes: 74 additions & 45 deletions cmd/goreleaser/internal/configure.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,22 +42,32 @@ const (
var (
ImagePrefixes = []string{DockerHub, GHCR}
Architectures = []string{"386", "amd64", "arm", "arm64", "ppc64le", "s390x"}
ArmVersions = []string{"7"}
DefaultConfigDists = map[string]bool{CoreDistro: true, ContribDistro: true}
MSIWindowsDists = map[string]bool{CoreDistro: true, ContribDistro: true, OTLPDistro: true}
K8sDockerSkipArchs = map[string]bool{"arm": true, "386": true}
K8sGoos = []string{"linux"}
K8sArchs = []string{"amd64", "arm64", "ppc64le", "s390x"}
)

func Generate(dist string) config.Project {
func GenerateContribBuildOnly(dist string, buildOrRest bool) config.Project {
return config.Project{
ProjectName: "opentelemetry-collector-releases",
Builds: Builds(dist, buildOrRest),
Version: 2,
Monorepo: config.Monorepo{
TagPrefix: "v",
},
}
}

func Generate(dist string, buildOrRest bool) config.Project {
return config.Project{
ProjectName: "opentelemetry-collector-releases",
Checksum: config.Checksum{
NameTemplate: fmt.Sprintf("{{ .ProjectName }}_%v_checksums.txt", dist),
},
Env: []string{"COSIGN_YES=true"},
Builds: Builds(dist),
Builds: Builds(dist, buildOrRest),
Archives: Archives(dist),
MSI: WinPackages(dist),
NFPMs: Packages(dist),
Expand All @@ -73,37 +83,41 @@ func Generate(dist string) config.Project {
}
}

func Builds(dist string) []config.Build {
func Builds(dist string, buildOrRest bool) []config.Build {
return []config.Build{
Build(dist),
Build(dist, buildOrRest),
}
}

// Build configures a goreleaser build.
// https://goreleaser.com/customization/build/
func Build(dist string) config.Build {
var goos []string
var archs []string
var ignore []config.IgnoredBuild
var armVersions []string
func Build(dist string, buildOrRest bool) config.Build {
goos := []string{"darwin", "linux", "windows"}
archs := Architectures

if dist == ContribDistro && !buildOrRest {
// only return build config for contrib build file
return config.Build{
ID: dist,
Builder: "prebuilt",
PreBuilt: config.PreBuiltOptions{
Path: "artifacts/otelcol-contrib_{{ .Target }}" +
"/otelcol-contrib{{- if eq .Os \"windows\" }}.exe{{ end }}",
},
Goos: goos,
Goarch: archs,
Goarm: ArmVersions(dist),
Dir: "_build",
Binary: dist,
Ignore: IgnoreBuildCombinations(dist),
}
}

if dist == K8sDistro {
goos = K8sGoos
archs = K8sArchs
ignore = make([]config.IgnoredBuild, 0)
armVersions = make([]string, 0)
} else {
goos = []string{"darwin", "linux", "windows"}
archs = Architectures
ignore = []config.IgnoredBuild{
{Goos: "darwin", Goarch: "386"},
{Goos: "darwin", Goarch: "arm"},
{Goos: "darwin", Goarch: "s390x"},
{Goos: "windows", Goarch: "arm"},
{Goos: "windows", Goarch: "arm64"},
{Goos: "windows", Goarch: "s390x"},
}
armVersions = ArmVersions
}

return config.Build{
ID: dist,
Dir: "_build",
Expand All @@ -115,12 +129,33 @@ func Build(dist string) config.Build {
},
Goos: goos,
Goarch: archs,
Goarm: armVersions,
Ignore: ignore,
Goarm: ArmVersions(dist),
Ignore: IgnoreBuildCombinations(dist),
}
}

func Archives(dist string) (r []config.Archive) {
func IgnoreBuildCombinations(dist string) []config.IgnoredBuild {
if dist == K8sDistro {
return nil
}
return []config.IgnoredBuild{
{Goos: "darwin", Goarch: "386"},
{Goos: "darwin", Goarch: "arm"},
{Goos: "darwin", Goarch: "s390x"},
{Goos: "windows", Goarch: "arm"},
{Goos: "windows", Goarch: "arm64"},
{Goos: "windows", Goarch: "s390x"},
}
}

func ArmVersions(dist string) []string {
if dist == K8sDistro {
return nil
}
return []string{"7"}
}

func Archives(dist string) []config.Archive {
return []config.Archive{
Archive(dist),
}
Expand All @@ -138,7 +173,7 @@ func Archive(dist string) config.Archive {

func WinPackages(dist string) []config.MSI {
if _, ok := MSIWindowsDists[dist]; !ok {
return []config.MSI{}
return nil
}
return []config.MSI{
WinPackage(dist),
Expand All @@ -160,9 +195,9 @@ func WinPackage(dist string) config.MSI {
}
}

func Packages(dist string) (r []config.NFPM) {
func Packages(dist string) []config.NFPM {
if dist == K8sDistro {
return []config.NFPM{}
return nil
}
return []config.NFPM{
Package(dist),
Expand Down Expand Up @@ -191,21 +226,17 @@ func Package(dist string) config.NFPM {
})
}
return config.NFPM{
ID: dist,
Builds: []string{dist},
Formats: []string{"deb", "rpm"},

ID: dist,
Builds: []string{dist},
Formats: []string{"deb", "rpm"},
License: "Apache 2.0",
Description: fmt.Sprintf("OpenTelemetry Collector - %s", dist),
Maintainer: "The OpenTelemetry Collector maintainers <[email protected]>",
Overrides: map[string]config.NFPMOverridables{
"rpm": {
Dependencies: []string{
"/bin/sh",
},
Dependencies: []string{"/bin/sh"},
},
},

NFPMOverridables: config.NFPMOverridables{
PackageName: dist,
Scripts: config.NFPMScripts{
Expand All @@ -219,16 +250,14 @@ func Package(dist string) config.NFPM {
}

func DockerImages(dist string) []config.Docker {
r := make([]config.Docker, 0)
var r []config.Docker
for _, arch := range Architectures {
if dist == K8sDistro {
if _, ok := K8sDockerSkipArchs[arch]; ok {
continue
}
if dist == K8sDistro && K8sDockerSkipArchs[arch] {
continue
}
switch arch {
case ArmArch:
for _, vers := range ArmVersions {
for _, vers := range ArmVersions(dist) {
r = append(r, DockerImage(dist, arch, vers))
}
default:
Expand Down Expand Up @@ -302,7 +331,7 @@ func DockerManifest(prefix, version, dist string) config.DockerManifest {
}
switch arch {
case ArmArch:
for _, armVers := range ArmVersions {
for _, armVers := range ArmVersions(dist) {
dockerArchTag := strings.ReplaceAll(archName(arch, armVers), "/", "")
imageTemplates = append(
imageTemplates,
Expand Down
10 changes: 9 additions & 1 deletion cmd/goreleaser/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,29 @@ import (
"log"
"os"

"github.com/goreleaser/goreleaser-pro/v2/pkg/config"
"gopkg.in/yaml.v3"

"github.com/open-telemetry/opentelemetry-collector-releases/cmd/goreleaser/internal"
)

var distFlag = flag.String("d", "", "Collector distributions to build")
var contribBuildOrRestFlag = flag.Bool("generate-build-step", false, "Collector Contrib distribution only - switch between build and package config file - set to true to generate build step, false to generate package step")

func main() {
flag.Parse()

if len(*distFlag) == 0 {
log.Fatal("no distribution to build")
}
var project config.Project

project := internal.Generate(*distFlag)
if *distFlag == internal.ContribDistro && *contribBuildOrRestFlag {
// Special care needs to be taken for otelcol-contrib since it has a split setup
project = internal.GenerateContribBuildOnly(*distFlag, *contribBuildOrRestFlag)
} else {
project = internal.Generate(*distFlag, *contribBuildOrRestFlag)
}

partial := map[string]any{
"partial": map[string]any{
Expand Down
Loading

0 comments on commit 1271690

Please sign in to comment.