From e31794da03b78520eef07e0e81cfb017d15e26e6 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 24 Aug 2020 11:06:23 +0200 Subject: [PATCH] Build docker images based on Red Hat UBI (#20576) Add an additional docker build that builds images based on Red Hat UBI, following Red Hat requirements for certified images. Additional checks have been added to packaging tests for labels and licenses. Additional changes done to support it also in Elastic Agent images: * Home directory is prepared in a different stage (#20356). * Allow the docker image to be run with random user ids (#18873). * Explicitly select a Dockerfile and entry point template. * Add NOTICE.txt file to all agent packages. * Actually run package tests after building packages, added flag to allow root user. * Improved checks on required packages, so they are not re-built if they already are. --- CHANGELOG-developer.next.asciidoc | 1 + dev-tools/mage/dockerbuilder.go | 19 +++--- dev-tools/mage/settings.go | 1 + dev-tools/packaging/package_test.go | 54 +++++++++++++++-- dev-tools/packaging/packages.yml | 48 +++++++++++++++ .../docker/Dockerfile.elastic-agent.tmpl | 59 ++++++++++++++----- .../templates/docker/Dockerfile.tmpl | 28 ++++++++- x-pack/elastic-agent/magefile.go | 56 +++++++++++------- 8 files changed, 211 insertions(+), 55 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index bb2dcf96345..29734e17fa8 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -100,3 +100,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Added SQL helper that can be used from any Metricbeat module {pull}18955[18955] - Update Go version to 1.14.4. {pull}19753[19753] - Update Go version to 1.14.7. {pull}20508[20508] +- Add packaging for docker image based on UBI minimal 8. {pull}20576[20576] diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 2da79983775..90a99434884 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -151,19 +151,14 @@ func isDockerFile(path string) bool { } func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error { - // has specific dockerfile - dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName) - _, err := os.Stat(filepath.Join(templatesDir, dockerfile)) - if err != nil { - // specific missing fallback to generic - dockerfile = "Dockerfile.tmpl" + dockerfile := "Dockerfile.tmpl" + if f, found := b.ExtraVars["dockerfile"]; found { + dockerfile = f } - entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName) - _, err = os.Stat(filepath.Join(templatesDir, entrypoint)) - if err != nil { - // specific missing fallback to generic - entrypoint = "docker-entrypoint.tmpl" + entrypoint := "docker-entrypoint.tmpl" + if e, found := b.ExtraVars["docker_entrypoint"]; found { + entrypoint = e } type fileExpansion struct { @@ -176,7 +171,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in ".tmpl", ) path := filepath.Join(templatesDir, file.source) - err = b.ExpandFile(path, target, data) + err := b.ExpandFile(path, target, data) if err != nil { return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target) } diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 2473202648e..f7de61e7db0 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -91,6 +91,7 @@ var ( "repo": GetProjectRepoInfo, "title": strings.Title, "tolower": strings.ToLower, + "contains": strings.Contains, } ) diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 96173cde880..2a74e80b7f4 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -48,13 +48,15 @@ const ( ) var ( - configFilePattern = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml$`) + configFilePattern = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml|elastic-agent\.yml$`) manifestFilePattern = regexp.MustCompile(`manifest.yml`) modulesDirPattern = regexp.MustCompile(`module/.+`) modulesDDirPattern = regexp.MustCompile(`modules.d/$`) modulesDFilePattern = regexp.MustCompile(`modules.d/.+`) monitorsDFilePattern = regexp.MustCompile(`monitors.d/.+`) systemdUnitFilePattern = regexp.MustCompile(`/lib/systemd/system/.*\.service`) + + licenseFiles = []string{"LICENSE.txt", "NOTICE.txt"} ) var ( @@ -122,6 +124,7 @@ func checkRPM(t *testing.T, file string) { checkModulesPresent(t, "/usr/share", p) checkModulesDPresent(t, "/etc/", p) checkMonitorsDPresent(t, "/etc", p) + checkLicensesPresent(t, "/usr/share", p) checkSystemdUnitPermissions(t, p) ensureNoBuildIDLinks(t, p) } @@ -141,6 +144,7 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) { checkModulesPresent(t, "./usr/share", p) checkModulesDPresent(t, "./etc/", p) checkMonitorsDPresent(t, "./etc/", p) + checkLicensesPresent(t, "./usr/share", p) checkModulesOwner(t, p, true) checkModulesPermissions(t, p) checkSystemdUnitPermissions(t, p) @@ -160,6 +164,7 @@ func checkTar(t *testing.T, file string) { checkModulesDPresent(t, "", p) checkModulesPermissions(t, p) checkModulesOwner(t, p, true) + checkLicensesPresent(t, "", p) } func checkZip(t *testing.T, file string) { @@ -174,6 +179,7 @@ func checkZip(t *testing.T, file string) { checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) checkModulesPermissions(t, p) + checkLicensesPresent(t, "", p) } func checkDocker(t *testing.T, file string) { @@ -190,6 +196,7 @@ func checkDocker(t *testing.T, file string) { checkManifestPermissionsWithMode(t, p, os.FileMode(0640)) checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) + checkLicensesPresent(t, "licenses/", p) } // Verify that the main configuration file is installed with a 0600 file mode. @@ -373,6 +380,22 @@ func checkMonitors(t *testing.T, name, prefix string, r *regexp.Regexp, p *packa }) } +func checkLicensesPresent(t *testing.T, prefix string, p *packageFile) { + for _, licenseFile := range licenseFiles { + t.Run("License file "+licenseFile, func(t *testing.T) { + for _, entry := range p.Contents { + if strings.HasPrefix(entry.File, prefix) && strings.HasSuffix(entry.File, "/"+licenseFile) { + return + } + } + if prefix != "" { + t.Fatalf("not found under %s", prefix) + } + t.Fatal("not found") + }) + } +} + func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) { expectedMode := os.FileMode(0755) @@ -402,7 +425,8 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri if vendor != "Elastic" { return } - t.Run(fmt.Sprintf("%s labels", p.Name), func(t *testing.T) { + + t.Run(fmt.Sprintf("%s license labels", p.Name), func(t *testing.T) { expectedLicense := "Elastic License" ossPrefix := strings.Join([]string{ info.Config.Labels["org.label-schema.name"], @@ -412,8 +436,24 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri if strings.HasPrefix(filepath.Base(file), ossPrefix) { expectedLicense = "ASL 2.0" } - if license, present := info.Config.Labels["license"]; !present || license != expectedLicense { - t.Errorf("unexpected license label: %s", license) + licenseLabels := []string{ + "license", + "org.label-schema.license", + } + for _, licenseLabel := range licenseLabels { + if license, present := info.Config.Labels[licenseLabel]; !present || license != expectedLicense { + t.Errorf("unexpected license label %s: %s", licenseLabel, license) + } + } + }) + + t.Run(fmt.Sprintf("%s required labels", p.Name), func(t *testing.T) { + // From https://redhat-connect.gitbook.io/partner-guide-for-red-hat-openshift-and-container/program-on-boarding/technical-prerequisites + requiredLabels := []string{"name", "vendor", "version", "release", "summary", "description"} + for _, label := range requiredLabels { + if value, present := info.Config.Labels[label]; !present || value == "" { + t.Errorf("missing required label %s", label) + } } }) } @@ -657,6 +697,12 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { if strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint { p.Contents[name] = entry } + // Add also licenses + for _, licenseFile := range licenseFiles { + if strings.Contains(name, licenseFile) { + p.Contents[name] = entry + } + } } } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index ea3ddfe76e0..f4261945233 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -28,6 +28,9 @@ shared: /usr/share/{{.BeatName}}/LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + /usr/share/{{.BeatName}}/NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 /usr/share/{{.BeatName}}/README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -117,6 +120,9 @@ shared: /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -186,6 +192,9 @@ shared: LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -307,6 +316,9 @@ shared: <<: *agent_binary_spec extra_vars: from: 'centos:7' + buildFrom: 'centos:7' + dockerfile: 'Dockerfile.elastic-agent.tmpl' + docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl' user: 'root' linux_capabilities: '' files: @@ -460,6 +472,7 @@ shared: <<: *binary_spec extra_vars: from: 'centos:7' + buildFrom: 'centos:7' user: '{{ .BeatName }}' linux_capabilities: '' files: @@ -468,6 +481,11 @@ shared: mode: 0600 config: true + - &docker_ubi_spec + extra_vars: + image_name: '{{.BeatName}}-ubi8' + from: 'registry.access.redhat.com/ubi8/ubi-minimal' + - &elastic_docker_spec extra_vars: repository: 'docker.elastic.co/beats' @@ -637,6 +655,14 @@ specs: <<: *elastic_docker_spec <<: *elastic_license_for_binaries + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + # Elastic Beat with Elastic License and binary taken the current directory. elastic_beat_xpack_reduced: ### @@ -721,6 +747,17 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + # Elastic Beat with Elastic License and binary taken from the x-pack dir. elastic_beat_agent_binaries: ### @@ -782,6 +819,17 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + types: [docker] + spec: + <<: *agent_docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + # Elastic Beat with Elastic License and binary taken from the x-pack dir. elastic_beat_agent_demo_binaries: diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index a38ea8701a3..a7242baa73b 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -2,11 +2,36 @@ {{- $beatBinary := printf "%s/%s" $beatHome .BeatName }} {{- $repoInfo := repo }} +# Prepare home in a different stage to avoid creating additional layers on +# the final image because of permission changes. +FROM {{ .buildFrom }} AS home + +COPY beat {{ $beatHome }} + +RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ + chown -R root:root {{ $beatHome }} && \ + find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ + find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ + chmod 0750 {{ $beatBinary }} && \ +{{- if .linux_capabilities }} + setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ +{{- end }} +{{- range $i, $modulesd := .ModulesDirs }} + chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ +{{- end }} + chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs + FROM {{ .from }} +{{- if contains .from "ubi-minimal" }} +RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/local/bin/jq && \ + chmod +x /usr/local/bin/jq +{{- else }} # Installing jq needs to be installed after epel-release and cannot be in the same yum install command. RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && yum install --setopt=tsflags=nodocs -y epel-release && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) RUN for iter in {1..10}; do yum update -y && yum install -y jq && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ @@ -18,33 +43,37 @@ LABEL \ org.label-schema.url="{{ .BeatURL }}" \ org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ org.label-schema.vcs-ref="{{ commit }}" \ + io.k8s.description="{{ .BeatDescription }}" \ + io.k8s.display-name="{{ .BeatName | title }} image" \ + org.opencontainers.image.created="{{ date }}" \ + org.opencontainers.image.licenses="{{ .License }}" \ + org.opencontainers.image.title="{{ .BeatName | title }}" \ + org.opencontainers.image.vendor="{{ .BeatVendor }}" \ + name="{{ .BeatName }}" \ + maintainer="infra@elastic.co" \ + vendor="{{ .BeatVendor }}" \ + version="{{ beat_version }}" \ + release="1" \ + url="{{ .BeatURL }}" \ + summary="{{ .BeatName }}" \ license="{{ .License }}" \ description="{{ .BeatDescription }}" ENV ELASTIC_CONTAINER "true" ENV PATH={{ $beatHome }}:$PATH -COPY beat {{ $beatHome }} COPY docker-entrypoint /usr/local/bin/docker-entrypoint RUN chmod 755 /usr/local/bin/docker-entrypoint -RUN groupadd --gid 1000 {{ .BeatName }} +COPY --from=home {{ $beatHome }} {{ $beatHome }} -RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ - chown -R root:{{ .BeatName }} {{ $beatHome }} && \ - find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ - find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ - chmod 0750 {{ $beatBinary }} && \ -{{- if .linux_capabilities }} - setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ -{{- end }} -{{- range $i, $modulesd := .ModulesDirs }} - chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ -{{- end }} - chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs +RUN mkdir /licenses +COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses +COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses {{- if ne .user "root" }} -RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }} +RUN groupadd --gid 1000 {{ .BeatName }} +RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} {{- end }} USER {{ .user }} diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 9eac254f822..8b7eb80745c 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -4,7 +4,7 @@ # Prepare home in a different stage to avoid creating additional layers on # the final image because of permission changes. -FROM {{ .from }} AS home +FROM {{ .buildFrom }} AS home COPY beat {{ $beatHome }} @@ -23,8 +23,13 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ FROM {{ .from }} -RUN yum -y --setopt=tsflags=nodocs update && \ - yum clean all +{{- if contains .from "ubi-minimal" }} +RUN microdnf -y --setopt=tsflags=nodocs update && \ + microdnf install shadow-utils && \ + microdnf clean all +{{- else }} +RUN yum -y --setopt=tsflags=nodocs update && yum clean all +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ @@ -36,6 +41,19 @@ LABEL \ org.label-schema.url="{{ .BeatURL }}" \ org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ org.label-schema.vcs-ref="{{ commit }}" \ + io.k8s.description="{{ .BeatDescription }}" \ + io.k8s.display-name="{{ .BeatName | title }} image" \ + org.opencontainers.image.created="{{ date }}" \ + org.opencontainers.image.licenses="{{ .License }}" \ + org.opencontainers.image.title="{{ .BeatName | title }}" \ + org.opencontainers.image.vendor="{{ .BeatVendor }}" \ + name="{{ .BeatName }}" \ + maintainer="infra@elastic.co" \ + vendor="{{ .BeatVendor }}" \ + version="{{ beat_version }}" \ + release="1" \ + url="{{ .BeatURL }}" \ + summary="{{ .BeatName }}" \ license="{{ .License }}" \ description="{{ .BeatDescription }}" @@ -47,6 +65,10 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint COPY --from=home {{ $beatHome }} {{ $beatHome }} +RUN mkdir /licenses +COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses +COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses + {{- if ne .user "root" }} RUN groupadd --gid 1000 {{ .BeatName }} RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 2d634d6fce1..5735470b54c 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -272,14 +272,30 @@ func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() - packageAgent([]string{ - "darwin-x86_64.tar.gz", - "linux-x86.tar.gz", - "linux-x86_64.tar.gz", - "windows-x86.zip", - "windows-x86_64.zip", - "linux-arm64.tar.gz", - }, devtools.UseElasticAgentPackaging) + platformPackages := []struct { + platform string + packages string + }{ + {"darwin/amd64", "darwin-x86_64.tar.gz"}, + {"linux/386", "linux-x86.tar.gz"}, + {"linux/amd64", "linux-x86_64.tar.gz"}, + {"linux/arm64", "linux-arm64.tar.gz"}, + {"windows/386", "windows-x86.zip"}, + {"windows/amd64", "windows-x86_64.zip"}, + } + + var requiredPackages []string + for _, p := range platformPackages { + if _, enabled := devtools.Platforms.Get(p.platform); enabled { + requiredPackages = append(requiredPackages, p.packages) + } + } + + if len(requiredPackages) == 0 { + panic("elastic-agent package is expected to include other packages") + } + + packageAgent(requiredPackages, devtools.UseElasticAgentPackaging) } func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool { @@ -297,7 +313,7 @@ func requiredPackagesPresent(basePath, beat, version string, requiredPackages [] // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return devtools.TestPackages() + return devtools.TestPackages(devtools.WithRootUserContainer()) } // RunGo runs go command and output the feedback to the stdout and the stderr. @@ -531,18 +547,16 @@ func packageAgent(requiredPackages []string, packagingFn func()) { panic(err) } - if requiredPackagesPresent(pwd, b, version, requiredPackages) { - continue - } + if !requiredPackagesPresent(pwd, b, version, requiredPackages) { + cmd := exec.Command("mage", "package") + cmd.Dir = pwd + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") - cmd := exec.Command("mage", "package") - cmd.Dir = pwd - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") - - if err := cmd.Run(); err != nil { - panic(err) + if err := cmd.Run(); err != nil { + panic(err) + } } // copy to new drop @@ -558,7 +572,7 @@ func packageAgent(requiredPackages []string, packagingFn func()) { mg.Deps(Update) mg.Deps(CrossBuild, CrossBuildGoDaemon) - mg.SerialDeps(devtools.Package) + mg.SerialDeps(devtools.Package, TestPackages) } func copyAll(from, to string) error {