(fleet) downloader flavors (#31840)

.gitattributes

@@ -10,7 +10,7 @@ go.sum -diff -merge linguist-generated=true
 pkg/security/probe/constantfetch/btfhub/constants.json -diff -merge linguist-generated=true
 pkg/security/seclwin/** -diff -merge linguist-generated=true
 # Fixtures should have LF line endings because they are checked against OCI packages built on Linux
-pkg/fleet/internal/fixtures/** text=auto eol=lf
+pkg/fleet/installer/fixtures/** text=auto eol=lf
 
 # Fix `git diff` when running on the below file formats.
 # Our windows build image uses MinGit which tries to use the astextplain diff algorithm (https://git-scm.com/docs/gitattributes#_setting_the_internal_diff_algorithm).
@@ -27,26 +27,25 @@ pkg/fleet/internal/fixtures/** text=auto eol=lf
 #	textconv = astextplain
 # ```
 
-
-*.doc	diff
-*.DOC	diff
-*.docx	diff
-*.DOCX	diff
-*.docm	diff
-*.DOCM	diff
-*.dot	diff
-*.DOT	diff
-*.dotx	diff
-*.DOTX	diff
-*.dotm	diff
-*.DOTM	diff
-*.pdf	diff
-*.PDF	diff
-*.rtf	diff
-*.RTF	diff
-*.ods	diff
-*.ODS	diff
-*.odf	diff
-*.ODF	diff
-*.odt	diff
-*.ODT	diff
+*.doc diff
+*.DOC diff
+*.docx diff
+*.DOCX diff
+*.docm diff
+*.DOCM diff
+*.dot diff
+*.DOT diff
+*.dotx diff
+*.DOTX diff
+*.dotm diff
+*.DOTM diff
+*.pdf diff
+*.PDF diff
+*.rtf diff
+*.RTF diff
+*.ods diff
+*.ODS diff
+*.odf diff
+*.ODF diff
+*.odt diff
+*.ODT diff

.gitlab/package_build/installer.yml

@@ -103,7 +103,8 @@ installer-install-scripts:
     - !reference [.retrieve_linux_go_deps]
     - echo "About to build for $RELEASE_VERSION"
     - mkdir -p $OMNIBUS_PACKAGE_DIR
-    - inv -e installer.build-linux-script && mv ./bin/installer/install.sh $OMNIBUS_PACKAGE_DIR/install-djm.sh
+    - inv -e installer.build-linux-script "databricks" "$RELEASE_VERSION"
+    - mv ./bin/installer/install-*.sh $OMNIBUS_PACKAGE_DIR/
     - ls -la $OMNIBUS_PACKAGE_DIR
   artifacts:
     expire_in: 2 weeks

cmd/installer-downloader/main.go

@@ -7,20 +7,94 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/exec"
+	"path/filepath"
 
-	"github.com/DataDog/datadog-agent/cmd/installer/subcommands/installer"
 	"github.com/DataDog/datadog-agent/cmd/installer/user"
-	"github.com/DataDog/datadog-agent/cmd/internal/runcmd"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/oci"
+	"github.com/DataDog/datadog-agent/pkg/fleet/telemetry"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+)
+
+const (
+	installerPackage = "datadog-installer"
+	installerBinPath = "bin/installer/installer"
+)
+
+var (
+	// Version is the version of the installer to download.
+	Version string
+	// Flavor is the flavor of the setup to run.
+	Flavor string
 )
 
 func main() {
+	if Version == "" || Flavor == "" {
+		fmt.Fprintln(os.Stderr, "Version and Flavor must be set at build time.")
+		os.Exit(1)
+	}
 	if !user.IsRoot() {
-		fmt.Fprintln(os.Stderr, "This command requires root privileges.")
+		fmt.Fprintln(os.Stderr, "This installer requires root privileges.")
 		os.Exit(1)
 	}
-	cmd := installer.BootstrapCommand()
-	cmd.SilenceUsage = true
-	os.Exit(runcmd.Run(cmd))
+	env := env.FromEnv()
+	ctx := context.Background()
+
+	t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor))
+	_ = t.Start(ctx)
+	defer func() { _ = t.Stop(ctx) }()
+	var err error
+	span, ctx := telemetry.StartSpanFromEnv(ctx, "downloader")
+	defer func() { span.Finish(tracer.WithError(err)) }()
+	err = runDownloader(ctx, env, Version, Flavor)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func runDownloader(ctx context.Context, env *env.Env, version string, flavor string) error {
+	downloaderPath, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("failed to get executable path: %w", err)
+	}
+	tmpDir, err := os.MkdirTemp(filepath.Dir(downloaderPath), "datadog-installer")
+	if err != nil {
+		return fmt.Errorf("failed to create temporary directory: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+	err = downloadInstaller(ctx, env, version, tmpDir)
+	if err != nil {
+		return fmt.Errorf("failed to download installer: %w", err)
+	}
+	cmd := exec.CommandContext(ctx, filepath.Join(tmpDir, installerBinPath), "setup", "--flavor", flavor)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = os.Environ()
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("failed to run installer: %w", err)
+	}
+	return nil
+}
+
+func downloadInstaller(ctx context.Context, env *env.Env, version string, tmpDir string) error {
+	url := oci.PackageURL(env, installerPackage, version)
+	downloader := oci.NewDownloader(env, env.HTTPClient())
+	downloadedPackage, err := downloader.Download(ctx, url)
+	if err != nil {
+		return fmt.Errorf("failed to download installer package: %w", err)
+	}
+	if downloadedPackage.Name != installerPackage {
+		return fmt.Errorf("unexpected package name: %s, expected %s", downloadedPackage.Name, installerPackage)
+	}
+	err = downloadedPackage.ExtractLayers(oci.DatadogPackageLayerMediaType, tmpDir)
+	if err != nil {
+		return fmt.Errorf("failed to extract layers: %w", err)
+	}
+	return nil
 }

cmd/installer/subcommands/installer/command.go

@@ -96,7 +96,7 @@ type cmd struct {
 func newCmd(operation string) *cmd {
 	env := env.FromEnv()
 	t := newTelemetry(env)
-	span, ctx := newSpan(operation)
+	span, ctx := telemetry.StartSpanFromEnv(context.Background(), operation)
 	setInstallerUmask(span)
 	return &cmd{
 		t:    t,
@@ -226,28 +226,15 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry {
 	if site == "" {
 		site = config.Site
 	}
-	t, err := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands
-	if err != nil {
-		fmt.Printf("failed to initialize telemetry: %v\n", err)
-		return nil
-	}
-	err = t.Start(context.Background())
+	t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands
+	err := t.Start(context.Background())
 	if err != nil {
 		fmt.Printf("failed to start telemetry: %v\n", err)
 		return nil
 	}
 	return t
 }
 
-func newSpan(operationName string) (ddtrace.Span, context.Context) {
-	var spanOptions []ddtrace.StartSpanOption
-	spanContext, ok := telemetry.SpanContextFromEnv()
-	if ok {
-		spanOptions = append(spanOptions, tracer.ChildOf(spanContext))
-	}
-	return tracer.StartSpanFromContext(context.Background(), operationName, spanOptions...)
-}
-
 func versionCommand() *cobra.Command {
 	return &cobra.Command{
 		Use:     "version",
@@ -287,16 +274,21 @@ func bootstrapCommand() *cobra.Command {
 }
 
 func setupCommand() *cobra.Command {
+	flavor := ""
 	cmd := &cobra.Command{
 		Use:     "setup",
 		Hidden:  true,
 		GroupID: "installer",
 		RunE: func(_ *cobra.Command, _ []string) (err error) {
 			cmd := newCmd("setup")
 			defer func() { cmd.Stop(err) }()
-			return setup.Setup(cmd.ctx, cmd.env)
+			if flavor == "" {
+				return setup.Agent7InstallScript(cmd.ctx, cmd.env)
+			}
+			return setup.Setup(cmd.ctx, cmd.env, flavor)
 		},
 	}
+	cmd.Flags().StringVar(&flavor, "flavor", "", "The setup flavor")
 	return cmd
 }
 

comp/updater/telemetry/telemetryimpl/telemetry.go

@@ -38,16 +38,13 @@ func newTelemetry(deps dependencies) (telemetry.Component, error) {
 	client := &http.Client{
 		Transport: httputils.CreateHTTPTransport(deps.Config),
 	}
-	telemetry, err := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon",
+	telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon",
 		fleettelemetry.WithSamplingRules(
 			tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1),
 			tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05),
 			tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05),
 		),
 	)
-	if err != nil {
-		return nil, err
-	}
 	deps.Lc.Append(fx.Hook{OnStart: telemetry.Start, OnStop: telemetry.Stop})
 	return telemetry, nil
 }

pkg/fleet/bootstrapper/bootstrapper.go

@@ -11,9 +11,9 @@ import (
 	"fmt"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/oci"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/bootstrap"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/exec"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/oci"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/paths"
 )
 

pkg/fleet/installer/default_packages.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/oci"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/oci"
 )
 
 // Package represents a package known to the installer

pkg/fleet/installer/default_packages_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/oci"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/oci"
 	"github.com/stretchr/testify/assert"
 )
 

pkg/fleet/installer/fixtures/README.md

@@ -0,0 +1,19 @@
+# Datadog Package fixtures
+
+This directory contains a few examples of Datadog Packages for use in the
+updater tests.
+
+*simple-v1*
+```bash
+datadog-package create --archive --version "v1" --archive-path "pkg/fleet/installer/fixtures/oci-layout-simple-v1.tar" --package "simple" --configs pkg/fleet/installer/fixtures/simple-v1-config pkg/fleet/installer/fixtures/simple-v1
+```
+
+*simple-v2*
+```bash
+datadog-package create --archive --version "v2" --archive-path "pkg/fleet/installer/fixtures/oci-layout-simple-v2.tar" --package "simple" --configs pkg/fleet/installer/fixtures/simple-v2-config pkg/fleet/installer/fixtures/simple-v2
+```
+
+*simple-v1-linux2-amd128*
+```bash
+datadog-package create --archive --version "v1" --os "linux2" --arch "amd128" --archive-path "pkg/fleet/installer/fixtures/oci-layout-simple-v1-linux2-amd128.tar" --package "simple" pkg/fleet/installer/fixtures/simple-v1
+```

pkg/fleet/internal/fixtures/server.go → pkg/fleet/installer/fixtures/server.go

@@ -18,7 +18,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/tar"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/tar"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/registry"
 	"github.com/google/go-containerregistry/pkg/v1/layout"

pkg/fleet/installer/installer.go

@@ -22,10 +22,10 @@ import (
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
 	installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/oci"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/repository"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/db"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/oci"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"github.com/DataDog/datadog-agent/pkg/version"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"

pkg/fleet/installer/installer_test.go

@@ -17,10 +17,10 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/fixtures"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/oci"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/repository"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/db"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/oci"
 )
 
 var testCtx = context.TODO()

pkg/fleet/internal/oci/download.go → pkg/fleet/installer/oci/download.go

@@ -35,7 +35,7 @@ import (
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
 	installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/tar"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/tar"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
 

pkg/fleet/internal/oci/download_test.go → pkg/fleet/installer/oci/download_test.go

@@ -19,7 +19,7 @@ import (
 	"golang.org/x/net/http2"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/fixtures"
 	"github.com/google/go-containerregistry/pkg/authn"
 	oci "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/google"

pkg/fleet/internal/oci/mirror_test.go → pkg/fleet/installer/oci/mirror_test.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/fixtures"
 	"github.com/stretchr/testify/require"
 )
 

pkg/fleet/installer/setup/common/config.go

@@ -15,9 +15,9 @@ import (
 	"path/filepath"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/env"
+	"github.com/DataDog/datadog-agent/pkg/fleet/installer/oci"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/packages"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/exec"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/oci"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/paths"
 	"gopkg.in/yaml.v2"
 )