(fleet) isolate the downloader in its own sub cmd (DataDog#31795)

.github/CODEOWNERS

@@ -240,6 +240,7 @@
 /cmd/systray/                           @DataDog/windows-agent
 /cmd/security-agent/                    @DataDog/agent-security
 /cmd/installer/                         @DataDog/fleet @DataDog/windows-agent
+/cmd/installer-downloader/              @DataDog/fleet
 
 /dev/                                   @DataDog/agent-devx-loops
 /devenv/                                @DataDog/agent-devx-loops

.gitlab/package_build/installer.yml

@@ -103,7 +103,7 @@ installer-install-scripts:
     - !reference [.retrieve_linux_go_deps]
     - echo "About to build for $RELEASE_VERSION"
     - mkdir -p $OMNIBUS_PACKAGE_DIR
-    - inv -e installer.build-linux-script && mv ./bin/installer/setup.sh $OMNIBUS_PACKAGE_DIR/install-djm.sh
+    - inv -e installer.build-linux-script && mv ./bin/installer/install.sh $OMNIBUS_PACKAGE_DIR/install-djm.sh
     - ls -la $OMNIBUS_PACKAGE_DIR
   artifacts:
     expire_in: 2 weeks

cmd/installer/main_bootstrapper.go → cmd/installer-downloader/main.go

@@ -3,9 +3,7 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2023-present Datadog, Inc.
 
-//go:build bootstrapper
-
-// Package main implements 'installer'.
+// Package main implements the installer downloader
 package main
 
 import (

cmd/installer/main.go

@@ -3,8 +3,6 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2023-present Datadog, Inc.
 
-//go:build !bootstrapper
-
 // Package main implements 'installer'.
 package main
 

pkg/fleet/installer/setup/install.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Installer for Datadog (www.datadoghq.com).
+# Copyright 2016-present Datadog, Inc.
+#
+set -e
+
+if [ "$(uname -s)" != "Linux" ] || { [ "$(uname -m)" != "x86_64" ] && [ "$(uname -m)" != "aarch64" ]; }; then
+  echo "This installer only supports linux running on amd64 or arm64." >&2
+  exit 1
+fi
+
+tmp_dir="/opt/datadog-packages/tmp"
+downloader_path="${tmp_dir}/download-installer"
+
+install() {
+  if [ "$UID" == "0" ]; then
+    sudo_cmd=''
+  else
+    sudo_cmd='sudo'
+  fi
+
+  $sudo_cmd mkdir -p "${tmp_dir}"
+  case "$(uname -m)" in
+  x86_64)
+    echo "${downloader_bin_linux_amd64}" | base64 -d | $sudo_cmd tee "${downloader_path}" >/dev/null
+    ;;
+  aarch64)
+    echo "${downloader_bin_linux_arm64}" | base64 -d | $sudo_cmd tee "${downloader_path}" >/dev/null
+    ;;
+  esac
+  $sudo_cmd chmod +x "${downloader_path}"
+  echo "Starting the Datadog installer..."
+  $sudo_cmd "${downloader_path}" "$@"
+  $sudo_cmd rm -f "${downloader_path}"
+}
+
+# Embedded binaries used to install Datadog.
+# Source: https://github.com/DataDog/datadog-agent/tree/INSTALLER_COMMIT/pkg/fleet/installer
+# DO NOT EDIT THIS SECTION MANUALLY.
+downloader_bin_linux_amd64=$(
+  cat <<EOM
+DOWNLOADER_BIN_LINUX_AMD64
+EOM
+)
+downloader_bin_linux_arm64=$(
+  cat <<EOM
+DOWNLOADER_BIN_LINUX_ARM64
+EOM
+)
+
+install "$@"
+exit 0

tasks/installer.py

@@ -13,15 +13,18 @@
 from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags
 from tasks.libs.releasing.version import get_version
 
-BIN_PATH = os.path.join(".", "bin", "installer")
+DIR_BIN = os.path.join(".", "bin", "installer")
+INSTALLER_BIN = os.path.join(DIR_BIN, bin_name("installer"))
+DOWNLOADER_BIN = os.path.join(DIR_BIN, bin_name("downloader"))
+INSTALL_SCRIPT = os.path.join(DIR_BIN, "install.sh")
+INSTALL_SCRIPT_TEMPLATE = os.path.join("pkg", "fleet", "installer", "setup", "install.sh")
 MAJOR_VERSION = '7'
 
 
 @task
 def build(
     ctx,
     output_bin=None,
-    bootstrapper=False,
     rebuild=False,
     race=False,
     install_path=None,
@@ -33,7 +36,7 @@ def build(
     no_cgo=False,
 ):
     """
-    Build the updater.
+    Build the installer.
     """
 
     ldflags, gcflags, env = get_build_flags(
@@ -48,20 +51,14 @@ def build(
         else filter_incompatible_tags(build_include.split(","))
     )
     build_exclude = [] if build_exclude is None else build_exclude.split(",")
-
     build_tags = get_build_tags(build_include, build_exclude)
-    if bootstrapper:
-        build_tags.append("bootstrapper")
 
     strip_flags = "" if no_strip_binary else "-s -w"
     race_opt = "-race" if race else ""
     build_type = "-a" if rebuild else ""
     go_build_tags = " ".join(build_tags)
 
-    installer_bin_name = "installer"
-    if bootstrapper:
-        installer_bin_name = "bootstrapper"
-    installer_bin = os.path.join(BIN_PATH, bin_name(installer_bin_name))
+    installer_bin = INSTALLER_BIN
     if output_bin:
         installer_bin = output_bin
 
@@ -76,58 +73,44 @@ def build(
     ctx.run(cmd, env=env)
 
 
+@task
+def build_downloader(
+    ctx,
+    os="linux",
+    arch="amd64",
+):
+    '''
+    Builds the installer downloader binary.
+    '''
+    ctx.run(
+        f'go build -ldflags="-s -w" -o {DOWNLOADER_BIN} {REPO_PATH}/cmd/installer-downloader',
+        env={'GOOS': os, 'GOARCH': arch, 'CGO_ENABLED': '0'},
+    )
+
+
 @task
 def build_linux_script(
     ctx,
-    signing_key_id=None,
 ):
     '''
     Builds the linux script that is used to install the agent on linux.
     '''
-    script_path = os.path.join(BIN_PATH, "setup.sh")
-    signed_script_path = os.path.join(BIN_PATH, "setup.sh.asc")
-    amd64_path = os.path.join(BIN_PATH, "bootstrapper-linux-amd64")
-    arm64_path = os.path.join(BIN_PATH, "bootstrapper-linux-arm64")
 
-    ctx.run(
-        f'inv -e installer.build --bootstrapper --no-no-strip-binary --output-bin {amd64_path} --no-cgo',
-        env={'GOOS': 'linux', 'GOARCH': 'amd64'},
-    )
-    ctx.run(
-        f'inv -e installer.build --bootstrapper --no-no-strip-binary --output-bin {arm64_path} --no-cgo',
-        env={'GOOS': 'linux', 'GOARCH': 'arm64'},
-    )
-    with open(amd64_path, 'rb') as f:
-        amd64_b64 = base64.encodebytes(f.read()).decode('utf-8')
-    with open(arm64_path, 'rb') as f:
-        arm64_b64 = base64.encodebytes(f.read()).decode('utf-8')
+    with open(INSTALL_SCRIPT_TEMPLATE) as f:
+        install_script = f.read()
 
-    with open('pkg/fleet/installer/setup.sh') as f:
-        setup_content = f.read()
-    setup_content = setup_content.replace('INSTALLER_BIN_LINUX_AMD64', amd64_b64)
-    setup_content = setup_content.replace('INSTALLER_BIN_LINUX_ARM64', arm64_b64)
+    archs = ['amd64', 'arm64']
+    for arch in archs:
+        build_downloader(ctx, os='linux', arch=arch)
+        with open(DOWNLOADER_BIN, 'rb') as f:
+            encoded_bin = base64.encodebytes(f.read()).decode('utf-8')
+        install_script = install_script.replace(f'DOWNLOADER_BIN_{arch.upper()}', encoded_bin)
 
     commit_sha = ctx.run('git rev-parse HEAD', hide=True).stdout.strip()
-    setup_content = setup_content.replace('INSTALLER_COMMIT', commit_sha)
-
-    with open(script_path, 'w') as f:
-        f.write(setup_content)
-
-    if signing_key_id:
-        ctx.run(
-            f'gpg --armor --batch --yes --output {signed_script_path} --clearsign --digest-algo SHA256 --default-key {signing_key_id} {script_path}',
-        )
-        # Add the signed footer to the setup.sh file
-        with (
-            open(signed_script_path) as signed_file,
-            open(script_path, 'w') as f,
-        ):
-            skip_header = False
-            for line in signed_file:
-                if skip_header:
-                    f.write(line)
-                elif line.strip() == "":  # Empty line marks end of header
-                    skip_header = True
+    install_script = install_script.replace('INSTALLER_COMMIT', commit_sha)
+
+    with open(INSTALL_SCRIPT, 'w') as f:
+        f.write(install_script)
 
 
 @task