(fleet) stop the previous installer/agent version on an upgrade (#32156)

pkg/fleet/installer/installer.go

@@ -631,6 +631,8 @@ func (i *installerImpl) promoteExperiment(ctx context.Context, pkg string) error
 
 func (i *installerImpl) preparePackage(ctx context.Context, pkg string, _ []string) error {
 	switch pkg {
+	case packageDatadogInstaller:
+		return packages.PrepareInstaller(ctx)
 	case packageDatadogAgent:
 		return packages.PrepareAgent(ctx)
 	default:

pkg/fleet/installer/packages/datadog_agent.go

@@ -23,7 +23,6 @@ import (
 
 const (
 	agentPackage      = "datadog-agent"
-	pathOldAgent      = "/opt/datadog-agent"
 	agentSymlink      = "/usr/bin/datadog-agent"
 	agentUnit         = "datadog-agent.service"
 	traceAgentUnit    = "datadog-agent-trace.service"
@@ -76,13 +75,18 @@ func PrepareAgent(ctx context.Context) (err error) {
 	span, ctx := telemetry.StartSpanFromContext(ctx, "prepare_agent")
 	defer func() { span.Finish(err) }()
 
-	// Check if the agent has been installed by a package manager, if yes remove it
-	if !oldAgentInstalled() {
-		return nil // Nothing to do
-	}
-	err = stopOldAgentUnits(ctx)
+	err = removeDebRPMPackage(ctx, "datadog-agent")
 	if err != nil {
-		return fmt.Errorf("failed to stop old agent units: %w", err)
+		return fmt.Errorf("failed to remove deb/rpm datadog-agent package: %w", err)
+	}
+
+	for _, unit := range stableUnits {
+		if err := stopUnit(ctx, unit); err != nil {
+			log.Warnf("Failed to stop %s: %s", unit, err)
+		}
+		if err := disableUnit(ctx, unit); err != nil {
+			log.Warnf("Failed to disable %s: %s", unit, err)
+		}
 	}
 	return removeDebRPMPackage(ctx, agentPackage)
 }
@@ -205,28 +209,6 @@ func RemoveAgent(ctx context.Context) error {
 	return nil
 }
 
-func oldAgentInstalled() bool {
-	_, err := os.Stat(pathOldAgent)
-	return err == nil
-}
-
-func stopOldAgentUnits(ctx context.Context) (err error) {
-	if !oldAgentInstalled() {
-		return nil
-	}
-	span, ctx := telemetry.StartSpanFromContext(ctx, "remove_old_agent_units")
-	defer span.Finish(err)
-	for _, unit := range stableUnits {
-		if err := stopUnit(ctx, unit); err != nil {
-			return fmt.Errorf("failed to stop %s: %v", unit, err)
-		}
-		if err := disableUnit(ctx, unit); err != nil {
-			return fmt.Errorf("failed to disable %s: %v", unit, err)
-		}
-	}
-	return nil
-}
-
 func chownRecursive(path string, uid int, gid int, ignorePaths []string) error {
 	return filepath.Walk(path, func(p string, _ os.FileInfo, err error) error {
 		if err != nil {

pkg/fleet/installer/packages/datadog_installer.go

@@ -41,6 +41,17 @@ func addDDAgentGroup(ctx context.Context) error {
 	return exec.CommandContext(ctx, "groupadd", "--system", "dd-agent").Run()
 }
 
+// PrepareInstaller prepares the installer
+func PrepareInstaller(ctx context.Context) error {
+	if err := stopUnit(ctx, installerUnit); err != nil {
+		log.Warnf("Failed to stop unit %s: %s", installerUnit, err)
+	}
+	if err := disableUnit(ctx, installerUnit); err != nil {
+		log.Warnf("Failed to disable %s: %s", installerUnit, err)
+	}
+	return nil
+}
+
 // SetupInstaller installs and starts the installer systemd units
 func SetupInstaller(ctx context.Context) (err error) {
 	defer func() {

pkg/fleet/installer/packages/datadog_installer_windows.go

@@ -11,16 +11,22 @@ package packages
 import (
 	"context"
 	"fmt"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/msi"
-	"github.com/DataDog/datadog-agent/pkg/fleet/internal/paths"
 	"os"
 	"path"
+
+	"github.com/DataDog/datadog-agent/pkg/fleet/internal/msi"
+	"github.com/DataDog/datadog-agent/pkg/fleet/internal/paths"
 )
 
 const (
 	datadogInstaller = "datadog-installer"
 )
 
+// PrepareInstaller prepares the installer
+func PrepareInstaller(_ context.Context) error {
+	return nil
+}
+
 // SetupInstaller installs and starts the installer
 func SetupInstaller(_ context.Context) error {
 	rootPath := ""

pkg/fleet/installer/packages/pkg_manager.go

@@ -8,44 +8,62 @@
 package packages
 
 import (
-	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"os/exec"
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/telemetry"
 )
 
+func dpkgInstalled() (bool, error) {
+	_, err := exec.LookPath("dpkg")
+	if err != nil && !errors.Is(err, exec.ErrNotFound) {
+		return false, err
+	}
+	return err == nil, nil
+}
+
+func rpmInstalled() (bool, error) {
+	_, err := exec.LookPath("rpm")
+	if err != nil && !errors.Is(err, exec.ErrNotFound) {
+		return false, err
+	}
+	return err == nil, nil
+}
+
 // removeDebRPMPackage removes a package installed via deb/rpm package manager
 // It doesn't remove dependencies or purge as we want to keep existing configuration files
 // and reinstall the package using the installer.
 // Note: we don't run the pre/post remove scripts as we want to avoid surprises for older agent versions (like removing config)
 func removeDebRPMPackage(ctx context.Context, pkg string) (err error) {
-	span, _ := telemetry.StartSpanFromContext(ctx, "remove_deb_rpm_package")
+	span, _ := telemetry.StartSpanFromContext(ctx, "removeDebRPMPackage")
 	defer func() { span.Finish(err) }()
-	// Compute the right command depending on the package manager
-	var cmd *exec.Cmd
-	if _, pathErr := exec.LookPath("dpkg"); pathErr == nil {
-		// Doesn't fail if the package isn't installed
-		cmd = exec.Command("dpkg", "-r", "--no-triggers", agentPackage)
-	} else if _, pathErr := exec.LookPath("rpm"); pathErr == nil {
-		// Check if package exist, else the command will fail
-		pkgErr := exec.Command("rpm", "-q", agentPackage).Run()
-		if pkgErr == nil {
-			cmd = exec.Command("rpm", "-e", "--nodeps", "--noscripts", agentPackage)
-		}
-	}
-
-	if cmd == nil {
-		// If we can't find a package manager or the package is not installed, ignore this step
+
+	dpkgInstalled, err := dpkgInstalled()
+	if err != nil {
+		return err
+	}
+	rpmInstalled, err := rpmInstalled()
+	if err != nil {
+		return err
+	}
+	var packageInstalled bool
+	var removeCmd *exec.Cmd
+	if dpkgInstalled {
+		removeCmd = exec.Command("dpkg", "-r", pkg)
+		packageInstalled = exec.Command("dpkg", "-s", pkg).Run() == nil
+	}
+	if rpmInstalled {
+		removeCmd = exec.Command("rpm", "-e", pkg)
+		packageInstalled = exec.Command("rpm", "-q", pkg).Run() == nil
+	}
+	if !packageInstalled {
 		return nil
 	}
-
-	// Run the command
-	var buf bytes.Buffer
-	cmd.Stderr = &buf
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("failed to uninstall deb/rpm package %s (%w): %s", pkg, err, buf.String())
+	out, err := removeCmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to uninstall deb/rpm package %s (%w): %s", pkg, err, out)
 	}
 	return nil
 }

test/new-e2e/tests/installer/unix/package_installer_test.go

@@ -6,6 +6,9 @@
 package installer
 
 import (
+	"strings"
+	"time"
+
 	awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host"
 	e2eos "github.com/DataDog/test-infra-definitions/components/os"
 	"github.com/stretchr/testify/assert"
@@ -102,20 +105,33 @@ func (s *packageInstallerSuite) TestReInstall() {
 func (s *packageInstallerSuite) TestUpdateInstallerOCI() {
 	// Install prod
 	err := s.RunInstallScriptProdOci(
+		"DD_REMOTE_UPDATES=true",
 		envForceVersion("datadog-installer", "7.58.0-installer-0.5.1-1"),
 	)
 	defer s.Purge()
 	assert.NoError(s.T(), err)
 
-	version := s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version")
-	assert.Equal(s.T(), "7.58.0-installer-0.5.1\n", version)
+	versionDisk := s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version")
+	assert.Equal(s.T(), "7.58.0-installer-0.5.1\n", versionDisk)
+	assert.Eventually(s.T(), func() bool {
+		versionRunning, err := s.Env().RemoteHost.Execute("sudo datadog-installer status")
+		s.T().Logf("checking version: %s, err: %v", versionRunning, err)
+		return err == nil && strings.Contains(versionRunning, "7.58.0-installer-0.5.1")
+	}, 30*time.Second, 1*time.Second)
 
 	// Install from QA registry
-	err = s.RunInstallScriptWithError()
+	err = s.RunInstallScriptWithError(
+		"DD_REMOTE_UPDATES=true",
+	)
 	assert.NoError(s.T(), err)
 
-	version = s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version")
-	assert.NotEqual(s.T(), "7.58.0-installer-0.5.1\n", version)
+	versionDisk = s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version")
+	assert.NotEqual(s.T(), "7.58.0-installer-0.5.1\n", versionDisk)
+	assert.Eventually(s.T(), func() bool {
+		versionRunning, err := s.Env().RemoteHost.Execute("sudo datadog-installer status")
+		s.T().Logf("checking version: %s, err: %v", versionRunning, err)
+		return err == nil && !strings.Contains(versionRunning, "7.58.0-installer-0.5.1")
+	}, 30*time.Second, 1*time.Second)
 }
 
 func (s *packageInstallerSuite) TestInstallWithUmask() {