diff --git a/agent/cloudinit/cloudinit.go b/agent/cloudinit/cloudinit.go index 6a327e3ec..9533d867d 100644 --- a/agent/cloudinit/cloudinit.go +++ b/agent/cloudinit/cloudinit.go @@ -4,6 +4,7 @@ package cloudinit import ( + "context" "encoding/base64" "fmt" "path/filepath" @@ -71,7 +72,7 @@ func (se ScriptExecutor) Execute(bootstrapScript string) error { } for _, cmd := range cloudInitData.CommandsToExecute { - err := se.RunCmdExecutor.RunCmd(cmd) + err := se.RunCmdExecutor.RunCmd(context.TODO(), cmd) if err != nil { return errors.Wrap(err, fmt.Sprintf("Error running the command %s", cmd)) } diff --git a/agent/cloudinit/cloudinit_test.go b/agent/cloudinit/cloudinit_test.go index b39d5a997..8d31ec3c8 100644 --- a/agent/cloudinit/cloudinit_test.go +++ b/agent/cloudinit/cloudinit_test.go @@ -135,7 +135,7 @@ runCmd: Expect(err).NotTo(HaveOccurred()) Expect(fakeCmdExecutor.RunCmdCallCount()).To(Equal(1)) - cmd := fakeCmdExecutor.RunCmdArgsForCall(0) + _, cmd := fakeCmdExecutor.RunCmdArgsForCall(0) Expect(cmd).To(Equal("echo 'some run command'")) }) diff --git a/agent/cloudinit/cloudinitfakes/fake_icmd_runner.go b/agent/cloudinit/cloudinitfakes/fake_icmd_runner.go index 8942db888..bc82b01d2 100644 --- a/agent/cloudinit/cloudinitfakes/fake_icmd_runner.go +++ b/agent/cloudinit/cloudinitfakes/fake_icmd_runner.go @@ -2,16 +2,18 @@ package cloudinitfakes import ( + "context" "sync" "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent/cloudinit" ) type FakeICmdRunner struct { - RunCmdStub func(string) error + RunCmdStub func(context.Context, string) error runCmdMutex sync.RWMutex runCmdArgsForCall []struct { - arg1 string + arg1 context.Context + arg2 string } runCmdReturns struct { result1 error @@ -23,18 +25,19 @@ type FakeICmdRunner struct { invocationsMutex sync.RWMutex } -func (fake *FakeICmdRunner) RunCmd(arg1 string) error { +func (fake *FakeICmdRunner) RunCmd(arg1 context.Context, arg2 string) error { fake.runCmdMutex.Lock() ret, specificReturn := fake.runCmdReturnsOnCall[len(fake.runCmdArgsForCall)] fake.runCmdArgsForCall = append(fake.runCmdArgsForCall, struct { - arg1 string - }{arg1}) + arg1 context.Context + arg2 string + }{arg1, arg2}) stub := fake.RunCmdStub fakeReturns := fake.runCmdReturns - fake.recordInvocation("RunCmd", []interface{}{arg1}) + fake.recordInvocation("RunCmd", []interface{}{arg1, arg2}) fake.runCmdMutex.Unlock() if stub != nil { - return stub(arg1) + return stub(arg1, arg2) } if specificReturn { return ret.result1 @@ -48,17 +51,17 @@ func (fake *FakeICmdRunner) RunCmdCallCount() int { return len(fake.runCmdArgsForCall) } -func (fake *FakeICmdRunner) RunCmdCalls(stub func(string) error) { +func (fake *FakeICmdRunner) RunCmdCalls(stub func(context.Context, string) error) { fake.runCmdMutex.Lock() defer fake.runCmdMutex.Unlock() fake.RunCmdStub = stub } -func (fake *FakeICmdRunner) RunCmdArgsForCall(i int) string { +func (fake *FakeICmdRunner) RunCmdArgsForCall(i int) (context.Context, string) { fake.runCmdMutex.RLock() defer fake.runCmdMutex.RUnlock() argsForCall := fake.runCmdArgsForCall[i] - return argsForCall.arg1 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeICmdRunner) RunCmdReturns(result1 error) { diff --git a/agent/cloudinit/cmd_runner.go b/agent/cloudinit/cmd_runner.go index 80acfe961..93c22c44f 100644 --- a/agent/cloudinit/cmd_runner.go +++ b/agent/cloudinit/cmd_runner.go @@ -4,13 +4,14 @@ package cloudinit import ( + "context" "os" "os/exec" ) //counterfeiter:generate . ICmdRunner type ICmdRunner interface { - RunCmd(string) error + RunCmd(context.Context, string) error } // CmdRunner default implementer of ICmdRunner @@ -19,8 +20,12 @@ type CmdRunner struct { } // RunCmd executes the command string -func (r CmdRunner) RunCmd(cmd string) error { - command := exec.Command("/bin/sh", "-c", cmd) +func (r CmdRunner) RunCmd(ctx context.Context, cmd string) error { + command := exec.CommandContext(ctx, "/bin/bash", "-c", cmd) command.Stderr = os.Stderr - return command.Run() + command.Stdout = os.Stdout + if err := command.Run(); err != nil { + return err + } + return nil } diff --git a/agent/main.go b/agent/main.go index eca094acc..29b2c0280 100644 --- a/agent/main.go +++ b/agent/main.go @@ -168,8 +168,7 @@ func main() { os.Exit(1) } } - // Handle kubeconfig flag - // first look in the byoh path for the kubeconfig + // Handle kubeconfig flag first look in the byoh path for the kubeconfig config, err := registration.LoadRESTClientConfig(registration.GetBYOHConfigPath()) if err != nil { logger.Error(err, "client config load failed") @@ -231,6 +230,7 @@ func main() { K8sInstaller: k8sInstaller, SkipK8sInstallation: skipInstallation, UseInstallerController: useInstallerController, + DownloadPath: downloadpath, } if err = hostReconciler.SetupWithManager(context.TODO(), mgr); err != nil { logger.Error(err, "unable to create controller") diff --git a/agent/reconciler/host_reconciler.go b/agent/reconciler/host_reconciler.go index 4a30cbd72..3320f5f56 100644 --- a/agent/reconciler/host_reconciler.go +++ b/agent/reconciler/host_reconciler.go @@ -45,6 +45,7 @@ type HostReconciler struct { K8sInstaller IK8sInstaller SkipK8sInstallation bool UseInstallerController bool + DownloadPath string } const ( @@ -94,6 +95,8 @@ func (r *HostReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctr func (r *HostReconciler) reconcileNormal(ctx context.Context, byoHost *infrastructurev1beta1.ByoHost) (ctrl.Result, error) { logger := ctrl.LoggerFrom(ctx) + logger = logger.WithValues("ByoHost", byoHost.Name) + logger.Info("reconcile normal") if byoHost.Status.MachineRef == nil { logger.Info("Machine ref not yet set") conditions.MarkFalse(byoHost, infrastructurev1beta1.K8sNodeBootstrapSucceeded, infrastructurev1beta1.WaitingForMachineRefReason, clusterv1.ConditionSeverityInfo, "") @@ -117,10 +120,20 @@ func (r *HostReconciler) reconcileNormal(ctx context.Context, byoHost *infrastru if r.SkipK8sInstallation { logger.Info("Skipping installation of k8s components") } else if r.UseInstallerController { - if byoHost.Spec.InstallationSecret == nil { - logger.Info("K8sInstallationSecret not ready") - conditions.MarkFalse(byoHost, infrastructurev1beta1.K8sNodeBootstrapSucceeded, infrastructurev1beta1.K8sInstallationSecretUnavailableReason, clusterv1.ConditionSeverityInfo, "") - return ctrl.Result{}, nil + if !conditions.IsTrue(byoHost, infrastructurev1beta1.K8sComponentsInstallationSucceeded) { + if byoHost.Spec.InstallationSecret == nil { + logger.Info("InstallationSecret not ready") + conditions.MarkFalse(byoHost, infrastructurev1beta1.K8sComponentsInstallationSucceeded, infrastructurev1beta1.K8sInstallationSecretUnavailableReason, clusterv1.ConditionSeverityInfo, "") + return ctrl.Result{}, nil + } + err = r.executeInstallerController(ctx, byoHost) + if err != nil { + return ctrl.Result{}, err + } + r.Recorder.Event(byoHost, corev1.EventTypeNormal, "InstallScriptExecutionSucceeded", "install script executed") + conditions.MarkTrue(byoHost, infrastructurev1beta1.K8sComponentsInstallationSucceeded) + } else { + logger.Info("install script already executed") } } else { err = r.installK8sComponents(ctx, byoHost) @@ -156,6 +169,34 @@ func (r *HostReconciler) reconcileNormal(ctx context.Context, byoHost *infrastru return ctrl.Result{}, nil } +func (r *HostReconciler) executeInstallerController(ctx context.Context, byoHost *infrastructurev1beta1.ByoHost) error { + logger := ctrl.LoggerFrom(ctx) + secret := &corev1.Secret{} + err := r.Client.Get(ctx, types.NamespacedName{Name: byoHost.Spec.InstallationSecret.Name, Namespace: byoHost.Spec.InstallationSecret.Namespace}, secret) + if err != nil { + logger.Error(err, "error getting install and uninstall script") + r.Recorder.Eventf(byoHost, corev1.EventTypeWarning, "ReadInstallationSecretFailed", "install and uninstall script %s not found", byoHost.Spec.InstallationSecret.Name) + return err + } + installScript := string(secret.Data["install"]) + uninstallScript := string(secret.Data["uninstall"]) + + byoHost.Spec.UninstallationScript = &uninstallScript + installScript, err = r.parseScript(ctx, installScript) + if err != nil { + return err + } + logger.Info("executing install script") + err = r.CmdRunner.RunCmd(ctx, installScript) + if err != nil { + logger.Error(err, "error executing installation script") + r.Recorder.Event(byoHost, corev1.EventTypeWarning, "InstallScriptExecutionFailed", "install script execution failed") + conditions.MarkFalse(byoHost, infrastructurev1beta1.K8sComponentsInstallationSucceeded, infrastructurev1beta1.K8sComponentsInstallationFailedReason, clusterv1.ConditionSeverityInfo, "") + return err + } + return nil +} + func (r *HostReconciler) reconcileDelete(ctx context.Context, byoHost *infrastructurev1beta1.ByoHost) (ctrl.Result, error) { return ctrl.Result{}, nil } @@ -171,6 +212,18 @@ func (r *HostReconciler) getBootstrapScript(ctx context.Context, dataSecretName, return bootstrapSecret, nil } +func (r *HostReconciler) parseScript(ctx context.Context, script string) (string, error) { + data, err := cloudinit.TemplateParser{ + Template: map[string]string{ + "BundleDownloadPath": r.DownloadPath, + }, + }.ParseTemplate(script) + if err != nil { + return "", fmt.Errorf("unable to apply install parsed template to the data object") + } + return data, nil +} + // SetupWithManager sets up the controller with the manager func (r *HostReconciler) SetupWithManager(ctx context.Context, mgr manager.Manager) error { return ctrl.NewControllerManagedBy(mgr). @@ -215,6 +268,23 @@ func (r *HostReconciler) hostCleanUp(ctx context.Context, byoHost *infrastructur } if r.SkipK8sInstallation { logger.Info("Skipping uninstallation of k8s components") + } else if r.UseInstallerController { + if byoHost.Spec.UninstallationScript == nil { + return fmt.Errorf("UninstallationScript not found in Byohost %s", byoHost.Name) + } + logger.Info("Executing Uninstall script") + uninstallScript := *byoHost.Spec.UninstallationScript + uninstallScript, err = r.parseScript(ctx, uninstallScript) + if err != nil { + logger.Error(err, "error parsing Uninstallation script") + return err + } + err = r.CmdRunner.RunCmd(ctx, uninstallScript) + if err != nil { + logger.Error(err, "error execting Uninstallation script") + r.Recorder.Event(byoHost, corev1.EventTypeWarning, "UninstallScriptExecutionFailed", "uninstall script execution failed") + return err + } } else { err = r.uninstallk8sComponents(ctx, byoHost) if err != nil { @@ -236,6 +306,8 @@ func (r *HostReconciler) hostCleanUp(ctx context.Context, byoHost *infrastructur return err } + byoHost.Spec.InstallationSecret = nil + byoHost.Spec.UninstallationScript = nil r.removeAnnotations(ctx, byoHost) conditions.MarkFalse(byoHost, infrastructurev1beta1.K8sNodeBootstrapSucceeded, infrastructurev1beta1.K8sNodeAbsentReason, clusterv1.ConditionSeverityInfo, "") return nil @@ -245,7 +317,7 @@ func (r *HostReconciler) resetNode(ctx context.Context, byoHost *infrastructurev logger := ctrl.LoggerFrom(ctx) logger.Info("Running kubeadm reset") - err := r.CmdRunner.RunCmd(KubeadmResetCommand) + err := r.CmdRunner.RunCmd(ctx, KubeadmResetCommand) if err != nil { r.Recorder.Event(byoHost, corev1.EventTypeWarning, "ResetK8sNodeFailed", "k8s Node Reset failed") return errors.Wrapf(err, "failed to exec kubeadm reset") diff --git a/agent/reconciler/reconciler_test.go b/agent/reconciler/reconciler_test.go index af5bd15c4..9bdd5e9b3 100644 --- a/agent/reconciler/reconciler_test.go +++ b/agent/reconciler/reconciler_test.go @@ -28,14 +28,16 @@ import ( var _ = Describe("Byohost Agent Tests", func() { var ( - ctx = context.TODO() - ns = "default" - hostName = "test-host" - byoHost *infrastructurev1beta1.ByoHost - byoMachine *infrastructurev1beta1.ByoMachine - byoHostLookupKey types.NamespacedName - bootstrapSecret *corev1.Secret - recorder *record.FakeRecorder + ctx = context.TODO() + ns = "default" + hostName = "test-host" + byoHost *infrastructurev1beta1.ByoHost + byoMachine *infrastructurev1beta1.ByoMachine + byoHostLookupKey types.NamespacedName + bootstrapSecret *corev1.Secret + installationSecret *corev1.Secret + recorder *record.FakeRecorder + uninstallScript string ) BeforeEach(func() { @@ -157,7 +159,7 @@ var _ = Describe("Byohost Agent Tests", func() { - path: fake/path content: blah runCmd: -- echo 'some run command'` +- echo 'run some command'` bootstrapSecret = builder.Secret(ns, "test-secret"). WithData(secretData). @@ -179,36 +181,6 @@ runCmd: Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) }) - Context("When use-installer-controller is set", func() { - BeforeEach(func() { - hostReconciler.UseInstallerController = true - }) - - It("should set the Reason to InstallationSecretUnavailableReason", func() { - result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ - NamespacedName: byoHostLookupKey, - }) - Expect(result).To(Equal(controllerruntime.Result{})) - Expect(reconcilerErr).ToNot(HaveOccurred()) - - updatedByoHost := &infrastructurev1beta1.ByoHost{} - err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) - Expect(err).ToNot(HaveOccurred()) - - byoHostRegistrationSucceeded := conditions.Get(updatedByoHost, infrastructurev1beta1.K8sNodeBootstrapSucceeded) - Expect(*byoHostRegistrationSucceeded).To(conditions.MatchCondition(clusterv1.Condition{ - Type: infrastructurev1beta1.K8sNodeBootstrapSucceeded, - Status: corev1.ConditionFalse, - Reason: infrastructurev1beta1.K8sInstallationSecretUnavailableReason, - Severity: clusterv1.ConditionSeverityInfo, - })) - }) - - AfterEach(func() { - hostReconciler.UseInstallerController = false - }) - }) - It("should set K8sComponentsInstallationSucceeded to false with Reason K8sComponentsInstallationFailedReason if Install fails", func() { hostReconciler.K8sInstaller = fakeInstaller fakeInstaller.InstallReturns(errors.New("k8s components install failed")) @@ -335,6 +307,183 @@ runCmd: Expect(fakeFileWriter.WriteToFileCallCount()).To(Equal(1)) }) + Context("When use-installer-controller is set", func() { + BeforeEach(func() { + hostReconciler.UseInstallerController = true + }) + + It("should set the Reason to InstallationSecretUnavailableReason", func() { + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).ToNot(HaveOccurred()) + + updatedByoHost := &infrastructurev1beta1.ByoHost{} + err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) + Expect(err).ToNot(HaveOccurred()) + + byoHostRegistrationSucceeded := conditions.Get(updatedByoHost, infrastructurev1beta1.K8sComponentsInstallationSucceeded) + Expect(*byoHostRegistrationSucceeded).To(conditions.MatchCondition(clusterv1.Condition{ + Type: infrastructurev1beta1.K8sComponentsInstallationSucceeded, + Status: corev1.ConditionFalse, + Reason: infrastructurev1beta1.K8sInstallationSecretUnavailableReason, + Severity: clusterv1.ConditionSeverityInfo, + })) + }) + + It("should return an error if we fail to load the installation secret", func() { + byoHost.Spec.InstallationSecret = &corev1.ObjectReference{ + Kind: "Secret", + Namespace: "non-existent", + Name: "non-existent", + } + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).To(MatchError("secrets \"non-existent\" not found")) + + // assert events + events := eventutils.CollectEvents(recorder.Events) + Expect(events).Should(ConsistOf([]string{ + fmt.Sprintf("Warning ReadInstallationSecretFailed install and uninstall script %s not found", byoHost.Spec.InstallationSecret.Name), + })) + }) + + Context("When installation secret is ready", func() { + BeforeEach(func() { + installScript := `echo "install"` + uninstallScript = `echo "uninstall"` + + installationSecret = builder.Secret(ns, "test-secret3"). + WithKeyData("install", installScript). + WithKeyData("uninstall", uninstallScript). + Build() + Expect(k8sClient.Create(ctx, installationSecret)).NotTo(HaveOccurred()) + + byoHost.Spec.InstallationSecret = &corev1.ObjectReference{ + Kind: "Secret", + Namespace: installationSecret.Namespace, + Name: installationSecret.Name, + } + + byoHost.Annotations = map[string]string{ + infrastructurev1beta1.K8sVersionAnnotation: "1.22", + infrastructurev1beta1.BundleLookupTagAnnotation: "byoh-bundle-tag", + infrastructurev1beta1.BundleLookupBaseRegistryAnnotation: "projects.blah.com", + } + + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + }) + + It("should return error if install script execution failed", func() { + fakeCommandRunner.RunCmdReturns(errors.New("failed to execute install script")) + invalidInstallationSecret := builder.Secret(ns, "invalid-test-secret"). + WithKeyData("install", "test"). + Build() + Expect(k8sClient.Create(ctx, invalidInstallationSecret)).NotTo(HaveOccurred()) + byoHost.Spec.InstallationSecret = &corev1.ObjectReference{ + Kind: "Secret", + Namespace: invalidInstallationSecret.Namespace, + Name: invalidInstallationSecret.Name, + } + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).To(HaveOccurred()) + + // assert events + events := eventutils.CollectEvents(recorder.Events) + Expect(events).Should(ConsistOf([]string{ + "Warning InstallScriptExecutionFailed install script execution failed", + })) + + }) + + It("should set uninstall script in byohost spec", func() { + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).NotTo(HaveOccurred()) + + updatedByoHost := &infrastructurev1beta1.ByoHost{} + err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) + Expect(err).ToNot(HaveOccurred()) + Expect(updatedByoHost.Spec.UninstallationScript).NotTo(BeNil()) + Expect(*updatedByoHost.Spec.UninstallationScript).To(Equal(uninstallScript)) + }) + + It("should set K8sComponentsInstallationSucceeded to true if Install succeeds", func() { + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).ToNot(HaveOccurred()) + + updatedByoHost := &infrastructurev1beta1.ByoHost{} + err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) + Expect(err).ToNot(HaveOccurred()) + + K8sComponentsInstallationSucceeded := conditions.Get(updatedByoHost, infrastructurev1beta1.K8sComponentsInstallationSucceeded) + Expect(*K8sComponentsInstallationSucceeded).To(conditions.MatchCondition(clusterv1.Condition{ + Type: infrastructurev1beta1.K8sComponentsInstallationSucceeded, + Status: corev1.ConditionTrue, + })) + + // assert events + events := eventutils.CollectEvents(recorder.Events) + Expect(events).Should(ConsistOf([]string{ + "Normal InstallScriptExecutionSucceeded install script executed", + "Normal BootstrapK8sNodeSucceeded k8s Node Bootstraped", + })) + }) + + It("should set K8sNodeBootstrapSucceeded to True if the boostrap execution succeeds", func() { + hostReconciler.K8sInstaller = fakeInstaller + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).ToNot(HaveOccurred()) + + Expect(fakeCommandRunner.RunCmdCallCount()).To(Equal(2)) + Expect(fakeFileWriter.WriteToFileCallCount()).To(Equal(1)) + + updatedByoHost := &infrastructurev1beta1.ByoHost{} + err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) + Expect(err).ToNot(HaveOccurred()) + + k8sNodeBootstrapSucceeded := conditions.Get(updatedByoHost, infrastructurev1beta1.K8sNodeBootstrapSucceeded) + Expect(*k8sNodeBootstrapSucceeded).To(conditions.MatchCondition(clusterv1.Condition{ + Type: infrastructurev1beta1.K8sNodeBootstrapSucceeded, + Status: corev1.ConditionTrue, + })) + + // assert events + events := eventutils.CollectEvents(recorder.Events) + Expect(events).Should(ConsistOf([]string{ + "Normal InstallScriptExecutionSucceeded install script executed", + "Normal BootstrapK8sNodeSucceeded k8s Node Bootstraped", + })) + }) + + AfterEach(func() { + Expect(k8sClient.Delete(ctx, installationSecret)).NotTo(HaveOccurred()) + }) + }) + + AfterEach(func() { + hostReconciler.UseInstallerController = false + }) + }) + AfterEach(func() { Expect(k8sClient.Delete(ctx, bootstrapSecret)).NotTo(HaveOccurred()) hostReconciler.SkipK8sInstallation = false @@ -397,7 +546,8 @@ runCmd: // assert kubeadm reset is called Expect(fakeCommandRunner.RunCmdCallCount()).To(Equal(1)) - Expect(fakeCommandRunner.RunCmdArgsForCall(0)).To(Equal(reconciler.KubeadmResetCommand)) + _, resetCommand := fakeCommandRunner.RunCmdArgsForCall(0) + Expect(resetCommand).To(Equal(reconciler.KubeadmResetCommand)) updatedByoHost := &infrastructurev1beta1.ByoHost{} err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) Expect(err).ToNot(HaveOccurred()) @@ -425,6 +575,105 @@ runCmd: })) }) + Context("When use-installer-controller is set", func() { + BeforeEach(func() { + hostReconciler.UseInstallerController = true + }) + + It("should return an error if we fail to load the uninstallation script", func() { + byoHost.Spec.UninstallationScript = nil + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).To(MatchError("UninstallationScript not found in Byohost " + byoHost.Name)) + }) + + It("should return error if uninstall script execution failed ", func() { + fakeCommandRunner.RunCmdReturnsOnCall(1, errors.New("failed to execute uninstall script")) + uninstallScript = `testcommand` + byoHost.Spec.UninstallationScript = &uninstallScript + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).To(HaveOccurred()) + + // assert events + events := eventutils.CollectEvents(recorder.Events) + Expect(events).Should(ConsistOf([]string{ + "Normal ResetK8sNodeSucceeded k8s Node Reset completed", + "Warning UninstallScriptExecutionFailed uninstall script execution failed", + })) + }) + + It("should set K8sComponentsInstallationSucceeded to false if uninstall succeeds", func() { + uninstallScript = `echo "uninstall success script"` + byoHost.Spec.UninstallationScript = &uninstallScript + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).ToNot(HaveOccurred()) + + updatedByoHost := &infrastructurev1beta1.ByoHost{} + err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) + Expect(err).ToNot(HaveOccurred()) + + K8sNodeBootstrapSucceeded := conditions.Get(updatedByoHost, infrastructurev1beta1.K8sNodeBootstrapSucceeded) + Expect(*K8sNodeBootstrapSucceeded).To(conditions.MatchCondition(clusterv1.Condition{ + Type: infrastructurev1beta1.K8sNodeBootstrapSucceeded, + Status: corev1.ConditionFalse, + Reason: infrastructurev1beta1.K8sNodeAbsentReason, + Severity: clusterv1.ConditionSeverityInfo, + })) + }) + + It("It should reset byoHost.Spec.InstallationSecret if uninstall succeeds", func() { + uninstallScript = `echo "uninstall success installation secret"` + byoHost.Spec.UninstallationScript = &uninstallScript + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).ToNot(HaveOccurred()) + + updatedByoHost := &infrastructurev1beta1.ByoHost{} + err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) + Expect(err).ToNot(HaveOccurred()) + Expect(updatedByoHost.Spec.InstallationSecret).To(BeNil()) + }) + + It("It should reset byoHost.Spec.UninstallationScript if uninstall succeeds", func() { + uninstallScript = `echo "uninstall script"` + byoHost.Spec.UninstallationScript = &uninstallScript + Expect(patchHelper.Patch(ctx, byoHost, patch.WithStatusObservedGeneration{})).NotTo(HaveOccurred()) + + result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ + NamespacedName: byoHostLookupKey, + }) + Expect(result).To(Equal(controllerruntime.Result{})) + Expect(reconcilerErr).ToNot(HaveOccurred()) + + updatedByoHost := &infrastructurev1beta1.ByoHost{} + err := k8sClient.Get(ctx, byoHostLookupKey, updatedByoHost) + Expect(err).ToNot(HaveOccurred()) + Expect(updatedByoHost.Spec.UninstallationScript).To(BeNil()) + + }) + + AfterEach(func() { + hostReconciler.UseInstallerController = false + }) + }) + It("should skip uninstallation if skip-installation flag is set", func() { hostReconciler.SkipK8sInstallation = true result, reconcilerErr := hostReconciler.Reconcile(ctx, controllerruntime.Request{ diff --git a/apis/infrastructure/v1beta1/byohost_types.go b/apis/infrastructure/v1beta1/byohost_types.go index 900040b0d..cf844e8ae 100644 --- a/apis/infrastructure/v1beta1/byohost_types.go +++ b/apis/infrastructure/v1beta1/byohost_types.go @@ -35,6 +35,11 @@ type ByoHostSpec struct { // generated by InstallerController for K8s installation // +optional InstallationSecret *corev1.ObjectReference `json:"installationSecret,omitempty"` + + // UninstallationScript is an optional field to store uninstall script + // generated by InstallerController + // +optional + UninstallationScript *string `json:"uninstallationScript,omitempty"` } // HostInfo is a set of details about the host platform. diff --git a/common/installer/internal/algo/ubuntu20_4k8s.go b/common/installer/internal/algo/ubuntu20_4k8s.go index 65b9da29f..919c2b525 100644 --- a/common/installer/internal/algo/ubuntu20_4k8s.go +++ b/common/installer/internal/algo/ubuntu20_4k8s.go @@ -67,7 +67,7 @@ func (s *Ubuntu20_04Installer) Uninstall() string { // contains the installation and uninstallation steps for the supported os and k8s var ( DoUbuntu20_4K8s1_22 = ` -set -euo pipefail +set -euox pipefail BUNDLE_DOWNLOAD_PATH={{.BundleDownloadPath}} BUNDLE_ADDR={{.BundleAddrs}} @@ -77,8 +77,19 @@ BUNDLE_PATH=$BUNDLE_DOWNLOAD_PATH/$BUNDLE_ADDR if ! command -v imgpkg >>/dev/null; then - echo "installing imgpkg" - wget -nv -O- github.com/vmware-tanzu/carvel-imgpkg/releases/download/$IMGPKG_VERSION/imgpkg-linux-$ARCH > /tmp/imgpkg + echo "installing imgpkg" + + if command -v wget >>/dev/null; then + dl_bin="wget -nv -O-" + elif command -v curl >>/dev/null; then + dl_bin="curl -s -L" + else + echo "installing curl" + apt-get install -y curl + dl_bin="curl -s -L" + fi + + $dl_bin github.com/vmware-tanzu/carvel-imgpkg/releases/download/$IMGPKG_VERSION/imgpkg-linux-$ARCH > /tmp/imgpkg mv /tmp/imgpkg /usr/local/bin/imgpkg chmod +x /usr/local/bin/imgpkg fi @@ -114,36 +125,36 @@ tar -C / -xvf "$BUNDLE_PATH/containerd.tar" systemctl daemon-reload && systemctl enable containerd && systemctl start containerd` UndoUbuntu20_4K8s1_22 = ` -set -euo pipefail +set -euox pipefail BUNDLE_DOWNLOAD_PATH={{.BundleDownloadPath}} BUNDLE_ADDR={{.BundleAddrs}} BUNDLE_PATH=$BUNDLE_DOWNLOAD_PATH/$BUNDLE_ADDR -## enable swap -swapon -a && sed -ri '/\sswap\s/s/^#?//' /etc/fstab - -## enable firewall -if command -v ufw >>/dev/null; then - ufw enable -fi - -## remove kernal modules -modprobe -r overlay && modprobe -r br_netfilter +## disabling containerd service +systemctl stop containerd && systemctl disable containerd && systemctl daemon-reload -## removing os configuration -tar tf "$BUNDLE_PATH/conf.tar" | xargs -n 1 echo '/' | sed 's/ //g' | xargs rm -f +## removing containerd configurations and cni plugins +rm -rf /opt/cni/ && rm -rf /opt/containerd/ && tar tf "$BUNDLE_PATH/containerd.tar" | xargs -n 1 echo '/' | sed 's/ //g' | grep -e '[^/]$' | xargs rm -f ## removing deb packages -for pkg in cri-tools kubernetes-cni kubectl kubeadm kubelet; do +for pkg in kubeadm kubelet kubectl kubernetes-cni cri-tools; do dpkg --purge $pkg done -## removing containerd configurations and cni plugins -rm -rf /opt/cni/ && rm -rf /opt/containerd/ && tar tf "$BUNDLE_PATH/containerd.tar" | xargs -n 1 echo '/' | sed 's/ //g' | grep -e '[^/]$' | xargs rm -f +## removing os configuration +tar tf "$BUNDLE_PATH/conf.tar" | xargs -n 1 echo '/' | sed 's/ //g' | grep -e "[^/]$" | xargs rm -f -## disabling containerd service -systemctl stop containerd && systemctl disable containerd && systemctl daemon-reload +## remove kernal modules +modprobe -rq overlay && modprobe -r br_netfilter + +## enable firewall +if command -v ufw >>/dev/null; then + ufw enable +fi + +## enable swap +swapon -a && sed -ri '/\sswap\s/s/^#?//' /etc/fstab rm -rf $BUNDLE_PATH` ) diff --git a/test/builder/builders.go b/test/builder/builders.go index 7f00985b7..96082961a 100644 --- a/test/builder/builders.go +++ b/test/builder/builders.go @@ -338,6 +338,15 @@ func (s *SecretBuilder) WithData(value string) *SecretBuilder { return s } +// WithKeyData adds the passed key and data to the SecretBuilder +func (s *SecretBuilder) WithKeyData(key, data string) *SecretBuilder { + if s.data == nil { + s.data = make(map[string][]byte) + } + s.data[key] = []byte(data) + return s +} + // Build returns a Secret with the attributes added to the SecretBuilder func (s *SecretBuilder) Build() *corev1.Secret { secret := &corev1.Secret{