diff --git a/controllers/export_test.go b/controllers/export_test.go index 5d8f51c..1acc211 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -129,6 +129,7 @@ var ( GetHelmReferenceResourceHash = getHelmReferenceResourceHash GetHelmChartValuesHash = getHelmChartValuesHash GetCredentialsAndCAFiles = getCredentialsAndCAFiles + GetInstantiatedChart = getInstantiatedChart InstantiateTemplateValues = instantiateTemplateValues diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index a402ac0..3e3a06b 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -34,6 +34,7 @@ import ( "github.com/gdexlab/go-render/render" "github.com/go-logr/logr" "github.com/pkg/errors" + "gopkg.in/yaml.v2" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" @@ -441,18 +442,23 @@ func getHelmRefs(clusterSummary *configv1beta1.ClusterSummary) []configv1beta1.P func handleCharts(ctx context.Context, clusterSummary *configv1beta1.ClusterSummary, c, remoteClient client.Client, kubeconfig string, logger logr.Logger) error { + mgmtResources, err := collectTemplateResourceRefs(ctx, clusterSummary) + if err != nil { + return err + } + // Before any helm release, managed by this ClusterSummary, is deployed, update ClusterSummary // Status. Order is important. If pod is restarted, it needs to rebuild internal state keeping // track of which ClusterSummary was managing which helm release. // Here only currently referenced helm releases are considered. If ClusterSummary was managing // an helm release and it is not referencing it anymore, such entry will be removed from ClusterSummary.Status // only after helm release is successfully undeployed. - clusterSummary, _, err := updateStatusForeferencedHelmReleases(ctx, c, clusterSummary, logger) + clusterSummary, _, err = updateStatusForeferencedHelmReleases(ctx, c, clusterSummary, mgmtResources, logger) if err != nil { return err } - releaseReports, chartDeployed, deployError := walkChartsAndDeploy(ctx, c, clusterSummary, kubeconfig, logger) + releaseReports, chartDeployed, deployError := walkChartsAndDeploy(ctx, c, clusterSummary, kubeconfig, mgmtResources, logger) // Even if there is a deployment error do not return just yet. Update various status and clean stale resources. // If there was an helm release previous managed by this ClusterSummary and currently not referenced @@ -503,12 +509,8 @@ func handleCharts(ctx context.Context, clusterSummary *configv1beta1.ClusterSumm // walkChartsAndDeploy walks all referenced helm charts. Deploys (install or upgrade) any chart // this clusterSummary is registered to manage. func walkChartsAndDeploy(ctx context.Context, c client.Client, clusterSummary *configv1beta1.ClusterSummary, - kubeconfig string, logger logr.Logger) ([]configv1beta1.ReleaseReport, []configv1beta1.Chart, error) { - - mgmtResources, err := collectTemplateResourceRefs(ctx, clusterSummary) - if err != nil { - return nil, nil, err - } + kubeconfig string, mgmtResources map[string]*unstructured.Unstructured, logger logr.Logger, +) ([]configv1beta1.ReleaseReport, []configv1beta1.Chart, error) { chartManager, err := chartmanager.GetChartManagerInstance(ctx, c) if err != nil { @@ -520,16 +522,22 @@ func walkChartsAndDeploy(ctx context.Context, c client.Client, clusterSummary *c chartDeployed := make([]configv1beta1.Chart, 0, len(clusterSummary.Spec.ClusterProfileSpec.HelmCharts)) for i := range clusterSummary.Spec.ClusterProfileSpec.HelmCharts { currentChart := &clusterSummary.Spec.ClusterProfileSpec.HelmCharts[i] + + instantiatedChart, err := getInstantiatedChart(ctx, clusterSummary, currentChart, mgmtResources, logger) + if err != nil { + return releaseReports, chartDeployed, err + } + // Eventual conflicts are already resolved before this method is called (in updateStatusForeferencedHelmReleases) // So it is safe to call CanManageChart here - if !chartManager.CanManageChart(clusterSummary, currentChart) { + if !chartManager.CanManageChart(clusterSummary, instantiatedChart) { var report *configv1beta1.ReleaseReport - report, err = createReportForUnmanagedHelmRelease(ctx, c, clusterSummary, currentChart, logger) + report, err = createReportForUnmanagedHelmRelease(ctx, c, clusterSummary, instantiatedChart, logger) if err != nil { return releaseReports, chartDeployed, err } releaseReports = append(releaseReports, *report) - conflictErrorMessage += generateConflictForHelmChart(ctx, clusterSummary, currentChart) + conflictErrorMessage += generateConflictForHelmChart(ctx, clusterSummary, instantiatedChart) // error is reported above, in updateHelmChartStatus. if clusterSummary.Spec.ClusterProfileSpec.ContinueOnConflict || clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun { @@ -546,11 +554,11 @@ func walkChartsAndDeploy(ctx context.Context, c client.Client, clusterSummary *c var report *configv1beta1.ReleaseReport var currentRelease *releaseInfo - currentRelease, report, err = handleChart(ctx, clusterSummary, mgmtResources, currentChart, kubeconfig, logger) + currentRelease, report, err = handleChart(ctx, clusterSummary, mgmtResources, instantiatedChart, kubeconfig, logger) if err != nil { return releaseReports, chartDeployed, err } - err = updateValueHashOnHelmChartSummary(ctx, currentChart, clusterSummary, logger) + err = updateValueHashOnHelmChartSummary(ctx, instantiatedChart, clusterSummary, logger) if err != nil { return releaseReports, chartDeployed, err } @@ -563,7 +571,7 @@ func walkChartsAndDeploy(ctx context.Context, c client.Client, clusterSummary *c if currentRelease.Status == release.StatusDeployed.String() { // Deployed chart is used for updating ClusterConfiguration. There is no ClusterConfiguration for mgmt cluster chartDeployed = append(chartDeployed, configv1beta1.Chart{ - RepoURL: currentChart.RepositoryURL, + RepoURL: instantiatedChart.RepositoryURL, Namespace: currentRelease.ReleaseNamespace, ReleaseName: currentRelease.ReleaseName, ChartVersion: currentRelease.ChartVersion, @@ -1586,7 +1594,8 @@ func undeployStaleReleases(ctx context.Context, c client.Client, clusterSummary // allowed to manage. // No action in DryRun mode. func updateStatusForeferencedHelmReleases(ctx context.Context, c client.Client, - clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) (*configv1beta1.ClusterSummary, bool, error) { + clusterSummary *configv1beta1.ClusterSummary, mgmtResources map[string]*unstructured.Unstructured, + logger logr.Logger) (*configv1beta1.ClusterSummary, bool, error) { // No-op in DryRun mode if clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun { @@ -1623,30 +1632,36 @@ func updateStatusForeferencedHelmReleases(ctx context.Context, c client.Client, helmReleaseSummaries := make([]configv1beta1.HelmChartSummary, len(currentClusterSummary.Spec.ClusterProfileSpec.HelmCharts)) for i := range currentClusterSummary.Spec.ClusterProfileSpec.HelmCharts { currentChart := ¤tClusterSummary.Spec.ClusterProfileSpec.HelmCharts[i] + + instantiatedChart, err := getInstantiatedChart(ctx, clusterSummary, currentChart, mgmtResources, logger) + if err != nil { + return err + } + var canManage bool - canManage, err = determineChartOwnership(ctx, c, clusterSummary, currentChart, logger) + canManage, err = determineChartOwnership(ctx, c, clusterSummary, instantiatedChart, logger) if err != nil { return err } if canManage { helmReleaseSummaries[i] = configv1beta1.HelmChartSummary{ - ReleaseName: currentChart.ReleaseName, - ReleaseNamespace: currentChart.ReleaseNamespace, + ReleaseName: instantiatedChart.ReleaseName, + ReleaseNamespace: instantiatedChart.ReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, - ValuesHash: getValueHashFromHelmChartSummary(currentChart, clusterSummary), // if a value is currently stored, keep it. + ValuesHash: getValueHashFromHelmChartSummary(instantiatedChart, clusterSummary), // if a value is currently stored, keep it. // after chart is deployed such value will be updated } - currentlyReferenced[helmInfo(currentChart.ReleaseNamespace, currentChart.ReleaseName)] = true + currentlyReferenced[helmInfo(instantiatedChart.ReleaseNamespace, instantiatedChart.ReleaseName)] = true } else { var managerName string managerName, err = chartManager.GetManagerForChart(currentClusterSummary.Spec.ClusterNamespace, - currentClusterSummary.Spec.ClusterName, currentClusterSummary.Spec.ClusterType, currentChart) + currentClusterSummary.Spec.ClusterName, currentClusterSummary.Spec.ClusterType, instantiatedChart) if err != nil { return err } helmReleaseSummaries[i] = configv1beta1.HelmChartSummary{ - ReleaseName: currentChart.ReleaseName, - ReleaseNamespace: currentChart.ReleaseNamespace, + ReleaseName: instantiatedChart.ReleaseName, + ReleaseNamespace: instantiatedChart.ReleaseNamespace, Status: configv1beta1.HelmChartStatusConflict, ConflictMessage: fmt.Sprintf("ClusterSummary %s managing it", managerName), } @@ -2696,3 +2711,29 @@ func requeueAllOtherClusterSummaries(ctx context.Context, c client.Client, return nil } + +func getInstantiatedChart(ctx context.Context, clusterSummary *configv1beta1.ClusterSummary, + currentChart *configv1beta1.HelmChart, mgmtResources map[string]*unstructured.Unstructured, + logger logr.Logger) (*configv1beta1.HelmChart, error) { + + // Marshal the struct to YAML + jsonData, err := yaml.Marshal(*currentChart) + if err != nil { + return nil, err + } + + instantiatedChartString, err := instantiateTemplateValues(ctx, getManagementClusterConfig(), + getManagementClusterClient(), clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, + clusterSummary.Spec.ClusterName, currentChart.ChartName, string(jsonData), mgmtResources, logger) + if err != nil { + return nil, err + } + + var instantiatedChart configv1beta1.HelmChart + err = yaml.Unmarshal([]byte(instantiatedChartString), &instantiatedChart) + if err != nil { + return nil, err + } + + return &instantiatedChart, nil +} diff --git a/controllers/handlers_helm_test.go b/controllers/handlers_helm_test.go index 834e09b..cef18a5 100644 --- a/controllers/handlers_helm_test.go +++ b/controllers/handlers_helm_test.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2/textlogger" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -49,6 +50,8 @@ var _ = Describe("HandlersHelm", func() { var clusterProfile *configv1beta1.ClusterProfile var clusterSummary *configv1beta1.ClusterSummary + const defaulNamespace = "default" + BeforeEach(func() { clusterNamespace := randomString() @@ -76,6 +79,7 @@ var _ = Describe("HandlersHelm", func() { Kind: configv1beta1.ClusterProfileKind, Name: clusterProfile.Name, APIVersion: "config.projectsveltos.io/v1beta1", + UID: types.UID(randomString()), }, }, }, @@ -182,44 +186,67 @@ var _ = Describe("HandlersHelm", func() { HelmCharts: []configv1beta1.HelmChart{*calicoChart}, } + clusterSummary.Namespace = defaulNamespace + clusterSummary.Spec.ClusterNamespace = defaulNamespace + + Expect(testEnv.Create(context.TODO(), clusterSummary)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, clusterSummary)).To(Succeed()) + // List a helm chart non referenced anymore as managed clusterSummary.Status = configv1beta1.ClusterSummaryStatus{ HelmReleaseSummaries: []configv1beta1.HelmChartSummary{ kyvernoSummary, }, } + Expect(testEnv.Status().Update(context.TODO(), clusterSummary)).To(Succeed()) - initObjects := []client.Object{ - clusterSummary, + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterSummary.Spec.ClusterName, + Namespace: clusterSummary.Spec.ClusterNamespace, + }, } - c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...).Build() + Expect(testEnv.Create(context.TODO(), cluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, cluster)).To(Succeed()) - manager, err := chartmanager.GetChartManagerInstance(context.TODO(), c) + manager, err := chartmanager.GetChartManagerInstance(context.TODO(), testEnv.Client) Expect(err).To(BeNil()) manager.RegisterClusterSummaryForCharts(clusterSummary) - clusterSummary, conflict, err := controllers.UpdateStatusForeferencedHelmReleases(context.TODO(), c, clusterSummary, - textlogger.NewLogger(textlogger.NewConfig())) + clusterSummary, conflict, err := controllers.UpdateStatusForeferencedHelmReleases(context.TODO(), + testEnv.Client, clusterSummary, nil, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) Expect(conflict).To(BeFalse()) - currentClusterSummary := &configv1beta1.ClusterSummary{} - Expect(c.Get(context.TODO(), - types.NamespacedName{Namespace: clusterSummary.Namespace, Name: clusterSummary.Name}, - currentClusterSummary)).To(Succeed()) - Expect(currentClusterSummary.Status.HelmReleaseSummaries).ToNot(BeNil()) - Expect(len(currentClusterSummary.Status.HelmReleaseSummaries)).To(Equal(2)) - Expect(currentClusterSummary.Status.HelmReleaseSummaries[0].Status).To(Equal(configv1beta1.HelmChartStatusManaging)) - Expect(currentClusterSummary.Status.HelmReleaseSummaries[0].ReleaseName).To(Equal(calicoChart.ReleaseName)) - Expect(currentClusterSummary.Status.HelmReleaseSummaries[0].ReleaseNamespace).To(Equal(calicoChart.ReleaseNamespace)) - - // UpdateStatusForeferencedHelmReleases adds status for referenced releases and does not remove any - // existing entry for non existing releases. - Expect(currentClusterSummary.Status.HelmReleaseSummaries[1].Status).To(Equal(kyvernoSummary.Status)) - Expect(currentClusterSummary.Status.HelmReleaseSummaries[1].ReleaseName).To(Equal(kyvernoSummary.ReleaseName)) - Expect(currentClusterSummary.Status.HelmReleaseSummaries[1].ReleaseNamespace).To(Equal(kyvernoSummary.ReleaseNamespace)) + Eventually(func() bool { + currentClusterSummary := &configv1beta1.ClusterSummary{} + err = testEnv.Get(context.TODO(), + types.NamespacedName{Namespace: clusterSummary.Namespace, Name: clusterSummary.Name}, + currentClusterSummary) + if err != nil { + return false + } + if currentClusterSummary.Status.HelmReleaseSummaries == nil { + return false + } + if len(currentClusterSummary.Status.HelmReleaseSummaries) != 2 { + return false + } + if currentClusterSummary.Status.HelmReleaseSummaries[0].Status != configv1beta1.HelmChartStatusManaging || + currentClusterSummary.Status.HelmReleaseSummaries[0].ReleaseName != calicoChart.ReleaseName || + currentClusterSummary.Status.HelmReleaseSummaries[0].ReleaseNamespace != calicoChart.ReleaseNamespace { + return false + } + + // UpdateStatusForeferencedHelmReleases adds status for referenced releases and does not remove any + // existing entry for non existing releases. + return currentClusterSummary.Status.HelmReleaseSummaries[1].Status == kyvernoSummary.Status && + currentClusterSummary.Status.HelmReleaseSummaries[1].ReleaseName == kyvernoSummary.ReleaseName && + currentClusterSummary.Status.HelmReleaseSummaries[1].ReleaseNamespace == kyvernoSummary.ReleaseNamespace + }, timeout, pollingInterval).Should(BeTrue()) + }) It("updateStatusForeferencedHelmReleases is no-op in DryRun mode", func() { @@ -244,7 +271,7 @@ var _ = Describe("HandlersHelm", func() { c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...).Build() - clusterSummary, conflict, err := controllers.UpdateStatusForeferencedHelmReleases(context.TODO(), c, clusterSummary, + clusterSummary, conflict, err := controllers.UpdateStatusForeferencedHelmReleases(context.TODO(), c, clusterSummary, nil, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) Expect(conflict).To(BeFalse()) @@ -406,6 +433,87 @@ var _ = Describe("HandlersHelm", func() { Expect(report.ReleaseNamespace).To(Equal(helmChart.ReleaseNamespace)) }) + It("getInstantiatedChart returns instantiated HelmChart matching passed in chart", func() { + helmChart := &configv1beta1.HelmChart{ + ReleaseName: randomString(), ReleaseNamespace: randomString(), + ChartName: randomString(), ChartVersion: randomString(), + RepositoryURL: randomString(), RepositoryName: randomString(), + HelmChartAction: configv1beta1.HelmChartActionInstall, + } + + clusterSummary.Namespace = defaulNamespace + clusterSummary.Spec.ClusterNamespace = defaulNamespace + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterSummary.Spec.ClusterName, + Namespace: clusterSummary.Spec.ClusterNamespace, + }, + } + + Expect(testEnv.Create(context.TODO(), cluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, cluster)).To(Succeed()) + + Expect(testEnv.Create(context.TODO(), clusterSummary)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, clusterSummary)).To(Succeed()) + + instaniatedChart, err := controllers.GetInstantiatedChart(context.TODO(), clusterSummary, helmChart, nil, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(instaniatedChart.ReleaseName).To(Equal(helmChart.ReleaseName)) + Expect(instaniatedChart.ReleaseNamespace).To(Equal(helmChart.ReleaseNamespace)) + Expect(instaniatedChart.ChartName).To(Equal(helmChart.ChartName)) + Expect(instaniatedChart.RepositoryURL).To(Equal(helmChart.RepositoryURL)) + Expect(instaniatedChart.RepositoryName).To(Equal(helmChart.RepositoryName)) + Expect(instaniatedChart.HelmChartAction).To(Equal(helmChart.HelmChartAction)) + Expect(instaniatedChart.ChartVersion).To(Equal(helmChart.ChartVersion)) + }) + + It("getInstantiatedChart returns instantiated HelmChart", func() { + helmChart := &configv1beta1.HelmChart{ + ReleaseName: randomString(), ReleaseNamespace: randomString(), + ChartName: randomString(), RepositoryURL: randomString(), + RepositoryName: randomString(), HelmChartAction: configv1beta1.HelmChartActionInstall, + } + + helmChart.ChartVersion = `{{$version := index .Cluster.metadata.labels "k8s-version" }}{{if eq $version "1.20"}}23.4.0 +{{else if eq $version "1.22"}}24.1.0 +{{else if eq $version "1.25"}}25.0.2 +{{ else }}23.4.0 +{{end}}` + + clusterSummary.Namespace = defaulNamespace + clusterSummary.Spec.ClusterNamespace = defaulNamespace + clusterSummary.Spec.ClusterType = libsveltosv1beta1.ClusterTypeSveltos + + cluster := &libsveltosv1beta1.SveltosCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterSummary.Spec.ClusterName, + Namespace: clusterSummary.Spec.ClusterNamespace, + Labels: map[string]string{ + "k8s-version": "1.25", + }, + }, + } + + Expect(testEnv.Create(context.TODO(), cluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, cluster)).To(Succeed()) + + Expect(testEnv.Create(context.TODO(), clusterSummary)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, clusterSummary)).To(Succeed()) + + instaniatedChart, err := controllers.GetInstantiatedChart(context.TODO(), clusterSummary, helmChart, nil, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(instaniatedChart.ReleaseName).To(Equal(helmChart.ReleaseName)) + Expect(instaniatedChart.ReleaseNamespace).To(Equal(helmChart.ReleaseNamespace)) + Expect(instaniatedChart.ChartName).To(Equal(helmChart.ChartName)) + Expect(instaniatedChart.RepositoryURL).To(Equal(helmChart.RepositoryURL)) + Expect(instaniatedChart.RepositoryName).To(Equal(helmChart.RepositoryName)) + Expect(instaniatedChart.HelmChartAction).To(Equal(helmChart.HelmChartAction)) + Expect(instaniatedChart.ChartVersion).To(Equal("25.0.2")) + }) + It("updateClusterReportWithHelmReports updates ClusterReports with HelmReports", func() { helmChart := &configv1beta1.HelmChart{ ReleaseName: randomString(), ReleaseNamespace: randomString(), @@ -502,6 +610,16 @@ var _ = Describe("HandlersHelm", func() { Expect(testEnv.Client.Create(context.TODO(), ns)).To(Succeed()) Expect(waitForObject(context.TODO(), testEnv.Client, ns)).To(Succeed()) + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterSummary.Spec.ClusterName, + Namespace: clusterSummary.Spec.ClusterNamespace, + }, + } + + Expect(testEnv.Create(context.TODO(), cluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, cluster)).To(Succeed()) + Expect(testEnv.Client.Create(context.TODO(), clusterProfile)).To(Succeed()) clusterSummary.OwnerReferences = []metav1.OwnerReference{ {