From a144bc301b87a72f426300caf6865ddc6bdcc81b Mon Sep 17 00:00:00 2001 From: Geoffrey MUSELLI Date: Mon, 14 Aug 2023 10:26:09 -0400 Subject: [PATCH] fix(appset): Fix helm valuesObject with ApplicationSet (#14912) (#14920) Signed-off-by: Geoffrey Muselli --- applicationset/utils/utils.go | 40 +++++++++-- applicationset/utils/utils_test.go | 108 +++++++++++++++++++++++++++++ test/e2e/applicationset_test.go | 80 +++++++++++++++++++++ 3 files changed, 222 insertions(+), 6 deletions(-) diff --git a/applicationset/utils/utils.go b/applicationset/utils/utils.go index 90e65ef5dafdc..ae069799a9d0c 100644 --- a/applicationset/utils/utils.go +++ b/applicationset/utils/utils.go @@ -17,6 +17,7 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/valyala/fasttemplate" + "sigs.k8s.io/yaml" log "github.com/sirupsen/logrus" @@ -51,6 +52,22 @@ func copyUnexported(copy, original reflect.Value) { copyValueIntoUnexported(copy, unexported) } +func IsJSONStr(str string) bool { + str = strings.TrimSpace(str) + return len(str) > 0 && str[0] == '{' +} + +func ConvertYAMLToJSON(str string) (string, error) { + if !IsJSONStr(str) { + jsonStr, err := yaml.YAMLToJSON([]byte(str)) + if err != nil { + return str, err + } + return string(jsonStr), nil + } + return str, nil +} + // This function is in charge of searching all String fields of the object recursively and apply templating // thanks to https://gist.github.com/randallmlough/1fd78ec8a1034916ca52281e3b886dc7 func (r *Render) deeplyReplace(copy, original reflect.Value, replaceMap map[string]interface{}, useGoTemplate bool, goTemplateOptions []string) error { @@ -86,11 +103,18 @@ func (r *Render) deeplyReplace(copy, original reflect.Value, replaceMap map[stri originalValue := original.Elem() // Create a new object. Now new gives us a pointer, but we want the value it // points to, so we have to call Elem() to unwrap it - copyValue := reflect.New(originalValue.Type()).Elem() - if err := r.deeplyReplace(copyValue, originalValue, replaceMap, useGoTemplate, goTemplateOptions); err != nil { - return err + + if originalValue.IsValid() { + reflectType := originalValue.Type() + + reflectValue := reflect.New(reflectType) + + copyValue := reflectValue.Elem() + if err := r.deeplyReplace(copyValue, originalValue, replaceMap, useGoTemplate, goTemplateOptions); err != nil { + return err + } + copy.Set(copyValue) } - copy.Set(copyValue) // If it is a struct we translate each field case reflect.Struct: @@ -99,10 +123,14 @@ func (r *Render) deeplyReplace(copy, original reflect.Value, replaceMap map[stri // specific case time if currentType == "time.Time" { copy.Field(i).Set(original.Field(i)) - } else if currentType == "Raw.k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" { + } else if currentType == "Raw.k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" || currentType == "Raw.k8s.io/apimachinery/pkg/runtime" { var unmarshaled interface{} originalBytes := original.Field(i).Bytes() - err := json.Unmarshal(originalBytes, &unmarshaled) + convertedToJson, err := ConvertYAMLToJSON(string(originalBytes)) + if err != nil { + return fmt.Errorf("error while converting template to json %q: %w", convertedToJson, err) + } + err = json.Unmarshal([]byte(convertedToJson), &unmarshaled) if err != nil { return fmt.Errorf("failed to unmarshal JSON field: %w", err) } diff --git a/applicationset/utils/utils_test.go b/applicationset/utils/utils_test.go index 0a62275befc9f..cfabd9ce088c0 100644 --- a/applicationset/utils/utils_test.go +++ b/applicationset/utils/utils_test.go @@ -2,6 +2,7 @@ package utils import ( "crypto/x509" + "encoding/json" "os" "path" "testing" @@ -198,6 +199,113 @@ func TestRenderTemplateParams(t *testing.T) { } +func TestRenderHelmValuesObjectJson(t *testing.T) { + + params := map[string]interface{}{ + "test": "Hello world", + } + + application := &argoappsv1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"annotation-key": "annotation-value", "annotation-key2": "annotation-value2"}, + Labels: map[string]string{"label-key": "label-value", "label-key2": "label-value2"}, + CreationTimestamp: metav1.NewTime(time.Now()), + UID: types.UID("d546da12-06b7-4f9a-8ea2-3adb16a20e2b"), + Name: "application-one", + Namespace: "default", + }, + Spec: argoappsv1.ApplicationSpec{ + Source: &argoappsv1.ApplicationSource{ + Path: "", + RepoURL: "", + TargetRevision: "", + Chart: "", + Helm: &argoappsv1.ApplicationSourceHelm{ + ValuesObject: &runtime.RawExtension{ + Raw: []byte(`{ + "some": { + "string": "{{.test}}" + } + }`), + }, + }, + }, + Destination: argoappsv1.ApplicationDestination{ + Server: "", + Namespace: "", + Name: "", + }, + Project: "", + }, + } + + // Render the cloned application, into a new application + render := Render{} + newApplication, err := render.RenderTemplateParams(application, nil, params, true, []string{}) + + assert.NoError(t, err) + assert.NotNil(t, newApplication) + + var unmarshaled interface{} + err = json.Unmarshal(newApplication.Spec.Source.Helm.ValuesObject.Raw, &unmarshaled) + + assert.NoError(t, err) + assert.Equal(t, unmarshaled.(map[string]interface{})["some"].(map[string]interface{})["string"], "Hello world") + +} + +func TestRenderHelmValuesObjectYaml(t *testing.T) { + + params := map[string]interface{}{ + "test": "Hello world", + } + + application := &argoappsv1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"annotation-key": "annotation-value", "annotation-key2": "annotation-value2"}, + Labels: map[string]string{"label-key": "label-value", "label-key2": "label-value2"}, + CreationTimestamp: metav1.NewTime(time.Now()), + UID: types.UID("d546da12-06b7-4f9a-8ea2-3adb16a20e2b"), + Name: "application-one", + Namespace: "default", + }, + Spec: argoappsv1.ApplicationSpec{ + Source: &argoappsv1.ApplicationSource{ + Path: "", + RepoURL: "", + TargetRevision: "", + Chart: "", + Helm: &argoappsv1.ApplicationSourceHelm{ + ValuesObject: &runtime.RawExtension{ + Raw: []byte(`some: + string: "{{.test}}"`), + }, + }, + }, + Destination: argoappsv1.ApplicationDestination{ + Server: "", + Namespace: "", + Name: "", + }, + Project: "", + }, + } + + // Render the cloned application, into a new application + render := Render{} + newApplication, err := render.RenderTemplateParams(application, nil, params, true, []string{}) + + assert.NoError(t, err) + assert.NotNil(t, newApplication) + + var unmarshaled interface{} + err = json.Unmarshal(newApplication.Spec.Source.Helm.ValuesObject.Raw, &unmarshaled) + + assert.NoError(t, err) + assert.Equal(t, unmarshaled.(map[string]interface{})["some"].(map[string]interface{})["string"], "Hello world") + +} + func TestRenderTemplateParamsGoTemplate(t *testing.T) { // Believe it or not, this is actually less complex than the equivalent solution using reflection diff --git a/test/e2e/applicationset_test.go b/test/e2e/applicationset_test.go index 80c406c1b62a3..27c5fd33b42c9 100644 --- a/test/e2e/applicationset_test.go +++ b/test/e2e/applicationset_test.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" argov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" @@ -352,6 +353,85 @@ func TestSimpleListGeneratorGoTemplate(t *testing.T) { } +func TestRenderHelmValuesObject(t *testing.T) { + + expectedApp := argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: application.ApplicationKind, + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster-guestbook", + Namespace: fixture.TestNamespace(), + Finalizers: []string{"resources-finalizer.argocd.argoproj.io"}, + Labels: map[string]string{ + LabelKeyAppSetInstance: "test-values-object", + }, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: &argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "helm-guestbook", + Helm: &argov1alpha1.ApplicationSourceHelm{ + ValuesObject: &runtime.RawExtension{ + // This will always be converted as yaml + Raw: []byte(`{"some":{"string":"Hello world"}}`), + }, + }, + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook", + }, + }, + } + + Given(t). + // Create a ListGenerator-based ApplicationSet + When().Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "test-values-object", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{.cluster}}-guestbook"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: &argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "helm-guestbook", + Helm: &argov1alpha1.ApplicationSourceHelm{ + ValuesObject: &runtime.RawExtension{ + Raw: []byte(`{"some":{"string":"{{.test}}"}}`), + }, + }, + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "{{.url}}", + Namespace: "guestbook", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + List: &v1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{{ + Raw: []byte(`{"cluster": "my-cluster","url": "https://kubernetes.default.svc", "test": "Hello world"}`), + }}, + }, + }, + }, + }, + }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedApp})). + // Delete the ApplicationSet, and verify it deletes the Applications + When(). + Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{expectedApp})) + +} + func TestSyncPolicyCreateUpdate(t *testing.T) { expectedApp := argov1alpha1.Application{