From b45d204126c5ee6b6fa22139e76b2a829d63dedd Mon Sep 17 00:00:00 2001 From: linglingye001 <143174321+linglingye001@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:18:47 +0800 Subject: [PATCH] Add more test cases for different scenarios (#64) * add more test for refresh scenarios * add more test cases * add more refresh test cases * resolve comments --- .../appconfigurationprovider_controller.go | 2 +- ...ppconfigurationprovider_controller_test.go | 1513 ++++++++++++++-- internal/controller/processor.go | 16 +- internal/controller/processor_test.go | 1537 +++++++++++++++++ 4 files changed, 2901 insertions(+), 167 deletions(-) create mode 100644 internal/controller/processor_test.go diff --git a/internal/controller/appconfigurationprovider_controller.go b/internal/controller/appconfigurationprovider_controller.go index 2530546..7dd3dcf 100644 --- a/internal/controller/appconfigurationprovider_controller.go +++ b/internal/controller/appconfigurationprovider_controller.go @@ -235,7 +235,7 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context processor := &AppConfigurationProviderProcessor{ Context: ctx, Provider: provider, - Retriever: &retriever, + Retriever: retriever, CurrentTime: metav1.Now(), ReconciliationState: reconciler.ProvidersReconcileState[req.NamespacedName], Settings: &loader.TargetKeyValueSettings{}, diff --git a/internal/controller/appconfigurationprovider_controller_test.go b/internal/controller/appconfigurationprovider_controller_test.go index b5adc28..10a4bde 100644 --- a/internal/controller/appconfigurationprovider_controller_test.go +++ b/internal/controller/appconfigurationprovider_controller_test.go @@ -95,6 +95,8 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) Expect(configmap.Data["testKey"]).Should(Equal("testValue")) Expect(createdProvider.Status.Phase).Should(Equal(acpv1.PhaseComplete)) + + _ = k8sClient.Delete(ctx, configProvider) }) It("Should create new configMap", func() { @@ -146,6 +148,8 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Data["testKey"]).Should(Equal("testValue")) Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + _ = k8sClient.Delete(ctx, configProvider) }) It("Should create new secret", func() { @@ -242,6 +246,73 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(string(secret2.Data["testSecretKey2"])).Should(Equal("testSecretValue2")) Expect(string(secret2.Data["testSecretKey3"])).Should(Equal("testSecretValue3")) Expect(secret2.Type).Should(Equal(corev1.SecretTypeOpaque)) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Should create empty secret successfully when secret section specified", func() { + By("even no Key Vault references loaded from AppConfig") + secretResult := make(map[string][]byte) + + secretName := "secret-to-be-created-empty" + allSettings := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretTypeOpaque, + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeOpaque, + SecretsMetadata: make(map[string]loader.KeyVaultSecretMetadata), + }, + }, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "test-appconfigurationprovider-emptysecret" + configMapName := "configmap-to-be-created-with-empty-secret" + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secret := &corev1.Secret{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, time.Second*5, interval).Should(BeTrue()) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(len(secret.Data)).Should(Equal(0)) + Expect(secret.Type).Should(Equal(corev1.SecretTypeOpaque)) + + _ = k8sClient.Delete(ctx, configProvider) }) It("Should create proper configmap and secret", func() { @@ -331,6 +402,8 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(string(secret.Data["testSecretKey2"])).Should(Equal("testSecretValue2")) Expect(string(secret.Data["testSecretKey3"])).Should(Equal("testSecretValue3")) Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + + _ = k8sClient.Delete(ctx, configProvider) }) It("Should create file style configMap", func() { @@ -384,6 +457,8 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Labels["bar"]).Should(Equal("barValue")) Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"testKey\":\"testValue\",\"testKey2\":\"testValue2\",\"testKey3\":\"testValue3\"}")) Expect(len(configmap.Data)).Should(Equal(1)) + + _ = k8sClient.Delete(ctx, configProvider) }) It("Should create file style ConfigMap with feature flag settings", func() { @@ -442,6 +517,8 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"testKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}")) Expect(len(configmap.Data)).Should(Equal(1)) + + _ = k8sClient.Delete(ctx, configProvider) }) It("Should refresh configMap", func() { @@ -523,6 +600,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Data["testKey2"]).Should(Equal("newtestValue2")) Expect(configmap.Data["testKey3"]).Should(Equal("newtestValue3")) + _ = k8sClient.Delete(ctx, configProvider) }) It("Should refresh configMap", func() { @@ -611,49 +689,24 @@ var _ = Describe("AppConfiguationProvider controller", func() { _ = k8sClient.Delete(ctx, configProvider) }) - It("Should refresh secret when data change", func() { - By("By enabling refresh on secret") - configMapResult := make(map[string]string) - configMapResult["testKey"] = "testValue" - configMapResult["testKey2"] = "testValue2" - configMapResult["testKey3"] = "testValue3" - - secretResult := make(map[string][]byte) - secretResult["testSecretKey"] = []byte("testSecretValue") - - secretName := "secret-to-be-refreshed-3" - var fakeId azsecrets.ID = "fakeSecretId" - secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) - secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ - SecretId: &fakeId, - } + It("Should refresh file style ConfigMap", func() { + By("when data change in App Configuration store") + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"testKey\":\"testValue\"}" allSettings := &loader.TargetKeyValueSettings{ - SecretSettings: map[string]corev1.Secret{ - secretName: { - Data: secretResult, - Type: corev1.SecretType("Opaque"), - }, - }, - ConfigMapSettings: configMapResult, - SecretReferences: map[string]*loader.TargetSecretReference{ - secretName: { - Type: corev1.SecretType("Opaque"), - SecretsMetadata: secretMetadata, - }, - }, + ConfigMapSettings: mapResult, } mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) ctx := context.Background() - providerName := "refresh-appconfigurationprovider-3" - configMapName := "configmap-to-be-refreshed-3" - + providerName := "test-appconfigurationprovider-8a" + configMapName := "file-style-configmap-to-be-created-8a" configProvider := &acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ APIVersion: "appconfig.kubernetes.config/v1", - Kind: "AppConfigurationProvider", + Kind: "AzureAppConfigurationProvider", }, ObjectMeta: metav1.ObjectMeta{ Name: providerName, @@ -663,143 +716,75 @@ var _ = Describe("AppConfiguationProvider controller", func() { Endpoint: &EndpointName, Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: configMapName, - }, - Secret: &acpv1.SecretReference{ - Target: acpv1.SecretGenerationParameters{ - SecretName: secretName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", }, - Refresh: &acpv1.RefreshSettings{ - Interval: "1m", + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Interval: "5s", Enabled: true, }, }, }, } Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + time.Sleep(time.Second * 5) //Wait few seconds to wait the second round reconcile complete configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} configmap := &corev1.ConfigMap{} Eventually(func() bool { err := k8sClient.Get(ctx, configmapLookupKey, configmap) - if err != nil { - fmt.Print(err.Error()) - } - return err == nil - }, timeout, interval).Should(BeTrue()) - - secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} - secret := &corev1.Secret{} - - Eventually(func() bool { - err := k8sClient.Get(ctx, secretLookupKey, secret) return err == nil }, timeout, interval).Should(BeTrue()) Expect(configmap.Name).Should(Equal(configMapName)) Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) - Expect(configmap.Data["testKey"]).Should(Equal("testValue")) - Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) - Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) - - Expect(secret.Namespace).Should(Equal(ProviderNamespace)) - Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) - Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) - - newSecretResult := make(map[string][]byte) - newSecretResult["testSecretKey"] = []byte("newTestSecretValue") - - newResolvedSecret := map[string]corev1.Secret{ - secretName: { - Data: newSecretResult, - Type: corev1.SecretType("Opaque"), - }, - } + Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"testKey\":\"testValue\"}")) + Expect(len(configmap.Data)).Should(Equal(1)) - var newFakeId azsecrets.ID = "newFakeSecretId" - newSecretMetadata := make(map[string]loader.KeyVaultSecretMetadata) - newSecretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ - SecretId: &newFakeId, - } - mockedSecretReference := make(map[string]*loader.TargetSecretReference) - mockedSecretReference[secretName] = &loader.TargetSecretReference{ - Type: corev1.SecretType("Opaque"), - SecretsMetadata: newSecretMetadata, + newResult := make(map[string]string) + newResult["filestyle.json"] = "{\"testKey\":\"newValue\"}" + newSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: newResult, } - newTargetSettings := &loader.TargetKeyValueSettings{ - SecretSettings: newResolvedSecret, - SecretReferences: mockedSecretReference, - } + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(newSettings, nil) - mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(newTargetSettings, nil) - // Refresh interval is 1 minute, wait for 65 seconds to make sure the refresh is triggered - time.Sleep(65 * time.Second) + time.Sleep(time.Second * 5) //Wait few seconds to wait the second round reconcile complete Eventually(func() bool { - err := k8sClient.Get(ctx, secretLookupKey, secret) + err := k8sClient.Get(ctx, configmapLookupKey, configmap) return err == nil }, timeout, interval).Should(BeTrue()) - Expect(secret.Namespace).Should(Equal(ProviderNamespace)) - Expect(string(secret.Data["testSecretKey"])).Should(Equal("newTestSecretValue")) - Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) - - // Mocked secret refresh scenario when secretMetadata is not changed - newTargetSettings2 := &loader.TargetKeyValueSettings{ - SecretSettings: make(map[string]corev1.Secret), - SecretReferences: mockedSecretReference, - } - - mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(newTargetSettings2, nil) - // Refresh interval is 1 minute, wait for 65 seconds to make sure the refresh is triggered - time.Sleep(65 * time.Second) - - Eventually(func() bool { - err := k8sClient.Get(ctx, secretLookupKey, secret) - return err == nil - }, timeout, interval).Should(BeTrue()) + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"testKey\":\"newValue\"}")) + Expect(len(configmap.Data)).Should(Equal(1)) - Expect(secret.Namespace).Should(Equal(ProviderNamespace)) - Expect(string(secret.Data["testSecretKey"])).Should(Equal("newTestSecretValue")) - Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + _ = k8sClient.Delete(ctx, configProvider) }) - It("Should refresh configMap by watching all keys", func() { - By("When key values updated in Azure App Configuration") + It("Should not refresh configMap", func() { + By("When sentinel value not changed in Azure App Configuration") mapResult := make(map[string]string) mapResult["testKey"] = "testValue" mapResult["testKey2"] = "testValue2" mapResult["testKey3"] = "testValue3" - keyValueEtags := make(map[acpv1.Selector][]*azcore.ETag) - featureFlagEtags := make(map[acpv1.Selector][]*azcore.ETag) - keyFilter := "*" - keyValueEtags[acpv1.Selector{KeyFilter: &keyFilter}] = []*azcore.ETag{} - allSettings := &loader.TargetKeyValueSettings{ ConfigMapSettings: mapResult, - KeyValueETags: keyValueEtags, - FeatureFlagETags: featureFlagEtags, - } - - mapResult2 := make(map[string]string) - mapResult2["testKey"] = "newtestValue" - mapResult2["testKey2"] = "newtestValue2" - mapResult2["testKey3"] = "newtestValue3" - - allSettings2 := &loader.TargetKeyValueSettings{ - ConfigMapSettings: mapResult2, - KeyValueETags: keyValueEtags, - FeatureFlagETags: featureFlagEtags, } mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) - mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) - mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings2, nil) + mockConfigurationSettings.EXPECT().CheckAndRefreshSentinels(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil) ctx := context.Background() - providerName := "refresh-appconfigurationprovider-4" - configMapName := "configmap-to-be-refresh-4" + providerName := "refresh-appconfigurationprovider-2a" + configMapName := "configmap-to-be-refresh-2a" configProvider := &acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ APIVersion: "appconfig.kubernetes.config/v1", @@ -819,6 +804,11 @@ var _ = Describe("AppConfiguationProvider controller", func() { Refresh: &acpv1.DynamicConfigurationRefreshParameters{ Interval: "5s", Enabled: true, + Monitoring: &acpv1.RefreshMonitoring{ + Sentinels: []acpv1.Sentinel{ + {Key: "testNewKey", Label: "testNewLabel"}, + }, + }, }, }, }, @@ -839,6 +829,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Data["testKey"]).Should(Equal("testValue")) Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + lastReconcileTime := configmap.Annotations["azconfig.io/LastReconcileTime"] time.Sleep(6 * time.Second) @@ -847,34 +838,27 @@ var _ = Describe("AppConfiguationProvider controller", func() { return err == nil }, timeout, interval).Should(BeTrue()) - Expect(configmap.Data["testKey"]).Should(Equal("newtestValue")) - Expect(configmap.Data["testKey2"]).Should(Equal("newtestValue2")) - Expect(configmap.Data["testKey3"]).Should(Equal("newtestValue3")) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).Should(Equal(lastReconcileTime)) _ = k8sClient.Delete(ctx, configProvider) }) - It("Should not refresh configMap by watching all keys", func() { - By("When key values not updated in Azure App Configuration") + It("Should not refresh configMap", func() { + By("When disabled configuration.refresh.enabled property") mapResult := make(map[string]string) - keyValueEtags := make(map[acpv1.Selector][]*azcore.ETag) - featureFlagEtags := make(map[acpv1.Selector][]*azcore.ETag) mapResult["testKey"] = "testValue" mapResult["testKey2"] = "testValue2" mapResult["testKey3"] = "testValue3" allSettings := &loader.TargetKeyValueSettings{ ConfigMapSettings: mapResult, - KeyValueETags: keyValueEtags, - FeatureFlagETags: featureFlagEtags, } mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) - mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil) ctx := context.Background() - providerName := "refresh-appconfigurationprovider-5" - configMapName := "configmap-to-be-refresh-5" + providerName := "refresh-appconfigurationprovider-2b" + configMapName := "configmap-to-be-refresh-2b" configProvider := &acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ APIVersion: "appconfig.kubernetes.config/v1", @@ -893,7 +877,12 @@ var _ = Describe("AppConfiguationProvider controller", func() { Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ Refresh: &acpv1.DynamicConfigurationRefreshParameters{ Interval: "5s", - Enabled: true, + Enabled: false, + Monitoring: &acpv1.RefreshMonitoring{ + Sentinels: []acpv1.Sentinel{ + {Key: "testNewKey", Label: "testNewLabel"}, + }, + }, }, }, }, @@ -914,6 +903,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Data["testKey"]).Should(Equal("testValue")) Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + lastReconcileTime := configmap.Annotations["azconfig.io/LastReconcileTime"] time.Sleep(6 * time.Second) @@ -922,26 +912,1071 @@ var _ = Describe("AppConfiguationProvider controller", func() { return err == nil }, timeout, interval).Should(BeTrue()) - Expect(configmap.Data["testKey"]).Should(Equal("testValue")) - Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) - Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).Should(Equal(lastReconcileTime)) _ = k8sClient.Delete(ctx, configProvider) }) - }) - Context("Verify exist non escaped value in label", func() { - It("Should return false if all character is escaped", func() { - Expect(hasNonEscapedValueInLabel(`some\,valid\,label`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`somevalidlabel`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel("")).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`some\*`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`\\some\,\*\valid\,\label\*`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`\,`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`\\`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`\`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`'\`)).Should(BeFalse()) - Expect(hasNonEscapedValueInLabel(`\\\,`)).Should(BeFalse()) + It("Should trigger reconciliation", func() { + By("Deleting ConfigMap") + mapResult := make(map[string]string) + mapResult["testKey"] = "testValue" + mapResult["testKey2"] = "testValue2" + mapResult["testKey3"] = "testValue3" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + } + + mapResult2 := make(map[string]string) + mapResult2["testKey"] = "newtestValue" + mapResult2["testKey2"] = "newtestValue2" + mapResult2["testKey3"] = "newtestValue3" + + allSettings2 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult2, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + mockConfigurationSettings.EXPECT().CheckAndRefreshSentinels(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil, nil) + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings2, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-2c" + configMapName := "configmap-to-be-refresh-2c" + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Labels: map[string]string{"foo": "fooValue", "bar": "barValue"}, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Interval: "5s", + Enabled: true, + Monitoring: &acpv1.RefreshMonitoring{ + Sentinels: []acpv1.Sentinel{ + {Key: "testKeyOne", Label: "testNewLabel"}, + {Key: "testKeyTwo", Label: "testLabel"}, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + lastReconcileTime := configmap.Annotations["azconfig.io/LastReconcileTime"] + + time.Sleep(6 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).Should(Equal(lastReconcileTime)) + + _ = k8sClient.Delete(ctx, configmap) + + time.Sleep(2 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Data["testKey"]).Should(Equal("newtestValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("newtestValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("newtestValue3")) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(lastReconcileTime)) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Should trigger reconciliation", func() { + By("Modifying ConfigMap") + configMapResult := make(map[string]string) + configMapResult["testKey"] = "testValue" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: configMapResult, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "appconfigurationprovider-modify-configmap" + configMapName := "configmap-to-be-modified" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + configmapLastReconcileTime := configmap.Annotations["azconfig.io/LastReconcileTime"] + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + configmap.Data["testKey"] = "newTestValue" + _ = k8sClient.Update(ctx, configmap) + + time.Sleep(2 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(configmapLastReconcileTime)) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Should trigger reconciliation", func() { + By("Modifying Secret") + configMapResult := make(map[string]string) + configMapResult["testKey"] = "testValue" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + + secretName := "secret-to-be-modified" + var fakeId azsecrets.ID = "fakeSecretId" + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + + allSettings := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + ConfigMapSettings: configMapResult, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "appconfigurationprovider-delete-secret" + configMapName := "configmap-not-deleted" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secret := &corev1.Secret{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + secretLastReconcileTime := secret.Annotations["azconfig.io/LastReconcileTime"] + configmapLastReconcileTime := configmap.Annotations["azconfig.io/LastReconcileTime"] + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + secret.Data["testSecretKey"] = []byte("newTestSecretValue") + _ = k8sClient.Update(ctx, secret) + + time.Sleep(2 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + Expect(secret.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(secretLastReconcileTime)) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(configmapLastReconcileTime)) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Should trigger reconciliation", func() { + By("Deleting Secret") + configMapResult := make(map[string]string) + configMapResult["testKey"] = "testValue" + configMapResult["testKey2"] = "testValue2" + configMapResult["testKey3"] = "testValue3" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + + secretName := "secret-to-be-deleted" + var fakeId azsecrets.ID = "fakeSecretId" + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + + allSettings := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + ConfigMapSettings: configMapResult, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "appconfigurationprovider-delete-secret" + configMapName := "configmap-not-to-be-deleted" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secret := &corev1.Secret{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + secretLastReconcileTime := secret.Annotations["azconfig.io/LastReconcileTime"] + configmapLastReconcileTime := configmap.Annotations["azconfig.io/LastReconcileTime"] + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + _ = k8sClient.Delete(ctx, secret) + + time.Sleep(2 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + Expect(secret.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(secretLastReconcileTime)) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(configmapLastReconcileTime)) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Should refresh secret when data change", func() { + By("By enabling refresh on secret") + configMapResult := make(map[string]string) + configMapResult["testKey"] = "testValue" + configMapResult["testKey2"] = "testValue2" + configMapResult["testKey3"] = "testValue3" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + + secretName := "secret-to-be-refreshed-3" + var fakeId azsecrets.ID = "fakeSecretId" + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + + allSettings := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + ConfigMapSettings: configMapResult, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-3" + configMapName := "configmap-to-be-refreshed-3" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secret := &corev1.Secret{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + + newSecretResult := make(map[string][]byte) + newSecretResult["testSecretKey"] = []byte("newTestSecretValue") + + newResolvedSecret := map[string]corev1.Secret{ + secretName: { + Data: newSecretResult, + Type: corev1.SecretType("Opaque"), + }, + } + + var newFakeId azsecrets.ID = "newFakeSecretId" + newSecretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + newSecretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &newFakeId, + } + mockedSecretReference := make(map[string]*loader.TargetSecretReference) + mockedSecretReference[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: newSecretMetadata, + } + + newTargetSettings := &loader.TargetKeyValueSettings{ + SecretSettings: newResolvedSecret, + SecretReferences: mockedSecretReference, + } + + mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(newTargetSettings, nil) + // Refresh interval is 1 minute, wait for 65 seconds to make sure the refresh is triggered + time.Sleep(65 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("newTestSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + + // Mocked secret refresh scenario when secretMetadata is not changed + newTargetSettings2 := &loader.TargetKeyValueSettings{ + SecretSettings: make(map[string]corev1.Secret), + SecretReferences: mockedSecretReference, + } + + mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(newTargetSettings2, nil) + // Refresh interval is 1 minute, wait for 65 seconds to make sure the refresh is triggered + time.Sleep(65 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("newTestSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Should refresh configMap by watching all keys", func() { + By("When key values updated in Azure App Configuration") + mapResult := make(map[string]string) + mapResult["testKey"] = "testValue" + mapResult["testKey2"] = "testValue2" + mapResult["testKey3"] = "testValue3" + + keyValueEtags := make(map[acpv1.Selector][]*azcore.ETag) + featureFlagEtags := make(map[acpv1.Selector][]*azcore.ETag) + keyFilter := "*" + keyValueEtags[acpv1.Selector{KeyFilter: &keyFilter}] = []*azcore.ETag{} + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: keyValueEtags, + FeatureFlagETags: featureFlagEtags, + } + + mapResult2 := make(map[string]string) + mapResult2["testKey"] = "newtestValue" + mapResult2["testKey2"] = "newtestValue2" + mapResult2["testKey3"] = "newtestValue3" + + allSettings2 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult2, + KeyValueETags: keyValueEtags, + FeatureFlagETags: featureFlagEtags, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings2, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-4" + configMapName := "configmap-to-be-refresh-4" + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Labels: map[string]string{"foo": "fooValue", "bar": "barValue"}, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Interval: "5s", + Enabled: true, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + time.Sleep(6 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Data["testKey"]).Should(Equal("newtestValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("newtestValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("newtestValue3")) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Should not refresh configMap by watching all keys", func() { + By("When key values not updated in Azure App Configuration") + mapResult := make(map[string]string) + keyValueEtags := make(map[acpv1.Selector][]*azcore.ETag) + featureFlagEtags := make(map[acpv1.Selector][]*azcore.ETag) + mapResult["testKey"] = "testValue" + mapResult["testKey2"] = "testValue2" + mapResult["testKey3"] = "testValue3" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: keyValueEtags, + FeatureFlagETags: featureFlagEtags, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-5" + configMapName := "configmap-to-be-refresh-5" + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Labels: map[string]string{"foo": "fooValue", "bar": "barValue"}, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Interval: "5s", + Enabled: true, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + time.Sleep(6 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + _ = k8sClient.Delete(ctx, configProvider) + }) + }) + + Context("AppConfigurationProvider can dynamically refresh feature flag data in ConfigMap", func() { + It("Should refresh configMap when both configuration.refresh and featureFlag.refresh enabled", func() { + By("When selected feattureFlags updated in Azure App Configuration") + mapResult := make(map[string]string) + keyValueEtags := make(map[acpv1.Selector][]*azcore.ETag) + featureFlagEtags := make(map[acpv1.Selector][]*azcore.ETag) + mapResult["filestyle.json"] = "{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: keyValueEtags, + FeatureFlagETags: featureFlagEtags, + } + + mapResult2 := make(map[string]string) + mapResult2["filestyle.json"] = "{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": false,\"conditions\": {\"client_filters\": []}}]}}" + + allSettings2 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult2, + KeyValueETags: keyValueEtags, + FeatureFlagETags: featureFlagEtags, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "test-appconfigurationprovider-7a" + configMapName := "file-style-configmap-to-be-created-7a" + wildcard := "*" + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Labels: map[string]string{"foo": "fooValue", "bar": "barValue"}, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Interval: "5s", + Enabled: true, + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &wildcard, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "5s", + Enabled: true, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + time.Sleep(time.Second * 5) //Wait few seconds to wait the second round reconcile complete + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}")) + Expect(len(configmap.Data)).Should(Equal(1)) + + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil).Times(2) + + time.Sleep(5 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}")) + Expect(len(configmap.Data)).Should(Equal(1)) + + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshFeatureFlagSettings(gomock.Any(), gomock.Any()).Return(allSettings2, nil) + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil) + + time.Sleep(5 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": false,\"conditions\": {\"client_filters\": []}}]}}")) + Expect(len(configmap.Data)).Should(Equal(1)) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + It("Feature flag refresh can work with secret refresh", func() { + By("By enabling refresh on secret and feature flag") + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + + secretName := "secret-to-be-refreshed-4" + var fakeId azsecrets.ID = "fakeSecretId" + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + + allSettings := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + ConfigMapSettings: mapResult, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-secret-ff" + configMapName := "configmap-to-be-refreshed-ff" + wildcard := "*" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &wildcard, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "55s", + Enabled: true, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secret := &corev1.Secret{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}")) + configmapLastReconcileTime := configmap.Annotations["azconfig.io/LastReconcileTime"] + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + secretLastReconcileTime := secret.Annotations["azconfig.io/LastReconcileTime"] + + mapResult2 := make(map[string]string) + mapResult2["filestyle.json"] = "{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": false,\"conditions\": {\"client_filters\": []}}]}}" + + allSettings2 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult2, + } + + // feature flag refresh + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshFeatureFlagSettings(gomock.Any(), gomock.Any()).Return(allSettings2, nil) + + time.Sleep(55 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"aKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": false,\"conditions\": {\"client_filters\": []}}]}}")) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(configmapLastReconcileTime)) + // feature flag refresh interval is shorter than secret refresh interval, so secret should not be refreshed when configMap is refreshed + Expect(secret.Annotations["azconfig.io/LastReconcileTime"]).Should(Equal(secretLastReconcileTime)) + // update configmap last reconcile time + configmapLastReconcileTime = configmap.Annotations["azconfig.io/LastReconcileTime"] + + newSecretResult := make(map[string][]byte) + newSecretResult["testSecretKey"] = []byte("newTestSecretValue") + + newResolvedSecret := map[string]corev1.Secret{ + secretName: { + Data: newSecretResult, + Type: corev1.SecretType("Opaque"), + }, + } + + var newFakeId azsecrets.ID = "newFakeSecretId" + newSecretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + newSecretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &newFakeId, + } + mockedSecretReference := make(map[string]*loader.TargetSecretReference) + mockedSecretReference[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: newSecretMetadata, + } + + newTargetSettings := &loader.TargetKeyValueSettings{ + SecretSettings: newResolvedSecret, + SecretReferences: mockedSecretReference, + } + + mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(newTargetSettings, nil) + + time.Sleep(5 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("newTestSecretValue")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + Expect(secret.Annotations["azconfig.io/LastReconcileTime"]).ShouldNot(Equal(secretLastReconcileTime)) + Expect(configmap.Annotations["azconfig.io/LastReconcileTime"]).Should(Equal(configmapLastReconcileTime)) + + _ = k8sClient.Delete(ctx, configProvider) + }) + + }) + + Context("Verify exist non escaped value in label", func() { + It("Should return false if all character is escaped", func() { + Expect(hasNonEscapedValueInLabel(`some\,valid\,label`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`somevalidlabel`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel("")).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`some\*`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`\\some\,\*\valid\,\label\*`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`\,`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`\\`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`\`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`'\`)).Should(BeFalse()) + Expect(hasNonEscapedValueInLabel(`\\\,`)).Should(BeFalse()) Expect(hasNonEscapedValueInLabel(`\a\\\,`)).Should(BeFalse()) Expect(hasNonEscapedValueInLabel(`\\\\\\\,`)).Should(BeFalse()) }) @@ -1041,6 +2076,94 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(verifyObject(configProviderSpec).Error()).Should(Equal("spec.target.configMapData.separator: separator field is not allowed when type is properties")) }) + It("Should return error if selector only uses labelFilter", func() { + configMapName := "test-configmap" + testLabelFilter := "testLabelFilter" + configProviderSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + LabelFilter: &testLabelFilter, + }, + }, + }, + } + + Expect(verifyObject(configProviderSpec).Error()).Should(Equal("spec.configuration.selectors: a selector uses 'labelFilter' but misses the 'keyFilter', 'keyFilter' is required for key-label pair filtering")) + }) + + It("Should return error set both 'keyFilter' and 'snapshotName' in one selector", func() { + configMapName := "test-configmap" + testLabelFilter := "testLabelFilter" + testKeyFilter := "testKeyFilter" + testSnapshotName := "testSnapshotName" + configProviderSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKeyFilter, + LabelFilter: &testLabelFilter, + SnapshotName: &testSnapshotName, + }, + }, + }, + } + + Expect(verifyObject(configProviderSpec).Error()).Should(Equal("spec.configuration.selectors: set both 'keyFilter' and 'snapshotName' in one selector causes ambiguity, only one of them should be set")) + }) + + It("Should return error set both 'labelFilter' and 'snapshotName' in one selector", func() { + configMapName := "test-configmap" + testLabelFilter := "testLabelFilter" + testSnapshotName := "testSnapshotName" + configProviderSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + LabelFilter: &testLabelFilter, + SnapshotName: &testSnapshotName, + }, + }, + }, + } + + Expect(verifyObject(configProviderSpec).Error()).Should(Equal("spec.configuration.selectors: 'labelFilter' is not allowed when 'snapshotName' is set")) + }) + + It("Should return error when there's non escaped value in labelFilter", func() { + configMapName := "test-configmap" + testLabelFilter := "," + testKeyFilter := "testKeyFilter" + configProviderSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + LabelFilter: &testLabelFilter, + KeyFilter: &testKeyFilter, + }, + }, + }, + } + + Expect(verifyObject(configProviderSpec).Error()).Should(Equal("spec.configuration.selectors: non-escaped reserved wildcard character '*' and multiple labels separator ',' are not supported in label filters")) + }) + It("Should return error if feature flag is set when data type is default", func() { configMapName := "test-configmap" testKey := "testKey" @@ -1256,6 +2379,80 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(verifyObject(configProviderSpec3).Error()).Should(Equal("spec.configuration.selectors: a selector uses 'labelFilter' but misses the 'keyFilter', 'keyFilter' is required for key-label pair filtering")) }) + + It("Should return error when configuration.refresh.interval is less than 1 second", func() { + configMapName := "test-configmap" + configProviderSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Interval: "500ms", + Enabled: true, + Monitoring: &acpv1.RefreshMonitoring{ + Sentinels: []acpv1.Sentinel{ + {Key: "testKey", Label: "testLabel"}, + }, + }, + }, + }, + } + + Expect(verifyObject(configProviderSpec).Error()).Should(Equal("configuration.refresh.interval: configuration.refresh.interval can not be shorter than 1s")) + }) + + It("Should return error when secret.refresh.interval is less than 1 minute", func() { + configMapName := "test-configmap" + secretName := "test" + configProviderSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1s", + Enabled: true, + }, + }, + } + + Expect(verifyObject(configProviderSpec).Error()).Should(Equal("secret.refresh.interval: secret.refresh.interval can not be shorter than 1m0s")) + }) + + It("Should return error when featureFlag.refresh.interval is less than 1 second", func() { + configMapName := "test-configmap" + wildcard := "*" + configProviderSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: acpv1.Json, + Key: "testKey", + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &wildcard, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "500ms", + Enabled: true, + }, + }, + } + + Expect(verifyObject(configProviderSpec).Error()).Should(Equal("featureFlag.refresh.interval: featureFlag.refresh.interval can not be shorter than 1s")) + }) + }) Context("Verify auth object", func() { diff --git a/internal/controller/processor.go b/internal/controller/processor.go index 90feede..4cb214a 100644 --- a/internal/controller/processor.go +++ b/internal/controller/processor.go @@ -20,7 +20,7 @@ import ( type AppConfigurationProviderProcessor struct { Context context.Context - Retriever *loader.ConfigurationSettingsRetriever + Retriever loader.ConfigurationSettingsRetriever Provider *acpv1.AzureAppConfigurationProvider Settings *loader.TargetKeyValueSettings ShouldReconcile bool @@ -68,7 +68,7 @@ func (processor *AppConfigurationProviderProcessor) PopulateSettings(existingCon } func (processor *AppConfigurationProviderProcessor) processFullReconciliation() error { - updatedSettings, err := (*processor.Retriever).CreateTargetSettings(processor.Context, processor.SecretReferenceResolver) + updatedSettings, err := (processor.Retriever).CreateTargetSettings(processor.Context, processor.SecretReferenceResolver) if err != nil { return err } @@ -108,7 +108,7 @@ func (processor *AppConfigurationProviderProcessor) processFeatureFlagRefresh(ex return nil } - if processor.RefreshOptions.featureFlagRefreshNeeded, err = (*processor.Retriever).CheckPageETags(processor.Context, reconcileState.FeatureFlagETags); err != nil { + if processor.RefreshOptions.featureFlagRefreshNeeded, err = (processor.Retriever).CheckPageETags(processor.Context, reconcileState.FeatureFlagETags); err != nil { return err } @@ -117,7 +117,7 @@ func (processor *AppConfigurationProviderProcessor) processFeatureFlagRefresh(ex return nil } - featureFlagRefreshedSettings, err := (*processor.Retriever).RefreshFeatureFlagSettings(processor.Context, &existingConfigMap.Data) + featureFlagRefreshedSettings, err := (processor.Retriever).RefreshFeatureFlagSettings(processor.Context, &existingConfigMap.Data) if err != nil { return err } @@ -156,11 +156,11 @@ func (processor *AppConfigurationProviderProcessor) processKeyValueRefresh(exist } if provider.Spec.Configuration.Refresh.Monitoring != nil { - if processor.RefreshOptions.sentinelChanged, processor.RefreshOptions.updatedSentinelETags, err = (*processor.Retriever).CheckAndRefreshSentinels(processor.Context, processor.Provider, reconcileState.SentinelETags); err != nil { + if processor.RefreshOptions.sentinelChanged, processor.RefreshOptions.updatedSentinelETags, err = (processor.Retriever).CheckAndRefreshSentinels(processor.Context, processor.Provider, reconcileState.SentinelETags); err != nil { return err } } else { - if processor.RefreshOptions.keyValuePageETagsChanged, err = (*processor.Retriever).CheckPageETags(processor.Context, reconcileState.KeyValueETags); err != nil { + if processor.RefreshOptions.keyValuePageETagsChanged, err = (processor.Retriever).CheckPageETags(processor.Context, reconcileState.KeyValueETags); err != nil { return err } } @@ -176,7 +176,7 @@ func (processor *AppConfigurationProviderProcessor) processKeyValueRefresh(exist existingConfigMapSettings = &processor.Settings.ConfigMapSettings } - keyValueRefreshedSettings, err := (*processor.Retriever).RefreshKeyValueSettings(processor.Context, existingConfigMapSettings, processor.SecretReferenceResolver) + keyValueRefreshedSettings, err := (processor.Retriever).RefreshKeyValueSettings(processor.Context, existingConfigMapSettings, processor.SecretReferenceResolver) if err != nil { return err } @@ -249,7 +249,7 @@ func (processor *AppConfigurationProviderProcessor) processSecretReferenceRefres } } - resolvedSecrets, err := (*processor.Retriever).ResolveSecretReferences(processor.Context, secretReferencesToSolve, processor.SecretReferenceResolver) + resolvedSecrets, err := (processor.Retriever).ResolveSecretReferences(processor.Context, secretReferencesToSolve, processor.SecretReferenceResolver) if err != nil { return err } diff --git a/internal/controller/processor_test.go b/internal/controller/processor_test.go new file mode 100644 index 0000000..13f1a92 --- /dev/null +++ b/internal/controller/processor_test.go @@ -0,0 +1,1537 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package controller + +import ( + "azappconfig/provider/internal/loader" + "context" + "time" + + acpv1 "azappconfig/provider/api/v1" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("AppConfiguationProvider processor", func() { + + // Define utility constants for object names and testing timeouts/durations and intervals. + const ( + ProviderName = "test-appconfigurationprovider" + ProviderNamespace = "default" + ) + + var ( + EndpointName = "https://fake-endpoint" + ) + + Context("Reconcile triggered by one component refresh scenarios", func() { + It("Should update reconcile state when sentinel Etag updated when configuration refresh enabled", func() { + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"testKey\":\"testValue\"}" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + } + + ctx := context.Background() + providerName := "test-appconfigurationprovider-sentinel" + configMapName := "configmap-sentinel" + testKey := "*" + + sentinelKey1 := "sentinel1" + sentinelKey2 := "sentinel2" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKey, + }, + }, + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Enabled: true, + Interval: "5s", + Monitoring: &acpv1.RefreshMonitoring{ + Sentinels: []acpv1.Sentinel{ + { + Key: sentinelKey1, + }, + { + Key: sentinelKey2, + }, + }, + }, + }, + }, + }, + } + + fakeEtag := azcore.ETag("fake-etag") + fakeResourceVersion := "1" + + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextKeyValueRefreshReconcileTime: metav1.Now(), + SentinelETags: map[acpv1.Sentinel]*azcore.ETag{ + { + Key: sentinelKey1, + }: &fakeEtag, + }, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.Now(), + RefreshOptions: &RefreshOptions{}, + } + + newFakeEtag1 := azcore.ETag("fake-etag-1") + newFakeEtag2 := azcore.ETag("fake-etag-2") + expectedNextKeyValueRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(5 * time.Second)) + + //Sentinel Etag is updated + mockConfigurationSettings.EXPECT().CheckAndRefreshSentinels(gomock.Any(), gomock.Any(), gomock.Any()).Return( + true, + map[acpv1.Sentinel]*azcore.ETag{ + { + Key: sentinelKey1, + }: &newFakeEtag1, + { + Key: sentinelKey2, + }: &newFakeEtag2, + }, + nil, + ) + + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings, nil) + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, make(map[string]corev1.Secret)) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.SentinelETags[acpv1.Sentinel{ + Key: sentinelKey1, + }]).Should(Equal(&newFakeEtag1)) + Expect(processor.ReconciliationState.SentinelETags[acpv1.Sentinel{ + Key: sentinelKey2, + }]).Should(Equal(&newFakeEtag2)) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(expectedNextKeyValueRefreshReconcileTime)) + }) + + It("Secret refresh can work with multiple version secrets when secret refresh enabled", func() { + ctx := context.Background() + providerName := "test-appconfigurationprovider-secret-2" + configMapName := "configmap-test-2" + secretName := "secret-test-2" + fakeSecretResourceVersion := "1" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + secretResult["testSecretKey2"] = []byte("testSecretValue2") + existingSecrets := make(map[string]corev1.Secret) + existingSecrets[secretName] = corev1.Secret{ + Data: secretResult, + ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeSecretResourceVersion, + }, + } + + var fakeId azsecrets.ID = "fakeSecretId" + var cachedFakeId azsecrets.ID = "cachedFakeSecretId" + + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata2 := make(map[string]loader.KeyVaultSecretMetadata) + cachedSecretReferences := make(map[string]*loader.TargetSecretReference) + // multiple version secrets + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + SecretVersion: "", + } + secretMetadata2["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &cachedFakeId, + SecretVersion: "", + } + secretMetadata2["testSecretKey2"] = loader.KeyVaultSecretMetadata{ + SecretId: &cachedFakeId, + SecretVersion: "fakeVersion", + } + cachedSecretReferences[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata2, + SecretResourceVersion: fakeSecretResourceVersion, + } + + allSettings := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + }, + } + + fakeResourceVersion := "1" + tmpTime := metav1.Now() + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextSecretReferenceRefreshReconcileTime: tmpTime, + ExistingSecretReferences: cachedSecretReferences, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.NewTime(tmpTime.Time.Add(1 * time.Minute)), + RefreshOptions: &RefreshOptions{}, + } + + // Only resolve non-version Key Vault references + mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings, nil) + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.ExistingSecretReferences[secretName].SecretsMetadata["testSecretKey"]).Should(Equal(allSettings.SecretReferences[secretName].SecretsMetadata["testSecretKey"])) + Expect(processor.ReconciliationState.ExistingSecretReferences[secretName].SecretsMetadata["testSecretKey2"]).Should(Equal(cachedSecretReferences[secretName].SecretsMetadata["testSecretKey2"])) + }) + }) + + Context("Reconcile triggered by two component refresh scenarios", func() { + // 4 scenarios: Both Etags updated, only keyValue Etag updated, only featureFlag Etag updated, both Etags not updated + It("Should update reconcile state when featureFlag pageEtag and keyValue pageEtag updated when configuration refresh and feature flag refresh enabled", func() { + ctx := context.Background() + providerName := "test-appconfigurationprovider-config-ff-etags" + configMapName := "configmap-config-ff-etags" + testKey := "*" + testFeatureFlagSelector := "*" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKey, + }, + }, + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Enabled: true, + Interval: "2s", + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testFeatureFlagSelector, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "2s", + Enabled: true, + }, + }, + }, + } + + fakeEtag := azcore.ETag("fake-etag") + fakeEtag2 := azcore.ETag("fake-etag2") + fakeResourceVersion := "1" + + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextFeatureFlagRefreshReconcileTime: metav1.Now(), + NextKeyValueRefreshReconcileTime: metav1.Now(), + KeyValueETags: map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &fakeEtag, + }, + }, + FeatureFlagETags: map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testFeatureFlagSelector, + }: { + &fakeEtag2, + }, + }, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.Now(), + RefreshOptions: &RefreshOptions{}, + } + + expectedNextKeyValueRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(2 * time.Second)) + expectedNextFeatureFlagRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(2 * time.Second)) + newFakeEtag := azcore.ETag("fake-etag-1") + newFakeEtag2 := azcore.ETag("fake-etag-2") + updatedKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &newFakeEtag, + }, + } + updatedFeatureFlagEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testFeatureFlagSelector, + }: { + &newFakeEtag2, + }, + } + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"testKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: updatedKeyValueEtags, + FeatureFlagETags: updatedFeatureFlagEtags, + } + + //Both Etags are updated + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshFeatureFlagSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings, nil) + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, make(map[string]corev1.Secret)) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }]).Should(Equal(updatedFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }])) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(expectedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime).Should(Equal(expectedNextFeatureFlagRefreshReconcileTime)) + + newKeyValueEtag := azcore.ETag("fake-keyValue-etag") + updatedKeyValueEtags2 := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &newKeyValueEtag, + }, + } + + allSettings2 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: updatedKeyValueEtags2, + } + + //Only keyValue Etag is updated + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil) + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings2, nil) + + time.Sleep(2 * time.Second) + processor.CurrentTime = metav1.Now() + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, make(map[string]corev1.Secret)) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }]).Should(Equal(updatedFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }])) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags2[acpv1.Selector{ + KeyFilter: &testKey, + }])) + + newFeatureFlagEtag := azcore.ETag("fake-ff-etag") + updatedFeatureFlagEtags2 := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testFeatureFlagSelector, + }: { + &newFeatureFlagEtag, + }, + } + + allSettings3 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + FeatureFlagETags: updatedFeatureFlagEtags2, + } + + //Only featureFlag Etag is updated + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshFeatureFlagSettings(gomock.Any(), gomock.Any()).Return(allSettings3, nil) + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil) + + time.Sleep(2 * time.Second) + processor.CurrentTime = metav1.Now() + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, make(map[string]corev1.Secret)) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }]).Should(Equal(updatedFeatureFlagEtags2[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }])) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags2[acpv1.Selector{ + KeyFilter: &testKey, + }])) + + //Both Etags are not updated + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil).Times(2) + + time.Sleep(2 * time.Second) + processor.CurrentTime = metav1.Now() + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, make(map[string]corev1.Secret)) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }]).Should(Equal(updatedFeatureFlagEtags2[acpv1.Selector{ + KeyFilter: &testFeatureFlagSelector, + }])) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags2[acpv1.Selector{ + KeyFilter: &testKey, + }])) + }) + + It("Should update reconcile state when keyValue pageEtag and secret references updated when configuration refresh and secret refresh enabled", func() { + ctx := context.Background() + providerName := "test-appconfigurationprovider-secret" + configMapName := "configmap-test" + secretName := "secret-test" + fakeSecretResourceVersion := "1" + testKey := "*" + + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"aKey\":\"testValue\"}" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + existingSecrets := make(map[string]corev1.Secret) + existingSecrets[secretName] = corev1.Secret{ + Data: secretResult, + ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeSecretResourceVersion, + }, + } + + var fakeId azsecrets.ID = "fakeSecretId" + var cachedFakeId azsecrets.ID = "cachedFakeSecretId" + + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata2 := make(map[string]loader.KeyVaultSecretMetadata) + cachedSecretReferences := make(map[string]*loader.TargetSecretReference) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + secretMetadata2["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &cachedFakeId, + } + cachedSecretReferences[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata2, + SecretResourceVersion: fakeSecretResourceVersion, + } + + newFakeEtag := azcore.ETag("fake-etag-1") + updatedKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &newFakeEtag, + }, + } + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: updatedKeyValueEtags, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKey, + }, + }, + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Enabled: true, + Interval: "5s", + }, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + }, + } + + fakeResourceVersion := "1" + fakeEtag := azcore.ETag("fake-etag") + nowTime := metav1.Now() + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextSecretReferenceRefreshReconcileTime: nowTime, + NextKeyValueRefreshReconcileTime: nowTime, + KeyValueETags: map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &fakeEtag, + }, + }, + ExistingSecretReferences: cachedSecretReferences, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.NewTime(nowTime.Time.Add(1 * time.Second)), + RefreshOptions: &RefreshOptions{}, + } + + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings, nil) + expectedNextKeyValueRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(5 * time.Second)) + expectedNextSecretReferenceRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(1 * time.Minute)) + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.ExistingSecretReferences).Should(Equal(allSettings.SecretReferences)) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(expectedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime).Should(Equal(expectedNextSecretReferenceRefreshReconcileTime)) + }) + }) + + Context("Reconcile triggered by three component refresh scenarios", func() { + It("Should update key value pageEtag and existing secret references when configuration, secret and feature flag refresh enabled", func() { + ctx := context.Background() + providerName := "test-appconfigurationprovider-secret" + configMapName := "configmap-test" + secretName := "secret-test" + fakeSecretResourceVersion := "1" + testKey := "*" + wildcard := "*" + + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"testKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + existingSecrets := make(map[string]corev1.Secret) + existingSecrets[secretName] = corev1.Secret{ + Data: secretResult, + ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeSecretResourceVersion, + }, + } + + fakeFeatureFlagEtag := azcore.ETag("fake-etag-1") + existingFeatureFlagEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &wildcard, + }: { + &fakeFeatureFlagEtag, + }, + } + + var fakeId azsecrets.ID = "fakeSecretId" + var cachedFakeId azsecrets.ID = "cachedFakeSecretId" + + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata2 := make(map[string]loader.KeyVaultSecretMetadata) + cachedSecretReferences := make(map[string]*loader.TargetSecretReference) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + secretMetadata2["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &cachedFakeId, + } + cachedSecretReferences[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata2, + SecretResourceVersion: fakeSecretResourceVersion, + } + + newFakeEtag := azcore.ETag("fake-etag-1") + updatedKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &newFakeEtag, + }, + } + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: updatedKeyValueEtags, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKey, + }, + }, + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Enabled: true, + Interval: "10s", + }, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1h", + Enabled: true, + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &wildcard, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "2m", + Enabled: true, + }, + }, + }, + } + + fakeResourceVersion := "1" + fakeEtag := azcore.ETag("fake-etag") + nowTime := metav1.Now() + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextSecretReferenceRefreshReconcileTime: metav1.NewTime(nowTime.Time.Add(1 * time.Hour)), + NextKeyValueRefreshReconcileTime: nowTime, + NextFeatureFlagRefreshReconcileTime: metav1.NewTime(nowTime.Time.Add(2 * time.Minute)), + KeyValueETags: map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &fakeEtag, + }, + }, + FeatureFlagETags: existingFeatureFlagEtags, + ExistingSecretReferences: cachedSecretReferences, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.NewTime(nowTime.Time.Add(2 * time.Second)), + RefreshOptions: &RefreshOptions{}, + } + + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings, nil) + expectedNextKeyValueRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(10 * time.Second)) + cachedNextSecretReferenceRefreshReconcileTime := processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime + cachedNextFeatureFlagRefreshReconcileTime := processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.ExistingSecretReferences).Should(Equal(allSettings.SecretReferences)) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + // FeatureFlag Etag should not be updated + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &wildcard, + }]).Should(Equal(existingFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &wildcard, + }])) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(expectedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime).Should(Equal(cachedNextSecretReferenceRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime).Should(Equal(cachedNextFeatureFlagRefreshReconcileTime)) + }) + + It("Should update feature flag pageEtag and existing secret references when configuration, secret and feature flag refresh enabled", func() { + ctx := context.Background() + providerName := "test-appconfigurationprovider-secret" + configMapName := "configmap-test" + secretName := "secret-test" + fakeSecretResourceVersion := "1" + testKey := "*" + wildcard := "*" + + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"testKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + existingSecrets := make(map[string]corev1.Secret) + existingSecrets[secretName] = corev1.Secret{ + Data: secretResult, + ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeSecretResourceVersion, + }, + } + + fakeKeyValueEtag := azcore.ETag("fake-etag-1") + existingKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &fakeKeyValueEtag, + }, + } + + var fakeId azsecrets.ID = "fakeSecretId" + var cachedFakeId azsecrets.ID = "cachedFakeSecretId" + + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata2 := make(map[string]loader.KeyVaultSecretMetadata) + cachedSecretReferences := make(map[string]*loader.TargetSecretReference) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + secretMetadata2["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &cachedFakeId, + } + cachedSecretReferences[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata2, + SecretResourceVersion: fakeSecretResourceVersion, + } + + newFakeEtag := azcore.ETag("fake-etag-1") + updatedFeatureFlagEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &wildcard, + }: { + &newFakeEtag, + }, + } + + allSettingsReturnedByFeatureFlagRefresh := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + FeatureFlagETags: updatedFeatureFlagEtags, + } + + allSettingsReturnedBySecretRefresh := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKey, + }, + }, + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Enabled: true, + Interval: "2m", + }, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1h", + Enabled: true, + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &wildcard, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + }, + } + + fakeResourceVersion := "1" + fakeEtag := azcore.ETag("fake-etag") + nowTime := metav1.Now() + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextSecretReferenceRefreshReconcileTime: nowTime, + NextKeyValueRefreshReconcileTime: metav1.NewTime(nowTime.Time.Add(2 * time.Minute)), + NextFeatureFlagRefreshReconcileTime: nowTime, + FeatureFlagETags: map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &wildcard, + }: { + &fakeEtag, + }, + }, + KeyValueETags: existingKeyValueEtags, + ExistingSecretReferences: cachedSecretReferences, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.NewTime(nowTime.Time.Add(2 * time.Second)), + RefreshOptions: &RefreshOptions{}, + } + + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshFeatureFlagSettings(gomock.Any(), gomock.Any()).Return(allSettingsReturnedByFeatureFlagRefresh, nil) + mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettingsReturnedBySecretRefresh, nil) + expectedNextFeatureFlagRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(1 * time.Minute)) + expectedNextSecretReferenceRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(1 * time.Hour)) + cachedNextKeyValueRefreshReconcileTime := processor.ReconciliationState.NextKeyValueRefreshReconcileTime + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.ExistingSecretReferences).Should(Equal(allSettingsReturnedBySecretRefresh.SecretReferences)) + // KeyValue Etag should not be updated + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(existingKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &wildcard, + }]).Should(Equal(updatedFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &wildcard, + }])) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(cachedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime).Should(Equal(expectedNextSecretReferenceRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime).Should(Equal(expectedNextFeatureFlagRefreshReconcileTime)) + }) + + It("Should update feature flag pageEtag and keyValue pageEtag when configuration, secret and feature flag refresh enabled", func() { + ctx := context.Background() + providerName := "test-appconfigurationprovider-secret" + configMapName := "configmap-test" + secretName := "secret-test" + fakeSecretResourceVersion := "1" + testKey := "*" + wildcard := "*" + + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"testKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + existingSecrets := make(map[string]corev1.Secret) + existingSecrets[secretName] = corev1.Secret{ + Data: secretResult, + ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeSecretResourceVersion, + }, + } + + fakeKeyValueEtag := azcore.ETag("fake-etag-1") + existingKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &fakeKeyValueEtag, + }, + } + + var fakeId azsecrets.ID = "fakeSecretId" + var cachedFakeId azsecrets.ID = "fakeSecretId" + + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata2 := make(map[string]loader.KeyVaultSecretMetadata) + cachedSecretReferences := make(map[string]*loader.TargetSecretReference) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + secretMetadata2["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &cachedFakeId, + } + cachedSecretReferences[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata2, + SecretResourceVersion: fakeSecretResourceVersion, + } + + newFakeEtag := azcore.ETag("fake-etag-1") + newFakeEtag2 := azcore.ETag("fake-etag-2") + updatedFeatureFlagEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &wildcard, + }: { + &newFakeEtag, + }, + } + updatedKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &newFakeEtag2, + }, + } + + allSettingsReturnedByFeatureFlagRefresh := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + FeatureFlagETags: updatedFeatureFlagEtags, + } + + allSettingsReturnedByKeyValueRefresh := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: updatedKeyValueEtags, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKey, + }, + }, + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Enabled: true, + Interval: "70s", + }, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1h", + Enabled: true, + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &wildcard, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + }, + } + + fakeResourceVersion := "1" + fakeEtag := azcore.ETag("fake-etag") + nowTime := metav1.Now() + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextSecretReferenceRefreshReconcileTime: metav1.NewTime(nowTime.Time.Add(1 * time.Hour)), + NextKeyValueRefreshReconcileTime: nowTime, + NextFeatureFlagRefreshReconcileTime: nowTime, + FeatureFlagETags: map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &wildcard, + }: { + &fakeEtag, + }, + }, + KeyValueETags: existingKeyValueEtags, + ExistingSecretReferences: cachedSecretReferences, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.NewTime(nowTime.Time.Add(2 * time.Second)), + RefreshOptions: &RefreshOptions{}, + } + + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshFeatureFlagSettings(gomock.Any(), gomock.Any()).Return(allSettingsReturnedByFeatureFlagRefresh, nil) + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettingsReturnedByKeyValueRefresh, nil) + expectedNextKeyValueRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(70 * time.Second)) + expectedNextFeatureFlagRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(1 * time.Minute)) + cachedNextSecretReferenceRefreshReconcileTime := processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + // SecretsMetadata keeps the same when no changes in KVR + Expect(*processor.ReconciliationState.ExistingSecretReferences[secretName].SecretsMetadata["testSecretKey"].SecretId).Should( + Equal(*allSettingsReturnedByKeyValueRefresh.SecretReferences[secretName].SecretsMetadata["testSecretKey"].SecretId)) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &wildcard, + }]).Should(Equal(updatedFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &wildcard, + }])) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(expectedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime).Should(Equal(cachedNextSecretReferenceRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime).Should(Equal(expectedNextFeatureFlagRefreshReconcileTime)) + }) + + // Should update the corresponding target's reconcile state, and those that shouldn't be updated should be as is. + It("Should update reconcile state when keyValue pageEtag, featureFlag pageEtag and secret references updated when configuration, secret and feature flag refresh enabled", func() { + ctx := context.Background() + providerName := "test-appconfigurationprovider-secret" + configMapName := "configmap-test" + secretName := "secret-test" + fakeSecretResourceVersion := "1" + testKey := "*" + wildcard := "*" + + mapResult := make(map[string]string) + mapResult["filestyle.json"] = "{\"testKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + existingSecrets := make(map[string]corev1.Secret) + existingSecrets[secretName] = corev1.Secret{ + Data: secretResult, + ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeSecretResourceVersion, + }, + } + + var fakeId azsecrets.ID = "fakeSecretId" + var cachedFakeId azsecrets.ID = "cachedFakeSecretId" + + secretMetadata := make(map[string]loader.KeyVaultSecretMetadata) + secretMetadata2 := make(map[string]loader.KeyVaultSecretMetadata) + cachedSecretReferences := make(map[string]*loader.TargetSecretReference) + secretMetadata["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &fakeId, + } + secretMetadata2["testSecretKey"] = loader.KeyVaultSecretMetadata{ + SecretId: &cachedFakeId, + } + cachedSecretReferences[secretName] = &loader.TargetSecretReference{ + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata2, + SecretResourceVersion: fakeSecretResourceVersion, + } + fakeKeyValueEtag := azcore.ETag("fake-etag-1") + existingKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &fakeKeyValueEtag, + }, + } + fakeFeatureFlagEtag := azcore.ETag("fake-etag-2") + existingFeatureFlagEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &wildcard, + }: { + &fakeFeatureFlagEtag, + }, + } + + newFakeEtag := azcore.ETag("fake-etag-1") + newFakeEtag2 := azcore.ETag("fake-etag-2") + updatedKeyValueEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &testKey, + }: { + &newFakeEtag, + }, + } + updatedFeatureFlagEtags := map[acpv1.Selector][]*azcore.ETag{ + { + KeyFilter: &wildcard, + }: { + &newFakeEtag2, + }, + } + + allSettingsReturnedByFeatureFlagRefresh := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: updatedKeyValueEtags, + FeatureFlagETags: updatedFeatureFlagEtags, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + allSettingsReturnedByKeyValueRefresh := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + KeyValueETags: updatedKeyValueEtags, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + SecretResourceVersion: fakeSecretResourceVersion, + }, + }, + } + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Generation: 1, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + ConfigMapData: &acpv1.ConfigMapDataOptions{ + Type: "json", + Key: "filestyle.json", + }, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &testKey, + }, + }, + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Enabled: true, + Interval: "45s", + }, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + FeatureFlag: &acpv1.AzureAppConfigurationFeatureFlagOptions{ + Selectors: []acpv1.Selector{ + { + KeyFilter: &wildcard, + }, + }, + Refresh: &acpv1.FeatureFlagRefreshSettings{ + Interval: "40s", + Enabled: true, + }, + }, + }, + } + + fakeResourceVersion := "1" + nowTime := metav1.Now() + processor := AppConfigurationProviderProcessor{ + Context: ctx, + Retriever: mockConfigurationSettings, + Provider: configProvider, + ShouldReconcile: false, + Settings: &loader.TargetKeyValueSettings{}, + ReconciliationState: &ReconciliationState{ + NextSecretReferenceRefreshReconcileTime: metav1.NewTime(nowTime.Time.Add(30 * time.Second)), + NextKeyValueRefreshReconcileTime: metav1.NewTime(nowTime.Time.Add(5 * time.Second)), + NextFeatureFlagRefreshReconcileTime: nowTime, + KeyValueETags: existingKeyValueEtags, + FeatureFlagETags: existingFeatureFlagEtags, + ExistingSecretReferences: cachedSecretReferences, + Generation: 1, + ConfigMapResourceVersion: &fakeResourceVersion, + }, + CurrentTime: metav1.NewTime(nowTime.Time.Add(1 * time.Second)), + RefreshOptions: &RefreshOptions{}, + } + + // only feature flag refresh + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshFeatureFlagSettings(gomock.Any(), gomock.Any()).Return(allSettingsReturnedByFeatureFlagRefresh, nil) + expectedNextFeatureFlagRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(40 * time.Second)) + cachedNextKeyValueRefreshReconcileTime := processor.ReconciliationState.NextKeyValueRefreshReconcileTime + cachedNextSecretReferenceRefreshReconcileTime := processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.ExistingSecretReferences).Should(Equal(cachedSecretReferences)) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(existingKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &wildcard, + }]).Should(Equal(updatedFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &wildcard, + }])) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(cachedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime).Should(Equal(cachedNextSecretReferenceRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime).Should(Equal(expectedNextFeatureFlagRefreshReconcileTime)) + + processor.CurrentTime = metav1.NewTime(processor.ReconciliationState.NextKeyValueRefreshReconcileTime.Time.Add(1 * time.Second)) + + // only key value refresh + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(true, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettingsReturnedByKeyValueRefresh, nil) + expectedNextKeyValueRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(45 * time.Second)) + cachedNextFeatureFlagRefreshReconcileTime := processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.ExistingSecretReferences).Should(Equal(allSettingsReturnedByKeyValueRefresh.SecretReferences)) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &wildcard, + }]).Should(Equal(updatedFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &wildcard, + }])) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(expectedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime).Should(Equal(cachedNextFeatureFlagRefreshReconcileTime)) + + processor.CurrentTime = metav1.NewTime(processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime.Time.Add(2 * time.Second)) + processor.RefreshOptions = &RefreshOptions{} + + allSettingsReturnedBySecretRefresh := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + SecretsMetadata: secretMetadata, + }, + }, + } + + // only secret refresh + mockConfigurationSettings.EXPECT().CheckPageETags(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettingsReturnedBySecretRefresh, nil) + expectedNextSecretReferenceRefreshReconcileTime := metav1.NewTime(processor.CurrentTime.Time.Add(1 * time.Minute)) + cachedNextKeyValueRefreshReconcileTime = processor.ReconciliationState.NextKeyValueRefreshReconcileTime + cachedNextFeatureFlagRefreshReconcileTime = processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime + + _ = processor.PopulateSettings(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: fakeResourceVersion, + }}, existingSecrets) + + _, _ = processor.Finish() + + Expect(processor.ReconciliationState.ExistingSecretReferences).Should(Equal(allSettingsReturnedBySecretRefresh.SecretReferences)) + Expect(processor.ReconciliationState.KeyValueETags[acpv1.Selector{ + KeyFilter: &testKey, + }]).Should(Equal(updatedKeyValueEtags[acpv1.Selector{ + KeyFilter: &testKey, + }])) + Expect(processor.ReconciliationState.FeatureFlagETags[acpv1.Selector{ + KeyFilter: &wildcard, + }]).Should(Equal(updatedFeatureFlagEtags[acpv1.Selector{ + KeyFilter: &wildcard, + }])) + Expect(processor.ReconciliationState.NextSecretReferenceRefreshReconcileTime).Should(Equal(expectedNextSecretReferenceRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextKeyValueRefreshReconcileTime).Should(Equal(cachedNextKeyValueRefreshReconcileTime)) + Expect(processor.ReconciliationState.NextFeatureFlagRefreshReconcileTime).Should(Equal(cachedNextFeatureFlagRefreshReconcileTime)) + }) + }) +})