fix(authz): ensures extauthz provider is removed from control plane during cleanup

components/kserve/servicemesh_setup.go

@@ -32,7 +32,10 @@ func (k *Kserve) defineServiceMeshFeatures() feature.FeaturesProvider {
 		kserveExtAuthzErr := feature.CreateFeature("kserve-external-authz").
 			For(handler).
 			Manifests(
-				path.Join(feature.KServeDir),
+				path.Join(feature.KServeDir, "activator-envoyfilter.tmpl"),
+				path.Join(feature.KServeDir, "envoy-oauth-temp-fix.tmpl"),
+				path.Join(feature.KServeDir, "kserve-predictor-authorizationpolicy.tmpl"),
+				path.Join(feature.KServeDir, "z-migrations"),
 			).
 			WithData(servicemesh.ClusterDetails).
 			Load()

pkg/feature/servicemesh/cleanup.go

@@ -12,7 +12,7 @@ import (
 )
 
 func RemoveExtensionProvider(f *feature.Feature) error {
-	ossmAuthzProvider := fmt.Sprintf("%s-odh-auth-provider", f.Spec.AppNamespace)
+	ossmAuthzProvider := fmt.Sprintf("%s-auth-provider", f.Spec.AppNamespace)
 
 	mesh := f.Spec.ControlPlane
 	smcp := &unstructured.Unstructured{}

tests/integration/features/features_suite_int_test.go

@@ -10,6 +10,7 @@ import (
 
 	ofapiv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	v1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
@@ -58,6 +59,7 @@ var _ = BeforeSuite(func() {
 
 	utilruntime.Must(v1.AddToScheme(testScheme))
 	utilruntime.Must(featurev1.AddToScheme(testScheme))
+	utilruntime.Must(apiextensionsv1.AddToScheme(testScheme))
 	utilruntime.Must(ofapiv1alpha1.AddToScheme(testScheme))
 
 	envTest = &envtest.Environment{

tests/integration/features/servicemesh_feature_test.go

@@ -2,6 +2,7 @@ package features_test
 
 import (
 	"context"
+	"path"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +15,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 
 	dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
+	infrav1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/infrastructure/v1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh"
@@ -24,7 +26,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("Service Mesh feature", func() {
+var _ = Describe("Service Mesh setup", func() {
 
 	var (
 		dsci          *dsciv1.DSCInitialization
@@ -43,136 +45,227 @@ var _ = Describe("Service Mesh feature", func() {
 		Expect(err).ToNot(HaveOccurred())
 	})
 
-	AfterEach(func() {
-
-	})
-
 	Describe("preconditions", func() {
 
-		When("operator is not installed", func() {
+		Context("operator setup", func() {
 
-			It("should fail using precondition check", func() {
-				// given
-				featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
-					verificationFeatureErr := feature.CreateFeature("no-service-mesh-operator-check").
-						For(handler).
-						UsingConfig(envTest.Config).
-						PreConditions(servicemesh.EnsureServiceMeshOperatorInstalled).
-						Load()
+			When("operator is not installed", func() {
 
-					Expect(verificationFeatureErr).ToNot(HaveOccurred())
+				It("should fail using precondition check", func() {
+					// given
+					featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
+						verificationFeatureErr := feature.CreateFeature("no-service-mesh-operator-check").
+							For(handler).
+							UsingConfig(envTest.Config).
+							PreConditions(servicemesh.EnsureServiceMeshOperatorInstalled).
+							Load()
 
-					return nil
-				})
+						Expect(verificationFeatureErr).ToNot(HaveOccurred())
+
+						return nil
+					})
 
-				// when
-				applyErr := featuresHandler.Apply()
+					// when
+					applyErr := featuresHandler.Apply()
 
-				// then
-				Expect(applyErr).To(MatchError(ContainSubstring("failed to find the pre-requisite operator subscription \"servicemeshoperator\"")))
+					// then
+					Expect(applyErr).To(MatchError(ContainSubstring("failed to find the pre-requisite operator subscription \"servicemeshoperator\"")))
+				})
 			})
-		})
 
-		When("operator is installed", func() {
-			var smcpCrdObj *apiextensionsv1.CustomResourceDefinition
+			When("operator is installed", func() {
+				var smcpCrdObj *apiextensionsv1.CustomResourceDefinition
 
-			BeforeEach(func() {
-				err := fixtures.CreateSubscription(fixtures.OssmSubscription, "openshift-operators", envTestClient)
-				Expect(err).ToNot(HaveOccurred())
+				BeforeEach(func() {
+					err := fixtures.CreateSubscription(fixtures.OssmSubscription, "openshift-operators", envTestClient)
+					Expect(err).ToNot(HaveOccurred())
+					smcpCrdObj = installServiceMeshCRD()
+				})
 
-				// Create SMCP CRD
-				smcpCrdObj = &apiextensionsv1.CustomResourceDefinition{}
-				Expect(yaml.Unmarshal([]byte(fixtures.ServiceMeshControlPlaneCRD), smcpCrdObj)).ToNot(HaveOccurred())
-				c, err := client.New(envTest.Config, client.Options{})
-				Expect(err).ToNot(HaveOccurred())
-				Expect(c.Create(context.TODO(), smcpCrdObj)).ToNot(HaveOccurred())
+				AfterEach(func() {
+					objectCleaner.DeleteAll(smcpCrdObj)
+				})
 
-				crdOptions := envtest.CRDInstallOptions{PollInterval: interval, MaxTime: timeout}
-				err = envtest.WaitForCRDs(envTest.Config, []*apiextensionsv1.CustomResourceDefinition{smcpCrdObj}, crdOptions)
-				Expect(err).ToNot(HaveOccurred())
-			})
+				It("should succeed using precondition check", func() {
+					// when
+					featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
+						verificationFeatureErr := feature.CreateFeature("service-mesh-operator-check").
+							For(handler).
+							UsingConfig(envTest.Config).
+							PreConditions(servicemesh.EnsureServiceMeshOperatorInstalled).
+							Load()
 
-			AfterEach(func() {
-				// Delete SMCP CRD
-				objectCleaner.DeleteAll(smcpCrdObj)
-			})
+						Expect(verificationFeatureErr).ToNot(HaveOccurred())
 
-			It("should succeed using precondition check", func() {
-				// when
-				featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
-					verificationFeatureErr := feature.CreateFeature("service-mesh-operator-check").
-						For(handler).
-						UsingConfig(envTest.Config).
-						PreConditions(servicemesh.EnsureServiceMeshOperatorInstalled).
-						Load()
+						return nil
+					})
 
-					Expect(verificationFeatureErr).ToNot(HaveOccurred())
+					// when
+					Expect(featuresHandler.Apply()).To(Succeed())
 
-					return nil
 				})
 
-				// when
-				Expect(featuresHandler.Apply()).To(Succeed())
+				It("should find installed Service Mesh Control Plane", func() {
+					// given
+					c, err := client.New(envTest.Config, client.Options{})
+					Expect(err).ToNot(HaveOccurred())
 
-			})
+					ns := envtestutil.AppendRandomNameTo(fixtures.TestNamespacePrefix)
+					nsResource := fixtures.NewNamespace(ns)
+					Expect(c.Create(context.Background(), nsResource)).To(Succeed())
+					defer objectCleaner.DeleteAll(nsResource)
+
+					createServiceMeshControlPlane("test-name", ns)
+					dsci.Spec.ServiceMesh.ControlPlane.Namespace = ns
+					dsci.Spec.ServiceMesh.ControlPlane.Name = "test-name"
 
-			It("should find installed Service Mesh Control Plane", func() {
-				// given
-				c, err := client.New(envTest.Config, client.Options{})
-				Expect(err).ToNot(HaveOccurred())
+					// when
+					featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
+						verificationFeatureErr := feature.CreateFeature("service-mesh-control-plane-check").
+							For(handler).
+							UsingConfig(envTest.Config).
+							PreConditions(servicemesh.EnsureServiceMeshInstalled).
+							Load()
+
+						Expect(verificationFeatureErr).ToNot(HaveOccurred())
+
+						return nil
+					})
+
+					// then
+					Expect(featuresHandler.Apply()).To(Succeed())
+				})
 
-				ns := envtestutil.AppendRandomNameTo(fixtures.TestNamespacePrefix)
-				nsResource := fixtures.NewNamespace(ns)
-				Expect(c.Create(context.Background(), nsResource)).To(Succeed())
-				defer objectCleaner.DeleteAll(nsResource)
+				It("should fail to find Service Mesh Control Plane if not present", func() {
+					// given
+					dsci.Spec.ServiceMesh.ControlPlane.Name = "test-name"
 
-				createServiceMeshControlPlane("test-name", ns)
-				dsci.Spec.ServiceMesh.ControlPlane.Namespace = ns
-				dsci.Spec.ServiceMesh.ControlPlane.Name = "test-name"
+					// when
+					featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
+						verificationFeatureErr := feature.CreateFeature("no-service-mesh-control-plane-check").
+							For(handler).
+							UsingConfig(envTest.Config).
+							PreConditions(servicemesh.EnsureServiceMeshInstalled).
+							Load()
 
-				// when
-				featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
-					verificationFeatureErr := feature.CreateFeature("service-mesh-control-plane-check").
-						For(handler).
-						UsingConfig(envTest.Config).
-						PreConditions(servicemesh.EnsureServiceMeshInstalled).
-						Load()
+						Expect(verificationFeatureErr).ToNot(HaveOccurred())
 
-					Expect(verificationFeatureErr).ToNot(HaveOccurred())
+						return nil
+					})
 
-					return nil
+					// then
+					Expect(featuresHandler.Apply()).To(MatchError(ContainSubstring("failed to find Service Mesh Control Plane")))
 				})
 
-				// then
-				Expect(featuresHandler.Apply()).To(Succeed())
 			})
+		})
+
+		Context("Control Plane configuration", func() {
+
+			When("setting up auth(z) provider", func() {
 
-			It("should fail to find Service Mesh Control Plane if not present", func() {
-				// given
-				dsci.Spec.ServiceMesh.ControlPlane.Name = "test-name"
+				var (
+					objectCleaner   *envtestutil.Cleaner
+					dsci            *dsciv1.DSCInitialization
+					serviceMeshSpec *infrav1.ServiceMeshSpec
+					smcpCrdObj      *apiextensionsv1.CustomResourceDefinition
+					namespace       = "test-ns"
+					name            = "minimal"
+				)
 
-				// when
-				featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
-					verificationFeatureErr := feature.CreateFeature("no-service-mesh-control-plane-check").
-						For(handler).
-						UsingConfig(envTest.Config).
-						PreConditions(servicemesh.EnsureServiceMeshInstalled).
-						Load()
+				BeforeEach(func() {
+					smcpCrdObj = installServiceMeshCRD()
+					objectCleaner = envtestutil.CreateCleaner(envTestClient, envTest.Config, timeout, interval)
+					dsci = newDSCInitialization(namespace)
 
-					Expect(verificationFeatureErr).ToNot(HaveOccurred())
+					serviceMeshSpec = &dsci.Spec.ServiceMesh
+
+					serviceMeshSpec.ControlPlane.Name = name
+					serviceMeshSpec.ControlPlane.Namespace = namespace
+				})
+
+				AfterEach(func() {
+					objectCleaner.DeleteAll(smcpCrdObj)
+				})
+
+				It("should be able to remove external provider on cleanup", func() {
+					// given
+					ns := fixtures.NewNamespace(namespace)
+					Expect(envTestClient.Create(context.Background(), ns)).To(Succeed())
+					defer objectCleaner.DeleteAll(ns)
+
+					serviceMeshSpec.Auth.Namespace = "auth-provider"
+
+					createServiceMeshControlPlane(name, namespace)
+
+					handler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error {
+						return feature.CreateFeature("control-plane-with-external-authz-provider").
+							For(handler).
+							Manifests(path.Join(feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl")).
+							OnDelete(servicemesh.RemoveExtensionProvider).
+							UsingConfig(envTest.Config).
+							Load()
+					})
+
+					// when
+					By("verifying extension provider has been added after applying feature", func() {
+						Expect(handler.Apply()).To(Succeed())
+						serviceMeshControlPlane, err := getServiceMeshControlPlane(namespace, name)
+						Expect(err).ToNot(HaveOccurred())
+
+						extensionProviders, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "techPreview", "meshConfig", "extensionProviders")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(found).To(BeTrue())
+
+						extensionProvider, ok := extensionProviders[0].(map[string]interface{})
+						if !ok {
+							Fail("extension provider has not been added after applying feature")
+						}
+						Expect(extensionProvider["name"]).To(Equal("test-ns-auth-provider"))
+
+						envoyExtAuthzGrpc, ok := extensionProvider["envoyExtAuthzGrpc"].(map[string]interface{})
+						if !ok {
+							Fail("extension provider envoyExtAuthzGrpc has not been added after applying feature")
+						}
+						Expect(envoyExtAuthzGrpc["service"]).To(Equal("authorino-authorino-authorization.auth-provider.svc.cluster.local"))
+					})
+
+					// then
+					By("verifying that extension provider has been removed", func() {
+						Expect(handler.Delete()).To(Succeed())
+						Eventually(func() []interface{} {
+
+							serviceMeshControlPlane, err := getServiceMeshControlPlane(namespace, name)
+							Expect(err).ToNot(HaveOccurred())
+
+							extensionProviders, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "techPreview", "meshConfig", "extensionProviders")
+							Expect(err).ToNot(HaveOccurred())
+							Expect(found).To(BeTrue())
+							return extensionProviders
+
+						}).WithTimeout(timeout).WithPolling(interval).Should(BeEmpty())
+					})
 
-					return nil
 				})
 
-				// then
-				Expect(featuresHandler.Apply()).To(MatchError(ContainSubstring("failed to find Service Mesh Control Plane")))
 			})
 
 		})
 
 	})
 })
 
+func installServiceMeshCRD() *apiextensionsv1.CustomResourceDefinition {
+	smcpCrdObj := &apiextensionsv1.CustomResourceDefinition{}
+	Expect(yaml.Unmarshal([]byte(fixtures.ServiceMeshControlPlaneCRD), smcpCrdObj)).ToNot(HaveOccurred())
+	Expect(envTestClient.Create(context.TODO(), smcpCrdObj)).ToNot(HaveOccurred())
+
+	crdOptions := envtest.CRDInstallOptions{PollInterval: interval, MaxTime: timeout}
+	Expect(envtest.WaitForCRDs(envTest.Config, []*apiextensionsv1.CustomResourceDefinition{smcpCrdObj}, crdOptions)).To(Succeed())
+
+	return smcpCrdObj
+}
+
 func getGateway(cfg *rest.Config, namespace, name string) (*unstructured.Unstructured, error) {
 	dynamicClient, err := dynamic.NewForConfig(cfg)
 	if err != nil {
@@ -241,3 +334,15 @@ func createSMCPInCluster(smcpObj *unstructured.Unstructured, namespace string) e
 
 	return envTestClient.Status().Update(context.TODO(), update)
 }
+
+func getServiceMeshControlPlane(namespace, name string) (*unstructured.Unstructured, error) {
+	smcpObj := &unstructured.Unstructured{}
+	smcpObj.SetGroupVersionKind(cluster.ServiceMeshControlPlaneGVK)
+
+	err := envTestClient.Get(context.TODO(), client.ObjectKey{
+		Namespace: namespace,
+		Name:      name,
+	}, smcpObj)
+
+	return smcpObj, err
+}