feat(authz): Authorino for Service Mesh

This first iteration is to cover authentication needs for KServe * Add templates to install Authorino * Add templates to configure Service Mesh to use Authorino to delegate Authorization * Add KServe-specific templates add ability to secure KServe Inference Services * Add relevant fields to DSCInitialization resource * Code for proper cleanup, in case of uninstalling Most (if not all) of this code comes from pull request opendatahub-io#605. Attribution to original authors: @bartoszmajsak, @aslakknutsen, @cam-garrison, et. al. Related opendatahub-io/kserve#128 Signed-off-by: Edgar Hernández <[email protected]>
israel-hdez · Dec 11, 2023 · 74a6f8a · 74a6f8a
1 parent 42b2bdd
commit 74a6f8a
Show file tree

Hide file tree

Showing 25 changed files with 645 additions and 68 deletions.
diff --git a/apis/dscinitialization/v1/zz_generated.deepcopy.go b/apis/dscinitialization/v1/zz_generated.deepcopy.go
diff --git a/components/kserve/kserve.go b/components/kserve/kserve.go
@@ -3,6 +3,7 @@ package kserve
 
 import (
 	"fmt"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh"
 )
 
 var (
@@ -148,6 +150,12 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci
 		}
 	}
 
+	if enabled {
+		if err := k.configureServiceMesh(cli, dscispec); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -206,3 +214,49 @@ func checkRequiredOperatorsInstalled(cli client.Client) error {
 
 	return multiErr.ErrorOrNil()
 }
+
+func (k *Kserve) configureServiceMesh(cli client.Client, dscispec *dsciv1.DSCInitializationSpec) error {
+	shouldConfigureServiceMesh, err := deploy.ShouldConfigureServiceMesh(cli, dscispec)
+	if err != nil {
+		return err
+	}
+
+	if shouldConfigureServiceMesh {
+		serviceMeshInitializer := feature.NewFeaturesInitializer(dscispec, k.defineServiceMeshFeatures(dscispec))
+
+		if err := serviceMeshInitializer.Prepare(); err != nil {
+			return err
+		}
+
+		if err := serviceMeshInitializer.Apply(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (k *Kserve) defineServiceMeshFeatures(dscispec *dsciv1.DSCInitializationSpec) feature.DefinedFeatures {
+	return func(s *feature.FeaturesInitializer) error {
+		var rootDir = filepath.Join(feature.BaseOutputDir, dscispec.ApplicationsNamespace)
+		if err := feature.CopyEmbeddedFiles("templates", rootDir); err != nil {
+			return err
+		}
+
+		kserve, err := feature.CreateFeature("configure-kserve-for-external-authz").
+			For(dscispec).
+			Manifests(
+				path.Join(rootDir, feature.KServeDir),
+			).
+			WithData(servicemesh.ClusterDetails).
+			Load()
+
+		if err != nil {
+			return err
+		}
+
+		s.Features = append(s.Features, kserve)
+
+		return nil
+	}
+}
diff --git a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml
@@ -89,6 +89,48 @@ spec:
                   user experience; e.g. it provides unified authentication giving
                   a Single Sign On experience.
                 properties:
+                  auth:
+                    description: Auth holds configuration of authentication and authorization
+                      services used by Service Mesh in Opendatahub.
+                    properties:
+                      authorino:
+                        description: Authorino holds configuration of Authorino service
+                          used as external authorization provider.
+                        properties:
+                          audiences:
+                            default:
+                            - https://kubernetes.default.svc
+                            description: Audiences is a list of the identifiers that
+                              the resource server presented with the token identifies
+                              as. Audience-aware token authenticators will verify
+                              that the token was intended for at least one of the
+                              audiences in this list. If no audiences are provided,
+                              the audience will default to the audience of the Kubernetes
+                              apiserver (kubernetes.default.svc).
+                            items:
+                              type: string
+                            type: array
+                          image:
+                            default: quay.io/kuadrant/authorino:v0.16.0
+                            description: Image allows to define a custom container
+                              image to be used when deploying Authorino's instance.
+                            type: string
+                          label:
+                            default: authorino/topic=odh
+                            description: Label narrows amount of AuthConfigs to process
+                              by Authorino service.
+                            type: string
+                          name:
+                            default: authorino-mesh-authz-provider
+                            description: Name specifies how external authorization
+                              provider should be called.
+                            type: string
+                        type: object
+                      namespace:
+                        default: auth-provider
+                        description: Namespace where it is deployed.
+                        type: string
+                    type: object
                   controlPlane:
                     description: ControlPlane holds configuration of Service Mesh
                       used by Opendatahub.

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -115,6 +115,30 @@ rules:
   - statefulsets
   verbs:
   - '*'
+- apiGroups:
+  - apps.openshift.io
+  resources:
+  - deploymentconfigs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps.openshift.io
+  resources:
+  - deploymentconfigs/instantiate
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - argoproj.io
   resources:
@@ -127,6 +151,12 @@ rules:
   - tokenreviews
   verbs:
   - create
+- apiGroups:
+  - authorino.kuadrant.io
+  resources:
+  - authconfigs
+  verbs:
+  - '*'
 - apiGroups:
   - authorization.k8s.io
   resources:
@@ -450,24 +480,7 @@ rules:
   resources:
   - secrets
   verbs:
-  - create
-  - delete
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets/finalizers
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
+  - '*'
 - apiGroups:
   - ""
   resources:
@@ -943,6 +956,12 @@ rules:
   - deletecollection
   - get
   - patch
+- apiGroups:
+  - networking.istio.io
+  resources:
+  - envoyfilters
+  verbs:
+  - '*'
 - apiGroups:
   - networking.istio.io
   resources:
@@ -1023,6 +1042,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - operator.authorino.kuadrant.io
+  resources:
+  - authorinos
+  verbs:
+  - '*'
 - apiGroups:
   - operator.knative.dev
   resources:
@@ -1163,6 +1188,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - security.istio.io
+  resources:
+  - authorizationpolicies
+  verbs:
+  - '*'
 - apiGroups:
   - security.openshift.io
   resources:

diff --git a/controllers/datasciencecluster/kubebuilder_rbac.go b/controllers/datasciencecluster/kubebuilder_rbac.go
@@ -4,14 +4,25 @@ package datasciencecluster
 //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters/finalizers,verbs=update;patch
 //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters,verbs=get;list;watch;create;update;patch;delete
 
-/* Service Mesh prerequisite */
-// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch
-
 /* Serverless prerequisite */
 // +kubebuilder:rbac:groups="networking.istio.io",resources=gateways,verbs=*
 // +kubebuilder:rbac:groups="operator.knative.dev",resources=knativeservings,verbs=*
 // +kubebuilder:rbac:groups="config.openshift.io",resources=ingresses,verbs=get
 
+/* Service Mesh Integration */
+// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch
+// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmemberrolls,verbs=create;get;list;patch;update;use;watch
+// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers,verbs=create;get;list;patch;update;use;watch
+// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch
+// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete
+// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=*
+// +kubebuilder:rbac:groups="networking.istio.io",resources=gateways,verbs=*
+// +kubebuilder:rbac:groups="networking.istio.io",resources=envoyfilters,verbs=*
+// +kubebuilder:rbac:groups="security.istio.io",resources=authorizationpolicies,verbs=*
+// +kubebuilder:rbac:groups="authorino.kuadrant.io",resources=authconfigs,verbs=*
+// +kubebuilder:rbac:groups="operator.authorino.kuadrant.io",resources=authorinos,verbs=*
+
 /* This is for DSP */
 //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/status,verbs=update;patch;get
 //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/finalizers,verbs=update;patch
@@ -93,10 +104,6 @@ package datasciencecluster
 // +kubebuilder:rbac:groups="networking.k8s.io",resources=networkpolicies,verbs=get;create;list;watch;delete;update;patch
 // +kubebuilder:rbac:groups="networking.k8s.io",resources=ingresses,verbs=create;delete;list;update;watch;patch;get
 
-// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete
-// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=*
-
 // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=servicemonitors,verbs=get;create;delete;update;watch;list;patch;deletecollection
 // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=podmonitors,verbs=get;create;delete;update;watch;list;patch
 // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheusrules,verbs=get;create;patch;delete;deletecollection
@@ -152,8 +159,7 @@ package datasciencecluster
 
 // +kubebuilder:rbac:groups="core",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
 
-// +kubebuilder:rbac:groups="core",resources=secrets,verbs=create;delete;list;update;watch;patch
-// +kubebuilder:rbac:groups="core",resources=secrets/finalizers,verbs=get;create;watch;update;patch;list;delete
+// +kubebuilder:rbac:groups="core",resources=secrets,verbs=*
 
 // +kubebuilder:rbac:groups="core",resources=rhmis,verbs=watch;list
 
@@ -187,7 +193,6 @@ package datasciencecluster
 
 // +kubebuilder:rbac:groups="cert-manager.io",resources=certificates;issuers,verbs=create;patch
 
-// OpenVino still need buildconfig
 // +kubebuilder:rbac:groups="build.openshift.io",resources=builds,verbs=create;patch;delete;list;watch
 // +kubebuilder:rbac:groups="build.openshift.io",resources=buildconfigs/instantiate,verbs=create;patch;delete;get;list;watch
 // +kubebuilder:rbac:groups="build.openshift.io",resources=buildconfigs,verbs=list;watch;create;patch;delete
@@ -218,6 +223,9 @@ package datasciencecluster
 // +kubebuilder:rbac:groups="*",resources=deployments,verbs=*
 // +kubebuilder:rbac:groups="extensions",resources=deployments,verbs=*
 
+// +kubebuilder:rbac:groups="apps.openshift.io",resources=deploymentconfigs,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups="apps.openshift.io",resources=deploymentconfigs/instantiate,verbs=get;list;watch;create;update;patch;delete
+
 // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch;create;patch;delete
 
 // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;create;update;delete;patch
@@ -232,11 +240,6 @@ package datasciencecluster
 
 // +kubebuilder:rbac:groups="*",resources=customresourcedefinitions,verbs=get;list;watch
 
-// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch
-// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmemberrolls,verbs=create;get;list;patch;update;use;watch
-// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers,verbs=create;get;list;patch;update;use;watch
-// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch
-
 /* Only for RHODS */
 // +kubebuilder:rbac:groups="user.openshift.io",resources=groups,verbs=get;create;list;watch;patch;delete
 // +kubebuilder:rbac:groups="console.openshift.io",resources=consolelinks,verbs=create;get;patch;delete
diff --git a/controllers/dscinitialization/servicemesh_setup.go b/controllers/dscinitialization/servicemesh_setup.go
@@ -100,5 +100,47 @@ func configureServiceMeshFeatures(s *feature.FeaturesInitializer) error {
 		s.Features = append(s.Features, metricsCollection)
 	}
 
+	if cfMaps, err := feature.CreateFeature("shared-config-maps").
+		For(s.DSCInitializationSpec).
+		WithResources(servicemesh.ConfigMaps).
+		Load(); err != nil {
+		return err
+	} else {
+		s.Features = append(s.Features, cfMaps)
+	}
+
+	if extAuthz, err := feature.CreateFeature("service-mesh-control-plane-setup-external-authorization").
+		For(s.DSCInitializationSpec).
+		Manifests(
+			path.Join(rootDir, feature.AuthDir, "auth-smm.tmpl"),
+			path.Join(rootDir, feature.AuthDir, "base"),
+			//path.Join(rootDir, feature.AuthDir, "rbac"),
+			path.Join(rootDir, feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl"),
+		).
+		WithData(servicemesh.ClusterDetails).
+		PreConditions(
+			feature.EnsureCRDIsInstalled("authconfigs.authorino.kuadrant.io"),
+			servicemesh.EnsureServiceMeshInstalled,
+			feature.CreateNamespaceIfNotExists(serviceMeshSpec.Auth.Namespace),
+		).
+		PostConditions(
+			feature.WaitForPodsToBeReady(serviceMeshSpec.ControlPlane.Namespace),
+			feature.WaitForPodsToBeReady(serviceMeshSpec.Auth.Namespace),
+			func(f *feature.Feature) error {
+				// We do not have the control over deployment resource creation.
+				// It is created by Authorino operator using Authorino CR
+				//
+				// To make it part of Service Mesh we have to patch it with injection
+				// enabled instead, otherwise it will not have proxy pod injected.
+				return f.ApplyManifest(path.Join(rootDir, feature.AuthDir, "deployment.injection.patch.tmpl"))
+			},
+		).
+		OnDelete(servicemesh.RemoveExtensionProvider).
+		Load(); err != nil {
+		return err
+	} else {
+		s.Features = append(s.Features, extAuthz)
+	}
+
 	return nil
 }