Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(reconciler): refactors Reconciler to simplify handling dependent resources #185

Merged
35 changes: 17 additions & 18 deletions controllers/inferenceservice_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
networkingv1 "k8s.io/api/networking/v1"
authv1 "k8s.io/api/rbac/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -40,22 +39,22 @@ import (

// OpenshiftInferenceServiceReconciler holds the controller configuration.
type OpenshiftInferenceServiceReconciler struct {
client client.Client
scheme *runtime.Scheme
log logr.Logger
MeshDisabled bool
mmISVCReconciler *reconcilers.ModelMeshInferenceServiceReconciler
kserveISVCReconciler *reconcilers.KserveInferenceServiceReconciler
client client.Client
log logr.Logger
MeshDisabled bool
mmISVCReconciler *reconcilers.ModelMeshInferenceServiceReconciler
kserveServerlessISVCReconciler *reconcilers.KserveServerlessInferenceServiceReconciler
kserveRawISVCReconciler *reconcilers.KserveRawInferenceServiceReconciler
}

func NewOpenshiftInferenceServiceReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger, meshDisabled bool) *OpenshiftInferenceServiceReconciler {
func NewOpenshiftInferenceServiceReconciler(client client.Client, log logr.Logger, meshDisabled bool) *OpenshiftInferenceServiceReconciler {
return &OpenshiftInferenceServiceReconciler{
client: client,
scheme: scheme,
log: log,
MeshDisabled: meshDisabled,
mmISVCReconciler: reconcilers.NewModelMeshInferenceServiceReconciler(client, scheme),
kserveISVCReconciler: reconcilers.NewKServeInferenceServiceReconciler(client, scheme),
client: client,
log: log,
MeshDisabled: meshDisabled,
mmISVCReconciler: reconcilers.NewModelMeshInferenceServiceReconciler(client),
kserveServerlessISVCReconciler: reconcilers.NewKServeServerlessInferenceServiceReconciler(client),
kserveRawISVCReconciler: reconcilers.NewKServeRawInferenceServiceReconciler(client),
}
}

Expand Down Expand Up @@ -98,10 +97,10 @@ func (r *OpenshiftInferenceServiceReconciler) Reconcile(ctx context.Context, req
err = r.mmISVCReconciler.Reconcile(ctx, log, isvc)
case utils.Serverless:
log.Info("Reconciling InferenceService for Kserve in mode Serverless")
err = r.kserveISVCReconciler.ReconcileServerless(ctx, log, isvc)
err = r.kserveServerlessISVCReconciler.Reconcile(ctx, log, isvc)
case utils.RawDeployment:
log.Info("Reconciling InferenceService for Kserve in mode RawDeployment")
err = r.kserveISVCReconciler.ReconcileRawDeployment(ctx, log, isvc)
err = r.kserveRawISVCReconciler.Reconcile(ctx, log, isvc)
}

return ctrl.Result{}, err
Expand Down Expand Up @@ -183,13 +182,13 @@ func (r *OpenshiftInferenceServiceReconciler) onDeletion(ctx context.Context, lo
}
if IsvcDeploymentMode == utils.Serverless {
log.V(1).Info("Deleting kserve inference resource (Serverless Mode)")
return r.kserveISVCReconciler.OnDeletionOfKserveInferenceService(ctx, log, inferenceService)
return r.kserveServerlessISVCReconciler.OnDeletionOfKserveInferenceService(ctx, log, inferenceService)
}
return nil
}

func (r *OpenshiftInferenceServiceReconciler) DeleteResourcesIfNoIsvcExists(ctx context.Context, log logr.Logger, isvcNamespace string) error {
if err := r.kserveISVCReconciler.DeleteKserveMetricsResourcesIfNoKserveIsvcExists(ctx, log, isvcNamespace); err != nil {
if err := r.kserveServerlessISVCReconciler.DeleteKserveMetricsResourcesIfNoKserveIsvcExists(ctx, log, isvcNamespace); err != nil {
return err
}
if err := r.mmISVCReconciler.DeleteModelMeshResourcesIfNoMMIsvcExists(ctx, log, isvcNamespace); err != nil {
Expand Down
4 changes: 1 addition & 3 deletions controllers/kserve_customcacert_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
corev1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -41,8 +40,7 @@ const (

type KServeCustomCACertReconciler struct {
client.Client
Scheme *runtime.Scheme
Log logr.Logger
Log logr.Logger
}

// reconcileConfigMap watch odh global ca cert and it will create/update/delete kserve custom cert configmap
Expand Down
2 changes: 0 additions & 2 deletions controllers/monitoring_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
k8srbacv1 "k8s.io/api/rbac/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"reflect"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -43,7 +42,6 @@ const (

type MonitoringReconciler struct {
client.Client
Scheme *runtime.Scheme
Log logr.Logger
MonitoringNS string
}
Expand Down
5 changes: 1 addition & 4 deletions controllers/mr_inferenceservice_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"google.golang.org/grpc/credentials/insecure"
corev1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
Expand All @@ -29,15 +28,13 @@ const modelRegistryFinalizer = "modelregistry.opendatahub.io/finalizer"
// ModelRegistryInferenceServiceReconciler holds the controller configuration.
type ModelRegistryInferenceServiceReconciler struct {
client client.Client
scheme *runtime.Scheme
log logr.Logger
deltaProcessor processors.DeltaProcessor
}

func NewModelRegistryInferenceServiceReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger) *ModelRegistryInferenceServiceReconciler {
func NewModelRegistryInferenceServiceReconciler(client client.Client, log logr.Logger) *ModelRegistryInferenceServiceReconciler {
return &ModelRegistryInferenceServiceReconciler{
client: client,
scheme: scheme,
log: log,
deltaProcessor: processors.NewDeltaProcessor(),
}
Expand Down
31 changes: 18 additions & 13 deletions controllers/reconcilers/kserve_authconfig_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,26 +27,25 @@ import (
"github.com/opendatahub-io/odh-model-controller/controllers/processors"
"github.com/opendatahub-io/odh-model-controller/controllers/resources"
k8serror "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var _ SubResourceReconciler = (*KserveAuthConfigReconciler)(nil)

type KserveAuthConfigReconciler struct {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

KserveAuthConfigReconciler should implement NoResourceRemoval interface as it have owner relationship with inference resource.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

client client.Client
scheme *runtime.Scheme
deltaProcessor processors.DeltaProcessor
detector resources.AuthTypeDetector
store resources.AuthConfigStore
templateLoader resources.AuthConfigTemplateLoader
hostExtractor resources.InferenceServiceHostExtractor
}

func NewKserveAuthConfigReconciler(client client.Client, scheme *runtime.Scheme) *KserveAuthConfigReconciler {
func NewKserveAuthConfigReconciler(client client.Client) *KserveAuthConfigReconciler {
return &KserveAuthConfigReconciler{
client: client,
scheme: scheme,
deltaProcessor: processors.NewDeltaProcessor(),
detector: resources.NewKServeAuthTypeDetector(client),
store: resources.NewClientAuthConfigStore(client),
Expand All @@ -56,6 +55,7 @@ func NewKserveAuthConfigReconciler(client client.Client, scheme *runtime.Scheme)
}

func (r *KserveAuthConfigReconciler) Reconcile(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error {
log.V(1).Info("Reconciling Authorino AuthConfig for InferenceService")

if isvc.Status.URL == nil {
log.V(1).Info("Inference Service not ready yet, waiting for URL")
Expand All @@ -77,7 +77,20 @@ func (r *KserveAuthConfigReconciler) Reconcile(ctx context.Context, log logr.Log
return err
}
return nil
}

func (r *KserveAuthConfigReconciler) Delete(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need explicit Delete function implementation for authorinov1beta2.AuthConfig type resource. As I understood, it already connected with owner reference of InferenceService?

Copy link
Contributor Author

@bartoszmajsak bartoszmajsak Apr 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I thought about it too. It's been like that before and I wonder if we need it. We have an internal store of processed templates, that's why there's this hook here, though it's not really about Deleting stuff in the cluster. @aslakknutsen can this be simplified somehow?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The store is just a wrapper around Client and has no logic per say. We could leave clean up to Kubernetes GC as suggested.

Copy link
Contributor Author

@bartoszmajsak bartoszmajsak Apr 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok then it would only leave route reconciler left with Delete, as it's a special case as route sits in istio ns. I will clean it up.

That's this piece in main:

func (r *KserveRouteReconciler) DeleteRoute(ctx context.Context, isvc *kservev1beta1.InferenceService) error {
return r.routeHandler.DeleteRoute(ctx, types.NamespacedName{Name: getKServeRouteName(isvc), Namespace: constants2.IstioNamespace})
}

log.V(1).Info("Deleting Kserve inference service authorino authconfig entry")
typeName := types.NamespacedName{
Name: isvc.GetName(),
Namespace: isvc.GetNamespace(),
}
return r.store.Remove(ctx, typeName)
}

func (r *KserveAuthConfigReconciler) Cleanup(_ context.Context, _ logr.Logger, _ string) error {
// NOOP
return nil
}

func (r *KserveAuthConfigReconciler) createDesiredResource(ctx context.Context, isvc *kservev1beta1.InferenceService) (*authorinov1beta2.AuthConfig, error) {
Expand All @@ -103,7 +116,7 @@ func (r *KserveAuthConfigReconciler) createDesiredResource(ctx context.Context,
}
template.Labels[constants.LabelAuthGroup] = "default"

ctrl.SetControllerReference(isvc, &template, r.scheme)
ctrl.SetControllerReference(isvc, &template, r.client.Scheme())

return &template, nil
}
Expand Down Expand Up @@ -152,11 +165,3 @@ func (r *KserveAuthConfigReconciler) processDelta(ctx context.Context, log logr.
}
return nil
}

func (r *KserveAuthConfigReconciler) Remove(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error {
typeName := types.NamespacedName{
Name: isvc.GetName(),
Namespace: isvc.GetNamespace(),
}
return r.store.Remove(ctx, typeName)
}
Loading