diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml index ec170590..f7bb5535 100644 --- a/.github/workflows/fvt.yml +++ b/.github/workflows/fvt.yml @@ -1,6 +1,7 @@ name: FVT on: + workflow_dispatch: pull_request: branches: - main diff --git a/.golangci.yaml b/.golangci.yaml index f716f984..0db2b287 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -243,11 +243,6 @@ issues: - staticcheck text: "SA9003:" - # Exclude some deprecation errors - - linters: - - staticcheck - text: "SA1019:" - # Exclude lll issues for long lines with go:generate - linters: - lll diff --git a/config/crd/bases/serving.kserve.io_clusterservingruntimes.yaml b/config/crd/bases/serving.kserve.io_clusterservingruntimes.yaml index cd7219d3..2b434b83 100644 --- a/config/crd/bases/serving.kserve.io_clusterservingruntimes.yaml +++ b/config/crd/bases/serving.kserve.io_clusterservingruntimes.yaml @@ -1,4 +1,4 @@ -# Copied from https://github.com/kserve/kserve/blob/v0.11.0/config/crd/serving.kserve.io_clusterservingruntimes.yaml +# Copied from https://github.com/kserve/kserve/blob/v0.11.1/config/crd/serving.kserve.io_clusterservingruntimes.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1133,6 +1133,10 @@ spec: type: boolean name: type: string + priority: + format: int32 + minimum: 1 + type: integer version: type: string required: diff --git a/config/crd/bases/serving.kserve.io_inferenceservices.yaml b/config/crd/bases/serving.kserve.io_inferenceservices.yaml index 0305c1d6..d572d99c 100644 --- a/config/crd/bases/serving.kserve.io_inferenceservices.yaml +++ b/config/crd/bases/serving.kserve.io_inferenceservices.yaml @@ -1,4 +1,4 @@ -# Copied from https://github.com/kserve/kserve/blob/v0.11.0/config/crd/serving.kserve.io_inferenceservices.yaml +# Copied from https://github.com/kserve/kserve/blob/v0.11.1/config/crd/serving.kserve.io_inferenceservices.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/config/crd/bases/serving.kserve.io_servingruntimes.yaml b/config/crd/bases/serving.kserve.io_servingruntimes.yaml index 12634391..68083880 100644 --- a/config/crd/bases/serving.kserve.io_servingruntimes.yaml +++ b/config/crd/bases/serving.kserve.io_servingruntimes.yaml @@ -1,4 +1,4 @@ -# Copied from https://github.com/kserve/kserve/blob/v0.11.0/config/crd/serving.kserve.io_servingruntimes.yaml +# Copied from https://github.com/kserve/kserve/blob/v0.11.1/config/crd/serving.kserve.io_servingruntimes.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1133,6 +1133,10 @@ spec: type: boolean name: type: string + priority: + format: int32 + minimum: 1 + type: integer version: type: string required: diff --git a/controllers/modelmesh/constraints.go b/controllers/modelmesh/constraints.go index dd7392d5..8964c722 100644 --- a/controllers/modelmesh/constraints.go +++ b/controllers/modelmesh/constraints.go @@ -18,6 +18,8 @@ import ( "errors" "strings" + "k8s.io/apimachinery/pkg/util/sets" + kserveapi "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -68,5 +70,5 @@ func (m *Deployment) addModelTypeConstraints(deployment *appsv1.Deployment) erro } func generateLabelsEnvVar(rts *kserveapi.ServingRuntimeSpec, restProxyEnabled bool, rtName string) string { - return strings.Join(GetServingRuntimeLabelSet(rts, restProxyEnabled, rtName).List(), ",") + return strings.Join(sets.List(GetServingRuntimeLabelSet(rts, restProxyEnabled, rtName)), ",") } diff --git a/controllers/modelmesh/model_type_labels.go b/controllers/modelmesh/model_type_labels.go index 5b212dcd..b6a67ebf 100644 --- a/controllers/modelmesh/model_type_labels.go +++ b/controllers/modelmesh/model_type_labels.go @@ -24,10 +24,10 @@ import ( ) func GetServingRuntimeLabelSets(rt *kserveapi.ServingRuntimeSpec, restProxyEnabled bool, rtName string) ( - mtLabels sets.String, pvLabels sets.String, rtLabel string) { + mtLabels sets.Set[string], pvLabels sets.Set[string], rtLabel string) { // model type labels - mtSet := make(sets.String, 2*len(rt.SupportedModelFormats)) + mtSet := make(sets.Set[string], 2*len(rt.SupportedModelFormats)) for _, t := range rt.SupportedModelFormats { // only include model type labels when autoSelect is true if t.AutoSelect != nil && *t.AutoSelect { @@ -38,7 +38,7 @@ func GetServingRuntimeLabelSets(rt *kserveapi.ServingRuntimeSpec, restProxyEnabl } } // protocol versions - pvSet := make(sets.String, len(rt.ProtocolVersions)) + pvSet := make(sets.Set[string], len(rt.ProtocolVersions)) for _, pv := range rt.ProtocolVersions { pvSet.Insert(fmt.Sprintf("pv:%s", pv)) if restProxyEnabled && pv == constants.ProtocolGRPCV2 { @@ -49,7 +49,7 @@ func GetServingRuntimeLabelSets(rt *kserveapi.ServingRuntimeSpec, restProxyEnabl return mtSet, pvSet, fmt.Sprintf("rt:%s", rtName) } -func GetServingRuntimeLabelSet(rt *kserveapi.ServingRuntimeSpec, restProxyEnabled bool, rtName string) sets.String { +func GetServingRuntimeLabelSet(rt *kserveapi.ServingRuntimeSpec, restProxyEnabled bool, rtName string) sets.Set[string] { s1, s2, l := GetServingRuntimeLabelSets(rt, restProxyEnabled, rtName) s1 = s1.Union(s2) s1.Insert(l) diff --git a/controllers/modelmesh/model_type_labels_test.go b/controllers/modelmesh/model_type_labels_test.go index 2a427062..dded972f 100644 --- a/controllers/modelmesh/model_type_labels_test.go +++ b/controllers/modelmesh/model_type_labels_test.go @@ -19,6 +19,8 @@ import ( "sort" "testing" + "k8s.io/apimachinery/pkg/util/sets" + kserveapi "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" "github.com/kserve/kserve/pkg/constants" api "github.com/kserve/modelmesh-serving/apis/serving/v1alpha1" @@ -82,11 +84,11 @@ func TestGetServingRuntimeLabelSets(t *testing.T) { if expectedRtLabel != rtLabel { t.Errorf("Missing expected entry [%s] in set: %v", expectedRtLabel, rtLabel) } - if !reflect.DeepEqual(mtLabelSet.List(), expectedMtLabels) { - t.Errorf("Labels [%s] don't match expected: %v", mtLabelSet.List(), expectedMtLabels) + if !reflect.DeepEqual(sets.List(mtLabelSet), expectedMtLabels) { + t.Errorf("Labels [%s] don't match expected: %v", sets.List(mtLabelSet), expectedMtLabels) } - if !reflect.DeepEqual(pvLabelSet.List(), expectedPvLabels) { - t.Errorf("Labels [%s] don't match expected: %v", pvLabelSet.List(), expectedPvLabels) + if !reflect.DeepEqual(sets.List(pvLabelSet), expectedPvLabels) { + t.Errorf("Labels [%s] don't match expected: %v", sets.List(pvLabelSet), expectedPvLabels) } } diff --git a/controllers/service_controller.go b/controllers/service_controller.go index ea756b03..4514b19d 100644 --- a/controllers/service_controller.go +++ b/controllers/service_controller.go @@ -119,24 +119,40 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct namespace = req.Name n := &corev1.Namespace{} if err := r.Client.Get(ctx, req.NamespacedName, n); err != nil { + // Previously, the controller kept checking namespaces even though the namespaces do not exist anymore. + // As a result, a lot of misleading error messages showed up in the log + if k8serr.IsNotFound(err) { + err = nil + } return ctrl.Result{}, err } if !modelMeshEnabled(n, r.ControllerDeployment.Namespace) { sl := &corev1.ServiceList{} - err := r.List(ctx, sl, client.HasLabels{"modelmesh-service"}, client.InNamespace(namespace)) - if err == nil { + //The logic is + // - If the namespace is not for modelmesh anymore, it will delete modelmesh Service when it exists. + // - If the namespace is being terminated, it does not need to delete the modelmesh Service because it will be gone with the namespace + if err := r.List(ctx, sl, client.HasLabels{"modelmesh-service"}, client.InNamespace(namespace)); err != nil { + return ctrl.Result{}, err + } else { for i := range sl.Items { s := &sl.Items[i] - if err2 := r.Delete(ctx, s); err2 != nil && err == nil { - err = err2 + if err := r.Delete(ctx, s); err != nil { + return ctrl.Result{}, err } } } + if mms := r.MMServices.Get(namespace); mms != nil { mms.Disconnect() r.MMServices.Delete(namespace) + //requeue is never expected here + //If the namespace is not for modelmesh anymore, it should trigger reconcileService for MMService list that manages the goroutines. + if _, err, _ := r.reconcileService(ctx, mms, namespace, owner); err != nil { + return ctrl.Result{}, err + } } - return ctrl.Result{}, err + + return ctrl.Result{}, nil } owner = n } else { @@ -222,10 +238,27 @@ func (r *ServiceReconciler) reconcileService(ctx context.Context, mms *mmesh.MMS return nil, errors.New("unexpected state - MMService uninitialized"), false } + if r.ClusterScope { + namespaceObj := &corev1.Namespace{} + // Get the namespace object to check label and state of the namespace + if err := r.Client.Get(ctx, types.NamespacedName{Name: namespace}, namespaceObj); err != nil { + return nil, err, false + } + // This will remove the goroutine when modelmesh is not enabled for a namespace. + // - when the namespace does not have the annotation modelmesh-enabled + // - when the namespace is under a Terminating state. + if !modelMeshEnabled(namespaceObj, r.ControllerDeployment.Namespace) { + r.ModelEventStream.RemoveWatchedService(serviceName, namespace) + r.Log.V(1).Info("Deleted Watched Service", "name", serviceName, "namespace", namespace) + return nil, nil, false + } + } + sl := &corev1.ServiceList{} if err := r.List(ctx, sl, client.HasLabels{"modelmesh-service"}, client.InNamespace(namespace)); err != nil { return nil, err, false } + var s *corev1.Service for i := range sl.Items { ss := &sl.Items[i] diff --git a/controllers/servingruntime_validator.go b/controllers/servingruntime_validator.go index 9eb8be87..39617a95 100644 --- a/controllers/servingruntime_validator.go +++ b/controllers/servingruntime_validator.go @@ -140,7 +140,7 @@ func validateVolumes(rts *kserveapi.ServingRuntimeSpec, _ *config.Config) error return nil } -func checkName(name string, internalNames sets.String, logStr string) error { +func checkName(name string, internalNames sets.Set[string], logStr string) error { if internalNames.Has(name) { return fmt.Errorf("%s %s is reserved for internal use", logStr, name) } @@ -151,29 +151,29 @@ func checkName(name string, internalNames sets.String, logStr string) error { return nil } -var internalContainerNames = sets.NewString( +var internalContainerNames = sets.New[string]( modelmesh.ModelMeshContainerName, modelmesh.RESTProxyContainerName, modelmesh.PullerContainerName, ) -var internalOnlyVolumeMounts = sets.NewString( +var internalOnlyVolumeMounts = sets.New[string]( modelmesh.ConfigStorageMount, modelmesh.EtcdVolume, modelmesh.InternalConfigMapName, modelmesh.SocketVolume, ) -var internalNamedPorts = sets.NewString("grpc", "http", "prometheus") +var internalNamedPorts = sets.New[string]("grpc", "http", "prometheus") -var internalPorts = sets.NewInt32( +var internalPorts = sets.New[int32]( 8080, // is used for LiteLinks communication in Model Mesh 8085, // is the port the built-in adapter listens on 8089, // is used for Model Mesh probes 8090, // is used for default preStop hooks ) -var internalVolumes = sets.NewString( +var internalVolumes = sets.New[string]( modelmesh.ConfigStorageMount, modelmesh.EtcdVolume, modelmesh.InternalConfigMapName, diff --git a/controllers/util.go b/controllers/util.go index ba18cbd8..e0fdabc4 100644 --- a/controllers/util.go +++ b/controllers/util.go @@ -18,12 +18,17 @@ import ( "context" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) func modelMeshEnabled(n *corev1.Namespace, controllerNamespace string) bool { if v, ok := n.Labels["modelmesh-enabled"]; ok { + // Returns false if the namespace state is terminating even though the namespace have the 'modelmesh-enabled=true' label. + if n.Status.Phase == corev1.NamespaceTerminating { + return false + } return v == "true" } return n.Name == controllerNamespace @@ -38,7 +43,10 @@ func modelMeshEnabled2(ctx context.Context, namespace, controllerNamespace strin } n := &corev1.Namespace{} if err := client.Get(ctx, types.NamespacedName{Name: namespace}, n); err != nil { - return false, err + if errors.IsNotFound(err) { + // If the namespace has already been deleted, it can not be modelmesh namespace + return false, nil + } } return modelMeshEnabled(n, controllerNamespace), nil } diff --git a/fvt/fvtclient.go b/fvt/fvtclient.go index fdee6faa..10b72ed1 100644 --- a/fvt/fvtclient.go +++ b/fvt/fvtclient.go @@ -59,8 +59,8 @@ import ( torchserveapi "github.com/kserve/modelmesh-serving/fvt/generated/torchserve/apis" ) -const PredictorTimeout = time.Second * 120 // absolute time to wait for predictor to become ready -const TimeForStatusToStabilize = time.Second * 5 // time to wait between watcher events before assuming a stable state +const PredictorTimeout = time.Second * 120 // absolute time to wait for predictor to become ready +const TimeForStatusToStabilize = time.Second * 10 // time to wait between watcher events before assuming a stable state type ModelServingConnectionType int diff --git a/fvt/predictor/predictor_suite_test.go b/fvt/predictor/predictor_suite_test.go index e06a4bb2..9dfa76f5 100644 --- a/fvt/predictor/predictor_suite_test.go +++ b/fvt/predictor/predictor_suite_test.go @@ -56,7 +56,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { FVTClientInstance.CreateTLSSecrets() // ensure a stable deploy state - WaitForStableActiveDeployState(time.Second * 30) + WaitForStableActiveDeployState(time.Second * 45) return nil }, func(_ []byte) { diff --git a/fvt/storage/storage_test.go b/fvt/storage/storage_test.go index 2b586fcd..ccb9294f 100644 --- a/fvt/storage/storage_test.go +++ b/fvt/storage/storage_test.go @@ -130,11 +130,21 @@ var _ = Describe("ISVCs", func() { // from the old to the new pod // make a shallow copy of default configmap (don't modify the DefaultConfig reference) - // keeping 1 pod per runtime and don't scale to 0 config := make(map[string]interface{}) for k, v := range DefaultConfig { config[k] = v } + + // scale to 0 for resource-constrained environments (only 2 CPUs on GH actions) + // to stop and remove runtimes which are not used for this test + // Warning FailedScheduling pod/modelmesh-serving-mlserver-1.x-... + // 0/1 nodes are available: 1 Insufficient cpu. preemption: 0/1 nodes are available: + // 1 No preemption victims found for incoming pod. + config["scaleToZero"] = map[string]interface{}{ + "enabled": true, + "gracePeriodSeconds": 5, + } + // update the model-serving-config to allow any PVC config["allowAnyPVC"] = true @@ -194,11 +204,21 @@ var _ = Describe("ISVCs", func() { It("should fail with non-existent PVC", func() { // make a shallow copy of default configmap (don't modify the DefaultConfig reference) - // keeping 1 pod per runtime and don't scale to 0 config := make(map[string]interface{}) for k, v := range DefaultConfig { config[k] = v } + + // scale to 0 for resource-constrained environments (only 2 CPUs on GH actions) + // to stop and remove runtimes which are not used for this test + // Warning FailedScheduling pod/modelmesh-serving-mlserver-1.x-... + // 0/1 nodes are available: 1 Insufficient cpu. preemption: 0/1 nodes are available: + // 1 No preemption victims found for incoming pod. + config["scaleToZero"] = map[string]interface{}{ + "enabled": true, + "gracePeriodSeconds": 5, + } + // update the model-serving-config to allow any PVC config["allowAnyPVC"] = true FVTClientInstance.ApplyUserConfigMap(config) diff --git a/go.mod b/go.mod index 37067fa2..1a8bdd30 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/go-logr/logr v1.2.4 github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.5.9 - github.com/kserve/kserve v0.11.0 + github.com/kserve/kserve v0.11.1 github.com/manifestival/controller-runtime-client v0.4.0 github.com/manifestival/manifestival v0.7.1 github.com/moverest/mnist v0.0.0-20160628192128-ec5d9d203b59 @@ -98,12 +98,12 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 6a6d7491..1cd6e05a 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kserve/kserve v0.11.0 h1:7ZPTjxuRd/qKFTygmYsTZKFCvuzOzjaM83NYAXu8DE4= -github.com/kserve/kserve v0.11.0/go.mod h1:oLeSCIZ6jQqoXYG1HyolzHN1PW0ioCNCWZGnin5yTDg= +github.com/kserve/kserve v0.11.1 h1:3gh2mmCkw2tbzhbN2zKKxtqDjt71V1K2MwpaiXF4KJI= +github.com/kserve/kserve v0.11.1/go.mod h1:qCEKO7gXwWm8sx4LGrKHYjK+SKVWQ35gAEVaE1a0Wug= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -664,8 +664,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -744,8 +744,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -821,15 +821,15 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -842,8 +842,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/scripts/delete.sh b/scripts/delete.sh index a8ebbfa5..ccb8950a 100755 --- a/scripts/delete.sh +++ b/scripts/delete.sh @@ -95,13 +95,7 @@ popd # Older versions of kustomize have different load restrictor flag formats. # Can be removed once Kubeflow installation stops requiring v3.2. -kustomize_version=$(kustomize version --short | grep -o -E "[0-9]\.[0-9]\.[0-9]") -kustomize_load_restrictor_arg="--load-restrictor LoadRestrictionsNone" -if [[ -n "$kustomize_version" && "$kustomize_version" < "3.4.0" ]]; then - kustomize_load_restrictor_arg="--load_restrictor none" -elif [[ -n "$kustomize_version" && "$kustomize_version" < "4.0.1" ]]; then - kustomize_load_restrictor_arg="--load_restrictor LoadRestrictionsNone" -fi +kustomize_load_restrictor_arg=$( kustomize build --help | grep -o -E "\-\-load.restrictor[^,]+" | sed -E "s/(--load.restrictor).+'(.*none)'/\1 \2/I" ) if [[ ! -z $user_ns_array ]]; then kustomize build runtimes ${kustomize_load_restrictor_arg} > runtimes.yaml @@ -122,6 +116,37 @@ if [[ ! -z $user_ns_array ]]; then rm runtimes.yaml fi +# If there is `modelmesh-webhook-server-cert` Certificate object in a namespace, it assumes that cert-manager operator is being used for generating a certificate. +# However, if there is no Certificate object in the namespace, it needs to exclude cert-manager part from kustomization.yaml to generate manifests properly. +export enable_self_signed_ca=true +if kubectl get certificates modelmesh-webhook-server-cert -n $namespace &> /dev/null; then + echo "Cert Manager is installed" + export enable_self_signed_ca=false +fi + +if [[ $enable_self_signed_ca == "true" ]]; then + echo "Enabled Self Signed CA: Update manifest" + if [[ ! -f certmanager/kustomization.yaml.ori ]]; then + cp certmanager/kustomization.yaml certmanager/kustomization.yaml.ori + fi + cd certmanager; kustomize edit remove resource certificate.yaml; cd ../ + + if [[ ! -f default/kustomization.yaml.ori ]]; then + cp default/kustomization.yaml default/kustomization.yaml.ori + fi + cd default; kustomize edit remove resource ../certmanager; cd ../ + + # comment out vars + configMapGeneratorStartLine=$(grep -n configMapGenerator ./default/kustomization.yaml |cut -d':' -f1) + configMapGeneratorBeforeLine=$((configMapGeneratorStartLine-1)) + sed -i.bak "1,${configMapGeneratorBeforeLine}s/^/#/g" default/kustomization.yaml + + # remove webhookcainjection_patch.yaml + sed -i.bak '/webhookcainjection_patch.yaml/d' default/kustomization.yaml + + rm default/kustomization.yaml.bak +fi + # Determine whether a modelmesh-controller-rolebinding clusterrolebinding exists and is # associated with the service account in this namespace. If not, don't delete the cluster level RBAC. set +e @@ -131,9 +156,15 @@ if [[ "$crb_ns" == "$namespace" ]]; then echo "deleting cluster scope RBAC" kustomize build rbac/cluster-scope | kubectl delete -f - --ignore-not-found=true fi + +# Determine whether deployment is namespace-scoped before deleting runtime resources +is_namespace_scoped=$(kubectl exec deploy/modelmesh-controller -- printenv NAMESPACE_SCOPE 2> /dev/null || echo "false") || : kustomize build default | kubectl delete -f - --ignore-not-found=true kustomize build rbac/namespace-scope | kubectl delete -f - --ignore-not-found=true -kustomize build runtimes ${kustomize_load_restrictor_arg} | kubectl delete -f - --ignore-not-found=true +if [[ ! "$is_namespace_scoped" == "true" ]]; then + kustomize build runtimes ${kustomize_load_restrictor_arg} | kubectl delete -f - --ignore-not-found=true +fi + kubectl delete -f dependencies/quickstart.yaml --ignore-not-found=true kubectl delete -f dependencies/fvt.yaml --ignore-not-found=true @@ -141,3 +172,9 @@ kubectl delete -f dependencies/fvt.yaml --ignore-not-found=true if [[ "$namespace" != "$old_namespace" ]]; then kubectl config set-context --current --namespace=${old_namespace} fi + +if [[ $enable_self_signed_ca == "true" ]]; then + cp certmanager/kustomization.yaml.ori certmanager/kustomization.yaml + cp default/kustomization.yaml.ori default/kustomization.yaml + rm certmanager/kustomization.yaml.ori default/kustomization.yaml.ori +fi diff --git a/scripts/install.sh b/scripts/install.sh index b9d1fafe..ecd8a65c 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -351,6 +351,8 @@ if [[ $namespace_scope_mode == "true" ]]; then # Reset crd/kustomization.yaml back to CSR crd since we used the same file for namespace scope mode installation sed -i.bak 's/#- bases\/serving.kserve.io_clusterservingruntimes.yaml/- bases\/serving.kserve.io_clusterservingruntimes.yaml/g' crd/kustomization.yaml rm crd/kustomization.yaml.bak +else + kubectl set env deploy/modelmesh-controller NAMESPACE_SCOPE=false fi if [[ -n $modelmesh_serving_image ]]; then @@ -376,13 +378,7 @@ wait_for_pods_ready "-l control-plane=modelmesh-controller" # Older versions of kustomize have different load restrictor flag formats. # Can be removed once Kubeflow installation stops requiring v3.2. -kustomize_version=$(kustomize version --short | grep -o -E "[0-9]\.[0-9]\.[0-9]") -kustomize_load_restrictor_arg="--load-restrictor LoadRestrictionsNone" -if [[ -n "$kustomize_version" && "$kustomize_version" < "3.4.0" ]]; then - kustomize_load_restrictor_arg="--load_restrictor none" -elif [[ -n "$kustomize_version" && "$kustomize_version" < "4.0.1" ]]; then - kustomize_load_restrictor_arg="--load_restrictor LoadRestrictionsNone" -fi +kustomize_load_restrictor_arg=$( kustomize build --help | grep -o -E "\-\-load.restrictor[^,]+" | sed -E "s/(--load.restrictor).+'(.*none)'/\1 \2/I" ) info "Installing ModelMesh Serving built-in runtimes" if [[ $namespace_scope_mode == "true" ]]; then diff --git a/scripts/setup_user_namespaces.sh b/scripts/setup_user_namespaces.sh index e02cbc77..c3b663fb 100755 --- a/scripts/setup_user_namespaces.sh +++ b/scripts/setup_user_namespaces.sh @@ -72,13 +72,7 @@ if [[ ! -z $user_ns_array ]]; then # Older versions of kustomize have different load restrictor flag formats. # Can be removed once Kubeflow installation stops requiring v3.2. - kustomize_version=$(kustomize version --short | grep -o -E "[0-9]\.[0-9]\.[0-9]") - kustomize_load_restrictor_arg="--load-restrictor LoadRestrictionsNone" - if [[ -n "$kustomize_version" && "$kustomize_version" < "3.4.0" ]]; then - kustomize_load_restrictor_arg="--load_restrictor none" - elif [[ -n "$kustomize_version" && "$kustomize_version" < "4.0.1" ]]; then - kustomize_load_restrictor_arg="--load_restrictor LoadRestrictionsNone" - fi + kustomize_load_restrictor_arg=$( kustomize build --help | grep -o -E "\-\-load.restrictor[^,]+" | sed -E "s/(--load.restrictor).+'(.*none)'/\1 \2/I" ) cp config/dependencies/minio-storage-secret.yaml . kustomize build config/runtimes ${kustomize_load_restrictor_arg} > runtimes.yaml