diff --git a/helm-charts/seldon-core-operator/templates/configmap_seldon-config.yaml b/helm-charts/seldon-core-operator/templates/configmap_seldon-config.yaml index befdf95e76..cf84054041 100644 --- a/helm-charts/seldon-core-operator/templates/configmap_seldon-config.yaml +++ b/helm-charts/seldon-core-operator/templates/configmap_seldon-config.yaml @@ -1,6 +1,7 @@ apiVersion: v1 data: credentials: "{\n \"gcs\" : {\n \"gcsCredentialFileName\": \"gcloud-application-credentials.json\"\n },\n \"s3\" : {\n \"s3AccessKeyIDName\": \"awsAccessKeyID\",\n \"s3SecretAccessKeyName\": \"awsSecretAccessKey\"\n }\n}" + storageInitializer: "{\n \"image\" : \"gcr.io/kfserving/storage-initializer:0.2.1\",\n \"memoryRequest\": \"100Mi\",\n \"memoryLimit\": \"1Gi\",\n \"cpuRequest\": \"100m\",\n \"cpuLimit\": \"1\"\n}" kind: ConfigMap metadata: labels: diff --git a/operator/config/manager/configmap.yaml b/operator/config/manager/configmap.yaml index eddc8f996f..1bec2edc9a 100644 --- a/operator/config/manager/configmap.yaml +++ b/operator/config/manager/configmap.yaml @@ -15,3 +15,11 @@ data: "s3SecretAccessKeyName": "awsSecretAccessKey" } } + storageInitializer: |- + { + "image" : "gcr.io/kfserving/storage-initializer:0.2.1", + "memoryRequest": "100Mi", + "memoryLimit": "1Gi", + "cpuRequest": "100m", + "cpuLimit": "1" + } \ No newline at end of file diff --git a/operator/controllers/model_initializer_injector.go b/operator/controllers/model_initializer_injector.go index a371273e12..8fec1dcb79 100644 --- a/operator/controllers/model_initializer_injector.go +++ b/operator/controllers/model_initializer_injector.go @@ -14,8 +14,7 @@ limitations under the License. package controllers import ( - - // "encoding/json" + "encoding/json" "fmt" "strings" @@ -30,17 +29,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// TODO: change image to seldon +// TODO: change image to seldon? is at least configurable by configmap now (with fixed version there) // TODO: check PVC const ( - DefaultModelLocalMountPath = "/mnt/models" - ModelInitializerContainerImage = "gcr.io/kfserving/model-initializer" - ModelInitializerContainerVersion = "latest" - PvcURIPrefix = "pvc://" - PvcSourceMountName = "kfserving-pvc-source" - PvcSourceMountPath = "/mnt/pvc" - ModelInitializerVolumeSuffix = "provision-location" - ModelInitializerContainerSuffix = "model-initializer" + DefaultModelLocalMountPath = "/mnt/models" + StorageInitializerConfigMapKeyName = "storageInitializer" + ModelInitializerContainerImage = "gcr.io/kfserving/model-initializer" + ModelInitializerContainerVersion = "latest" + PvcURIPrefix = "pvc://" + PvcSourceMountName = "kfserving-pvc-source" + PvcSourceMountPath = "/mnt/pvc" + ModelInitializerVolumeSuffix = "provision-location" + ModelInitializerContainerSuffix = "model-initializer" ) var ( @@ -48,6 +48,14 @@ var ( ControllerConfigMapName = "seldon-config" ) +type StorageInitializerConfig struct { + Image string `json:"image"` + CpuRequest string `json:"cpuRequest"` + CpuLimit string `json:"cpuLimit"` + MemoryRequest string `json:"memoryRequest"` + MemoryLimit string `json:"memoryLimit"` +} + func credentialsBuilder(Client client.Client) (credentialsBuilder *credentials.CredentialBuilder, err error) { clientset := kubernetes.NewForConfigOrDie(ctrl.GetConfigOrDie()) @@ -61,6 +69,39 @@ func credentialsBuilder(Client client.Client) (credentialsBuilder *credentials.C return credentialBuilder, nil } +func getStorageInitializerConfigs(Client client.Client) (*StorageInitializerConfig, error) { + clientset := kubernetes.NewForConfigOrDie(ctrl.GetConfigOrDie()) + configMap, err := clientset.CoreV1().ConfigMaps(ControllerNamespace).Get(ControllerConfigMapName, metav1.GetOptions{}) + if err != nil { + //log.Error(err, "Failed to find config map", "name", ControllerConfigMapName) + return nil, err + } + return getStorageInitializerConfigsFromMap(configMap) +} + +func getStorageInitializerConfigsFromMap(configMap *corev1.ConfigMap) (*StorageInitializerConfig, error) { + storageInitializerConfig := &StorageInitializerConfig{} + if initializerConfig, ok := configMap.Data[StorageInitializerConfigMapKeyName]; ok { + err := json.Unmarshal([]byte(initializerConfig), &storageInitializerConfig) + if err != nil { + panic(fmt.Errorf("Unable to unmarshall %v json string due to %v ", StorageInitializerConfigMapKeyName, err)) + } + } + //Ensure that we set proper values for CPU/Memory Limit/Request + resourceDefaults := []string{storageInitializerConfig.MemoryRequest, + storageInitializerConfig.MemoryLimit, + storageInitializerConfig.CpuRequest, + storageInitializerConfig.CpuLimit} + for _, key := range resourceDefaults { + _, err := resource.ParseQuantity(key) + if err != nil { + return storageInitializerConfig, err + } + } + + return storageInitializerConfig, nil +} + // InjectModelInitializer injects an init container to provision model data func InjectModelInitializer(deployment *appsv1.Deployment, containerName string, srcURI string, serviceAccountName string, envSecretRefName string, Client client.Client) (deploy *appsv1.Deployment, err error) { @@ -150,10 +191,20 @@ func InjectModelInitializer(deployment *appsv1.Deployment, containerName string, } modelInitializerMounts = append(modelInitializerMounts, sharedVolumeWriteMount) + config, err := getStorageInitializerConfigs(Client) + if err != nil { + return nil, err + } + + storageInitializerImage := ModelInitializerContainerImage + ":" + ModelInitializerContainerVersion + if config != nil && config.Image != "" { + storageInitializerImage = config.Image + } + // Add an init container to run provisioning logic to the PodSpec (with defaults to pass comparison later) initContainer := &corev1.Container{ Name: ModelInitializerContainerName, - Image: ModelInitializerContainerImage + ":" + ModelInitializerContainerVersion, + Image: storageInitializerImage, ImagePullPolicy: corev1.PullIfNotPresent, Args: []string{ srcURI, @@ -163,8 +214,14 @@ func InjectModelInitializer(deployment *appsv1.Deployment, containerName string, TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile, Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{"cpu": resource.MustParse("1"), "memory": resource.MustParse("4Gi")}, - Requests: corev1.ResourceList{"cpu": resource.MustParse("0.5"), "memory": resource.MustParse("1Gi")}, + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse(config.CpuLimit), + corev1.ResourceMemory: resource.MustParse(config.MemoryLimit), + }, + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse(config.CpuRequest), + corev1.ResourceMemory: resource.MustParse(config.MemoryRequest), + }, }, }