From 4c8550ed13a65be7c90228fbfceb31f4bf32fe89 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Fri, 12 Jul 2024 21:47:45 +0100 Subject: [PATCH 01/11] feat: Initial database support (#246) * Initial database support - Add status checking - Add better storage flags - Add spec.storage.format validation - Add DDL -Add HIBERNATE format to DB (test) - Update service image - Revert identifier to DATABASE - Update CR options (remove mandatory data) * Remove default DDL generation env var * Update service image to latest tag * Add migration awareness * Add updating pods for migration * Change JDBC url from mysql to mariadb * Fix TLS mount * Revert images * Remove redundant logic * Fix comments --- api/v1alpha1/trustyaiservice_types.go | 39 +- ...styai.opendatahub.io_trustyaiservices.yaml | 12 +- controllers/constants.go | 11 +- controllers/deployment.go | 100 +- controllers/deployment_test.go | 877 ++++++++++++------ controllers/monitor_test.go | 2 +- controllers/route.go | 1 + controllers/route_test.go | 102 +- controllers/secrets.go | 60 ++ controllers/service_accounts_test.go | 4 +- controllers/statuses.go | 44 +- controllers/statuses_test.go | 185 +++- controllers/storage_test.go | 16 +- controllers/suite_test.go | 87 +- .../templates/service/deployment.tmpl.yaml | 73 +- controllers/trustyaiservice_controller.go | 63 +- 16 files changed, 1270 insertions(+), 406 deletions(-) create mode 100644 controllers/secrets.go diff --git a/api/v1alpha1/trustyaiservice_types.go b/api/v1alpha1/trustyaiservice_types.go index 59f2bb7c..7ac65ae6 100644 --- a/api/v1alpha1/trustyaiservice_types.go +++ b/api/v1alpha1/trustyaiservice_types.go @@ -34,14 +34,17 @@ type TrustyAIService struct { } type StorageSpec struct { - Format string `json:"format"` - Folder string `json:"folder"` - Size string `json:"size"` + // Format only supports "PVC" or "DATABASE" values + // +kubebuilder:validation:Enum=PVC;DATABASE + Format string `json:"format"` + Folder string `json:"folder,omitempty"` + Size string `json:"size,omitempty"` + DatabaseConfigurations string `json:"databaseConfigurations,omitempty"` } type DataSpec struct { - Filename string `json:"filename"` - Format string `json:"format"` + Filename string `json:"filename,omitempty"` + Format string `json:"format,omitempty"` } type MetricsSpec struct { @@ -55,7 +58,7 @@ type TrustyAIServiceSpec struct { // +optional Replicas *int32 `json:"replicas"` Storage StorageSpec `json:"storage"` - Data DataSpec `json:"data"` + Data DataSpec `json:"data,omitempty"` Metrics MetricsSpec `json:"metrics"` } @@ -90,6 +93,30 @@ func init() { SchemeBuilder.Register(&TrustyAIService{}, &TrustyAIServiceList{}) } +// IsDatabaseConfigurationsSet returns true if the DatabaseConfigurations field is set. +func (s *StorageSpec) IsDatabaseConfigurationsSet() bool { + return s.DatabaseConfigurations != "" +} + +// IsStoragePVC returns true if the storage is set to PVC. +func (s *StorageSpec) IsStoragePVC() bool { + return s.Format == "PVC" +} + +// IsStorageDatabase returns true if the storage is set to database. +func (s *StorageSpec) IsStorageDatabase() bool { + return s.Format == "DATABASE" +} + +// IsMigration returns true if the migration fields are set. +func (t *TrustyAIService) IsMigration() bool { + if t.Spec.Storage.Format == "DATABASE" && t.Spec.Storage.Folder != "" && t.Spec.Data.Filename != "" { + return true + } else { + return false + } +} + // SetStatus sets the status of the TrustyAIService func (t *TrustyAIService) SetStatus(condType, reason, message string, status corev1.ConditionStatus) { now := metav1.Now() diff --git a/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml b/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml index 56921595..076a8082 100644 --- a/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml +++ b/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml @@ -41,9 +41,6 @@ spec: type: string format: type: string - required: - - filename - - format type: object metrics: properties: @@ -60,19 +57,22 @@ spec: type: integer storage: properties: + databaseConfigurations: + type: string folder: type: string format: + description: Format only supports "PVC" or "DATABASE" values + enum: + - PVC + - DATABASE type: string size: type: string required: - - folder - format - - size type: object required: - - data - metrics - storage type: object diff --git a/controllers/constants.go b/controllers/constants.go index adb130f5..f55da3a2 100644 --- a/controllers/constants.go +++ b/controllers/constants.go @@ -12,7 +12,14 @@ const ( modelMeshLabelKey = "modelmesh-service" modelMeshLabelValue = "modelmesh-serving" volumeMountName = "volume" - defaultRequeueDelay = 10 * time.Second + defaultRequeueDelay = 30 * time.Second + dbCredentialsSuffix = "-db-credentials" +) + +// Allowed storage formats +const ( + STORAGE_PVC = "PVC" + STORAGE_DATABASE = "DATABASE" ) // Configuration constants @@ -56,3 +63,5 @@ const ( EventReasonInferenceServiceConfigured = "InferenceServiceConfigured" EventReasonServiceMonitorCreated = "ServiceMonitorCreated" ) + +const migrationAnnotationKey = "trustyai.opendatahub.io/db-migration" diff --git a/controllers/deployment.go b/controllers/deployment.go index d498a2a6..816c9c17 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -37,13 +37,14 @@ type DeploymentConfig struct { PVCClaimName string CustomCertificatesBundle CustomCertificatesBundle Version string + BatchSize int } // createDeploymentObject returns a Deployment for the TrustyAI Service instance func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, serviceImage string, caBunble CustomCertificatesBundle) (*appsv1.Deployment, error) { var batchSize int - // If not batch size is provided, assume the default one + // If no batch size is provided, assume the default one if instance.Spec.Metrics.BatchSize == nil { batchSize = defaultBatchSize } else { @@ -66,6 +67,7 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, PVCClaimName: pvcName, CustomCertificatesBundle: caBunble, Version: Version, + BatchSize: batchSize, } var deployment *appsv1.Deployment @@ -81,42 +83,78 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, // reconcileDeployment returns a Deployment object with the same name/namespace as the cr func (r *TrustyAIServiceReconciler) createDeployment(ctx context.Context, cr *trustyaiopendatahubiov1alpha1.TrustyAIService, imageName string, caBundle CustomCertificatesBundle) error { - pvcName := generatePVCName(cr) + if !cr.Spec.Storage.IsDatabaseConfigurationsSet() { - pvc := &corev1.PersistentVolumeClaim{} - pvcerr := r.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: cr.Namespace}, pvc) - if pvcerr != nil { - log.FromContext(ctx).Error(pvcerr, "PVC not found") - return pvcerr - } - if pvcerr == nil { - // The PVC is ready. We can now create the Deployment. - deployment, err := r.createDeploymentObject(ctx, cr, imageName, caBundle) - if err != nil { - // Error creating the deployment resource object - return err - } + pvcName := generatePVCName(cr) - if err := ctrl.SetControllerReference(cr, deployment, r.Scheme); err != nil { - log.FromContext(ctx).Error(err, "Error setting TrustyAIService as owner of Deployment.") - return err + pvc := &corev1.PersistentVolumeClaim{} + pvcerr := r.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: cr.Namespace}, pvc) + if pvcerr != nil { + log.FromContext(ctx).Error(pvcerr, "PVC not found") + return pvcerr } - log.FromContext(ctx).Info("Creating Deployment.") - err = r.Create(ctx, deployment) - if err != nil { - log.FromContext(ctx).Error(err, "Error creating Deployment.") - return err + } + + // We can now create the Deployment. + deployment, err := r.createDeploymentObject(ctx, cr, imageName, caBundle) + if err != nil { + // Error creating the deployment resource object + return err + } + + if err := ctrl.SetControllerReference(cr, deployment, r.Scheme); err != nil { + log.FromContext(ctx).Error(err, "Error setting TrustyAIService as owner of Deployment.") + return err + } + log.FromContext(ctx).Info("Creating Deployment.") + err = r.Create(ctx, deployment) + if err != nil { + log.FromContext(ctx).Error(err, "Error creating Deployment.") + return err + } + // Created successfully + return nil + +} + +// updateDeployment returns a Deployment object with the same name/namespace as the cr +func (r *TrustyAIServiceReconciler) updateDeployment(ctx context.Context, cr *trustyaiopendatahubiov1alpha1.TrustyAIService, imageName string, caBundle CustomCertificatesBundle) error { + + if !cr.Spec.Storage.IsDatabaseConfigurationsSet() { + + pvcName := generatePVCName(cr) + + pvc := &corev1.PersistentVolumeClaim{} + pvcerr := r.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: cr.Namespace}, pvc) + if pvcerr != nil { + log.FromContext(ctx).Error(pvcerr, "PVC not found") + return pvcerr } - // Created successfully - return nil + } - } else { - return ErrPVCNotReady + // We can now create the Deployment object. + deployment, err := r.createDeploymentObject(ctx, cr, imageName, caBundle) + if err != nil { + // Error creating the deployment resource object + return err + } + + if err := ctrl.SetControllerReference(cr, deployment, r.Scheme); err != nil { + log.FromContext(ctx).Error(err, "Error setting TrustyAIService as owner of Deployment.") + return err + } + log.FromContext(ctx).Info("Updating Deployment.") + err = r.Update(ctx, deployment) + if err != nil { + log.FromContext(ctx).Error(err, "Error updating Deployment.") + return err } + // Created successfully + return nil } -func (r *TrustyAIServiceReconciler) ensureDeployment(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, caBundle CustomCertificatesBundle) error { +func (r *TrustyAIServiceReconciler) ensureDeployment(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, caBundle CustomCertificatesBundle, migration bool) error { // Get image and tag from ConfigMap // If there's a ConfigMap with custom images, it is only applied when the operator is first deployed @@ -138,6 +176,12 @@ func (r *TrustyAIServiceReconciler) ensureDeployment(ctx context.Context, instan // Some other error occurred when trying to get the Deployment return err } + // Deployment exists, but we are migrating + if migration { + log.FromContext(ctx).Info("Found migration annotation. Migrating.") + return r.updateDeployment(ctx, instance, image, caBundle) + } + // Deployment is ready and using the PVC return nil } diff --git a/controllers/deployment_test.go b/controllers/deployment_test.go index b513edeb..de80a1c4 100644 --- a/controllers/deployment_test.go +++ b/controllers/deployment_test.go @@ -27,6 +27,297 @@ func printKubeObject(obj interface{}) { } } +func setupAndTestDeploymentDefault(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) + + deployment := &appsv1.Deployment{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, deployment) + Expect(err).ToNot(HaveOccurred()) + Expect(deployment).ToNot(BeNil()) + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + + Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("quay.io/trustyai/trustyai-service:latest")) + Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal("registry.redhat.io/openshift4/ose-oauth-proxy:latest")) + + WaitFor(func() error { + service, _ := reconciler.reconcileService(ctx, instance) + return reconciler.Create(ctx, service) + }, "failed to create service") + + service := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) + }, "failed to get Service") + + Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) + Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) + Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) + Expect(service.Namespace).Should(Equal(namespace)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance, caBundle) + return err + }, "failed to create oauth service") + + desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) + Expect(err).ToNot(HaveOccurred()) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + + // Check if the OAuth service has the expected labels + Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + +} + +func setupAndTestDeploymentConfigMap(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + serviceImage := "custom-service-image:foo" + oauthImage := "custom-oauth-proxy:bar" + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + + WaitFor(func() error { + configMap := createConfigMap(operatorNamespace, oauthImage, serviceImage) + return k8sClient.Create(ctx, configMap) + }, "failed to create ConfigMap") + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to reconcile deployment") + + deployment := &appsv1.Deployment{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: namespace}, deployment) + }, "failed to get updated deployment") + Expect(deployment).ToNot(BeNil()) + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + + Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(serviceImage)) + Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal(oauthImage)) + + WaitFor(func() error { + service, _ := reconciler.reconcileService(ctx, instance) + return reconciler.Create(ctx, service) + }, "failed to create service") + + service := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) + }, "failed to get Service") + + Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) + Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) + Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) + Expect(service.Namespace).Should(Equal(namespace)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance, caBundle) + return err + }, "failed to create oauth service") + + desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) + Expect(err).ToNot(HaveOccurred()) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + + // Check if the OAuth service has the expected labels + Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + +} + +func setupAndTestDeploymentNoCustomCABundle(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) + + customCertificatesBundleVolumeName := caBundle + for _, volume := range deployment.Spec.Template.Spec.Volumes { + Expect(volume.Name).ToNot(Equal(customCertificatesBundleVolumeName)) + } + + for _, container := range deployment.Spec.Template.Spec.Containers { + for _, volumeMount := range container.VolumeMounts { + Expect(volumeMount.Name).ToNot(Equal(customCertificatesBundleVolumeName)) + } + for _, arg := range container.Args { + Expect(arg).ToNot(ContainSubstring("--openshift-ca")) + } + } + +} + +func setupAndTestDeploymentCustomCABundle(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + caBundleConfigMap := createTrustedCABundleConfigMap(namespace) + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + Expect(k8sClient.Create(ctx, caBundleConfigMap)).To(Succeed()) + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) + + foundCustomCertificatesBundleVolumeMount := false + + customCertificatesBundleMountPath := "/etc/ssl/certs/ca-bundle.crt" + for _, container := range deployment.Spec.Template.Spec.Containers { + for _, volumeMount := range container.VolumeMounts { + if volumeMount.Name == caBundleName && volumeMount.MountPath == customCertificatesBundleMountPath { + foundCustomCertificatesBundleVolumeMount = true + } + } + } + Expect(foundCustomCertificatesBundleVolumeMount).To(BeTrue(), caBundleName+" volume mount not found in any container") + + Expect(k8sClient.Delete(ctx, caBundleConfigMap)).To(Succeed(), "failed to delete custom certificates bundle ConfigMap") + +} + +func setupAndTestDeploymentServiceAccount(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string, mode string) { + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + if mode == "PVC" { + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + } + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) +} + +func setupAndTestDeploymentInferenceService(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string, mode string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + WaitFor(func() error { + return createTestPVC(ctx, k8sClient, instance) + }, "failed to create PVC") + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + // Creating the InferenceService + inferenceService := createInferenceService("my-model", namespace) + WaitFor(func() error { + return k8sClient.Create(ctx, inferenceService) + }, "failed to create deployment") + + Expect(reconciler.patchKServe(ctx, instance, *inferenceService, namespace, instance.Name, false)).ToNot(HaveOccurred()) + + deployment := &appsv1.Deployment{} + WaitFor(func() error { + // Define defaultServiceName for the deployment created by the operator + namespacedNamed := types.NamespacedName{ + Namespace: namespace, + Name: instance.Name, + } + return k8sClient.Get(ctx, namespacedNamed, deployment) + }, "failed to get Deployment") + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance, caBundle) + return err + }, "failed to create oauth service") + + desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) + Expect(err).ToNot(HaveOccurred()) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + + // Check if the OAuth service has the expected labels + Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + +} + var _ = Describe("TrustyAI operator", func() { BeforeEach(func() { @@ -59,173 +350,153 @@ var _ = Describe("TrustyAI operator", func() { Context("When deploying with default settings without an InferenceService", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + It("Creates a deployment and a service with the default configuration in PVC-mode", func() { + namespace := "trusty-ns-a-1-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentDefault(instance, namespace) + }) + It("Creates a deployment and a service with the default configuration in DB-mode (mysql)", func() { + namespace := "trusty-ns-a-1-db" + instance = createDefaultDBCustomResource(namespace) + WaitFor(func() error { + secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mysql") + return k8sClient.Create(ctx, secret) + }, "failed to create ConfigMap") + setupAndTestDeploymentDefault(instance, namespace) + }) + It("Creates a deployment and a service with the default configuration in DB-mode (mariadb)", func() { + namespace := "trusty-ns-a-1-db" + instance = createDefaultDBCustomResource(namespace) + WaitFor(func() error { + secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mariadb") + return k8sClient.Create(ctx, secret) + }, "failed to create ConfigMap") + setupAndTestDeploymentDefault(instance, namespace) + }) + + }) - It("Creates a deployment and a service with the default configuration", func() { + Context("When deploying with a ConfigMap and without an InferenceService", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + + It("Creates a deployment and a service with the ConfigMap configuration in PVC-mode", func() { + namespace := "trusty-ns-a-1-cm-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentConfigMap(instance, namespace) + }) + It("Creates a deployment and a service with the ConfigMap configuration in DB-mode", func() { + namespace := "trusty-ns-a-1-cm-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentConfigMap(instance, namespace) + }) + + }) - namespace := "trusty-ns-a-1" + Context("When deploying with default settings without an InferenceService", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + + It("should set environment variables correctly in PVC mode", func() { + + namespace := "trusty-ns-a-4-pvc" + instance = createDefaultPVCCustomResource(namespace) + //printKubeObject(instance) Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - instance = createDefaultCR(namespace) caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - Expect(reconciler.ensureDeployment(ctx, instance, caBundle)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) deployment := &appsv1.Deployment{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, deployment) - Expect(err).ToNot(HaveOccurred()) - Expect(deployment).ToNot(BeNil()) - - Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) - Expect(deployment.Namespace).Should(Equal(namespace)) - Expect(deployment.Name).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - - Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) - Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("quay.io/trustyai/trustyai-service:latest")) - Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal("registry.redhat.io/openshift4/ose-oauth-proxy:latest")) - - WaitFor(func() error { - service, _ := reconciler.reconcileService(ctx, instance) - return reconciler.Create(ctx, service) - }, "failed to create service") - - service := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) - }, "failed to get Service") - - Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) - Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) - Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) - Expect(service.Namespace).Should(Equal(namespace)) - - WaitFor(func() error { - err := reconciler.reconcileOAuthService(ctx, instance, caBundle) - return err - }, "failed to create oauth service") + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) - desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) - Expect(err).ToNot(HaveOccurred()) + //printKubeObject(deployment) - oauthService := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) - }, "failed to get OAuth Service") + foundEnvVar := func(envVars []corev1.EnvVar, name string) *corev1.EnvVar { + for _, env := range envVars { + if env.Name == name { + return &env + } + } + return nil + } - // Check if the OAuth service has the expected labels - Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + var trustyaiServiceContainer *corev1.Container + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == "trustyai-service" { + trustyaiServiceContainer = &container + break + } + } - }) - }) + Expect(trustyaiServiceContainer).NotTo(BeNil(), "trustyai-service container not found") - Context("When deploying with a ConfigMap and without an InferenceService", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + // Checking the environment variables of the trustyai-service container + var envVar *corev1.EnvVar - It("Creates a deployment and a service with the ConfigMap configuration", func() { + //envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_BATCH_SIZE") + //Expect(envVar).NotTo(BeNil(), "Env var SERVICE_BATCH_SIZE not found") + //Expect(envVar.Value).To(Equal("5000")) - namespace := "trusty-ns-a-1-cm" - serviceImage := "custom-service-image:foo" - oauthImage := "custom-oauth-proxy:bar" - Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FILENAME") + Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FILENAME not found") + Expect(envVar.Value).To(Equal("data.csv")) - WaitFor(func() error { - configMap := createConfigMap(operatorNamespace, oauthImage, serviceImage) - return k8sClient.Create(ctx, configMap) - }, "failed to create ConfigMap") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_STORAGE_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_STORAGE_FORMAT not found") + Expect(envVar.Value).To(Equal("PVC")) - instance = createDefaultCR(namespace) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FOLDER") + Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FOLDER not found") + Expect(envVar.Value).To(Equal("/data")) - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_DATA_FORMAT not found") + Expect(envVar.Value).To(Equal("CSV")) - Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) - Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to reconcile deployment") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_METRICS_SCHEDULE") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_METRICS_SCHEDULE not found") + Expect(envVar.Value).To(Equal("5s")) - deployment := &appsv1.Deployment{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: namespace}, deployment) - }, "failed to get updated deployment") - Expect(deployment).ToNot(BeNil()) - - Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) - Expect(deployment.Namespace).Should(Equal(namespace)) - Expect(deployment.Name).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - - Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) - Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(serviceImage)) - Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal(oauthImage)) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_HIBERNATE_ORM_ACTIVE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_HIBERNATE_ORM_ACTIVE not found") + Expect(envVar.Value).To(Equal("false")) - WaitFor(func() error { - service, _ := reconciler.reconcileService(ctx, instance) - return reconciler.Create(ctx, service) - }, "failed to create service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_DB_KIND") + Expect(envVar).To(BeNil()) - service := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) - }, "failed to get Service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_MAX_SIZE") + Expect(envVar).To(BeNil()) - Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) - Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) - Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) - Expect(service.Namespace).Should(Equal(namespace)) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_USERNAME") + Expect(envVar).To(BeNil()) - WaitFor(func() error { - err := reconciler.reconcileOAuthService(ctx, instance, caBundle) - return err - }, "failed to create oauth service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_PASSWORD") + Expect(envVar).To(BeNil()) - desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) - Expect(err).ToNot(HaveOccurred()) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_SERVICE") + Expect(envVar).To(BeNil()) - oauthService := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) - }, "failed to get OAuth Service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_PORT") + Expect(envVar).To(BeNil()) - // Check if the OAuth service has the expected labels - Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") + Expect(envVar).To(BeNil()) }) - }) - Context("When deploying with default settings without an InferenceService", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - - It("should set environment variables correctly", func() { + It("should set environment variables correctly in DB mode", func() { - namespace := "trusty-ns-a-4" - instance = createDefaultCR(namespace) + namespace := "trusty-ns-a-4-db" + instance = createDefaultDBCustomResource(namespace) Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - Expect(reconciler.ensureDeployment(ctx, instance, caBundle)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) deployment := &appsv1.Deployment{} namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} @@ -253,135 +524,255 @@ var _ = Describe("TrustyAI operator", func() { // Checking the environment variables of the trustyai-service container var envVar *corev1.EnvVar - envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_BATCH_SIZE") - Expect(envVar).NotTo(BeNil(), "Env var SERVICE_BATCH_SIZE not found") - Expect(envVar.Value).To(Equal("5000")) - envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FILENAME") - Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FILENAME not found") - Expect(envVar.Value).To(Equal("data.csv")) + Expect(envVar).To(BeNil()) envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_STORAGE_FORMAT") Expect(envVar).NotTo(BeNil(), "Env var SERVICE_STORAGE_FORMAT not found") - Expect(envVar.Value).To(Equal("PVC")) + Expect(envVar.Value).To(Equal(STORAGE_DATABASE)) envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FOLDER") - Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FOLDER not found") - Expect(envVar.Value).To(Equal("/data")) + Expect(envVar).To(BeNil()) - envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") - Expect(envVar).NotTo(BeNil(), "Env var SERVICE_DATA_FORMAT not found") - Expect(envVar.Value).To(Equal("CSV")) + //envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") + //Expect(envVar).To(BeNil()) envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_METRICS_SCHEDULE") Expect(envVar).NotTo(BeNil(), "Env var SERVICE_METRICS_SCHEDULE not found") Expect(envVar.Value).To(Equal("5s")) - }) - }) - Context("When deploying with no custom CA bundle ConfigMap", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_HIBERNATE_ORM_ACTIVE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_HIBERNATE_ORM_ACTIVE not found") + Expect(envVar.Value).To(Equal("true")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_DB_KIND") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseKind"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_MAX_SIZE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_MAX_SIZE not found") + Expect(envVar.Value).To(Equal("16")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_USERNAME") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseUsername"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_PASSWORD") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePassword"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_SERVICE") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_SERVICE not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_SERVICE does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_SERVICE is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseService"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_PORT") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_PORT not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_PORT does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_PORT is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePort"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_URL not found") + Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database")) + + }) - It("should use the correct service account and not include CustomCertificatesBundle", func() { + It("should set environment variables correctly in migration mode", func() { - namespace := "trusty-ns-a-7" - instance = createDefaultCR(namespace) + namespace := "trusty-ns-a-4-migration" + instance = createDefaultMigrationCustomResource(namespace) Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - + //printKubeObject(instance) caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) deployment := &appsv1.Deployment{} namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) - Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) - - customCertificatesBundleVolumeName := caBundle - for _, volume := range deployment.Spec.Template.Spec.Volumes { - Expect(volume.Name).ToNot(Equal(customCertificatesBundleVolumeName)) + foundEnvVar := func(envVars []corev1.EnvVar, name string) *corev1.EnvVar { + for _, env := range envVars { + if env.Name == name { + return &env + } + } + return nil } + var trustyaiServiceContainer *corev1.Container for _, container := range deployment.Spec.Template.Spec.Containers { - for _, volumeMount := range container.VolumeMounts { - Expect(volumeMount.Name).ToNot(Equal(customCertificatesBundleVolumeName)) - } - for _, arg := range container.Args { - Expect(arg).ToNot(ContainSubstring("--openshift-ca")) + if container.Name == "trustyai-service" { + trustyaiServiceContainer = &container + break } } + + Expect(trustyaiServiceContainer).NotTo(BeNil(), "trustyai-service container not found") + + // Checking the environment variables of the trustyai-service container + var envVar *corev1.EnvVar + + //envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_BATCH_SIZE") + //Expect(envVar).To(BeNil(), "Env var SERVICE_BATCH_SIZE not found") + //Expect(envVar.Value).To(Equal("5000")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FILENAME") + Expect(envVar).ToNot(BeNil()) + Expect(envVar.Value).To(Equal("data.csv")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_STORAGE_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_STORAGE_FORMAT not found") + Expect(envVar.Value).To(Equal(STORAGE_DATABASE)) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FOLDER") + Expect(envVar).ToNot(BeNil()) + Expect(envVar.Value).To(Equal("/data")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") + Expect(envVar).ToNot(BeNil()) + Expect(envVar.Value).To(Equal("CSV")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_METRICS_SCHEDULE") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_METRICS_SCHEDULE not found") + Expect(envVar.Value).To(Equal("5s")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_HIBERNATE_ORM_ACTIVE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_HIBERNATE_ORM_ACTIVE not found") + Expect(envVar.Value).To(Equal("true")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_DB_KIND") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseKind"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_MAX_SIZE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_MAX_SIZE not found") + Expect(envVar.Value).To(Equal("16")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_USERNAME") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseUsername"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_PASSWORD") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePassword"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_SERVICE") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_SERVICE not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_SERVICE does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_SERVICE is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseService"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_PORT") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_PORT not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_PORT does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_PORT is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePort"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_URL not found") + Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database")) + }) + }) - Context("When deploying with a custom CA bundle ConfigMap", func() { + Context("When deploying with no custom CA bundle ConfigMap", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("should use the correct service account and include CustomCertificatesBundle", func() { + It("should use the correct service account and not include CustomCertificatesBundle in PVC-mode", func() { - namespace := "trusty-ns-a-8" - instance = createDefaultCR(namespace) - caBundleConfigMap := createTrustedCABundleConfigMap(namespace) - Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - Expect(k8sClient.Create(ctx, caBundleConfigMap)).To(Succeed()) + namespace := "trusty-ns-a-7-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentNoCustomCABundle(instance, namespace) + }) + It("should use the correct service account and not include CustomCertificatesBundle in DB-mode", func() { - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + namespace := "trusty-ns-a-7-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentNoCustomCABundle(instance, namespace) + }) + It("should use the correct service account and not include CustomCertificatesBundle in migration-mode", func() { - Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) - Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") + namespace := "trusty-ns-a-7-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentNoCustomCABundle(instance, namespace) + }) - deployment := &appsv1.Deployment{} - namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} - Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + }) - Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) + Context("When deploying with a custom CA bundle ConfigMap", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - foundCustomCertificatesBundleVolumeMount := false + It("should use the correct service account and include CustomCertificatesBundle in PVC-mode", func() { - customCertificatesBundleMountPath := "/etc/ssl/certs/ca-bundle.crt" - for _, container := range deployment.Spec.Template.Spec.Containers { - for _, volumeMount := range container.VolumeMounts { - if volumeMount.Name == caBundleName && volumeMount.MountPath == customCertificatesBundleMountPath { - foundCustomCertificatesBundleVolumeMount = true - } - } - } - Expect(foundCustomCertificatesBundleVolumeMount).To(BeTrue(), caBundleName+" volume mount not found in any container") + namespace := "trusty-ns-a-8-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentCustomCABundle(instance, namespace) + }) + It("should use the correct service account and include CustomCertificatesBundle in DB-mode", func() { - Expect(k8sClient.Delete(ctx, caBundleConfigMap)).To(Succeed(), "failed to delete custom certificates bundle ConfigMap") + namespace := "trusty-ns-a-8-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentCustomCABundle(instance, namespace) + }) + It("should use the correct service account and include CustomCertificatesBundle in migration-mode", func() { + namespace := "trusty-ns-a-8-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentCustomCABundle(instance, namespace) }) }) Context("When deploying with default settings without an InferenceService", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("should use the correct service account", func() { + It("should use the correct service account in PVC-mode", func() { - namespace := "trusty-ns-a-6" - instance = createDefaultCR(namespace) - Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + namespace := "trusty-ns-a-6-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentServiceAccount(instance, namespace, "PVC") - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + }) + It("should use the correct service account in DB-mode", func() { - Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) - Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") + namespace := "trusty-ns-a-6-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentServiceAccount(instance, namespace, "DATABASE") - deployment := &appsv1.Deployment{} - namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} - Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + }) + It("should use the correct service account in migration-mode", func() { + + namespace := "trusty-ns-a-6-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentServiceAccount(instance, namespace, "DATABASE") - Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) }) }) @@ -402,72 +793,28 @@ var _ = Describe("TrustyAI operator", func() { Context("When deploying with an associated InferenceService", func() { - It("Sets up the InferenceService and links it to the TrustyAIService deployment", func() { + It("Sets up the InferenceService and links it to the TrustyAIService deployment in PVC-mode", func() { - namespace := "trusty-ns-2" - instance := createDefaultCR(namespace) - WaitFor(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, "failed to create namespace") + namespace := "trusty-ns-2-pvc" + instance := createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentInferenceService(instance, namespace, "PVC") - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) - - WaitFor(func() error { - return createTestPVC(ctx, k8sClient, instance) - }, "failed to create PVC") - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") - - // Creating the InferenceService - inferenceService := createInferenceService("my-model", namespace) - WaitFor(func() error { - return k8sClient.Create(ctx, inferenceService) - }, "failed to create deployment") - - Expect(reconciler.patchKServe(ctx, instance, *inferenceService, namespace, instance.Name, false)).ToNot(HaveOccurred()) - - deployment := &appsv1.Deployment{} - WaitFor(func() error { - // Define defaultServiceName for the deployment created by the operator - namespacedNamed := types.NamespacedName{ - Namespace: namespace, - Name: instance.Name, - } - return k8sClient.Get(ctx, namespacedNamed, deployment) - }, "failed to get Deployment") - - Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) - Expect(deployment.Namespace).Should(Equal(namespace)) - Expect(deployment.Name).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - - WaitFor(func() error { - err := reconciler.reconcileOAuthService(ctx, instance, caBundle) - return err - }, "failed to create oauth service") + }) + It("Sets up the InferenceService and links it to the TrustyAIService deployment in DB-mode", func() { - desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) - Expect(err).ToNot(HaveOccurred()) + namespace := "trusty-ns-2-db" + instance := createDefaultDBCustomResource(namespace) + setupAndTestDeploymentInferenceService(instance, namespace, "DATABASE") - oauthService := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) - }, "failed to get OAuth Service") + }) + It("Sets up the InferenceService and links it to the TrustyAIService deployment in migration-mode", func() { - // Check if the OAuth service has the expected labels - Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + namespace := "trusty-ns-2-migration" + instance := createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentInferenceService(instance, namespace, "DATABASE") }) + }) }) @@ -493,7 +840,7 @@ var _ = Describe("TrustyAI operator", func() { It("Deploys services with defaults in each specified namespace", func() { for i, namespace := range namespaces { - instances[i] = createDefaultCR(namespace) + instances[i] = createDefaultPVCCustomResource(namespace) instances[i].Namespace = namespace WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) @@ -507,7 +854,7 @@ var _ = Describe("TrustyAI operator", func() { return createTestPVC(ctx, k8sClient, instance) }, "failed to create PVC") WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) + return reconciler.ensureDeployment(ctx, instance, caBundle, false) }, "failed to create deployment") //Expect(k8sClient.Create(ctx, instance)).Should(Succeed()) deployment := &appsv1.Deployment{} diff --git a/controllers/monitor_test.go b/controllers/monitor_test.go index 41cd8fd7..ce9bc9c9 100644 --- a/controllers/monitor_test.go +++ b/controllers/monitor_test.go @@ -60,7 +60,7 @@ var _ = Describe("Service Monitor Reconciliation", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("Should have correct values", func() { namespace := "sm-test-namespace-1" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) diff --git a/controllers/route.go b/controllers/route.go index 6d5963e5..3016a321 100644 --- a/controllers/route.go +++ b/controllers/route.go @@ -100,6 +100,7 @@ func (r *TrustyAIServiceReconciler) checkRouteReady(ctx context.Context, cr *tru err := r.Client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, existingRoute) if err != nil { + log.FromContext(ctx).Info("Unable to find the Route") if errors.IsNotFound(err) { return false, nil } diff --git a/controllers/route_test.go b/controllers/route_test.go index b84f237b..da35a689 100644 --- a/controllers/route_test.go +++ b/controllers/route_test.go @@ -11,6 +11,42 @@ import ( "k8s.io/client-go/tools/record" ) +func setupAndTestRouteCreation(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + err := reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + Expect(err).ToNot(HaveOccurred()) + + route := &routev1.Route{} + err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) + Expect(err).ToNot(HaveOccurred()) + Expect(route).ToNot(BeNil()) + Expect(route.Spec.To.Name).To(Equal(instance.Name + "-tls")) + +} + +func setupAndTestSameRouteCreation(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + // Create a Route with the expected spec + existingRoute, err := reconciler.createRouteObject(ctx, instance) + Expect(err).ToNot(HaveOccurred()) + Expect(reconciler.Client.Create(ctx, existingRoute)).To(Succeed()) + + err = reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + Expect(err).ToNot(HaveOccurred()) + + // Fetch the Route + route := &routev1.Route{} + err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) + Expect(err).ToNot(HaveOccurred()) + Expect(route.Spec).To(Equal(existingRoute.Spec)) +} + var _ = Describe("Route Reconciliation", func() { BeforeEach(func() { @@ -25,47 +61,41 @@ var _ = Describe("Route Reconciliation", func() { Context("When Route does not exist", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should create Route successfully", func() { - namespace := "route-test-namespace-1" - instance = createDefaultCR(namespace) - - WaitFor(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, "failed to create namespace") - - err := reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) - Expect(err).ToNot(HaveOccurred()) - - route := &routev1.Route{} - err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) - Expect(err).ToNot(HaveOccurred()) - Expect(route).ToNot(BeNil()) - Expect(route.Spec.To.Name).To(Equal(instance.Name + "-tls")) + It("Should create Route successfully in PVC-mode", func() { + namespace := "route-test-namespace-1-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestRouteCreation(instance, namespace) + }) + It("Should create Route successfully in DB-mode", func() { + namespace := "route-test-namespace-1-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestRouteCreation(instance, namespace) + }) + It("Should create Route successfully in migration-mode", func() { + namespace := "route-test-namespace-1-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestRouteCreation(instance, namespace) }) + }) Context("When Route exists and is the same", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should not update Route", func() { - namespace := "route-test-namespace-2" - instance = createDefaultCR(namespace) - WaitFor(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, "failed to create namespace") - - // Create a Route with the expected spec - existingRoute, err := reconciler.createRouteObject(ctx, instance) - Expect(err).ToNot(HaveOccurred()) - Expect(reconciler.Client.Create(ctx, existingRoute)).To(Succeed()) - - err = reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) - Expect(err).ToNot(HaveOccurred()) - - // Fetch the Route - route := &routev1.Route{} - err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) - Expect(err).ToNot(HaveOccurred()) - Expect(route.Spec).To(Equal(existingRoute.Spec)) + It("Should not update Route in PVC-mode", func() { + namespace := "route-test-namespace-2-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestSameRouteCreation(instance, namespace) + }) + It("Should not update Route in DB-mode", func() { + namespace := "route-test-namespace-2-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestSameRouteCreation(instance, namespace) }) + It("Should not update Route in migration-mode", func() { + namespace := "route-test-namespace-2-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestSameRouteCreation(instance, namespace) + }) + }) }) diff --git a/controllers/secrets.go b/controllers/secrets.go new file mode 100644 index 00000000..6e2d49b7 --- /dev/null +++ b/controllers/secrets.go @@ -0,0 +1,60 @@ +package controllers + +import ( + "context" + "fmt" + trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// findDatabaseSecret finds the DB configuration secret named (specified or default) in the same namespace as the CR +func (r *TrustyAIServiceReconciler) findDatabaseSecret(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) (*corev1.Secret, error) { + + databaseConfigurationsName := instance.Spec.Storage.DatabaseConfigurations + defaultDatabaseConfigurationsName := instance.Name + dbCredentialsSuffix + + secret := &corev1.Secret{} + + if databaseConfigurationsName != "" { + secret := &corev1.Secret{} + err := r.Get(ctx, client.ObjectKey{Name: databaseConfigurationsName, Namespace: instance.Namespace}, secret) + if err == nil { + return secret, nil + } + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", databaseConfigurationsName, instance.Namespace, err) + } + } else { + // If specified not found, try the default + + err := r.Get(ctx, client.ObjectKey{Name: defaultDatabaseConfigurationsName, Namespace: instance.Namespace}, secret) + if err == nil { + return secret, nil + } + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", defaultDatabaseConfigurationsName, instance.Namespace, err) + } + + } + + return nil, fmt.Errorf("neither secret %s nor %s found in namespace %s", databaseConfigurationsName, defaultDatabaseConfigurationsName, instance.Namespace) +} + +// validateDatabaseSecret validates the DB configuration secret +func (r *TrustyAIServiceReconciler) validateDatabaseSecret(secret *corev1.Secret) error { + + mandatoryKeys := []string{"databaseKind", "databaseUsername", "databasePassword", "databaseService", "databasePort"} + + for _, key := range mandatoryKeys { + value, exists := secret.Data[key] + if !exists { + return fmt.Errorf("mandatory key %s is missing from database configuration", key) + } + if len(value) == 0 { + return fmt.Errorf("mandatory key %s is empty on database configuration", key) + } + } + return nil +} diff --git a/controllers/service_accounts_test.go b/controllers/service_accounts_test.go index 4b64837d..17b9d660 100644 --- a/controllers/service_accounts_test.go +++ b/controllers/service_accounts_test.go @@ -26,7 +26,7 @@ var _ = Describe("Service Accounts", func() { It("Should create SAs, CRBs successfully", func() { namespace1 := "sa-test-namespace-1" - instance1 := createDefaultCR(namespace1) + instance1 := createDefaultPVCCustomResource(namespace1) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace1) @@ -36,7 +36,7 @@ var _ = Describe("Service Accounts", func() { namespace2 := "sa-test-namespace-2" - instance2 := createDefaultCR(namespace2) + instance2 := createDefaultPVCCustomResource(namespace2) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace2) diff --git a/controllers/statuses.go b/controllers/statuses.go index b54036ac..b88ba47c 100644 --- a/controllers/statuses.go +++ b/controllers/statuses.go @@ -2,6 +2,7 @@ package controllers import ( "context" + trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" v1 "k8s.io/api/core/v1" "k8s.io/client-go/util/retry" @@ -10,12 +11,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -// IsAllReady checks if all the necessary readiness fields are true. -func (rs *AvailabilityStatus) IsAllReady() bool { - return rs.PVCReady && rs.DeploymentReady && rs.RouteReady +// IsAllReady checks if all the necessary readiness fields are true for the specific mode +func (rs *AvailabilityStatus) IsAllReady(mode string) bool { + return (rs.PVCReady && rs.DeploymentReady && rs.RouteReady && mode == STORAGE_PVC) || (rs.DeploymentReady && rs.RouteReady && mode == STORAGE_DATABASE) } -// AvailabilityStatus holds the readiness status of various resources. +// AvailabilityStatus has the readiness status of various resources. type AvailabilityStatus struct { PVCReady bool DeploymentReady bool @@ -44,37 +45,39 @@ func (r *TrustyAIServiceReconciler) updateStatus(ctx context.Context, original * return saved, err } -// reconcileStatuses checks the readiness status of PVC, Deployment, Route and Inference Services +// reconcileStatuses checks the readiness status of required resources func (r *TrustyAIServiceReconciler) reconcileStatuses(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) (ctrl.Result, error) { var err error status := AvailabilityStatus{} - // Check for PVC readiness - status.PVCReady, err = r.checkPVCReady(ctx, instance) - if err != nil || !status.PVCReady { - // PVC not ready, requeue - return Requeue() + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + // Check for PVC readiness + status.PVCReady, err = r.checkPVCReady(ctx, instance) + if err != nil || !status.PVCReady { + // PVC not ready, requeue + return RequeueWithDelayMessage(ctx, defaultRequeueDelay, "PVC not ready") + } } // Check for deployment readiness status.DeploymentReady, err = r.checkDeploymentReady(ctx, instance) if err != nil || !status.DeploymentReady { // Deployment not ready, requeue - return Requeue() + return RequeueWithDelayMessage(ctx, defaultRequeueDelay, "Deployment not ready") } // Check for route readiness status.RouteReady, err = r.checkRouteReady(ctx, instance) if err != nil || !status.RouteReady { // Route not ready, requeue - return Requeue() + return RequeueWithDelayMessage(ctx, defaultRequeueDelay, "Route not ready") } // Check if InferenceServices present status.InferenceServiceReady, err = r.checkInferenceServicesPresent(ctx, instance.Namespace) // All checks passed, resources are ready - if status.IsAllReady() { + if status.IsAllReady(instance.Spec.Storage.Format) { _, updateErr := r.updateStatus(ctx, instance, func(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) { if status.InferenceServiceReady { @@ -83,7 +86,9 @@ func (r *TrustyAIServiceReconciler) reconcileStatuses(ctx context.Context, insta UpdateInferenceServiceNotPresent(saved) } - UpdatePVCAvailable(saved) + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + UpdatePVCAvailable(saved) + } UpdateRouteAvailable(saved) UpdateTrustyAIServiceAvailable(saved) saved.Status.Phase = "Ready" @@ -101,11 +106,14 @@ func (r *TrustyAIServiceReconciler) reconcileStatuses(ctx context.Context, insta UpdateInferenceServiceNotPresent(saved) } - if status.PVCReady { - UpdatePVCAvailable(saved) - } else { - UpdatePVCNotAvailable(saved) + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + if status.PVCReady { + UpdatePVCAvailable(saved) + } else { + UpdatePVCNotAvailable(saved) + } } + if status.RouteReady { UpdateRouteAvailable(saved) } else { diff --git a/controllers/statuses_test.go b/controllers/statuses_test.go index 6d110816..9e395df4 100644 --- a/controllers/statuses_test.go +++ b/controllers/statuses_test.go @@ -3,6 +3,7 @@ package controllers import ( "context" "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" @@ -26,6 +27,27 @@ func checkCondition(conditions []trustyaiopendatahubiov1alpha1.Condition, condit return nil, false, fmt.Errorf("%s condition not found", conditionType) } +func setupAndTestStatusNoComponent(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + // Call the reconcileStatuses function + _, _ = reconciler.reconcileStatuses(ctx, instance) + + readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) + Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") + if readyCondition != nil { + Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Ready condition should be true") + } + + availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionFalse, true) + Expect(err).NotTo(HaveOccurred(), "Error checking Available condition") + if availableCondition != nil { + Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Available condition should be false") + } +} + var _ = Describe("Status and condition tests", func() { BeforeEach(func() { @@ -41,37 +63,163 @@ var _ = Describe("Status and condition tests", func() { Context("When no component exists", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should not be available", func() { - namespace := "statuses-test-namespace-1" - instance = createDefaultCR(namespace) + It("Should not be available in PVC-mode", func() { + namespace := "statuses-test-namespace-1-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestStatusNoComponent(instance, namespace) + }) + It("Should not be available in DB-mode", func() { + namespace := "statuses-test-namespace-1-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestStatusNoComponent(instance, namespace) + }) + It("Should not be available in migration-mode", func() { + namespace := "statuses-test-namespace-1-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestStatusNoComponent(instance, namespace) + }) + + }) + Context("When route, deployment and PVC component, but not inference service, exist", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + It("Should be available in PVC-mode", func() { + namespace := "statuses-test-namespace-2-pvc" + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + WaitFor(func() error { + return reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + }, "failed to create route") + WaitFor(func() error { + return makeRouteReady(ctx, k8sClient, instance) + }, "failed to make route ready") + WaitFor(func() error { + return reconciler.ensurePVC(ctx, instance) + }, "failed to create PVC") + WaitFor(func() error { + return makePVCReady(ctx, k8sClient, instance) + }, "failed to bind PVC") + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + WaitFor(func() error { + return makeDeploymentReady(ctx, k8sClient, instance) + }, "failed to make deployment ready") + WaitFor(func() error { + return k8sClient.Create(ctx, instance) + }, "failed to create TrustyAIService") // Call the reconcileStatuses function - _, _ = reconciler.reconcileStatuses(ctx, instance) + WaitFor(func() error { + _, err := reconciler.reconcileStatuses(ctx, instance) + return err + }, "failed to update statuses") + + // Fetch the updated instance + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, "failed to get updated instance") readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") if readyCondition != nil { - Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Ready condition should be true") + Expect(statusMatch).To(Equal(corev1.ConditionTrue), "Ready condition should be true") } - availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionFalse, true) + availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionTrue, false) Expect(err).NotTo(HaveOccurred(), "Error checking Available condition") - if availableCondition != nil { - Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Available condition should be false") - } + Expect(availableCondition).NotTo(BeNil(), "Available condition should not be null") + Expect(statusMatch).To(Equal(true), "Ready condition should be true") + + routeAvailableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeRouteAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking RouteAvailable condition") + Expect(routeAvailableCondition).NotTo(BeNil(), "RouteAvailable condition should not be null") + Expect(statusMatch).To(Equal(true), "RouteAvailable condition should be true") + + pvcAvailableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypePVCAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking PVCAvailable condition") + Expect(pvcAvailableCondition).NotTo(BeNil(), "PVCAvailable condition should not be null") + Expect(statusMatch).To(Equal(true), "PVCAvailable condition should be true") + ISPresentCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeInferenceServicesPresent, corev1.ConditionFalse, false) + Expect(err).NotTo(HaveOccurred(), "Error checking InferenceServicePresent condition") + Expect(ISPresentCondition).NotTo(BeNil(), "InferenceServicePresent condition should not be null") + Expect(statusMatch).To(Equal(true), "InferenceServicePresent condition should be false") }) - }) + It("Should be available in DB-mode", func() { + namespace := "statuses-test-namespace-2-db" + instance = createDefaultDBCustomResource(namespace) + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) - Context("When route, deployment and PVC component, but not inference service, exist", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should be available", func() { - namespace := "statuses-test-namespace-2" - instance = createDefaultCR(namespace) + WaitFor(func() error { + return reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + }, "failed to create route") + WaitFor(func() error { + return makeRouteReady(ctx, k8sClient, instance) + }, "failed to make route ready") + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + WaitFor(func() error { + return makeDeploymentReady(ctx, k8sClient, instance) + }, "failed to make deployment ready") + WaitFor(func() error { + return k8sClient.Create(ctx, instance) + }, "failed to create TrustyAIService") + + // Call the reconcileStatuses function + WaitFor(func() error { + _, err := reconciler.reconcileStatuses(ctx, instance) + return err + }, "failed to update statuses") + + // Fetch the updated instance + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, "failed to get updated instance") + + readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) + Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") + if readyCondition != nil { + Expect(statusMatch).To(Equal(corev1.ConditionTrue), "Ready condition should be true") + } + + availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking Available condition") + Expect(availableCondition).NotTo(BeNil(), "Available condition should not be null") + Expect(statusMatch).To(Equal(true), "Ready condition should be true") + + routeAvailableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeRouteAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking RouteAvailable condition") + Expect(routeAvailableCondition).NotTo(BeNil(), "RouteAvailable condition should not be null") + Expect(statusMatch).To(Equal(true), "RouteAvailable condition should be true") + + pvcAvailableCondition, _, err := checkCondition(instance.Status.Conditions, StatusTypePVCAvailable, corev1.ConditionTrue, false) + Expect(err).To(HaveOccurred(), "Error checking PVCAvailable condition") + Expect(pvcAvailableCondition).To(BeNil(), "PVCAvailable condition should be null") + + ISPresentCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeInferenceServicesPresent, corev1.ConditionFalse, false) + Expect(err).NotTo(HaveOccurred(), "Error checking InferenceServicePresent condition") + Expect(ISPresentCondition).NotTo(BeNil(), "InferenceServicePresent condition should not be null") + Expect(statusMatch).To(Equal(true), "InferenceServicePresent condition should be false") + + }) + It("Should be available in migration-mode", func() { + namespace := "statuses-test-namespace-2-migration" + instance = createDefaultMigrationCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -90,7 +238,7 @@ var _ = Describe("Status and condition tests", func() { return makePVCReady(ctx, k8sClient, instance) }, "failed to bind PVC") WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) + return reconciler.ensureDeployment(ctx, instance, caBundle, false) }, "failed to create deployment") WaitFor(func() error { return makeDeploymentReady(ctx, k8sClient, instance) @@ -138,6 +286,7 @@ var _ = Describe("Status and condition tests", func() { Expect(err).NotTo(HaveOccurred(), "Error checking InferenceServicePresent condition") Expect(ISPresentCondition).NotTo(BeNil(), "InferenceServicePresent condition should not be null") Expect(statusMatch).To(Equal(true), "InferenceServicePresent condition should be false") + }) }) @@ -145,7 +294,7 @@ var _ = Describe("Status and condition tests", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("Should be available", func() { namespace := "statuses-test-namespace-2" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -164,7 +313,7 @@ var _ = Describe("Status and condition tests", func() { return makePVCReady(ctx, k8sClient, instance) }, "failed to bind PVC") WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) + return reconciler.ensureDeployment(ctx, instance, caBundle, false) }, "failed to create deployment") WaitFor(func() error { return makeDeploymentReady(ctx, k8sClient, instance) diff --git a/controllers/storage_test.go b/controllers/storage_test.go index f11e6394..c5799035 100644 --- a/controllers/storage_test.go +++ b/controllers/storage_test.go @@ -31,7 +31,7 @@ var _ = Describe("PVC Reconciliation", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("should create a new PVC and emit an event", func() { namespace := "pvc-test-namespace-1" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -54,7 +54,7 @@ var _ = Describe("PVC Reconciliation", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("should not attempt to create the PVC", func() { namespace := "pvc-test-namespace-2" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -77,4 +77,16 @@ var _ = Describe("PVC Reconciliation", func() { }) }) + Context("when a migration CR is made", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + It("Check all fields are correct", func() { + namespace := "pvc-test-namespace-3" + instance = createDefaultMigrationCustomResource(namespace) + + Expect(instance.IsMigration()).To(BeTrue()) + Expect(instance.Spec.Storage.IsStoragePVC()).To(BeFalse()) + Expect(instance.Spec.Storage.IsStorageDatabase()).To(BeTrue()) + }) + }) + }) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index ec35b5f9..7ee75a93 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -65,8 +65,9 @@ var ( ) const ( - defaultServiceName = "example-trustyai-service" - operatorNamespace = "system" + defaultServiceName = "example-trustyai-service" + defaultDatabaseConfigurationName = defaultServiceName + "-db-credentials" + operatorNamespace = "system" ) const ( @@ -86,8 +87,8 @@ func WaitFor(operation func() error, errorMsg string) { Eventually(operation, defaultTimeout, defaultPolling).Should(Succeed(), errorMsg) } -// createDefaultCR creates a TrustyAIService instance with default values -func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { +// createDefaultPVCCustomResource creates a TrustyAIService instance with default values and PVC backend +func createDefaultPVCCustomResource(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { service := trustyaiopendatahubiov1alpha1.TrustyAIService{ ObjectMeta: metav1.ObjectMeta{ Name: defaultServiceName, @@ -96,7 +97,7 @@ func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.Tru }, Spec: trustyaiopendatahubiov1alpha1.TrustyAIServiceSpec{ Storage: trustyaiopendatahubiov1alpha1.StorageSpec{ - Format: "PVC", + Format: STORAGE_PVC, Folder: "/data", Size: "1Gi", }, @@ -112,6 +113,54 @@ func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.Tru return &service } +// createDefaultDBCustomResource creates a TrustyAIService instance with default values and DB backend +func createDefaultDBCustomResource(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { + service := trustyaiopendatahubiov1alpha1.TrustyAIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultServiceName, + Namespace: namespaceCurrent, + UID: types.UID(uuid.New().String()), + }, + Spec: trustyaiopendatahubiov1alpha1.TrustyAIServiceSpec{ + Storage: trustyaiopendatahubiov1alpha1.StorageSpec{ + Format: STORAGE_DATABASE, + DatabaseConfigurations: defaultDatabaseConfigurationName, + }, + Metrics: trustyaiopendatahubiov1alpha1.MetricsSpec{ + Schedule: "5s", + }, + }, + } + return &service +} + +// createDefaultMigrationCustomResource creates a TrustyAIService instance with default values and both PVC and DB backend +func createDefaultMigrationCustomResource(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { + service := trustyaiopendatahubiov1alpha1.TrustyAIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultServiceName, + Namespace: namespaceCurrent, + UID: types.UID(uuid.New().String()), + }, + Spec: trustyaiopendatahubiov1alpha1.TrustyAIServiceSpec{ + Storage: trustyaiopendatahubiov1alpha1.StorageSpec{ + Format: STORAGE_DATABASE, + DatabaseConfigurations: defaultDatabaseConfigurationName, + Folder: "/data", + Size: "1Gi", + }, + Data: trustyaiopendatahubiov1alpha1.DataSpec{ + Filename: "data.csv", + Format: "CSV", + }, + Metrics: trustyaiopendatahubiov1alpha1.MetricsSpec{ + Schedule: "5s", + }, + }, + } + return &service +} + // createNamespace creates a new namespace func createNamespace(ctx context.Context, k8sClient client.Client, namespace string) error { ns := &corev1.Namespace{ @@ -148,6 +197,34 @@ func createConfigMap(namespace string, oauthImage string, trustyaiServiceImage s } } +// createSecret creates a secret in the specified namespace +func createSecret(namespace string, secretName string, data map[string]string) *corev1.Secret { + // Convert the data map values from string to byte array + byteData := make(map[string][]byte) + for key, value := range data { + byteData[key] = []byte(value) + } + + // Define the Secret with the necessary data + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: byteData, + } +} + +func createDatabaseConfiguration(namespace string, name string, dbKind string) *corev1.Secret { + return createSecret(namespace, name, map[string]string{ + "databaseKind": dbKind, + "databaseUsername": "foo", + "databasePassword": "bar", + "databaseService": "mariadb-service", + "databasePort": "3306", + }) +} + // createTrustedCABundleConfigMap creates a ConfigMap in the specified namespace // with the label to inject the trusted CA bundle by OpenShift func createTrustedCABundleConfigMap(namespace string) *corev1.ConfigMap { diff --git a/controllers/templates/service/deployment.tmpl.yaml b/controllers/templates/service/deployment.tmpl.yaml index b9f2392e..840421f0 100644 --- a/controllers/templates/service/deployment.tmpl.yaml +++ b/controllers/templates/service/deployment.tmpl.yaml @@ -12,6 +12,11 @@ metadata: app.kubernetes.io/part-of: trustyai app.kubernetes.io/version: {{ .Version }} spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 replicas: 1 selector: matchLabels: @@ -38,25 +43,85 @@ spec: - name: trustyai-service image: {{ .ServiceImage }} env: - - name: STORAGE_DATA_FILENAME - value: {{ .Instance.Spec.Data.Filename }} - name: SERVICE_STORAGE_FORMAT value: {{ .Instance.Spec.Storage.Format }} + {{ if eq .Instance.Spec.Storage.Format "PVC" }} + - name: STORAGE_DATA_FILENAME + value: {{ .Instance.Spec.Data.Filename }} + - name: STORAGE_DATA_FOLDER + value: {{ .Instance.Spec.Storage.Folder }} + - name: SERVICE_DATA_FORMAT + value: {{ .Instance.Spec.Data.Format }} + - name: QUARKUS_HIBERNATE_ORM_ACTIVE + value: false + {{ end }} + {{ if .Instance.IsMigration }} + - name: STORAGE_DATA_FILENAME + value: {{ .Instance.Spec.Data.Filename }} - name: STORAGE_DATA_FOLDER value: {{ .Instance.Spec.Storage.Folder }} - name: SERVICE_DATA_FORMAT value: {{ .Instance.Spec.Data.Format }} + {{ end }} + {{ if eq .Instance.Spec.Storage.Format "DATABASE" }} + - name: QUARKUS_HIBERNATE_ORM_ACTIVE + value: true + - name: QUARKUS_DATASOURCE_DB_KIND + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseKind + - name: QUARKUS_DATASOURCE_JDBC_MAX_SIZE + value: 16 + - name: QUARKUS_DATASOURCE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseUsername + - name: QUARKUS_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databasePassword + - name: DATABASE_SERVICE + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseService + - name: DATABASE_PORT + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databasePort + - name: QUARKUS_DATASOURCE_JDBC_URL + value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database" + - name: SERVICE_DATA_FORMAT + value: "HIBERNATE" + - name: QUARKUS_DATASOURCE_GENERATION + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseGeneration + {{ end }} - name: SERVICE_METRICS_SCHEDULE value: {{ .Instance.Spec.Metrics.Schedule }} - name: SERVICE_BATCH_SIZE - value: {{ .Schedule }} + value: {{ .BatchSize }} + {{ if .Instance.IsMigration }} + - name: STORAGE_MIGRATION_CONFIG_FROM_FOLDER + value: {{ .Instance.Spec.Storage.Folder }} + - name: STORAGE_MIGRATION_CONFIG_FROM_FILENAME + value: {{ .Instance.Spec.Data.Filename }} + {{ end }} volumeMounts: - name: {{ .Instance.Name }}-internal readOnly: false mountPath: /etc/tls/internal + {{ if or (eq .Instance.Spec.Storage.Format "PVC") (.Instance.IsMigration) }} - name: {{ .VolumeMountName }} mountPath: {{ .Instance.Spec.Storage.Folder }} readOnly: false + {{ end }} - resources: limits: cpu: 100m @@ -125,9 +190,11 @@ spec: "pods", "verb": "get"}} serviceAccount: {{ .Instance.Name }}-proxy volumes: + {{ if or (eq .Instance.Spec.Storage.Format "PVC") ( .Instance.IsMigration) }} - name: volume persistentVolumeClaim: claimName: {{ .PVCClaimName }} + {{ end }} {{ if .CustomCertificatesBundle.IsDefined }} - name: {{ .CustomCertificatesBundle.VolumeName}} configMap: diff --git a/controllers/trustyaiservice_controller.go b/controllers/trustyaiservice_controller.go index 00600da7..d37fb267 100644 --- a/controllers/trustyaiservice_controller.go +++ b/controllers/trustyaiservice_controller.go @@ -136,25 +136,57 @@ func (r *TrustyAIServiceReconciler) Reconcile(ctx context.Context, req ctrl.Requ return RequeueWithDelayMessage(ctx, time.Minute, "Not all replicas are ready, requeue the reconcile request") } - // Ensure PVC - err = r.ensurePVC(ctx, instance) - if err != nil { - // PVC not found condition - log.FromContext(ctx).Error(err, "Error creating PVC storage.") - _, updateErr := r.updateStatus(ctx, instance, UpdatePVCNotAvailable) - if updateErr != nil { - return RequeueWithErrorMessage(ctx, err, "Failed to update status") - } + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + // Ensure PVC + err = r.ensurePVC(ctx, instance) + if err != nil { + // PVC not found condition + log.FromContext(ctx).Error(err, "Error creating PVC storage.") + _, updateErr := r.updateStatus(ctx, instance, UpdatePVCNotAvailable) + if updateErr != nil { + return RequeueWithErrorMessage(ctx, err, "Failed to update status") + } - // If there was an error finding the PV, requeue the request - return RequeueWithErrorMessage(ctx, err, "Could not find requested PersistentVolumeClaim.") + // If there was an error finding the PV, requeue the request + return RequeueWithErrorMessage(ctx, err, "Could not find requested PersistentVolumeClaim.") + } + } + if instance.Spec.Storage.IsStorageDatabase() { + // Get database configuration + secret, err := r.findDatabaseSecret(ctx, instance) + if err != nil { + return RequeueWithErrorMessage(ctx, err, "Service configured to use database storage but no database configuration found.") + } + err = r.validateDatabaseSecret(secret) + if err != nil { + return RequeueWithErrorMessage(ctx, err, "Database configuration contains errors.") + } } - // Ensure Deployment object - err = r.ensureDeployment(ctx, instance, caBundle) - if err != nil { - return RequeueWithError(err) + // Check for migration annotation + if _, ok := instance.Annotations[migrationAnnotationKey]; ok { + log.FromContext(ctx).Info("Found migration annotation. Migrating.") + err = r.ensureDeployment(ctx, instance, caBundle, true) + //err = r.redeployForMigration(ctx, instance) + + if err != nil { + return RequeueWithErrorMessage(ctx, err, "Retrying to restart deployment during migration.") + } + + // Remove the migration annotation after processing to avoid restarts + delete(instance.Annotations, migrationAnnotationKey) + log.FromContext(ctx).Info("Deleting annotation") + if err := r.Update(ctx, instance); err != nil { + return RequeueWithErrorMessage(ctx, err, "Failed to remove migration annotation.") + } + } else { + // Ensure Deployment object + err = r.ensureDeployment(ctx, instance, caBundle, false) + log.FromContext(ctx).Info("No annotation found") + if err != nil { + return RequeueWithError(err) + } } // Fetch the TrustyAIService instance @@ -235,6 +267,7 @@ func (r *TrustyAIServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&trustyaiopendatahubiov1alpha1.TrustyAIService{}). + Owns(&appsv1.Deployment{}). Watches(&source.Kind{Type: &kservev1beta1.InferenceService{}}, &handler.EnqueueRequestForObject{}). Watches(&source.Kind{Type: &kservev1alpha1.ServingRuntime{}}, &handler.EnqueueRequestForObject{}). Complete(r) From e9a0204b5df2c90e49473c58ffae3b2e78a21d61 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Mon, 22 Jul 2024 10:40:35 +0100 Subject: [PATCH 02/11] feat: Add TLS certificate mount on ModelMesh (#255) * feat: Add TLS certificate mount on ModelMesh * Revert from http to https until https://github.com/kserve/modelmesh/pull/147 is merged --- controllers/certificates.go | 31 ++++++++++++++++ controllers/constants.go | 2 + controllers/inference_services.go | 62 +++++++++++++++++++++++++++++-- controllers/utils.go | 5 ++- 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/controllers/certificates.go b/controllers/certificates.go index 7a3fc1fb..16faa1f9 100644 --- a/controllers/certificates.go +++ b/controllers/certificates.go @@ -3,10 +3,41 @@ package controllers import ( "context" trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) +const ( + tlsMountPath = "/etc/trustyai/tls" +) + +// TLSCertVolumes holds the volume and volume mount for the TLS certificates +type TLSCertVolumes struct { + volume corev1.Volume + volumeMount corev1.VolumeMount +} + +// createFor creates the required volumes and volume mount for the TLS certificates for a specific Kubernetes secret +func (cert *TLSCertVolumes) createFor(instance *trustyaiopendatahubiov1alpha1.TrustyAIService) { + volume := corev1.Volume{ + Name: instance.Name + "-internal", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: instance.Name + "-internal", + }, + }, + } + + volumeMount := corev1.VolumeMount{ + Name: instance.Name + "-internal", + MountPath: tlsMountPath, + ReadOnly: true, + } + cert.volume = volume + cert.volumeMount = volumeMount +} + func (r *TrustyAIServiceReconciler) GetCustomCertificatesBundle(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) CustomCertificatesBundle { var customCertificatesBundle CustomCertificatesBundle diff --git a/controllers/constants.go b/controllers/constants.go index f55da3a2..1fc4eb90 100644 --- a/controllers/constants.go +++ b/controllers/constants.go @@ -9,6 +9,8 @@ const ( serviceMonitorName = "trustyai-metrics" finalizerName = "trustyai.opendatahub.io/finalizer" payloadProcessorName = "MM_PAYLOAD_PROCESSORS" + tlsKeyCertPathName = "MM_TLS_KEY_CERT_PATH" + mmContainerName = "mm" modelMeshLabelKey = "modelmesh-service" modelMeshLabelValue = "modelmesh-serving" volumeMountName = "volume" diff --git a/controllers/inference_services.go b/controllers/inference_services.go index bde29551..773d4924 100644 --- a/controllers/inference_services.go +++ b/controllers/inference_services.go @@ -13,6 +13,10 @@ import ( ) func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, deployments []appsv1.Deployment, envVarName string, url string, remove bool) (bool, error) { + // Create volume and volume mount for this intance's TLS secrets + certVolumes := TLSCertVolumes{} + certVolumes.createFor(instance) + // Loop over the Deployments for _, deployment := range deployments { @@ -23,8 +27,31 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte return false, nil } + // If the secret volume doesn't exist, add it + volumeExists := false + for _, vol := range deployment.Spec.Template.Spec.Volumes { + if vol.Name == instance.Name+"-internal" { + volumeExists = true + break + } + } + if !volumeExists { + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, certVolumes.volume) + } + // Loop over all containers in the Deployment's Pod template for i := range deployment.Spec.Template.Spec.Containers { + mountExists := false + for _, mount := range deployment.Spec.Template.Spec.Containers[i].VolumeMounts { + if mount.Name == instance.Name+"-internal" { + mountExists = true + break + } + } + if !mountExists { + deployment.Spec.Template.Spec.Containers[i].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[i].VolumeMounts, certVolumes.volumeMount) + } + // Store the original environment variable list // Get the existing env var var envVar *corev1.EnvVar @@ -50,14 +77,17 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte } else if envVar != nil { // If the env var exists and already contains the value, don't do anything existingValues := strings.Split(envVar.Value, " ") + valueExists := false for _, v := range existingValues { if v == url { - continue + valueExists = true + break } } - // Modify the existing env var based on the remove flag and current value - envVar.Value = generateEnvVarValue(envVar.Value, url, remove) + if !valueExists { + envVar.Value = generateEnvVarValue(envVar.Value, url, remove) + } } // Only update the deployment if the var value has to change, or we are removing it @@ -70,6 +100,32 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte r.eventModelMeshConfigured(instance) log.FromContext(ctx).Info("Updating Deployment " + deployment.Name + ", container spec " + deployment.Spec.Template.Spec.Containers[i].Name + ", env var " + envVarName + " to " + url) } + + // Check TLS environment variable on ModelMesh + if deployment.Spec.Template.Spec.Containers[i].Name == mmContainerName { + tlsKeyCertPathEnvValue := tlsMountPath + "/tls.crt" + tlsKeyCertPathExists := false + for _, envVar := range deployment.Spec.Template.Spec.Containers[i].Env { + if envVar.Name == tlsKeyCertPathName { + tlsKeyCertPathExists = true + break + } + } + + // Doesn't exist, so we can add + if !tlsKeyCertPathExists { + deployment.Spec.Template.Spec.Containers[i].Env = append(deployment.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ + Name: tlsKeyCertPathName, + Value: tlsKeyCertPathEnvValue, + }) + + if err := r.Update(ctx, &deployment); err != nil { + log.FromContext(ctx).Error(err, "Could not update Deployment", "Deployment", deployment.Name) + return false, err + } + log.FromContext(ctx).Info("Added environment variable " + tlsKeyCertPathName + " to deployment " + deployment.Name + " for container " + mmContainerName) + } + } } } diff --git a/controllers/utils.go b/controllers/utils.go index 4a8415e1..91ee9bdb 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -2,9 +2,10 @@ package controllers import ( "context" + "os" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" - "os" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -63,5 +64,5 @@ func (r *TrustyAIServiceReconciler) GetDeploymentsByLabel(ctx context.Context, n // generateServiceURL generates an internal URL for a TrustyAI service func generateServiceURL(crName string, namespace string) string { - return "http://" + crName + "." + namespace + ".svc.cluster.local" + return "http://" + crName + "." + namespace + ".svc" } From c4ef8021149a6c9facba7e7549d9b9689232e1f0 Mon Sep 17 00:00:00 2001 From: Rob Geada Date: Mon, 29 Jul 2024 10:34:16 +0100 Subject: [PATCH 03/11] Pin oc version, ubi version (#263) --- tests/Dockerfile | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 9465467b..5677c360 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,18 +1,17 @@ -FROM registry.access.redhat.com/ubi8:8.10-901.1716497712 +FROM registry.access.redhat.com/ubi8:8.10-1020 ARG ORG=trustyai-explainability ARG BRANCH=main ARG ODS_CI_REPO=https://github.com/red-hat-data-services/ods-ci # This git reference should always reference a stable commit from ods-ci that supports ODH # This hash corresponds to a March 24th, 2023 commit -ARG ODS_CI_GITREF=867a617bc224726cf98fa3354293f8e50b4f5eb5 -ARG OC_CLI_URL=https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/latest/openshift-client-linux.tar.gz +ARG ODS_CI_GITREF=a8cf770b37caa4ef7ce6596acc8bdd6866cc7772 +ARG OC_CLI_URL=https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/4.14.33/openshift-client-linux.tar.gz ENV HOME /root WORKDIR /root -RUN dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm &&\ - dnf install -y jq bc git go-toolset python3.11 python3.11-pip python3.11-devel unzip && \ +RUN dnf install -y jq bc git go-toolset python3.11 python3.11-devel python3.11-pip unzip && \ dnf clean all && \ git clone https://github.com/opendatahub-io/peak $HOME/peak && \ cd $HOME/peak && \ @@ -22,14 +21,6 @@ RUN dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.25.1/yq_linux_amd64 -o /usr/bin/yq &&\ chmod +x /usr/bin/yq -RUN mkdir -p $HOME/src && \ - cd $HOME/src && \ - git clone --depth=1 --branch ${BRANCH} https://github.com/${ORG}/trustyai-explainability && \ - # Clone ods-ci repo at specified git ref for the ODH Dashboard webUI tests - git clone --depth=1 ${ODS_CI_REPO} ods-ci && cd ods-ci && \ - git fetch origin ${ODS_CI_GITREF} && git checkout FETCH_HEAD && \ - chmod -R 777 $HOME/src - # Use a specific destination file name in case the url download name changes ADD ${OC_CLI_URL} $HOME/peak/oc-cli.tar.gz RUN tar -C /usr/local/bin -xvf $HOME/peak/oc-cli.tar.gz && \ @@ -37,16 +28,6 @@ RUN tar -C /usr/local/bin -xvf $HOME/peak/oc-cli.tar.gz && \ COPY Pipfile Pipfile.lock $HOME/peak/ -RUN pip3 install micropipenv &&\ - ln -s `which pip3` /usr/bin/pip &&\ - cd $HOME/peak &&\ - micropipenv install - -# Install poetry to support the exeuction of ods-ci test framework -RUN curl -sSL https://install.python-poetry.org | python3 - -ENV PATH="${PATH}:$HOME/.local/bin" -RUN cd $HOME/src/ods-ci && poetry install - ## Grab CI scripts from single-source-of-truth RUN mkdir -p $HOME/peak/operator-tests/trustyai-explainability/ &&\ mkdir $HOME/kfdef/ &&\ From 883c3fbb18656943b3a3b35e418b20c75e13296e Mon Sep 17 00:00:00 2001 From: Rob Geada Date: Mon, 29 Jul 2024 11:46:00 +0100 Subject: [PATCH 04/11] Restore checkout of trustyai-exp (#265) --- tests/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/Dockerfile b/tests/Dockerfile index 5677c360..e39ebdfd 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -21,6 +21,11 @@ RUN dnf install -y jq bc git go-toolset python3.11 python3.11-devel python3.11-p RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.25.1/yq_linux_amd64 -o /usr/bin/yq &&\ chmod +x /usr/bin/yq +RUN mkdir -p $HOME/src && \ + cd $HOME/src && \ + git clone --depth=1 --branch ${BRANCH} https://github.com/${ORG}/trustyai-explainability && \ + chmod -R 777 $HOME/src + # Use a specific destination file name in case the url download name changes ADD ${OC_CLI_URL} $HOME/peak/oc-cli.tar.gz RUN tar -C /usr/local/bin -xvf $HOME/peak/oc-cli.tar.gz && \ From 4a52d652652501a63c4ed450ce8b13c159510d6a Mon Sep 17 00:00:00 2001 From: Rob Geada Date: Mon, 29 Jul 2024 14:40:58 +0100 Subject: [PATCH 05/11] Add operator installation robustness (#266) --- tests/scripts/install.sh | 63 ++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/tests/scripts/install.sh b/tests/scripts/install.sh index 46eb7d29..cc47a549 100755 --- a/tests/scripts/install.sh +++ b/tests/scripts/install.sh @@ -14,14 +14,44 @@ if ! [ -z "${SKIP_OPERATOR_INSTALL}" ]; then ./setup.sh -t ~/peak/operatorsetup 2>&1 else echo "Installing operator from community marketplace" - while [[ $retry -gt 0 ]]; do - # patch bug in peak setup script - sed -i "s/path=\"{.status.channels.*/ | jq '.status.channels | .[0].currentCSVDesc.installModes | map(select(.type == \"AllNamespaces\")) | .[0].supported')/" setup.sh - sed -i "s/csource=.*/echo \$3; csource=\$3/" setup.sh - sed -i 's/installop \$.*/installop \${vals[0]} \${vals[1]} \${vals[3]}/' setup.sh + start_t=$(date +%s) 2>&1 + ready=false 2>&1 + while ! $ready; do + CATALOG_SOURCES=$(oc get catalogsources -n openshift-marketplace 2> /dev/null | grep 'community-operators') + if [ ! -z "${CATALOG_SOURCES}" ]; then + echo $CATALOG_SOURCES + ready=true 2>&1 + else + sleep 10 + fi + if [ $(($(date +%s)-start_t)) -gt 300 ]; then + echo "Marketplace pods never started" + exit 1 + fi + done + + start_t=$(date +%s) 2>&1 + ready=false 2>&1 + while ! $ready; do + MANIFESTS=$(oc get packagemanifests -n openshift-marketplace 2> /dev/null | grep 'opendatahub') + echo $MANIFESTS + if [ ! -z "${MANIFESTS}" ]; then + echo $MANIFESTS + ready=true 2>&1 + else + sleep 10 + fi + if [ $(($(date +%s)-start_t)) -gt 900 ]; then + echo "Package manifests never downloaded" + exit 1 + fi + done + + while [[ $retry -gt 0 ]]; do + ./setup.sh -o ~/peak/operatorsetup\ - ./setup.sh -o ~/peak/operatorsetup + # approve installplans if [ $? -eq 0 ]; then retry=-1 else @@ -31,11 +61,16 @@ else fi retry=$(( retry - 1)) + sleep 30 + echo "Approving Install Plans, if needed" + oc patch installplan $(oc get installplan -n openshift-operators | grep $ODH_VERSION | awk '{print $1}') -n openshift-operators --type merge --patch '{"spec":{"approved":true}}' || true + oc patch installplan $(oc get installplan -n openshift-operators | grep authorino | awk '{print $1}') -n openshift-operators --type merge --patch '{"spec":{"approved":true}}' || true + finished=false 2>&1 start_t=$(date +%s) 2>&1 echo "Verifying installation of ODH operator" while ! $finished; do - if [ ! -z "$(oc get pods -n openshift-operators | grep 'opendatahub-operator-controller-manager' | grep '1/1')" ]; then + if [ ! -z "$(oc get pods -n openshift-operators | grep 'opendatahub-operator-controller-manager' | grep '1/1')" ]; then finished=true 2>&1 else sleep 10 @@ -50,20 +85,6 @@ else done fi -#popd -### Grabbing and applying the patch in the PR we are testing -#pushd ~/src/${REPO_NAME} -#if [ -z "$PULL_NUMBER" ]; then -# echo "No pull number, assuming nightly run" -#else -# if [ $REPO_OWNER == "trustyai-explainability" ]; then -# curl -O -L https://github.com/${REPO_OWNER}/${REPO_NAME}/pull/${PULL_NUMBER}.patch -# echo "Applying followng patch:" -# cat ${PULL_NUMBER}.patch > ${ARTIFACT_DIR}/github-pr-${PULL_NUMBER}.patch -# git apply ${PULL_NUMBER}.patch -# fi -#fi - popd ## Point manifests repo uri in the KFDEF to the manifests in the PR pushd ~/kfdef From ec4462ed07b7b6f0d86e985bb83db015f175ea29 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Mon, 29 Jul 2024 21:07:58 +0100 Subject: [PATCH 06/11] fix: Skip InferenceService patching for KServe RawDeployment (#262) --- controllers/inference_services.go | 36 ++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/controllers/inference_services.go b/controllers/inference_services.go index 773d4924..5f22e476 100644 --- a/controllers/inference_services.go +++ b/controllers/inference_services.go @@ -12,6 +12,12 @@ import ( "strings" ) +const ( + DEPLOYMENT_MODE_MODELMESH = "ModelMesh" + DEPLOYMENT_MODE_RAW = "RawDeployment" + DEPLOYMENT_MODE_SERVERLESS = "Serverless" +) + func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, deployments []appsv1.Deployment, envVarName string, url string, remove bool) (bool, error) { // Create volume and volume mount for this intance's TLS secrets certVolumes := TLSCertVolumes{} @@ -199,20 +205,26 @@ func (r *TrustyAIServiceReconciler) handleInferenceServices(ctx context.Context, for _, infService := range inferenceServices.Items { annotations := infService.GetAnnotations() - // Check the annotation "serving.kserve.io/deploymentMode: ModelMesh" - if val, ok := annotations["serving.kserve.io/deploymentMode"]; ok && val == "ModelMesh" { - shouldContinue, err := r.patchEnvVarsByLabelForDeployments(ctx, instance, namespace, labelKey, labelValue, envVarName, crName, remove) - if err != nil { - log.FromContext(ctx).Error(err, "Could not patch environment variables for ModelMesh deployments.") - return shouldContinue, err - } - } else { - err := r.patchKServe(ctx, instance, infService, namespace, crName, remove) - if err != nil { - log.FromContext(ctx).Error(err, "Could not path InferenceLogger for KServe deployment.") - return false, err + + // Check the annotation "serving.kserve.io/deploymentMode" + if val, ok := annotations["serving.kserve.io/deploymentMode"]; ok { + if val == DEPLOYMENT_MODE_RAW { + log.FromContext(ctx).Info("RawDeployment mode not supported by TrustyAI") + continue + } else if val == DEPLOYMENT_MODE_MODELMESH { + shouldContinue, err := r.patchEnvVarsByLabelForDeployments(ctx, instance, namespace, labelKey, labelValue, envVarName, crName, remove) + if err != nil { + log.FromContext(ctx).Error(err, "could not patch environment variables for ModelMesh deployments") + return shouldContinue, err + } + continue } } + err := r.patchKServe(ctx, instance, infService, namespace, crName, remove) + if err != nil { + log.FromContext(ctx).Error(err, "could not patch InferenceLogger for KServe deployment") + return false, err + } } return true, nil } From c537e14a45b1b9d537e764fc1e6d6a0ff76a56c7 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Wed, 31 Jul 2024 12:56:21 +0100 Subject: [PATCH 07/11] feat: ConfigMap key to disable KServe Serverless configuration (#267) --- config/base/kustomization.yaml | 7 ++++++ config/base/params.env | 1 + controllers/config_maps.go | 38 +++++++++++++++++++++++++++++++ controllers/constants.go | 7 +++--- controllers/inference_services.go | 16 +++++++++---- 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index ab0b9738..0247b9bb 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -41,3 +41,10 @@ vars: apiVersion: v1 fieldref: fieldpath: data.oauthProxyImage + - name: kServeServerless + objref: + kind: ConfigMap + name: config + apiVersion: v1 + fieldref: + fieldpath: data.kServeServerless \ No newline at end of file diff --git a/config/base/params.env b/config/base/params.env index 68d67aae..a0f5419b 100644 --- a/config/base/params.env +++ b/config/base/params.env @@ -1,3 +1,4 @@ trustyaiServiceImage=quay.io/trustyai/trustyai-service:latest trustyaiOperatorImage=quay.io/trustyai/trustyai-service-operator:latest oauthProxyImage=quay.io/openshift/origin-oauth-proxy:4.14.0 +kServeServerless=disabled \ No newline at end of file diff --git a/controllers/config_maps.go b/controllers/config_maps.go index 6eceb845..a4885ab4 100644 --- a/controllers/config_maps.go +++ b/controllers/config_maps.go @@ -46,6 +46,44 @@ func (r *TrustyAIServiceReconciler) getImageFromConfigMap(ctx context.Context, k } } +// getKServeServerlessConfig checks the kServeServerless value in a ConfigMap in the operator's namespace +func (r *TrustyAIServiceReconciler) getKServeServerlessConfig(ctx context.Context) (bool, error) { + + if r.Namespace != "" { + // Define the key for the ConfigMap + configMapKey := types.NamespacedName{ + Namespace: r.Namespace, + Name: imageConfigMap, + } + + // Create an empty ConfigMap object + var cm corev1.ConfigMap + + // Try to get the ConfigMap + if err := r.Get(ctx, configMapKey, &cm); err != nil { + if errors.IsNotFound(err) { + // ConfigMap not found, return false as the default behavior + return false, nil + } + // Other error occurred when trying to fetch the ConfigMap + return false, fmt.Errorf("error reading configmap %s", configMapKey) + } + + // ConfigMap is found, extract the kServeServerless value + kServeServerless, ok := cm.Data[configMapkServeServerlessKey] + + if !ok || kServeServerless != "enabled" { + // Key is missing or its value is not "enabled", return false + return false, nil + } + + // kServeServerless is "enabled" + return true, nil + } else { + return false, nil + } +} + // getConfigMapNamesWithLabel retrieves the names of ConfigMaps that have the specified label func (r *TrustyAIServiceReconciler) getConfigMapNamesWithLabel(ctx context.Context, namespace string, labelSelector client.MatchingLabels) ([]string, error) { configMapList := &corev1.ConfigMapList{} diff --git a/controllers/constants.go b/controllers/constants.go index 1fc4eb90..2c7081b0 100644 --- a/controllers/constants.go +++ b/controllers/constants.go @@ -26,9 +26,10 @@ const ( // Configuration constants const ( - imageConfigMap = "trustyai-service-operator-config" - configMapOAuthProxyImageKey = "oauthProxyImage" - configMapServiceImageKey = "trustyaiServiceImage" + imageConfigMap = "trustyai-service-operator-config" + configMapOAuthProxyImageKey = "oauthProxyImage" + configMapServiceImageKey = "trustyaiServiceImage" + configMapkServeServerlessKey = "kServeServerless" ) // OAuth constants diff --git a/controllers/inference_services.go b/controllers/inference_services.go index 5f22e476..db426d4a 100644 --- a/controllers/inference_services.go +++ b/controllers/inference_services.go @@ -199,6 +199,12 @@ func (r *TrustyAIServiceReconciler) handleInferenceServices(ctx context.Context, return false, err } + kServeServerlessEnabled, err := r.getKServeServerlessConfig(ctx) + if err != nil { + log.FromContext(ctx).Error(err, "Could not read KServeServerless configuration. Defaulting to disabled") + kServeServerlessEnabled = false + } + if len(inferenceServices.Items) == 0 { return true, nil } @@ -220,10 +226,12 @@ func (r *TrustyAIServiceReconciler) handleInferenceServices(ctx context.Context, continue } } - err := r.patchKServe(ctx, instance, infService, namespace, crName, remove) - if err != nil { - log.FromContext(ctx).Error(err, "could not patch InferenceLogger for KServe deployment") - return false, err + if kServeServerlessEnabled { + err := r.patchKServe(ctx, instance, infService, namespace, crName, remove) + if err != nil { + log.FromContext(ctx).Error(err, "could not patch InferenceLogger for KServe deployment") + return false, err + } } } return true, nil From 1515872f008fd8c646e726a25f0881025dea1ee1 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Thu, 1 Aug 2024 18:54:49 +0100 Subject: [PATCH 08/11] feat: Add support for custom certificates in database connection (#259) --- controllers/deployment.go | 15 ++++++++ controllers/secrets.go | 38 +++++++++++-------- .../templates/service/deployment.tmpl.yaml | 17 ++++++++- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/controllers/deployment.go b/controllers/deployment.go index 816c9c17..81be18a5 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -38,6 +38,7 @@ type DeploymentConfig struct { CustomCertificatesBundle CustomCertificatesBundle Version string BatchSize int + UseDBTLSCerts bool } // createDeploymentObject returns a Deployment for the TrustyAI Service instance @@ -70,6 +71,20 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, BatchSize: batchSize, } + if instance.Spec.Storage.IsStorageDatabase() { + _, err := r.getSecret(ctx, instance.Name+"-db-tls", instance.Namespace) + if err != nil { + deploymentConfig.UseDBTLSCerts = false + log.FromContext(ctx).Error(err, "Using insecure database connection. Certificates "+instance.Name+"-db-tls not found") + } else { + deploymentConfig.UseDBTLSCerts = true + log.FromContext(ctx).Info("Using secure database connection with certificates " + instance.Name + "-db-tls") + } + } else { + deploymentConfig.UseDBTLSCerts = false + log.FromContext(ctx).Info("No need to check database secrets. Using PVC-mode.") + } + var deployment *appsv1.Deployment deployment, err = templateParser.ParseResource[appsv1.Deployment](deploymentTemplatePath, deploymentConfig, reflect.TypeOf(&appsv1.Deployment{})) if err != nil { diff --git a/controllers/secrets.go b/controllers/secrets.go index 6e2d49b7..090ab58d 100644 --- a/controllers/secrets.go +++ b/controllers/secrets.go @@ -9,34 +9,42 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// getSecret retrieves a secret if it exists, returns an error if not +func (r *TrustyAIServiceReconciler) getSecret(ctx context.Context, name, namespace string) (*corev1.Secret, error) { + secret := &corev1.Secret{} + err := r.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, secret) + if err != nil { + if errors.IsNotFound(err) { + return nil, fmt.Errorf("secret %s not found in namespace %s: %w", name, namespace, err) + } + return nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", name, namespace, err) + } + return secret, nil +} + // findDatabaseSecret finds the DB configuration secret named (specified or default) in the same namespace as the CR func (r *TrustyAIServiceReconciler) findDatabaseSecret(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) (*corev1.Secret, error) { databaseConfigurationsName := instance.Spec.Storage.DatabaseConfigurations defaultDatabaseConfigurationsName := instance.Name + dbCredentialsSuffix - secret := &corev1.Secret{} - if databaseConfigurationsName != "" { - secret := &corev1.Secret{} - err := r.Get(ctx, client.ObjectKey{Name: databaseConfigurationsName, Namespace: instance.Namespace}, secret) - if err == nil { - return secret, nil + secret, err := r.getSecret(ctx, databaseConfigurationsName, instance.Namespace) + if err != nil { + return nil, err } - if !errors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", databaseConfigurationsName, instance.Namespace, err) + if secret != nil { + return secret, nil } } else { // If specified not found, try the default - - err := r.Get(ctx, client.ObjectKey{Name: defaultDatabaseConfigurationsName, Namespace: instance.Namespace}, secret) - if err == nil { - return secret, nil + secret, err := r.getSecret(ctx, defaultDatabaseConfigurationsName, instance.Namespace) + if err != nil { + return nil, err } - if !errors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", defaultDatabaseConfigurationsName, instance.Namespace, err) + if secret != nil { + return secret, nil } - } return nil, fmt.Errorf("neither secret %s nor %s found in namespace %s", databaseConfigurationsName, defaultDatabaseConfigurationsName, instance.Namespace) diff --git a/controllers/templates/service/deployment.tmpl.yaml b/controllers/templates/service/deployment.tmpl.yaml index 840421f0..fa8a9c76 100644 --- a/controllers/templates/service/deployment.tmpl.yaml +++ b/controllers/templates/service/deployment.tmpl.yaml @@ -94,7 +94,11 @@ spec: name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} key: databasePort - name: QUARKUS_DATASOURCE_JDBC_URL + {{ if .UseDBTLSCerts }} + value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database?sslMode=verify-ca&serverSslCert=/etc/tls/db/tls.crt" + {{ else }} value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database" + {{ end }} - name: SERVICE_DATA_FORMAT value: "HIBERNATE" - name: QUARKUS_DATASOURCE_GENERATION @@ -121,7 +125,12 @@ spec: - name: {{ .VolumeMountName }} mountPath: {{ .Instance.Spec.Storage.Folder }} readOnly: false - {{ end }} + {{ end }} + {{ if .UseDBTLSCerts }} + - name: db-tls-certs + mountPath: /etc/tls/db + readOnly: true + {{ end }} - resources: limits: cpu: 100m @@ -209,3 +218,9 @@ spec: secret: secretName: {{ .Instance.Name }}-internal defaultMode: 420 + {{ if .UseDBTLSCerts }} + - name: db-tls-certs + secret: + secretName: {{ .Instance.Name }}-db-tls + defaultMode: 420 + {{ end }} From 91ba2c56fb0ae4aeaa17c610d637e3df394bb10a Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Thu, 1 Aug 2024 18:55:17 +0100 Subject: [PATCH 09/11] Add TLS endpoint for ModelMesh payload processors. (#268) Keep non-TLS endpoint for KServe Serverless (disabled by default) --- controllers/inference_services.go | 4 ++-- controllers/utils.go | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/controllers/inference_services.go b/controllers/inference_services.go index db426d4a..b49a5267 100644 --- a/controllers/inference_services.go +++ b/controllers/inference_services.go @@ -147,7 +147,7 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsByLabelForDeployments(ctx contex } // Build the payload processor endpoint - url := generateServiceURL(crName, namespace) + "/consumer/kserve/v2" + url := generateTLSServiceURL(crName, namespace) + "/consumer/kserve/v2" // Patch environment variables for the Deployments if shouldContinue, err := r.patchEnvVarsForDeployments(ctx, instance, deployments, envVarName, url, remove); err != nil { @@ -240,7 +240,7 @@ func (r *TrustyAIServiceReconciler) handleInferenceServices(ctx context.Context, // patchKServe adds a TrustyAI service as an InferenceLogger to a KServe InferenceService func (r *TrustyAIServiceReconciler) patchKServe(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, infService kservev1beta1.InferenceService, namespace string, crName string, remove bool) error { - url := generateServiceURL(crName, namespace) + url := generateNonTLSServiceURL(crName, namespace) if remove { if infService.Spec.Predictor.Logger == nil || *infService.Spec.Predictor.Logger.URL != url { diff --git a/controllers/utils.go b/controllers/utils.go index 91ee9bdb..e96fc4e5 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -62,7 +62,12 @@ func (r *TrustyAIServiceReconciler) GetDeploymentsByLabel(ctx context.Context, n return deployments.Items, nil } -// generateServiceURL generates an internal URL for a TrustyAI service -func generateServiceURL(crName string, namespace string) string { +// generateTLSServiceURL generates an internal URL for a TLS-enabled TrustyAI service +func generateTLSServiceURL(crName string, namespace string) string { + return "https://" + crName + "." + namespace + ".svc" +} + +// generateNonTLSServiceURL generates an internal URL for a TrustyAI service +func generateNonTLSServiceURL(crName string, namespace string) string { return "http://" + crName + "." + namespace + ".svc" } From d552762ee3ca7173e71932d91cad61afe2bdf80a Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Mon, 5 Aug 2024 13:37:38 +0100 Subject: [PATCH 10/11] fix: Correct maxSurge and maxUnavailable (#275) --- controllers/templates/service/deployment.tmpl.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/templates/service/deployment.tmpl.yaml b/controllers/templates/service/deployment.tmpl.yaml index fa8a9c76..da69b24e 100644 --- a/controllers/templates/service/deployment.tmpl.yaml +++ b/controllers/templates/service/deployment.tmpl.yaml @@ -15,8 +15,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxUnavailable: 0 - maxSurge: 1 + maxUnavailable: 1 + maxSurge: 0 replicas: 1 selector: matchLabels: From 4458d0d726118cd25ec97671f63f7d750ac110d4 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Wed, 7 Aug 2024 19:26:30 +0100 Subject: [PATCH 11/11] feat: Add support for custom DB names (#257) * feat: Add support for custom DB names * fix: Correct custom DB name --- controllers/deployment_test.go | 22 +++++++++++++++---- controllers/secrets.go | 9 +++++++- controllers/suite_test.go | 3 ++- .../templates/service/deployment.tmpl.yaml | 9 ++++++-- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/controllers/deployment_test.go b/controllers/deployment_test.go index de80a1c4..0aa0f925 100644 --- a/controllers/deployment_test.go +++ b/controllers/deployment_test.go @@ -359,7 +359,7 @@ var _ = Describe("TrustyAI operator", func() { namespace := "trusty-ns-a-1-db" instance = createDefaultDBCustomResource(namespace) WaitFor(func() error { - secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mysql") + secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mysql", "trustyai_service") return k8sClient.Create(ctx, secret) }, "failed to create ConfigMap") setupAndTestDeploymentDefault(instance, namespace) @@ -368,7 +368,7 @@ var _ = Describe("TrustyAI operator", func() { namespace := "trusty-ns-a-1-db" instance = createDefaultDBCustomResource(namespace) WaitFor(func() error { - secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mariadb") + secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mariadb", "trustyai_service") return k8sClient.Create(ctx, secret) }, "failed to create ConfigMap") setupAndTestDeploymentDefault(instance, namespace) @@ -584,9 +584,16 @@ var _ = Describe("TrustyAI operator", func() { Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePort"), "Secret key does not match") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_NAME") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_NAME not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_NAME does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_NAME is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseName"), "Secret key does not match") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_URL not found") - Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database")) + Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/${DATABASE_NAME}")) }) @@ -695,9 +702,16 @@ var _ = Describe("TrustyAI operator", func() { Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePort"), "Secret key does not match") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_NAME") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_NAME not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_NAME does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_NAME is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseName"), "Secret key does not match") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_URL not found") - Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database")) + Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/${DATABASE_NAME}")) }) diff --git a/controllers/secrets.go b/controllers/secrets.go index 090ab58d..499d768b 100644 --- a/controllers/secrets.go +++ b/controllers/secrets.go @@ -53,7 +53,14 @@ func (r *TrustyAIServiceReconciler) findDatabaseSecret(ctx context.Context, inst // validateDatabaseSecret validates the DB configuration secret func (r *TrustyAIServiceReconciler) validateDatabaseSecret(secret *corev1.Secret) error { - mandatoryKeys := []string{"databaseKind", "databaseUsername", "databasePassword", "databaseService", "databasePort"} + mandatoryKeys := []string{ + "databaseKind", + "databaseUsername", + "databasePassword", + "databaseService", + "databasePort", + "databaseName", + } for _, key := range mandatoryKeys { value, exists := secret.Data[key] diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 7ee75a93..e61f4ac4 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -215,13 +215,14 @@ func createSecret(namespace string, secretName string, data map[string]string) * } } -func createDatabaseConfiguration(namespace string, name string, dbKind string) *corev1.Secret { +func createDatabaseConfiguration(namespace string, name string, dbKind string, databaseName string) *corev1.Secret { return createSecret(namespace, name, map[string]string{ "databaseKind": dbKind, "databaseUsername": "foo", "databasePassword": "bar", "databaseService": "mariadb-service", "databasePort": "3306", + "databaseName": databaseName, }) } diff --git a/controllers/templates/service/deployment.tmpl.yaml b/controllers/templates/service/deployment.tmpl.yaml index da69b24e..f1e429f1 100644 --- a/controllers/templates/service/deployment.tmpl.yaml +++ b/controllers/templates/service/deployment.tmpl.yaml @@ -93,11 +93,16 @@ spec: secretKeyRef: name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} key: databasePort + - name: DATABASE_NAME + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseName - name: QUARKUS_DATASOURCE_JDBC_URL {{ if .UseDBTLSCerts }} - value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database?sslMode=verify-ca&serverSslCert=/etc/tls/db/tls.crt" + value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/${DATABASE_NAME}?sslMode=verify-ca&serverSslCert=/etc/tls/db/tls.crt" {{ else }} - value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database" + value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/${DATABASE_NAME}" {{ end }} - name: SERVICE_DATA_FORMAT value: "HIBERNATE"