feat: persistent volumes work on multi node k8s clusters (#1943)

feat: use storage-class in the kurtosis-config.yaml; users should make sure that they have the right storage class configured in their `kurtosis-config.yaml`
kurtosis-tech · Dec 15, 2023 · b2fd9f2 · b2fd9f2
1 parent 9cc5f77
commit b2fd9f2
Show file tree

Hide file tree

Showing 20 changed files with 78 additions and 297 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -59,7 +59,7 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s
               type: kubernetes
               config:
                 kubernetes-cluster-name: k3d-k3s-default
-                storage-class: standard
+                storage-class: local-path
                 enclave-size-in-megabytes: 2048
           EOF
     # Set the K3S cluster with some previous steps to force this cluster type in the cluster-setting file. This save us to start the engine with the cluster set command

diff --git a/cli/cli/kurtosis_config/resolved_config/kurtosis_cluster_config.go b/cli/cli/kurtosis_config/resolved_config/kurtosis_cluster_config.go
@@ -145,7 +145,7 @@ func getSuppliers(clusterId string, clusterType KurtosisClusterType, kubernetesC
 		}
 
 		backendSupplier = func(ctx context.Context) (backend_interface.KurtosisBackend, error) {
-			backend, err := kubernetes_kurtosis_backend.GetCLIBackend(ctx)
+			backend, err := kubernetes_kurtosis_backend.GetCLIBackend(ctx, *kubernetesConfig.StorageClass)
 			if err != nil {
 				return nil, stacktrace.Propagate(
 					err,

diff --git a/cli/cli/kurtosis_gateway/connection/provider.go b/cli/cli/kurtosis_gateway/connection/provider.go
@@ -22,6 +22,8 @@ import (
 const (
 	grpcPortIdStr           = "grpc"
 	httpApplicationProtocol = "http"
+	// this doesn't have any effect as this is just the gateway
+	emptyStorageClassName = ""
 )
 
 var noWait *port_spec.Wait = nil
@@ -42,7 +44,7 @@ func NewGatewayConnectionProvider(ctx context.Context, kubernetesConfig *restcli
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "Expected to be able to get config for Kubernetes client set, instead a non nil error was returned")
 	}
-	kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig)
+	kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig, emptyStorageClassName)
 
 	return &GatewayConnectionProvider{
 		config:                          kubernetesConfig,

diff --git a/...b/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go b/...b/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go
@@ -67,8 +67,9 @@ func NewAPIContainerKubernetesKurtosisBackend(
 	kubernetesManager *kubernetes_manager.KubernetesManager,
 	ownEnclaveUuid enclave.EnclaveUUID,
 	ownNamespaceName string,
+	storageClassName string,
 ) *KubernetesKurtosisBackend {
-	modeArgs := shared_helpers.NewApiContainerModeArgs(ownEnclaveUuid, ownNamespaceName)
+	modeArgs := shared_helpers.NewApiContainerModeArgs(ownEnclaveUuid, ownNamespaceName, storageClassName)
 	return newKubernetesKurtosisBackend(
 		kubernetesManager,
 		nil,

diff --git a/...s/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_enclave_functions.go b/...s/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_enclave_functions.go
@@ -36,8 +36,6 @@ type enclaveKubernetesResources struct {
 	// StopEnclave
 	services []apiv1.Service
 
-	persistentVolumes []apiv1.PersistentVolume
-
 	clusterRoles []rbacv1.ClusterRole
 
 	clusterRoleBindings []rbacv1.ClusterRoleBinding
@@ -101,7 +99,6 @@ func (backend *KubernetesKurtosisBackend) CreateEnclave(
 		namespace:           enclaveNamespace,
 		pods:                []apiv1.Pod{},
 		services:            nil,
-		persistentVolumes:   []apiv1.PersistentVolume{},
 		clusterRoles:        []rbacv1.ClusterRole{},
 		clusterRoleBindings: []rbacv1.ClusterRoleBinding{},
 	}
@@ -299,21 +296,6 @@ func (backend *KubernetesKurtosisBackend) DestroyEnclaves(
 			}
 		}
 
-		// Remove persistent volume
-		if resources.persistentVolumes != nil {
-			for _, persistentVolume := range resources.persistentVolumes {
-				if err := backend.kubernetesManager.RemovePersistentVolume(ctx, persistentVolume.Name); err != nil {
-					erroredEnclaveIds[enclaveId] = stacktrace.Propagate(
-						err,
-						"An error occurred removing persistent volume '%v' for enclave '%v'",
-						persistentVolume.Name,
-						enclaveId,
-					)
-					continue
-				}
-			}
-		}
-
 		// Remove custom API container Cluster Role Bindings
 		if resources.clusterRoleBindings != nil {
 			for _, clusterRoleBinding := range resources.clusterRoleBindings {
@@ -519,7 +501,7 @@ func (backend *KubernetesKurtosisBackend) createGetEnclaveResourcesOperation(
 		}
 
 		// Pods and Services
-		podsList, servicesList, persistentVolumesList, clusterRolesList, clusterRoleBindingsList, err := backend.kubernetesManager.GetAllEnclaveResourcesByLabels(
+		podsList, servicesList, clusterRolesList, clusterRoleBindingsList, err := backend.kubernetesManager.GetAllEnclaveResourcesByLabels(
 			ctx,
 			namespaceName,
 			enclaveWithIDMatchLabels,
@@ -534,9 +516,6 @@ func (backend *KubernetesKurtosisBackend) createGetEnclaveResourcesOperation(
 		var services []apiv1.Service
 		services = append(services, servicesList.Items...)
 
-		var persistentVolumes []apiv1.PersistentVolume
-		persistentVolumes = append(persistentVolumes, persistentVolumesList.Items...)
-
 		var clusterRoles []rbacv1.ClusterRole
 		clusterRoles = append(clusterRoles, clusterRolesList.Items...)
 
@@ -547,7 +526,6 @@ func (backend *KubernetesKurtosisBackend) createGetEnclaveResourcesOperation(
 			namespace:           namespace,
 			pods:                pods,
 			services:            services,
-			persistentVolumes:   persistentVolumes,
 			clusterRoles:        clusterRoles,
 			clusterRoleBindings: clusterRoleBindings,
 		}

diff --git a/...ackend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_helper.go b/...ackend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_helper.go
@@ -21,7 +21,7 @@ var kubeConfigFileFilepath = filepath.Join(
 	os.Getenv("HOME"), ".kube", "config",
 )
 
-func GetCLIBackend(ctx context.Context) (backend_interface.KurtosisBackend, error) {
+func GetCLIBackend(ctx context.Context, storageClass string) (backend_interface.KurtosisBackend, error) {
 	kubernetesConfig, err := clientcmd.BuildConfigFromFlags(emptyMasterURL, kubeConfigFileFilepath)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred creating kubernetes configuration from flags in file '%v'", kubeConfigFileFilepath)
@@ -35,6 +35,7 @@ func GetCLIBackend(ctx context.Context) (backend_interface.KurtosisBackend, erro
 		ctx,
 		kubernetesConfig,
 		backendSupplier,
+		storageClass,
 	)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred wrapping the CLI Kubernetes backend")
@@ -44,7 +45,7 @@ func GetCLIBackend(ctx context.Context) (backend_interface.KurtosisBackend, erro
 }
 
 func GetEngineServerBackend(
-	ctx context.Context,
+	ctx context.Context, storageClass string,
 ) (backend_interface.KurtosisBackend, error) {
 	kubernetesConfig, err := rest.InClusterConfig()
 	if err != nil {
@@ -61,6 +62,7 @@ func GetEngineServerBackend(
 		ctx,
 		kubernetesConfig,
 		backendSupplier,
+		storageClass,
 	)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred wrapping the Kurtosis Engine Kubernetes backend")
@@ -71,6 +73,7 @@ func GetEngineServerBackend(
 
 func GetApiContainerBackend(
 	ctx context.Context,
+	storageClass string,
 ) (backend_interface.KurtosisBackend, error) {
 	kubernetesConfig, err := rest.InClusterConfig()
 	if err != nil {
@@ -103,13 +106,15 @@ func GetApiContainerBackend(
 			kubernetesManager,
 			enclaveId,
 			namespaceName,
+			storageClass,
 		), nil
 	}
 
 	wrappedBackend, err := getWrappedKubernetesKurtosisBackend(
 		ctx,
 		kubernetesConfig,
 		backendSupplier,
+		storageClass,
 	)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred wrapping the APIC Kubernetes backend")
@@ -127,13 +132,14 @@ func getWrappedKubernetesKurtosisBackend(
 	ctx context.Context,
 	kubernetesConfig *rest.Config,
 	kurtosisBackendSupplier func(context.Context, *kubernetes_manager.KubernetesManager) (*KubernetesKurtosisBackend, error),
+	storageClass string,
 ) (*metrics_reporting.MetricsReportingKurtosisBackend, error) {
 	clientSet, err := kubernetes.NewForConfig(kubernetesConfig)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "Expected to be able to create kubernetes client set using Kubernetes config '%+v', instead a non nil error was returned", kubernetesConfig)
 	}
 
-	kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig)
+	kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig, storageClass)
 
 	kubernetesBackend, err := kurtosisBackendSupplier(ctx, kubernetesManager)
 	if err != nil {

diff --git a/...lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers/shared_helpers.go b/...lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers/shared_helpers.go
@@ -102,11 +102,11 @@ type dumpPodResult struct {
 
 func NewApiContainerModeArgs(
 	ownEnclaveId enclave.EnclaveUUID,
-	ownNamespaceName string) *ApiContainerModeArgs {
+	ownNamespaceName string, storageClassName string) *ApiContainerModeArgs {
 	return &ApiContainerModeArgs{
 		ownEnclaveId:     ownEnclaveId,
 		ownNamespaceName: ownNamespaceName,
-		storageClassName: "",
+		storageClassName: storageClassName,
 		filesArtifactExpansionVolumeSizeInMegabytes: 0,
 	}
 }

diff --git a/.../kubernetes/kubernetes_kurtosis_backend/user_services_functions/persistent_directories.go b/.../kubernetes/kubernetes_kurtosis_backend/user_services_functions/persistent_directories.go
@@ -12,14 +12,12 @@ import (
 )
 
 type kubernetesVolumeWithClaim struct {
-	VolumeName string
-
 	VolumeClaimName string
 }
 
 func (volumeAndClaim *kubernetesVolumeWithClaim) GetVolume() *apiv1.Volume {
 	return &apiv1.Volume{
-		Name: volumeAndClaim.VolumeName,
+		Name: volumeAndClaim.VolumeClaimName,
 		VolumeSource: apiv1.VolumeSource{
 			HostPath:             nil,
 			EmptyDir:             nil,
@@ -59,7 +57,7 @@ func (volumeAndClaim *kubernetesVolumeWithClaim) GetVolume() *apiv1.Volume {
 
 func (volumeAndClaim *kubernetesVolumeWithClaim) GetVolumeMount(mountPath string) *apiv1.VolumeMount {
 	return &apiv1.VolumeMount{
-		Name:             volumeAndClaim.VolumeName,
+		Name:             volumeAndClaim.VolumeClaimName,
 		ReadOnly:         false,
 		MountPath:        mountPath,
 		SubPath:          "",
@@ -77,7 +75,6 @@ func preparePersistentDirectoriesResources(
 	kubernetesManager *kubernetes_manager.KubernetesManager,
 ) (map[string]*kubernetesVolumeWithClaim, error) {
 	shouldDeleteVolumesAndClaimsCreated := true
-	volumesCreated := map[string]*apiv1.PersistentVolume{}
 	volumeClaimsCreated := map[string]*apiv1.PersistentVolumeClaim{}
 
 	persistentVolumesAndClaims := map[string]*kubernetesVolumeWithClaim{}
@@ -96,16 +93,7 @@ func preparePersistentDirectoriesResources(
 
 		persistentVolumeSize := int64(persistentDirectory.Size)
 
-		var persistentVolume *apiv1.PersistentVolume
-		if persistentVolume, err = kubernetesManager.GetPersistentVolume(ctx, volumeName); err != nil {
-			persistentVolume, err = kubernetesManager.CreatePersistentVolume(ctx, namespace, volumeName, volumeLabelsStrs, persistentVolumeSize)
-			if err != nil {
-				return nil, stacktrace.Propagate(err, "An error occurred creating the persistent volume for '%s'", persistentDirectory.PersistentKey)
-			}
-			volumesCreated[persistentVolume.Name] = persistentVolume
-		}
-
-		// For now, we have a 1:1 mapping between volume and volume claims, so it's fine giving it the same name
+		// This claim works with a dynamic driver - it will spin up its own volume - the volume will get deleted when said claims is deleted
 		var persistentVolumeClaim *apiv1.PersistentVolumeClaim
 		if persistentVolumeClaim, err = kubernetesManager.GetPersistentVolumeClaim(ctx, namespace, volumeName); err != nil {
 			persistentVolumeClaim, err = kubernetesManager.CreatePersistentVolumeClaim(ctx, namespace, volumeName, volumeLabelsStrs, persistentVolumeSize)
@@ -116,7 +104,6 @@ func preparePersistentDirectoriesResources(
 		}
 
 		persistentVolumesAndClaims[dirPath] = &kubernetesVolumeWithClaim{
-			VolumeName:      persistentVolume.Name,
 			VolumeClaimName: persistentVolumeClaim.Name,
 		}
 	}
@@ -136,17 +123,6 @@ func preparePersistentDirectoriesResources(
 				logrus.Warnf("You'll need to clean up volume claim '%v' manually!", volumeClaimNameStr)
 			}
 		}
-		for volumeNameStr := range volumesCreated {
-			// Background context so we still run this even if the input context was cancelled
-			if err := kubernetesManager.RemovePersistentVolumeClaim(context.Background(), namespace, volumeNameStr); err != nil {
-				logrus.Warnf(
-					"Creating persistent directory volumes didn't complete successfully so we tried to delete volume '%v' that we created, but doing so threw an error:\n%v",
-					volumeNameStr,
-					err,
-				)
-				logrus.Warnf("You'll need to clean up volume '%v' manually!", volumeNameStr)
-			}
-		}
 	}()
 
 	shouldDeleteVolumesAndClaimsCreated = false

diff --git a/...pls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go b/...pls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go
@@ -372,15 +372,10 @@ func createStartServiceOperation(
 			}
 			for _, volumeAndClaim := range createVolumesWithClaims {
 				volumeClaimName := volumeAndClaim.VolumeClaimName
-				volumeName := volumeAndClaim.VolumeName
 				if err := kubernetesManager.RemovePersistentVolumeClaim(ctx, namespaceName, volumeClaimName); err != nil {
 					logrus.Errorf("Starting service didn't complete successfully so we tried to remove the persistent volume claim we created but doing so threw an error:\n%v", err)
 					logrus.Errorf("ACTION REQUIRED: You'll need to remove persistent volume claim '%v' in '%v' manually!!!", volumeClaimName, namespaceName)
 				}
-				if err := kubernetesManager.RemovePersistentVolume(ctx, volumeAndClaim.VolumeName); err != nil {
-					logrus.Errorf("Starting service didn't complete successfully so we tried to remove the persistent volume we created but doing so threw an error:\n%v", err)
-					logrus.Errorf("ACTION REQUIRED: You'll need to remove persistent volume '%v' manually!!!", volumeName)
-				}
 			}
 		}()