Allow users to take GCS-based backups

This commit adds first-pass support for exposing Solr's GcsBackupRepository through our operator configuration. This WIP support has a number of caveats and downsides: - GCS backups eschew the "persistence" step that currently follows normal backups - GCS backups are only included in Solr 8.9+, but there's no check for this currently. - operator logic currently assumes that exactly 1 type of backup config will be provided on a given solrcloud object (i.e. GCS backups and 'local' PV backups are mutually exclusive for a solrcloud. - no automated tests have been added - no documentation of has been added, beyond the examples on issue apache#301
gerlowskija · Aug 5, 2021 · 8e6f69d · 8e6f69d
1 parent c9d556c
commit 8e6f69d
Show file tree

Hide file tree

Showing 10 changed files with 202 additions and 58 deletions.
diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go
@@ -43,7 +43,7 @@ type SolrBackupSpec struct {
 	Collections []string `json:"collections,omitempty"`
 
 	// Persistence is the specification on how to persist the backup data.
-	Persistence PersistenceSource `json:"persistence"`
+	Persistence PersistenceSource `json:"persistence,omitempty"`
 }
 
 func (spec *SolrBackupSpec) withDefaults(backupName string) (changed bool) {

diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go
@@ -331,14 +331,28 @@ type SolrBackupRestoreOptions struct {
 	// Since the volume will be mounted to all solrNodes, it must be able to be written from multiple pods.
 	// If a PVC reference is given, the PVC must have `accessModes: - ReadWriteMany`.
 	// Other options are to use a NFS volume.
-	Volume corev1.VolumeSource `json:"volume"`
+	Volume *corev1.VolumeSource `json:"volume,omitempty"`
+
+	// Configuration used to store backups in Google Cloud Storage ("GCS").
+	Gcs *GcsStorage `json:"gcsStorage,omitempty"`
 
 	// Select a custom directory name to mount the backup/restore data from the given volume.
 	// If not specified, then the name of the solrcloud will be used by default.
 	// +optional
 	Directory string `json:"directory,omitempty"`
 }
 
+type GcsStorage struct {
+	// The name of the GCS bucket that all backup data will be stored in
+	Bucket string `json:"bucket"`
+
+	// The name of a Kubernetes secret holding a Google cloud service account key
+	GcsCredentialSecret string `json:"gcsCredentialSecret"`
+	// JEGERLOW TODO Should 'baseLocation' be optional?
+	// A chroot within the bucket to store data in.  If specified this should already exist
+	BaseLocation string `json:"baseLocation"`
+}
+
 type SolrAddressabilityOptions struct {
 	// External defines the way in which this SolrCloud nodes should be made addressable externally, from outside the Kubernetes cluster.
 	// If none is provided, the Solr Cloud will not be made addressable externally.

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml
@@ -1052,7 +1052,6 @@ spec:
                 description: A reference to the SolrCloud to create a backup for
                 type: string
             required:
-            - persistence
             - solrCloud
             type: object
           status:

diff --git a/config/crd/bases/solr.apache.org_solrclouds.yaml b/config/crd/bases/solr.apache.org_solrclouds.yaml
@@ -3426,6 +3426,23 @@ spec:
                       directory:
                         description: Select a custom directory name to mount the backup/restore data from the given volume. If not specified, then the name of the solrcloud will be used by default.
                         type: string
+                      gcsStorage:
+                        description: Configuration used to store backups in Google Cloud Storage ("GCS").
+                        properties:
+                          baseLocation:
+                            description: JEGERLOW TODO Should 'baseLocation' be optional? A chroot within the bucket to store data in.  If specified this should already exist
+                            type: string
+                          bucket:
+                            description: The name of the GCS bucket that all backup data will be stored in
+                            type: string
+                          gcsCredentialSecret:
+                            description: The name of a Kubernetes secret holding a Google cloud service account key
+                            type: string
+                        required:
+                        - baseLocation
+                        - bucket
+                        - gcsCredentialSecret
+                        type: object
                       volume:
                         description: 'This is a volumeSource for a volume that will be mounted to all solrNodes to store backups and load restores. The data within the volume will be namespaces for this instance, so feel free to use the same volume for multiple clouds. Since the volume will be mounted to all solrNodes, it must be able to be written from multiple pods. If a PVC reference is given, the PVC must have `accessModes: - ReadWriteMany`. Other options are to use a NFS volume.'
                         properties:
@@ -4321,8 +4338,6 @@ spec:
                             - volumePath
                             type: object
                         type: object
-                    required:
-                    - volume
                     type: object
                   ephemeral:
                     description: "EphemeralStorage is the specification for how the ephemeral Solr data storage should be configured. \n This option cannot be used with the \"persistent\" option. Ephemeral storage is used by default if neither \"persistent\" or \"ephemeral\" is provided."

diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go
@@ -104,9 +104,20 @@ func (r *SolrBackupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error)
 		if allCollectionsComplete && !backup.Status.Finished {
 			// We will count on the Job updates to be notifified
 			requeueOrNot = reconcile.Result{}
-			err = persistSolrCloudBackups(r, backup, solrCloud)
-			if err != nil {
-				r.Log.Error(err, "Error while persisting SolrCloud backup")
+			if solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume != nil {
+				err = persistSolrCloudBackups(r, backup, solrCloud)
+				if err != nil {
+					r.Log.Error(err, "Error while persisting SolrCloud backup")
+				}
+			} else { // No volume, so no persistence.  Mark all as successful
+				tru := true
+				now := metav1.Now()
+				backup.Status.PersistenceStatus.Successful = &tru
+				backup.Status.PersistenceStatus.InProgress = false
+				backup.Status.PersistenceStatus.Finished = true
+				backup.Status.PersistenceStatus.FinishTime = &now
+				backup.Status.Finished = true
+				backup.Status.Successful = backup.Status.PersistenceStatus.Successful
 			}
 		}
 	}

diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go
@@ -631,11 +631,22 @@ func reconcileCloudStatus(r *SolrCloudReconciler, solrCloud *solr.SolrCloud, log
 			newStatus.ReadyReplicas += 1
 		}
 
+		// JEGERLOW TODO Reduce duplication here
 		// Get Volumes for backup/restore
 		if solrCloud.Spec.StorageOptions.BackupRestoreOptions != nil {
-			for _, volume := range p.Spec.Volumes {
-				if volume.Name == util.BackupRestoreVolume {
-					backupRestoreReadyPods += 1
+			// Check that the backup-data storage volume is present
+			if solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume != nil {
+				for _, volume := range p.Spec.Volumes {
+					if volume.Name == util.BackupRestoreVolume {
+						backupRestoreReadyPods += 1
+					}
+				}
+			} else if solrCloud.Spec.StorageOptions.BackupRestoreOptions.Gcs != nil {
+				// Check that the GCS credential volume is present
+				for _, volume := range p.Spec.Volumes {
+					if volume.Name == util.BackupRestoreCredentialVolume {
+						backupRestoreReadyPods += 1
+					}
 				}
 			}
 		}
@@ -676,7 +687,6 @@ func reconcileCloudStatus(r *SolrCloudReconciler, solrCloud *solr.SolrCloud, log
 	for idx, nodeName := range nodeNames {
 		newStatus.SolrNodes[idx] = nodeStatusMap[nodeName]
 	}
-
 	if backupRestoreReadyPods == int(*solrCloud.Spec.Replicas) && backupRestoreReadyPods > 0 {
 		newStatus.BackupRestoreReady = true
 	}

diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go
@@ -33,10 +33,12 @@ import (
 )
 
 const (
-	BaseBackupRestorePath = "/var/solr/data/backup-restore"
-	TarredFile            = "/var/solr/data/backup-restore/backup.tgz"
-	CleanupCommand        = " && rm -rf " + BaseBackupRestorePath + "/*"
-	BackupTarCommand      = "cd " + BaseBackupRestorePath + " && tar -czf /tmp/backup.tgz * " + CleanupCommand + " && mv /tmp/backup.tgz " + TarredFile + " && chmod -R a+rwx " + TarredFile + " && cd - && "
+	BackupRestoreCredentialDirPath  = "/backup-restore-credential"
+	BackupRestoreCredentialFilePath = BackupRestoreCredentialDirPath + "/service-account-key.json"
+	BaseBackupRestorePath           = "/var/solr/data/backup-restore"
+	TarredFile                      = "/var/solr/data/backup-restore/backup.tgz"
+	CleanupCommand                  = " && rm -rf " + BaseBackupRestorePath + "/*"
+	BackupTarCommand                = "cd " + BaseBackupRestorePath + " && tar -czf /tmp/backup.tgz * " + CleanupCommand + " && mv /tmp/backup.tgz " + TarredFile + " && chmod -R a+rwx " + TarredFile + " && cd - && "
 
 	AWSSecretDir = "/var/aws"
 
@@ -58,10 +60,17 @@ func RestoreSubPathForCloud(directoryOverride string, cloud string, restoreName
 	return BackupRestoreSubPathForCloud(directoryOverride, cloud) + "/restores/" + restoreName
 }
 
-func BackupPath(backupName string) string {
+func LocalBackupPath(backupName string) string {
 	return BaseBackupRestorePath + "/backups/" + backupName
 }
 
+func GcsBackupPath(override *string) string {
+	if override != nil {
+		return *override
+	}
+	return "/"
+}
+
 func RestorePath(backupName string) string {
 	return BaseBackupRestorePath + "/restores/" + backupName
 }
@@ -96,7 +105,7 @@ func GenerateBackupPersistenceJobForCloud(backup *solr.SolrBackup, solrCloud *so
 	var backupVolume corev1.VolumeSource
 	var solrCloudBackupDirectoryOverride string
 	if solrCloud.Spec.StorageOptions.BackupRestoreOptions != nil {
-		backupVolume = solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume
+		backupVolume = *solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume
 		solrCloudBackupDirectoryOverride = solrCloud.Spec.StorageOptions.BackupRestoreOptions.Directory
 	}
 	return GenerateBackupPersistenceJob(backup, backupVolume, BackupSubPathForCloud(solrCloudBackupDirectoryOverride, solrCloud.Name, backup.Name))
@@ -322,8 +331,12 @@ func StartBackupForCollection(cloud *solr.SolrCloud, collection string, backupNa
 	queryParams.Add("action", "BACKUP")
 	queryParams.Add("collection", collection)
 	queryParams.Add("name", collection)
-	queryParams.Add("location", BackupPath(backupName))
 	queryParams.Add("async", AsyncIdForCollectionBackup(collection, backupName))
+	if cloud.Spec.StorageOptions.BackupRestoreOptions.Volume != nil {
+		queryParams.Add("location", LocalBackupPath(backupName))
+	} else {
+		queryParams.Add("location", GcsBackupPath(&cloud.Spec.StorageOptions.BackupRestoreOptions.Gcs.BaseLocation))
+	}
 
 	resp := &solr_api.SolrAsyncResponse{}
 
@@ -387,14 +400,18 @@ func DeleteAsyncInfoForBackup(cloud *solr.SolrCloud, collection string, backupNa
 }
 
 func EnsureDirectoryForBackup(solrCloud *solr.SolrCloud, backup string, config *rest.Config) (err error) {
-	backupPath := BackupPath(backup)
-	// Create an empty directory for the backup
-	return RunExecForPod(
-		solrCloud.GetAllSolrNodeNames()[0],
-		solrCloud.Namespace,
-		[]string{"/bin/bash", "-c", "rm -rf " + backupPath + " && mkdir -p " + backupPath},
-		*config,
-	)
+	// Directory creation only required/possible for local backups using a mounted volume
+	if solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume != nil {
+		backupPath := LocalBackupPath(backup)
+		// Create an empty directory for the backup
+		return RunExecForPod(
+			solrCloud.GetAllSolrNodeNames()[0],
+			solrCloud.Namespace,
+			[]string{"/bin/bash", "-c", "rm -rf " + backupPath + " && mkdir -p " + backupPath},
+			*config,
+		)
+	}
+	return nil
 }
 
 func RunExecForPod(podName string, namespace string, command []string, config rest.Config) (err error) {

diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
@@ -39,6 +39,8 @@ import (
 const (
 	SolrClientPortName  = "solr-client"
 	BackupRestoreVolume = "backup-restore"
+	BackupRestoreCredentialVolume = "backup-restore-credential"
+	BackupRestoreCredentialSecretKey = "service-account-key.json"
 
 	SolrNodeContainer = "solrcloud-node"
 
@@ -215,16 +217,37 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
 	}
 	// Add backup volumes
 	if solrCloud.Spec.StorageOptions.BackupRestoreOptions != nil {
-		solrVolumes = append(solrVolumes, corev1.Volume{
-			Name:         BackupRestoreVolume,
-			VolumeSource: solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume,
-		})
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      BackupRestoreVolume,
-			MountPath: BaseBackupRestorePath,
-			SubPath:   BackupRestoreSubPathForCloud(solrCloud.Spec.StorageOptions.BackupRestoreOptions.Directory, solrCloud.Name),
-			ReadOnly:  false,
-		})
+		// Backups can be configured to write to either a local drive, or to GCS.  Each requires a volume
+		if solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume != nil {
+			solrVolumes = append(solrVolumes, corev1.Volume{
+				Name:         BackupRestoreVolume,
+				VolumeSource: *solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume,
+			})
+			volumeMounts = append(volumeMounts, corev1.VolumeMount{
+				Name:      BackupRestoreVolume,
+				MountPath: BaseBackupRestorePath,
+				SubPath:   BackupRestoreSubPathForCloud(solrCloud.Spec.StorageOptions.BackupRestoreOptions.Directory, solrCloud.Name),
+				ReadOnly:  false,
+			})
+		} else if solrCloud.Spec.StorageOptions.BackupRestoreOptions.Gcs != nil {
+			fals := false
+			solrVolumes = append(solrVolumes, corev1.Volume{
+				Name: BackupRestoreCredentialVolume,
+				VolumeSource: corev1.VolumeSource {
+					Secret: &corev1.SecretVolumeSource {
+						SecretName: solrCloud.Spec.StorageOptions.BackupRestoreOptions.Gcs.GcsCredentialSecret,
+						Items: []corev1.KeyToPath{{Key: BackupRestoreCredentialSecretKey, Path: BackupRestoreCredentialSecretKey}},
+						Optional: &fals,
+					},
+				},
+				// the local baackup stuff has this handled by the user, but for the credential I need to derive it from the secret.
+			})
+			volumeMounts = append(volumeMounts, corev1.VolumeMount{
+				Name:      BackupRestoreCredentialVolume,
+				MountPath: BackupRestoreCredentialDirPath,
+				ReadOnly:  true,
+			})
+		}
 	}
 
 	if nil != customPodOptions {
@@ -602,14 +625,16 @@ func generateSolrSetupInitContainers(solrCloud *solr.SolrCloud, solrCloudStatus
 	// Add prep for backup-restore volume
 	// This entails setting the correct permissions for the directory
 	if solrCloud.Spec.StorageOptions.BackupRestoreOptions != nil {
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      BackupRestoreVolume,
-			MountPath: "/backup-restore",
-			SubPath:   BackupRestoreSubPathForCloud(solrCloud.Spec.StorageOptions.BackupRestoreOptions.Directory, solrCloud.Name),
-			ReadOnly:  false,
-		})
+		if solrCloud.Spec.StorageOptions.BackupRestoreOptions.Volume != nil {
+			volumeMounts = append(volumeMounts, corev1.VolumeMount{
+				Name:      BackupRestoreVolume,
+				MountPath: "/backup-restore",
+				SubPath:   BackupRestoreSubPathForCloud(solrCloud.Spec.StorageOptions.BackupRestoreOptions.Directory, solrCloud.Name),
+				ReadOnly:  false,
+			})
 
-		setupCommands = append(setupCommands, fmt.Sprintf("chown -R %d:%d /backup-restore", DefaultSolrUser, DefaultSolrGroup))
+			setupCommands = append(setupCommands, fmt.Sprintf("chown -R %d:%d /backup-restore", DefaultSolrUser, DefaultSolrGroup))
+		}
 	}
 
 	volumePrepInitContainer := corev1.Container{
@@ -641,15 +666,7 @@ func GenerateConfigMap(solrCloud *solr.SolrCloud) *corev1.ConfigMap {
 		annotations = MergeLabelsOrAnnotations(annotations, customOptions.Annotations)
 	}
 
-	configMap := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        solrCloud.ConfigMapName(),
-			Namespace:   solrCloud.GetNamespace(),
-			Labels:      labels,
-			Annotations: annotations,
-		},
-		Data: map[string]string{
-			"solr.xml": `<?xml version="1.0" encoding="UTF-8" ?>
+	solrXml := `<?xml version="1.0" encoding="UTF-8" ?>
 <solr>
   <solrcloud>
     <str name="host">${host:}</str>
@@ -668,7 +685,30 @@ func GenerateConfigMap(solrCloud *solr.SolrCloud) *corev1.ConfigMap {
     <int name="connTimeout">${connTimeout:60000}</int>
   </shardHandlerFactory>
 </solr>
-`,
+`
+	// Inject GCS backup configuration into solr.xml if needed
+	if solrCloud.Spec.StorageOptions.BackupRestoreOptions != nil && solrCloud.Spec.StorageOptions.BackupRestoreOptions.Gcs != nil {
+		gcsXmlConfig :=`
+<str name="sharedLib">/opt/solr/dist,/opt/solr/contrib/gcs-repository/lib</str>
+<backup>
+  <repository name="gcs_backup" class="org.apache.solr.gcs.GCSBackupRepository" default="true">
+    <str name="gcsBucket">` + solrCloud.Spec.StorageOptions.BackupRestoreOptions.Gcs.Bucket + `</str>
+    <str name="gcsCredentialPath">` + BackupRestoreCredentialFilePath + `</str>
+  </repository>
+</backup>
+`
+		solrXml = strings.Replace(solrXml, "</solr>", gcsXmlConfig + "</solr>", 1)
+	}
+
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        solrCloud.ConfigMapName(),
+			Namespace:   solrCloud.GetNamespace(),
+			Labels:      labels,
+			Annotations: annotations,
+		},
+		Data: map[string]string{
+			"solr.xml": solrXml,
 		},
 	}