Dc backup restore (#325)

deploy/crds/db.orange.com_cassandrabackups_crd.yaml

@@ -80,7 +80,6 @@ spec:
                 type: string
             required:
             - cassandraCluster
-            - datacenter
             - snapshotTag
             - storageLocation
             type: object

deploy/crds/db.orange.com_cassandrarestores_crd.yaml

@@ -48,6 +48,10 @@ spec:
                   the cloud. Defaults to 10
                 format: int32
                 type: integer
+              datacenter:
+                description: Cassandra DC name to restore, used to find the cassandra
+                  nodes in the CassandraCluster
+                type: string
               entities:
                 description: Database entities to restore, it might be either only
                   keyspaces or only tables prefixed by their respective keyspace,

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-openapi/spec v0.19.4
 	github.com/golang/protobuf v1.4.2 // indirect
-	github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20210115143219-869427dc2595
+	github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20210427160512-5264f1cbba08
 	github.com/jarcoal/httpmock v1.0.4
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2

go.sum

@@ -579,6 +579,8 @@ github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20201103172213-3e9c43
 github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20201103172213-3e9c434b280c/go.mod h1:2+9I3yZFu2UU6G+fRrnJqUH9tl1iq2W3dCUkjzQTMBM=
 github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20210115143219-869427dc2595 h1:wbD83uZxDmoTmO4F4kmSL0yba6vYgHsyvjEIRWxSdzU=
 github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20210115143219-869427dc2595/go.mod h1:2+9I3yZFu2UU6G+fRrnJqUH9tl1iq2W3dCUkjzQTMBM=
+github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20210427160512-5264f1cbba08 h1:kUQehfMbAgZR4PklnZ5YVAj7a5DoWf9tIpVbZud22xU=
+github.com/instaclustr/instaclustr-icarus-go-client v0.0.0-20210427160512-5264f1cbba08/go.mod h1:2+9I3yZFu2UU6G+fRrnJqUH9tl1iq2W3dCUkjzQTMBM=
 github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ=
 github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
 github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=

helm/cassandra-operator/crds/db.orange.com_cassandrabackups_crd.yaml

@@ -80,7 +80,6 @@ spec:
                 type: string
             required:
             - cassandraCluster
-            - datacenter
             - snapshotTag
             - storageLocation
             type: object

helm/cassandra-operator/crds/db.orange.com_cassandrarestores_crd.yaml

@@ -48,6 +48,10 @@ spec:
                   the cloud. Defaults to 10
                 format: int32
                 type: integer
+              datacenter:
+                description: Cassandra DC name to restore, used to find the cassandra
+                  nodes in the CassandraCluster
+                type: string
               entities:
                 description: Database entities to restore, it might be either only
                   keyspaces or only tables prefixed by their respective keyspace,

multi-casskop/helm/multi-casskop/crds/db.orange.com_cassandrabackups_crd.yaml

@@ -80,7 +80,6 @@ spec:
                 type: string
             required:
             - cassandraCluster
-            - datacenter
             - snapshotTag
             - storageLocation
             type: object

multi-casskop/helm/multi-casskop/crds/db.orange.com_cassandrarestores_crd.yaml

@@ -48,6 +48,10 @@ spec:
                   the cloud. Defaults to 10
                 format: int32
                 type: integer
+              datacenter:
+                description: Cassandra DC name to restore, used to find the cassandra
+                  nodes in the CassandraCluster
+                type: string
               entities:
                 description: Database entities to restore, it might be either only
                   keyspaces or only tables prefixed by their respective keyspace,

pkg/apis/db/v1alpha1/cassandrabackup_types.go

@@ -23,7 +23,7 @@ type CassandraBackupSpec struct {
 	// Name of the CassandraCluster to backup
 	CassandraCluster 	  string `json:"cassandraCluster"`
 	// Cassandra DC name to back up, used to find the cassandra nodes in the CassandraCluster
-	Datacenter 		 	  string `json:"datacenter"`
+	Datacenter 		 	  string `json:"datacenter,omitempty"`
 	// URI for the backup target location e.g. s3 bucket, filepath
 	StorageLocation  	  string `json:"storageLocation"`
 	// Specify a schedule to assigned to the backup. The schedule doesn't enforce anything so if you schedule multiple

pkg/apis/db/v1alpha1/cassandracluster_types.go

@@ -36,7 +36,7 @@ const (
 
 	defaultCassandraImage     = "cassandra:3.11"
 	defaultBootstrapImage     = "orangeopensource/cassandra-bootstrap:0.1.8"
-	DefaultBackRestImage      = "gcr.io/cassandra-operator/instaclustr-icarus:1.0.9"
+	DefaultBackRestImage      = "gcr.io/cassandra-operator/instaclustr-icarus:1.1.0"
 	defaultServiceAccountName = "cassandra-cluster-node"
 	InitContainerCmd          = "cp -vr /etc/cassandra/* /bootstrap"
 	defaultMaxPodUnavailable  = 1

pkg/apis/db/v1alpha1/cassandrarestore_types.go

@@ -49,6 +49,8 @@ func (r RestoreConditionType) IsCompleted() bool {
 type CassandraRestoreSpec struct {
 	// Name of the CassandraCluster the restore belongs to
 	CassandraCluster 		string `json:"cassandraCluster"`
+	// Cassandra DC name to restore, used to find the cassandra nodes in the CassandraCluster
+	Datacenter 		 	  string `json:"datacenter,omitempty"`
 	// Name of the CassandraBackup to restore
 	CassandraBackup 		string `json:"cassandraBackup"`
 	// Maximum number of threads used to download files from the cloud. Defaults to 10

pkg/apis/db/v1alpha1/cassandrarestore_types_test.go

@@ -10,7 +10,6 @@ import (
 )
 
 const (
-	state             = "PENDING"
 	stateGetById      = "RUNNING"
 	operationID       = "d3262073-8101-450f-9a11-c851760abd57"
 	k8sSecretName     = "cloud-backup-secrets"

pkg/backrest/backrest.go

@@ -54,6 +54,7 @@ func (c *Client) PerformRestore(restore *api.CassandraRestore,
 	backup *api.CassandraBackup) (*api.BackRestStatus, error) {
 	restoreOperationRequest := &icarus.RestoreOperationRequest {
 		Type_: "restore",
+		Dc: restore.Spec.Datacenter,
 		StorageLocation: backup.Spec.StorageLocation,
 		SnapshotTag: backup.Spec.SnapshotTag,
 		NoDeleteTruncates: restore.Spec.NoDeleteTruncates,

pkg/controller/cassandrarestore/reconcile.go

@@ -101,7 +101,7 @@ func (r ReconcileCassandraRestore) Reconcile(request reconcile.Request) (reconci
 		r.recorder.Event(cassandraRestore,
 			v1.EventTypeNormal,
 			"RestoreRequired",
-			r.restoreEventMessage(cassandraBackup, ""))
+			r.restoreEventMessage(cassandraBackup,  cassandraRestore.Spec.Datacenter,""))
 		return common.Reconciled()
 	}
 
@@ -116,7 +116,7 @@ func (r ReconcileCassandraRestore) Reconcile(request reconcile.Request) (reconci
 					cassandraRestore,
 					v1.EventTypeWarning,
 					"PerformRestoreOperationFailed",
-					r.restoreEventMessage(cassandraBackup, " failed to run, will retry"))
+					r.restoreEventMessage(cassandraBackup, cassandraRestore.Spec.Datacenter, " failed to run, will retry"))
 				return controllerruntime.Result{
 					RequeueAfter: time.Duration(15) * time.Second,
 				}, nil
@@ -127,7 +127,7 @@ func (r ReconcileCassandraRestore) Reconcile(request reconcile.Request) (reconci
 		r.recorder.Event(cassandraRestore,
 			v1.EventTypeNormal,
 			"RestoreInitiated",
-			r.restoreEventMessage(cassandraBackup, ""))
+			r.restoreEventMessage(cassandraBackup, cassandraRestore.Spec.Datacenter, ""))
 
 		return common.Reconciled()
 	}
@@ -148,7 +148,7 @@ func (r ReconcileCassandraRestore) Reconcile(request reconcile.Request) (reconci
 				r.recorder.Event(cassandraRestore,
 					v1.EventTypeNormal,
 					"RestoreFailed",
-					r.restoreEventMessage(cassandraBackup, err.Error()))
+					r.restoreEventMessage(cassandraBackup, cassandraRestore.Spec.Datacenter, err.Error()))
 				return common.Reconciled()
 			default:
 				return common.RequeueWithError(reqLogger, err.Error(), err)
@@ -157,16 +157,16 @@ func (r ReconcileCassandraRestore) Reconcile(request reconcile.Request) (reconci
 		r.recorder.Event(cassandraRestore,
 			v1.EventTypeNormal,
 			"RestoreCompleted",
-			r.restoreEventMessage(cassandraBackup, ""))
+			r.restoreEventMessage(cassandraBackup, cassandraRestore.Spec.Datacenter, ""))
 	}
 	return common.Reconciled()
 }
 
 func (r ReconcileCassandraRestore) restoreEventMessage(cassandraBackup *v1alpha1.CassandraBackup,
-	message string) string {
+	datacenter string, message string) string {
 	return fmt.Sprintf("Restore of backup %s of datacenter %s of cluster %s to %s " +
 		"under snapshot %s. %s", cassandraBackup.Name,
-		cassandraBackup.Spec.Datacenter, cassandraBackup.Spec.CassandraCluster, cassandraBackup.Spec.StorageLocation,
+		datacenter, cassandraBackup.Spec.CassandraCluster, cassandraBackup.Spec.StorageLocation,
 		cassandraBackup.Spec.SnapshotTag, message)
 }
 

pkg/k8s/util.go

@@ -53,7 +53,10 @@ func LabelsForCassandraDC(cc *api.CassandraCluster, dcName string) map[string]st
 	m := map[string]string{
 		"app":                                "cassandracluster",
 		"cassandracluster":                   cc.GetName(),
-		"cassandraclusters.db.orange.com.dc": dcName,
+	}
+
+	if len(dcName) > 0{
+		m["cassandraclusters.db.orange.com.dc"] = dcName
 	}
 	return MergeLabels(cc.GetLabels(), m)
 }

website/docs/5_operations/3_5_backup_restore.md

@@ -99,3 +99,10 @@ With the object above, table k1.t1 will be restored under k1.t2 using the backup
 
 In the restore phase, you can specify a subset of the entities specified in the backup. For instance, you can backup 2
 tables and only restore one.
+
+### Datacenter
+
+It can be specified in a backup or a restore and declares where data must be backed up or restored. If not specified it 
+will run everywhere and entities must exist if they're specified. Specifying it in a restore will declare where data 
+will be restored but as icarus truncates entities it restores, it won't prevent the truncate from cleaning data in non 
+chosen datacenters.

website/docs/6_references/5_cassandra_backup.md

@@ -35,7 +35,7 @@ spec:
 |bandwidth|string|Specify the bandwidth to not exceed when uploading files to the cloud. Format supported is \d+[KMG] case insensitive. You can use values like 10M (meaning 10MB), 1024, 1024K, 2G, etc...|no|-|
 |cassandraCluster|string|Name of the CassandraCluster to backup|Yes|-|
 |concurrentConnections|int32|Maximum number of threads used to download files from the cloud. Defaults to 10|No|-|
-|datacenter|string|Cassandra DC name to back up, used to find the cassandra nodes in the CassandraCluster|Yes|-|
+|datacenter|string|Cassandra DC name to back up, used to find the cassandra nodes in the CassandraCluster|No|-|
 |duration|string|Specify a duration the backup should try to last. See https://golang.org/pkg/time/#ParseDuration for an exhaustive list of the supported units. You can use values like .25h, 15m, 900s all meaning 15 minutes|No|-|
 |entities|string|Database entities to backup, it might be either only keyspaces or only tables prefixed by their respective keyspace, e.g. 'k1,k2' if one wants to backup whole keyspaces or 'ks1.t1,ks2.t2' if one wants to restore specific tables. These formats are mutually exclusive so 'k1,k2.t2' is invalid. An empty field will backup all keyspaces|No|-|
 |schedule|string|Specify a schedule to assigned to the backup. The schedule doesn't enforce anything so if you schedule multiple backups around the same time they would conflict. See https://godoc.org/github.com/robfig/cron for more information regarding the supported formats|No|-|

website/docs/6_references/6_cassandra_restore.md

@@ -32,6 +32,7 @@ spec:
 |cassandraBackup|string|Name of the [CassandraBackup](/casskop/docs/6_references/5_cassandra_backup) to restore|Yes|-|
 |cassandraCluster|string|Name of the CassandraCluster the restore belongs to|Yes|-|
 |cassandraDirectory|string|Directory of Cassandra where data folder resides. Defaults to /var/lib/cassandra|No|-|
+|datacenter|string|Cassandra DC name to restore to, a restore will truncate tables but restore only to this datacenter if specified|No|-|
 |concurrentConnection|int32|Maximum number of threads used to download files from the cloud. Defaults to 10|No|-|
 |entities|string|Database entities to restore, it might be either only keyspaces or only tables prefixed by their respective keyspace, e.g. 'k1,k2' if one wants to backup whole keyspaces or 'ks1.t1,ks2.t2' if one wants to restore specific tables. These formats are mutually exclusive so 'k1,k2.t2' is invalid. An empty field will restore all keyspaces|No|-|
 |exactSchemaVersion|boolean|When set a running node's schema version must match the snapshot's schema version. There might be cases when we want to restore a table for which its CQL schema has not changed but it has changed for other table / keyspace but a schema for that node has changed by doing that. Defaults to False|No|false|