Reaper sidecar (#116)

* initial support for reaper sidecar mode

* debug deployment issues

* using the PodIP for mgmt api calls

* calculate liveness probe init delay based on cluster size

This commit also updates the role binding for helm chart

* create service to expose reaper ui

* fix test failure

* bring back GetPodFromHost function

* add reaper field to CRD

* add checks to see if reaper is enabled and add unit tests for schema job

* add test for buildInitReaperSchemaJob

* add unit tests for reaper service

* bump mgmt api image version

Switch to used mgmt api image v0.1.4 to get the fix for
k8ssandra/management-api-for-apache-cassandra#18 which
was blocking me. Without the fix, I could not deploy a CassandraDatacenter and
then enable Reaper.

Due to the changes in
k8ssandra/management-api-for-apache-cassandra@1a225f9#diff-3aaf645b2133cebc86740ab63150b507
I also had to add the IGNORE_DEFAULTS=true env var to the cassandra container
in order for seeds to get configured properly.

* add integration test and delete service when reaper is disabled

The integration test does the following:

* stand up a cluster
* enable reaper
* verify cluster reaches ready state
* verify that pods include reaper containers
* disable reaper
* verify cluster reaches ready state
* verify pods do not include reaper containers

The CheckReaperService function has been refactored to delete the service if
reaper is disabled.

* switch default image back to official reaper image

* fix unit test failures

* updates from PR review

* fix bad merge after rebase and add integration test

* removed env var that is no longer needed

* add nil checks since the Reaper field is now a pointer and fix missing back tick

charts/cass-operator-chart/templates/customresourcedefinition.yaml

@@ -5896,6 +5896,17 @@ spec:
                 - name
                 type: object
               type: array
+            reaper:
+              properties:
+                enabled:
+                  type: boolean
+                image:
+                  type: string
+                imagePullPolicy:
+                  description: PullPolicy describes a policy for if/when to pull a
+                    container image
+                  type: string
+              type: object
             replaceNodes:
               description: A list of pod names that need to be replaced.
               items:

charts/cass-operator-chart/templates/role.yaml

@@ -63,3 +63,9 @@ rules:
   - '*'
   verbs:
   - '*'
+- apiGroups:
+    - batch
+  resources:
+    - '*'
+  verbs:
+    - '*'

operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml

@@ -5886,6 +5886,17 @@ spec:
                 - name
                 type: object
               type: array
+            reaper:
+              properties:
+                enabled:
+                  type: boolean
+                image:
+                  type: string
+                imagePullPolicy:
+                  description: PullPolicy describes a policy for if/when to pull a
+                    container image
+                  type: string
+              type: object
             replaceNodes:
               description: A list of pod names that need to be replaced.
               items:

operator/deploy/role.yaml

@@ -64,4 +64,10 @@ rules:
   - '*'
   verbs:
   - '*'
+- apiGroups:
+    - batch
+  resources:
+    - '*'
+  verbs:
+    - '*'
 

operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -211,6 +211,8 @@ type CassandraDatacenterSpec struct {
 	Users []CassandraUser `json:"users,omitempty"`
 
 	AdditionalSeeds []string `json:"additionalSeeds,omitempty"`
+
+	Reaper *ReaperConfig `json:"reaper,omitempty"`
 }
 
 type StorageConfig struct {
@@ -338,6 +340,14 @@ type ManagementApiAuthConfig struct {
 	// other strategy configs (e.g. Cert Manager) go here
 }
 
+type ReaperConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+
+	Image string `json:"image,omitempty"`
+
+	ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // CassandraDatacenterList contains a list of CassandraDatacenter

operator/pkg/httphelper/client_test.go

@@ -101,4 +101,4 @@ func Test_parseMetadataEndpointsResponseBody(t *testing.T) {
 	assert.Equal(t, 2, len(endpoints.Entity))
 	assert.Equal(t, "10.233.90.45", endpoints.Entity[0].RpcAddress)
 	assert.Equal(t, "95c157dc-2811-446a-a541-9faaab2e6930", endpoints.Entity[0].HostID)
-}
+}

operator/pkg/reconciliation/constructor.go

@@ -8,6 +8,7 @@ package reconciliation
 import (
 	"fmt"
 	"os"
+	"k8s.io/api/batch/v1"
 
 	api "github.com/datastax/cass-operator/operator/pkg/apis/cassandra/v1beta1"
 	"github.com/datastax/cass-operator/operator/pkg/httphelper"
@@ -415,7 +416,13 @@ func buildContainers(dc *api.CassandraDatacenter, serverVolumeMounts []corev1.Vo
 	}
 	loggerContainer.VolumeMounts = []corev1.VolumeMount{cassServerLogsMount}
 
-	return []corev1.Container{cassContainer, loggerContainer}, nil
+	containers := []corev1.Container{cassContainer, loggerContainer}
+	if dc.Spec.Reaper != nil && dc.Spec.Reaper.Enabled && dc.Spec.ServerType == "cassandra" {
+		reaperContainer := buildReaperContainer(dc)
+		containers = append(containers, reaperContainer)
+	}
+
+	return containers, nil
 }
 
 func buildInitContainers(dc *api.CassandraDatacenter, rackName string) ([]corev1.Container, error) {
@@ -512,3 +519,46 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, zone string, rackName str
 
 	return baseTemplate, nil
 }
+
+func buildInitReaperSchemaJob(dc *api.CassandraDatacenter) *v1.Job {
+	return &v1.Job{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Job",
+			APIVersion: "batch/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: dc.Namespace,
+			Name:      getReaperSchemaInitJobName(dc),
+			Labels:    dc.GetDatacenterLabels(),
+		},
+		Spec: v1.JobSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					RestartPolicy: corev1.RestartPolicyOnFailure,
+					Containers: []corev1.Container{
+						{
+							Name:            getReaperSchemaInitJobName(dc),
+							Image:           ReaperSchemaInitJobImage,
+							ImagePullPolicy: corev1.PullIfNotPresent,
+							Env: []corev1.EnvVar{
+								{
+									Name:  "KEYSPACE",
+									Value: ReaperKeyspace,
+								},
+								{
+									Name:  "CONTACT_POINTS",
+									Value: dc.GetSeedServiceName(),
+								},
+								{
+									Name:  "REPLICATION",
+									Value: getReaperReplication(dc),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+

operator/pkg/reconciliation/reconcile_racks.go

@@ -1989,6 +1989,14 @@ func (rc *ReconciliationContext) ReconcileAllRacks() (reconcile.Result, error) {
 		return recResult.Output()
 	}
 
+	if recResult := rc.CheckReaperService(); recResult.Completed() {
+		return recResult.Output()
+	}
+
+	if recResult := rc.CheckReaperSchemaInitialized(); recResult.Completed() {
+		return recResult.Output()
+	}
+
 	if recResult := rc.CheckRollingRestart(); recResult.Completed() {
 		return recResult.Output()
 	}