core: restart ceph daemons when network updated

We need to restart all the ceph daemons whenever cephCluster network settings are modified like requiremsgr2, encryption and compression. This required for Ceph to consider the new settings it require new ceph daemons all over. Signed-off-by: subhamkrai <[email protected]>
red-hat-storage · Sep 1, 2023 · 29d2b6a · 29d2b6a
1 parent 0432e09
commit 29d2b6a
Show file tree

Hide file tree

Showing 13 changed files with 139 additions and 18 deletions.
diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go
@@ -184,7 +184,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
 			},
 		},
 		Env: append(
-			controller.DaemonEnvVars(c.spec.CephVersion.Image),
+			controller.DaemonEnvVars(&c.spec),
 			c.cephMgrOrchestratorModuleEnvs()...,
 		),
 		Resources:       cephv1.GetMgrResources(c.spec.Resources),
@@ -246,7 +246,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe
 		Image:           c.spec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount),
-		Env:             append(controller.DaemonEnvVars(c.spec.CephVersion.Image), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}),
+		Env:             append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}),
 		Resources:       cephv1.GetMgrResources(c.spec.Resources),
 		SecurityContext: controller.PodSecurityContext(),
 	}

diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go
@@ -269,7 +269,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container
 		VolumeMounts:    controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
 		SecurityContext: controller.PodSecurityContext(),
 		// filesystem creation does not require ports to be exposed
-		Env:       controller.DaemonEnvVars(c.spec.CephVersion.Image),
+		Env:       controller.DaemonEnvVars(&c.spec),
 		Resources: cephv1.GetMonResources(c.spec.Resources),
 	}
 }
@@ -308,7 +308,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
 			},
 		},
 		Env: append(
-			controller.DaemonEnvVars(c.spec.CephVersion.Image),
+			controller.DaemonEnvVars(&c.spec),
 			k8sutil.PodIPEnvVar(podIPEnvVar),
 		),
 		Resources:     cephv1.GetMonResources(c.spec.Resources),

diff --git a/pkg/operator/ceph/cluster/nodedaemon/crash.go b/pkg/operator/ceph/cluster/nodedaemon/crash.go
@@ -167,10 +167,9 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container
 }
 
 func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container {
-	cephImage := cephCluster.Spec.CephVersion.Image
 	dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
 	crashEnvVar := generateCrashEnvVar()
-	envVars := append(controller.DaemonEnvVars(cephImage), crashEnvVar)
+	envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), crashEnvVar)
 	volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath)
 	volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector())
 
@@ -179,7 +178,7 @@ func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver
 		Command: []string{
 			"ceph-crash",
 		},
-		Image:           cephImage,
+		Image:           cephCluster.Spec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
 		Env:             envVars,
 		VolumeMounts:    volumeMounts,

diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go
@@ -171,7 +171,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
 	volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin())
 
 	envVars := append(
-		controller.DaemonEnvVars(cephCluster.Spec.CephVersion.Image),
+		controller.DaemonEnvVars(&cephCluster.Spec),
 		v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())})
 
 	container := corev1.Container{

diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go
@@ -170,8 +170,7 @@ func (r *ReconcileNode) deletev1betaJob(objectMeta metav1.ObjectMeta) {
 }
 
 func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container {
-	cephImage := cephCluster.Spec.CephVersion.Image
-	envVars := append(controller.DaemonEnvVars(cephImage), generateCrashEnvVar())
+	envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), generateCrashEnvVar())
 	dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
 	volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath)
 	volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector())
@@ -188,7 +187,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver.
 		Args: []string{
 			fmt.Sprintf("%d", cephCluster.Spec.CrashCollector.DaysToRetain),
 		},
-		Image:           cephImage,
+		Image:           cephCluster.Spec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
 		Env:             envVars,
 		VolumeMounts:    volumeMounts,

diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go
@@ -583,6 +583,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 			"",
 		))
 
+	envVars = append(envVars, controller.ApplyNetworkEnv(&c.spec)...)
 	podTemplateSpec := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   AppName,

diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go
@@ -122,7 +122,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem
 		Image:           r.cephClusterSpec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
-		Env:             controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(r.cephClusterSpec),
 		Resources:       rbdMirror.Spec.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 		WorkingDir:      config.VarLogCephDir,

diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go
@@ -394,13 +394,41 @@ func ContainerEnvVarReference(envVarName string) string {
 }
 
 // DaemonEnvVars returns the container environment variables used by all Ceph daemons.
-func DaemonEnvVars(image string) []v1.EnvVar {
+func DaemonEnvVars(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar {
+	networkEnv := ApplyNetworkEnv(cephClusterSpec)
+	cephDaemonsEnvVars := append(k8sutil.ClusterDaemonEnvVars(cephClusterSpec.CephVersion.Image), networkEnv...)
+
 	return append(
-		k8sutil.ClusterDaemonEnvVars(image),
+		cephDaemonsEnvVars,
 		config.StoredMonHostEnvVars()...,
 	)
 }
 
+func ApplyNetworkEnv(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar {
+	if cephClusterSpec.Network.Connections != nil {
+		msgr2Required := false
+		encryptionEnabled := false
+		compressionEnabled := false
+		if cephClusterSpec.Network.Connections.RequireMsgr2 {
+			msgr2Required = true
+		}
+		if cephClusterSpec.Network.Connections.Encryption != nil && cephClusterSpec.Network.Connections.Encryption.Enabled {
+			encryptionEnabled = true
+		}
+		if cephClusterSpec.Network.Connections.Compression != nil && cephClusterSpec.Network.Connections.Compression.Enabled {
+			compressionEnabled = true
+		}
+		envVarValue := fmt.Sprintf("msgr2_%t_encryption_%t_compression_%t", msgr2Required, encryptionEnabled, compressionEnabled)
+
+		rookMsgr2Env := []v1.EnvVar{{
+			Name:  "ROOK_MSGR2",
+			Value: envVarValue,
+		}}
+		return rookMsgr2Env
+	}
+	return []v1.EnvVar{}
+}
+
 // AppLabels returns labels common for all Rook-Ceph applications which may be useful for admins.
 // App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc.
 func AppLabels(appName, namespace string) map[string]string {

diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go
@@ -463,3 +463,97 @@ func TestGetContainerImagePullPolicy(t *testing.T) {
 		assert.Equal(t, exepctedImagePullPolicy, imagePullPolicy)
 	})
 }
+
+func TestDaemonEnvVars(t *testing.T) {
+	// No network settings specified
+	want := []v1.EnvVar{}
+
+	clusterSpec := &cephv1.ClusterSpec{}
+	got := ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Encryption is enabled
+	connections := &cephv1.ConnectionsSpec{
+		Encryption: &cephv1.EncryptionSpec{Enabled: true},
+	}
+
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_false_encryption_true_compression_false",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Compression is enabled
+	connections = &cephv1.ConnectionsSpec{
+		Compression: &cephv1.CompressionSpec{
+			Enabled: true,
+		},
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_false_encryption_false_compression_true",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Msgr2 is enabled
+	connections = &cephv1.ConnectionsSpec{
+		RequireMsgr2: true,
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_true_encryption_false_compression_false",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Msgr2, Compression, Encryption are enabled
+	connections = &cephv1.ConnectionsSpec{
+		RequireMsgr2: true,
+		Encryption: &cephv1.EncryptionSpec{
+			Enabled: true,
+		},
+		Compression: &cephv1.CompressionSpec{
+			Enabled: true,
+		},
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_true_encryption_true_compression_true",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Msgr2 is enabled but Compression, Encryption are disabled
+	connections = &cephv1.ConnectionsSpec{
+		RequireMsgr2: true,
+		Encryption: &cephv1.EncryptionSpec{
+			Enabled: false,
+		},
+		Compression: &cephv1.CompressionSpec{
+			Enabled: false,
+		},
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_true_encryption_false_compression_false",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+}
diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go
@@ -147,7 +147,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
 		Image:           c.clusterSpec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath),
-		Env:             append(controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), k8sutil.PodIPEnvVar(podIPEnvVar)),
+		Env:             append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)),
 		Resources:       c.fs.Spec.MetadataServer.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 		StartupProbe:    controller.GenerateStartupProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID),

diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go
@@ -119,7 +119,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig
 		Image:           r.cephClusterSpec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
-		Env:             controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(r.cephClusterSpec),
 		Resources:       fsMirror.Spec.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 		// TODO:

diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go
@@ -243,7 +243,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
 			nfsConfigMount,
 			dbusMount,
 		},
-		Env:             controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(r.cephClusterSpec),
 		Resources:       nfs.Spec.Server.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 	}

diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go
@@ -359,7 +359,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
 			controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath),
 			c.mimeTypesVolumeMount(),
 		),
-		Env:             controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(c.clusterSpec),
 		Resources:       c.store.Spec.Gateway.Resources,
 		StartupProbe:    startupProbe,
 		LivenessProbe:   noLivenessProbe(),