Merge pull request #541 from avanthakkar/fix-odf-2248850

Bug 2248850: exporter: run exporter with specific keyring

pkg/operator/ceph/cluster/cluster.go

@@ -489,6 +489,12 @@ func (c *cluster) postMonStartupActions() error {
 		return errors.Wrap(err, "failed to create crash collector kubernetes secret")
 	}
 
+	// Create exporter Kubernetes Secret
+	err = nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo)
+	if err != nil {
+		return errors.Wrap(err, "failed to create exporter kubernetes secret")
+	}
+
 	// Always ensure the skip mds sanity checks setting is cleared, for all Pacific deployments
 	if c.ClusterInfo.CephVersion.IsPacific() {
 		if err := c.skipMDSSanityChecks(false); err != nil {

pkg/operator/ceph/cluster/nodedaemon/exporter.go

@@ -47,6 +47,8 @@ const (
 	statsPeriod                      = "5"
 	DefaultMetricsPort        uint16 = 9926
 	exporterServiceMetricName        = "ceph-exporter-http-metrics"
+	exporterKeyringUsername          = "client.ceph-exporter"
+	exporterKeyName                  = "rook-ceph-exporter-keyring"
 )
 
 var (
@@ -81,7 +83,7 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations
 
 	volumes := append(
 		controller.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), "", cephCluster.Spec.DataDirHostPath),
-		keyring.Volume().Admin())
+		keyring.Volume().Exporter())
 
 	mutateFunc := func() error {
 
@@ -165,12 +167,10 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
 	cephImage := cephCluster.Spec.CephVersion.Image
 	dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
 	volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath)
-	// FIX: Use an exporter keyring instead of the admin keyring
-	volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin())
+	volumeMounts = append(volumeMounts, keyring.VolumeMount().Exporter())
 
-	envVars := append(
-		controller.DaemonEnvVars(&cephCluster.Spec),
-		v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())})
+	exporterEnvVar := generateExporterEnvVar()
+	envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), exporterEnvVar)
 
 	container := corev1.Container{
 		Name:            "ceph-exporter",
@@ -265,3 +265,10 @@ func applyPrometheusAnnotations(cephCluster cephv1.CephCluster, objectMeta *meta
 		t.ApplyToObjectMeta(objectMeta)
 	}
 }
+
+func generateExporterEnvVar() corev1.EnvVar {
+	val := fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -n %s -k %s", exporterKeyringUsername, keyring.VolumeMount().ExporterKeyringFilePath())
+	env := corev1.EnvVar{Name: "CEPH_ARGS", Value: val}
+
+	return env
+}

pkg/operator/ceph/cluster/nodedaemon/keyring.go

@@ -30,11 +30,20 @@ import (
 
 const (
 	crashClient          = `client.crash`
+	exporterClient       = `client.ceph-exporter`
 	crashKeyringTemplate = `
 [client.crash]
 	key = %s
 	caps mon = "allow profile crash"
 	caps mgr = "allow rw"
+`
+	exporterKeyringTemplate = `
+[client.ceph-exporter]
+	key = %s
+	caps mon = "allow profile ceph-exporter"
+	caps mgr = "allow r"
+	caps osd = "allow r"
+	caps mds = "allow r"
 `
 )
 
@@ -102,3 +111,69 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo
 	logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace)
 	return nil
 }
+
+func CreateExporterSecret(context *clusterd.Context, clusterInfo *client.ClusterInfo) error {
+	k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo)
+
+	// Create exporter Ceph key
+	exporterSecretKey, err := createExporterKeyring(k)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create %q ceph keyring", exporterKeyringUsername)
+	}
+
+	// Create or update Kubernetes CSI secret
+	if err := createOrUpdateExporterSecret(clusterInfo, exporterSecretKey, k); err != nil {
+		return errors.Wrap(err, "failed to create kubernetes csi secret")
+	}
+
+	return nil
+}
+
+func createExporterKeyringCaps() []string {
+	return []string{
+		"mon", "allow profile ceph-exporter",
+		"mgr", "allow r",
+		"osd", "allow r",
+		"mds", "allow r",
+	}
+}
+
+func createExporterKeyring(s *keyring.SecretStore) (string, error) {
+	key, err := s.GenerateKey(exporterKeyringUsername, createExporterKeyringCaps())
+	if err != nil {
+		return "", err
+	}
+
+	return key, nil
+}
+
+func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecretKey string, k *keyring.SecretStore) error {
+
+	keyring := fmt.Sprintf(exporterKeyringTemplate, exporterSecretKey)
+
+	exporterSecret := map[string][]byte{
+		"keyring": []byte(keyring),
+	}
+
+	s := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      exporterKeyName,
+			Namespace: clusterInfo.Namespace,
+		},
+		Data: exporterSecret,
+		Type: k8sutil.RookType,
+	}
+	err := clusterInfo.OwnerInfo.SetControllerReference(s)
+	if err != nil {
+		return errors.Wrapf(err, "failed to set owner reference to exporter controller secret %q", s.Name)
+	}
+
+	// Create Kubernetes Secret
+	err = k.CreateSecret(s)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
+	}
+
+	logger.Infof("created kubernetes exporter secret for cluster %q", clusterInfo.Namespace)
+	return nil
+}

pkg/operator/ceph/cluster/nodedaemon/keyring_test.go

@@ -26,3 +26,8 @@ func TestCephCrashCollectorKeyringCaps(t *testing.T) {
 	caps := cephCrashCollectorKeyringCaps()
 	assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow rw"})
 }
+
+func TestExporterKeyringCaps(t *testing.T) {
+	caps := createExporterKeyringCaps()
+	assert.Equal(t, caps, []string{"mon", "allow profile ceph-exporter", "mgr", "allow r", "osd", "allow r", "mds", "allow r"})
+}

pkg/operator/ceph/config/keyring/admin.go

@@ -29,6 +29,7 @@ import (
 const (
 	adminKeyringResourceName          = "rook-ceph-admin"
 	crashCollectorKeyringResourceName = "rook-ceph-crash-collector"
+	exporterKeyringResourceName       = "rook-ceph-exporter"
 
 	adminKeyringTemplate = `
 [client.admin]

pkg/operator/ceph/config/keyring/volume.go

@@ -13,6 +13,7 @@ const (
 	// mounted independently
 	adminKeyringDir          = "/etc/ceph/admin-keyring-store/"
 	crashCollectorKeyringDir = "/etc/ceph/crash-collector-keyring-store/"
+	exporterKeyringDir       = "/etc/ceph/exporter-keyring-store/"
 )
 
 // VolumeBuilder is a helper for creating Kubernetes pod volumes with content sourced by keyrings
@@ -47,6 +48,11 @@ func (v *VolumeBuilder) CrashCollector() v1.Volume {
 	return v.Resource(crashCollectorKeyringResourceName)
 }
 
+// Exporter returns a kubernetes pod volume whose content is sourced by the SecretStore exporter keyring.
+func (v *VolumeBuilder) Exporter() v1.Volume {
+	return v.Resource(exporterKeyringResourceName)
+}
+
 // VolumeMount returns a VolumeMountBuilder.
 func VolumeMount() *VolumeMountBuilder { return &VolumeMountBuilder{} }
 
@@ -80,6 +86,16 @@ func (*VolumeMountBuilder) CrashCollector() v1.VolumeMount {
 	}
 }
 
+// Exporter returns a Kubernetes container volume mount that mounts the content from the matching
+// VolumeBuilder Exporter volume.
+func (*VolumeMountBuilder) Exporter() v1.VolumeMount {
+	return v1.VolumeMount{
+		Name:      keyringSecretName(exporterKeyringResourceName),
+		ReadOnly:  true, // should be no reason to write to the keyring in pods, so enforce this
+		MountPath: exporterKeyringDir,
+	}
+}
+
 // KeyringFilePath returns the full path to the regular keyring file within a container.
 func (*VolumeMountBuilder) KeyringFilePath() string {
 	return path.Join(keyringDir, keyringFileName)
@@ -94,3 +110,8 @@ func (*VolumeMountBuilder) AdminKeyringFilePath() string {
 func (*VolumeMountBuilder) CrashCollectorKeyringFilePath() string {
 	return path.Join(crashCollectorKeyringDir, keyringFileName)
 }
+
+// ExporterKeyringFilePath returns the full path to the admin keyring file within a container.
+func (*VolumeMountBuilder) ExporterKeyringFilePath() string {
+	return path.Join(exporterKeyringDir, keyringFileName)
+}