ROX-18428: report secret data to fleet-manager

dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml

@@ -75,6 +75,10 @@ spec:
           value: {{ required "fleetshardSync.managedDB.securityGroup is required when fleetshardSync.managedDB.enabled = true" .Values.fleetshardSync.managedDB.securityGroup }}
         - name: MANAGED_DB_PERFORMANCE_INSIGHTS
           value: {{ .Values.fleetshardSync.managedDB.performanceInsights | quote }}
+        - name: SECRET_ENCRYPTION_TYPE
+          value: {{ .Values.fleetshardSync.secretEncryption.type | quote }}
+        - name: SECRET_ENCRYPTION_KEY_ID
+          value: {{ .Values.fleetshardSync.secretEncryption.keyID | quote }}
         {{- end }}
         - name: AWS_REGION
           value: {{ .Values.fleetshardSync.aws.region }}

dp-terraform/helm/rhacs-terraform/terraform_cluster.sh

@@ -168,6 +168,8 @@ invoke_helm "${SCRIPT_DIR}" rhacs-terraform \
   --set fleetshardSync.imageCredentials.registry="quay.io" \
   --set fleetshardSync.imageCredentials.username="${QUAY_READ_ONLY_USERNAME}" \
   --set fleetshardSync.imageCredentials.password="${QUAY_READ_ONLY_PASSWORD}" \
+  --set fleetshardSync.secretEncryption.type="kms" \
+  --set fleetshardSync.secretEncryption.keyID="${CLUSTER_SECRET_ENCRYPTION_KEY_ID}" \
   --set cloudwatch.aws.accessKeyId="${CLOUDWATCH_EXPORTER_AWS_ACCESS_KEY_ID:-}" \
   --set cloudwatch.aws.secretAccessKey="${CLOUDWATCH_EXPORTER_AWS_SECRET_ACCESS_KEY:-}" \
   --set cloudwatch.clusterName="${CLUSTER_NAME}" \

dp-terraform/helm/rhacs-terraform/values.yaml

@@ -35,6 +35,9 @@ fleetshardSync:
     subnetGroup: ""
     securityGroup: ""
     performanceInsights: true
+  secretEncryption:
+    type: "" # local or kms
+    keyID: ""
   aws:
     region: "us-east-1" # TODO(2023-05-01): Remove the default value here as we now set it explicitly
     roleARN: ""

fleetshard/config/config.go

@@ -11,6 +11,11 @@ import (
 	"github.com/pkg/errors"
 )
 
+const (
+	// EnvDev is the expected value of the environment variable "ENVIRONMENT" for dev deployments of fleetshard-sync
+	EnvDev = "dev"
+)
+
 // Config contains this application's runtime configuration.
 type Config struct {
 	FleetManagerEndpoint string        `env:"FLEET_MANAGER_ENDPOINT" envDefault:"http://127.0.0.1:8000"`
@@ -30,9 +35,10 @@ type Config struct {
 	EgressProxyImage     string        `env:"EGRESS_PROXY_IMAGE"`
 	DefaultBaseCRDURL    string        `env:"DEFAULT_BASE_CRD_URL" envDefault:"https://raw.githubusercontent.com/stackrox/stackrox/%s/operator/bundle/manifests/"`
 
-	ManagedDB    ManagedDB
-	Telemetry    Telemetry
-	AuditLogging AuditLogging
+	ManagedDB        ManagedDB
+	Telemetry        Telemetry
+	AuditLogging     AuditLogging
+	SecretEncryption SecretEncryption
 }
 
 // ManagedDB for configuring managed DB specific parameters
@@ -58,6 +64,12 @@ type Telemetry struct {
 	StorageKey      string `env:"TELEMETRY_STORAGE_KEY"`
 }
 
+// SecretEncryption defines paramaters to configure encryption of tenant secrest
+type SecretEncryption struct {
+	Type  string `env:"SECRET_ENCRYPTION_TYPE" envDefault:"local"`
+	KeyID string `env:"SECRET_ENCRYPTION_KEY_ID"`
+}
+
 // GetConfig retrieves the current runtime configuration from the environment and returns it.
 func GetConfig() (*Config, error) {
 	c := Config{}
@@ -76,6 +88,7 @@ func GetConfig() (*Config, error) {
 		configErrors.AddError(errors.New("AUTH_TYPE unset in the environment"))
 	}
 	validateManagedDBConfig(c, &configErrors)
+	validateSecretEncryptionConfig(c, &configErrors)
 
 	cfgErr := configErrors.ToError()
 	if cfgErr != nil {
@@ -99,3 +112,17 @@ func (a *AuditLogging) Endpoint(withScheme bool) string {
 	}
 	return fmt.Sprintf("%s:%d", a.AuditLogTargetHost, a.AuditLogTargetPort)
 }
+
+func validateSecretEncryptionConfig(c Config, configErrors *errorhelpers.ErrorList) {
+	if !isDevEnvironment(c) && c.SecretEncryption.Type == "local" {
+		configErrors.AddError(errors.New("SECRET_ENCRYPTION_TYPE == local not allowed for non dev environments")) // pragma: allowlist secret
+	}
+
+	if c.SecretEncryption.Type == "kms" && c.SecretEncryption.KeyID == "" {
+		configErrors.AddError(errors.New("SECRET_ENCRYPTION_TYPE == kms and SECRET_ENCRYPTION_KEY_ID unset in the environment")) // pragma: allowlist secret
+	}
+}
+
+func isDevEnvironment(c Config) bool {
+	return c.Environment == EnvDev || c.Environment == ""
+}

fleetshard/pkg/central/reconciler/reconciler.go

@@ -4,10 +4,12 @@ package reconciler
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/url"
 	"reflect"
+	"sort"
 	"strconv"
 	"sync/atomic"
 	"time"
@@ -21,6 +23,7 @@ import (
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/charts"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/cloudprovider"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/postgres"
+	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/cipher"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/util"
 	centralConstants "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
@@ -106,6 +109,8 @@ type CentralReconciler struct {
 	useRoutes        bool
 	Resources        bool
 	routeService     *k8s.RouteService
+	secretBackup     *k8s.SecretBackup
+	secretCipher     cipher.Cipher
 	egressProxyImage string
 	telemetry        config.Telemetry
 	clusterName      string
@@ -746,9 +751,90 @@ func (r *CentralReconciler) collectReconciliationStatus(ctx context.Context, rem
 		}
 	}
 
+	// Only report secrets if Central is ready, to ensure we're not trying to get secrets before they are created.
+	// Only report secrets once. Ensures we don't overwrite initial secrets with corrupted secrets
+	// from the cluster state.
+	if isRemoteCentralReady(remoteCentral) && !r.areSecretsStored(remoteCentral.Metadata.SecretsStored) {
+		secrets, err := r.collectSecretsEncrypted(ctx, remoteCentral)
+		if err != nil {
+			return nil, err
+		}
+		status.Secrets = secrets // pragma: allowlist secret
+	}
+
 	return status, nil
 }
 
+func (r *CentralReconciler) areSecretsStored(secretsStored []string) bool {
+	expectedSecrets := k8s.GetWatchedSecrets()
+	if len(secretsStored) != len(expectedSecrets) {
+		return false
+	}
+
+	sort.Strings(secretsStored)
+
+	for i := 0; i < len(secretsStored); i++ {
+		if secretsStored[i] != expectedSecrets[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (r *CentralReconciler) collectSecrets(ctx context.Context, remoteCentral *private.ManagedCentral) (map[string]*corev1.Secret, error) {
+	namespace := remoteCentral.Metadata.Namespace
+	secrets, err := r.secretBackup.CollectSecrets(ctx, namespace)
+	if err != nil {
+		return secrets, fmt.Errorf("collecting secrets for namespace %s: %w", namespace, err)
+	}
+
+	// remove ResourceVersion and owner reference as this is only intended to recreate non-existent
+	// resources instead of updating existing ones, the owner reference might get invalid in case of
+	// central namespace recreation
+	for _, secret := range secrets { // pragma: allowlist secret
+		secret.ObjectMeta.ResourceVersion = ""
+		secret.ObjectMeta.OwnerReferences = []metav1.OwnerReference{}
+	}
+
+	return secrets, nil
+}
+
+func (r *CentralReconciler) collectSecretsEncrypted(ctx context.Context, remoteCentral *private.ManagedCentral) (map[string]string, error) {
+	secrets, err := r.collectSecrets(ctx, remoteCentral)
+	if err != nil {
+		return nil, err
+	}
+
+	encryptedSecrets, err := r.encryptSecrets(secrets)
+	if err != nil {
+		return nil, fmt.Errorf("encrypting secrets for namespace: %s: %w", remoteCentral.Metadata.Namespace, err)
+	}
+
+	return encryptedSecrets, nil
+}
+
+func (r *CentralReconciler) encryptSecrets(secrets map[string]*corev1.Secret) (map[string]string, error) {
+	encryptedSecrets := map[string]string{}
+
+	for key, secret := range secrets { // pragma: allowlist secret
+		secretBytes, err := json.Marshal(secret)
+		if err != nil {
+			return nil, fmt.Errorf("error marshaling secret for encryption: %s: %w", key, err)
+		}
+
+		encryptedBytes, err := r.secretCipher.Encrypt(secretBytes)
+		if err != nil {
+			return nil, fmt.Errorf("encrypting secret: %s: %w", key, err)
+		}
+
+		encryptedSecrets[key] = base64.StdEncoding.EncodeToString(encryptedBytes)
+	}
+
+	return encryptedSecrets, nil
+
+}
+
 func (r *CentralReconciler) ensureDeclarativeConfigurationSecretCleaned(ctx context.Context, remoteCentralNamespace string) error {
 	secret := &corev1.Secret{}
 	secretKey := ctrlClient.ObjectKey{ // pragma: allowlist secret
@@ -1395,6 +1481,7 @@ func (r *CentralReconciler) ensureSecretExists(
 // NewCentralReconciler ...
 func NewCentralReconciler(k8sClient ctrlClient.Client, central private.ManagedCentral,
 	managedDBProvisioningClient cloudprovider.DBClient, managedDBInitFunc postgres.CentralDBInitFunc,
+	secretCipher cipher.Cipher,
 	opts CentralReconcilerOptions,
 ) *CentralReconciler {
 	return &CentralReconciler{
@@ -1404,6 +1491,8 @@ func NewCentralReconciler(k8sClient ctrlClient.Client, central private.ManagedCe
 		useRoutes:         opts.UseRoutes,
 		wantsAuthProvider: opts.WantsAuthProvider,
 		routeService:      k8s.NewRouteService(k8sClient),
+		secretBackup:      k8s.NewSecretBackup(k8sClient),
+		secretCipher:      secretCipher, // pragma: allowlist secret
 		egressProxyImage:  opts.EgressProxyImage,
 		telemetry:         opts.Telemetry,
 		clusterName:       opts.ClusterName,

fleetshard/pkg/central/reconciler/reconciler_test.go

@@ -5,16 +5,17 @@ import (
 	"context"
 	"embed"
 	"fmt"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/stackrox/rox/pkg/declarativeconfig"
 	"github.com/stackrox/rox/pkg/utils"
 	"gopkg.in/yaml.v2"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	"strings"
-	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 
@@ -26,6 +27,7 @@ import (
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/cloudprovider"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/cloudprovider/awsclient"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/postgres"
+	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/cipher"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/testutils"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/util"
@@ -120,6 +122,12 @@ var simpleManagedCentral = private.ManagedCentral{
 //go:embed testdata
 var testdata embed.FS
 
+func createBase64Cipher(t *testing.T) cipher.Cipher {
+	b64Cipher, err := cipher.NewLocalBase64Cipher()
+	require.NoError(t, err, "creating base64 cipher for test")
+	return b64Cipher
+}
+
 func getClientTrackerAndReconciler(
 	t *testing.T,
 	centralConfig private.ManagedCentral,
@@ -133,6 +141,7 @@ func getClientTrackerAndReconciler(
 		centralConfig,
 		managedDBClient,
 		centralDBInitFunc,
+		createBase64Cipher(t),
 		reconcilerOptions,
 	)
 	return fakeClient, tracker, reconciler
@@ -142,6 +151,24 @@ func centralDBInitFunc(_ context.Context, _ postgres.DBConnection, _, _ string)
 	return nil
 }
 
+func centralTLSSecretObject() *v1.Secret {
+	return &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "central-tls",
+			Namespace: centralNamespace,
+		},
+	}
+}
+
+func centralDBPasswordSecretObject() *v1.Secret {
+	return &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "central-db-password",
+			Namespace: centralNamespace,
+		},
+	}
+}
+
 func conditionForType(conditions []private.DataPlaneClusterUpdateStatusRequestConditions, conditionType string) (*private.DataPlaneClusterUpdateStatusRequestConditions, bool) {
 	for _, c := range conditions {
 		if c.Type == conditionType {
@@ -364,6 +391,8 @@ func TestReconcileLastHashSetOnSuccess(t *testing.T) {
 			},
 		},
 		centralDeploymentObject(),
+		centralTLSSecretObject(),
+		centralDBPasswordSecretObject(),
 	)
 
 	managedCentral := simpleManagedCentral
@@ -431,6 +460,8 @@ func TestIgnoreCacheForCentralForceReconcileAlways(t *testing.T) {
 			},
 		},
 		centralDeploymentObject(),
+		centralTLSSecretObject(),
+		centralDBPasswordSecretObject(),
 	)
 
 	managedCentral := simpleManagedCentral

fleetshard/pkg/cipher/cipher.go

@@ -1,10 +1,31 @@
 // Package cipher defines encryption and decryption methods used by fleetshard-sync
 package cipher
 
+import (
+	"fmt"
+
+	"github.com/stackrox/acs-fleet-manager/fleetshard/config"
+)
+
 //go:generate moq -out cipher_moq.go . Cipher
 
 // Cipher is the interface used to encrypt and decrypt content
 type Cipher interface {
 	Encrypt(plaintext []byte) ([]byte, error)
 	Decrypt(ciphertext []byte) ([]byte, error)
 }
+
+// NewCipher returns a new object implementing cipher, based on the Type defined in config
+func NewCipher(config *config.Config) (Cipher, error) {
+	encryptionType := config.SecretEncryption.Type
+
+	if encryptionType == "local" {
+		return NewLocalBase64Cipher()
+	}
+
+	if encryptionType == "kms" {
+		return NewKMSCipher(config.SecretEncryption.KeyID)
+	}
+
+	return nil, fmt.Errorf("no Cipher implementation for SecretEncryption.Type: %s", encryptionType)
+}

fleetshard/pkg/cipher/local_base64_cipher.go

@@ -0,0 +1,33 @@
+package cipher
+
+import (
+	"encoding/base64"
+	"fmt"
+)
+
+// LocalBase64Cipher simulates encryption by using base64 encoding and decoding
+// Warning: Only use this for development it does not encrypt data
+type LocalBase64Cipher struct {
+}
+
+// NewLocalBase64Cipher returns a new Cipher using the given key
+func NewLocalBase64Cipher() (Cipher, error) {
+	return LocalBase64Cipher{}, nil
+}
+
+var _ Cipher = LocalBase64Cipher{}
+
+// Encrypt implementes the logic to encode plaintext with base64
+func (a LocalBase64Cipher) Encrypt(plaintext []byte) ([]byte, error) {
+	enc := base64.StdEncoding.EncodeToString(plaintext)
+	return []byte(enc), nil
+}
+
+// Decrypt implements the logic to decode base64 text to plaintext
+func (a LocalBase64Cipher) Decrypt(ciphertext []byte) ([]byte, error) {
+	plaintext, err := base64.StdEncoding.DecodeString(string(ciphertext))
+	if err != nil {
+		return nil, fmt.Errorf("decoding base64 string %w", err)
+	}
+	return plaintext, nil
+}