diff --git a/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml b/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml index 087becd4d6..487e914d57 100644 --- a/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml +++ b/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml @@ -75,6 +75,10 @@ spec: value: {{ required "fleetshardSync.managedDB.securityGroup is required when fleetshardSync.managedDB.enabled = true" .Values.fleetshardSync.managedDB.securityGroup }} - name: MANAGED_DB_PERFORMANCE_INSIGHTS value: {{ .Values.fleetshardSync.managedDB.performanceInsights | quote }} + - name: SECRET_ENCRYPTION_TYPE + value: {{ .Values.fleetshardSync.secretEncryption.type | quote }} + - name: SECRET_ENCRYPTION_KEY_ID + value: {{ .Values.fleetshardSync.secretEncryption.keyID | quote }} {{- end }} - name: AWS_REGION value: {{ .Values.fleetshardSync.aws.region }} diff --git a/dp-terraform/helm/rhacs-terraform/terraform_cluster.sh b/dp-terraform/helm/rhacs-terraform/terraform_cluster.sh index 6e54edc343..c510ec634f 100755 --- a/dp-terraform/helm/rhacs-terraform/terraform_cluster.sh +++ b/dp-terraform/helm/rhacs-terraform/terraform_cluster.sh @@ -168,6 +168,8 @@ invoke_helm "${SCRIPT_DIR}" rhacs-terraform \ --set fleetshardSync.imageCredentials.registry="quay.io" \ --set fleetshardSync.imageCredentials.username="${QUAY_READ_ONLY_USERNAME}" \ --set fleetshardSync.imageCredentials.password="${QUAY_READ_ONLY_PASSWORD}" \ + --set fleetshardSync.secretEncryption.type="kms" \ + --set fleetshardSync.secretEncryption.keyID="${CLUSTER_SECRET_ENCRYPTION_KEY_ID}" \ --set cloudwatch.aws.accessKeyId="${CLOUDWATCH_EXPORTER_AWS_ACCESS_KEY_ID:-}" \ --set cloudwatch.aws.secretAccessKey="${CLOUDWATCH_EXPORTER_AWS_SECRET_ACCESS_KEY:-}" \ --set cloudwatch.clusterName="${CLUSTER_NAME}" \ diff --git a/dp-terraform/helm/rhacs-terraform/values.yaml b/dp-terraform/helm/rhacs-terraform/values.yaml index cb3b44bc81..23c78ceb75 100644 --- a/dp-terraform/helm/rhacs-terraform/values.yaml +++ b/dp-terraform/helm/rhacs-terraform/values.yaml @@ -35,6 +35,9 @@ fleetshardSync: subnetGroup: "" securityGroup: "" performanceInsights: true + secretEncryption: + type: "" # local or kms + keyID: "" aws: region: "us-east-1" # TODO(2023-05-01): Remove the default value here as we now set it explicitly roleARN: "" diff --git a/fleetshard/config/config.go b/fleetshard/config/config.go index 7df6d8126e..0f17347861 100644 --- a/fleetshard/config/config.go +++ b/fleetshard/config/config.go @@ -11,6 +11,11 @@ import ( "github.com/pkg/errors" ) +const ( + // EnvDev is the expected value of the environment variable "ENVIRONMENT" for dev deployments of fleetshard-sync + EnvDev = "dev" +) + // Config contains this application's runtime configuration. type Config struct { FleetManagerEndpoint string `env:"FLEET_MANAGER_ENDPOINT" envDefault:"http://127.0.0.1:8000"` @@ -30,9 +35,10 @@ type Config struct { EgressProxyImage string `env:"EGRESS_PROXY_IMAGE"` DefaultBaseCRDURL string `env:"DEFAULT_BASE_CRD_URL" envDefault:"https://raw.githubusercontent.com/stackrox/stackrox/%s/operator/bundle/manifests/"` - ManagedDB ManagedDB - Telemetry Telemetry - AuditLogging AuditLogging + ManagedDB ManagedDB + Telemetry Telemetry + AuditLogging AuditLogging + SecretEncryption SecretEncryption } // ManagedDB for configuring managed DB specific parameters @@ -58,6 +64,12 @@ type Telemetry struct { StorageKey string `env:"TELEMETRY_STORAGE_KEY"` } +// SecretEncryption defines paramaters to configure encryption of tenant secrest +type SecretEncryption struct { + Type string `env:"SECRET_ENCRYPTION_TYPE" envDefault:"local"` + KeyID string `env:"SECRET_ENCRYPTION_KEY_ID"` +} + // GetConfig retrieves the current runtime configuration from the environment and returns it. func GetConfig() (*Config, error) { c := Config{} @@ -76,6 +88,7 @@ func GetConfig() (*Config, error) { configErrors.AddError(errors.New("AUTH_TYPE unset in the environment")) } validateManagedDBConfig(c, &configErrors) + validateSecretEncryptionConfig(c, &configErrors) cfgErr := configErrors.ToError() if cfgErr != nil { @@ -99,3 +112,17 @@ func (a *AuditLogging) Endpoint(withScheme bool) string { } return fmt.Sprintf("%s:%d", a.AuditLogTargetHost, a.AuditLogTargetPort) } + +func validateSecretEncryptionConfig(c Config, configErrors *errorhelpers.ErrorList) { + if !isDevEnvironment(c) && c.SecretEncryption.Type == "local" { + configErrors.AddError(errors.New("SECRET_ENCRYPTION_TYPE == local not allowed for non dev environments")) // pragma: allowlist secret + } + + if c.SecretEncryption.Type == "kms" && c.SecretEncryption.KeyID == "" { + configErrors.AddError(errors.New("SECRET_ENCRYPTION_TYPE == kms and SECRET_ENCRYPTION_KEY_ID unset in the environment")) // pragma: allowlist secret + } +} + +func isDevEnvironment(c Config) bool { + return c.Environment == EnvDev || c.Environment == "" +} diff --git a/fleetshard/pkg/central/reconciler/reconciler.go b/fleetshard/pkg/central/reconciler/reconciler.go index be219ce393..73dad97ea2 100644 --- a/fleetshard/pkg/central/reconciler/reconciler.go +++ b/fleetshard/pkg/central/reconciler/reconciler.go @@ -4,10 +4,12 @@ package reconciler import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "net/url" "reflect" + "sort" "strconv" "sync/atomic" "time" @@ -21,6 +23,7 @@ import ( "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/charts" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/cloudprovider" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/postgres" + "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/cipher" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/util" centralConstants "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants" @@ -106,6 +109,8 @@ type CentralReconciler struct { useRoutes bool Resources bool routeService *k8s.RouteService + secretBackup *k8s.SecretBackup + secretCipher cipher.Cipher egressProxyImage string telemetry config.Telemetry clusterName string @@ -746,9 +751,90 @@ func (r *CentralReconciler) collectReconciliationStatus(ctx context.Context, rem } } + // Only report secrets if Central is ready, to ensure we're not trying to get secrets before they are created. + // Only report secrets once. Ensures we don't overwrite initial secrets with corrupted secrets + // from the cluster state. + if isRemoteCentralReady(remoteCentral) && !r.areSecretsStored(remoteCentral.Metadata.SecretsStored) { + secrets, err := r.collectSecretsEncrypted(ctx, remoteCentral) + if err != nil { + return nil, err + } + status.Secrets = secrets // pragma: allowlist secret + } + return status, nil } +func (r *CentralReconciler) areSecretsStored(secretsStored []string) bool { + expectedSecrets := k8s.GetWatchedSecrets() + if len(secretsStored) != len(expectedSecrets) { + return false + } + + sort.Strings(secretsStored) + + for i := 0; i < len(secretsStored); i++ { + if secretsStored[i] != expectedSecrets[i] { + return false + } + } + + return true +} + +func (r *CentralReconciler) collectSecrets(ctx context.Context, remoteCentral *private.ManagedCentral) (map[string]*corev1.Secret, error) { + namespace := remoteCentral.Metadata.Namespace + secrets, err := r.secretBackup.CollectSecrets(ctx, namespace) + if err != nil { + return secrets, fmt.Errorf("collecting secrets for namespace %s: %w", namespace, err) + } + + // remove ResourceVersion and owner reference as this is only intended to recreate non-existent + // resources instead of updating existing ones, the owner reference might get invalid in case of + // central namespace recreation + for _, secret := range secrets { // pragma: allowlist secret + secret.ObjectMeta.ResourceVersion = "" + secret.ObjectMeta.OwnerReferences = []metav1.OwnerReference{} + } + + return secrets, nil +} + +func (r *CentralReconciler) collectSecretsEncrypted(ctx context.Context, remoteCentral *private.ManagedCentral) (map[string]string, error) { + secrets, err := r.collectSecrets(ctx, remoteCentral) + if err != nil { + return nil, err + } + + encryptedSecrets, err := r.encryptSecrets(secrets) + if err != nil { + return nil, fmt.Errorf("encrypting secrets for namespace: %s: %w", remoteCentral.Metadata.Namespace, err) + } + + return encryptedSecrets, nil +} + +func (r *CentralReconciler) encryptSecrets(secrets map[string]*corev1.Secret) (map[string]string, error) { + encryptedSecrets := map[string]string{} + + for key, secret := range secrets { // pragma: allowlist secret + secretBytes, err := json.Marshal(secret) + if err != nil { + return nil, fmt.Errorf("error marshaling secret for encryption: %s: %w", key, err) + } + + encryptedBytes, err := r.secretCipher.Encrypt(secretBytes) + if err != nil { + return nil, fmt.Errorf("encrypting secret: %s: %w", key, err) + } + + encryptedSecrets[key] = base64.StdEncoding.EncodeToString(encryptedBytes) + } + + return encryptedSecrets, nil + +} + func (r *CentralReconciler) ensureDeclarativeConfigurationSecretCleaned(ctx context.Context, remoteCentralNamespace string) error { secret := &corev1.Secret{} secretKey := ctrlClient.ObjectKey{ // pragma: allowlist secret @@ -1395,6 +1481,7 @@ func (r *CentralReconciler) ensureSecretExists( // NewCentralReconciler ... func NewCentralReconciler(k8sClient ctrlClient.Client, central private.ManagedCentral, managedDBProvisioningClient cloudprovider.DBClient, managedDBInitFunc postgres.CentralDBInitFunc, + secretCipher cipher.Cipher, opts CentralReconcilerOptions, ) *CentralReconciler { return &CentralReconciler{ @@ -1404,6 +1491,8 @@ func NewCentralReconciler(k8sClient ctrlClient.Client, central private.ManagedCe useRoutes: opts.UseRoutes, wantsAuthProvider: opts.WantsAuthProvider, routeService: k8s.NewRouteService(k8sClient), + secretBackup: k8s.NewSecretBackup(k8sClient), + secretCipher: secretCipher, // pragma: allowlist secret egressProxyImage: opts.EgressProxyImage, telemetry: opts.Telemetry, clusterName: opts.ClusterName, diff --git a/fleetshard/pkg/central/reconciler/reconciler_test.go b/fleetshard/pkg/central/reconciler/reconciler_test.go index b105777a0d..11598abeee 100644 --- a/fleetshard/pkg/central/reconciler/reconciler_test.go +++ b/fleetshard/pkg/central/reconciler/reconciler_test.go @@ -5,6 +5,10 @@ import ( "context" "embed" "fmt" + "strings" + "testing" + "time" + "github.com/stackrox/rox/pkg/declarativeconfig" "github.com/stackrox/rox/pkg/utils" "gopkg.in/yaml.v2" @@ -12,9 +16,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "strings" - "testing" - "time" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" @@ -26,6 +27,7 @@ import ( "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/cloudprovider" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/cloudprovider/awsclient" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/postgres" + "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/cipher" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/testutils" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/util" @@ -120,6 +122,12 @@ var simpleManagedCentral = private.ManagedCentral{ //go:embed testdata var testdata embed.FS +func createBase64Cipher(t *testing.T) cipher.Cipher { + b64Cipher, err := cipher.NewLocalBase64Cipher() + require.NoError(t, err, "creating base64 cipher for test") + return b64Cipher +} + func getClientTrackerAndReconciler( t *testing.T, centralConfig private.ManagedCentral, @@ -133,6 +141,7 @@ func getClientTrackerAndReconciler( centralConfig, managedDBClient, centralDBInitFunc, + createBase64Cipher(t), reconcilerOptions, ) return fakeClient, tracker, reconciler @@ -142,6 +151,24 @@ func centralDBInitFunc(_ context.Context, _ postgres.DBConnection, _, _ string) return nil } +func centralTLSSecretObject() *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "central-tls", + Namespace: centralNamespace, + }, + } +} + +func centralDBPasswordSecretObject() *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "central-db-password", + Namespace: centralNamespace, + }, + } +} + func conditionForType(conditions []private.DataPlaneClusterUpdateStatusRequestConditions, conditionType string) (*private.DataPlaneClusterUpdateStatusRequestConditions, bool) { for _, c := range conditions { if c.Type == conditionType { @@ -364,6 +391,8 @@ func TestReconcileLastHashSetOnSuccess(t *testing.T) { }, }, centralDeploymentObject(), + centralTLSSecretObject(), + centralDBPasswordSecretObject(), ) managedCentral := simpleManagedCentral @@ -431,6 +460,8 @@ func TestIgnoreCacheForCentralForceReconcileAlways(t *testing.T) { }, }, centralDeploymentObject(), + centralTLSSecretObject(), + centralDBPasswordSecretObject(), ) managedCentral := simpleManagedCentral diff --git a/fleetshard/pkg/cipher/cipher.go b/fleetshard/pkg/cipher/cipher.go index 9cfedabbfd..99321dd6dd 100644 --- a/fleetshard/pkg/cipher/cipher.go +++ b/fleetshard/pkg/cipher/cipher.go @@ -1,6 +1,12 @@ // Package cipher defines encryption and decryption methods used by fleetshard-sync package cipher +import ( + "fmt" + + "github.com/stackrox/acs-fleet-manager/fleetshard/config" +) + //go:generate moq -out cipher_moq.go . Cipher // Cipher is the interface used to encrypt and decrypt content @@ -8,3 +14,18 @@ type Cipher interface { Encrypt(plaintext []byte) ([]byte, error) Decrypt(ciphertext []byte) ([]byte, error) } + +// NewCipher returns a new object implementing cipher, based on the Type defined in config +func NewCipher(config *config.Config) (Cipher, error) { + encryptionType := config.SecretEncryption.Type + + if encryptionType == "local" { + return NewLocalBase64Cipher() + } + + if encryptionType == "kms" { + return NewKMSCipher(config.SecretEncryption.KeyID) + } + + return nil, fmt.Errorf("no Cipher implementation for SecretEncryption.Type: %s", encryptionType) +} diff --git a/fleetshard/pkg/cipher/local_base64_cipher.go b/fleetshard/pkg/cipher/local_base64_cipher.go new file mode 100644 index 0000000000..422a9a1737 --- /dev/null +++ b/fleetshard/pkg/cipher/local_base64_cipher.go @@ -0,0 +1,33 @@ +package cipher + +import ( + "encoding/base64" + "fmt" +) + +// LocalBase64Cipher simulates encryption by using base64 encoding and decoding +// Warning: Only use this for development it does not encrypt data +type LocalBase64Cipher struct { +} + +// NewLocalBase64Cipher returns a new Cipher using the given key +func NewLocalBase64Cipher() (Cipher, error) { + return LocalBase64Cipher{}, nil +} + +var _ Cipher = LocalBase64Cipher{} + +// Encrypt implementes the logic to encode plaintext with base64 +func (a LocalBase64Cipher) Encrypt(plaintext []byte) ([]byte, error) { + enc := base64.StdEncoding.EncodeToString(plaintext) + return []byte(enc), nil +} + +// Decrypt implements the logic to decode base64 text to plaintext +func (a LocalBase64Cipher) Decrypt(ciphertext []byte) ([]byte, error) { + plaintext, err := base64.StdEncoding.DecodeString(string(ciphertext)) + if err != nil { + return nil, fmt.Errorf("decoding base64 string %w", err) + } + return plaintext, nil +} diff --git a/fleetshard/pkg/cipher/local_base64_cipher_test.go b/fleetshard/pkg/cipher/local_base64_cipher_test.go new file mode 100644 index 0000000000..0aedd13a02 --- /dev/null +++ b/fleetshard/pkg/cipher/local_base64_cipher_test.go @@ -0,0 +1,22 @@ +package cipher + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBase64EncryptDecryptMatch(t *testing.T) { + plaintext := []byte("test plaintext") + + b64Cipher, err := NewLocalBase64Cipher() + require.NoError(t, err, "creating cipher") + + cipher, err := b64Cipher.Encrypt(plaintext) + require.NoError(t, err, "encyrpting plaintext") + + decrypted, err := b64Cipher.Decrypt(cipher) + require.NoError(t, err, "decrypting ciphertext") + + require.Equal(t, string(plaintext), string(decrypted), "decrypted string does not match plaintext") +} diff --git a/fleetshard/pkg/k8s/route.go b/fleetshard/pkg/k8s/route.go index 8b807e0a5d..94fe3c02e5 100644 --- a/fleetshard/pkg/k8s/route.go +++ b/fleetshard/pkg/k8s/route.go @@ -8,8 +8,6 @@ import ( "github.com/pkg/errors" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/private" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" @@ -18,13 +16,12 @@ import ( const ( centralReencryptRouteName = "managed-central-reencrypt" centralPassthroughRouteName = "managed-central-passthrough" - centralTLSSecretName = "central-tls" // pragma: allowlist secret centralReencryptTimeoutAnnotationKey = "haproxy.router.openshift.io/timeout" centralReencryptTimeoutAnnotationValue = "10m" ) -// ErrCentralTLSSecretNotFound returned when central-tls secret is not found during creation of the reencrypt route +// ErrCentralTLSSecretNotFound returned when central-tls secret is not found var ErrCentralTLSSecretNotFound = errors.New("central-tls secret not found") // RouteService is responsible for performing read and write operations on the OpenShift Route objects in the cluster. @@ -83,18 +80,14 @@ func isAdmitted(ingress openshiftRouteV1.RouteIngress) bool { // CreateReencryptRoute creates a new managed central reencrypt route. func (s *RouteService) CreateReencryptRoute(ctx context.Context, remoteCentral private.ManagedCentral) error { - centralTLSSecret := &v1.Secret{} namespace := remoteCentral.Metadata.Namespace - err := s.client.Get(ctx, ctrlClient.ObjectKey{Namespace: namespace, Name: centralTLSSecretName}, centralTLSSecret) + centralTLSSecret, err := getSecret(ctx, s.client, centralTLSSecretName, namespace) if err != nil { - if apiErrors.IsNotFound(err) { - return ErrCentralTLSSecretNotFound - } - return errors.Wrapf(err, "get central TLS secret %s/%s", namespace, remoteCentral.Metadata.Name) + return fmt.Errorf("getting central-tls secret for tenant %s: %w", remoteCentral.Metadata.Name, err) } centralCA, ok := centralTLSSecret.Data["ca.pem"] if !ok { - return errors.Errorf("could not find centrals ca certificate 'ca.pem' in secret/%s", centralTLSSecretName) + return fmt.Errorf("could not find centrals ca certificate 'ca.pem' in secret/%s", centralTLSSecretName) } annotations := map[string]string{ diff --git a/fleetshard/pkg/k8s/secret.go b/fleetshard/pkg/k8s/secret.go new file mode 100644 index 0000000000..fd9c567ad6 --- /dev/null +++ b/fleetshard/pkg/k8s/secret.go @@ -0,0 +1,67 @@ +package k8s + +import ( + "context" + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + centralTLSSecretName = "central-tls" // pragma: allowlist secret + centralDBPasswordSecretName = "central-db-password" // pragma: allowlist secret +) + +var secretsToWatch = []string{ + centralTLSSecretName, + centralDBPasswordSecretName, +} + +// SecretBackup is responsible for reading secrets to Backup for a tenant. +type SecretBackup struct { + client ctrlClient.Client +} + +// NewSecretBackup creates a new instance of SecretService. +func NewSecretBackup(client ctrlClient.Client) *SecretBackup { + return &SecretBackup{client: client} +} + +// GetWatchedSecrets return a sorted list of secrets watched by this package +func GetWatchedSecrets() []string { + secrets := make([]string, len(secretsToWatch)) + copy(secrets, secretsToWatch) + sort.Strings(secrets) + return secrets +} + +// CollectSecrets returns a map of secret name to secret object for all secrets +// watched by SecretServices +func (s *SecretBackup) CollectSecrets(ctx context.Context, namespace string) (map[string]*corev1.Secret, error) { + secrets := map[string]*corev1.Secret{} + for _, secretname := range secretsToWatch { // pragma: allowlist secret + secret, err := getSecret(ctx, s.client, secretname, namespace) + if err != nil { + return nil, err + } + secrets[secretname] = secret // pragma: allowlist secret + } + + return secrets, nil +} + +func getSecret(ctx context.Context, client ctrlClient.Client, secretname, namespace string) (*corev1.Secret, error) { + centralSecret := &corev1.Secret{} + err := client.Get(ctx, ctrlClient.ObjectKey{Namespace: namespace, Name: secretname}, centralSecret) + if err != nil { + if apiErrors.IsNotFound(err) { + return centralSecret, fmt.Errorf("%s secret not found", secretname) + } + return centralSecret, fmt.Errorf("getting secret %s/%s: %w", namespace, secretname, err) + } + + return centralSecret, nil +} diff --git a/fleetshard/pkg/runtime/runtime.go b/fleetshard/pkg/runtime/runtime.go index 1ee5b090a1..42875c8eb6 100644 --- a/fleetshard/pkg/runtime/runtime.go +++ b/fleetshard/pkg/runtime/runtime.go @@ -15,6 +15,7 @@ import ( "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/operator" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/postgres" centralReconciler "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/reconciler" + "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/cipher" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/fleetshardmetrics" "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/private" @@ -55,6 +56,7 @@ type Runtime struct { dbProvisionClient cloudprovider.DBClient statusResponseCh chan private.DataPlaneCentralStatus operatorManager *operator.ACSOperatorManager + secretCipher cipher.Cipher } // NewRuntime creates a new runtime @@ -92,6 +94,10 @@ func NewRuntime(config *config.Config, k8sClient ctrlClient.Client) (*Runtime, e } operatorManager := operator.NewACSOperatorManager(k8sClient) + secretCipher, err := cipher.NewCipher(config) + if err != nil { + return nil, fmt.Errorf("creating secretCipher: %w", err) + } return &Runtime{ config: config, @@ -101,6 +107,7 @@ func NewRuntime(config *config.Config, k8sClient ctrlClient.Client) (*Runtime, e dbProvisionClient: dbProvisionClient, reconcilers: make(reconcilerRegistry), operatorManager: operatorManager, + secretCipher: secretCipher, // pragma: allowlist secret }, nil } @@ -149,7 +156,7 @@ func (r *Runtime) Start() error { for _, central := range list.Items { if _, ok := r.reconcilers[central.Id]; !ok { r.reconcilers[central.Id] = centralReconciler.NewCentralReconciler(r.k8sClient, central, - r.dbProvisionClient, postgres.InitializeDatabase, reconcilerOpts) + r.dbProvisionClient, postgres.InitializeDatabase, r.secretCipher, reconcilerOpts) } reconciler := r.reconcilers[central.Id] diff --git a/internal/dinosaur/pkg/api/dbapi/central_request_types.go b/internal/dinosaur/pkg/api/dbapi/central_request_types.go index b641d346f3..0e1957ff92 100644 --- a/internal/dinosaur/pkg/api/dbapi/central_request_types.go +++ b/internal/dinosaur/pkg/api/dbapi/central_request_types.go @@ -75,8 +75,11 @@ type CentralRequest struct { RoutesCreated bool `json:"routes_created"` // Namespace is the namespace of the provisioned central instance. // We store this in the database to ensure that old centrals whose namespace contained "owner-" information will continue to work. - Namespace string `json:"namespace"` - RoutesCreationID string `json:"routes_creation_id"` + + // Secrets stores the encrypted secrets reported for a central tenant + Secrets api.JSON `json:"secrets"` + Namespace string `json:"namespace"` + RoutesCreationID string `json:"routes_creation_id"` // DeletionTimestamp stores the timestamp of the DELETE api call for the resource. DeletionTimestamp *time.Time `json:"deletionTimestamp"` @@ -158,6 +161,16 @@ func (k *CentralRequest) SetRoutes(routes []DataPlaneCentralRoute) error { return nil } +// SetSecrets sets CentralRequest.Secret field by converting secrets to api.JSON +func (k *CentralRequest) SetSecrets(secrets map[string]string) error { + r, err := json.Marshal(secrets) + if err != nil { + return fmt.Errorf("marshalling secrets into JSON: %w", err) + } + k.Secrets = r + return nil +} + // GetUIHost returns host for CLI/GUI/API connections func (k *CentralRequest) GetUIHost() string { if k.Host == "" { diff --git a/internal/dinosaur/pkg/api/dbapi/data_plane_central_status.go b/internal/dinosaur/pkg/api/dbapi/data_plane_central_status.go index 81b33976b0..6e7b6b4b5f 100644 --- a/internal/dinosaur/pkg/api/dbapi/data_plane_central_status.go +++ b/internal/dinosaur/pkg/api/dbapi/data_plane_central_status.go @@ -10,6 +10,7 @@ type DataPlaneCentralStatus struct { Conditions []DataPlaneCentralStatusCondition // Going to ignore the rest of fields (like capacity and versions) for now, until when they are needed Routes []DataPlaneCentralRoute + Secrets map[string]string CentralVersion string CentralOperatorVersion string } diff --git a/internal/dinosaur/pkg/api/private/api/openapi.yaml b/internal/dinosaur/pkg/api/private/api/openapi.yaml index 83b8b77c3f..bfb0ddeae2 100644 --- a/internal/dinosaur/pkg/api/private/api/openapi.yaml +++ b/internal/dinosaur/pkg/api/private/api/openapi.yaml @@ -358,6 +358,11 @@ components: items: $ref: '#/components/schemas/DataPlaneCentralStatus_routes' type: array + secrets: + additionalProperties: + type: string + description: Map of Secrets created for a Central + type: object type: object DataPlaneCentralStatusUpdateRequest: additionalProperties: @@ -412,6 +417,10 @@ components: $ref: '#/components/schemas/ManagedCentral_allOf_metadata_annotations' deletionTimestamp: type: string + secretsStored: + items: + type: string + type: array ManagedCentral_allOf_spec_auth: properties: clientSecret: diff --git a/internal/dinosaur/pkg/api/private/model_data_plane_central_status.go b/internal/dinosaur/pkg/api/private/model_data_plane_central_status.go index beac0faa02..b4c5abbea8 100644 --- a/internal/dinosaur/pkg/api/private/model_data_plane_central_status.go +++ b/internal/dinosaur/pkg/api/private/model_data_plane_central_status.go @@ -16,4 +16,6 @@ type DataPlaneCentralStatus struct { Conditions []DataPlaneClusterUpdateStatusRequestConditions `json:"conditions,omitempty"` // Routes created for a Central Routes []DataPlaneCentralStatusRoutes `json:"routes,omitempty"` + // Map of Secrets created for a Central + Secrets map[string]string `json:"secrets,omitempty"` } diff --git a/internal/dinosaur/pkg/api/private/model_managed_central_all_of_metadata.go b/internal/dinosaur/pkg/api/private/model_managed_central_all_of_metadata.go index 5fe8f3e6a3..f90ab28f7a 100644 --- a/internal/dinosaur/pkg/api/private/model_managed_central_all_of_metadata.go +++ b/internal/dinosaur/pkg/api/private/model_managed_central_all_of_metadata.go @@ -17,4 +17,5 @@ type ManagedCentralAllOfMetadata struct { Internal bool `json:"internal,omitempty"` Annotations ManagedCentralAllOfMetadataAnnotations `json:"annotations,omitempty"` DeletionTimestamp string `json:"deletionTimestamp,omitempty"` + SecretsStored []string `json:"secretsStored,omitempty"` } diff --git a/internal/dinosaur/pkg/migrations/20230802000000_add_secrets_field_to_central_request.go b/internal/dinosaur/pkg/migrations/20230802000000_add_secrets_field_to_central_request.go new file mode 100644 index 0000000000..ed7bc0857a --- /dev/null +++ b/internal/dinosaur/pkg/migrations/20230802000000_add_secrets_field_to_central_request.go @@ -0,0 +1,31 @@ +package migrations + +// Migrations should NEVER use types from other packages. Types can change +// and then migrations run on a _new_ database will fail or behave unexpectedly. +// Instead of importing types, always re-create the type in the migration, as +// is done here, even though the same type is defined in pkg/api + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "github.com/stackrox/acs-fleet-manager/pkg/api" + "github.com/stackrox/acs-fleet-manager/pkg/db" + "gorm.io/gorm" +) + +func addSecretFieldToCentralRequests() *gormigrate.Migration { + type CentralRequest struct { + db.Model + Secrets api.JSON `json:"secrets"` + } + migrationID := "20230802000000" + + return &gormigrate.Migration{ + ID: migrationID, + Migrate: func(tx *gorm.DB) error { + return addColumnIfNotExists(tx, &CentralRequest{}, "secrets") + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + } +} diff --git a/internal/dinosaur/pkg/migrations/migrations.go b/internal/dinosaur/pkg/migrations/migrations.go index 339fd713f8..fff67c8476 100644 --- a/internal/dinosaur/pkg/migrations/migrations.go +++ b/internal/dinosaur/pkg/migrations/migrations.go @@ -2,6 +2,7 @@ package migrations import ( "fmt" + "github.com/go-gormigrate/gormigrate/v2" "github.com/stackrox/acs-fleet-manager/pkg/db" "gorm.io/gorm" @@ -46,6 +47,7 @@ func getMigrations() []*gormigrate.Migration { addForceReconcileToCentralRequest(), addOperatorImageFields(), removeAvailableOperatorField(), + addSecretFieldToCentralRequests(), } } diff --git a/internal/dinosaur/pkg/presenters/data_plane_dinosaur_status.go b/internal/dinosaur/pkg/presenters/data_plane_dinosaur_status.go index 2ed3f1add2..e1b0789f5d 100644 --- a/internal/dinosaur/pkg/presenters/data_plane_dinosaur_status.go +++ b/internal/dinosaur/pkg/presenters/data_plane_dinosaur_status.go @@ -29,10 +29,12 @@ func ConvertDataPlaneDinosaurStatus(status map[string]private.DataPlaneCentralSt }) } } + res = append(res, &dbapi.DataPlaneCentralStatus{ CentralClusterID: k, Conditions: c, Routes: routes, + Secrets: v.Secrets, // pragma: allowlist secret }) } diff --git a/internal/dinosaur/pkg/presenters/managedcentral.go b/internal/dinosaur/pkg/presenters/managedcentral.go index 567a704fad..52892066fc 100644 --- a/internal/dinosaur/pkg/presenters/managedcentral.go +++ b/internal/dinosaur/pkg/presenters/managedcentral.go @@ -59,7 +59,8 @@ func (c *ManagedCentralPresenter) PresentManagedCentral(from *dbapi.CentralReque MasId: from.ID, MasPlacementId: from.PlacementID, }, - Internal: from.Internal, + Internal: from.Internal, + SecretsStored: getSecretNames(from), // pragma: allowlist secret }, Spec: private.ManagedCentralAllOfSpec{ Owners: []string{ @@ -163,3 +164,20 @@ func orDefaultInt32(i int32, def int32) int32 { } return def } + +func getSecretNames(from *dbapi.CentralRequest) []string { + secrets, err := from.Secrets.Object() + if err != nil { + glog.Errorf("Failed to get Secrets as JSON object for Central request %q/%s: %v", from.Name, from.ClusterID, err) + return []string{} + } + + secretNames := make([]string, len(secrets)) + i := 0 + for k := range secrets { + secretNames[i] = k + i++ + } + + return secretNames +} diff --git a/internal/dinosaur/pkg/services/data_plane_dinosaur.go b/internal/dinosaur/pkg/services/data_plane_dinosaur.go index f057259a09..faae4a64e2 100644 --- a/internal/dinosaur/pkg/services/data_plane_dinosaur.go +++ b/internal/dinosaur/pkg/services/data_plane_dinosaur.go @@ -73,9 +73,8 @@ func (d *dataPlaneCentralService) UpdateDataPlaneCentralService(ctx context.Cont var e *serviceError.ServiceError switch s := getStatus(ks); s { case statusReady: - // Only store the routes (and create them) when the Dinosaurs are ready, as by the time they are ready, - // the routes should definitely be there. - e = d.persistCentralRoutes(dinosaur, ks, cluster) + // Persist values only known once central is in statusReady e.g. routes, secrets + e = d.persistCentralValues(dinosaur, ks, cluster) if e == nil { e = d.setCentralClusterReady(dinosaur) } @@ -217,7 +216,23 @@ func (d *dataPlaneCentralService) checkCentralRequestCurrentStatus(centralReques return matchStatus, nil } -func (d *dataPlaneCentralService) persistCentralRoutes(centralRequest *dbapi.CentralRequest, centralStatus *dbapi.DataPlaneCentralStatus, cluster *api.Cluster) *serviceError.ServiceError { +func (d *dataPlaneCentralService) persistCentralValues(centralRequest *dbapi.CentralRequest, centralStatus *dbapi.DataPlaneCentralStatus, cluster *api.Cluster) *serviceError.ServiceError { + if err := d.addRoutesToRequest(centralRequest, centralStatus, cluster); err != nil { + return err + } + + if err := d.addSecretsToRequest(centralRequest, centralStatus, cluster); err != nil { + return err + } + + if err := d.dinosaurService.Update(centralRequest); err != nil { + return serviceError.NewWithCause(err.Code, err, "failed to update routes for central cluster %s", centralRequest.ID) + } + + return nil +} + +func (d *dataPlaneCentralService) addRoutesToRequest(centralRequest *dbapi.CentralRequest, centralStatus *dbapi.DataPlaneCentralStatus, cluster *api.Cluster) *serviceError.ServiceError { if centralRequest.Routes != nil { logger.Logger.V(10).Infof("skip persisting routes for Central %s as they are already stored", centralRequest.ID) return nil @@ -238,9 +253,20 @@ func (d *dataPlaneCentralService) persistCentralRoutes(centralRequest *dbapi.Cen return serviceError.NewWithCause(serviceError.ErrorGeneral, err, "failed to set routes for central %s", centralRequest.ID) } - if err := d.dinosaurService.Update(centralRequest); err != nil { - return serviceError.NewWithCause(err.Code, err, "failed to update routes for central cluster %s", centralRequest.ID) + return nil +} + +func (d *dataPlaneCentralService) addSecretsToRequest(centralRequest *dbapi.CentralRequest, centralStatus *dbapi.DataPlaneCentralStatus, cluster *api.Cluster) *serviceError.ServiceError { + if centralRequest.Secrets != nil { // pragma: allowlist secret + logger.Logger.V(10).Infof("skip persisting secrets for Central %s as they are already stored", centralRequest.ID) + return nil } + logger.Logger.Infof("store secret information for central %s", centralRequest.ID) + + if err := centralRequest.SetSecrets(centralStatus.Secrets); err != nil { + return serviceError.NewWithCause(serviceError.ErrorGeneral, err, "failed to set secrets for central %s", centralRequest.ID) + } + return nil } diff --git a/openapi/fleet-manager-private.yaml b/openapi/fleet-manager-private.yaml index 0019e2f1cb..19795a685b 100644 --- a/openapi/fleet-manager-private.yaml +++ b/openapi/fleet-manager-private.yaml @@ -234,6 +234,10 @@ components: type: string deletionTimestamp: type: string + secretsStored: + type: array + items: + type: string spec: type: object properties: @@ -416,6 +420,12 @@ components: type: string router: type: string + secrets: + description: "Map of Secrets created for a Central" + type: object + additionalProperties: + type: string + example: $ref: "#/components/examples/DataPlaneCentralStatusRequestExample"