Release 2023 09 12.1.f6d113e (#1255)

## Description
<!-- Please include a summary of the change and a link to the JIRA
ticket. Please add any additional motivation and context as needed.
Screenshots are also welcome -->
Release 2023 09 12.1.f6d113e

.secrets.baseline

@@ -462,78 +462,78 @@
         "filename": "templates/service-template.yml",
         "hashed_secret": "13032f402fed753c2248419ea4f69f99931f6dbc",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "30025f80f6e22cdafb85db387d50f90ea884576a",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "355f24fd038bcaf85617abdcaa64af51ed19bbcf",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "3d8a1dcd2c3c765ce35c9a9552d23273cc4ddace",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "4ac7b0522761eba972467942cd5cd7499dd2c361",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "7639ab2a6bcf2ea30a055a99468c9cd844d4c22a",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "b56360daf4793d2a74991a972b34d95bc00fb2da",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "c9a73ef9ee8ce9f38437227801c70bcc6740d1a1",
         "is_verified": false,
-        "line_number": 552
+        "line_number": 557
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "14736999d9940728c5294277831a702f7882dece",
         "is_verified": false,
-        "line_number": 589
+        "line_number": 594
       },
       {
         "type": "Secret Keyword",
         "filename": "templates/service-template.yml",
         "hashed_secret": "4e199b4a1c40b497a95fcd1cd896351733849949",
         "is_verified": false,
-        "line_number": 676,
+        "line_number": 681,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "templates/service-template.yml",
         "hashed_secret": "9d51dabe59aa776bef2909d3689374ebb93ab2be",
         "is_verified": false,
-        "line_number": 720
+        "line_number": 725
       }
     ],
     "test/support/certs.json": [
@@ -564,5 +564,5 @@
       }
     ]
   },
-  "generated_at": "2023-07-19T10:20:12Z"
+  "generated_at": "2023-09-06T14:19:26Z"
 }

README.md

@@ -133,7 +133,7 @@ To contact the people that created this template go to [zulip](https://bf2.zulip
 - [Deploying fleet manager via Service Delivery](docs/legacy/onboarding-with-service-delivery.md)
 - [Data Plane Setup](docs/legacy/data-plane-osd-cluster-options.md)
 - [Access Control](docs/legacy/access-control.md)
-- [Quota Management](docs/legacy/quota-management-list-configuration.md)
+- [Quota Management](docs/quota/quota.md)
 - [Explanation of JWT token claims used across the fleet-manager](docs/auth/jwt-claims.md)
 
 ## Contributing

dev/env/defaults/05-docker.env

@@ -0,0 +1,5 @@
+# shellcheck shell=bash
+
+if [[ "$CLUSTER_NAME" == "docker" ]]; then
+    export CLUSTER_TYPE_DEFAULT="docker"
+fi

dev/env/defaults/cluster-type-docker/env

@@ -0,0 +1,7 @@
+export ENABLE_DB_PORT_FORWARDING_DEFAULT="true"
+export ENABLE_FM_PORT_FORWARDING_DEFAULT="true"
+export OPERATOR_SOURCE_DEFAULT="quay"
+export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
+export INSTALL_OLM_DEFAULT="true"
+export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
+export AWS_AUTH_HELPER_DEFAULT="aws-saml"

dev/env/scripts/exec_fleetshard_sync.sh

@@ -20,12 +20,12 @@ export AWS_AUTH_HELPER="${AWS_AUTH_HELPER:-aws-saml}"
 source "${GITROOT}/scripts/lib/external_config.sh"
 init_chamber
 
-CLUSTER_NAME="cluster-acs-dev-dp-01"
+CLUSTER_NAME="${CLUSTER_NAME:-cluster-acs-dev-dp-01}"
 
-ARGS="CLUSTER_ID=${CLUSTER_ID:-$(chamber read ${CLUSTER_NAME} ID -q -b ssm)} \
-    MANAGED_DB_SECURITY_GROUP=${MANAGED_DB_SECURITY_GROUP:-$(chamber read ${CLUSTER_NAME} MANAGED_DB_SECURITY_GROUP -q -b ssm)} \
-    MANAGED_DB_SUBNET_GROUP=${MANAGED_DB_SUBNET_GROUP:-$(chamber read ${CLUSTER_NAME} MANAGED_DB_SUBNET_GROUP -q -b ssm)} \
-    SECRET_ENCRYPTION_KEY_ID=${SECRET_ENCRYPTION_KEY_ID:-$(chamber read ${CLUSTER_NAME} SECRET_ENCRYPTION_KEY_ID -q -b ssm)} \
+ARGS="CLUSTER_ID=${CLUSTER_ID:-$(chamber read "${CLUSTER_NAME}" ID -q -b ssm)} \
+    MANAGED_DB_SECURITY_GROUP=${MANAGED_DB_SECURITY_GROUP:-$(chamber read "${CLUSTER_NAME}" MANAGED_DB_SECURITY_GROUP -q -b ssm)} \
+    MANAGED_DB_SUBNET_GROUP=${MANAGED_DB_SUBNET_GROUP:-$(chamber read "${CLUSTER_NAME}" MANAGED_DB_SUBNET_GROUP -q -b ssm)} \
+    SECRET_ENCRYPTION_KEY_ID=${SECRET_ENCRYPTION_KEY_ID:-$(chamber read "${CLUSTER_NAME}" SECRET_ENCRYPTION_KEY_ID -q -b ssm)} \
     AWS_ROLE_ARN=${FLEETSHARD_SYNC_AWS_ROLE_ARN:-$(chamber read fleetshard-sync AWS_ROLE_ARN -q -b ssm)} \
     $ARGS"
 

dev/env/scripts/lib.sh

@@ -299,7 +299,7 @@ inject_exported_env_vars() {
 
 is_local_cluster() {
     local cluster_type=${1:-}
-    if [[ "$cluster_type" == "minikube" || "$cluster_type" == "colima" || "$cluster_type" == "rancher-desktop" ]]; then
+    if [[ "$cluster_type" == "minikube" || "$cluster_type" == "colima" || "$cluster_type" == "rancher-desktop" || "$cluster_type" == "docker" ]]; then
         return 0
     else
         return 1

dev/env/scripts/up.sh

@@ -81,7 +81,7 @@ if [[ "$SPAWN_LOGGER" == "true" && -n "${LOG_DIR:-}" ]]; then
 fi
 
 # Sanity check.
-wait_for_container_to_become_ready "$ACSMS_NAMESPACE" "application=fleetshard-sync" "fleetshard-sync"
+wait_for_container_to_become_ready "$ACSMS_NAMESPACE" "application=fleetshard-sync" "fleetshard-sync" 500
 # Prerequisite for port-forwarding are pods in ready state.
 wait_for_container_to_become_ready "$ACSMS_NAMESPACE" "application=fleet-manager" "fleet-manager"
 

docs/development/howto-e2e-test-rds.md

@@ -25,8 +25,8 @@ At the point in time this documentation was written AWS RDS DB creation and dele
     # Prepare environment
     export AWS_AUTH_HELPER=aws-saml
     export MANAGED_DB_ENABLED=true
-
-    # flip the PublicAcessible flag to true in rds.go line 354
+    export CLUSTER_NAME=local_cluster
+    # flip the PubliclyAccessible flag to true in rds.go line 514
     make binary
 
     ./dev/env/scripts/exec_fleetshard_sync.sh

docs/legacy/architecture/quota-service-implementation.md

@@ -10,7 +10,7 @@ We have also provided another implementation based on the [quota-management-list
 
 When it is enabled, the following diagram describes the architecture for quota management service:
 
-![Quota Service Interface](../images/quoata-service.png)
+![Quota Service Interface](../images/quota-service.png)
 
 The `QuotaService` is defined in the [services package](../../internal/dinosaur/pkg/services/quota.go).
 

fleetshard/main.go

@@ -40,7 +40,10 @@ func main() {
 	glog.Infof("ManagedDB.SecurityGroup: %s", config.ManagedDB.SecurityGroup)
 	glog.Infof("ManagedDB.SubnetGroup: %s", config.ManagedDB.SubnetGroup)
 
-	runtime, err := runtime.NewRuntime(config, k8s.CreateClientOrDie())
+	glog.Info("Creating k8s client...")
+	k8sClient := k8s.CreateClientOrDie()
+	glog.Info("Creating runtime...")
+	runtime, err := runtime.NewRuntime(config, k8sClient)
 	if err != nil {
 		glog.Fatal(err)
 	}
@@ -52,6 +55,7 @@ func main() {
 		}
 	}()
 
+	glog.Info("Creating metrics server...")
 	metricServer := fleetshardmetrics.NewMetricsServer(config.MetricsAddress)
 	go func() {
 		if err := metricServer.ListenAndServe(); err != nil {
@@ -60,8 +64,10 @@ func main() {
 	}()
 
 	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, os.Interrupt, unix.SIGTERM)
+	notifySignals := []os.Signal{os.Interrupt, unix.SIGTERM}
+	signal.Notify(sigs, notifySignals...)
 
+	glog.Infof("Application started. Will shut down gracefully on %s.", notifySignals)
 	sig := <-sigs
 	runtime.Stop()
 	if err := metricServer.Close(); err != nil {

fleetshard/pkg/central/cloudprovider/awsclient/rds.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -15,11 +16,13 @@ import (
 	"github.com/stackrox/acs-fleet-manager/fleetshard/config"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/cloudprovider"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/postgres"
+	"k8s.io/apimachinery/pkg/util/rand"
 )
 
 const (
 	dbAvailableStatus = "available"
 	dbDeletingStatus  = "deleting"
+	dbBackingUpStatus = "backing-up"
 	dbUser            = "rhacs_master"
 	dbPrefix          = "rhacs-"
 	dbInstanceSuffix  = "-db-instance"
@@ -105,7 +108,15 @@ func (r *RDS) EnsureDBDeprovisioned(databaseID string, skipFinalSnapshot bool) e
 // to construct a PostgreSQL connection string. It expects that the database was already provisioned.
 func (r *RDS) GetDBConnection(databaseID string) (postgres.DBConnection, error) {
 	dbCluster, err := r.describeDBCluster(getClusterID(databaseID))
+
 	if err != nil {
+		var awsErr awserr.Error
+		if errors.As(err, &awsErr) {
+			if awsErr.Code() == rds.ErrCodeDBClusterNotFoundFault {
+				err = errors.Join(cloudprovider.ErrDBNotFound, err)
+			}
+		}
+
 		return postgres.DBConnection{}, err
 	}
 
@@ -154,7 +165,13 @@ func (r *RDS) ensureDBClusterCreated(clusterID, acsInstanceID, masterPassword st
 		return nil
 	}
 
+	finalSnapshotID, err := r.getFinalSnapshotIDIfExists(clusterID)
+	if err != nil {
+		return err
+	}
+
 	glog.Infof("Initiating provisioning of RDS database cluster %s.", clusterID)
+
 	input := &createCentralDBClusterInput{
 		clusterID:            clusterID,
 		acsInstanceID:        acsInstanceID,
@@ -164,14 +181,60 @@ func (r *RDS) ensureDBClusterCreated(clusterID, acsInstanceID, masterPassword st
 		dataplaneClusterName: r.dataplaneClusterName,
 		isTestInstance:       isTestInstance,
 	}
-	_, err = r.rdsClient.CreateDBCluster(newCreateCentralDBClusterInput(input))
+
+	rdsCreateDBClusterInput := newCreateCentralDBClusterInput(input)
+
+	if finalSnapshotID != "" {
+		glog.Infof("Restoring DB cluster: %s from snasphot: %s", clusterID, finalSnapshotID)
+		return r.restoreDBClusterFromSnapshot(finalSnapshotID, rdsCreateDBClusterInput)
+	}
+
+	return r.createDBCluster(rdsCreateDBClusterInput)
+}
+
+func (r *RDS) restoreDBClusterFromSnapshot(snapshotID string, clusterInput *rds.CreateDBClusterInput) error {
+	_, err := r.rdsClient.RestoreDBClusterFromSnapshot(newRestoreCentralDBClusterInput(snapshotID, clusterInput))
+	if err != nil {
+		return fmt.Errorf("restoring DB cluster: %w", err)
+	}
+
+	return nil
+}
+
+func (r *RDS) createDBCluster(clusterInput *rds.CreateDBClusterInput) error {
+	_, err := r.rdsClient.CreateDBCluster(clusterInput)
 	if err != nil {
 		return fmt.Errorf("creating DB cluster: %w", err)
 	}
 
 	return nil
 }
 
+func (r *RDS) getFinalSnapshotIDIfExists(clusterID string) (string, error) {
+	snapshotsOut, err := r.rdsClient.DescribeDBClusterSnapshots(&rds.DescribeDBClusterSnapshotsInput{
+		DBClusterIdentifier: &clusterID,
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("checking if final snapshot for clusterID: %s exists: %w", clusterID, err)
+	}
+
+	var mostRecentSnapshotID string
+	var mostRecentSnapshotTime *time.Time
+	for _, snapshot := range snapshotsOut.DBClusterSnapshots {
+		if !strings.Contains(*snapshot.DBClusterSnapshotIdentifier, "final") {
+			continue
+		}
+
+		if mostRecentSnapshotTime == nil || mostRecentSnapshotTime.Before(*snapshot.SnapshotCreateTime) {
+			mostRecentSnapshotID = *snapshot.DBClusterSnapshotIdentifier
+			mostRecentSnapshotTime = snapshot.SnapshotCreateTime
+		}
+	}
+
+	return mostRecentSnapshotID, nil
+}
+
 func (r *RDS) ensureDBInstanceCreated(instanceID, clusterID, acsInstanceID string, isTestInstance bool) error {
 	instanceExists, _, err := r.instanceStatus(instanceID)
 	if err != nil {
@@ -227,6 +290,10 @@ func (r *RDS) ensureClusterDeleted(clusterID string, skipFinalSnapshot bool) err
 		return nil
 	}
 
+	if clusterStatus == dbBackingUpStatus {
+		return cloudprovider.ErrDBBackupInProgress
+	}
+
 	if clusterStatus != dbDeletingStatus {
 		glog.Infof("Initiating deprovisioning of RDS database cluster %s.", clusterID)
 		_, err := r.rdsClient.DeleteDBCluster(newDeleteCentralDBClusterInput(clusterID, skipFinalSnapshot))
@@ -415,6 +482,23 @@ func newCreateCentralDBClusterInput(input *createCentralDBClusterInput) *rds.Cre
 	return awsInput
 }
 
+func newRestoreCentralDBClusterInput(snapshotID string, input *rds.CreateDBClusterInput) *rds.RestoreDBClusterFromSnapshotInput {
+	restoreInput := &rds.RestoreDBClusterFromSnapshotInput{
+		DBClusterIdentifier:              input.DBClusterIdentifier,
+		Engine:                           input.Engine,
+		EngineVersion:                    input.EngineVersion,
+		VpcSecurityGroupIds:              input.VpcSecurityGroupIds,
+		PubliclyAccessible:               input.PubliclyAccessible,
+		DBSubnetGroupName:                input.DBSubnetGroupName,
+		ServerlessV2ScalingConfiguration: input.ServerlessV2ScalingConfiguration,
+		Tags:                             input.Tags,
+		SnapshotIdentifier:               &snapshotID,
+		EnableCloudwatchLogsExports:      input.EnableCloudwatchLogsExports,
+	}
+
+	return restoreInput
+}
+
 type createCentralDBInstanceInput struct {
 	clusterID            string
 	instanceID           string
@@ -482,7 +566,7 @@ func newRdsClient() (*rds.RDS, error) {
 }
 
 func getFinalSnapshotID(clusterID string) *string {
-	return aws.String(fmt.Sprintf("%s-%s", clusterID, "final"))
+	return aws.String(fmt.Sprintf("%s-%s-%s", clusterID, rand.String(10), "final"))
 }
 
 func getInstanceType(isTestInstance bool) string {