Release 2023 01 24.1.e67c2c6 (#760)

.secrets.baseline

@@ -322,7 +322,7 @@
         "filename": "fleetshard/pkg/central/cloudprovider/dbclient_moq.go",
         "hashed_secret": "80519927d0f3ce1efe933f46ca9e05e68e491adc",
         "is_verified": false,
-        "line_number": 106
+        "line_number": 118
       }
     ],
     "internal/dinosaur/pkg/api/public/api/openapi.yaml": [
@@ -444,78 +444,78 @@
         "filename": "templates/service-template.yml",
         "hashed_secret": "13032f402fed753c2248419ea4f69f99931f6dbc",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "30025f80f6e22cdafb85db387d50f90ea884576a",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "355f24fd038bcaf85617abdcaa64af51ed19bbcf",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "3d8a1dcd2c3c765ce35c9a9552d23273cc4ddace",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "4ac7b0522761eba972467942cd5cd7499dd2c361",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "7639ab2a6bcf2ea30a055a99468c9cd844d4c22a",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "b56360daf4793d2a74991a972b34d95bc00fb2da",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "c9a73ef9ee8ce9f38437227801c70bcc6740d1a1",
         "is_verified": false,
-        "line_number": 542
+        "line_number": 547
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "templates/service-template.yml",
         "hashed_secret": "14736999d9940728c5294277831a702f7882dece",
         "is_verified": false,
-        "line_number": 579
+        "line_number": 584
       },
       {
         "type": "Secret Keyword",
         "filename": "templates/service-template.yml",
         "hashed_secret": "4e199b4a1c40b497a95fcd1cd896351733849949",
         "is_verified": false,
-        "line_number": 666,
+        "line_number": 671,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "templates/service-template.yml",
         "hashed_secret": "9d51dabe59aa776bef2909d3689374ebb93ab2be",
         "is_verified": false,
-        "line_number": 710
+        "line_number": 715
       }
     ],
     "test/support/certs.json": [
@@ -546,5 +546,5 @@
       }
     ]
   },
-  "generated_at": "2023-01-13T14:02:09Z"
+  "generated_at": "2023-01-23T01:51:04Z"
 }

CHANGELOG.md

@@ -9,6 +9,15 @@ This Changelog should be updated for:
 ## [NEXT RELEASE]
 ### Added
 ### Changed
+- Collected logs in AWS CloudWatch are grouped by log type instead of namespace
+### Deprecated
+### Removed
+
+## 2023-01-17.1.f4e71a7
+### Added
+### Changed
+- Updated operator to version 3.73.1
+- Request the "api.iam.clients" for dynamic client API calls
 ### Deprecated
 ### Removed
 

Makefile

@@ -481,7 +481,7 @@ db/start:
 .PHONY: db/start
 
 db/migrate:
-	OCM_ENV=integration $(GO) run ./cmd/fleet-manager migrate
+	$(GO) run ./cmd/fleet-manager migrate
 .PHONY: db/migrate
 
 db/teardown:

cmd/fleet-manager/main_test.go

@@ -38,7 +38,7 @@ func TestInjections(t *testing.T) {
 
 	var bootList []environments.BootService
 	env.MustResolve(&bootList)
-	Expect(len(bootList)).To(Equal(4))
+	Expect(len(bootList)).To(Equal(5))
 
 	_, ok := bootList[0].(*server.APIServer)
 	Expect(ok).To(Equal(true))

dp-terraform/helm/rhacs-terraform/charts/logging/templates/01-logging-03-cluster-log-forwarder.yaml

@@ -12,7 +12,8 @@ spec:
     - name: cloudwatch-output
       type: cloudwatch
       cloudwatch:
-        groupBy: namespaceName
+        groupBy: "logType"
+        groupPrefix: {{ .Values.groupPrefix | quote }}
         region: {{ .Values.aws.region | quote }}
       secret:
         name: cloudwatch

dp-terraform/helm/rhacs-terraform/charts/logging/values.yaml

@@ -2,6 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
+groupPrefix: ""
 aws:
   region: "us-east-1"
   accessKeyId: ""

dp-terraform/helm/rhacs-terraform/terraform_cluster.sh

@@ -105,6 +105,7 @@ helm upgrade rhacs-terraform "${SCRIPT_DIR}" ${HELM_DEBUG_FLAGS:-} \
   --set fleetshardSync.aws.roleARN="${FLEETSHARD_SYNC_AWS_ROLE_ARN}" \
   --set fleetshardSync.telemetry.storage.endpoint="${FLEETSHARD_SYNC_TELEMETRY_STORAGE_ENDPOINT:-}" \
   --set fleetshardSync.telemetry.storage.key="${FLEETSHARD_SYNC_TELEMETRY_STORAGE_KEY:-}" \
+  --set logging.groupPrefix="${CLUSTER_NAME}" \
   --set logging.aws.accessKeyId="${LOGGING_AWS_ACCESS_KEY_ID}" \
   --set logging.aws.secretAccessKey="${LOGGING_AWS_SECRET_ACCESS_KEY}" \
   --set observability.github.accessToken="${OBSERVABILITY_GITHUB_ACCESS_TOKEN}" \

dp-terraform/helm/rhacs-terraform/values.yaml

@@ -67,6 +67,7 @@ observability:
 # - enabled flag is used to completely enable/disable logging sub-chart
 logging:
   enabled: true
+  groupPrefix: ""
   aws:
     accessKeyId: ""
     secretAccessKey: ""

fleetshard/pkg/central/cloudprovider/awsclient/rds.go

@@ -55,66 +55,82 @@ type RDS struct {
 }
 
 // EnsureDBProvisioned is a blocking function that makes sure that an RDS database was provisioned for a Central
-func (r *RDS) EnsureDBProvisioned(ctx context.Context, databaseID, masterPassword string) (*postgres.DBConnection, error) {
+func (r *RDS) EnsureDBProvisioned(ctx context.Context, databaseID, masterPassword string) error {
 	clusterID := getClusterID(databaseID)
 	instanceID := getInstanceID(databaseID)
 
 	if err := r.ensureDBClusterCreated(clusterID, masterPassword); err != nil {
-		return nil, fmt.Errorf("ensuring DB cluster %s exists: %w", clusterID, err)
+		return fmt.Errorf("ensuring DB cluster %s exists: %w", clusterID, err)
 	}
 
 	if err := r.ensureDBInstanceCreated(instanceID, clusterID); err != nil {
-		return nil, fmt.Errorf("ensuring DB instance %s exists in cluster %s: %w", instanceID, clusterID, err)
+		return fmt.Errorf("ensuring DB instance %s exists in cluster %s: %w", instanceID, clusterID, err)
 	}
 
 	return r.waitForInstanceToBeAvailable(ctx, instanceID, clusterID)
 }
 
 // EnsureDBDeprovisioned is a function that initiates the deprovisioning of the RDS database of a Central
 // Unlike EnsureDBProvisioned, this function does not block until the DB is deprovisioned
-func (r *RDS) EnsureDBDeprovisioned(databaseID string) (bool, error) {
+func (r *RDS) EnsureDBDeprovisioned(databaseID string) error {
 	clusterID := getClusterID(databaseID)
 	instanceID := getInstanceID(databaseID)
 
 	instanceExists, err := r.instanceExists(instanceID)
 	if err != nil {
-		return false, fmt.Errorf("checking if DB instance exists: %w", err)
+		return fmt.Errorf("checking if DB instance exists: %w", err)
 	}
 	if instanceExists {
 		status, err := r.instanceStatus(instanceID)
 		if err != nil {
-			return false, fmt.Errorf("getting DB instance status: %w", err)
+			return fmt.Errorf("getting DB instance status: %w", err)
 		}
 		if status != dbDeletingStatus {
 			glog.Infof("Initiating deprovisioning of RDS database instance %s.", instanceID)
 			// TODO(ROX-13692): do not skip taking a final DB snapshot
 			_, err := r.rdsClient.DeleteDBInstance(newDeleteCentralDBInstanceInput(instanceID, true))
 			if err != nil {
-				return false, fmt.Errorf("deleting DB instance: %w", err)
+				return fmt.Errorf("deleting DB instance: %w", err)
 			}
 		}
 	}
 
 	clusterExists, err := r.clusterExists(clusterID)
 	if err != nil {
-		return false, fmt.Errorf("checking if DB cluster exists: %w", err)
+		return fmt.Errorf("checking if DB cluster exists: %w", err)
 	}
 	if clusterExists {
 		status, err := r.clusterStatus(clusterID)
 		if err != nil {
-			return false, fmt.Errorf("getting DB cluster status: %w", err)
+			return fmt.Errorf("getting DB cluster status: %w", err)
 		}
 		if status != dbDeletingStatus {
 			glog.Infof("Initiating deprovisioning of RDS database cluster %s.", clusterID)
 			// TODO(ROX-13692): do not skip taking a final DB snapshot
 			_, err := r.rdsClient.DeleteDBCluster(newDeleteCentralDBClusterInput(clusterID, true))
 			if err != nil {
-				return false, fmt.Errorf("deleting DB cluster: %w", err)
+				return fmt.Errorf("deleting DB cluster: %w", err)
 			}
 		}
 	}
 
-	return true, nil
+	return nil
+}
+
+// GetDBConnection returns a postgres.DBConnection struct, which contains the data necessary
+// to construct a PostgreSQL connection string. It expects that the database was already provisioned.
+func (r *RDS) GetDBConnection(databaseID string) (postgres.DBConnection, error) {
+	dbCluster, err := r.describeDBCluster(getClusterID(databaseID))
+	if err != nil {
+		return postgres.DBConnection{}, err
+	}
+
+	connection, err := postgres.NewDBConnection(*dbCluster.Endpoint, dbPostgresPort, dbUser, dbName)
+	if err != nil {
+		return postgres.DBConnection{}, fmt.Errorf("incorrect DB connection parameters: %w", err)
+	}
+
+	return connection, nil
 }
 
 func (r *RDS) ensureDBClusterCreated(clusterID, masterPassword string) error {
@@ -234,25 +250,15 @@ func (r *RDS) describeDBCluster(clusterID string) (*rds.DBCluster, error) {
 	return result.DBClusters[0], nil
 }
 
-func (r *RDS) waitForInstanceToBeAvailable(ctx context.Context, instanceID string, clusterID string) (*postgres.DBConnection, error) {
+func (r *RDS) waitForInstanceToBeAvailable(ctx context.Context, instanceID string, clusterID string) error {
 	for {
 		dbInstanceStatus, err := r.instanceStatus(instanceID)
 		if err != nil {
-			return nil, err
+			return err
 		}
 
 		if dbInstanceStatus == dbAvailableStatus {
-			dbCluster, err := r.describeDBCluster(clusterID)
-			if err != nil {
-				return nil, err
-			}
-
-			connection, err := postgres.NewDBConnection(*dbCluster.Endpoint, dbPostgresPort, dbUser, dbName)
-			if err != nil {
-				return nil, fmt.Errorf("incorrect DB connection parameters: %w", err)
-			}
-
-			return &connection, nil
+			return nil
 		}
 
 		glog.Infof("RDS instance status: %s (instance ID: %s)", dbInstanceStatus, instanceID)
@@ -261,7 +267,7 @@ func (r *RDS) waitForInstanceToBeAvailable(ctx context.Context, instanceID strin
 		case <-ticker.C:
 			continue
 		case <-ctx.Done():
-			return nil, fmt.Errorf("waiting for RDS instance to be available: %w", ctx.Err())
+			return fmt.Errorf("waiting for RDS instance to be available: %w", ctx.Err())
 		}
 	}
 }

fleetshard/pkg/central/cloudprovider/awsclient/rds_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/rds"
 	"github.com/google/uuid"
@@ -16,6 +17,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const awsTimeoutMinutes = 15
+
 func newTestRDS() (*RDS, error) {
 	rdsClient, err := newTestRDSClient()
 	if err != nil {
@@ -71,7 +74,7 @@ func TestRDSProvisioning(t *testing.T) {
 	rdsClient, err := newTestRDS()
 	require.NoError(t, err)
 
-	ctx, cancel := context.WithTimeout(context.TODO(), 15*time.Minute)
+	ctx, cancel := context.WithTimeout(context.TODO(), awsTimeoutMinutes*time.Minute)
 	defer cancel()
 
 	dbID := "test-" + uuid.New().String()
@@ -89,7 +92,10 @@ func TestRDSProvisioning(t *testing.T) {
 	require.NoError(t, err)
 	require.False(t, instanceExists)
 
-	_, err = rdsClient.EnsureDBProvisioned(ctx, dbID, dbMasterPassword)
+	err = rdsClient.EnsureDBProvisioned(ctx, dbID, dbMasterPassword)
+	assert.NoError(t, err)
+
+	_, err = rdsClient.GetDBConnection(dbID)
 	assert.NoError(t, err)
 
 	clusterExists, err = rdsClient.clusterExists(clusterID)
@@ -108,14 +114,27 @@ func TestRDSProvisioning(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, instanceStatus, dbAvailableStatus)
 
-	deletionStarted, err := rdsClient.EnsureDBDeprovisioned(dbID)
+	err = rdsClient.EnsureDBDeprovisioned(dbID)
 	assert.NoError(t, err)
-	assert.True(t, deletionStarted)
 
-	deleteCtx, deleteCancel := context.WithTimeout(context.TODO(), 10*time.Minute)
+	deleteCtx, deleteCancel := context.WithTimeout(context.TODO(), awsTimeoutMinutes*time.Minute)
 	defer deleteCancel()
 
 	clusterDeleted, err := waitForClusterToBeDeleted(deleteCtx, rdsClient, clusterID)
 	require.NoError(t, err)
 	assert.True(t, clusterDeleted)
 }
+
+func TestGetDBConnection(t *testing.T) {
+	if os.Getenv("RUN_RDS_TESTS") != "true" {
+		t.Skip("Skip RDS tests. Set RUN_RDS_TESTS=true env variable to enable RDS tests.")
+	}
+
+	rdsClient, err := newTestRDS()
+	require.NoError(t, err)
+
+	_, err = rdsClient.GetDBConnection("test-" + uuid.New().String())
+	var awsErr awserr.Error
+	require.ErrorAs(t, err, &awsErr)
+	assert.Equal(t, awsErr.Code(), rds.ErrCodeDBClusterNotFoundFault)
+}

fleetshard/pkg/central/cloudprovider/dbclient.go

@@ -13,8 +13,11 @@ import (
 type DBClient interface {
 	// EnsureDBProvisioned is a blocking function that makes sure that a database with the given databaseID was provisioned,
 	// using the master password given as parameter
-	EnsureDBProvisioned(ctx context.Context, databaseID, passwordSecretName string) (*postgres.DBConnection, error)
+	EnsureDBProvisioned(ctx context.Context, databaseID, passwordSecretName string) error
 	// EnsureDBDeprovisioned is a non-blocking function that makes sure that a managed DB is deprovisioned (more
 	// specifically, that its deletion was initiated)
-	EnsureDBDeprovisioned(databaseID string) (bool, error)
+	EnsureDBDeprovisioned(databaseID string) error
+	// GetDBConnection returns a postgres.DBConnection struct, which contains the data necessary
+	// to construct a PostgreSQL connection string. It expects that the database was already provisioned.
+	GetDBConnection(databaseID string) (postgres.DBConnection, error)
 }