ROX-19013 Add gitops to fleetmanager (#1233)

internal/dinosaur/pkg/gitops/README.md

@@ -0,0 +1,14 @@
+# GitOps Workflow
+
+![GitOps workflow](gitops-workflow.png)
+
+1. `fleetshard` polls `fleetmanager` for a list of `Centrals` by sending an api request
+2. `fleetmanager` lists the central instances from the database
+3. `fleetmanager` applies the default configuration to the central instances
+4. `fleetmanager` retrieves the gitops configuration
+5. `fleetmanager` applies the gitops configuration to the central instances
+6. `fleetmanager` returns the list of central instances to `fleetshard`
+7. `fleetshard` applies the cluster-specific configuration/overrides to the central instances
+8. `fleetshard` performs reconciliation of the central instances
+
+The `gitops` configuration repository is located at https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/config

internal/dinosaur/pkg/gitops/config.go

@@ -3,8 +3,8 @@ package gitops
 
 import (
 	"github.com/stackrox/rox/operator/apis/platform/v1alpha1"
-	"gopkg.in/yaml.v2"
-	field "k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"sigs.k8s.io/yaml"
 )
 
 // Config represents the gitops configuration
@@ -14,8 +14,6 @@ type Config struct {
 
 // CentralsConfig represents the declarative configuration for Central instances defaults and overrides.
 type CentralsConfig struct {
-	// Default configuration for Central instances.
-	Default v1alpha1.Central `json:"default"`
 	// Overrides are the overrides for Central instances.
 	Overrides []CentralOverride `json:"overrides"`
 }

internal/dinosaur/pkg/gitops/config_test.go

@@ -24,20 +24,19 @@ func TestValidateGitOpsConfig(t *testing.T) {
 			},
 			yaml: `
 centrals:
-  default: {}
   overrides:
-  - instanceId: id1
+  - instanceIds:
+    - id1
     patch: |
       {}`,
 		}, {
 			name: "invalid yaml in patch",
 			assert: func(t *testing.T, c *Config, err field.ErrorList) {
 				require.Len(t, err, 1)
-				assert.Equal(t, field.Invalid(field.NewPath("centrals", "overrides").Index(0).Child("patch"), "foo", "invalid patch: yaml: unmarshal errors:\n  line 1: cannot unmarshal !!str `foo` into v1alpha1.Central"), err[0])
+				assert.Equal(t, field.Invalid(field.NewPath("centrals", "overrides").Index(0).Child("patch"), "foo", "invalid patch: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type v1alpha1.Central"), err[0])
 			},
 			yaml: `
 centrals:
-  default: {}
   overrides:
   - instanceIds:
     - id1
@@ -47,11 +46,10 @@ centrals:
 			name: "patch contains un-mergeable fields",
 			assert: func(t *testing.T, c *Config, err field.ErrorList) {
 				require.Len(t, err, 1)
-				assert.Equal(t, field.Invalid(field.NewPath("centrals", "overrides").Index(0).Child("patch"), "spec: 123\n", "invalid patch: yaml: unmarshal errors:\n  line 1: cannot unmarshal !!int `123` into v1alpha1.CentralSpec"), err[0])
+				assert.Equal(t, field.Invalid(field.NewPath("centrals", "overrides").Index(0).Child("patch"), "spec: 123\n", "invalid patch: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal number into Go struct field Central.spec of type v1alpha1.CentralSpec"), err[0])
 			},
 			yaml: `
 centrals:
-  default: {}
   overrides:
   - instanceIds:
     - id1

internal/dinosaur/pkg/gitops/default_central.go

@@ -0,0 +1,6 @@
+package gitops
+
+import _ "embed"
+
+//go:embed default_central.yaml
+var defaultCentralTemplate []byte

internal/dinosaur/pkg/gitops/default_central.yaml

@@ -0,0 +1,52 @@
+metadata:
+  name: "{{ .Name }}"
+  namespace: "{{ .Namespace }}"
+  labels:
+    rhacs.redhat.com/instance-type: "{{ .InstanceType }}"
+    rhacs.redhat.com/org-id: "{{ .OrganizationID }}"
+    rhacs.redhat.com/tenant: "{{ .ID }}"
+  annotations:
+    platform.stackrox.io/managed-services: "true"
+    rhacs.redhat.com/org-name: {{ .OrganizationName }}
+spec:
+  central:
+    adminPasswordGenerationDisabled: true #pragma: allowlist secret
+    # db: {} -- managed by fleetshard-sync
+    # exposure: {} -- managed by fleetshard-sync
+    monitoring:
+      exposeEndpoint: Enabled
+      openshift:
+        enabled: false
+    resources:
+      limits:
+        cpu: "4"
+        memory: 8Gi
+      requests:
+        cpu: "2"
+        memory: 4Gi
+    # telemetry: {} -- managed by fleetshard-sync
+  scanner:
+    analyzer:
+      resources:
+        limits:
+          cpu: "3"
+          memory: 8Gi
+        requests:
+          cpu: "1.5"
+          memory: 4Gi
+      scaling:
+        autoScaling: Enabled
+        maxReplicas: 3
+        minReplicas: 1
+        replicas: 1
+    scannerComponent: Enabled
+    db:
+      resources:
+        limits:
+          cpu: "2.5"
+          memory: 4Gi
+        requests:
+          cpu: "1.25"
+          memory: 2Gi
+    monitoring:
+      exposeEndpoint: Enabled

internal/dinosaur/pkg/gitops/gitops-workflow.puml

@@ -0,0 +1,28 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+
+autonumber
+
+box FleetShard
+participant FS as "FleetShard"
+end box
+
+box "FleetManager"
+participant FM as "FleetManager"
+participant DC as "Default Central"
+participant DB as "Database"
+participant GitOps as GitOps
+end box
+
+FS -> FM: Poll Centrals
+FM -> DB: List Instances
+FM -> DC: Get Default Central
+FM -> FM: Apply Defaults to List
+FM -> GitOps: Get GitOps Config
+FM -> FM: Apply GitOps Config to List
+FM -> FS: Central List
+FS -> FS: Apply Cluster-Specific Defaults
+FS -> FS: Reconcile
+
+
+@enduml

internal/dinosaur/pkg/gitops/provider.go

@@ -0,0 +1,76 @@
+package gitops
+
+import (
+	"sync/atomic"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	errorCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "dinosaur_gitops_config_provider_error_total",
+		Help: "Number of errors encountered by the GitOps configuration provider.",
+	}, []string{})
+)
+
+func init() {
+	prometheus.MustRegister(errorCounter)
+}
+
+// ConfigProvider is the interface for GitOps configuration providers.
+type ConfigProvider interface {
+	// Get returns the GitOps configuration.
+	Get() (Config, error)
+}
+
+type validationFn func(config Config) error
+
+type provider struct {
+	reader            Reader
+	lastWorkingConfig atomic.Pointer[Config]
+	validationFn      validationFn
+}
+
+// NewProvider returns a new ConfigProvider.
+func NewProvider(reader Reader) ConfigProvider {
+	return &provider{
+		reader:            reader,
+		lastWorkingConfig: atomic.Pointer[Config]{},
+		validationFn: func(config Config) error {
+			return ValidateConfig(config).ToAggregate()
+		},
+	}
+}
+
+// Get implements ConfigProvider.Get
+func (p *provider) Get() (Config, error) {
+	// Load the config from the reader
+	cfg, err := p.reader.Read()
+	if err != nil {
+		p.increaseErrorCount()
+		return p.tryGetLastWorkingConfig(errors.Wrap(err, "failed to read GitOps configuration"))
+	}
+	// Validate the config
+	if err := p.validationFn(cfg); err != nil {
+		p.increaseErrorCount()
+		return p.tryGetLastWorkingConfig(errors.Wrap(err, "failed to validate GitOps configuration"))
+	}
+	// Store the config as the last working config
+	p.lastWorkingConfig.Store(&cfg)
+	return cfg, nil
+}
+
+func (p *provider) increaseErrorCount() {
+	errorCounter.WithLabelValues().Inc()
+}
+
+func (p *provider) tryGetLastWorkingConfig(err error) (Config, error) {
+	lastWorkingConfig := p.lastWorkingConfig.Load()
+	if lastWorkingConfig == nil {
+		return Config{}, errors.Wrap(err, "no last working gitops config available")
+	}
+	glog.Warningf("Failed to get GitOps configuration. Using last working config: %s", err)
+	return *lastWorkingConfig, nil
+}

internal/dinosaur/pkg/gitops/provider_test.go

@@ -0,0 +1,133 @@
+package gitops
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestProvider_Get(t *testing.T) {
+
+	var failingValidation validationFn = func(config Config) error {
+		return assert.AnError
+	}
+	var successfulValidation validationFn = func(config Config) error {
+		return nil
+	}
+	var failingReader Reader = &mockReader{err: assert.AnError}
+	var successfulReader Reader = &mockReader{config: Config{}}
+
+	type tc struct {
+		name                     string
+		hasLastWorkingConfig     bool
+		reader                   Reader
+		validator                validationFn
+		expectedErrorMetricCount int
+		expectError              bool
+	}
+	tcs := []tc{
+		{
+			name:                     "Successful without last working config",
+			hasLastWorkingConfig:     false,
+			reader:                   successfulReader,
+			validator:                successfulValidation,
+			expectedErrorMetricCount: 0,
+			expectError:              false,
+		}, {
+			name:                     "Successful with last working config",
+			hasLastWorkingConfig:     true,
+			reader:                   successfulReader,
+			validator:                successfulValidation,
+			expectedErrorMetricCount: 0,
+			expectError:              false,
+		}, {
+			name:                     "Reader fails without last working config",
+			hasLastWorkingConfig:     false,
+			reader:                   failingReader,
+			validator:                successfulValidation,
+			expectedErrorMetricCount: 1,
+			expectError:              true,
+		}, {
+			name:                     "Reader fails with last working config",
+			hasLastWorkingConfig:     true,
+			reader:                   failingReader,
+			validator:                successfulValidation,
+			expectedErrorMetricCount: 1,
+			expectError:              false,
+		}, {
+			name:                     "Validation fails without last working config",
+			hasLastWorkingConfig:     false,
+			reader:                   failingReader,
+			validator:                failingValidation,
+			expectedErrorMetricCount: 1,
+			expectError:              true,
+		}, {
+			name:                     "Validation fails with last working config",
+			hasLastWorkingConfig:     true,
+			reader:                   failingReader,
+			validator:                failingValidation,
+			expectedErrorMetricCount: 1,
+			expectError:              false,
+		},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			p := &provider{}
+			p.lastWorkingConfig = atomic.Pointer[Config]{}
+
+			if tc.hasLastWorkingConfig {
+				// Get the config once to set the last working config
+				p.reader = successfulReader
+				p.validationFn = successfulValidation
+				_, err := p.Get()
+				require.NoError(t, err)
+			}
+
+			p.reader = tc.reader
+			p.validationFn = tc.validator
+
+			errorCounter.Reset()
+			_, err := p.Get()
+			if tc.expectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			count := testutil.CollectAndCount(errorCounter)
+			assert.Equal(t, tc.expectedErrorMetricCount, count)
+
+		})
+	}
+}
+
+type mockReader struct {
+	config Config
+	err    error
+}
+
+func NewMockReader(config Config) *mockReader {
+	return &mockReader{
+		config: Config{},
+		err:    nil,
+	}
+}
+
+func (r *mockReader) Read() (Config, error) {
+	return r.config, r.err
+}
+
+func (r *mockReader) WillFail() *mockReader {
+	r.err = assert.AnError
+	return r
+}
+
+func (r *mockReader) WillSucceed() *mockReader {
+	r.err = nil
+	return r
+}
+
+var _ Reader = &mockReader{}