ROX-26039: Create tenant argoCd app

fleetshard/config/config.go

@@ -35,13 +35,16 @@ type Config struct {
 	// If it is empty, nothing is injected (for example, it is not required when running on OpenShift).
 	// It is required when central images need to fetched from a private Quay registry.
 	// It needs to given as Docker Config JSON object.
-	TenantImagePullSecret string `env:"TENANT_IMAGE_PULL_SECRET"`
-	ManagedDB             ManagedDB
-	Telemetry             Telemetry
-	AuditLogging          AuditLogging
-	SecretEncryption      SecretEncryption
-	RouteParameters       RouteConfig
-	FleetshardAddonName   string `env:"FLEETSHARD_ADDON_NAME" envDefault:"acs-fleetshard"`
+	TenantImagePullSecret               string `env:"TENANT_IMAGE_PULL_SECRET"`
+	DefaultTenantArgoCdAppSourceRepoURL string `env:"TENANT_ARGOCD_APP_SOURCE_REPO_URL_DEFAULT" envDefault:"https://github.com/stackrox/acscs-manifests.git"`
+	DefaultTenantArgoCdAppSourceRef     string `env:"TENANT_ARGOCD_APP_SOURCE_REF_DEFAULT" envDefault:"HEAD"`
+	DefaultTenantArgoCdAppSourcePath    string `env:"TENANT_ARGOCD_APP_SOURCE_PATH_DEFAULT" envDefault:"tenant-resources"`
+	ManagedDB                           ManagedDB
+	Telemetry                           Telemetry
+	AuditLogging                        AuditLogging
+	SecretEncryption                    SecretEncryption
+	RouteParameters                     RouteConfig
+	FleetshardAddonName                 string `env:"FLEETSHARD_ADDON_NAME" envDefault:"acs-fleetshard"`
 
 	// The SecureTenantNetwork option controls whether the Tenant's K8s
 	// Namespace will be secured at the network level, e.g. by using

fleetshard/pkg/central/reconciler/reconciler.go

@@ -13,6 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	argocd "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/golang/glog"
 	"github.com/hashicorp/go-multierror"
 	openshiftRouteV1 "github.com/openshift/api/route/v1"
@@ -40,6 +41,7 @@ import (
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/pointer"
@@ -115,16 +117,19 @@ type encryptedSecrets struct {
 
 // CentralReconcilerOptions are the static options for creating a reconciler.
 type CentralReconcilerOptions struct {
-	UseRoutes             bool
-	WantsAuthProvider     bool
-	ManagedDBEnabled      bool
-	Telemetry             config.Telemetry
-	ClusterName           string
-	Environment           string
-	AuditLogging          config.AuditLogging
-	TenantImagePullSecret string
-	RouteParameters       config.RouteConfig
-	SecureTenantNetwork   bool
+	UseRoutes                           bool
+	WantsAuthProvider                   bool
+	ManagedDBEnabled                    bool
+	Telemetry                           config.Telemetry
+	ClusterName                         string
+	Environment                         string
+	AuditLogging                        config.AuditLogging
+	TenantImagePullSecret               string
+	RouteParameters                     config.RouteConfig
+	SecureTenantNetwork                 bool
+	DefaultTenantArgoCdAppSourceRepoURL string
+	DefaultTenantArgoCdAppSourceRef     string
+	DefaultTenantArgoCdAppSourcePath    string
 }
 
 // CentralReconciler is a reconciler tied to a one Central instance. It installs, updates and deletes Central instances
@@ -163,6 +168,10 @@ type CentralReconciler struct {
 	areSecretsStoredFunc      areSecretsStoredFunc
 	needsReconcileFunc        needsReconcileFunc
 	restoreCentralSecretsFunc restoreCentralSecretsFunc
+
+	defaultTenantArgoCdAppSourceRepoURL string
+	defaultTenantArgoCdAppSourceRef     string
+	defaultTenantArgoCdAppSourcePath    string
 }
 
 // Reconcile takes a private.ManagedCentral and tries to install it into the cluster managed by the fleet-shard.
@@ -236,8 +245,26 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 		return nil, err
 	}
 
-	if err := r.ensureChartResourcesExist(ctx, remoteCentral); err != nil {
-		return nil, errors.Wrapf(err, "unable to install chart resource for central %s/%s", central.GetNamespace(), central.GetName())
+	if remoteCentral.Spec.ArgoCd.Enabled {
+		if err := r.ensureArgoCdApplicationExists(ctx, remoteCentral); err != nil {
+			return nil, errors.Wrapf(err, "unable to install ArgoCD application for central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name)
+		}
+	} else {
+
+		{
+			// This little part would only happen if we enable, then disable ArgoCD for a tenant
+			ok, err := r.ensureArgoCdApplicationDeleted(ctx, remoteCentral)
+			if err != nil {
+				return nil, errors.Wrapf(err, "unable to delete ArgoCD application for central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name)
+			}
+			if !ok {
+				return nil, errors.New("ArgoCD application not yet deleted")
+			}
+		}
+
+		if err := r.ensureChartResourcesExist(ctx, remoteCentral); err != nil {
+			return nil, errors.Wrapf(err, "unable to install chart resource for central %s/%s", central.GetNamespace(), central.GetName())
+		}
 	}
 
 	if err = r.reconcileCentralDBConfig(ctx, &remoteCentral, central); err != nil {
@@ -1106,6 +1133,12 @@ func (r *CentralReconciler) ensureCentralDeleted(ctx context.Context, remoteCent
 		globalDeleted = globalDeleted && secretDeleted
 	}
 
+	argoCdAppDeleted, err := r.ensureArgoCdApplicationDeleted(ctx, *remoteCentral)
+	if err != nil {
+		return false, err
+	}
+	globalDeleted = globalDeleted && argoCdAppDeleted
+
 	chartResourcesDeleted, err := r.ensureChartResourcesDeleted(ctx, remoteCentral)
 	if err != nil {
 		return false, err
@@ -1767,6 +1800,120 @@ func (r *CentralReconciler) isTenantResourcesChartObject(existingObject *unstruc
 		existingObject.GetNamespace() == remoteCentral.Metadata.Namespace
 }
 
+func (r *CentralReconciler) ensureArgoCdApplicationExists(ctx context.Context, remoteCentral private.ManagedCentral) error {
+	const lastAppliedHashLabel = "last-applied-hash"
+
+	want, err := r.getArgoCDApplication(remoteCentral)
+	if err != nil {
+		return fmt.Errorf("getting ArgoCD application: %w", err)
+	}
+
+	hash, err := util.MD5SumFromJSONStruct(want)
+	if err != nil {
+		return fmt.Errorf("calculating MD5 from JSON: %w", err)
+	}
+	if want.Labels == nil {
+		want.Labels = map[string]string{}
+	}
+	want.Labels[lastAppliedHashLabel] = fmt.Sprintf("%x", hash)
+
+	var existing argocd.Application
+	err = r.client.Get(ctx, ctrlClient.ObjectKey{Namespace: want.Namespace, Name: want.Namespace}, &existing)
+	if err != nil {
+		if !apiErrors.IsNotFound(err) {
+			return fmt.Errorf("getting ArgoCD application: %w", err)
+		}
+		if err := r.client.Create(ctx, want); err != nil {
+			return fmt.Errorf("creating ArgoCD application: %w", err)
+		}
+		return nil
+	}
+
+	if existing.Labels == nil || existing.Labels[lastAppliedHashLabel] != want.Labels[lastAppliedHashLabel] {
+		want.ResourceVersion = existing.ResourceVersion
+		if err := r.client.Update(ctx, want); err != nil {
+			return fmt.Errorf("updating ArgoCD application: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (r *CentralReconciler) getArgoCDApplication(remoteCentral private.ManagedCentral) (*argocd.Application, error) {
+
+	values := map[string]interface{}{
+		"tenant": map[string]interface{}{
+			"organizationId":   remoteCentral.Spec.Auth.OwnerOrgId,
+			"organizationName": remoteCentral.Spec.Auth.OwnerOrgName,
+			"id":               remoteCentral.Id,
+			"instanceType":     remoteCentral.Spec.InstanceType,
+		},
+		"centralRdsCidrBlock": "10.1.0.0/16",
+		"vpa": map[string]interface{}{
+			"central": map[string]interface{}{
+				"enabled": true,
+			},
+		},
+	}
+
+	valuesBytes, err := json.Marshal(values)
+	if err != nil {
+		return nil, fmt.Errorf("marshalling values: %w", err)
+	}
+
+	return &argocd.Application{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      remoteCentral.Metadata.Namespace,
+			Namespace: openshiftGitopsNamespace,
+		},
+		Spec: argocd.ApplicationSpec{
+			Project: "default",
+			Source: &argocd.ApplicationSource{
+				RepoURL: r.defaultTenantArgoCdAppSourceRepoURL,
+				Ref:     r.defaultTenantArgoCdAppSourceRef,
+				Path:    r.defaultTenantArgoCdAppSourcePath,
+				Helm: &argocd.ApplicationSourceHelm{
+					ValuesObject: &runtime.RawExtension{
+						Raw: valuesBytes,
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+func (r *CentralReconciler) ensureArgoCdApplicationDeleted(ctx context.Context, remoteCentral private.ManagedCentral) (bool, error) {
+	app := &argocd.Application{}
+	objectKey := ctrlClient.ObjectKey{Namespace: openshiftGitopsNamespace, Name: remoteCentral.Metadata.Namespace}
+
+	for i := 0; i < 5; i++ {
+
+		if i > 0 {
+			time.Sleep(time.Duration(i) * time.Millisecond * 200)
+		}
+
+		err := r.client.Get(ctx, objectKey, app)
+		if err != nil {
+			if apiErrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, fmt.Errorf("getting ArgoCD application: %w", err)
+		}
+
+		if app.DeletionTimestamp != nil {
+			continue
+		}
+
+		if err := r.client.Delete(ctx, app); err != nil {
+			return false, fmt.Errorf("deleting ArgoCD application: %w", err)
+		}
+
+	}
+
+	return false, nil
+
+}
+
 func (r *CentralReconciler) ensureRoutesExist(ctx context.Context, remoteCentral private.ManagedCentral) error {
 	err := r.ensureReencryptRouteExists(ctx, remoteCentral)
 	if err != nil {
@@ -2106,6 +2253,10 @@ func NewCentralReconciler(k8sClient ctrlClient.Client, fleetmanagerClient *fleet
 
 		resourcesChart: resourcesChart,
 		clock:          realClock{},
+
+		defaultTenantArgoCdAppSourcePath:    opts.DefaultTenantArgoCdAppSourcePath,
+		defaultTenantArgoCdAppSourceRepoURL: opts.DefaultTenantArgoCdAppSourceRepoURL,
+		defaultTenantArgoCdAppSourceRef:     opts.DefaultTenantArgoCdAppSourceRef,
 	}
 	r.needsReconcileFunc = r.needsReconcile
 

fleetshard/pkg/central/reconciler/reconciler_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	argocd "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	openshiftRouteV1 "github.com/openshift/api/route/v1"
@@ -2701,3 +2702,72 @@ func TestEncyrptionSHASumSameObject(t *testing.T) {
 		require.Equal(t, sums[i-1], sums[i], "hash of the same object should always be equal but was not")
 	}
 }
+
+func TestArgoCDApplication_CanBeToggleOnAndOff(t *testing.T) {
+	ctx := context.Background()
+	chartFiles, err := charts.TraverseChart(testdata, "testdata/tenant-resources")
+	require.NoError(t, err)
+	chart, err := loader.LoadFiles(chartFiles)
+	require.NoError(t, err)
+	managedCentral := simpleManagedCentral
+
+	cli, _, r := getClientTrackerAndReconciler(
+		t,
+		managedCentral,
+		nil,
+		defaultReconcilerOptions,
+	)
+	r.resourcesChart = chart
+
+	assertLegacyChartPresent := func(t *testing.T, present bool) {
+		var netPol networkingv1.NetworkPolicy
+		err := cli.Get(ctx, client.ObjectKey{Name: "dummy", Namespace: managedCentral.Metadata.Namespace}, &netPol)
+		if present {
+			require.NoError(t, err)
+		} else {
+			require.True(t, k8sErrors.IsNotFound(err))
+		}
+	}
+
+	assertArgoCdAppPresent := func(t *testing.T, present bool) {
+		var app argocd.Application
+		objectKey := client.ObjectKey{Name: managedCentral.Metadata.Namespace, Namespace: openshiftGitopsNamespace}
+		err := cli.Get(ctx, objectKey, &app)
+		if present {
+			require.NoError(t, err)
+		} else {
+			require.True(t, k8sErrors.IsNotFound(err))
+		}
+	}
+
+	{
+		// Ensure argocd application is created
+		managedCentral.Spec.ArgoCd.Enabled = true
+		_, err := r.Reconcile(ctx, managedCentral)
+		require.NoError(t, err)
+
+		assertArgoCdAppPresent(t, true)
+		assertLegacyChartPresent(t, false)
+	}
+
+	{
+		// Ensure argocd application is deleted
+		managedCentral.Spec.ArgoCd.Enabled = false
+		_, err := r.Reconcile(ctx, managedCentral)
+		require.NoError(t, err)
+
+		assertArgoCdAppPresent(t, false)
+		assertLegacyChartPresent(t, true)
+	}
+
+	{
+		// Ensure argocd and charts application are deleted
+		managedCentral.Metadata.DeletionTimestamp = time.Now().Format(time.RFC3339)
+		_, err = r.Reconcile(ctx, managedCentral)
+		require.ErrorIs(t, err, ErrDeletionInProgress)
+
+		assertArgoCdAppPresent(t, false)
+		assertLegacyChartPresent(t, false)
+	}
+
+}

fleetshard/pkg/runtime/runtime.go

@@ -132,16 +132,19 @@ func (r *Runtime) Start() error {
 	routesAvailable := r.routesAvailable()
 
 	reconcilerOpts := centralReconciler.CentralReconcilerOptions{
-		UseRoutes:             routesAvailable,
-		WantsAuthProvider:     r.config.CreateAuthProvider,
-		ManagedDBEnabled:      r.config.ManagedDB.Enabled,
-		Telemetry:             r.config.Telemetry,
-		ClusterName:           r.config.ClusterName,
-		Environment:           r.config.Environment,
-		AuditLogging:          r.config.AuditLogging,
-		TenantImagePullSecret: r.config.TenantImagePullSecret, // pragma: allowlist secret
-		RouteParameters:       r.config.RouteParameters,
-		SecureTenantNetwork:   r.config.SecureTenantNetwork,
+		UseRoutes:                           routesAvailable,
+		WantsAuthProvider:                   r.config.CreateAuthProvider,
+		ManagedDBEnabled:                    r.config.ManagedDB.Enabled,
+		Telemetry:                           r.config.Telemetry,
+		ClusterName:                         r.config.ClusterName,
+		Environment:                         r.config.Environment,
+		AuditLogging:                        r.config.AuditLogging,
+		TenantImagePullSecret:               r.config.TenantImagePullSecret, // pragma: allowlist secret
+		RouteParameters:                     r.config.RouteParameters,
+		SecureTenantNetwork:                 r.config.SecureTenantNetwork,
+		DefaultTenantArgoCdAppSourceRef:     r.config.DefaultTenantArgoCdAppSourceRef,
+		DefaultTenantArgoCdAppSourcePath:    r.config.DefaultTenantArgoCdAppSourcePath,
+		DefaultTenantArgoCdAppSourceRepoURL: r.config.DefaultTenantArgoCdAppSourceRepoURL,
 	}
 
 	ticker := concurrency.NewRetryTicker(func(ctx context.Context) (timeToNextTick time.Duration, err error) {

fleetshard/pkg/testutils/k8s.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"testing"
 
+	argoCd "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/hashicorp/go-multierror"
 	"github.com/openshift/addon-operator/apis/addons"
 	openshiftOperatorV1 "github.com/openshift/api/operator/v1"
@@ -97,6 +98,7 @@ func NewScheme(t *testing.T) *runtime.Scheme {
 	require.NoError(t, openshiftRouteV1.Install(scheme))
 	require.NoError(t, openshiftOperatorV1.Install(scheme))
 	require.NoError(t, addons.AddToScheme(scheme))
+	require.NoError(t, argoCd.AddToScheme(scheme))
 
 	return scheme
 }