From 688f7a6d6ada19cc9ca2edb6642dceb153548ad9 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Thu, 14 Nov 2024 09:02:31 +0100 Subject: [PATCH] ROX-26039: Create tenant argoCd app (#2009) --- .openshift-ci/tests/e2e.sh | 1 + Makefile | 2 +- dev/config/gitops-config.yaml | 18 +- .../defaults/cluster-type-infra-openshift/env | 2 + dev/env/manifests/argocd/00-namespace.yaml | 4 + dev/env/manifests/argocd/01-secret.yaml | 12 + .../fleetshard-operator/28-deployment.yaml | 2 - .../fleetshard-operator/51-fleetshard-cr.yaml | 1 + .../openshift-gitops/00-namespace.yaml | 5 + .../manifests/openshift-gitops/01-secret.yaml | 12 + ...ator-group.yaml => 02-operator-group.yaml} | 0 ...subscription.yaml => 03-subscription.yaml} | 0 dev/env/scripts/bootstrap.sh | 14 +- docs/development/secret-management.md | 6 + e2e/e2e_canary_upgrade_test.go | 20 +- fleetshard/config/config.go | 18 +- .../central/cloudprovider/awsclient/rds.go | 1 + .../pkg/central/reconciler/reconciler.go | 239 ++++++++++++++++-- .../pkg/central/reconciler/reconciler_test.go | 82 +++++- fleetshard/pkg/runtime/runtime.go | 4 + fleetshard/pkg/testutils/k8s.go | 2 + .../dinosaur/pkg/api/private/api/openapi.yaml | 6 - .../model_managed_central_all_of_spec.go | 1 - ...del_managed_central_all_of_spec_argo_cd.go | 16 -- openapi/fleet-manager-private.yaml | 5 - 25 files changed, 400 insertions(+), 73 deletions(-) create mode 100644 dev/env/manifests/argocd/00-namespace.yaml create mode 100644 dev/env/manifests/argocd/01-secret.yaml create mode 100644 dev/env/manifests/openshift-gitops/01-secret.yaml rename dev/env/manifests/openshift-gitops/{01-operator-group.yaml => 02-operator-group.yaml} (100%) rename dev/env/manifests/openshift-gitops/{02-subscription.yaml => 03-subscription.yaml} (100%) delete mode 100644 internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec_argo_cd.go diff --git a/.openshift-ci/tests/e2e.sh b/.openshift-ci/tests/e2e.sh index b7984193b3..a500e211ca 100755 --- a/.openshift-ci/tests/e2e.sh +++ b/.openshift-ci/tests/e2e.sh @@ -33,6 +33,7 @@ if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then done export STATIC_TOKEN="${FLEET_STATIC_TOKEN:-}" export STATIC_TOKEN_ADMIN="${FLEET_STATIC_TOKEN_ADMIN:-}" + export GITHUB_TOKEN="${GITHUB_TOKEN:-}" export CLUSTER_TYPE="openshift-ci" export GOARGS="-mod=mod" # For some reason we need this in the official base images. export GINKGO_FLAGS="--no-color -v" diff --git a/Makefile b/Makefile index ad7faddec7..2f68479be4 100644 --- a/Makefile +++ b/Makefile @@ -920,7 +920,7 @@ deploy/dev-fast/fleet-manager: image/build deploy/dev-fast/fleetshard-sync: image/build kubectl -n $(NAMESPACE) set image deploy/fleetshard-sync fleetshard-sync=$(SHORT_IMAGE_REF) - kubectl -n $(NAMESPACE) delete pod -l application=fleetshard-sync + kubectl -n $(NAMESPACE) delete pod -l app=fleetshard-sync deploy/probe: IMAGE_REGISTRY?="$(external_image_registry)" deploy/probe: IMAGE_REPOSITORY?="$(probe_image_repository)" diff --git a/dev/config/gitops-config.yaml b/dev/config/gitops-config.yaml index a4ac803aad..0de769801f 100644 --- a/dev/config/gitops-config.yaml +++ b/dev/config/gitops-config.yaml @@ -1,11 +1,11 @@ rhacsOperators: crdUrls: - - https://raw.githubusercontent.com/stackrox/stackrox/4.4.4/operator/bundle/manifests/platform.stackrox.io_securedclusters.yaml - - https://raw.githubusercontent.com/stackrox/stackrox/4.4.4/operator/bundle/manifests/platform.stackrox.io_centrals.yaml + - https://raw.githubusercontent.com/stackrox/stackrox/4.5.4/operator/bundle/manifests/platform.stackrox.io_securedclusters.yaml + - https://raw.githubusercontent.com/stackrox/stackrox/4.5.4/operator/bundle/manifests/platform.stackrox.io_centrals.yaml operators: - - deploymentName: "rhacs-operator-fast-stream" - image: "quay.io/rhacs-eng/stackrox-operator:4.6.x-233-gd6b1869836-fast" - centralLabelSelector: "rhacs.redhat.com/version-selector=fast-stream" + - deploymentName: "rhacs-operator-dev" + image: "quay.io/rhacs-eng/stackrox-operator:4.5.4" + centralLabelSelector: "rhacs.redhat.com/version-selector=dev" securedClusterReconcilerEnabled: false verticalPodAutoscaling: recommenders: [] @@ -21,6 +21,8 @@ tenantResources: rhacs.redhat.com/org-name: "{{ .OrganizationName }}" secureTenantNetwork: false centralRdsCidrBlock: "10.1.0.0/16" + argoCd: + enabled: true verticalPodAutoscalers: central: enabled: true @@ -41,12 +43,14 @@ centrals: - instanceIds: - "*" patch: | - # Set label for all centrals to fast-stream + # Set label for all centrals to dev metadata: labels: - rhacs.redhat.com/version-selector: "fast-stream" + rhacs.redhat.com/version-selector: "dev" # Adjust centrals for development environment spec: + network: + policies: Disabled monitoring: openshift: enabled: false diff --git a/dev/env/defaults/cluster-type-infra-openshift/env b/dev/env/defaults/cluster-type-infra-openshift/env index dd8a889004..040021c618 100644 --- a/dev/env/defaults/cluster-type-infra-openshift/env +++ b/dev/env/defaults/cluster-type-infra-openshift/env @@ -1,2 +1,4 @@ export FLEETSHARD_SYNC_RESOURCES_DEFAULT='{"requests":{"cpu":"400m","memory":"1000Mi"},"limits":{"cpu":"400m","memory":"1000Mi"}}' export EXPOSE_OPENSHIFT_ROUTER_DEFAULT="true" +export ENABLE_EXTERNAL_CONFIG_DEFAULT="true" +export AWS_AUTH_HELPER_DEFAULT="aws-saml" diff --git a/dev/env/manifests/argocd/00-namespace.yaml b/dev/env/manifests/argocd/00-namespace.yaml new file mode 100644 index 0000000000..a040f2ba58 --- /dev/null +++ b/dev/env/manifests/argocd/00-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: argocd diff --git a/dev/env/manifests/argocd/01-secret.yaml b/dev/env/manifests/argocd/01-secret.yaml new file mode 100644 index 0000000000..cabd4f875c --- /dev/null +++ b/dev/env/manifests/argocd/01-secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: acscs-manifests.stackrox.github.com + namespace: argocd + labels: + argocd.argoproj.io/secret-type: repository +stringData: + url: https://github.com/stackrox/acscs-manifests + username: not-used + password: "$GITHUB_TOKEN" # pragma: allowlist secret diff --git a/dev/env/manifests/fleetshard-operator/28-deployment.yaml b/dev/env/manifests/fleetshard-operator/28-deployment.yaml index bdcffd9b19..ae7d544e0b 100644 --- a/dev/env/manifests/fleetshard-operator/28-deployment.yaml +++ b/dev/env/manifests/fleetshard-operator/28-deployment.yaml @@ -32,8 +32,6 @@ spec: fieldPath: metadata.namespace - name: ADDON_NAME value: acs-fleetshard-dev - - name: ARGOCD_NAMESPACE - value: "${ARGOCD_NAMESPACE}" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/dev/env/manifests/fleetshard-operator/51-fleetshard-cr.yaml b/dev/env/manifests/fleetshard-operator/51-fleetshard-cr.yaml index 3c4cb54ff5..d22e1821a2 100644 --- a/dev/env/manifests/fleetshard-operator/51-fleetshard-cr.yaml +++ b/dev/env/manifests/fleetshard-operator/51-fleetshard-cr.yaml @@ -15,6 +15,7 @@ spec: environment: "dev" fleetManagerEndpoint: "http://fleet-manager:8000" secureTenantNetwork: $SECURE_TENANT_NETWORK + argoCdNamespace: $ARGOCD_NAMESPACE managedDB: enabled: $MANAGED_DB_ENABLED subnetGroup: "$MANAGED_DB_SUBNET_GROUP" diff --git a/dev/env/manifests/openshift-gitops/00-namespace.yaml b/dev/env/manifests/openshift-gitops/00-namespace.yaml index 07436ba5c2..eddd06e9bb 100644 --- a/dev/env/manifests/openshift-gitops/00-namespace.yaml +++ b/dev/env/manifests/openshift-gitops/00-namespace.yaml @@ -2,3 +2,8 @@ apiVersion: v1 kind: Namespace metadata: name: openshift-gitops-operator +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-gitops diff --git a/dev/env/manifests/openshift-gitops/01-secret.yaml b/dev/env/manifests/openshift-gitops/01-secret.yaml new file mode 100644 index 0000000000..3cb1d47c52 --- /dev/null +++ b/dev/env/manifests/openshift-gitops/01-secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: acscs-manifests.stackrox.github.com + namespace: openshift-gitops + labels: + argocd.argoproj.io/secret-type: repository +stringData: + url: https://github.com/stackrox/acscs-manifests + username: not-used + password: "$GITHUB_TOKEN" # pragma: allowlist secret diff --git a/dev/env/manifests/openshift-gitops/01-operator-group.yaml b/dev/env/manifests/openshift-gitops/02-operator-group.yaml similarity index 100% rename from dev/env/manifests/openshift-gitops/01-operator-group.yaml rename to dev/env/manifests/openshift-gitops/02-operator-group.yaml diff --git a/dev/env/manifests/openshift-gitops/02-subscription.yaml b/dev/env/manifests/openshift-gitops/03-subscription.yaml similarity index 100% rename from dev/env/manifests/openshift-gitops/02-subscription.yaml rename to dev/env/manifests/openshift-gitops/03-subscription.yaml diff --git a/dev/env/scripts/bootstrap.sh b/dev/env/scripts/bootstrap.sh index 905313fa7a..f8f1dd89c6 100755 --- a/dev/env/scripts/bootstrap.sh +++ b/dev/env/scripts/bootstrap.sh @@ -6,8 +6,18 @@ export GITROOT source "${GITROOT}/dev/env/scripts/lib.sh" # shellcheck source=/dev/null source "${GITROOT}/dev/env/scripts/docker.sh" +# shellcheck source=/dev/null +source "${GITROOT}/scripts/lib/external_config.sh" init +if [[ "$ENABLE_EXTERNAL_CONFIG" == "true" ]]; then + init_chamber + export CHAMBER_SECRET_BACKEND=secretsmanager +else + add_bin_to_path + ensure_tool_installed chamber + export CHAMBER_SECRET_BACKEND=null +fi log "** Preparing ACSCS Environment **" print_env @@ -54,11 +64,11 @@ fi if [[ "$INSTALL_ARGOCD" == "true" ]]; then log "Installing ArgoCD" - kubectl create ns argocd || true + chamber exec gitops -- apply "${MANIFESTS_DIR}/argocd" kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml elif [[ "$INSTALL_OPENSHIFT_GITOPS" == "true" ]]; then log "Installing Openshift GitOps" - apply "${MANIFESTS_DIR}/openshift-gitops" + chamber exec gitops -- apply "${MANIFESTS_DIR}/openshift-gitops" else log "One of ArgoCD or OpenShift GitOps must be installed" exit 1 diff --git a/docs/development/secret-management.md b/docs/development/secret-management.md index cf46093213..73b856434d 100644 --- a/docs/development/secret-management.md +++ b/docs/development/secret-management.md @@ -68,3 +68,9 @@ Check if your OS user matches the company User ID. If not, then specify it expli ```shell kinit bob ``` + +## E2E + +See instructions on managing secrets in CI https://docs.ci.openshift.org/docs/how-tos/adding-a-new-secret-to-ci/ + +Vault mount for e2e https://vault.ci.openshift.org/ui/vault/secrets/kv/kv/selfservice%2Frhacs-ms-e2e-tests%2Fcredentials/details diff --git a/e2e/e2e_canary_upgrade_test.go b/e2e/e2e_canary_upgrade_test.go index a4274588ee..720d09d453 100644 --- a/e2e/e2e_canary_upgrade_test.go +++ b/e2e/e2e_canary_upgrade_test.go @@ -31,14 +31,14 @@ const ( namespace = "rhacs" gitopsConfigmapName = "gitops-config" gitopsConfigmapDataKey = "config.yaml" - operatorVersion1 = "4.4.1" - operatorVersion2 = "4.4.2" + operatorVersion1 = "4.5.4" + operatorVersion2 = "4.5.3" ) var ( defaultCRDUrls = []string{ - "https://raw.githubusercontent.com/stackrox/stackrox/4.4.2/operator/bundle/manifests/platform.stackrox.io_securedclusters.yaml", - "https://raw.githubusercontent.com/stackrox/stackrox/4.4.2/operator/bundle/manifests/platform.stackrox.io_centrals.yaml", + "https://raw.githubusercontent.com/stackrox/stackrox/4.5.4/operator/bundle/manifests/platform.stackrox.io_securedclusters.yaml", + "https://raw.githubusercontent.com/stackrox/stackrox/4.5.4/operator/bundle/manifests/platform.stackrox.io_centrals.yaml", } operatorConfig1 = operatorConfigForVersion(operatorVersion1) operatorConfig2 = operatorConfigForVersion(operatorVersion2) @@ -528,6 +528,8 @@ spec: monitoring: openshift: enabled: false + network: + policies: Disabled central: db: resources: @@ -615,9 +617,9 @@ func defaultGitopsConfig() gitops.Config { CRDURLs: defaultCRDUrls, Configs: []operator.OperatorConfig{ { - "deploymentName": "rhacs-operator-4.4.2", - "image": "quay.io/rhacs-eng/stackrox-operator:4.4.2", - "centralLabelSelector": "rhacs.redhat.com/version-selector=4.4.2", + "deploymentName": "rhacs-operator-dev", + "image": "quay.io/rhacs-eng/stackrox-operator:4.5.4", + "centralLabelSelector": "rhacs.redhat.com/version-selector=dev", "securedClusterReconcilerEnabled": false, }, }, @@ -629,7 +631,7 @@ func defaultGitopsConfig() gitops.Config { Patch: ` metadata: labels: - rhacs.redhat.com/version-selector: "4.4.2"`, + rhacs.redhat.com/version-selector: "dev"`, }, { InstanceIDs: []string{"*"}, Patch: ` @@ -637,6 +639,8 @@ spec: monitoring: openshift: enabled: false + network: + policies: Disabled central: db: resources: diff --git a/fleetshard/config/config.go b/fleetshard/config/config.go index df9879693f..a485d0fa5e 100644 --- a/fleetshard/config/config.go +++ b/fleetshard/config/config.go @@ -35,13 +35,17 @@ type Config struct { // If it is empty, nothing is injected (for example, it is not required when running on OpenShift). // It is required when central images need to fetched from a private Quay registry. // It needs to given as Docker Config JSON object. - TenantImagePullSecret string `env:"TENANT_IMAGE_PULL_SECRET"` - ManagedDB ManagedDB - Telemetry Telemetry - AuditLogging AuditLogging - SecretEncryption SecretEncryption - RouteParameters RouteConfig - FleetshardAddonName string `env:"FLEETSHARD_ADDON_NAME" envDefault:"acs-fleetshard"` + TenantImagePullSecret string `env:"TENANT_IMAGE_PULL_SECRET"` + DefaultTenantArgoCdAppSourceRepoURL string `env:"DEFAULT_TENANT_ARGOCD_APP_SOURCE_REPO_URL" envDefault:"https://github.com/stackrox/acscs-manifests.git"` + DefaultTenantArgoCdAppSourceTargetRevision string `env:"DEFAULT_TENANT_ARGOCD_APP_SOURCE_TARGET_REVISION" envDefault:"HEAD"` + DefaultTenantArgoCdAppSourcePath string `env:"DEFAULT_TENANT_ARGOCD_APP_SOURCE_PATH" envDefault:"tenant-resources"` + ArgoCdNamespace string `env:"ARGOCD_NAMESPACE" envDefault:"openshift-gitops"` + ManagedDB ManagedDB + Telemetry Telemetry + AuditLogging AuditLogging + SecretEncryption SecretEncryption + RouteParameters RouteConfig + FleetshardAddonName string `env:"FLEETSHARD_ADDON_NAME" envDefault:"acs-fleetshard"` // The SecureTenantNetwork option controls whether the Tenant's K8s // Namespace will be secured at the network level, e.g. by using diff --git a/fleetshard/pkg/central/cloudprovider/awsclient/rds.go b/fleetshard/pkg/central/cloudprovider/awsclient/rds.go index fb8040b5b4..c7f41344c2 100644 --- a/fleetshard/pkg/central/cloudprovider/awsclient/rds.go +++ b/fleetshard/pkg/central/cloudprovider/awsclient/rds.go @@ -519,6 +519,7 @@ func newCreateCentralDBInstanceInput(input *createCentralDBInstanceInput) *rds.C PromotionTier: aws.Int64(dbInstancePromotionTier), CACertificateIdentifier: aws.String(dbCACertificateType), AutoMinorVersionUpgrade: aws.Bool(dbAutoVersionUpgrade), + Tags: []*rds.Tag{ { Key: aws.String(dataplaneClusterNameKey), diff --git a/fleetshard/pkg/central/reconciler/reconciler.go b/fleetshard/pkg/central/reconciler/reconciler.go index b5a4274b81..c5e8624620 100644 --- a/fleetshard/pkg/central/reconciler/reconciler.go +++ b/fleetshard/pkg/central/reconciler/reconciler.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + argocd "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/golang/glog" "github.com/hashicorp/go-multierror" openshiftRouteV1 "github.com/openshift/api/route/v1" @@ -40,6 +41,7 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/pointer" @@ -118,16 +120,20 @@ type encryptedSecrets struct { // CentralReconcilerOptions are the static options for creating a reconciler. type CentralReconcilerOptions struct { - UseRoutes bool - WantsAuthProvider bool - ManagedDBEnabled bool - Telemetry config.Telemetry - ClusterName string - Environment string - AuditLogging config.AuditLogging - TenantImagePullSecret string - RouteParameters config.RouteConfig - SecureTenantNetwork bool + UseRoutes bool + WantsAuthProvider bool + ManagedDBEnabled bool + Telemetry config.Telemetry + ClusterName string + Environment string + AuditLogging config.AuditLogging + TenantImagePullSecret string + RouteParameters config.RouteConfig + SecureTenantNetwork bool + DefaultTenantArgoCdAppSourceRepoURL string + DefaultTenantArgoCdAppSourceTargetRevision string + DefaultTenantArgoCdAppSourcePath string + ArgoCdNamespace string } // CentralReconciler is a reconciler tied to a one Central instance. It installs, updates and deletes Central instances @@ -166,6 +172,11 @@ type CentralReconciler struct { areSecretsStoredFunc areSecretsStoredFunc needsReconcileFunc needsReconcileFunc restoreCentralSecretsFunc restoreCentralSecretsFunc + + defaultTenantArgoCdAppSourceRepoURL string + defaultTenantArgoCdAppSourceTargetRevision string + defaultTenantArgoCdAppSourcePath string + argoCdNamespace string } // Reconcile takes a private.ManagedCentral and tries to install it into the cluster managed by the fleet-shard. @@ -239,8 +250,27 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private return nil, err } - if err := r.ensureChartResourcesExist(ctx, remoteCentral); err != nil { - return nil, errors.Wrapf(err, "unable to install chart resource for central %s/%s", central.GetNamespace(), central.GetName()) + if isArgoCdEnabledForTenant(remoteCentral) { + if err := r.ensureArgoCdApplicationExists(ctx, remoteCentral); err != nil { + return nil, errors.Wrapf(err, "unable to install ArgoCD application for central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name) + } + } else { + + { + // This part handles the case where we enable, then disable ArgoCD for a tenant + // to make sure the ArgoCD application is cleaned up. + ok, err := r.ensureArgoCdApplicationDeleted(ctx, remoteCentral) + if err != nil { + return nil, errors.Wrapf(err, "unable to delete ArgoCD application for central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name) + } + if !ok { + return nil, errors.New("ArgoCD application not yet deleted") + } + } + + if err := r.ensureChartResourcesExist(ctx, remoteCentral); err != nil { + return nil, errors.Wrapf(err, "unable to install chart resource for central %s/%s", central.GetNamespace(), central.GetName()) + } } if err = r.reconcileCentralDBConfig(ctx, &remoteCentral, central); err != nil { @@ -311,6 +341,30 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private return status, nil } +func isArgoCdEnabledForTenant(remoteCentral private.ManagedCentral) bool { + tenantResourceValues := remoteCentral.Spec.TenantResourcesValues + if tenantResourceValues == nil { + return false + } + argoCdIntf, ok := tenantResourceValues["argoCd"] + if !ok { + return false + } + argoCd, ok := argoCdIntf.(map[string]interface{}) + if !ok { + return false + } + enabled, ok := argoCd["enabled"] + if !ok { + return false + } + enabledBool, ok := enabled.(bool) + if !ok { + return false + } + return enabledBool +} + func (r *CentralReconciler) getInstanceConfig(remoteCentral *private.ManagedCentral) (*v1alpha1.Central, error) { var central = new(v1alpha1.Central) if err := yaml2.Unmarshal([]byte(remoteCentral.Spec.CentralCRYAML), central); err != nil { @@ -1109,6 +1163,12 @@ func (r *CentralReconciler) ensureCentralDeleted(ctx context.Context, remoteCent globalDeleted = globalDeleted && secretDeleted } + argoCdAppDeleted, err := r.ensureArgoCdApplicationDeleted(ctx, *remoteCentral) + if err != nil { + return false, err + } + globalDeleted = globalDeleted && argoCdAppDeleted + chartResourcesDeleted, err := r.ensureChartResourcesDeleted(ctx, remoteCentral) if err != nil { return false, err @@ -1204,7 +1264,7 @@ func (r *CentralReconciler) createImagePullSecret(ctx context.Context, namespace } func (r *CentralReconciler) reconcileNamespace(ctx context.Context, c private.ManagedCentral) error { - desiredNamespace := getDesiredNamespace(c) + desiredNamespace := r.getDesiredNamespace(c) existingNamespace, err := r.getNamespace(desiredNamespace.Name) if err != nil { @@ -1241,12 +1301,12 @@ func (r *CentralReconciler) reconcileNamespace(ctx context.Context, c private.Ma return nil } -func getDesiredNamespace(c private.ManagedCentral) *corev1.Namespace { +func (r *CentralReconciler) getDesiredNamespace(c private.ManagedCentral) *corev1.Namespace { return &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: c.Metadata.Namespace, Annotations: getNamespaceAnnotations(c), - Labels: getNamespaceLabels(c), + Labels: r.getNamespaceLabels(c), }, } } @@ -1770,6 +1830,143 @@ func (r *CentralReconciler) isTenantResourcesChartObject(existingObject *unstruc existingObject.GetNamespace() == remoteCentral.Metadata.Namespace } +func (r *CentralReconciler) ensureArgoCdApplicationExists(ctx context.Context, remoteCentral private.ManagedCentral) error { + const lastAppliedHashLabel = "last-applied-hash" + + want, err := r.makeDesiredArgoCDApplication(remoteCentral) + if err != nil { + return fmt.Errorf("getting ArgoCD application: %w", err) + } + + hash, err := util.MD5SumFromJSONStruct(want) + if err != nil { + return fmt.Errorf("calculating MD5 from JSON: %w", err) + } + if want.Labels == nil { + want.Labels = map[string]string{} + } + want.Labels[lastAppliedHashLabel] = fmt.Sprintf("%x", hash) + + var existing argocd.Application + err = r.client.Get(ctx, ctrlClient.ObjectKey{Namespace: want.Namespace, Name: want.Name}, &existing) + if err != nil { + if !apiErrors.IsNotFound(err) { + return fmt.Errorf("getting ArgoCD application: %w", err) + } + if err := r.client.Create(ctx, want); err != nil { + return fmt.Errorf("creating ArgoCD application: %w", err) + } + return nil + } + + if existing.Labels == nil || existing.Labels[lastAppliedHashLabel] != want.Labels[lastAppliedHashLabel] { + want.ResourceVersion = existing.ResourceVersion + if err := r.client.Update(ctx, want); err != nil { + return fmt.Errorf("updating ArgoCD application: %w", err) + } + } + + return nil +} + +func (r *CentralReconciler) makeDesiredArgoCDApplication(remoteCentral private.ManagedCentral) (*argocd.Application, error) { + + values := map[string]interface{}{ + "tenant": map[string]interface{}{ + "organizationId": remoteCentral.Spec.Auth.OwnerOrgId, + "organizationName": remoteCentral.Spec.Auth.OwnerOrgName, + "id": remoteCentral.Id, + "instanceType": remoteCentral.Spec.InstanceType, + "name": remoteCentral.Metadata.Name, + }, + "centralRdsCidrBlock": "10.1.0.0/16", + "vpa": map[string]interface{}{ + "central": map[string]interface{}{ + "enabled": true, + }, + }, + } + + valuesBytes, err := json.Marshal(values) + if err != nil { + return nil, fmt.Errorf("marshalling values: %w", err) + } + + return &argocd.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getArgoCdAppName(remoteCentral), + Namespace: r.getArgoCdAppNamespace(remoteCentral), + }, + Spec: argocd.ApplicationSpec{ + Project: "default", + SyncPolicy: &argocd.SyncPolicy{ + Automated: &argocd.SyncPolicyAutomated{ + Prune: true, + SelfHeal: true, + }, + }, + Source: &argocd.ApplicationSource{ + RepoURL: r.defaultTenantArgoCdAppSourceRepoURL, + TargetRevision: r.defaultTenantArgoCdAppSourceTargetRevision, + Path: r.defaultTenantArgoCdAppSourcePath, + Helm: &argocd.ApplicationSourceHelm{ + ValuesObject: &runtime.RawExtension{ + Raw: valuesBytes, + }, + }, + }, + Destination: argocd.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: remoteCentral.Metadata.Namespace, + }, + }, + }, nil +} + +func (r *CentralReconciler) ensureArgoCdApplicationDeleted(ctx context.Context, remoteCentral private.ManagedCentral) (bool, error) { + app := &argocd.Application{} + objectKey := r.getArgoCdAppObjectKey(remoteCentral) + + err := wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + err := r.client.Get(ctx, objectKey, app) + if apiErrors.IsNotFound(err) { + return true, nil + } else if err != nil { + return false, fmt.Errorf("getting ArgoCD application: %w", err) + } + + if app.DeletionTimestamp != nil { + return false, nil + } + + if err := r.client.Delete(ctx, app); err != nil { + return false, fmt.Errorf("deleting ArgoCD application: %w", err) + } + + return false, nil + }) + if err != nil { + return false, fmt.Errorf("waiting for ArgoCD application deletion: %w", err) + } + + return true, nil +} + +func (r *CentralReconciler) getArgoCdAppName(remoteCentral private.ManagedCentral) string { + return fmt.Sprintf("rhacs-%s", remoteCentral.Id) +} + +func (r *CentralReconciler) getArgoCdAppNamespace(_ private.ManagedCentral) string { + return r.argoCdNamespace +} + +func (r *CentralReconciler) getArgoCdAppObjectKey(remoteCentral private.ManagedCentral) ctrlClient.ObjectKey { + return ctrlClient.ObjectKey{ + Namespace: r.getArgoCdAppNamespace(remoteCentral), + Name: r.getArgoCdAppName(remoteCentral), + } +} + func (r *CentralReconciler) ensureRoutesExist(ctx context.Context, remoteCentral private.ManagedCentral) error { err := r.ensureReencryptRouteExists(ctx, remoteCentral) if err != nil { @@ -1879,12 +2076,12 @@ func getTenantAnnotations(c private.ManagedCentral) map[string]string { } } -func getNamespaceLabels(c private.ManagedCentral) map[string]string { +func (r *CentralReconciler) getNamespaceLabels(c private.ManagedCentral) map[string]string { ret := map[string]string{} for k, v := range getTenantLabels(c) { ret[k] = v } - ret[argoCdManagedBy] = openshiftGitopsNamespace + ret[argoCdManagedBy] = r.argoCdNamespace return ret } @@ -2113,6 +2310,14 @@ func NewCentralReconciler(k8sClient ctrlClient.Client, fleetmanagerClient *fleet resourcesChart: resourcesChart, clock: realClock{}, + + // Todo: Allow overriding the tenant source path, repo URL, and ref + // on a per-tenant basis. + + defaultTenantArgoCdAppSourcePath: opts.DefaultTenantArgoCdAppSourcePath, + defaultTenantArgoCdAppSourceRepoURL: opts.DefaultTenantArgoCdAppSourceRepoURL, + defaultTenantArgoCdAppSourceTargetRevision: opts.DefaultTenantArgoCdAppSourceTargetRevision, + argoCdNamespace: opts.ArgoCdNamespace, } r.needsReconcileFunc = r.needsReconcile diff --git a/fleetshard/pkg/central/reconciler/reconciler_test.go b/fleetshard/pkg/central/reconciler/reconciler_test.go index 78afffc8ae..c0c6ec4f77 100644 --- a/fleetshard/pkg/central/reconciler/reconciler_test.go +++ b/fleetshard/pkg/central/reconciler/reconciler_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + argocd "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" openshiftRouteV1 "github.com/openshift/api/route/v1" @@ -63,7 +64,9 @@ const ( var ( defaultCentralConfig = private.ManagedCentral{} - defaultReconcilerOptions = CentralReconcilerOptions{} + defaultReconcilerOptions = CentralReconcilerOptions{ + ArgoCdNamespace: "openshift-gitops", + } useRoutesReconcilerOptions = CentralReconcilerOptions{UseRoutes: true} secureTenantNetworkReconcilerOptions = CentralReconcilerOptions{SecureTenantNetwork: true} @@ -2701,3 +2704,80 @@ func TestEncyrptionSHASumSameObject(t *testing.T) { require.Equal(t, sums[i-1], sums[i], "hash of the same object should always be equal but was not") } } + +func TestArgoCDApplication_CanBeToggleOnAndOff(t *testing.T) { + ctx := context.Background() + chartFiles, err := charts.TraverseChart(testdata, "testdata/tenant-resources") + require.NoError(t, err) + chart, err := loader.LoadFiles(chartFiles) + require.NoError(t, err) + managedCentral := simpleManagedCentral + + cli, _, r := getClientTrackerAndReconciler( + t, + managedCentral, + nil, + defaultReconcilerOptions, + ) + r.resourcesChart = chart + + assertLegacyChartPresent := func(t *testing.T, present bool) { + var netPol networkingv1.NetworkPolicy + err := cli.Get(ctx, client.ObjectKey{Name: "dummy", Namespace: managedCentral.Metadata.Namespace}, &netPol) + if present { + require.NoError(t, err) + } else { + require.True(t, k8sErrors.IsNotFound(err)) + } + } + + assertArgoCdAppPresent := func(t *testing.T, present bool) { + var app argocd.Application + objectKey := r.getArgoCdAppObjectKey(managedCentral) + err := cli.Get(ctx, objectKey, &app) + if present { + require.NoError(t, err) + } else { + require.True(t, k8sErrors.IsNotFound(err)) + } + } + + { + // Ensure argocd application is created + managedCentral.Spec.TenantResourcesValues = map[string]interface{}{ + "argoCd": map[string]interface{}{ + "enabled": true, + }, + } + _, err := r.Reconcile(ctx, managedCentral) + require.NoError(t, err) + + assertArgoCdAppPresent(t, true) + assertLegacyChartPresent(t, false) + } + + { + // Ensure argocd application is deleted + managedCentral.Spec.TenantResourcesValues = map[string]interface{}{ + "argoCd": map[string]interface{}{ + "enabled": false, + }, + } + _, err := r.Reconcile(ctx, managedCentral) + require.NoError(t, err) + + assertArgoCdAppPresent(t, false) + assertLegacyChartPresent(t, true) + } + + { + // Ensure argocd and charts application are deleted + managedCentral.Metadata.DeletionTimestamp = time.Now().Format(time.RFC3339) + _, err = r.Reconcile(ctx, managedCentral) + require.ErrorIs(t, err, ErrDeletionInProgress) + + assertArgoCdAppPresent(t, false) + assertLegacyChartPresent(t, false) + } + +} diff --git a/fleetshard/pkg/runtime/runtime.go b/fleetshard/pkg/runtime/runtime.go index 21a4891ed6..b3636af8b6 100644 --- a/fleetshard/pkg/runtime/runtime.go +++ b/fleetshard/pkg/runtime/runtime.go @@ -142,6 +142,10 @@ func (r *Runtime) Start() error { TenantImagePullSecret: r.config.TenantImagePullSecret, // pragma: allowlist secret RouteParameters: r.config.RouteParameters, SecureTenantNetwork: r.config.SecureTenantNetwork, + DefaultTenantArgoCdAppSourceTargetRevision: r.config.DefaultTenantArgoCdAppSourceTargetRevision, + DefaultTenantArgoCdAppSourcePath: r.config.DefaultTenantArgoCdAppSourcePath, + DefaultTenantArgoCdAppSourceRepoURL: r.config.DefaultTenantArgoCdAppSourceRepoURL, + ArgoCdNamespace: r.config.ArgoCdNamespace, } ticker := concurrency.NewRetryTicker(func(ctx context.Context) (timeToNextTick time.Duration, err error) { diff --git a/fleetshard/pkg/testutils/k8s.go b/fleetshard/pkg/testutils/k8s.go index 99e84f535a..3658a30af3 100644 --- a/fleetshard/pkg/testutils/k8s.go +++ b/fleetshard/pkg/testutils/k8s.go @@ -5,6 +5,7 @@ import ( "fmt" "testing" + argoCd "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/hashicorp/go-multierror" "github.com/openshift/addon-operator/apis/addons" openshiftOperatorV1 "github.com/openshift/api/operator/v1" @@ -97,6 +98,7 @@ func NewScheme(t *testing.T) *runtime.Scheme { require.NoError(t, openshiftRouteV1.Install(scheme)) require.NoError(t, openshiftOperatorV1.Install(scheme)) require.NoError(t, addons.AddToScheme(scheme)) + require.NoError(t, argoCd.AddToScheme(scheme)) return scheme } diff --git a/internal/dinosaur/pkg/api/private/api/openapi.yaml b/internal/dinosaur/pkg/api/private/api/openapi.yaml index f677979ff3..ba85ccc411 100644 --- a/internal/dinosaur/pkg/api/private/api/openapi.yaml +++ b/internal/dinosaur/pkg/api/private/api/openapi.yaml @@ -713,10 +713,6 @@ components: properties: host: type: string - ManagedCentral_allOf_spec_argoCd: - properties: - enabled: - type: boolean ManagedCentral_allOf_spec: properties: instanceType: @@ -740,8 +736,6 @@ components: $ref: '#/components/schemas/ManagedCentral_allOf_spec_uiEndpoint' dataEndpoint: $ref: '#/components/schemas/ManagedCentral_allOf_spec_dataEndpoint' - argoCd: - $ref: '#/components/schemas/ManagedCentral_allOf_spec_argoCd' ManagedCentral_allOf: properties: metadata: diff --git a/internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec.go b/internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec.go index 134f099f20..16e3357f6f 100644 --- a/internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec.go +++ b/internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec.go @@ -20,5 +20,4 @@ type ManagedCentralAllOfSpec struct { AdditionalAuthProvider ManagedCentralAllOfSpecAdditionalAuthProvider `json:"additionalAuthProvider,omitempty"` UiEndpoint ManagedCentralAllOfSpecUiEndpoint `json:"uiEndpoint,omitempty"` DataEndpoint ManagedCentralAllOfSpecDataEndpoint `json:"dataEndpoint,omitempty"` - ArgoCd ManagedCentralAllOfSpecArgoCd `json:"argoCd,omitempty"` } diff --git a/internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec_argo_cd.go b/internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec_argo_cd.go deleted file mode 100644 index 1e8783becc..0000000000 --- a/internal/dinosaur/pkg/api/private/model_managed_central_all_of_spec_argo_cd.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Red Hat Advanced Cluster Security Service Fleet Manager - * - * Red Hat Advanced Cluster Security (RHACS) Service Fleet Manager APIs that are used by internal services e.g fleetshard operators. - * - * API version: 1.4.0 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -// Code generated by OpenAPI Generator (https://openapi-generator.tech). DO NOT EDIT. -package private - -// ManagedCentralAllOfSpecArgoCd struct for ManagedCentralAllOfSpecArgoCd -type ManagedCentralAllOfSpecArgoCd struct { - Enabled bool `json:"enabled,omitempty"` -} diff --git a/openapi/fleet-manager-private.yaml b/openapi/fleet-manager-private.yaml index cf702f4218..2663ea3440 100644 --- a/openapi/fleet-manager-private.yaml +++ b/openapi/fleet-manager-private.yaml @@ -376,11 +376,6 @@ components: properties: host: type: string - argoCd: - type: object - properties: - enabled: - type: boolean requestStatus: type: string