diff --git a/fleetshard/pkg/central/reconciler/doc.go b/fleetshard/pkg/central/reconciler/doc.go new file mode 100644 index 0000000000..cc55a2c0ee --- /dev/null +++ b/fleetshard/pkg/central/reconciler/doc.go @@ -0,0 +1,3 @@ +package reconciler + +//go:generate go run gen/main.go diff --git a/fleetshard/pkg/central/reconciler/gen/main.go b/fleetshard/pkg/central/reconciler/gen/main.go new file mode 100644 index 0000000000..275c63c443 --- /dev/null +++ b/fleetshard/pkg/central/reconciler/gen/main.go @@ -0,0 +1,236 @@ +// This program generates a list of GVKs used by the tenant-resources helm chart for enabling garbage collection. +// +// To enable garbage collection without this list, we would need to... +// - Manually maintain a list of GVKs present in the chart, which would be error-prone. +// - Or have the reconciler list all objects for all possible GVKs, which would be very expensive. +// - Switch to the native helm client or the helm operator, which would require a lot of changes. +// +// This program automatically generates that list. +// +// Process: +// - It extract the GVKs that are present in the tenant-resources chart manifests +// - It extract the GVKs that are already declared in the generated file +// - It combines the two lists +// - It writes the combined list to the generated file + +package main + +import ( + "fmt" + "github.com/pkg/errors" + "io/fs" + "k8s.io/apimachinery/pkg/runtime/schema" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" +) + +var ( + reconcilerDir = path.Join(path.Dir(getCurrentFile()), "..") + outFile = fmt.Sprintf("%s/zzz_managed_resources.go", reconcilerDir) + tenantResourcesChartDir = path.Join(path.Dir(getCurrentFile()), "../../charts/data/tenant-resources/templates") + gvkRegex = regexp.MustCompile(`schema.GroupVersionKind{Group: "(.*)", Version: "(.*)", Kind: "(.*)"},`) +) + +const ( + apiVersionPrefix = "apiVersion: " + kindPrefix = "kind: " +) + +func main() { + if err := generate(); err != nil { + panic(err) + } +} + +type resourceMap map[schema.GroupVersionKind]bool + +func generate() error { + + // Holds a map of GVKs that will be in the output file + // The value `bool` indicates that the resource is present in the tenant-resources helm chart + // If false, this means that the resource is only present in the generated file (for GVKs removed from the chart) + // But the GVK is still needed for garbage collection. + seen := resourceMap{} + + // Finding GVKs used in the tenant-resources chart + if err := findGVKsInChart(tenantResourcesChartDir, seen); err != nil { + return err + } + + // Finding GVKs already declared in the generated file + if err := findGVKsInGeneratedFile(seen); err != nil { + return err + } + + // Re-generating the file + if err := generateGVKsList(seen); err != nil { + return err + } + + return nil +} + +func getCurrentFile() string { + _, file, _, _ := runtime.Caller(0) + return file +} + +func generateGVKsList(seen resourceMap) error { + // Making sure resources are ordered (for deterministic output) + sorted := sortResourceKeys(seen) + + builder := strings.Builder{} + builder.WriteString("// Code generated by fleetshard/pkg/central/reconciler/gen/main.go. DO NOT EDIT.\n") + builder.WriteString("package reconciler\n\n") + builder.WriteString("import (\n") + builder.WriteString("\t\"k8s.io/apimachinery/pkg/runtime/schema\"\n") + builder.WriteString(")\n\n") + + builder.WriteString("// tenantChartResourceGVKs is a list of GroupVersionKind that...\n") + builder.WriteString("// - are present in the tenant-resources helm chart\n") + builder.WriteString("// - were present in a previous version of the chart. A comment will indicate that manual removal from the list is required.\n") + + builder.WriteString("var tenantChartResourceGVKs = []schema.GroupVersionKind{\n") + for _, k := range sorted { + builder.WriteString(fmt.Sprintf("\tschema.GroupVersionKind{Group: %q, Version: %q, Kind: %q},", k.Group, k.Version, k.Kind)) + stillInChart := seen[k] + if !stillInChart { + builder.WriteString(" // This resource was present in a previous version of the chart. Manual removal is required.") + } + builder.WriteString("\n") + } + builder.WriteString("}\n") + + genFile, err := os.Create(outFile) + if err != nil { + return err + } + defer genFile.Close() + + genFile.WriteString(builder.String()) + return nil +} + +func sortResourceKeys(seen resourceMap) []schema.GroupVersionKind { + sorted := make([]schema.GroupVersionKind, 0, len(seen)) + for k := range seen { + sorted = append(sorted, k) + } + sort.Slice(sorted, func(i, j int) bool { + left := sorted[i] + right := sorted[j] + return left.String() < right.String() + }) + return sorted +} + +func findGVKsInGeneratedFile(seen resourceMap) error { + if _, err := os.Stat(outFile); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + file, err := os.Open(outFile) + if err != nil { + return err + } + defer file.Close() + + fileBytes, err := os.ReadFile(outFile) + if err != nil { + return err + } + + lines := strings.Split(string(fileBytes), "\n") + + for _, line := range lines { + matches := gvkRegex.FindStringSubmatch(line) + if len(matches) != 4 { + continue + } + gvk := schema.GroupVersionKind{Group: matches[1], Version: matches[2], Kind: matches[3]} + isAlreadyPresent := seen[gvk] + seen[gvk] = isAlreadyPresent + } + return nil +} + +func findGVKsInChart(chartDir string, seen resourceMap) error { + if err := filepath.WalkDir(chartDir, func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + + ext := filepath.Ext(path) + if ext != ".yaml" && ext != ".yml" { + return nil + } + + fileBytes, err := os.ReadFile(path) + if err != nil { + return err + } + + if err := findGVKsInFile(fileBytes, seen); err != nil { + return fmt.Errorf("failed to parse file %q: %w", path, err) + } + + return nil + }); err != nil { + return err + } + return nil +} + +func splitFileIntoResources(fileBytes []byte) []string { + fileStr := string(fileBytes) + return strings.Split(fileStr, "---") +} + +func findGVKsInFile(fileBytes []byte, seen resourceMap) error { + resources := splitFileIntoResources(fileBytes) + for _, resource := range resources { + gvk, ok, err := findResourceGVK(resource) + if err != nil { + return fmt.Errorf("failed to parse resource: %w", err) + } + if !ok { + return errors.New("resource GVK could not be parsed") + } + seen[gvk] = true + } + return nil +} + +func findResourceGVK(resource string) (schema.GroupVersionKind, bool, error) { + lines := strings.Split(resource, "\n") + apiVersion := "" + kind := "" + for i := 0; i < len(lines); i++ { + if strings.HasPrefix(lines[i], apiVersionPrefix) { + apiVersion = strings.TrimSpace(strings.TrimPrefix(lines[i], apiVersionPrefix)) + continue + } + if strings.HasPrefix(lines[i], kindPrefix) { + kind = strings.TrimSpace(strings.TrimPrefix(lines[i], kindPrefix)) + continue + } + } + if len(apiVersion) == 0 || len(kind) == 0 { + return schema.GroupVersionKind{}, false, nil + } + + groupVersion, err := schema.ParseGroupVersion(apiVersion) + if err != nil { + return schema.GroupVersionKind{}, false, err + } + + return groupVersion.WithKind(kind), true, nil +} diff --git a/fleetshard/pkg/central/reconciler/reconciler.go b/fleetshard/pkg/central/reconciler/reconciler.go index ba542c5e18..d7ebc1bbe8 100644 --- a/fleetshard/pkg/central/reconciler/reconciler.go +++ b/fleetshard/pkg/central/reconciler/reconciler.go @@ -94,6 +94,9 @@ const ( additionalAuthProviderConfigKey = "additional-auth-provider" tenantImagePullSecretName = "stackrox" // pragma: allowlist secret + + helmChartLabelKey = "helm.sh/chart" + helmChartNameLabel = "helm.sh/chart-name" ) type verifyAuthProviderExistsFunc func(ctx context.Context, central private.ManagedCentral, client ctrlClient.Client) (bool, error) @@ -1549,6 +1552,14 @@ func (r *CentralReconciler) disablePauseReconcileIfPresent(ctx context.Context, } func (r *CentralReconciler) ensureChartResourcesExist(ctx context.Context, remoteCentral private.ManagedCentral) error { + getObjectKey := func(obj *unstructured.Unstructured) string { + return fmt.Sprintf("%s/%s/%s", + obj.GetAPIVersion(), + obj.GetKind(), + obj.GetName(), + ) + } + vals, err := r.chartValues(remoteCentral) if err != nil { return fmt.Errorf("obtaining values for resources chart: %w", err) @@ -1558,55 +1569,125 @@ func (r *CentralReconciler) ensureChartResourcesExist(ctx context.Context, remot if err != nil { return fmt.Errorf("rendering resources chart: %w", err) } + + helmChartLabelValue := r.getTenantResourcesChartHelmLabelValue() + + // objectsThatShouldExist stores the keys of the objects we want to exist + var objectsThatShouldExist = map[string]struct{}{} + for _, obj := range objs { + objectsThatShouldExist[getObjectKey(obj)] = struct{}{} + if obj.GetNamespace() == "" { obj.SetNamespace(remoteCentral.Metadata.Namespace) } + if obj.GetLabels() == nil { + obj.SetLabels(map[string]string{}) + } + labels := obj.GetLabels() + labels[managedByLabelKey] = labelManagedByFleetshardValue + labels[helmChartLabelKey] = helmChartLabelValue + labels[helmChartNameLabel] = r.resourcesChart.Name() + obj.SetLabels(labels) err := charts.InstallOrUpdateChart(ctx, obj, r.client) if err != nil { return fmt.Errorf("failed to update central tenant object %w", err) } } + // Perform garbage collection + for _, gvk := range tenantChartResourceGVKs { + gvk := gvk + var existingObjects unstructured.UnstructuredList + existingObjects.SetGroupVersionKind(gvk) + + if err := r.client.List(ctx, &existingObjects, + ctrlClient.InNamespace(remoteCentral.Metadata.Namespace), + ctrlClient.MatchingLabels{helmChartNameLabel: r.resourcesChart.Name()}, + ); err != nil { + return fmt.Errorf("failed to list tenant resources chart objects %v: %w", gvk, err) + } + + for _, existingObject := range existingObjects.Items { + existingObject := &existingObject + if _, shouldExist := objectsThatShouldExist[getObjectKey(existingObject)]; shouldExist { + continue + } + + // Re-check that the helm label is present & namespace matches. + // Failsafe against some potential k8s-client bug when listing objects with a label selector + if !r.isTenantResourcesChartObject(existingObject, &remoteCentral) { + continue + } + + if existingObject.GetDeletionTimestamp() != nil { + continue + } + + // The object exists but it should not. Delete it. + if err := r.client.Delete(ctx, existingObject); err != nil { + if !apiErrors.IsNotFound(err) { + return fmt.Errorf("failed to delete central tenant object %v %q in namespace %s: %w", gvk, existingObject.GetName(), remoteCentral.Metadata.Namespace, err) + } + } + } + } + return nil } +func (r *CentralReconciler) getTenantResourcesChartHelmLabelValue() string { + // the objects rendered by the helm chart will have a label in the format + // helm.sh/chart: - + return fmt.Sprintf("%s-%s", r.resourcesChart.Name(), r.resourcesChart.Metadata.Version) +} + func (r *CentralReconciler) ensureChartResourcesDeleted(ctx context.Context, remoteCentral *private.ManagedCentral) (bool, error) { - vals, err := r.chartValues(*remoteCentral) - if err != nil { - return false, fmt.Errorf("obtaining values for resources chart: %w", err) - } - objs, err := charts.RenderToObjects(helmReleaseName, remoteCentral.Metadata.Namespace, r.resourcesChart, vals) - if err != nil { - return false, fmt.Errorf("rendering resources chart: %w", err) - } + allObjectsDeleted := true - waitForDelete := false - for _, obj := range objs { - key := ctrlClient.ObjectKey{Namespace: obj.GetNamespace(), Name: obj.GetName()} - if key.Namespace == "" { - key.Namespace = remoteCentral.Metadata.Namespace + for _, gvk := range tenantChartResourceGVKs { + gvk := gvk + var existingObjects unstructured.UnstructuredList + existingObjects.SetGroupVersionKind(gvk) + + if err := r.client.List(ctx, &existingObjects, + ctrlClient.InNamespace(remoteCentral.Metadata.Namespace), + ctrlClient.MatchingLabels{helmChartNameLabel: r.resourcesChart.Name()}, + ); err != nil { + return false, fmt.Errorf("failed to list tenant resources chart objects %v in namespace %s: %w", gvk, remoteCentral.Metadata.Namespace, err) } - var out unstructured.Unstructured - out.SetGroupVersionKind(obj.GroupVersionKind()) - err := r.client.Get(ctx, key, &out) - if err != nil { - if apiErrors.IsNotFound(err) { + + for _, existingObject := range existingObjects.Items { + existingObject := &existingObject + + // Re-check that the helm label is present & namespace matches. + // Failsafe against some potential k8s-client bug when listing objects with a label selector + if !r.isTenantResourcesChartObject(existingObject, remoteCentral) { continue } - return false, fmt.Errorf("retrieving object %s/%s of type %v: %w", key.Namespace, key.Name, obj.GroupVersionKind(), err) - } - if out.GetDeletionTimestamp() != nil { - waitForDelete = true - continue - } - err = r.client.Delete(ctx, &out) - if err != nil && !apiErrors.IsNotFound(err) { - return false, fmt.Errorf("retrieving object %s/%s of type %v: %w", key.Namespace, key.Name, obj.GroupVersionKind(), err) + + if existingObject.GetDeletionTimestamp() != nil { + allObjectsDeleted = false + continue + } + + if err := r.client.Delete(ctx, existingObject); err != nil { + if !apiErrors.IsNotFound(err) { + return false, fmt.Errorf("failed to delete central tenant object %v in namespace %q: %w", gvk, remoteCentral.Metadata.Namespace, err) + } + } } } - return !waitForDelete, nil + + return allObjectsDeleted, nil +} + +func (r *CentralReconciler) isTenantResourcesChartObject(existingObject *unstructured.Unstructured, remoteCentral *private.ManagedCentral) bool { + return existingObject.GetLabels() != nil && + existingObject.GetLabels()[helmChartNameLabel] == r.resourcesChart.Name() && + existingObject.GetLabels()[managedByLabelKey] == labelManagedByFleetshardValue && + existingObject.GetNamespace() == remoteCentral.Metadata.Namespace } func (r *CentralReconciler) ensureRoutesExist(ctx context.Context, remoteCentral private.ManagedCentral) error { diff --git a/fleetshard/pkg/central/reconciler/reconciler_test.go b/fleetshard/pkg/central/reconciler/reconciler_test.go index 5adac82ab7..8a89f4d35d 100644 --- a/fleetshard/pkg/central/reconciler/reconciler_test.go +++ b/fleetshard/pkg/central/reconciler/reconciler_test.go @@ -2622,6 +2622,7 @@ func TestReconciler_ensureChartResourcesExist_labelsAndAnnotations(t *testing.T) "rhacs.redhat.com/tenant": "cb45idheg5ip6dq1jo4g", "app.kubernetes.io/component": "egress-proxy", "app.kubernetes.io/name": "central-tenant-resources", + "helm.sh/chart-name": "central-tenant-resources", "helm.sh/chart": "central-tenant-resources-0.0.0", }, egressProxyDeployment.ObjectMeta.Labels) diff --git a/fleetshard/pkg/central/reconciler/zzz_managed_resources.go b/fleetshard/pkg/central/reconciler/zzz_managed_resources.go new file mode 100644 index 0000000000..d0769db30f --- /dev/null +++ b/fleetshard/pkg/central/reconciler/zzz_managed_resources.go @@ -0,0 +1,16 @@ +// Code generated by fleetshard/pkg/central/reconciler/gen/main.go. DO NOT EDIT. +package reconciler + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// tenantChartResourceGVKs is a list of GroupVersionKind that... +// - are present in the tenant-resources helm chart +// - were present in a previous version of the chart. A comment will indicate that manual removal from the list is required. +var tenantChartResourceGVKs = []schema.GroupVersionKind{ + schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"}, + schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"}, + schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + schema.GroupVersionKind{Group: "networking.k8s.io", Version: "v1", Kind: "NetworkPolicy"}, +}