Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: appset preserve labels and global preserve fields #15445

Merged
merged 7 commits into from
Sep 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions applicationset/controllers/applicationset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,12 @@ type ApplicationSetReconciler struct {
Policy argov1alpha1.ApplicationsSyncPolicy
EnablePolicyOverride bool
utils.Renderer
ArgoCDNamespace string
ApplicationSetNamespaces []string
EnableProgressiveSyncs bool
SCMRootCAPath string
ArgoCDNamespace string
ApplicationSetNamespaces []string
EnableProgressiveSyncs bool
SCMRootCAPath string
GlobalPreservedAnnotations []string
GlobalPreservedLabels []string
}

// +kubebuilder:rbac:groups=argoproj.io,resources=applicationsets,verbs=get;list;watch;create;update;patch;delete
Expand Down Expand Up @@ -617,9 +619,21 @@ func (r *ApplicationSetReconciler) createOrUpdateInCluster(ctx context.Context,
}

preservedAnnotations := make([]string, 0)
preservedLabels := make([]string, 0)

if applicationSet.Spec.PreservedFields != nil {
preservedAnnotations = append(preservedAnnotations, applicationSet.Spec.PreservedFields.Annotations...)
preservedLabels = append(preservedLabels, applicationSet.Spec.PreservedFields.Labels...)
}

if len(r.GlobalPreservedAnnotations) > 0 {
preservedAnnotations = append(preservedAnnotations, r.GlobalPreservedAnnotations...)
}

if len(r.GlobalPreservedLabels) > 0 {
preservedLabels = append(preservedLabels, r.GlobalPreservedLabels...)
}

// Preserve specially treated argo cd annotations:
// * https://github.com/argoproj/applicationset/issues/180
// * https://github.com/argoproj/argo-cd/issues/10500
Expand All @@ -633,6 +647,16 @@ func (r *ApplicationSetReconciler) createOrUpdateInCluster(ctx context.Context,
generatedApp.Annotations[key] = state
}
}

for _, key := range preservedLabels {
if state, exists := found.ObjectMeta.Labels[key]; exists {
if generatedApp.Labels == nil {
generatedApp.Labels = map[string]string{}
}
generatedApp.Labels[key] = state
}
}

found.ObjectMeta.Annotations = generatedApp.Annotations

found.ObjectMeta.Finalizers = generatedApp.Finalizers
Expand Down
6 changes: 6 additions & 0 deletions assets/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -5894,6 +5894,12 @@
"items": {
"type": "string"
}
},
"labels": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ func NewCommand() *cobra.Command {
maxConcurrentReconciliations int
scmRootCAPath string
allowedScmProviders []string
globalPreservedAnnotations []string
globalPreservedLabels []string
)
scheme := runtime.NewScheme()
_ = clientgoscheme.AddToScheme(scheme)
Expand Down Expand Up @@ -200,20 +202,22 @@ func NewCommand() *cobra.Command {
}

if err = (&controllers.ApplicationSetReconciler{
Generators: topLevelGenerators,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("applicationset-controller"),
Renderer: &utils.Render{},
Policy: policyObj,
EnablePolicyOverride: enablePolicyOverride,
ArgoAppClientset: appSetConfig,
KubeClientset: k8sClient,
ArgoDB: argoCDDB,
ArgoCDNamespace: namespace,
ApplicationSetNamespaces: applicationSetNamespaces,
EnableProgressiveSyncs: enableProgressiveSyncs,
SCMRootCAPath: scmRootCAPath,
Generators: topLevelGenerators,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("applicationset-controller"),
Renderer: &utils.Render{},
Policy: policyObj,
EnablePolicyOverride: enablePolicyOverride,
ArgoAppClientset: appSetConfig,
KubeClientset: k8sClient,
ArgoDB: argoCDDB,
ArgoCDNamespace: namespace,
ApplicationSetNamespaces: applicationSetNamespaces,
EnableProgressiveSyncs: enableProgressiveSyncs,
SCMRootCAPath: scmRootCAPath,
GlobalPreservedAnnotations: globalPreservedAnnotations,
GlobalPreservedLabels: globalPreservedLabels,
}).SetupWithManager(mgr, enableProgressiveSyncs, maxConcurrentReconciliations); err != nil {
log.Error(err, "unable to create controller", "controller", "ApplicationSet")
os.Exit(1)
Expand Down Expand Up @@ -251,6 +255,8 @@ func NewCommand() *cobra.Command {
command.Flags().IntVar(&repoServerTimeoutSeconds, "repo-server-timeout-seconds", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_REPO_SERVER_TIMEOUT_SECONDS", 60, 0, math.MaxInt64), "Repo server RPC call timeout seconds.")
command.Flags().IntVar(&maxConcurrentReconciliations, "concurrent-reconciliations", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_CONCURRENT_RECONCILIATIONS", 10, 1, 100), "Max concurrent reconciliations limit for the controller")
command.Flags().StringVar(&scmRootCAPath, "scm-root-ca-path", env.StringFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_SCM_ROOT_CA_PATH", ""), "Provide Root CA Path for self-signed TLS Certificates")
command.Flags().StringSliceVar(&globalPreservedAnnotations, "preserved-annotations", env.StringsFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS", []string{}, ","), "Sets global preserved field values for annotations")
command.Flags().StringSliceVar(&globalPreservedLabels, "preserved-labels", env.StringsFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS", []string{}, ","), "Sets global preserved field values for labels")
return &command
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -160,37 +160,44 @@ cd applicationset/manifests
kubectl apply -n argocd -f install.yaml
```

## Preserving changes made to an Applications annotations
## Preserving changes made to an Applications annotations and labels

It is common practice in Kubernetes to store state in annotations, operators will often make use of this. To allow for this, it is possible to configure a list of annotations that the ApplicationSet should preserve when reconciling.

For example, imagine that we have an Application created from an ApplicationSet, but a custom annotation has since been added (to the Application) that does not exist in the `ApplicationSet` resource:
For example, imagine that we have an Application created from an ApplicationSet, but a custom annotation and label has since been added (to the Application) that does not exist in the `ApplicationSet` resource:
```yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
# This annotation and label exists only on this Application, and not in
# the parent ApplicationSet template:
annotations:
# This annotation exists only on this Application, and not in
# the parent ApplicationSet template:
my-custom-annotation: some-value
labels:
my-custom-label: some-value
spec:
# (...)
```

To preserve this annotation we can use the `preservedFields` property of the `ApplicationSet` like so:
To preserve this annotation and label we can use the `preservedFields` property of the `ApplicationSet` like so:
```yaml
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
spec:
# (...)
preservedFields:
annotations: ["my-custom-annotation"]
labels: ["my-custom-label"]
```

The ApplicationSet controller will leave this annotation as-is when reconciling, even though it is not an annotation defined in the metadata of the ApplicationSet itself.
The ApplicationSet controller will leave this annotation and label as-is when reconciling, even though it is not defined in the metadata of the ApplicationSet itself.

By default, the Argo CD notifications and the Argo CD refresh type annotations are also preserved.

!!!note
One can also set global preserved fields for the controller by passing a comma separated list of annotations and labels to
`ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS` and `ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS` respectively.

## Limitations: what isn't supported as of the current release

Here is a list of commonly requested resource modification features which are not supported as of the current release. This lack of support is *not* necessarily by design; rather these behaviours are documented here to provide clear, concise descriptions of the current state of the feature.
Expand All @@ -210,9 +217,9 @@ As of this writing, there is [an issue open](https://github.com/argoproj/applica

### Limitation: ApplicationSet controller will not selectively ignore changes to individual fields

Currently, you can only instruct the ApplicationSet controller to ignore changes to Application annotations.
Currently, you can only instruct the ApplicationSet controller to ignore changes to Application annotations and labels.

For example, imagine that we have an Application created from an ApplicationSet, but a user has attempted to add a custom annotation (to the Application) that does not exist in the `ApplicationSet` resource:
For example, imagine that we have an Application created from an ApplicationSet, but a user has attempted to add a custom annotation/label (to the Application) that does not exist in the `ApplicationSet` resource:
```yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
Expand All @@ -227,12 +234,12 @@ spec:

As above, the `ApplicationSet` resource does not have a `my-custom-label: some-value` label in the `.spec.template.labels` for the Application.

Since this field is not in the ApplicationSet template, as soon as a user adds this custom annotation, it will be immediately reverted (removed) by the ApplicationSet controller.
Since this field is not in the ApplicationSet template, as soon as a user adds this custom label, it will be immediately reverted (removed) by the ApplicationSet controller.

There is currently no support for disabling or customizing this behaviour.
If the labels/annotations are not mentioned in appset preserved fields, there is currently no way for disabling or customizing this behaviour.

To some extent this is by design: the main principle of ApplicationSets is that we maintain a 1-to-many relationship between the ApplicationSet and the Applications that it owns, such that all the Applications necessarily conform to a strict template.

This provides the advantages of the 'cattle not pets' philosophy of microservice/cloud native application resource management, wherein you don't need to worry about individual Applications differing from each other in subtle ways: they will all necessarily be reconciled to be consistent with the parent template.

BUT, support exists for preserving changes to Application annotations as documented [above](#preserving-changes-made-to-an-applications-annotations).
BUT, support exists for preserving changes to Application annotations and labels as documented [above](#preserving-changes-made-to-an-applications-annotations-and-labels).
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,18 @@ spec:
- containerPort: 8080
name: metrics
env:
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.annotations
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.labels
name: argocd-cmd-params-cm
optional: true
- name: NAMESPACE
valueFrom:
fieldRef:
Expand Down
16 changes: 16 additions & 0 deletions manifests/core-install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17492,6 +17492,10 @@ spec:
items:
type: string
type: array
labels:
items:
type: string
type: array
type: object
strategy:
properties:
Expand Down Expand Up @@ -18786,6 +18790,18 @@ spec:
- args:
- /usr/local/bin/argocd-applicationset-controller
env:
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.annotations
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.labels
name: argocd-cmd-params-cm
optional: true
- name: NAMESPACE
valueFrom:
fieldRef:
Expand Down
4 changes: 4 additions & 0 deletions manifests/crds/applicationset-crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13001,6 +13001,10 @@ spec:
items:
type: string
type: array
labels:
items:
type: string
type: array
type: object
strategy:
properties:
Expand Down
16 changes: 16 additions & 0 deletions manifests/ha/install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17492,6 +17492,10 @@ spec:
items:
type: string
type: array
labels:
items:
type: string
type: array
type: object
strategy:
properties:
Expand Down Expand Up @@ -20024,6 +20028,18 @@ spec:
- args:
- /usr/local/bin/argocd-applicationset-controller
env:
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.annotations
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.labels
name: argocd-cmd-params-cm
optional: true
- name: NAMESPACE
valueFrom:
fieldRef:
Expand Down
12 changes: 12 additions & 0 deletions manifests/ha/namespace-install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1524,6 +1524,18 @@ spec:
- args:
- /usr/local/bin/argocd-applicationset-controller
env:
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.annotations
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.labels
name: argocd-cmd-params-cm
optional: true
- name: NAMESPACE
valueFrom:
fieldRef:
Expand Down
16 changes: 16 additions & 0 deletions manifests/install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17492,6 +17492,10 @@ spec:
items:
type: string
type: array
labels:
items:
type: string
type: array
type: object
strategy:
properties:
Expand Down Expand Up @@ -19125,6 +19129,18 @@ spec:
- args:
- /usr/local/bin/argocd-applicationset-controller
env:
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.annotations
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.labels
name: argocd-cmd-params-cm
optional: true
- name: NAMESPACE
valueFrom:
fieldRef:
Expand Down
12 changes: 12 additions & 0 deletions manifests/namespace-install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,18 @@ spec:
- args:
- /usr/local/bin/argocd-applicationset-controller
env:
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_ANNOTATIONS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.annotations
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS
valueFrom:
configMapKeyRef:
key: applicationsetcontroller.global.preserved.labels
name: argocd-cmd-params-cm
optional: true
- name: NAMESPACE
valueFrom:
fieldRef:
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/api-rules/violation_exceptions.list
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/ap
API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1,AppProjectSpec,SourceRepos
API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1,ApplicationMatchExpression,Values
API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1,ApplicationPreservedFields,Annotations
API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1,ApplicationPreservedFields,Labels
API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1,ApplicationSetRolloutStep,MatchExpressions
API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1,ApplicationSetRolloutStrategy,Steps
API rule violation: list_type_missing,github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1,ApplicationSetSpec,Generators
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/application/v1alpha1/applicationset_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ type ApplicationSetSpec struct {

type ApplicationPreservedFields struct {
Annotations []string `json:"annotations,omitempty" protobuf:"bytes,1,name=annotations"`
Labels []string `json:"labels,omitempty" protobuf:"bytes,2,name=labels"`
}

// ApplicationSetStrategy configures how generated Applications are updated in sequence.
Expand Down
Loading