Allow setting MaxConcurrentReconciles through env variable (#665)

* update common/config.go

* Support using sync.Map in common Config

* Read max concurrent reconcile from env

* Fix typo

* Update utils.go

* Add GetMaxConcurrentReconciles test

* Allow configuring a default MAX_CONCURRENT_RECONCILES for all CRs

* Set MaxconcurrentReconciles for RuntimeComponent only

* Add max concurrent reconcile tests

* Update kustomize files

* Use quotes when setting max concurrent reconcile in YAML

* Add kustomize env for max concurrent reconciles

* Apply max concurrent reconcile as kustomize overlay

* Add instructions for using max concurrent reconciles in kustomize

* Add install command for kustomize

* Add 1.4.1 readme.adoc instructions with max concurrent reconciles

* Update readme.adoc

* Add defaults in readme.adoc

* Fix wording

* Update properties file name

* Update kustomization.yaml

* Fix readme typos

deploy/releases/1.4.1/readme.adoc

@@ -0,0 +1,88 @@
+= Runtime Component Operator v1.4.1
+
+== Supported Environments
+
+The Runtime Component Operator is available for the following CPU architectures:
+
+- Linux® x86_64 (amd64)
+- Linux® on IBM® Z (s390x)
+- Linux® on Power® (ppc64le)
+
+Red Hat OpenShift 4.12 or higher is supported. The operator is also supported on Kubernetes environments with v1.25 or higher.
+
+== Installation
+
+=== Option 1: Install using Operator Lifecycle Manager (OLM) on OpenShift
+
+Runtime Component Operator is available from Red Hat's Certified OpenShift Operators catalog. 
+
+To install using the integrated OperatorHub in OpenShift UI, select `Operators` from the navigation panel, then select `OperatorHub` and then search for `Runtime Component`. Click on the tile with `Runtime Component` to install the Operator. Select the channel `v1.4` and install.
+
+To install using the `oc` CLI, use the following `Subscription`. Replace `<install_namespace>` with the namespace you want to install the Operator to:
+
+```
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: runtime-component-operator-certified
+  namespace: <install_namespace>
+spec:
+  channel: v1.4
+  name: runtime-component-operator-certified
+  source: certified-operators
+  sourceNamespace: openshift-marketplace
+```
+
+==== OPTIONAL: Set multiple concurrent reconcile workers
+
+You can configure multiple workers to reconcile Runtime Component custom resource instances concurrently. If not set, the number of workers defaults to `1`.
+
+To set this using the integrated OperatorHub in OpenShift UI, select `Manual` Update Approval and install the operator. Under the `Installed Operators` tab, you should see the operator pending installation with message "Upgrade available". Click the name to enter the Subscription resource then navigate to the YAML tab to nest the following configuration as the Subscription's `.spec.config` field.
+
+```
+config:
+  env:
+    - name: MAX_CONCURRENT_RECONCILES
+      value: "8"
+```
+To set this using the `oc` CLI, use the following `Subscription`.
+
+```
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: runtime-component-operator-certified
+  namespace: <install_namespace>
+spec:
+  channel: v1.4
+  name: runtime-component-operator-certified
+  source: certified-operators
+  sourceNamespace: openshift-marketplace
+  config:
+    env:
+      - name: MAX_CONCURRENT_RECONCILES
+        value: "8"
+```
+
+You can verify if the configuration has been successfully set by viewing the Pod logs under the `rco-controller-manager` Deployment which outputs the newly modified `worker count` as `8`.
+
+```
+2024-11-15T18:10:35Z	INFO	Starting workers	{"controller": "runtimeoperation", "controllerGroup": "rc.app.stacks", "controllerKind": "RuntimeOperation", "worker count": 1}
+2024-11-15T18:10:35Z	INFO	Starting workers	{"controller": "runtimecomponent", "controllerGroup": "rc.app.stacks", "controllerKind": "RuntimeComponent", "worker count": 8}
+```
+
+
+=== Option 2: Install using kubectl
+
+See the instructions link:++kubectl/++[here].
+
+=== Option 3: Install using kustomize
+
+See the instructions link:++kustomize/++[here].
+
+
+== Limitations
+
+* Knative support is limited. Values specified for `.spec.autoscaling`, `.spec.resources` and `.spec.replicas` fields would not apply for Knative when enabled using `.spec.createKnativeService` field.
+* Monitoring feature does not support integration with Knative Service. Prometheus Operator is required to use ServiceMonitor.
+* After the initial deployment of `RuntimeComponent`, any changes to its labels would be applied only when one of the fields from `.spec` is updated.

internal/controller/runtimecomponent_controller.go

@@ -616,14 +616,16 @@ func (r *RuntimeComponentReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		},
 	}
 
+	maxConcurrentReconciles := appstacksutils.GetMaxConcurrentReconciles()
+
 	b := ctrl.NewControllerManagedBy(mgr).For(&appstacksv1.RuntimeComponent{}, builder.WithPredicates(pred)).
 		Owns(&corev1.Service{}, builder.WithPredicates(predSubResource)).
 		Owns(&corev1.Secret{}, builder.WithPredicates(predSubResource)).
 		Owns(&appsv1.Deployment{}, builder.WithPredicates(predSubResWithGenCheck)).
 		Owns(&appsv1.StatefulSet{}, builder.WithPredicates(predSubResWithGenCheck)).
 		Owns(&autoscalingv1.HorizontalPodAutoscaler{}, builder.WithPredicates(predSubResource)).
 		WithOptions(kcontroller.Options{
-			MaxConcurrentReconciles: 8,
+			MaxConcurrentReconciles: maxConcurrentReconciles,
 		})
 
 	ok, _ := r.IsGroupVersionSupported(routev1.SchemeGroupVersion.String(), "Route")

internal/controller/runtimeoperation_controller.go

@@ -31,6 +31,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	kcontroller "sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -192,6 +193,9 @@ func (r *RuntimeOperationReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&appstacksv1.RuntimeOperation{}, builder.WithPredicates(pred)).
+		WithOptions(kcontroller.Options{
+			MaxConcurrentReconciles: 1,
+		}).
 		Complete(r)
 }
 

internal/deploy/kubectl/readme.adoc

@@ -26,11 +26,13 @@ NOTE: Ensure that you replace  `<SPECIFY_OPERATOR_NAMESPACE_HERE>` and `<SPECIFY
 +
     * To watch all namespaces in the cluster, set `WATCH_NAMESPACE='""'`
 +
-
+    * To optionally reconcile multiple `RuntimeComponent` instances concurrently, set `MAX_CONCURRENT_RECONCILES=<SPECIFY_MAX_CONCURRENT_RECONCILES_HERE>` (default is `1`)
++
 [source,sh]
 ----
     OPERATOR_NAMESPACE=<SPECIFY_OPERATOR_NAMESPACE_HERE>
     WATCH_NAMESPACE=<SPECIFY_WATCH_NAMESPACE_HERE>
+    MAX_CONCURRENT_RECONCILES=<OPTIONALLY_SPECIFY_MAX_CONCURRENT_RECONCILES_HERE>
 ----
 
 .. _Optional_: Install roles and bindings to watch another namespace or all namespaces.  This step can be skipped if the operator is only watching own namespace.
@@ -60,6 +62,7 @@ curl -L https://raw.githubusercontent.com/application-stacks/runtime-component-o
 ----
 curl -L https://raw.githubusercontent.com/application-stacks/runtime-component-operator/main/internal/deploy/kubectl/runtime-component-operator.yaml \
       | sed -e "s/RUNTIME_COMPONENT_WATCH_NAMESPACE/${WATCH_NAMESPACE}/" \
+      | sed -e "s/RUNTIME_COMPONENT_MAX_CONCURRENT_RECONCILES/'${MAX_CONCURRENT_RECONCILES}'/" \
       | kubectl apply -n ${OPERATOR_NAMESPACE} -f -
 ----
 

internal/deploy/kubectl/runtime-component-operator.yaml

@@ -301,6 +301,8 @@ spec:
               fieldPath: metadata.namespace
         - name: WATCH_NAMESPACE
           value: RUNTIME_COMPONENT_WATCH_NAMESPACE
+        - name: MAX_CONCURRENT_RECONCILES
+          value: RUNTIME_COMPONENT_MAX_CONCURRENT_RECONCILES
         - name: RELATED_IMAGE_LIBERTY_SAMPLE_APP
           value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:3999aa86f788e601d305896e48a043a91861cdbf71951a1959887151390b3650
         - name: RELATED_IMAGE_RUNTIME_COMPONENT_OPERATOR

internal/deploy/kustomize/daily/README.adoc

@@ -44,4 +44,9 @@ To install, run: `kubectl create -k overlays/watch-all-namespaces`
 This example overlay builds on the previous example and demonstrates how to change
 the namespace that the operator installs into. In this example, the operator installs
 into a namespace that is called 'rco-ns' and watches for Runtime Component custom resource
-instances in any namespaces. To install, run: `kubectl create -k examples/watch-all-namespaces`
+instances in any namespaces. To install, run: `kubectl create -k examples/watch-all-namespaces`
+
+== Setting max concurrent reconciles
+
+=== overlays/set-max-concurrent-reconciles
+This overlay updates the base configuration by setting the `MAX_CONCURRENT_RECONCILES` environment variable in the operator's Deployment. It enables multiple workers to reconcile the RuntimeComponent custom resource instances concurrently. There are `8` workers set by default, and this setting can be customized in the `overlays/set-max-concurrent-reconciles/operator.properties` file. To install, run: `kubectl apply -k overlays/set-max-concurrent-reconciles`

internal/deploy/kustomize/daily/base/kustomization.yaml

@@ -7,4 +7,3 @@ resources:
   - runtime-component-crd.yaml
   - runtime-component-operator.yaml
   - runtime-component-roles.yaml
-

internal/deploy/kustomize/daily/base/runtime-component-operator.yaml

@@ -49,6 +49,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: MAX_CONCURRENT_RECONCILES
+          value: "1"
         - name: RELATED_IMAGE_LIBERTY_SAMPLE_APP
           value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:3999aa86f788e601d305896e48a043a91861cdbf71951a1959887151390b3650
         - name: RELATED_IMAGE_RUNTIME_COMPONENT_OPERATOR

internal/deploy/kustomize/daily/overlays/set-max-concurrent-reconciles/kustomization.yaml

@@ -0,0 +1,25 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: runtime-component
+
+
+resources:
+- ../../base
+
+configMapGenerator:
+  - envs:
+      - operator.properties
+    name: operator-config
+
+replacements:
+- source:
+    kind: ConfigMap
+    name: operator-config
+    fieldPath: data.MAX_CONCURRENT_RECONCILES
+  targets:
+  - select:
+      name: rco-controller-manager
+      kind: Deployment
+    fieldPaths:
+      - spec.template.spec.containers.[name=manager].env.[name=MAX_CONCURRENT_RECONCILES].value

internal/deploy/kustomize/daily/overlays/set-max-concurrent-reconciles/operator.properties

@@ -0,0 +1 @@
+MAX_CONCURRENT_RECONCILES=8

utils/utils.go

@@ -1890,3 +1890,23 @@ func ShouldDeleteRoute(ba common.BaseComponent) bool {
 	}
 	return false
 }
+
+// Returns the configured operator max concurrent reconciles setting
+func GetMaxConcurrentReconciles() int {
+	envValue := os.Getenv("MAX_CONCURRENT_RECONCILES")
+	if intVal := parseEnvAsPositiveInt(envValue); intVal != -1 {
+		return intVal
+	}
+	return 1
+}
+
+// Parses env as positive int or returns -1 on failure
+func parseEnvAsPositiveInt(envValue string) int {
+	if envValue != "" {
+		positiveIntVal, err := strconv.ParseInt(envValue, 10, 64)
+		if err == nil && positiveIntVal >= 1 {
+			return int(positiveIntVal)
+		}
+	}
+	return -1
+}

utils/utils_test.go

@@ -763,6 +763,40 @@ func TestGetWatchNamespaces(t *testing.T) {
 	verifyTests(configMapConstTests, t)
 }
 
+func TestGetMaxConcurrentReconciles(t *testing.T) {
+	// Set the logger to development mode for verbose logs
+	logger := zap.New()
+	logf.SetLogger(logger)
+
+	os.Setenv("MAX_CONCURRENT_RECONCILES", "1")
+	maxConcurrentReconciles := GetMaxConcurrentReconciles()
+	maxConcurrentReconcilesTests := []Test{
+		{"max concurrent reconcile (env set to 1)", 1, maxConcurrentReconciles},
+	}
+	verifyTests(maxConcurrentReconcilesTests, t)
+
+	os.Setenv("MAX_CONCURRENT_RECONCILES", "-1")
+	maxConcurrentReconciles = GetMaxConcurrentReconciles()
+	maxConcurrentReconcilesTests = []Test{
+		{"max concurrent reconcile (env set to -1)", 1, maxConcurrentReconciles},
+	}
+	verifyTests(maxConcurrentReconcilesTests, t)
+
+	os.Setenv("MAX_CONCURRENT_RECONCILES", "8")
+	maxConcurrentReconciles = GetMaxConcurrentReconciles()
+	maxConcurrentReconcilesTests = []Test{
+		{"max concurrent reconcile (env set to 8)", 8, maxConcurrentReconciles},
+	}
+	verifyTests(maxConcurrentReconcilesTests, t)
+
+	os.Setenv("MAX_CONCURRENT_RECONCILES", "tenthousand")
+	maxConcurrentReconciles = GetMaxConcurrentReconciles()
+	maxConcurrentReconcilesTests = []Test{
+		{"max concurrent reconcile (env set to NaN)", 1, maxConcurrentReconciles},
+	}
+	verifyTests(maxConcurrentReconcilesTests, t)
+}
+
 func TestShouldDeleteRoute(t *testing.T) {
 	logger := zap.New()
 	logf.SetLogger(logger)