fix(apache#592): Introduce "dependencies" build order strategy

- Introduce new build order strategy "dependencies"
- Strategy looks at the list of dependencies required by an Integration and queues builds that may reuse base images produced by other scheduled builds in order to leverage the incremental build option
- The strategy allows non-matching builds to run in parallel to each other

config/crd/bases/camel.apache.org_builds.yaml

@@ -100,6 +100,7 @@ spec:
                   orderStrategy:
                     description: the build order strategy to adopt
                     enum:
+                    - dependencies
                     - fifo
                     - sequential
                     type: string
@@ -215,6 +216,7 @@ spec:
                             orderStrategy:
                               description: the build order strategy to adopt
                               enum:
+                              - dependencies
                               - fifo
                               - sequential
                               type: string

config/crd/bases/camel.apache.org_integrationkits.yaml

@@ -212,8 +212,8 @@ spec:
                           of memory required by the pod builder.
                         type: string
                       orderStrategy:
-                        description: The build order strategy to use, either `fifo`
-                          or `sequential` (default sequential)
+                        description: The build order strategy to use, either `dependencies`,
+                          `fifo` or `sequential` (default sequential)
                         type: string
                       properties:
                         description: A list of properties to be provided to the build

config/crd/bases/camel.apache.org_integrationplatforms.yaml

@@ -119,6 +119,7 @@ spec:
                       orderStrategy:
                         description: the build order strategy to adopt
                         enum:
+                        - dependencies
                         - fifo
                         - sequential
                         type: string
@@ -469,8 +470,8 @@ spec:
                           of memory required by the pod builder.
                         type: string
                       orderStrategy:
-                        description: The build order strategy to use, either `fifo`
-                          or `sequential` (default sequential)
+                        description: The build order strategy to use, either `dependencies`,
+                          `fifo` or `sequential` (default sequential)
                         type: string
                       properties:
                         description: A list of properties to be provided to the build
@@ -1766,6 +1767,7 @@ spec:
                       orderStrategy:
                         description: the build order strategy to adopt
                         enum:
+                        - dependencies
                         - fifo
                         - sequential
                         type: string
@@ -2164,8 +2166,8 @@ spec:
                           of memory required by the pod builder.
                         type: string
                       orderStrategy:
-                        description: The build order strategy to use, either `fifo`
-                          or `sequential` (default sequential)
+                        description: The build order strategy to use, either `dependencies`,
+                          `fifo` or `sequential` (default sequential)
                         type: string
                       properties:
                         description: A list of properties to be provided to the build

config/crd/bases/camel.apache.org_integrations.yaml

@@ -6195,8 +6195,8 @@ spec:
                           of memory required by the pod builder.
                         type: string
                       orderStrategy:
-                        description: The build order strategy to use, either `fifo`
-                          or `sequential` (default sequential)
+                        description: The build order strategy to use, either `dependencies`,
+                          `fifo` or `sequential` (default sequential)
                         type: string
                       properties:
                         description: A list of properties to be provided to the build

config/crd/bases/camel.apache.org_kameletbindings.yaml

@@ -6470,8 +6470,8 @@ spec:
                               of memory required by the pod builder.
                             type: string
                           orderStrategy:
-                            description: The build order strategy to use, either `fifo`
-                              or `sequential` (default sequential)
+                            description: The build order strategy to use, either `dependencies`,
+                              `fifo` or `sequential` (default sequential)
                             type: string
                           properties:
                             description: A list of properties to be provided to the

config/crd/bases/camel.apache.org_pipes.yaml

@@ -6467,8 +6467,8 @@ spec:
                               of memory required by the pod builder.
                             type: string
                           orderStrategy:
-                            description: The build order strategy to use, either `fifo`
-                              or `sequential` (default sequential)
+                            description: The build order strategy to use, either `dependencies`,
+                              `fifo` or `sequential` (default sequential)
                             type: string
                           properties:
                             description: A list of properties to be provided to the

docs/modules/ROOT/pages/architecture/cr/build.adoc

@@ -43,6 +43,7 @@ You can choose from different build order strategies. The strategy defines in wh
 At the moment the available strategies are:
 
 - buildOrderStrategy: sequential (runs builds strictly sequential so that only one single build per operator namespace is running at a time.)
+- buildOrderStrategy: dependencies (strategy looks at the list of dependencies required by an Integration and queues builds that may reuse base images produced by other scheduled builds in order to leverage the incremental build option. The strategy allows non-matching builds to run in parallel to each other.)
 - buildOrderStrategy: fifo (performs the builds with first in first out strategy based on the creation timestamp. The strategy allows builds to run in parallel to each other but oldest builds will be run first.)
 
 [[build-queue]]

docs/modules/ROOT/partials/apis/camel-k-crds.adoc

@@ -5412,7 +5412,7 @@ string
 |
 
 
-The build order strategy to use, either `fifo` or `sequential` (default sequential)
+The build order strategy to use, either `dependencies`, `fifo` or `sequential` (default sequential)
 
 |`requestCPU` +
 string

docs/modules/traits/pages/builder.adoc

@@ -42,7 +42,7 @@ The following configuration options are available:
 
 | builder.order-strategy
 | string
-| The build order strategy to use, either `fifo` or `sequential` (default sequential)
+| The build order strategy to use, either `dependencies`, `fifo` or `sequential` (default sequential)
 
 | builder.request-cpu
 | string

e2e/builder/build_test.go

@@ -214,6 +214,80 @@ func TestKitMaxBuildLimitFIFOStrategy(t *testing.T) {
 	})
 }
 
+func TestKitMaxBuildLimitDependencyMatchingStrategy(t *testing.T) {
+	WithNewTestNamespace(t, func(ns string) {
+		createOperator(ns, "8m0s", "--global", "--force")
+
+		pl := Platform(ns)()
+		// set maximum number of running builds and order strategy
+		pl.Spec.Build.MaxRunningBuilds = 2
+		pl.Spec.Build.BuildConfiguration.OrderStrategy = v1.BuildOrderStrategyDependencies
+		if err := TestClient().Update(TestContext, pl); err != nil {
+			t.Error(err)
+			t.FailNow()
+		}
+
+		buildA := "integration-a"
+		buildB := "integration-b"
+		buildC := "integration-c"
+
+		doKitBuildInNamespace(buildA, ns, TestTimeoutShort, kitOptions{
+			operatorID: fmt.Sprintf("camel-k-%s", ns),
+			dependencies: []string{
+				"camel:timer", "camel:log",
+			},
+			traits: []string{
+				"builder.properties=build-property=A",
+			},
+		}, v1.BuildPhaseRunning, v1.IntegrationKitPhaseBuildRunning)
+
+		doKitBuildInNamespace(buildB, ns, TestTimeoutShort, kitOptions{
+			operatorID: fmt.Sprintf("camel-k-%s", ns),
+			dependencies: []string{
+				"camel:cron", "camel:log", "camel:joor",
+			},
+			traits: []string{
+				"builder.properties=build-property=B",
+			},
+		}, v1.BuildPhaseRunning, v1.IntegrationKitPhaseBuildRunning)
+
+		doKitBuildInNamespace(buildC, ns, TestTimeoutShort, kitOptions{
+			operatorID: fmt.Sprintf("camel-k-%s", ns),
+			dependencies: []string{
+				"camel:timer", "camel:log", "camel:joor", "camel:http",
+			},
+			traits: []string{
+				"builder.properties=build-property=C",
+			},
+		}, v1.BuildPhaseScheduling, v1.IntegrationKitPhaseNone)
+
+		var notExceedsMaxBuildLimit = func(runningBuilds int) bool {
+			return runningBuilds <= 2
+		}
+
+		limit := 0
+		for limit < 5 && BuildPhase(ns, buildA)() == v1.BuildPhaseRunning {
+			// verify that number of running builds does not exceed max build limit
+			Consistently(BuildsRunning(BuildPhase(ns, buildA), BuildPhase(ns, buildB), BuildPhase(ns, buildC)), TestTimeoutShort, 10*time.Second).Should(Satisfy(notExceedsMaxBuildLimit))
+			limit++
+		}
+
+		// make sure we have verified max build limit at least once
+		if limit == 0 {
+			t.Error(errors.New(fmt.Sprintf("Unexpected build phase '%s' for %s - not able to verify max builds limit", BuildPhase(ns, buildA)(), buildA)))
+			t.FailNow()
+		}
+
+		// verify that all builds are successful
+		Eventually(BuildPhase(ns, buildA), TestTimeoutLong).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(KitPhase(ns, buildA), TestTimeoutLong).Should(Equal(v1.IntegrationKitPhaseReady))
+		Eventually(BuildPhase(ns, buildB), TestTimeoutLong).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(KitPhase(ns, buildB), TestTimeoutLong).Should(Equal(v1.IntegrationKitPhaseReady))
+		Eventually(BuildPhase(ns, buildC), TestTimeoutLong).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(KitPhase(ns, buildC), TestTimeoutLong).Should(Equal(v1.IntegrationKitPhaseReady))
+	})
+}
+
 func TestKitTimerToLogFullBuild(t *testing.T) {
 	doKitFullBuild(t, "timer-to-log", "8m0s", TestTimeoutLong, kitOptions{
 		dependencies: []string{

e2e/common/traits/builder_test.go

@@ -63,6 +63,32 @@ func TestBuilderTrait(t *testing.T) {
 		Expect(Kamel("reset", "-n", ns).Execute()).To(Succeed())
 	})
 
+	t.Run("Run build order strategy dependencies", func(t *testing.T) {
+		name := "java-fifo-strategy"
+		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",
+			"--name", name,
+			"-t", "builder.order-strategy=dependencies").Execute()).To(Succeed())
+
+		Eventually(IntegrationPodPhase(ns, name), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+
+		integrationKitName := IntegrationKit(ns, name)()
+		builderKitName := fmt.Sprintf("camel-k-%s-builder", integrationKitName)
+		Eventually(BuildConfig(ns, integrationKitName)().Strategy, TestTimeoutShort).Should(Equal(v1.BuildStrategyRoutine))
+		Eventually(BuildConfig(ns, integrationKitName)().OrderStrategy, TestTimeoutShort).Should(Equal(v1.BuildOrderStrategyDependencies))
+		// Default resource CPU Check
+		Eventually(BuildConfig(ns, integrationKitName)().RequestCPU, TestTimeoutShort).Should(Equal(""))
+		Eventually(BuildConfig(ns, integrationKitName)().LimitCPU, TestTimeoutShort).Should(Equal(""))
+		Eventually(BuildConfig(ns, integrationKitName)().RequestMemory, TestTimeoutShort).Should(Equal(""))
+		Eventually(BuildConfig(ns, integrationKitName)().LimitMemory, TestTimeoutShort).Should(Equal(""))
+
+		Eventually(BuilderPod(ns, builderKitName), TestTimeoutShort).Should(BeNil())
+
+		// We need to remove the kit as well
+		Expect(Kamel("reset", "-n", ns).Execute()).To(Succeed())
+	})
+
 	t.Run("Run build order strategy fifo", func(t *testing.T) {
 		name := "java-fifo-strategy"
 		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",

e2e/commonwithcustominstall/build_order_strategy_test.go

@@ -0,0 +1,155 @@
+//go:build integration
+// +build integration
+
+// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "integration"
+
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commonwithcustominstall
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+
+	. "github.com/apache/camel-k/v2/e2e/support"
+	v1 "github.com/apache/camel-k/v2/pkg/apis/camel/v1"
+	"github.com/apache/camel-k/v2/pkg/util/defaults"
+)
+
+func TestRunBuildOrderStrategyMatchingDependencies(t *testing.T) {
+	WithNewTestNamespace(t, func(ns string) {
+		operatorID := "camel-k-build-order-deps"
+		Expect(KamelInstallWithID(operatorID, ns, "--build-order-strategy", string(v1.BuildOrderStrategyDependencies)).Execute()).To(Succeed())
+
+		integrationA := "java-a"
+		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",
+			"--name", integrationA,
+		).Execute()).To(Succeed())
+		integrationKitNameA := IntegrationKit(ns, integrationA)()
+		Eventually(BuildPhase(ns, integrationKitNameA)).Should(Equal(v1.BuildPhaseScheduling))
+
+		integrationB := "java-b"
+		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",
+			"--name", integrationB,
+			"-d", "camel:joor",
+		).Execute()).To(Succeed())
+
+		integrationC := "java-c"
+		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",
+			"--name", integrationC,
+			"-d", "camel:joor",
+			"-d", "camel:zipfile",
+		).Execute()).To(Succeed())
+
+		integrationZ := "groovy-z"
+		Expect(KamelRunWithID(operatorID, ns, "files/simple.groovy",
+			"--name", integrationZ,
+		).Execute()).To(Succeed())
+
+		integrationKitNameB := IntegrationKit(ns, integrationB)()
+		integrationKitNameC := IntegrationKit(ns, integrationC)()
+		integrationKitNameZ := IntegrationKit(ns, integrationZ)()
+
+		Eventually(BuildPhase(ns, integrationKitNameA)).Should(Equal(v1.BuildPhaseRunning))
+		Eventually(BuildPhase(ns, integrationKitNameB)).Should(Equal(v1.BuildPhaseScheduling))
+		Eventually(BuildPhase(ns, integrationKitNameC)).Should(Equal(v1.BuildPhaseScheduling))
+		Eventually(BuildPhase(ns, integrationKitNameZ)).Should(Equal(v1.BuildPhaseRunning))
+		Eventually(BuildPhase(ns, integrationKitNameB)).Should(Equal(v1.BuildPhaseScheduling))
+		Eventually(BuildPhase(ns, integrationKitNameC)).Should(Equal(v1.BuildPhaseScheduling))
+
+		Eventually(BuildPhase(ns, integrationKitNameA)).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(IntegrationPodPhase(ns, integrationA), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, integrationA, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, integrationA), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+		Eventually(Kit(ns, integrationKitNameA)().Status.BaseImage).Should(Equal(defaults.BaseImage()))
+
+		Eventually(BuildPhase(ns, integrationKitNameB)).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(IntegrationPodPhase(ns, integrationB), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, integrationB, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, integrationB), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+		Eventually(Kit(ns, integrationKitNameB)().Status.BaseImage).Should(ContainSubstring(integrationKitNameA))
+
+		Eventually(BuildPhase(ns, integrationKitNameC)).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(IntegrationPodPhase(ns, integrationC), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, integrationC, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, integrationC), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+		Eventually(Kit(ns, integrationKitNameC)().Status.BaseImage).Should(ContainSubstring(integrationKitNameB))
+
+		Eventually(BuildPhase(ns, integrationKitNameZ)).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(IntegrationPodPhase(ns, integrationZ), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, integrationZ, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, integrationZ), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+		Eventually(Kit(ns, integrationKitNameZ)().Status.BaseImage).Should(Equal(defaults.BaseImage()))
+
+		Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed())
+	})
+}
+
+func TestRunBuildOrderStrategyFIFO(t *testing.T) {
+	WithNewTestNamespace(t, func(ns string) {
+		operatorID := "camel-k-build-order-fifo"
+		Expect(KamelInstallWithID(operatorID, ns, "--build-order-strategy", string(v1.BuildOrderStrategyFIFO)).Execute()).To(Succeed())
+
+		integrationA := "java-a"
+		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",
+			"--name", integrationA,
+		).Execute()).To(Succeed())
+
+		integrationB := "java-b"
+		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",
+			"--name", integrationB,
+			"-d", "camel:joor",
+		).Execute()).To(Succeed())
+
+		integrationZ := "groovy-z"
+		Expect(KamelRunWithID(operatorID, ns, "files/simple.groovy",
+			"--name", integrationZ,
+		).Execute()).To(Succeed())
+
+		integrationKitNameA := IntegrationKit(ns, integrationA)()
+		integrationKitNameB := IntegrationKit(ns, integrationB)()
+		integrationKitNameZ := IntegrationKit(ns, integrationZ)()
+
+		Eventually(BuildPhase(ns, integrationKitNameA)).Should(Equal(v1.BuildPhaseRunning))
+		Eventually(BuildPhase(ns, integrationKitNameB)).Should(Equal(v1.BuildPhaseRunning))
+		Eventually(BuildPhase(ns, integrationKitNameZ)).Should(Equal(v1.BuildPhaseRunning))
+
+		Eventually(BuildPhase(ns, integrationKitNameA)).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(IntegrationPodPhase(ns, integrationA), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, integrationA, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, integrationA), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+		Eventually(Kit(ns, integrationKitNameA)().Status.BaseImage).Should(Equal(defaults.BaseImage()))
+
+		Eventually(BuildPhase(ns, integrationKitNameB)).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(IntegrationPodPhase(ns, integrationB), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, integrationB, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, integrationB), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+		Eventually(Kit(ns, integrationKitNameB)().Status.BaseImage).Should(Equal(defaults.BaseImage()))
+
+		Eventually(BuildPhase(ns, integrationKitNameZ)).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(IntegrationPodPhase(ns, integrationZ), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, integrationZ, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, integrationZ), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+		Eventually(Kit(ns, integrationKitNameZ)().Status.BaseImage).Should(Equal(defaults.BaseImage()))
+
+		Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed())
+	})
+}