fix(apache#592): Introduce build order strategy

- Run builds on same operator namespace with user defined strategy
- Default strategy is "sequential" running only one single build at a time
- Also support "fifo" strategy where builds are run/queued based on their creation timestamp. This strategy allows parallel builds as long as individual build dependency lists are not colliding
- Users may adjust/overwrite the build order strategy via install command option, in the (local) integration platform settings or via the builder trait option
- Max number of running builds limitation is not affected by the build order strategy (ensure to always obey the limitation)

docs/modules/ROOT/pages/architecture/cr/build.adoc

@@ -36,6 +36,15 @@ At the moment the available strategies are:
 - buildStrategy: pod (each build is run in a separate pod, the operator monitors the pod state)
 - buildStrategy: routine (each build is run as a go routine inside the operator pod)
 
+[[build-order-strategy]]
+== Build order strategy
+
+You can choose from different build order strategies. The strategy defines in which order queued builds are run.
+At the moment the available strategies are:
+
+- buildOrderStrategy: sequential (runs builds strictly sequential so that only one single build per operator namespace is running at a time.)
+- buildOrderStrategy: fifo (performs the builds with first in first out strategy based on the creation timestamp. The strategy allows builds to run in parallel to each other but oldest builds will be run first.)
+
 [[build-queue]]
 == Build queues
 

docs/modules/ROOT/pages/installation/advanced/advanced.adoc

@@ -38,6 +38,7 @@ We have several configuration used to influence the building of an integration:
 --build-publish-strategy string               Set the build publish strategy
 --build-publish-strategy-option stringArray   Add a build publish strategy option, as <name=value>
 --build-strategy string                       Set the build strategy
+--build-order-strategy string                 Set the build order strategy
 --build-timeout string                        Set how long the build process can last
 ```
 A very important set of configuration you can provide is related to Maven:
@@ -85,4 +86,4 @@ We have also certain configuration that let you control how to deploy your Camel
 --global                                      Configure the operator to watch all namespaces. No integration platform is created. You can run integrations in a namespace by installing an integration platform: 'kamel install --skip-operator-setup -n my-namespace'
 --operator-id string                          Set the operator id that is used to select the resources this operator should manage (default "camel-k")
 ```
-Learn more about xref:installation/advanced/multi.adoc[Camel K multi-tenancy].
+Learn more about xref:installation/advanced/multi.adoc[Camel K multi-tenancy].

e2e/builder/build_test.go

@@ -45,8 +45,9 @@ func TestKitMaxBuildLimit(t *testing.T) {
 		createOperator(ns, "8m0s", "--global", "--force")
 
 		pl := Platform(ns)()
-		// set maximum number of running builds
+		// set maximum number of running builds and order strategy
 		pl.Spec.Build.MaxRunningBuilds = 2
+		pl.Spec.Build.BuildConfiguration.OrderStrategy = v1.BuildOrderStrategySequential
 		if err := TestClient().Update(TestContext, pl); err != nil {
 			t.Error(err)
 			t.FailNow()
@@ -139,6 +140,80 @@ func TestKitMaxBuildLimit(t *testing.T) {
 	})
 }
 
+func TestKitMaxBuildLimitFIFOStrategy(t *testing.T) {
+	WithNewTestNamespace(t, func(ns string) {
+		createOperator(ns, "8m0s", "--global", "--force")
+
+		pl := Platform(ns)()
+		// set maximum number of running builds and order strategy
+		pl.Spec.Build.MaxRunningBuilds = 2
+		pl.Spec.Build.BuildConfiguration.OrderStrategy = v1.BuildOrderStrategyFIFO
+		if err := TestClient().Update(TestContext, pl); err != nil {
+			t.Error(err)
+			t.FailNow()
+		}
+
+		buildA := "integration-a"
+		buildB := "integration-b"
+		buildC := "integration-c"
+
+		doKitBuildInNamespace(buildA, ns, TestTimeoutShort, kitOptions{
+			operatorID: fmt.Sprintf("camel-k-%s", ns),
+			dependencies: []string{
+				"camel:timer", "camel:log",
+			},
+			traits: []string{
+				"builder.properties=build-property=A",
+			},
+		}, v1.BuildPhaseRunning, v1.IntegrationKitPhaseBuildRunning)
+
+		doKitBuildInNamespace(buildB, ns, TestTimeoutShort, kitOptions{
+			operatorID: fmt.Sprintf("camel-k-%s", ns),
+			dependencies: []string{
+				"camel:timer", "camel:log",
+			},
+			traits: []string{
+				"builder.properties=build-property=B",
+			},
+		}, v1.BuildPhaseRunning, v1.IntegrationKitPhaseBuildRunning)
+
+		doKitBuildInNamespace(buildC, ns, TestTimeoutShort, kitOptions{
+			operatorID: fmt.Sprintf("camel-k-%s", ns),
+			dependencies: []string{
+				"camel:timer", "camel:log",
+			},
+			traits: []string{
+				"builder.properties=build-property=C",
+			},
+		}, v1.BuildPhaseScheduling, v1.IntegrationKitPhaseNone)
+
+		var notExceedsMaxBuildLimit = func(runningBuilds int) bool {
+			return runningBuilds <= 2
+		}
+
+		limit := 0
+		for limit < 5 && BuildPhase(ns, buildA)() == v1.BuildPhaseRunning {
+			// verify that number of running builds does not exceed max build limit
+			Consistently(BuildsRunning(BuildPhase(ns, buildA), BuildPhase(ns, buildB), BuildPhase(ns, buildC)), TestTimeoutShort, 10*time.Second).Should(Satisfy(notExceedsMaxBuildLimit))
+			limit++
+		}
+
+		// make sure we have verified max build limit at least once
+		if limit == 0 {
+			t.Error(errors.New(fmt.Sprintf("Unexpected build phase '%s' for %s - not able to verify max builds limit", BuildPhase(ns, buildA)(), buildA)))
+			t.FailNow()
+		}
+
+		// verify that all builds are successful
+		Eventually(BuildPhase(ns, buildA), TestTimeoutLong).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(KitPhase(ns, buildA), TestTimeoutLong).Should(Equal(v1.IntegrationKitPhaseReady))
+		Eventually(BuildPhase(ns, buildB), TestTimeoutLong).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(KitPhase(ns, buildB), TestTimeoutLong).Should(Equal(v1.IntegrationKitPhaseReady))
+		Eventually(BuildPhase(ns, buildC), TestTimeoutLong).Should(Equal(v1.BuildPhaseSucceeded))
+		Eventually(KitPhase(ns, buildC), TestTimeoutLong).Should(Equal(v1.IntegrationKitPhaseReady))
+	})
+}
+
 func TestKitTimerToLogFullBuild(t *testing.T) {
 	doKitFullBuild(t, "timer-to-log", "8m0s", TestTimeoutLong, kitOptions{
 		dependencies: []string{

e2e/common/traits/builder_test.go

@@ -51,6 +51,32 @@ func TestBuilderTrait(t *testing.T) {
 		integrationKitName := IntegrationKit(ns, name)()
 		builderKitName := fmt.Sprintf("camel-k-%s-builder", integrationKitName)
 		Eventually(BuildConfig(ns, integrationKitName)().Strategy, TestTimeoutShort).Should(Equal(v1.BuildStrategyRoutine))
+		Eventually(BuildConfig(ns, integrationKitName)().OrderStrategy, TestTimeoutShort).Should(Equal(v1.BuildOrderStrategySequential))
+		// Default resource CPU Check
+		Eventually(BuildConfig(ns, integrationKitName)().RequestCPU, TestTimeoutShort).Should(Equal(""))
+		Eventually(BuildConfig(ns, integrationKitName)().LimitCPU, TestTimeoutShort).Should(Equal(""))
+		Eventually(BuildConfig(ns, integrationKitName)().RequestMemory, TestTimeoutShort).Should(Equal(""))
+		Eventually(BuildConfig(ns, integrationKitName)().LimitMemory, TestTimeoutShort).Should(Equal(""))
+
+		Eventually(BuilderPod(ns, builderKitName), TestTimeoutShort).Should(BeNil())
+
+		// We need to remove the kit as well
+		Expect(Kamel("reset", "-n", ns).Execute()).To(Succeed())
+	})
+
+	t.Run("Run build order strategy fifo", func(t *testing.T) {
+		Expect(KamelRunWithID(operatorID, ns, "files/Java.java",
+			"--name", name,
+			"-t", "builder.order-strategy=fifo").Execute()).To(Succeed())
+
+		Eventually(IntegrationPodPhase(ns, name), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+		Eventually(IntegrationConditionStatus(ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+		Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+
+		integrationKitName := IntegrationKit(ns, name)()
+		builderKitName := fmt.Sprintf("camel-k-%s-builder", integrationKitName)
+		Eventually(BuildConfig(ns, integrationKitName)().Strategy, TestTimeoutShort).Should(Equal(v1.BuildStrategyPod))
+		Eventually(BuildConfig(ns, integrationKitName)().OrderStrategy, TestTimeoutShort).Should(Equal(v1.BuildOrderStrategyFIFO))
 		// Default resource CPU Check
 		Eventually(BuildConfig(ns, integrationKitName)().RequestCPU, TestTimeoutShort).Should(Equal(""))
 		Eventually(BuildConfig(ns, integrationKitName)().LimitCPU, TestTimeoutShort).Should(Equal(""))
@@ -80,6 +106,7 @@ func TestBuilderTrait(t *testing.T) {
 		builderKitName := fmt.Sprintf("camel-k-%s-builder", integrationKitName)
 
 		Eventually(BuildConfig(ns, integrationKitName)().Strategy, TestTimeoutShort).Should(Equal(v1.BuildStrategyPod))
+		Eventually(BuildConfig(ns, integrationKitName)().OrderStrategy, TestTimeoutShort).Should(Equal(v1.BuildOrderStrategySequential))
 		Eventually(BuildConfig(ns, integrationKitName)().RequestCPU, TestTimeoutShort).Should(Equal("500m"))
 		Eventually(BuildConfig(ns, integrationKitName)().LimitCPU, TestTimeoutShort).Should(Equal("1000m"))
 		Eventually(BuildConfig(ns, integrationKitName)().RequestMemory, TestTimeoutShort).Should(Equal("2Gi"))

e2e/commonwithcustominstall/local_platform_test.go

@@ -82,6 +82,7 @@ func TestLocalPlatform(t *testing.T) {
 			local := Platform(ns1)()
 			Expect(local.Status.Build.PublishStrategy).To(Equal(pl.Status.Build.PublishStrategy))
 			Expect(local.Status.Build.BuildConfiguration.Strategy).To(Equal(pl.Status.Build.BuildConfiguration.Strategy))
+			Expect(local.Status.Build.BuildConfiguration.OrderStrategy).To(Equal(pl.Status.Build.BuildConfiguration.OrderStrategy))
 			Expect(local.Status.Build.Maven.LocalRepository).To(Equal(pl.Status.Build.Maven.LocalRepository))
 			Expect(local.Status.Build.Maven.CLIOptions).To(ContainElements(pl.Status.Build.Maven.CLIOptions))
 			Expect(local.Status.Build.Maven.Extension).To(BeEmpty())

pkg/apis/camel/v1/build_types_support.go

@@ -201,3 +201,28 @@ func (c BuildCondition) GetReason() string {
 func (c BuildCondition) GetMessage() string {
 	return c.Message
 }
+
+func (bl BuildList) HasRunningBuilds() bool {
+	for _, b := range bl.Items {
+		if b.Status.Phase == BuildPhasePending || b.Status.Phase == BuildPhaseRunning {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (bl BuildList) HasScheduledBuildsBefore(build *Build) bool {
+	for _, b := range bl.Items {
+		if b.Name == build.Name {
+			continue
+		}
+
+		if (b.Status.Phase == BuildPhaseInitialization || b.Status.Phase == BuildPhaseScheduling) &&
+			b.CreationTimestamp.Before(&build.CreationTimestamp) {
+			return true
+		}
+	}
+
+	return false
+}

pkg/apis/camel/v1/common_types.go

@@ -43,6 +43,8 @@ type BuildConfiguration struct {
 	BuilderPodNamespace string `json:"operatorNamespace,omitempty"`
 	// the strategy to adopt
 	Strategy BuildStrategy `property:"strategy" json:"strategy,omitempty"`
+	// the build order strategy to adopt
+	OrderStrategy BuildOrderStrategy `property:"order-strategy" json:"orderStrategy,omitempty"`
 	// The minimum amount of CPU required. Only used for `pod` strategy
 	RequestCPU string `property:"request-cpu" json:"requestCPU,omitempty"`
 	// The minimum amount of memory required. Only used for `pod` strategy
@@ -67,6 +69,12 @@ const (
 	// BuildStrategyPod performs the build in a `Pod` (will schedule a new builder ephemeral `Pod` which will take care of the build action).
 	// This strategy has the limitation that every build will have to download all the dependencies required by the Maven build.
 	BuildStrategyPod BuildStrategy = "pod"
+
+	// BuildOrderStrategyFIFO performs the builds with first in first out strategy based on the creation timestamp.
+	// The strategy allows builds to run in parallel to each other but oldest builds will be run first.
+	BuildOrderStrategyFIFO BuildOrderStrategy = "fifo"
+	// BuildOrderStrategySequential runs builds strictly sequential so that only one single build per operator namespace is running at a time.
+	BuildOrderStrategySequential BuildOrderStrategy = "sequential"
 )
 
 // BuildStrategies is a list of strategies allowed for the build
@@ -75,6 +83,16 @@ var BuildStrategies = []BuildStrategy{
 	BuildStrategyPod,
 }
 
+// BuildOrderStrategy specifies how builds are reconciled and queued.
+// +kubebuilder:validation:Enum=fifo;sequential
+type BuildOrderStrategy string
+
+// BuildOrderStrategies is a list of order strategies allowed for the build
+var BuildOrderStrategies = []BuildOrderStrategy{
+	BuildOrderStrategyFIFO,
+	BuildOrderStrategySequential,
+}
+
 // ConfigurationSpec represents a generic configuration specification
 type ConfigurationSpec struct {
 	// represents the type of configuration, ie: property, configmap, secret, ...

pkg/apis/camel/v1/common_types_support.go

@@ -159,6 +159,7 @@ var _ json.Unmarshaler = (*RawMessage)(nil)
 // IsEmpty -- .
 func (bc *BuildConfiguration) IsEmpty() bool {
 	return bc.Strategy == "" &&
+		bc.OrderStrategy == "" &&
 		bc.RequestCPU == "" &&
 		bc.RequestMemory == "" &&
 		bc.LimitCPU == "" &&

pkg/apis/camel/v1/trait/builder.go

@@ -29,6 +29,8 @@ type BuilderTrait struct {
 	Properties []string `property:"properties" json:"properties,omitempty"`
 	// The strategy to use, either `pod` or `routine` (default routine)
 	Strategy string `property:"strategy" json:"strategy,omitempty"`
+	// The build order strategy to use, either `fifo` or `sequential` (default sequential)
+	OrderStrategy string `property:"order-strategy" json:"orderStrategy,omitempty"`
 	// When using `pod` strategy, the minimum amount of CPU required by the pod builder.
 	RequestCPU string `property:"request-cpu" json:"requestCPU,omitempty"`
 	// When using `pod` strategy, the minimum amount of memory required by the pod builder.

pkg/cmd/install.go

@@ -107,6 +107,7 @@ func newCmdInstall(rootCmdOptions *RootCmdOptions) (*cobra.Command, *installCmdO
 	cmd.Flags().String("operator-image", "", "Set the operator Image used for the operator deployment")
 	cmd.Flags().String("operator-image-pull-policy", "", "Set the operator ImagePullPolicy used for the operator deployment")
 	cmd.Flags().String("build-strategy", "", "Set the build strategy")
+	cmd.Flags().String("build-order-strategy", "", "Set the build order strategy")
 	cmd.Flags().String("build-publish-strategy", "", "Set the build publish strategy")
 	cmd.Flags().StringArray("build-publish-strategy-option", nil, "Add a build publish strategy option, as <name=value>")
 	cmd.Flags().String("build-timeout", "", "Set how long the build process can last")
@@ -187,6 +188,7 @@ type installCmdOptions struct {
 	OperatorImage               string   `mapstructure:"operator-image"`
 	OperatorImagePullPolicy     string   `mapstructure:"operator-image-pull-policy"`
 	BuildStrategy               string   `mapstructure:"build-strategy"`
+	BuildOrderStrategy          string   `mapstructure:"build-order-strategy"`
 	BuildPublishStrategy        string   `mapstructure:"build-publish-strategy"`
 	BuildPublishStrategyOptions []string `mapstructure:"build-publish-strategy-options"`
 	BuildTimeout                string   `mapstructure:"build-timeout"`
@@ -535,6 +537,9 @@ func (o *installCmdOptions) setupIntegrationPlatform(c client.Client, namespace
 	if o.BuildStrategy != "" {
 		platform.Spec.Build.BuildConfiguration.Strategy = v1.BuildStrategy(o.BuildStrategy)
 	}
+	if o.BuildOrderStrategy != "" {
+		platform.Spec.Build.BuildConfiguration.OrderStrategy = v1.BuildOrderStrategy(o.BuildOrderStrategy)
+	}
 	if o.BuildPublishStrategy != "" {
 		platform.Spec.Build.PublishStrategy = v1.IntegrationPlatformBuildPublishStrategy(o.BuildPublishStrategy)
 	}
@@ -767,6 +772,23 @@ func (o *installCmdOptions) validate(_ *cobra.Command, _ []string) error {
 		}
 	}
 
+	if o.BuildOrderStrategy != "" {
+		found := false
+		for _, s := range v1.BuildOrderStrategies {
+			if string(s) == o.BuildOrderStrategy {
+				found = true
+				break
+			}
+		}
+		if !found {
+			var strategies []string
+			for _, s := range v1.BuildOrderStrategies {
+				strategies = append(strategies, string(s))
+			}
+			return fmt.Errorf("unknown build order strategy: %s. One of [%s] is expected", o.BuildOrderStrategy, strings.Join(strategies, ", "))
+		}
+	}
+
 	if o.BuildPublishStrategy != "" {
 		found := false
 		for _, s := range v1.IntegrationPlatformBuildPublishStrategies {

pkg/cmd/install_test.go

@@ -107,6 +107,13 @@ func TestInstallBuildStrategyFlag(t *testing.T) {
 	assert.Equal(t, "someString", installCmdOptions.BuildStrategy)
 }
 
+func TestInstallBuildOrderStrategyFlag(t *testing.T) {
+	installCmdOptions, rootCmd, _ := initializeInstallCmdOptions(t)
+	_, err := test.ExecuteCommand(rootCmd, cmdInstall, "--build-order-strategy", "someString")
+	assert.Nil(t, err)
+	assert.Equal(t, "someString", installCmdOptions.BuildOrderStrategy)
+}
+
 func TestInstallBuildTimeoutFlag(t *testing.T) {
 	installCmdOptions, rootCmd, _ := initializeInstallCmdOptions(t)
 	_, err := test.ExecuteCommand(rootCmd, cmdInstall, "--build-timeout", "10")

pkg/controller/build/build_controller.go

@@ -147,7 +147,8 @@ func (r *reconcileBuild) Reconcile(ctx context.Context, request reconcile.Reques
 		return reconcile.Result{}, err
 	}
 	buildMonitor := Monitor{
-		maxRunningBuilds: ip.Status.Build.MaxRunningBuilds,
+		maxRunningBuilds:   ip.Status.Build.MaxRunningBuilds,
+		buildOrderStrategy: ip.Status.Build.BuildConfiguration.OrderStrategy,
 	}
 
 	switch instance.BuilderConfiguration().Strategy {