feat(install): use a PVC to share artifacts among concurrent builds

Closes apache#3831

build/Dockerfile

@@ -40,10 +40,11 @@ RUN ${MVNW_DIR}/mvnw --version | grep "Maven home:" | sed 's/Maven home: //' >>
     && rm $(cat ${MVNW_DIR}default)/lib/maven-slf4j-provider*
 ENV MAVEN_OPTS="${MAVEN_OPTS} -Dlogback.configurationFile=${MAVEN_HOME}/conf/logback.xml"
 
-ADD build/_maven_output /tmp/artifacts/m2
+ADD build/_maven_output /tmp/local/m2
 ADD build/_kamelets /kamelets
 
-RUN chgrp -R 0 /tmp/artifacts/m2 \
+RUN mkdir -p /tmp/artifacts/m2 \
+    && chgrp -R 0 /tmp/artifacts/m2 \
     && chmod -R g=u /tmp/artifacts/m2 \
     && chgrp -R 0 /kamelets \
     && chmod -R g=u /kamelets \

config/crd/bases/camel.apache.org_camelcatalogs.yaml

@@ -404,7 +404,7 @@ spec:
             description: the actual state of the catalog
             properties:
               conditions:
-                description: Conditions --
+                description: a list of events happened for the CamelCatalog
                 items:
                   description: CamelCatalogCondition describes the state of a resource
                     at a certain point.
@@ -437,15 +437,16 @@ spec:
                   type: object
                 type: array
               image:
-                description: Image --
+                description: the container image available for building an application
+                  with this catalog
                 type: string
               observedGeneration:
                 description: ObservedGeneration is the most recent generation observed
                   for this Catalog.
                 format: int64
                 type: integer
               phase:
-                description: Phase --
+                description: the actual phase
                 type: string
             type: object
         type: object

config/manager/kustomization.yaml

@@ -20,6 +20,7 @@ kind: Kustomization
 
 resources:
 - operator-deployment.yaml
+- operator-storage.yaml
 - operator-service-account.yaml
 
 patchesStrategicMerge:

config/manager/operator-deployment.yaml

@@ -44,6 +44,10 @@ spec:
         app.kubernetes.io/version: "2.0.0-SNAPSHOT"
     spec:
       serviceAccountName: camel-k-operator
+      volumes:
+        - name: camel-k-maven-repo
+          persistentVolumeClaim:
+            claimName: camel-k-maven-repo
       containers:
         - name: camel-k-operator
           image: docker.io/apache/camel-k:2.0.0-SNAPSHOT
@@ -78,3 +82,6 @@ spec:
               port: 8081
             initialDelaySeconds: 20
             periodSeconds: 10
+          volumeMounts:
+            - mountPath: "/tmp/artifacts/m2"
+              name: camel-k-maven-repo

config/manager/operator-storage.yaml

@@ -0,0 +1,27 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: camel-k-maven-repo
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 20Gi

docs/modules/ROOT/partials/apis/camel-k-crds.adoc

@@ -947,21 +947,21 @@ ObservedGeneration is the most recent generation observed for this Catalog.
 |
 
 
-Phase --
+the actual phase
 
 |`conditions` +
 *xref:#_camel_apache_org_v1_CamelCatalogCondition[[\]CamelCatalogCondition]*
 |
 
 
-Conditions --
+a list of events happened for the CamelCatalog
 
 |`image` +
 string
 |
 
 
-Image --
+the container image available for building an application with this catalog
 
 
 |===

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/onsi/gomega v1.27.1
 	github.com/openshift/api v3.9.1-0.20190927182313-d4a64ec2cbd8+incompatible
 	github.com/operator-framework/api v0.13.0
+	github.com/otiai10/copy v1.9.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.60.0
 	github.com/prometheus/client_golang v1.14.0

go.sum

@@ -986,10 +986,14 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
 github.com/operator-framework/api v0.13.0 h1:V1vUluRwajSBdDPCnzgTWDnn5LYxLk66VPVGMw3B7Uc=
 github.com/operator-framework/api v0.13.0/go.mod h1:FTiYGm11fZQ3cSX+EQHc/UWoGZAwkGfyeHU+wMJ8jmA=
 github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
+github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4=
+github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4=
+github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

helm/camel-k/crds/crd-camel-catalog.yaml

@@ -404,7 +404,7 @@ spec:
             description: the actual state of the catalog
             properties:
               conditions:
-                description: Conditions --
+                description: a list of events happened for the CamelCatalog
                 items:
                   description: CamelCatalogCondition describes the state of a resource
                     at a certain point.
@@ -437,15 +437,16 @@ spec:
                   type: object
                 type: array
               image:
-                description: Image --
+                description: the container image available for building an application
+                  with this catalog
                 type: string
               observedGeneration:
                 description: ObservedGeneration is the most recent generation observed
                   for this Catalog.
                 format: int64
                 type: integer
               phase:
-                description: Phase --
+                description: the actual phase
                 type: string
             type: object
         type: object

pkg/cmd/dump.go

@@ -89,6 +89,21 @@ func dumpNamespace(ctx context.Context, c client.Client, ns string, out io.Write
 	if err != nil {
 		return err
 	}
+
+	its, err := camelClient.CamelV1().Integrations(ns).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(out, "Found %d integrations:\n", len(its.Items))
+	for _, integration := range its.Items {
+		ref := integration
+		pdata, err := kubernetes.ToYAML(&ref)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(out, "---\n%s\n---\n", string(pdata))
+	}
+
 	pls, err := camelClient.CamelV1().IntegrationPlatforms(ns).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return err
@@ -103,18 +118,18 @@ func dumpNamespace(ctx context.Context, c client.Client, ns string, out io.Write
 		fmt.Fprintf(out, "---\n%s\n---\n", string(pdata))
 	}
 
-	its, err := camelClient.CamelV1().Integrations(ns).List(ctx, metav1.ListOptions{})
+	cat, err := camelClient.CamelV1().CamelCatalogs(ns).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return err
 	}
-	fmt.Fprintf(out, "Found %d integrations:\n", len(its.Items))
-	for _, integration := range its.Items {
-		ref := integration
-		pdata, err := kubernetes.ToYAML(&ref)
+	fmt.Fprintf(out, "Found %d catalogs:\n", len(pls.Items))
+	for _, c := range cat.Items {
+		ref := c
+		cdata, err := kubernetes.ToYAML(&ref)
 		if err != nil {
 			return err
 		}
-		fmt.Fprintf(out, "---\n%s\n---\n", string(pdata))
+		fmt.Fprintf(out, "---\n%s\n---\n", string(cdata))
 	}
 
 	iks, err := camelClient.CamelV1().IntegrationKits(ns).List(ctx, metav1.ListOptions{})

pkg/controller/build/build_pod.go

@@ -18,10 +18,8 @@ limitations under the License.
 package build
 
 import (
-	"bufio"
 	"context"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -39,7 +37,6 @@ import (
 	"github.com/apache/camel-k/pkg/platform"
 	"github.com/apache/camel-k/pkg/util/defaults"
 	"github.com/apache/camel-k/pkg/util/kubernetes"
-	"github.com/apache/camel-k/pkg/util/log"
 )
 
 const (
@@ -204,14 +201,29 @@ func buildPodName(build *v1.Build) string {
 }
 
 func addBuildTaskToPod(build *v1.Build, taskName string, pod *corev1.Pod) {
-	if !hasBuilderVolume(pod) {
-		// Add the EmptyDir volume used to share the build state across tasks
-		pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
-			Name: builderVolume,
-			VolumeSource: corev1.VolumeSource{
-				EmptyDir: &corev1.EmptyDirVolumeSource{},
+	if !hasVolume(pod, builderVolume) {
+		pod.Spec.Volumes = append(pod.Spec.Volumes,
+			// EmptyDir volume used to share the build state across tasks
+			corev1.Volume{
+				Name: builderVolume,
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
 			},
-		})
+		)
+	}
+	if !hasVolume(pod, "camel-k-maven-repo") {
+		pod.Spec.Volumes = append(pod.Spec.Volumes,
+			// Maven repo volume
+			corev1.Volume{
+				Name: "camel-k-maven-repo",
+				VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: "camel-k-maven-repo",
+					},
+				},
+			},
+		)
 	}
 
 	container := corev1.Container{
@@ -235,15 +247,6 @@ func addBuildTaskToPod(build *v1.Build, taskName string, pod *corev1.Pod) {
 	addContainerToPod(build, container, pod)
 }
 
-func readSpectrumLogs(newStdOut io.Reader) {
-	scanner := bufio.NewScanner(newStdOut)
-
-	for scanner.Scan() {
-		line := scanner.Text()
-		log.Infof(line)
-	}
-}
-
 func addBuildahTaskToPod(ctx context.Context, c ctrl.Reader, build *v1.Build, task *v1.BuildahTask, pod *corev1.Pod) error {
 	var bud []string
 
@@ -473,19 +476,25 @@ func addKanikoTaskToPod(ctx context.Context, c ctrl.Reader, build *v1.Build, tas
 }
 
 func addContainerToPod(build *v1.Build, container corev1.Container, pod *corev1.Pod) {
-	if hasBuilderVolume(pod) {
+	if hasVolume(pod, builderVolume) {
 		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
 			Name:      builderVolume,
 			MountPath: filepath.Join(builderDir, build.Name),
 		})
 	}
+	if hasVolume(pod, "camel-k-maven-repo") {
+		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+			Name:      "camel-k-maven-repo",
+			MountPath: "/tmp/artifacts/m2",
+		})
+	}
 
 	pod.Spec.InitContainers = append(pod.Spec.InitContainers, container)
 }
 
-func hasBuilderVolume(pod *corev1.Pod) bool {
+func hasVolume(pod *corev1.Pod, name string) bool {
 	for _, volume := range pod.Spec.Volumes {
-		if volume.Name == builderVolume {
+		if volume.Name == name {
 			return true
 		}
 	}

pkg/install/common.go

@@ -110,6 +110,7 @@ func ObjectOrCollect(ctx context.Context, c client.Client, namespace string, col
 		if err := c.Create(ctx, obj); err != nil && !errors.IsAlreadyExists(err) {
 			return err
 		}
+		return nil
 	}
 
 	if force {

pkg/install/operator.go

@@ -456,6 +456,7 @@ func installKubernetesRoles(ctx context.Context, c client.Client, namespace stri
 
 func installOperator(ctx context.Context, c client.Client, namespace string, customizer ResourceCustomizer, collection *kubernetes.Collection, force bool) error {
 	return ResourcesOrCollect(ctx, c, namespace, collection, force, customizer,
+		"/manager/operator-storage.yaml",
 		"/manager/operator-deployment.yaml",
 	)
 }

pkg/install/optional.go

@@ -24,10 +24,16 @@ import (
 	"github.com/apache/camel-k/pkg/client"
 	"github.com/apache/camel-k/pkg/util/defaults"
 	logutil "github.com/apache/camel-k/pkg/util/log"
+	cp "github.com/otiai10/copy"
 )
 
 // OperatorStartupOptionalTools tries to install optional tools at operator startup and warns if something goes wrong.
 func OperatorStartupOptionalTools(ctx context.Context, c client.Client, namespace string, operatorNamespace string, log logutil.Logger) {
+	// Try to copy any local runtime dependency to maven repository
+	if err := cp.Copy("/tmp/local/m2", "/tmp/artifacts/m2"); err != nil {
+		log.Info("Could not copy local runtime dependencies due to", err.Error())
+	}
+
 	// Try to register the OpenShift CLI Download link if possible
 	if err := OpenShiftConsoleDownloadLink(ctx, c); err != nil {
 		log.Info("Cannot install OpenShift CLI download link: skipping.")

pkg/platform/defaults.go

@@ -69,24 +69,17 @@ func ConfigureDefaults(ctx context.Context, c client.Client, p *v1.IntegrationPl
 	}
 
 	if p.Status.Build.PublishStrategy == "" {
-		log.Debugf("Integration Platform [%s]: setting publishing strategy", p.Namespace)
 		if p.Status.Cluster == v1.IntegrationPlatformClusterOpenShift {
 			p.Status.Build.PublishStrategy = v1.IntegrationPlatformBuildPublishStrategyS2I
 		} else {
 			p.Status.Build.PublishStrategy = v1.IntegrationPlatformBuildPublishStrategySpectrum
 		}
+		log.Debugf("Integration Platform [%s]: setting publishing strategy %s", p.Namespace, p.Status.Build.PublishStrategy)
 	}
 
 	if p.Status.Build.BuildStrategy == "" {
-		log.Debugf("Integration Platform [%s]: setting build strategy", p.Namespace)
-		// Use the fastest strategy that they support (routine when possible)
-		if p.Status.Build.PublishStrategy == v1.IntegrationPlatformBuildPublishStrategyS2I ||
-			p.Status.Build.PublishStrategy == v1.IntegrationPlatformBuildPublishStrategySpectrum {
-			p.Status.Build.BuildStrategy = v1.BuildStrategyRoutine
-		} else {
-			// The build output has to be shared via a volume
-			p.Status.Build.BuildStrategy = v1.BuildStrategyPod
-		}
+		p.Status.Build.BuildStrategy = v1.BuildStrategyPod
+		log.Debugf("Integration Platform [%s]: setting build strategy %s", p.Namespace, p.Status.Build.BuildStrategy)
 	}
 
 	err := setPlatformDefaults(p, verbose)

pkg/trait/builder.go

@@ -151,11 +151,12 @@ func (t *builderTrait) Apply(e *Environment) error {
 
 	if t.Strategy != "" {
 		t.L.Infof("User defined build strategy %s", t.Strategy)
-		if t.Strategy == string(v1.BuildStrategyPod) {
+		switch t.Strategy {
+		case string(v1.BuildStrategyPod):
 			e.BuildStrategy = v1.BuildStrategyPod
-		} else if t.Strategy == string(v1.BuildStrategyRoutine) {
+		case string(v1.BuildStrategyRoutine):
 			e.BuildStrategy = v1.BuildStrategyRoutine
-		} else {
+		default:
 			return fmt.Errorf("Must specify either pod or routine build strategy, unknown %s", t.Strategy)
 		}
 	}