feat(driver/kubernetes): support mount buildkit.toml and qemu installing

Signed-off-by: Morlay <[email protected]>

docs/reference/buildx_create.md

@@ -132,6 +132,8 @@ Passes additional driver-specific options. Details for each driver:
     - `nodeselector="label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
     - `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
     - `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"
+    - `qemu.install=(true|false)` - Install QEMU emulation for multi platforms support.
+    - `qemu.image=IMAGE` - Sets the QEMU emulation image. Defaults to `tonistiigi/binfmt:latest`
 
 ### <a name="leave"></a> Remove a node from a builder (--leave)
 

driver/bkimage/bkimage.go

@@ -2,5 +2,6 @@ package bkimage
 
 const (
 	DefaultImage         = "moby/buildkit:buildx-stable-1" // TODO: make this verified
+	QemuImage            = "tonistiigi/binfmt:latest"      // TODO: make this verified
 	DefaultRootlessImage = DefaultImage + "-rootless"
 )

driver/kubernetes/driver.go

@@ -18,6 +18,8 @@ import (
 	"github.com/moby/buildkit/util/tracing/detect"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	clientappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
@@ -39,15 +41,18 @@ type Driver struct {
 	factory          driver.Factory
 	minReplicas      int
 	deployment       *appsv1.Deployment
+	configMap        *corev1.ConfigMap
 	clientset        *kubernetes.Clientset
 	deploymentClient clientappsv1.DeploymentInterface
 	podClient        clientcorev1.PodInterface
+	configMapClient  clientcorev1.ConfigMapInterface
 	podChooser       podchooser.PodChooser
 }
 
 func (d *Driver) IsMobyDriver() bool {
 	return false
 }
+
 func (d *Driver) Config() driver.InitConfig {
 	return d.InitConfig
 }
@@ -56,7 +61,24 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
 	return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
 		_, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
 		if err != nil {
-			// TODO: return err if err != ErrNotFound
+			if !apierrors.IsNotFound(err) {
+				return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name)
+			}
+
+			if d.configMap != nil {
+				// create ConfigMap first if exists
+				_, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{})
+				if err != nil {
+					if !apierrors.IsAlreadyExists(err) {
+						return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name)
+					}
+					_, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{})
+					if err != nil {
+						return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name)
+					}
+				}
+			}
+
 			_, err = d.deploymentClient.Create(ctx, d.deployment, metav1.CreateOptions{})
 			if err != nil {
 				return errors.Wrapf(err, "error while calling deploymentClient.Create for %q", d.deployment.Name)
@@ -145,7 +167,16 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
 
 func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 	if err := d.deploymentClient.Delete(ctx, d.deployment.Name, metav1.DeleteOptions{}); err != nil {
-		return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
+		if !apierrors.IsNotFound(err) {
+			return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
+		}
+	}
+	if d.configMap != nil {
+		if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil {
+			if !apierrors.IsNotFound(err) {
+				return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name)
+			}
+		}
 	}
 	return nil
 }

driver/kubernetes/factory.go

@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"context"
+	"os"
 	"strconv"
 	"strings"
 
@@ -59,11 +60,13 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 	if err != nil {
 		return nil, err
 	}
+
 	d := &Driver{
 		factory:    f,
 		InitConfig: cfg,
 		clientset:  clientset,
 	}
+
 	deploymentOpt := &manifest.DeploymentOpt{
 		Name:          deploymentName,
 		Image:         bkimage.DefaultImage,
@@ -72,12 +75,25 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		Rootless:      false,
 		Platforms:     cfg.Platforms,
 	}
+
+	deploymentOpt.Qemu.Image = bkimage.QemuImage
+
+	if cfg.ConfigFile != "" {
+		buildkitConfig, err := os.ReadFile(cfg.ConfigFile)
+		if err != nil {
+			return nil, err
+		}
+		deploymentOpt.BuildkitConfig = buildkitConfig
+	}
+
 	loadbalance := LoadbalanceSticky
-	imageOverride := ""
+
 	for k, v := range cfg.DriverOpts {
 		switch k {
 		case "image":
-			imageOverride = v
+			if v != "" {
+				deploymentOpt.Image = v
+			}
 		case "namespace":
 			namespace = v
 		case "replicas":
@@ -117,20 +133,31 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 				return nil, errors.Errorf("invalid loadbalance %q", v)
 			}
 			loadbalance = v
+		case "qemu.install":
+			deploymentOpt.Qemu.Install, err = strconv.ParseBool(v)
+			if err != nil {
+				return nil, err
+			}
+		case "qemu.image":
+			if v != "" {
+				deploymentOpt.Qemu.Image = v
+			}
 		default:
 			return nil, errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
 		}
 	}
-	if imageOverride != "" {
-		deploymentOpt.Image = imageOverride
-	}
-	d.deployment, err = manifest.NewDeployment(deploymentOpt)
+
+	d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt)
 	if err != nil {
 		return nil, err
 	}
+
 	d.minReplicas = deploymentOpt.Replicas
+
 	d.deploymentClient = clientset.AppsV1().Deployments(namespace)
 	d.podClient = clientset.CoreV1().Pods(namespace)
+	d.configMapClient = clientset.CoreV1().ConfigMaps(namespace)
+
 	switch loadbalance {
 	case LoadbalanceSticky:
 		d.podChooser = &podchooser.StickyPodChooser{

driver/kubernetes/manifest/manifest.go

@@ -12,11 +12,23 @@ import (
 )
 
 type DeploymentOpt struct {
-	Namespace      string
-	Name           string
-	Image          string
-	Replicas       int
-	BuildkitFlags  []string
+	Namespace string
+	Name      string
+	Image     string
+	Replicas  int
+
+	// Qemu
+	Qemu struct {
+		// when true, will install binfmt
+		Install bool
+		Image   string
+	}
+
+	BuildkitFlags []string
+	// BuildkitConfig
+	// when not empty, will create configmap with buildkit.toml and mounted
+	BuildkitConfig []byte
+
 	Rootless       bool
 	NodeSelector   map[string]string
 	RequestsCPU    string
@@ -31,7 +43,7 @@ const (
 	AnnotationPlatform = "buildx.docker.com/platform"
 )
 
-func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
+func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) {
 	labels := map[string]string{
 		"app": opt.Name,
 	}
@@ -44,7 +56,7 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
 		annotations[AnnotationPlatform] = strings.Join(platformutil.Format(opt.Platforms), ",")
 	}
 
-	d := &appsv1.Deployment{
+	d = &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: appsv1.SchemeGroupVersion.String(),
 			Kind:       "Deployment",
@@ -91,9 +103,56 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
 			},
 		},
 	}
+
+	if len(opt.BuildkitConfig) > 0 {
+		c = &corev1.ConfigMap{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: corev1.SchemeGroupVersion.String(),
+				Kind:       "ConfigMap",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace:   opt.Namespace,
+				Name:        opt.Name + "-config",
+				Annotations: annotations,
+			},
+			Data: map[string]string{
+				"buildkitd.toml": string(opt.BuildkitConfig),
+			},
+		}
+
+		d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{
+			Name:      "config",
+			MountPath: "/etc/buildkit",
+		}}
+
+		d.Spec.Template.Spec.Volumes = []corev1.Volume{{
+			Name: "config",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: c.Name,
+					},
+				},
+			},
+		}}
+	}
+
+	if opt.Qemu.Install {
+		d.Spec.Template.Spec.InitContainers = []corev1.Container{
+			{
+				Name:  "qemu",
+				Image: opt.Qemu.Image,
+				Args:  []string{"--install", "all"},
+				SecurityContext: &corev1.SecurityContext{
+					Privileged: &privileged,
+				},
+			},
+		}
+	}
+
 	if opt.Rootless {
 		if err := toRootless(d); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 
@@ -104,36 +163,36 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
 	if opt.RequestsCPU != "" {
 		reqCPU, err := resource.ParseQuantity(opt.RequestsCPU)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = reqCPU
 	}
 
 	if opt.RequestsMemory != "" {
 		reqMemory, err := resource.ParseQuantity(opt.RequestsMemory)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = reqMemory
 	}
 
 	if opt.LimitsCPU != "" {
 		limCPU, err := resource.ParseQuantity(opt.LimitsCPU)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU] = limCPU
 	}
 
 	if opt.LimitsMemory != "" {
 		limMemory, err := resource.ParseQuantity(opt.LimitsMemory)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = limMemory
 	}
 
-	return d, nil
+	return
 }
 
 func toRootless(d *appsv1.Deployment) error {

go.mod

@@ -1,18 +1,16 @@
 module github.com/docker/buildx
 
-go 1.13
+go 1.16
 
 require (
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
-	github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect
 	github.com/bugsnag/bugsnag-go v1.4.1 // indirect
 	github.com/bugsnag/panicwrap v1.2.0 // indirect
 	github.com/cenkalti/backoff v2.1.1+incompatible // indirect
 	github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e // indirect
 	github.com/compose-spec/compose-go v0.0.0-20210706130854-69459d4976b5
 	github.com/containerd/console v1.0.2
 	github.com/containerd/containerd v1.5.2
-	github.com/denisenkom/go-mssqldb v0.0.0-20190315220205-a8ed825ac853 // indirect
 	github.com/docker/cli v20.10.7+incompatible
 	github.com/docker/compose-on-kubernetes v0.4.19-0.20190128150448-356b2919c496 // indirect
 	github.com/docker/distribution v2.7.1+incompatible
@@ -22,7 +20,6 @@ require (
 	github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4 // indirect
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
 	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
-	github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
 	github.com/fvbommel/sortorder v1.0.1 // indirect
 	github.com/gofrs/flock v0.7.3
 	github.com/gofrs/uuid v3.3.0+incompatible // indirect
@@ -33,10 +30,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.8.2
 	github.com/jinzhu/gorm v1.9.2 // indirect
 	github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
-	github.com/jinzhu/now v1.0.0 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/lib/pq v1.10.0 // indirect
-	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/moby/buildkit v0.8.2-0.20210702160134-1a7543a10527
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1