Merge pull request #12081 from spowelljr/buildkitSocket

buildkit socket: Add systemd unit

Makefile

@@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v1.23.1
+ISO_VERSION ?= v1.23.1-1633115168-12081
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 DEB_REVISION ?= 0
@@ -286,7 +286,7 @@ minikube_iso: deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/b
 	if [ ! -d $(BUILD_DIR)/buildroot ]; then \
 		mkdir -p $(BUILD_DIR); \
 		git clone --depth=1 --branch=$(BUILDROOT_BRANCH) https://github.com/buildroot/buildroot $(BUILD_DIR)/buildroot; \
-		cp $(PWD)/deploy/iso/minikube-iso/go.hash $(BUILD_DIR)/buildroot/package/go/go.hash; \
+		cp deploy/iso/minikube-iso/go.hash $(BUILD_DIR)/buildroot/package/go/go.hash; \
 	fi;
 	$(MAKE) BR2_EXTERNAL=../../deploy/iso/minikube-iso minikube_defconfig -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS)
 	$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) host-python

deploy/iso/minikube-iso/board/coreos/minikube/users

@@ -1 +1 @@
-docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf,podman -
+docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf,podman,buildkit -

deploy/iso/minikube-iso/package/buildkit-bin/51-buildkit.preset

@@ -0,0 +1 @@
+disable buildkit.service

deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk

@@ -12,6 +12,10 @@ BUILDKIT_BIN_SOURCE = buildkit-$(BUILDKIT_BIN_VERSION).linux-amd64.tar.gz
 # https://github.com/opencontainers/runc.git
 BUILDKIT_RUNC_VERSION = 12644e614e25b05da6fd08a38ffa0cfe1903fdec
 
+define BUILDKIT_BIN_USERS
+	- -1 buildkit -1 - - - - -
+endef
+
 define BUILDKIT_BIN_INSTALL_TARGET_CMDS
 	$(INSTALL) -D -m 0755 \
 		$(@D)/buildctl \
@@ -25,6 +29,24 @@ define BUILDKIT_BIN_INSTALL_TARGET_CMDS
 	$(INSTALL) -D -m 0755 \
 		$(@D)/buildkitd \
 		$(TARGET_DIR)/usr/sbin
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_PKGDIR)/buildkit.conf \
+		$(TARGET_DIR)/usr/lib/tmpfiles.d/buildkit.conf
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_PKGDIR)/buildkitd.toml \
+		$(TARGET_DIR)/etc/buildkit/buildkitd.toml
+endef
+
+define BUILDKIT_BIN_INSTALL_INIT_SYSTEMD
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_PKGDIR)/buildkit.service \
+		$(TARGET_DIR)/usr/lib/systemd/system/buildkit.service
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_PKGDIR)/buildkit.socket \
+		$(TARGET_DIR)/usr/lib/systemd/system/buildkit.socket
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_PKGDIR)/51-buildkit.preset \
+		$(TARGET_DIR)/usr/lib/systemd/system-preset/51-buildkit.preset
 endef
 
 $(eval $(generic-package))

deploy/iso/minikube-iso/package/buildkit-bin/buildkit.conf

@@ -0,0 +1 @@
+d /run/buildkit 0770 root buildkit

deploy/iso/minikube-iso/package/buildkit-bin/buildkit.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=BuildKit
+Requires=buildkit.socket
+After=buildkit.socket
+Documentation=https://github.com/moby/buildkit
+
+[Service]
+ExecStart=/usr/sbin/buildkitd --addr fd://
+
+[Install]
+WantedBy=multi-user.target

deploy/iso/minikube-iso/package/buildkit-bin/buildkit.socket

@@ -0,0 +1,12 @@
+[Unit]
+Description=BuildKit
+Documentation=https://github.com/moby/buildkit
+
+[Socket]
+ListenStream=%t/buildkit/buildkitd.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=buildkit
+
+[Install]
+WantedBy=sockets.target

deploy/iso/minikube-iso/package/buildkit-bin/buildkitd.toml

@@ -0,0 +1,5 @@
+[worker.oci]
+  enabled = false
+[worker.containerd]
+  enabled = true
+  namespace = "k8s.io"

deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash

@@ -21,5 +21,5 @@ sha256 74a4e916acddc6cf47ab5752bdebb6732ce2c028505ef57b7edc21d2da9039b6 v1.18.4.
 sha256 fc8a8e61375e3ce30563eeb0fd6534c4f48fc20300a72e6ff51cc99cb2703516 v1.19.0.tar.gz
 sha256 6165c5b8212ea03be2a465403177318bfe25a54c3e8d66d720344643913a0223 v1.19.1.tar.gz
 sha256 76fd7543bc92d4364a11060f43a5131893a76c6e6e9d6de3a6bb6292c110b631 v1.20.0.tar.gz
-sha256 1c01d4a76cdcfe3ac24147eb1d5f6ebd782bd98fb0ac0c19b79bd5a6560b1481 v1.20.2.tar.gz
+sha256 36d9f4cf4966342e2d4099e44d8156c55c6a10745c67ce4f856aa9f6dcc2d9ba v1.20.2.tar.gz
 sha256 bc53ea8977e252bd9812974c33ff654ee22076598e901464468c5c105a5ef773 v1.22.0.tar.gz

deploy/kicbase/Dockerfile

@@ -142,14 +142,21 @@ COPY deploy/kicbase/containerd-fuse-overlayfs.service /etc/systemd/system/contai
 # install buildkit
 RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \
  && echo "Installing buildkit ..." \
+    && addgroup --system buildkit \
     && export BUILDKIT_BASE_URL="https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}" \
     && curl -sSL --retry 5 --output /tmp/buildkit.tgz "${BUILDKIT_BASE_URL}/buildkit-${BUILDKIT_VERSION}.linux-${ARCH}.tar.gz" \
     && tar -C /usr/local -xzvf /tmp/buildkit.tgz \
     && rm -rf /tmp/buildkit.tgz \
+    && mkdir -p /usr/local/lib/systemd/system \
+    && curl -L --retry 5 --output /usr/local/lib/systemd/system/buildkit.service "https://raw.githubusercontent.com/moby/buildkit/${BUILDKIT_VERSION}/examples/systemd/buildkit.service" \
+    && curl -L --retry 5 --output /usr/local/lib/systemd/system/buildkit.socket "https://raw.githubusercontent.com/moby/buildkit/${BUILDKIT_VERSION}/examples/systemd/buildkit.socket" \
+    && mkdir -p /etc/buildkit \
+    && echo "[worker.oci]\n  enabled = false\n[worker.containerd]\n  enabled = true\n  namespace = \"k8s.io\"" > /etc/buildkit/buildkitd.toml \
     && chmod 755 /usr/local/bin/buildctl \
     && chmod 755 /usr/local/bin/buildkit-runc \
     && chmod 755 /usr/local/bin/buildkit-qemu-* \
-    && chmod 755 /usr/local/bin/buildkitd
+    && chmod 755 /usr/local/bin/buildkitd \
+    && systemctl enable buildkit.socket
 
 # Install cri-o/podman dependencies:
 RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
@@ -210,6 +217,7 @@ EXPOSE 22
 RUN adduser --ingroup docker --disabled-password --gecos '' docker 
 RUN adduser docker sudo
 RUN adduser docker podman
+RUN adduser docker buildkit
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER docker
 RUN mkdir /home/docker/.ssh

pkg/drivers/kic/types.go

@@ -24,13 +24,13 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.27"
+	Version = "v0.0.27-1633027942-12081"
 	// SHA of the kic base image
-	baseImageSHA = "89b4738ee74ba28684676e176752277f0db46f57d27f0e08c3feec89311e22de"
+	baseImageSHA = "4780f1897569d2bf77aafb3d133a08d42b4fe61127f06fcfc90c2c5d902d893c"
 	// The name of the GCR kicbase repository
-	gcrRepo = "gcr.io/k8s-minikube/kicbase"
+	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository
-	dockerhubRepo = "docker.io/kicbase/stable"
+	dockerhubRepo = "docker.io/kicbase/build"
 )
 
 var (

pkg/minikube/cruntime/containerd.go

@@ -393,10 +393,6 @@ func downloadRemote(cr CommandRunner, src string) (string, error) {
 
 // BuildImage builds an image into this runtime
 func (r *Containerd) BuildImage(src string, file string, tag string, push bool, env []string, opts []string) error {
-	if err := r.initBuildkitDaemon(); err != nil {
-		return fmt.Errorf("failed to init buildkit daemon: %v", err)
-	}
-
 	// download url if not already present
 	dir, err := downloadRemote(r.Runner, src)
 	if err != nil {
@@ -456,24 +452,6 @@ func (r *Containerd) PushImage(name string) error {
 	}
 	return nil
 }
-func (r *Containerd) initBuildkitDaemon() error {
-	// if daemon is already running, do nothing
-	cmd := exec.Command("pgrep", "buildkitd")
-	if _, err := r.Runner.RunCmd(cmd); err == nil {
-		return nil
-	}
-
-	// otherwise, start daemon
-	cmd = exec.Command("/bin/bash", "-c", "sudo -b buildkitd --oci-worker false --containerd-worker true --containerd-worker-namespace k8s.io &> /dev/null")
-	if _, err := r.Runner.RunCmd(cmd); err != nil {
-		return fmt.Errorf("failed to start buildkit daemon: %v", err)
-	}
-
-	// give the daemon time to finish starting up or image build will fail
-	time.Sleep(1 * time.Second)
-
-	return nil
-}
 
 // CGroupDriver returns cgroup driver ("cgroupfs" or "systemd")
 func (r *Containerd) CGroupDriver() (string, error) {

pkg/minikube/download/iso.go

@@ -40,7 +40,7 @@ const fileScheme = "file"
 // DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order
 func DefaultISOURLs() []string {
 	v := version.GetISOVersion()
-	isoBucket := "minikube/iso"
+	isoBucket := "minikube-builds/iso/12081"
 	return []string{
 		fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", isoBucket, v),
 		fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/%s/minikube-%s.iso", v, v),

site/content/en/docs/commands/start.md

@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.27@sha256:89b4738ee74ba28684676e176752277f0db46f57d27f0e08c3feec89311e22de")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.27-1633027942-12081@sha256:4780f1897569d2bf77aafb3d133a08d42b4fe61127f06fcfc90c2c5d902d893c")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cert-expiration duration          Duration until minikube certificate expiration, defaults to three years (26280h). (default 26280h0m0s)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
@@ -66,7 +66,7 @@ minikube start [flags]
       --insecure-registry strings         Insecure Docker registries to pass to the Docker daemon.  The default service CIDR range will automatically be added.
       --install-addons                    If set, install addons. Defaults to true. (default true)
       --interactive                       Allow user prompts for more information (default true)
-      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.23.1.iso,https://github.com/kubernetes/minikube/releases/download/v1.23.1/minikube-v1.23.1.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.23.1.iso])
+      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/12081/minikube-v1.23.1-1633115168-12081.iso,https://github.com/kubernetes/minikube/releases/download/v1.23.1-1633115168-12081/minikube-v1.23.1-1633115168-12081.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.23.1-1633115168-12081.iso])
       --keep-context                      This will keep the existing kubectl context and will create a minikube context.
       --kubernetes-version string         The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.22.2, 'latest' for v1.22.3-rc.0). Defaults to 'stable'.
       --kvm-gpu                           Enable experimental NVIDIA GPU support in minikube

test/integration/functional_test.go

@@ -255,6 +255,10 @@ func validateImageCommands(ctx context.Context, t *testing.T, profile string) {
 	t.Run("ImageBuild", func(t *testing.T) {
 		MaybeParallel(t)
 
+		if _, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "pgrep", "buildkitd")); err == nil {
+			t.Errorf("buildkitd process is running, should not be running until `minikube image build` is ran")
+		}
+
 		newImage := fmt.Sprintf("localhost/my-image:%s", profile)
 
 		// try to build the new image with minikube