implement kind build node-image --arch

- switch to pulling images with containerd in the running base, instead of to the host. this breaks offline kubernetes development but the previous approach with docker pull / save / export cannot be used reliably for multi-arch
- fully qualify images to prepull (needed by ctr)
- fix image architecture in upstream built images, some of which are have the wrong metadata
- move architecture support to warning

pkg/build/nodeimage/build.go

@@ -31,8 +31,7 @@ func Build(options ...Option) error {
 		image:     DefaultImage,
 		baseImage: DefaultBaseImage,
 		logger:    log.NoopLogger{},
-		// TODO: only host arch supported. changing this will be tricky
-		arch: runtime.GOARCH,
+		arch:      runtime.GOARCH,
 	}
 
 	// apply user options
@@ -44,7 +43,7 @@ func Build(options ...Option) error {
 
 	// verify that we're using a supported arch
 	if !supportedArch(ctx.arch) {
-		return errors.Errorf("unsupported architecture %q", ctx.arch)
+		ctx.logger.Warnf("unsupported architecture %q", ctx.arch)
 	}
 
 	// locate sources if no kubernetes source was specified
@@ -78,16 +77,3 @@ func supportedArch(arch string) bool {
 	}
 	return true
 }
-
-// buildContext is used to build the kind node image, and contains
-// build configuration
-type buildContext struct {
-	// option fields
-	image     string
-	baseImage string
-	logger    log.Logger
-	// non-option fields
-	arch     string // TODO(bentheelder): this should be an option
-	kubeRoot string
-	builder  kube.Builder
-}

pkg/build/nodeimage/build_impl.go → pkg/build/nodeimage/buildcontext.go

@@ -33,8 +33,22 @@ import (
 	"sigs.k8s.io/kind/pkg/errors"
 	"sigs.k8s.io/kind/pkg/exec"
 	"sigs.k8s.io/kind/pkg/fs"
+	"sigs.k8s.io/kind/pkg/log"
 )
 
+// buildContext is used to build the kind node image, and contains
+// build configuration
+type buildContext struct {
+	// option fields
+	image     string
+	baseImage string
+	logger    log.Logger
+	arch      string
+	kubeRoot  string
+	// non-option fields
+	builder kube.Builder
+}
+
 // Build builds the cluster node image, the sourcedir must be set on
 // the buildContext
 func (c *buildContext) Build() (err error) {
@@ -188,7 +202,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 	fixRepository := func(repository string) string {
 		if strings.HasSuffix(repository, archSuffix) {
 			fixed := strings.TrimSuffix(repository, archSuffix)
-			fmt.Println("fixed: " + repository + " -> " + fixed)
+			c.logger.V(1).Info("fixed: " + repository + " -> " + fixed)
 			repository = fixed
 		}
 		return repository
@@ -206,7 +220,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		fixedImages.Insert(registry + ":" + tag)
 	}
 	builtImages = fixedImages
-	c.logger.V(0).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
+	c.logger.V(1).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
 
 	// gets the list of images required by kubeadm
 	requiredImages, err := exec.OutputLines(cmder.Command(
@@ -262,38 +276,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		return nil, errors.Wrap(err, "failed to make images dir")
 	}
 
-	fns := []func() error{}
-	pulledImages := make(chan string, len(requiredImages))
-	for i, image := range requiredImages {
-		i, image := i, image // https://golang.org/doc/faq#closures_and_goroutines
-		fns = append(fns, func() error {
-			if !builtImages.Has(image) {
-				fmt.Printf("Pulling: %s\n", image)
-				err := docker.Pull(c.logger, image, 2)
-				if err != nil {
-					c.logger.Warnf("Failed to pull %s with error: %v", image, err)
-				}
-				// TODO(bentheelder): generate a friendlier name
-				pullName := fmt.Sprintf("%d.tar", i)
-				pullTo := path.Join(imagesDir, pullName)
-				err = docker.Save(image, pullTo)
-				if err != nil {
-					return err
-				}
-				pulledImages <- pullTo
-			}
-			return nil
-		})
-	}
-	if err := errors.AggregateConcurrent(fns); err != nil {
-		return nil, err
-	}
-	close(pulledImages)
-	pulled := []string{}
-	for image := range pulledImages {
-		pulled = append(pulled, image)
-	}
-
 	// setup image importer
 	importer := newContainerdImporter(cmder)
 	if err := importer.Prepare(); err != nil {
@@ -308,19 +290,28 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		}
 	}()
 
-	// create a plan of image loading
-	loadFns := []func() error{}
-	for _, image := range pulled {
-		image := image // capture loop var
-		loadFns = append(loadFns, func() error {
-			f, err := os.Open(image)
-			if err != nil {
-				return err
+	fns := []func() error{}
+	for _, image := range requiredImages {
+		image := image // https://golang.org/doc/faq#closures_and_goroutines
+		fns = append(fns, func() error {
+			if !builtImages.Has(image) {
+				err := importer.Pull(image, dockerBuildOsAndArch(c.arch))
+				if err != nil {
+					c.logger.Warnf("Failed to pull %s with error: %v", image, err)
+					runE := exec.RunErrorForError(err)
+					c.logger.Warn(string(runE.Output))
+				}
+				// TODO(bentheelder): generate a friendlier name
 			}
-			defer f.Close()
-			return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout).SetStdin(f).Run()
+			return nil
 		})
 	}
+	if err := errors.AggregateConcurrent(fns); err != nil {
+		return nil, err
+	}
+
+	// create a plan of image loading
+	loadFns := []func() error{}
 	for _, image := range bits.ImagePaths() {
 		image := image // capture loop var
 		loadFns = append(loadFns, func() error {
@@ -332,7 +323,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 			//return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stderr).SetStdin(f).Run()
 			// we will rewrite / correct the tags as we load the image
 			if err := exec.RunWithStdinWriter(importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout), func(w io.Writer) error {
-				return docker.EditArchiveRepositories(f, w, fixRepository)
+				return docker.EditArchive(f, w, fixRepository, c.arch)
 			}); err != nil {
 				return err
 			}
@@ -352,7 +343,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 func (c *buildContext) createBuildContainer() (id string, err error) {
 	// attempt to explicitly pull the image if it doesn't exist locally
 	// we don't care if this errors, we'll still try to run which also pulls
-	_, _ = docker.PullIfNotPresent(c.logger, c.baseImage, 4)
+	_ = docker.Pull(c.logger, c.baseImage, dockerBuildOsAndArch(c.arch), 4)
 	// this should be good enough: a specific prefix, the current unix time,
 	// and a little random bits in case we have multiple builds simultaneously
 	random := rand.New(rand.NewSource(time.Now().UnixNano())).Int31()
@@ -364,6 +355,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
 			// the container should hang forever so we can exec in it
 			"--entrypoint=sleep",
 			"--name=" + id,
+			"--platform=" + dockerBuildOsAndArch(c.arch),
 		},
 		[]string{
 			"infinity", // sleep infinitely to keep the container around
@@ -374,3 +366,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
 	}
 	return id, nil
 }
+
+func dockerBuildOsAndArch(arch string) string {
+	return "linux/" + arch
+}

pkg/build/nodeimage/const_cni.go

@@ -20,7 +20,7 @@ package nodeimage
 The default CNI manifest and images are our own tiny kindnet
 */
 
-var defaultCNIImages = []string{"kindest/kindnetd:v20210326-1e038dc5"}
+var defaultCNIImages = []string{"docker.io/kindest/kindnetd:v20210326-1e038dc5"}
 
 // TODO: migrate to fully patching and deprecate the template
 const defaultCNIManifest = `
@@ -95,7 +95,7 @@ spec:
       serviceAccountName: kindnet
       containers:
       - name: kindnet-cni
-        image: kindest/kindnetd:v20210326-1e038dc5
+        image: docker.io/kindest/kindnetd:v20210326-1e038dc5
         env:
         - name: HOST_IP
           valueFrom:

pkg/build/nodeimage/const_storage.go

@@ -25,7 +25,7 @@ NOTE: we have customized it in the following ways:
 - install as the default storage class
 */
 
-var defaultStorageImages = []string{"rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}
+var defaultStorageImages = []string{"docker.io/rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}
 
 const defaultStorageManifest = `
 # kind customized https://github.com/rancher/local-path-provisioner manifest
@@ -95,7 +95,7 @@ spec:
       serviceAccountName: local-path-provisioner-service-account
       containers:
       - name: local-path-provisioner
-        image: rancher/local-path-provisioner:v0.0.14
+        image: docker.io/rancher/local-path-provisioner:v0.0.14
         imagePullPolicy: IfNotPresent
         command:
         - local-path-provisioner

pkg/build/nodeimage/defaults.go

@@ -20,4 +20,4 @@ package nodeimage
 const DefaultImage = "kindest/node:latest"
 
 // DefaultBaseImage is the default base image used
-const DefaultBaseImage = "kindest/base:v20210328-c17ca167@sha256:0311870f4d35b0f68e2fedb5d703a552a8e7eb438acc67a3bd13982c2bda7487"
+const DefaultBaseImage = "kindest/base:v20210328-c17ca167"

pkg/build/nodeimage/imageimporter.go

@@ -20,25 +20,16 @@ import (
 	"sigs.k8s.io/kind/pkg/exec"
 )
 
-type imageImporter interface {
-	Prepare() error
-	LoadCommand() exec.Cmd
-	ListImported() ([]string, error)
-	End() error
-}
-
 type containerdImporter struct {
 	containerCmder exec.Cmder
 }
 
-func newContainerdImporter(containerCmder exec.Cmder) imageImporter {
+func newContainerdImporter(containerCmder exec.Cmder) *containerdImporter {
 	return &containerdImporter{
 		containerCmder: containerCmder,
 	}
 }
 
-var _ imageImporter = &containerdImporter{}
-
 func (c *containerdImporter) Prepare() error {
 	if err := c.containerCmder.Command(
 		"bash", "-c", "nohup containerd > /dev/null 2>&1 &",
@@ -53,6 +44,12 @@ func (c *containerdImporter) End() error {
 	return c.containerCmder.Command("pkill", "containerd").Run()
 }
 
+func (c *containerdImporter) Pull(image, platform string) error {
+	return c.containerCmder.Command(
+		"ctr", "--namespace=k8s.io", "images", "pull", "--platform="+platform, image,
+	).Run()
+}
+
 func (c *containerdImporter) LoadCommand() exec.Cmd {
 	return c.containerCmder.Command(
 		// TODO: ideally we do not need this in the future. we have fixed at least one image

pkg/build/nodeimage/internal/container/docker/archive.go

@@ -87,7 +87,7 @@ func GetArchiveTags(path string) ([]string, error) {
 // https://github.com/moby/moby/blob/master/image/spec/v1.md
 // https://github.com/moby/moby/blob/master/image/spec/v1.1.md
 // https://github.com/moby/moby/blob/master/image/spec/v1.2.md
-func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositories func(string) string) error {
+func EditArchive(reader io.Reader, writer io.Writer, editRepositories func(string) string, architectureOverride string) error {
 	tarReader := tar.NewReader(reader)
 	tarWriter := tar.NewWriter(writer)
 	// iterate all entries in the tarball
@@ -117,6 +117,15 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie
 				return err
 			}
 			hdr.Size = int64(len(b))
+			// edit image config when we find that
+		} else if strings.HasSuffix(hdr.Name, ".json") {
+			if architectureOverride != "" {
+				b, err = editConfigArchitecture(b, architectureOverride)
+				if err != nil {
+					return err
+				}
+				hdr.Size = int64(len(b))
+			}
 		}
 
 		// write to the output tarball
@@ -133,6 +142,19 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie
 
 /* helpers */
 
+func editConfigArchitecture(raw []byte, architectureOverride string) ([]byte, error) {
+	var cfg map[string]interface{}
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, err
+	}
+	const architecture = "architecture"
+	if _, ok := cfg[architecture]; !ok {
+		return raw, nil
+	}
+	cfg[architecture] = architectureOverride
+	return json.Marshal(cfg)
+}
+
 // archiveRepositories represents repository:tag:ref
 //
 // https://github.com/moby/moby/blob/master/image/spec/v1.md

pkg/build/nodeimage/internal/container/docker/pull.go

@@ -23,33 +23,17 @@ import (
 	"sigs.k8s.io/kind/pkg/log"
 )
 
-// PullIfNotPresent will pull an image if it is not present locally
-// retrying up to retries times
-// it returns true if it attempted to pull, and any errors from pulling
-func PullIfNotPresent(logger log.Logger, image string, retries int) (pulled bool, err error) {
-	// TODO(bentheelder): switch most (all) of the logging here to debug level
-	// once we have configurable log levels
-	// if this did not return an error, then the image exists locally
-	cmd := exec.Command("docker", "inspect", "--type=image", image)
-	if err := cmd.Run(); err == nil {
-		logger.V(1).Infof("Image: %s present locally", image)
-		return false, nil
-	}
-	// otherwise try to pull it
-	return true, Pull(logger, image, retries)
-}
-
 // Pull pulls an image, retrying up to retries times
-func Pull(logger log.Logger, image string, retries int) error {
-	logger.V(1).Infof("Pulling image: %s ...", image)
-	err := exec.Command("docker", "pull", image).Run()
+func Pull(logger log.Logger, image string, platform string, retries int) error {
+	logger.V(1).Infof("Pulling image: %s for platform %s ...", image, platform)
+	err := exec.Command("docker", "pull", "--platform="+platform, image).Run()
 	// retry pulling up to retries times if necessary
 	if err != nil {
 		for i := 0; i < retries; i++ {
 			time.Sleep(time.Second * time.Duration(i+1))
 			logger.V(1).Infof("Trying again to pull image: %q ... %v", image, err)
 			// TODO(bentheelder): add some backoff / sleep?
-			err = exec.Command("docker", "pull", image).Run()
+			err = exec.Command("docker", "pull", "--platform="+platform, image).Run()
 			if err == nil {
 				break
 			}

pkg/build/nodeimage/internal/kube/builder_docker.go

@@ -21,11 +21,11 @@ import (
 	"path/filepath"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/util/version"
+
 	"sigs.k8s.io/kind/pkg/errors"
 	"sigs.k8s.io/kind/pkg/exec"
 	"sigs.k8s.io/kind/pkg/log"
-
-	"k8s.io/apimachinery/pkg/util/version"
 )
 
 // TODO(bentheelder): plumb through arch

pkg/build/nodeimage/options.go

@@ -62,3 +62,13 @@ func WithLogger(logger log.Logger) Option {
 		return nil
 	})
 }
+
+// WithArch sets the architecture to build for
+func WithArch(arch string) Option {
+	return optionAdapter(func(b *buildContext) error {
+		if arch != "" {
+			b.arch = arch
+		}
+		return nil
+	})
+}