feat(erofs): initial commit for erofs support

Fixes opencontainers/image-spec#1190

Signed-off-by: Ramkumar Chinchani <[email protected]>

cmd/stacker/build.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	cli "github.com/urfave/cli/v2"
+	"stackerbuild.io/stacker/pkg/erofs"
 	"stackerbuild.io/stacker/pkg/squashfs"
 	"stackerbuild.io/stacker/pkg/stacker"
 	"stackerbuild.io/stacker/pkg/types"
@@ -52,13 +53,17 @@ func initCommonBuildFlags() []cli.Flag {
 		},
 		&cli.StringSliceFlag{
 			Name:  "layer-type",
-			Usage: "set the output layer type (supported values: tar, squashfs); can be supplied multiple times",
+			Usage: "set the output layer type (supported values: tar, squashfs, erofs); can be supplied multiple times",
 			Value: cli.NewStringSlice("tar"),
 		},
 		&cli.BoolFlag{
 			Name:  "no-squashfs-verity",
 			Usage: "do not append dm-verity data to squashfs archives",
 		},
+		&cli.BoolFlag{
+			Name:  "no-verity",
+			Usage: "do not append dm-verity data to non-tar archives",
+		},
 		&cli.BoolFlag{
 			Name:  "require-hash",
 			Usage: "require all remote imports to have a hash provided in stackerfiles",
@@ -105,6 +110,7 @@ func newBuildArgs(ctx *cli.Context) (stacker.BuildArgs, error) {
 	var err error
 	verity := squashfs.VerityMetadata(!ctx.Bool("no-squashfs-verity"))
 	args.LayerTypes, err = types.NewLayerTypes(ctx.StringSlice("layer-type"), verity)
+	verity := erofs.VerityMetadata(!ctx.Bool("no-erofs-verity"))
 	return args, err
 }
 

pkg/erofs/erofs.go

@@ -0,0 +1,182 @@
+// This package is a small go "library" (read: exec wrapper) around the
+// mkfs.erofs binary that provides some useful primitives.
+package erofs
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+var checkZstdSupported sync.Once
+var zstdIsSuspported bool
+
+// ExcludePaths represents a list of paths to exclude in a erofs listing.
+// Users should do something like filepath.Walk() over the whole filesystem,
+// calling AddExclude() or AddInclude() based on whether they want to include
+// or exclude a particular file. Note that if e.g. /usr is excluded, then
+// everyting underneath is also implicitly excluded. The
+// AddExclude()/AddInclude() methods do the math to figure out what is the
+// correct set of things to exclude or include based on what paths have been
+// previously included or excluded.
+type ExcludePaths struct {
+	exclude map[string]bool
+	include []string
+}
+
+func NewExcludePaths() *ExcludePaths {
+	return &ExcludePaths{
+		exclude: map[string]bool{},
+		include: []string{},
+	}
+}
+
+func (eps *ExcludePaths) AddExclude(p string) {
+	for _, inc := range eps.include {
+		// If /usr/bin/ls has changed but /usr hasn't, we don't want to list
+		// /usr in the include paths any more, so let's be sure to only
+		// add things which aren't prefixes.
+		if strings.HasPrefix(inc, p) {
+			return
+		}
+	}
+	eps.exclude[p] = true
+}
+
+func (eps *ExcludePaths) AddInclude(orig string, isDir bool) {
+	// First, remove this thing and all its parents from exclude.
+	p := orig
+
+	// normalize to the first dir
+	if !isDir {
+		p = path.Dir(p)
+	}
+	for {
+		// our paths are all absolute, so this is a base case
+		if p == "/" {
+			break
+		}
+
+		delete(eps.exclude, p)
+		p = filepath.Dir(p)
+	}
+
+	// now add it to the list of includes, so we don't accidentally re-add
+	// anything above.
+	eps.include = append(eps.include, orig)
+}
+
+func (eps *ExcludePaths) String() (string, error) {
+	var buf bytes.Buffer
+	for p := range eps.exclude {
+		_, err := buf.WriteString(p)
+		if err != nil {
+			return "", err
+		}
+		_, err = buf.WriteString("\n")
+		if err != nil {
+			return "", err
+		}
+	}
+
+	_, err := buf.WriteString("\n")
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}
+
+func MakeErofs(tempdir string, rootfs string, eps *ExcludePaths, verity VerityMetadata) (io.ReadCloser, string, string, error) {
+	var excludesFile string
+	var err error
+	var toExclude string
+	var rootHash string
+
+	if eps != nil {
+		toExclude, err = eps.String()
+		if err != nil {
+			return nil, "", rootHash, errors.Wrapf(err, "couldn't create exclude path list")
+		}
+	}
+
+	if len(toExclude) != 0 {
+		excludes, err := os.CreateTemp(tempdir, "stacker-erofs-exclude-")
+		if err != nil {
+			return nil, "", rootHash, err
+		}
+		defer os.Remove(excludes.Name())
+
+		excludesFile = excludes.Name()
+		_, err = excludes.WriteString(toExclude)
+		excludes.Close()
+		if err != nil {
+			return nil, "", rootHash, err
+		}
+	}
+
+	tmpErofs, err := os.CreateTemp(tempdir, "stacker-erofs-img-")
+	if err != nil {
+		return nil, "", rootHash, err
+	}
+	tmpErofs.Close()
+	os.Remove(tmpErofs.Name())
+	defer os.Remove(tmpErofs.Name())
+	args := []string{rootfs, tmpErofs.Name()}
+	compression := GzipCompression
+	if mkerofsSupportsZstd() {
+		args = append(args, "-z", "zstd")
+		compression = ZstdCompression
+	}
+	if len(toExclude) != 0 {
+		args = append(args, "--exclude-path", excludesFile)
+	}
+	cmd := exec.Command("mkfs.erofs", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err = cmd.Run(); err != nil {
+		return nil, "", rootHash, errors.Wrap(err, "couldn't build erofs")
+	}
+
+	if verity {
+		rootHash, err = appendVerityData(tmpErofs.Name())
+		if err != nil {
+			return nil, "", rootHash, err
+		}
+	}
+
+	blob, err := os.Open(tmpErofs.Name())
+	if err != nil {
+		return nil, "", rootHash, errors.WithStack(err)
+	}
+
+	return blob, GenerateErofsMediaType(compression, verity), rootHash, nil
+}
+
+func mkerofsSupportsZstd() bool {
+	checkZstdSupported.Do(func() {
+		var stdoutBuffer strings.Builder
+		var stderrBuffer strings.Builder
+
+		cmd := exec.Command("mkfs.erofs", "--help")
+		cmd.Stdout = &stdoutBuffer
+		cmd.Stderr = &stderrBuffer
+
+		// Ignore errs here as `mkerofs --help` exit status code is 1
+		_ = cmd.Run()
+
+		if strings.Contains(stdoutBuffer.String(), "zstd") ||
+			strings.Contains(stderrBuffer.String(), "zstd") {
+			zstdIsSuspported = true
+		}
+	})
+
+	return zstdIsSuspported
+}

pkg/erofs/mediatype.go

@@ -0,0 +1,37 @@
+package erofs
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ErofsCompression string
+type VerityMetadata bool
+
+const (
+	BaseMediaTypeLayerErofs = "application/vnd.stacker.image.layer.erofs"
+
+	GzipCompression ErofsCompression = "gzip"
+	ZstdCompression ErofsCompression = "zstd"
+
+	veritySuffix = "verity"
+
+	VerityMetadataPresent VerityMetadata = true
+	VerityMetadataMissing VerityMetadata = false
+)
+
+func IsErofsMediaType(mediaType string) bool {
+	return strings.HasPrefix(mediaType, BaseMediaTypeLayerErofs)
+}
+
+func GenerateErofsMediaType(comp ErofsCompression, verity VerityMetadata) string {
+	verityString := ""
+	if verity {
+		verityString = fmt.Sprintf("+%s", veritySuffix)
+	}
+	return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerErofs, comp, verityString)
+}
+
+func HasVerityMetadata(mediaType string) VerityMetadata {
+	return VerityMetadata(strings.HasSuffix(mediaType, veritySuffix))
+}

pkg/erofs/superblock.go

@@ -0,0 +1,84 @@
+package erofs
+
+const (
+	// Definitions for superblock.
+	SuperBlockMagicV1 = 0xe0f5e1e2
+	SuperBlockOffset  = 1024
+
+	// Inode slot size in bit shift.
+	InodeSlotBits = 5
+
+	// Max file name length.
+	MaxNameLen = 255
+)
+
+// Bit definitions for Inode*::Format.
+const (
+	InodeLayoutBit  = 0
+	InodeLayoutBits = 1
+
+	InodeDataLayoutBit  = 1
+	InodeDataLayoutBits = 3
+)
+
+// Inode layouts.
+const (
+	InodeLayoutCompact  = 0
+	InodeLayoutExtended = 1
+)
+
+// Inode data layouts.
+const (
+	InodeDataLayoutFlatPlain = iota
+	InodeDataLayoutFlatCompressionLegacy
+	InodeDataLayoutFlatInline
+	InodeDataLayoutFlatCompression
+	InodeDataLayoutChunkBased
+	InodeDataLayoutMax
+)
+
+// Features w/ backward compatibility.
+// This is not exhaustive, unused features are not listed.
+const (
+	FeatureCompatSuperBlockChecksum = 0x00000001
+)
+
+// Features w/o backward compatibility.
+//
+// Any features that aren't in FeatureIncompatSupported are incompatible
+// with this implementation.
+//
+// This is not exhaustive, unused features are not listed.
+const (
+	FeatureIncompatSupported = 0x0
+)
+
+// Sizes of on-disk structures in bytes.
+const (
+	SuperBlockSize    = 128
+	InodeCompactSize  = 32
+	InodeExtendedSize = 64
+	DirentSize        = 12
+)
+
+type superblock struct {
+	Magic           uint32
+	Checksum        uint32
+	FeatureCompat   uint32
+	BlockSizeBits   uint8
+	ExtSlots        uint8
+	RootNid         uint16
+	Inodes          uint64
+	BuildTime       uint64
+	BuildTimeNsec   uint32
+	Blocks          uint32
+	MetaBlockAddr   uint32
+	XattrBlockAddr  uint32
+	UUID            [16]uint8
+	VolumeName      [16]uint8
+	FeatureIncompat uint32
+	Union1          uint16
+	ExtraDevices    uint16
+	DevTableSlotOff uint16
+	Reserved        [38]uint8
+}