feat: support AUFS -> OverlayFS whiteout conversion for squashfs

Add an option, `OptConvertWhiteout`, for `squashfsConvertor`, that
when set true will convert AUFS whiteout files in the incoming tar, to
OverlayFS whiteouts in the outgoing squashfs.

A `.wh.file` marker indicating a whiteout of `file` is replaced by
`file` as a 0:0 character device.

A `dir/.wh..wh..opq` marker indicating an opaque directory is replaced
with the `trusted.overlay.opaque="y"` xattr on `dir`.

A two-pass approach is required. AUFS directory opaque markers are not
required to be adjacent to the directory to which they apply, in the
tar stream. As `tar2sqfs` and `sqfstar` do not support tar files with
two instances of the same dir/file, we cannot append an additional tar
entry if we come across an opaque marker distant from the target
directory entry.

Fixes sylabs#25

pkg/mutate/squashfs.go

@@ -5,12 +5,14 @@
 package mutate
 
 import (
+	"archive/tar"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -19,10 +21,16 @@ import (
 
 const layerMediaType types.MediaType = "application/vnd.sylabs.image.layer.v1.squashfs"
 
+const (
+	aufsWhiteoutPrefix = ".wh."
+	aufsOpaqueMarker   = ".wh..wh..opq"
+)
+
 type squashfsConverter struct {
-	converter string   // Path to converter program.
-	args      []string // Arguments required for converter program.
-	dir       string   // Working directory.
+	converter       string   // Path to converter program.
+	args            []string // Arguments required for converter program.
+	dir             string   // Working directory.
+	convertWhiteout bool     // Convert whiteout markers from AUFS -> OverlayFS
 }
 
 // SquashfsConverterOpt are used to specify squashfs converter options.
@@ -45,6 +53,15 @@ func OptSquashfsLayerConverter(converter string) SquashfsConverterOpt {
 
 var errSquashfsConverterNotSupported = errors.New("squashfs converter not supported")
 
+// OptConvertWhiteout is set to convert whiteout / opaque markers from AUFS to
+// OverlayFS format during the tar -> squashfs conversion.
+func OptConvertWhiteout(b bool) SquashfsConverterOpt {
+	return func(c *squashfsConverter) error {
+		c.convertWhiteout = b
+		return nil
+	}
+}
+
 // SquashfsLayer converts the base layer into a layer using the squashfs format. A dir must be
 // specified, which is used as a working directory during conversion. The caller is responsible for
 // cleaning up dir.
@@ -104,23 +121,57 @@ func SquashfsLayer(base v1.Layer, dir string, opts ...SquashfsConverterOpt) (v1.
 }
 
 // makeSquashfs returns the path to a squashfs file that contains the contents of the uncompressed
-// TAR stream from r.
-func (c *squashfsConverter) makeSquashfs(r io.Reader) (string, error) {
+// TAR stream from l.Uncompressed().
+func (c *squashfsConverter) makeSquashfs(l v1.Layer) (string, error) {
 	dir, err := os.MkdirTemp(c.dir, "")
 	if err != nil {
 		return "", err
 	}
 
 	path := filepath.Join(dir, "layer.sqfs")
 
+	tarIn, err := l.Uncompressed()
+	if err != nil {
+		return "", err
+	}
+	defer tarIn.Close()
+
+	var opaquePaths map[string]bool
+	filterErr := make(chan error, 1)
+
+	if c.convertWhiteout {
+		opqrc, err := l.Uncompressed()
+		if err != nil {
+			return "", err
+		}
+		defer opqrc.Close()
+		opaquePaths, err = scanAUFSOpaque(opqrc)
+		if err != nil {
+			return "", err
+		}
+
+		filterIn := tarIn
+		pr, pw := io.Pipe()
+		tarIn = pr
+		go func() {
+			filterErr <- whiteoutFilter(filterIn, pw, opaquePaths)
+		}()
+	}
+
 	//nolint:gosec // Arguments are created programatically.
 	cmd := exec.Command(c.converter, append(c.args, path)...)
-	cmd.Stdin = r
+	cmd.Stdin = tarIn
 
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return "", fmt.Errorf("%s error: %w, output: %s", c.converter, err, out)
 	}
 
+	if c.convertWhiteout {
+		if err := <-filterErr; err != nil {
+			return "", err
+		}
+	}
+
 	return path, nil
 }
 
@@ -170,13 +221,7 @@ func (l *squashfsLayer) populate() error {
 		return nil
 	}
 
-	rc, err := l.base.Uncompressed()
-	if err != nil {
-		return err
-	}
-	defer rc.Close()
-
-	path, err := l.converter.makeSquashfs(rc)
+	path, err := l.converter.makeSquashfs(l.base)
 	if err != nil {
 		return err
 	}
@@ -241,3 +286,96 @@ func (l *squashfsLayer) Size() (int64, error) {
 func (l *squashfsLayer) MediaType() (types.MediaType, error) {
 	return layerMediaType, nil
 }
+
+// whiteOutFilter streams a tar file from in to out, replacing AUFS whiteout
+// markers with OverlayFS whiteout markers. Due to unrestricted ordering of
+// markers vs their target, the list of opaquePaths must be obtained prior to
+// filtering and provided to this filter.
+func whiteoutFilter(in io.ReadCloser, out io.WriteCloser, opaquePaths map[string]bool) error {
+	tr := tar.NewReader(in)
+	tw := tar.NewWriter(out)
+	defer out.Close()
+	defer tw.Close()
+
+	fmt.Fprintf(os.Stderr, "OPAQUES: %v\n", opaquePaths)
+
+	for {
+		header, err := tr.Next()
+
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		// Must force to PAX format, to accommodate xattrs
+		header.Format = tar.FormatPAX
+
+		clean := filepath.Clean(header.Name)
+		base := filepath.Base(header.Name)
+		parent := filepath.Dir(header.Name)
+
+		// Don't include .wh..wh..opq opaque directory markers in output.
+		if filepath.Base(header.Name) == aufsOpaqueMarker {
+			// If we don't know the target should be opaque, then provided opaquePaths is incorrect.
+			if !opaquePaths[parent] {
+				return fmt.Errorf("unexpected opaque marker found for %q", parent)
+			}
+			continue
+		}
+		// Set overlayfs xattr on a dir that was previously found to contain a .wh..wh..opq marker.
+		fmt.Fprintf(os.Stderr, "%v\n", header.Name)
+		if opq := opaquePaths[clean]; opq {
+			if header.PAXRecords == nil {
+				header.PAXRecords = map[string]string{}
+			}
+			header.PAXRecords["SCHILY.xattr."+"trusted.overlay.opaque"] = "y"
+			fmt.Fprintf(os.Stderr, "PAX: %v", header.PAXRecords)
+		}
+		// Replace a `.wh.<name>` marker with a char dev 0 at <name>
+		if strings.HasPrefix(base, aufsWhiteoutPrefix) {
+			target := filepath.Join(parent, strings.TrimPrefix(base, aufsWhiteoutPrefix))
+			cHeader := header
+			cHeader.Name = target
+			cHeader.Typeflag = tar.TypeChar
+			cHeader.Devmajor = 0
+			cHeader.Devminor = 0
+			if err := tw.WriteHeader(header); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		if _, err := io.Copy(tw, tr); err != nil {
+			return err
+		}
+
+	}
+}
+
+func scanAUFSOpaque(in io.Reader) (opaquePaths map[string]bool, err error) {
+	opaquePaths = map[string]bool{}
+	tr := tar.NewReader(in)
+	for {
+		header, err := tr.Next()
+
+		if err == io.EOF {
+			return opaquePaths, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		base := filepath.Base(header.Name)
+		parent := filepath.Dir(header.Name)
+
+		if base == aufsOpaqueMarker {
+			opaquePaths[parent] = true
+		}
+	}
+}

pkg/mutate/squashfs_test.go

@@ -49,18 +49,21 @@ func diffSquashFS(tb testing.TB, pathA, pathB string, diffArgs ...string) {
 
 func Test_SquashfsLayer(t *testing.T) {
 	tests := []struct {
-		name      string
-		layer     v1.Layer
-		converter string
-		diffArgs  []string
+		name            string
+		layer           v1.Layer
+		converter       string
+		convertWhiteout bool
+		diffArgs        []string
 	}{
+		// HelloWorld layer contains no whiteouts - convertWhiteout should have no effect on output.
 		{
 			name: "HelloWorldBlob_sqfstar",
 			layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{
 				Algorithm: "sha256",
 				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
 			}),
-			converter: "sqfstar",
+			converter:       "sqfstar",
+			convertWhiteout: false,
 			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
 			// differences in ownership.
 			diffArgs: []string{"--no-owner"},
@@ -71,7 +74,89 @@ func Test_SquashfsLayer(t *testing.T) {
 				Algorithm: "sha256",
 				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
 			}),
-			converter: "tar2sqfs",
+			converter:       "tar2sqfs",
+			convertWhiteout: false,
+		},
+		{
+			name: "HelloWorldBlob_sqfstar_convertWhiteout",
+			layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
+			}),
+			converter:       "sqfstar",
+			convertWhiteout: true,
+			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
+			// differences in ownership.
+			diffArgs: []string{"--no-owner"},
+		},
+		{
+			name: "HelloWorldBlob_tar2sqfs_convertWhiteout",
+			layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
+			}),
+			converter:       "tar2sqfs",
+			convertWhiteout: true,
+		},
+		// AUFS layer contains whiteouts. Should be converted to overlayfs form when convertWhiteout = true.
+		//
+		//    Original (AUFS)
+		//		All regular files.
+		//
+		//        [drwxr-xr-x]  .
+		//			├── [drwxr-xr-x]  dir
+		//			│   └── [-rw-r--r--]  .wh..wh..opq
+		//			└── [-rw-r--r--]  .wh.file
+		//
+		//    Converted (OverlayFS)
+		//		.wh.file becomes file as a char 0:0 device
+		// 		dir/.wh..wh..opq becomes trusted.overlay.opaque="y" xattr on dir
+		//
+		//			[drwxr-xr-x]  .
+		//			├── [drwxr-xr-x]  dir
+		//			└── [crw-r--r--]  file
+		//
+		{
+			name: "AUFSBlob_sqfstar",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:       "sqfstar",
+			convertWhiteout: false,
+			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
+			// differences in ownership.
+			diffArgs: []string{"--no-owner"},
+		},
+		{
+			name: "AUFSBlob_tar2sqfs",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:       "tar2sqfs",
+			convertWhiteout: false,
+		},
+		{
+			name: "AUFSBlob_sqfstar_convertWhiteout",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:       "sqfstar",
+			convertWhiteout: true,
+			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
+			// differences in ownership.
+			diffArgs: []string{"--no-owner"},
+		},
+		{
+			name: "AUFSBlob_tar2sqfs_convertWhiteout",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:       "tar2sqfs",
+			convertWhiteout: true,
 		},
 	}
 	for _, tt := range tests {
@@ -82,7 +167,10 @@ func Test_SquashfsLayer(t *testing.T) {
 				t.Skip(err)
 			}
 
-			l, err := SquashfsLayer(tt.layer, t.TempDir(), OptSquashfsLayerConverter(tt.converter))
+			l, err := SquashfsLayer(tt.layer, t.TempDir(),
+				OptSquashfsLayerConverter(tt.converter),
+				OptConvertWhiteout(tt.convertWhiteout),
+			)
 			if err != nil {
 				t.Fatal(err)
 			}

test/images/aufs-docker-v2-manifest/blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a

@@ -0,0 +1 @@
+{"architecture":"arm64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"],"Image":"sha256:cc0fff24c4ece63ade5d9f549e42c926cf569112c4f5c439a4a57f3f33f5588b","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"b2af51419cbf516f3c99b877a64906b21afedc175bd3cd082eb5798e2f277bb4","container_config":{"Hostname":"b2af51419cbf","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/hello\"]"],"Image":"sha256:cc0fff24c4ece63ade5d9f549e42c926cf569112c4f5c439a4a57f3f33f5588b","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":{}},"created":"2022-03-19T16:12:58.923371954Z","docker_version":"20.10.12","history":[{"created":"2022-03-19T16:12:58.834095198Z","created_by":"/bin/sh -c #(nop) COPY file:a79dd5bda1e77203401956a93401d3aef45221fc750295a4291896f3386f4f54 in / "},{"created":"2022-03-19T16:12:58.923371954Z","created_by":"/bin/sh -c #(nop)  CMD [\"/hello\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:efb53921da3394806160641b72a2cbd34ca1a9a8345ac670a85a04ad3d0e3507"]},"variant":"v8"}

test/images/aufs-docker-v2-manifest/blobs/sha256/6c9c1b8d1adba535a40046b52c051cccf33d85f60827bd269c978fd95f05c3c9

@@ -0,0 +1,16 @@
+{
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+    "config": {
+       "mediaType": "application/vnd.docker.container.image.v1+json",
+       "size": 2,
+       "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
+    },
+    "layers": [
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 252,
+          "digest": "sha256:da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf"
+       }
+    ]
+ }

test/images/aufs-docker-v2-manifest/index.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":525,"digest":"sha256:6c9c1b8d1adba535a40046b52c051cccf33d85f60827bd269c978fd95f05c3c9","platform":{"architecture":"arm64","os":"linux","variant":"v8"}}]}

test/images/aufs-docker-v2-manifest/oci-layout

@@ -0,0 +1,3 @@
+{
+    "imageLayoutVersion": "1.0.0"
+}