Merge pull request #55 from dtrudg/issue53

feat: OverlayFS -> AUFS whiteout conversion

pkg/mutate/squashfs.go

@@ -178,7 +178,7 @@ func (c *squashfsConverter) Uncompressed(l v1.Layer) (io.ReadCloser, error) {
 	pr, pw := io.Pipe()
 	go func() {
 		defer rc.Close()
-		pw.CloseWithError(whiteoutFilter(rc, pw, opaquePaths))
+		pw.CloseWithError(whiteoutsToOverlayFS(rc, pw, opaquePaths))
 	}()
 	return pr, nil
 }

pkg/mutate/whiteout.go

@@ -16,14 +16,14 @@ import (
 const (
 	aufsWhiteoutPrefix = ".wh."
 	aufsOpaqueMarker   = ".wh..wh..opq"
+	schilyOpaqueXattr  = "SCHILY.xattr.trusted.overlay.opaque"
 )
 
 var errUnexpectedOpaque = errors.New("unexpected opaque marker")
 
 // scanAUFSWhiteouts reads a TAR stream, returning a map of <path>:true for
 // directories in the tar that contain an AUFS .wh..wh..opq opaque directory
 // marker file, and a boolean indicating the presence of any .wh.<file> markers.
-// Note that paths returned are clean, per filepath.Clean.
 func scanAUFSWhiteouts(in io.Reader) (map[string]bool, bool, error) {
 	opaquePaths := map[string]bool{}
 	fileWhiteout := false
@@ -39,10 +39,9 @@ func scanAUFSWhiteouts(in io.Reader) (map[string]bool, bool, error) {
 			return nil, false, err
 		}
 
-		base := filepath.Base(header.Name)
+		parent, base := filepath.Split(header.Name)
 
 		if base == aufsOpaqueMarker {
-			parent := filepath.Dir(header.Name)
 			opaquePaths[parent] = true
 		}
 
@@ -52,11 +51,11 @@ func scanAUFSWhiteouts(in io.Reader) (map[string]bool, bool, error) {
 	}
 }
 
-// whiteOutFilter streams a tar file from in to out, replacing AUFS whiteout
-// markers with OverlayFS whiteout markers. Due to unrestricted ordering of
-// markers vs their target, the list of opaquePaths must be obtained prior to
-// filtering and provided to this filter.
-func whiteoutFilter(in io.Reader, out io.Writer, opaquePaths map[string]bool) error {
+// whiteoutsToOverlayFS streams a tar file from in to out, replacing AUFS
+// whiteout markers with OverlayFS whiteout markers. Due to unrestricted
+// ordering of markers vs their target, the list of opaquePaths must be obtained
+// prior to filtering and provided to this filter.
+func whiteoutsToOverlayFS(in io.Reader, out io.Writer, opaquePaths map[string]bool) error {
 	tr := tar.NewReader(in)
 	tw := tar.NewWriter(out)
 	defer tw.Close()
@@ -74,10 +73,7 @@ func whiteoutFilter(in io.Reader, out io.Writer, opaquePaths map[string]bool) er
 		// Must force to PAX format, to accommodate xattrs
 		header.Format = tar.FormatPAX
 
-		clean := filepath.Clean(header.Name)
-		base := filepath.Base(header.Name)
-		parent := filepath.Dir(header.Name)
-
+		parent, base := filepath.Split(header.Name)
 		// Don't include .wh..wh..opq opaque directory markers in output.
 		if base == aufsOpaqueMarker {
 			// If we don't know the target should be opaque, then provided opaquePaths is incorrect.
@@ -87,15 +83,15 @@ func whiteoutFilter(in io.Reader, out io.Writer, opaquePaths map[string]bool) er
 			continue
 		}
 		// Set overlayfs xattr on a dir that was previously found to contain a .wh..wh..opq marker.
-		if opq := opaquePaths[clean]; opq {
+		if opq := opaquePaths[header.Name]; opq {
 			if header.PAXRecords == nil {
 				header.PAXRecords = map[string]string{}
 			}
-			header.PAXRecords["SCHILY.xattr."+"trusted.overlay.opaque"] = "y"
+			header.PAXRecords[schilyOpaqueXattr] = "y"
 		}
 		// Replace a `.wh.<name>` marker with a char dev 0 at <name>
 		if strings.HasPrefix(base, aufsWhiteoutPrefix) {
-			target := filepath.Join(parent, strings.TrimPrefix(base, aufsWhiteoutPrefix))
+			target := parent + strings.TrimPrefix(base, aufsWhiteoutPrefix)
 			header.Name = target
 			header.Typeflag = tar.TypeChar
 			header.Devmajor = 0
@@ -119,3 +115,69 @@ func whiteoutFilter(in io.Reader, out io.Writer, opaquePaths map[string]bool) er
 		}
 	}
 }
+
+// whiteoutsToAUFS streams a tar file from in to out, replacing OverlayFS
+// whiteout markers with AUFS whiteout markers.
+func whiteoutsToAUFS(in io.Reader, out io.Writer) error {
+	tr := tar.NewReader(in)
+	tw := tar.NewWriter(out)
+	defer tw.Close()
+
+	for {
+		header, err := tr.Next()
+
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		// <dir> with opaque xattr -> write both <dir> & <dir>/.wh..wh..opq
+		if header.Typeflag == tar.TypeDir && header.PAXRecords[schilyOpaqueXattr] == "y" {
+			// Write directory entry, without the xattr.
+			delete(header.PAXRecords, schilyOpaqueXattr)
+			if err := tw.WriteHeader(header); err != nil {
+				return err
+			}
+			// Write opaque marker file inside the directory.
+			trimmedName := strings.TrimSuffix(header.Name, string(filepath.Separator))
+			opqName := trimmedName + string(filepath.Separator) + aufsOpaqueMarker
+			if err := tw.WriteHeader(&tar.Header{
+				Typeflag:   tar.TypeReg,
+				Name:       opqName,
+				Size:       0,
+				Mode:       0o600,
+				Uid:        header.Uid,
+				Gid:        header.Gid,
+				Uname:      header.Uname,
+				Gname:      header.Gname,
+				AccessTime: header.AccessTime,
+				ChangeTime: header.ChangeTime,
+			}); err != nil {
+				return err
+			}
+			continue
+		}
+
+		// <file> as 0:0 char dev -> becomes .wh..wh.<file>
+		if header.Typeflag == tar.TypeChar && header.Devmajor == 0 && header.Devminor == 0 {
+			parent, base := filepath.Split(header.Name)
+			header.Typeflag = tar.TypeReg
+			header.Name = parent + aufsWhiteoutPrefix + base
+			header.Size = 0
+			header.Mode = 0o600
+		}
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		// Disable gosec G110: Potential DoS vulnerability via decompression bomb.
+		// We are just filtering a flow directly from tar reader to tar writer - we aren't reading
+		// into memory beyond the stdlib buffering.
+		//nolint:gosec
+		if _, err := io.Copy(tw, tr); err != nil {
+			return err
+		}
+	}
+}

pkg/mutate/whiteout_test.go

@@ -5,10 +5,12 @@
 package mutate
 
 import (
+	"bytes"
 	"maps"
 	"testing"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/sebdah/goldie/v2"
 )
 
 func Test_scanAUFSOpaque(t *testing.T) {
@@ -40,7 +42,7 @@ func Test_scanAUFSOpaque(t *testing.T) {
 				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
 			}),
 			expectOpaque: map[string]bool{
-				"dir": true,
+				"./dir/": true,
 			},
 			expectFileWhiteout: true,
 		},
@@ -66,3 +68,57 @@ func Test_scanAUFSOpaque(t *testing.T) {
 		})
 	}
 }
+
+func Test_WhiteoutRoundTrip(t *testing.T) {
+	// AUFS layer contains a single opaque marker on dir
+	//        [drwxr-xr-x]  .
+	//			├── [drwxr-xr-x]  dir
+	//			│   └── [-rw-r--r--]  .wh..wh..opq
+	//			└── [-rw-r--r--]  .wh.file
+	layer := testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+		Algorithm: "sha256",
+		Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+	})
+
+	g := goldie.New(t,
+		goldie.WithTestNameForDir(true),
+	)
+
+	// To an OverlayFS layer
+	//       [drwxr-xr-x]  .
+	// 		   ├── [drwxr-xr-x]  ./dir		(xattr trusted.overlay.opaque=y)
+	//         └── [crw-r--r--]  ./file		(0,0 character device)
+	rc, err := layer.Uncompressed()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rc.Close()
+
+	opaques, _, err := scanAUFSWhiteouts(rc)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rc, err = layer.Uncompressed()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rc.Close()
+
+	overlayfsTar := bytes.Buffer{}
+	if err := whiteoutsToOverlayFS(rc, &overlayfsTar, opaques); err != nil {
+		t.Fatal(err)
+	}
+	g.Assert(t, "overlayfs", overlayfsTar.Bytes())
+
+	// Back to an AUFS layer
+	//        [drwxr-xr-x]  .
+	//			├── [drwxr-xr-x]  dir
+	//			│   └── [-rw-r--r--]  .wh..wh..opq
+	//			└── [-rw-r--r--]  .wh.file
+	aufsTar := bytes.Buffer{}
+	if err := whiteoutsToAUFS(&overlayfsTar, &aufsTar); err != nil {
+		t.Fatal(err)
+	}
+	g.Assert(t, "aufs", aufsTar.Bytes())
+}