From 4df812eed799cede14846bcf1ceb4889e20470ff Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 31 May 2024 16:40:52 +0100 Subject: [PATCH] feat: OverlayFS -> AUFS whiteout conversion Allow OverlayFS whiteouts to be converted to AUFS, by filtering a tar stream. In this direction the conversion is a single pass, single operation. There is no requirement to pre-scan the tar. Close #53 --- pkg/mutate/squashfs.go | 2 +- .../Test_WhiteoutRoundTrip/aufs.golden | Bin 0 -> 4096 bytes .../Test_WhiteoutRoundTrip/overlayfs.golden | Bin 0 -> 3584 bytes pkg/mutate/whiteout.go | 79 ++++++++++++++++-- pkg/mutate/whiteout_test.go | 56 +++++++++++++ 5 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 pkg/mutate/testdata/Test_WhiteoutRoundTrip/aufs.golden create mode 100644 pkg/mutate/testdata/Test_WhiteoutRoundTrip/overlayfs.golden diff --git a/pkg/mutate/squashfs.go b/pkg/mutate/squashfs.go index 95b11ae..1a49b88 100644 --- a/pkg/mutate/squashfs.go +++ b/pkg/mutate/squashfs.go @@ -178,7 +178,7 @@ func (c *squashfsConverter) Uncompressed(l v1.Layer) (io.ReadCloser, error) { pr, pw := io.Pipe() go func() { defer rc.Close() - pw.CloseWithError(whiteoutFilter(rc, pw, opaquePaths)) + pw.CloseWithError(whiteoutsToOverlayFS(rc, pw, opaquePaths)) }() return pr, nil } diff --git a/pkg/mutate/testdata/Test_WhiteoutRoundTrip/aufs.golden b/pkg/mutate/testdata/Test_WhiteoutRoundTrip/aufs.golden new file mode 100644 index 0000000000000000000000000000000000000000..8960745e4f3ddb59311661014ec7b13b8843f077 GIT binary patch literal 4096 zcmeHJJ#T|B5cS+&!5^sc0mf6OE~%8EQm2lt?6#4RNQ_D0-}kO0DT>liDRIQ$laN2o zx_f>PpUqPi+yt7El3@c%!nh?D8G}MXu5!pq^g~g~j3qKSGEC4mUOR+jylc(JebZO^ ztvQi+%0KKw(B{ZaM4082S-JFCcMGeHbqx=Kdtl_3nxnB0!#}Z`#&sYRlq~6P!Ps8^ zTjZKyjslXG$Hmj~m3P{E$MF<2-WpzSt*dm;>$Tpr_MyLZ>|-09EbET@k+$z&-hJ6S)cro?KY$-P$7$v|Cf(mxl?{oH7@5-l@csjQCqgMQ74;9PNsJvChzvvqz5#iP Bkxc*q literal 0 HcmV?d00001 diff --git a/pkg/mutate/testdata/Test_WhiteoutRoundTrip/overlayfs.golden b/pkg/mutate/testdata/Test_WhiteoutRoundTrip/overlayfs.golden new file mode 100644 index 0000000000000000000000000000000000000000..1cf28af4bd9f554a4cfaf4857fbe9a4390aea678 GIT binary patch literal 3584 zcmeHJv2Md45M|C+_y>YH17_-!C6zK%>XeZRT1B#y1aM{leL1<7qbi9j#g0613-X=y z?%f0KpgEb{STrRii3=zRmn(+Q2nY#WaVW|d2T{s`WO8<9F+uHvc9D_^t~Ot5-&Xp= zUr9U`@9JP!9NHCxMY_zv=+dfEWAU z{hwq1m!goV|KHPes@UhDvQuH*!omG7g}9vi{W}0dXv6%6_)Y-Hg(4Zh1vsU@@Bd@- XeJtM9F$}pWIVP;6KvEznFoOa=eW`Iz literal 0 HcmV?d00001 diff --git a/pkg/mutate/whiteout.go b/pkg/mutate/whiteout.go index 467a1ae..418e08e 100644 --- a/pkg/mutate/whiteout.go +++ b/pkg/mutate/whiteout.go @@ -16,6 +16,7 @@ import ( const ( aufsWhiteoutPrefix = ".wh." aufsOpaqueMarker = ".wh..wh..opq" + schilyOpaqueXattr = "SCHILY.xattr.trusted.overlay.opaque" ) var errUnexpectedOpaque = errors.New("unexpected opaque marker") @@ -52,11 +53,11 @@ func scanAUFSWhiteouts(in io.Reader) (map[string]bool, bool, error) { } } -// whiteOutFilter streams a tar file from in to out, replacing AUFS whiteout -// markers with OverlayFS whiteout markers. Due to unrestricted ordering of -// markers vs their target, the list of opaquePaths must be obtained prior to -// filtering and provided to this filter. -func whiteoutFilter(in io.Reader, out io.Writer, opaquePaths map[string]bool) error { +// whiteoutsToOverlayFS streams a tar file from in to out, replacing AUFS +// whiteout markers with OverlayFS whiteout markers. Due to unrestricted +// ordering of markers vs their target, the list of opaquePaths must be obtained +// prior to filtering and provided to this filter. +func whiteoutsToOverlayFS(in io.Reader, out io.Writer, opaquePaths map[string]bool) error { tr := tar.NewReader(in) tw := tar.NewWriter(out) defer tw.Close() @@ -91,7 +92,7 @@ func whiteoutFilter(in io.Reader, out io.Writer, opaquePaths map[string]bool) er if header.PAXRecords == nil { header.PAXRecords = map[string]string{} } - header.PAXRecords["SCHILY.xattr."+"trusted.overlay.opaque"] = "y" + header.PAXRecords[schilyOpaqueXattr] = "y" } // Replace a `.wh.` marker with a char dev 0 at if strings.HasPrefix(base, aufsWhiteoutPrefix) { @@ -119,3 +120,69 @@ func whiteoutFilter(in io.Reader, out io.Writer, opaquePaths map[string]bool) er } } } + +// whiteoutsToAUFS streams a tar file from in to out, replacing OverlayFS +// whiteout markers with AUFS whiteout markers. +func whiteoutsToAUFS(in io.Reader, out io.Writer) error { + tr := tar.NewReader(in) + tw := tar.NewWriter(out) + defer tw.Close() + + for { + header, err := tr.Next() + + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + // with opaque xattr -> write both & /.wh..wh..opq + if header.Typeflag == tar.TypeDir && header.PAXRecords[schilyOpaqueXattr] == "y" { + // Write directory entry, without the xattr. + delete(header.PAXRecords, schilyOpaqueXattr) + if err := tw.WriteHeader(header); err != nil { + return err + } + // Write opaque marker file inside the directory. + // Disable gosec 305: File traversal when extracting zip/tar archive. + // We are modifying an existing tar stream. No extraction here. + //nolint:gosec + if err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: filepath.Join(header.Name, aufsOpaqueMarker), + Size: 0, + Mode: 0o600, + Uid: header.Uid, + Gid: header.Gid, + Uname: header.Uname, + Gname: header.Gname, + AccessTime: header.AccessTime, + ChangeTime: header.ChangeTime, + }); err != nil { + return err + } + continue + } + + // as 0:0 char dev -> becomes .wh..wh. + if header.Typeflag == tar.TypeChar && header.Devmajor == 0 && header.Devminor == 0 { + header.Typeflag = tar.TypeReg + header.Name = filepath.Join(filepath.Dir(header.Name), aufsWhiteoutPrefix+filepath.Base(header.Name)) + header.Size = 0 + header.Mode = 0o600 + } + + if err := tw.WriteHeader(header); err != nil { + return err + } + // Disable gosec G110: Potential DoS vulnerability via decompression bomb. + // We are just filtering a flow directly from tar reader to tar writer - we aren't reading + // into memory beyond the stdlib buffering. + //nolint:gosec + if _, err := io.Copy(tw, tr); err != nil { + return err + } + } +} diff --git a/pkg/mutate/whiteout_test.go b/pkg/mutate/whiteout_test.go index 1921e6e..e432b50 100644 --- a/pkg/mutate/whiteout_test.go +++ b/pkg/mutate/whiteout_test.go @@ -5,10 +5,12 @@ package mutate import ( + "bytes" "maps" "testing" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/sebdah/goldie/v2" ) func Test_scanAUFSOpaque(t *testing.T) { @@ -66,3 +68,57 @@ func Test_scanAUFSOpaque(t *testing.T) { }) } } + +func Test_WhiteoutRoundTrip(t *testing.T) { + // AUFS layer contains a single opaque marker on dir + // [drwxr-xr-x] . + // ├── [drwxr-xr-x] dir + // │   └── [-rw-r--r--] .wh..wh..opq + // └── [-rw-r--r--] .wh.file + layer := testLayer(t, "aufs-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf", + }) + + g := goldie.New(t, + goldie.WithTestNameForDir(true), + ) + + // To an OverlayFS layer + // [drwxr-xr-x] . + // ├── [drwxr-xr-x] ./dir (xattr trusted.overlay.opaque=y) + // └── [crw-r--r--] ./file (0,0 character device) + rc, err := layer.Uncompressed() + if err != nil { + t.Fatal(err) + } + defer rc.Close() + + opaques, _, err := scanAUFSWhiteouts(rc) + if err != nil { + t.Fatal(err) + } + + rc, err = layer.Uncompressed() + if err != nil { + t.Fatal(err) + } + defer rc.Close() + + overlayfsTar := bytes.Buffer{} + if err := whiteoutsToOverlayFS(rc, &overlayfsTar, opaques); err != nil { + t.Fatal(err) + } + g.Assert(t, "overlayfs", overlayfsTar.Bytes()) + + // Back to an AUFS layer + // [drwxr-xr-x] . + // ├── [drwxr-xr-x] dir + // │   └── [-rw-r--r--] .wh..wh..opq + // └── [-rw-r--r--] .wh.file + aufsTar := bytes.Buffer{} + if err := whiteoutsToAUFS(&overlayfsTar, &aufsTar); err != nil { + t.Fatal(err) + } + g.Assert(t, "aufs", aufsTar.Bytes()) +}