From a3315e31ff71bc2fa2ee2def70fa308df4ccf26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Mon, 9 Jan 2023 21:15:10 +0100 Subject: [PATCH] testutil/pkgdata: Create deb programatically Currently we only test with base-files deb which is embedded as base64 encoded binary in testutil/pkgdata.go. If we want to change the contents of the data files, we have to un-ar it, un-tar it, change the data files, tar it and ar it. Furthermore, these operations modify attributes based on the workstation on which this process is run. So we end up with arbitrary user/group ids/names names and dates. Fix it by creating debs programatically. The new interface provides way to create debs with arbitrary content of data tarball. --- internal/testutil/pkgdata.go | 115 +++++++++ internal/testutil/pkgdata_test.go | 372 ++++++++++++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 internal/testutil/pkgdata_test.go diff --git a/internal/testutil/pkgdata.go b/internal/testutil/pkgdata.go index 7dc9dc20..2b7f0221 100644 --- a/internal/testutil/pkgdata.go +++ b/internal/testutil/pkgdata.go @@ -1,7 +1,13 @@ package testutil import ( + "archive/tar" + "bytes" "encoding/base64" + "time" + + "github.com/blakesmith/ar" + "github.com/klauspost/compress/zstd" ) var PackageData = map[string][]byte{} @@ -150,3 +156,112 @@ BIsAGAJD+LGgIzDiD2ntudH6JU80wD0rklyRGBV2dEx8VJG5LVwpcXb8PI0lMdi+wwUPoBkQGEIN 7UwLQKMDjg8YkdmAmRrEgCSRRhMWBgBBSuI7S4BVs7zODfj1ja0TF+KbjdQDGANIcRlZhwzQSlla abj2Z7KoYMBXz9dwNNP2Aw13FguKkogezW5cqy4lCg== ` + +type TarEntry struct { + Header tar.Header + NoFixup bool + Content []byte +} + +var zeroTime time.Time +var epochStartTime time.Time = time.Unix(0, 0) + +func fixupTarEntry(entry *TarEntry) { + if entry.NoFixup { + return + } + hdr := &entry.Header + if hdr.Typeflag == 0 { + if hdr.Linkname != "" { + hdr.Typeflag = tar.TypeSymlink + } else if hdr.Name[len(hdr.Name)-1] == '/' { + hdr.Typeflag = tar.TypeDir + } else { + hdr.Typeflag = tar.TypeReg + } + } + if hdr.Mode == 0 { + switch hdr.Typeflag { + case tar.TypeDir: + hdr.Mode = 0755 + case tar.TypeSymlink: + hdr.Mode = 0777 + default: + hdr.Mode = 0644 + } + } + if hdr.Size == 0 && entry.Content != nil { + hdr.Size = int64(len(entry.Content)) + } + if hdr.Uid == 0 && hdr.Uname == "" { + hdr.Uname = "root" + } + if hdr.Gid == 0 && hdr.Gname == "" { + hdr.Gname = "root" + } + if hdr.ModTime == zeroTime { + hdr.ModTime = epochStartTime + } + if hdr.Format == 0 { + hdr.Format = tar.FormatGNU + } +} + +func makeTar(entries []TarEntry) ([]byte, error) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, entry := range entries { + fixupTarEntry(&entry) + if err := tw.WriteHeader(&entry.Header); err != nil { + return nil, err + } + if entry.Content != nil { + if _, err := tw.Write(entry.Content); err != nil { + return nil, err + } + } + } + return buf.Bytes(), nil +} + +func compressBytesZstd(input []byte) ([]byte, error) { + var buf bytes.Buffer + writer, err := zstd.NewWriter(&buf) + if _, err = writer.Write(input); err != nil { + return nil, err + } + if err = writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func MakeDeb(entries []TarEntry) ([]byte, error) { + var buf bytes.Buffer + + tarData, err := makeTar(entries) + if err != nil { + return nil, err + } + compTarData, err := compressBytesZstd(tarData) + if err != nil { + return nil, err + } + + writer := ar.NewWriter(&buf) + if err := writer.WriteGlobalHeader(); err != nil { + return nil, err + } + dataHeader := ar.Header{ + Name: "data.tar.zst", + Mode: 0644, + Size: int64(len(compTarData)), + } + if err := writer.WriteHeader(&dataHeader); err != nil { + return nil, err + } + if _, err = writer.Write(compTarData); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/internal/testutil/pkgdata_test.go b/internal/testutil/pkgdata_test.go new file mode 100644 index 00000000..53283b3a --- /dev/null +++ b/internal/testutil/pkgdata_test.go @@ -0,0 +1,372 @@ +package testutil_test + +import ( + "archive/tar" + "bytes" + "io" + "time" + + "github.com/blakesmith/ar" + "github.com/canonical/chisel/internal/testutil" + "github.com/klauspost/compress/zstd" + . "gopkg.in/check.v1" +) + +type pkgdataSuite struct{} + +var _ = Suite(&pkgdataSuite{}) + +type checkTarEntry struct { + tarEntry testutil.TarEntry + checkHeader tar.Header +} + +var epochStartTime time.Time = time.Unix(0, 0) + +var pkgdataCheckEntries = []checkTarEntry{{ + testutil.TarEntry{ + Header: tar.Header{ + Name: "./", + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./", + Mode: 00755, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./admin/", + Mode: 00700, + Uname: "admin", + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./admin/", + Mode: 00700, + Uname: "admin", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./admin/password", + Mode: 00600, + Uname: "admin", + }, + Content: []byte("swordf1sh"), + }, + tar.Header{ + Typeflag: tar.TypeReg, + Name: "./admin/password", + Size: 9, + Mode: 00600, + Uname: "admin", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./admin/setpassword", + Mode: 04711, + Uname: "admin", + }, + Content: []byte{0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01}, + }, + tar.Header{ + Typeflag: tar.TypeReg, + Name: "./admin/setpassword", + Size: 7, + Mode: 04711, + Uname: "admin", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./data/", + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./data/", + Mode: 00755, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./data/invoice.txt", + }, + Content: []byte("$ 10"), + }, + tar.Header{ + Typeflag: tar.TypeReg, + Name: "./data/invoice.txt", + Size: 4, + Mode: 00644, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./data/logs/", + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./data/logs/", + Mode: 00755, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./data/logs/task.log", + ModTime: time.Date(2022, 3, 1, 12, 0, 0, 0, time.Local), + }, + Content: []byte("starting\nfinished\n"), + }, + tar.Header{ + Typeflag: tar.TypeReg, + Name: "./data/logs/task.log", + Size: 18, + Mode: 00644, + Uname: "root", + Gname: "root", + ModTime: time.Date(2022, 3, 1, 12, 0, 0, 0, time.Local), + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./data/shared/", + Mode: 02777, + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./data/shared/", + Mode: 02777, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./home/", + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./home/", + Mode: 00755, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./home/alice/", + Uid: 1000, + Gid: 1000, + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./home/alice/", + Mode: 00755, + Uid: 1000, + Gid: 1000, + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./home/alice/notes", + Uid: 1000, + }, + Content: []byte("check the cat"), + }, + tar.Header{ + Typeflag: tar.TypeReg, + Name: "./home/alice/notes", + Size: 13, + Mode: 00644, + Uid: 1000, + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./home/bob/", + Uname: "bob", + Uid: 1001, + }, + }, + tar.Header{ + Typeflag: tar.TypeDir, + Name: "./home/bob/", + Mode: 00755, + Uid: 1001, + Uname: "bob", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./home/bob/task.sh", + Mode: 00700, + Uname: "bob", + Uid: 1001, + }, + Content: []byte("#!/bin/sh\n"), + }, + tar.Header{ + Typeflag: tar.TypeReg, + Name: "./home/bob/task.sh", + Size: 10, + Mode: 00700, + Uid: 1001, + Uname: "bob", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Name: "./logs/", + Linkname: "data/logs", + }, + }, + tar.Header{ + Typeflag: tar.TypeSymlink, + Name: "./logs/", + Linkname: "data/logs", + Mode: 00777, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Typeflag: tar.TypeFifo, + Name: "./pipe", + }, + }, + tar.Header{ + Typeflag: tar.TypeFifo, + Name: "./pipe", + Mode: 00644, + Uname: "root", + Gname: "root", + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}, { + testutil.TarEntry{ + Header: tar.Header{ + Typeflag: tar.TypeReg, + Name: "./restricted.txt", + Size: 3, + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, + Content: []byte("123"), + NoFixup: true, + }, + tar.Header{ + Typeflag: tar.TypeReg, + Name: "./restricted.txt", + Size: 3, + ModTime: epochStartTime, + Format: tar.FormatGNU, + }, +}} + +func (s *pkgdataSuite) TestMakeDeb(c *C) { + var size int64 + var err error + + inputEntries := make([]testutil.TarEntry, len(pkgdataCheckEntries)) + for i, checkEntry := range pkgdataCheckEntries { + inputEntries[i] = checkEntry.tarEntry + } + debBytes, err := testutil.MakeDeb(inputEntries) + c.Assert(err, IsNil) + + debBuf := bytes.NewBuffer(debBytes) + arReader := ar.NewReader(debBuf) + + arHeader, err := arReader.Next() + c.Assert(err, IsNil) + c.Assert(arHeader.Name, Equals, "data.tar.zst") + c.Assert(arHeader.Mode, Equals, int64(0644)) + c.Assert(int(arHeader.Size), testutil.IntGreaterThan, 0) + + var tarZstdBuf bytes.Buffer + size, err = io.Copy(&tarZstdBuf, arReader) + c.Assert(err, IsNil) + c.Assert(int(size), testutil.IntGreaterThan, 0) + + var tarBuf bytes.Buffer + zstdReader, err := zstd.NewReader(&tarZstdBuf) + size, err = zstdReader.WriteTo(&tarBuf) + c.Assert(err, IsNil) + c.Assert(int(size), testutil.IntGreaterThan, 0) + + tarReader := tar.NewReader(&tarBuf) + + for _, checkEntry := range pkgdataCheckEntries { + tarHeader, err := tarReader.Next() + c.Assert(err, IsNil) + c.Assert(*tarHeader, DeepEquals, checkEntry.checkHeader) + var dataBuf bytes.Buffer + size, err = io.Copy(&dataBuf, tarReader) + c.Assert(err, IsNil) + if checkEntry.tarEntry.Content != nil { + c.Assert(dataBuf.Bytes(), DeepEquals, checkEntry.tarEntry.Content) + } else { + c.Assert(int(size), Equals, 0) + } + } + + _, err = tarReader.Next() + c.Assert(err, Equals, io.EOF) + + _, err = arReader.Next() + c.Assert(err, Equals, io.EOF) +}