feat: report mutated files (#131)

The paths with "mutable: true" property can be "mutated" with mutation
scripts. Thus, the files matching those paths may have a different size
and hash after the mutation scripts have been run. The report should
reflect these changes by updating the entries of the mutated paths.

Paths can also have the "until: mutate" property. This means that
those files are available until the mutation scripts have been run and
are not present in the final file system. Thus, they should not be part
of the report either.

This commit adds support for both -- it updates the changed properties
of mutated files and makes sure not to add report entries for
"until: mutate".

internal/fsutil/create.go

@@ -45,6 +45,7 @@ func Create(options *CreateOptions) (*Entry, error) {
 			return nil, err
 		}
 	}
+
 	switch o.Mode & fs.ModeType {
 	case 0:
 		err = createFile(o)
@@ -60,9 +61,13 @@ func Create(options *CreateOptions) (*Entry, error) {
 		return nil, err
 	}
 
+	s, err := os.Lstat(o.Path)
+	if err != nil {
+		return nil, err
+	}
 	entry := &Entry{
 		Path: o.Path,
-		Mode: o.Mode,
+		Mode: s.Mode(),
 		Hash: hash,
 		Size: rp.size,
 		Link: o.Link,

internal/fsutil/create_test.go

@@ -2,7 +2,6 @@ package fsutil_test
 
 import (
 	"bytes"
-	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -80,6 +79,20 @@ var createTests = []createTest{{
 		// mode is not updated.
 		"/foo/": "dir 0765",
 	},
+}, {
+	options: fsutil.CreateOptions{
+		Path: "foo",
+		// Mode should be ignored for existing entry.
+		Mode: 0644,
+		Data: bytes.NewBufferString("changed"),
+	},
+	hackdir: func(c *C, dir string) {
+		c.Assert(os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666), IsNil)
+	},
+	result: map[string]string{
+		// mode is not updated.
+		"/foo": "file 0666 d67e2e94",
+	},
 }}
 
 func (s *S) TestCreate(c *C) {
@@ -111,33 +124,12 @@ func (s *S) TestCreate(c *C) {
 		c.Assert(testutil.TreeDump(dir), DeepEquals, test.result)
 		// [fsutil.Create] does not return information about parent directories
 		// created implicitly. We only check for the requested path.
-		c.Assert(dumpFSEntry(entry, dir)[test.options.Path], DeepEquals, test.result[test.options.Path])
-	}
-}
-
-// dumpFSEntry returns the file entry in the same format as [testutil.TreeDump].
-func dumpFSEntry(fsEntry *fsutil.Entry, root string) map[string]string {
-	result := make(map[string]string)
-	path := strings.TrimPrefix(fsEntry.Path, root)
-	fperm := fsEntry.Mode.Perm()
-	if fsEntry.Mode&fs.ModeSticky != 0 {
-		fperm |= 01000
-	}
-	switch fsEntry.Mode.Type() {
-	case fs.ModeDir:
-		result[path+"/"] = fmt.Sprintf("dir %#o", fperm)
-	case fs.ModeSymlink:
-		result[path] = fmt.Sprintf("symlink %s", fsEntry.Link)
-	case 0: // Regular
-		var entry string
-		if fsEntry.Size == 0 {
-			entry = fmt.Sprintf("file %#o empty", fsEntry.Mode.Perm())
-		} else {
-			entry = fmt.Sprintf("file %#o %s", fperm, fsEntry.Hash[:8])
+		entry.Path = strings.TrimPrefix(entry.Path, dir)
+		// Add the slashes that TreeDump adds to the path.
+		slashPath := "/" + test.options.Path
+		if test.options.Mode.IsDir() {
+			slashPath = slashPath + "/"
 		}
-		result[path] = entry
-	default:
-		panic(fmt.Errorf("unknown file type %d: %s", fsEntry.Mode.Type(), path))
+		c.Assert(testutil.TreeDumpEntry(entry), DeepEquals, test.result[slashPath])
 	}
-	return result
 }

internal/scripts/scripts.go

@@ -1,13 +1,16 @@
 package scripts
 
 import (
-	"go.starlark.net/resolve"
-	"go.starlark.net/starlark"
-
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"go.starlark.net/resolve"
+	"go.starlark.net/starlark"
+
+	"github.com/canonical/chisel/internal/fsutil"
 )
 
 func init() {
@@ -33,6 +36,9 @@ type ContentValue struct {
 	RootDir    string
 	CheckRead  func(path string) error
 	CheckWrite func(path string) error
+	// OnWrite has to be called after a successful write with the entry resulting
+	// from the write.
+	OnWrite func(entry *fsutil.Entry) error
 }
 
 // Content starlark.Value interface
@@ -171,10 +177,18 @@ func (c *ContentValue) Write(thread *starlark.Thread, fn *starlark.Builtin, args
 
 	// No mode parameter for now as slices are supposed to list files
 	// explicitly instead.
-	err = os.WriteFile(fpath, fdata, 0644)
+	entry, err := fsutil.Create(&fsutil.CreateOptions{
+		Path: fpath,
+		Data: bytes.NewReader(fdata),
+		Mode: 0644,
+	})
 	if err != nil {
 		return nil, c.polishError(path, err)
 	}
+	err = c.OnWrite(entry)
+	if err != nil {
+		return nil, err
+	}
 	return starlark.None, nil
 }
 

internal/scripts/scripts_test.go

@@ -4,9 +4,11 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	. "gopkg.in/check.v1"
 
+	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/scripts"
 	"github.com/canonical/chisel/internal/testutil"
 )
@@ -17,6 +19,7 @@ type scriptsTest struct {
 	hackdir func(c *C, dir string)
 	script  string
 	result  map[string]string
+	mutated map[string]string
 	checkr  func(path string) error
 	checkw  func(path string) error
 	error   string
@@ -44,6 +47,10 @@ var scriptsTests = []scriptsTest{{
 		"/foo/file1.txt": "file 0644 5b41362b",
 		"/foo/file2.txt": "file 0644 d98cf53e",
 	},
+	mutated: map[string]string{
+		"/foo/file1.txt": "file 0644 5b41362b",
+		"/foo/file2.txt": "file 0644 d98cf53e",
+	},
 }, {
 	summary: "Read a file",
 	content: map[string]string{
@@ -59,6 +66,9 @@ var scriptsTests = []scriptsTest{{
 		"/foo/file1.txt": "file 0644 5b41362b",
 		"/foo/file2.txt": "file 0644 5b41362b",
 	},
+	mutated: map[string]string{
+		"/foo/file2.txt": "file 0644 5b41362b",
+	},
 }, {
 	summary: "List a directory",
 	content: map[string]string{
@@ -77,6 +87,53 @@ var scriptsTests = []scriptsTest{{
 		"/bar/":          "dir 0755",
 		"/bar/file3.txt": "file 0644 5b41362b",
 	},
+}, {
+	summary: "OnWrite is called for modified files only",
+	content: map[string]string{
+		"foo/file1.txt": `placeholder`,
+		"foo/file2.txt": `placeholder`,
+		// This file is not mutable, it cannot be written to.
+		"foo/file3.txt": `placeholder`,
+	},
+	script: `
+		content.write("/foo/file1.txt", "data1")
+		content.write("/foo/file2.txt", "data2")
+	`,
+	checkw: func(p string) error {
+		if p == "foo/file3.txt" {
+			return fmt.Errorf("no write: %s", p)
+		}
+		return nil
+	},
+	result: map[string]string{
+		"/foo/":          "dir 0755",
+		"/foo/file1.txt": "file 0644 5b41362b",
+		"/foo/file2.txt": "file 0644 d98cf53e",
+		"/foo/file3.txt": "file 0644 40978892",
+	},
+	mutated: map[string]string{
+		"/foo/file1.txt": "file 0644 5b41362b",
+		"/foo/file2.txt": "file 0644 d98cf53e",
+	},
+}, {
+	summary: "Mode is not changed when writing to a file",
+	content: map[string]string{
+		"foo/file1.txt": ``,
+		"foo/file2.txt": ``,
+	},
+	hackdir: func(c *C, dir string) {
+		fpath1 := filepath.Join(dir, "foo/file1.txt")
+		_ = os.Chmod(fpath1, 0744)
+	},
+	script: `
+		content.write("/foo/file1.txt", "data1")
+		content.write("/foo/file2.txt", "data2")
+	`,
+	result: map[string]string{
+		"/foo/":          "dir 0755",
+		"/foo/file1.txt": "file 0744 5b41362b",
+		"/foo/file2.txt": "file 0644 d98cf53e",
+	},
 }, {
 	summary: "Forbid relative paths",
 	content: map[string]string{
@@ -217,10 +274,17 @@ func (s *S) TestScripts(c *C) {
 			test.hackdir(c, rootDir)
 		}
 
+		mutatedFiles := map[string]string{}
 		content := &scripts.ContentValue{
 			RootDir:    rootDir,
 			CheckRead:  test.checkr,
 			CheckWrite: test.checkw,
+			OnWrite: func(entry *fsutil.Entry) error {
+				// Set relative path.
+				entry.Path = strings.TrimPrefix(entry.Path, rootDir)
+				mutatedFiles[entry.Path] = testutil.TreeDumpEntry(entry)
+				return nil
+			},
 		}
 		namespace := map[string]scripts.Value{
 			"content": content,
@@ -237,6 +301,10 @@ func (s *S) TestScripts(c *C) {
 		}
 
 		c.Assert(testutil.TreeDump(rootDir), DeepEquals, test.result)
+
+		if test.mutated != nil {
+			c.Assert(mutatedFiles, DeepEquals, test.mutated)
+		}
 	}
 }
 

internal/slicer/report.go

@@ -11,12 +11,13 @@ import (
 )
 
 type ReportEntry struct {
-	Path   string
-	Mode   fs.FileMode
-	Hash   string
-	Size   int
-	Slices map[*setup.Slice]bool
-	Link   string
+	Path      string
+	Mode      fs.FileMode
+	Hash      string
+	Size      int
+	Slices    map[*setup.Slice]bool
+	Link      string
+	FinalHash string
 }
 
 // Report holds the information about files and directories created when slicing
@@ -30,31 +31,32 @@ type Report struct {
 
 // NewReport returns an empty report for content that will be based at the
 // provided root path.
-func NewReport(root string) *Report {
-	return &Report{
+func NewReport(root string) (*Report, error) {
+	if !filepath.IsAbs(root) {
+		return nil, fmt.Errorf("cannot use relative path for report root: %q", root)
+	}
+	report := &Report{
 		Root:    filepath.Clean(root) + "/",
 		Entries: make(map[string]ReportEntry),
 	}
+	return report, nil
 }
 
 func (r *Report) Add(slice *setup.Slice, fsEntry *fsutil.Entry) error {
-	if !strings.HasPrefix(fsEntry.Path, r.Root) {
-		return fmt.Errorf("cannot add path %q outside of root %q", fsEntry.Path, r.Root)
-	}
-	relPath := filepath.Clean("/" + strings.TrimPrefix(fsEntry.Path, r.Root))
-	if fsEntry.Mode.IsDir() {
-		relPath = relPath + "/"
+	relPath, err := r.sanitizeAbsPath(fsEntry.Path, fsEntry.Mode.IsDir())
+	if err != nil {
+		return fmt.Errorf("cannot add path to report: %s", err)
 	}
 
 	if entry, ok := r.Entries[relPath]; ok {
 		if fsEntry.Mode != entry.Mode {
-			return fmt.Errorf("path %q reported twice with diverging mode: %q != %q", relPath, fsEntry.Mode, entry.Mode)
+			return fmt.Errorf("path %s reported twice with diverging mode: 0%03o != 0%03o", relPath, fsEntry.Mode, entry.Mode)
 		} else if fsEntry.Link != entry.Link {
-			return fmt.Errorf("path %q reported twice with diverging link: %q != %q", relPath, fsEntry.Link, entry.Link)
+			return fmt.Errorf("path %s reported twice with diverging link: %q != %q", relPath, fsEntry.Link, entry.Link)
 		} else if fsEntry.Size != entry.Size {
-			return fmt.Errorf("path %q reported twice with diverging size: %d != %d", relPath, fsEntry.Size, entry.Size)
+			return fmt.Errorf("path %s reported twice with diverging size: %d != %d", relPath, fsEntry.Size, entry.Size)
 		} else if fsEntry.Hash != entry.Hash {
-			return fmt.Errorf("path %q reported twice with diverging hash: %q != %q", relPath, fsEntry.Hash, entry.Hash)
+			return fmt.Errorf("path %s reported twice with diverging hash: %q != %q", relPath, fsEntry.Hash, entry.Hash)
 		}
 		entry.Slices[slice] = true
 		r.Entries[relPath] = entry
@@ -70,3 +72,38 @@ func (r *Report) Add(slice *setup.Slice, fsEntry *fsutil.Entry) error {
 	}
 	return nil
 }
+
+// Mutate updates the FinalHash and Size of an existing path entry.
+func (r *Report) Mutate(fsEntry *fsutil.Entry) error {
+	relPath, err := r.sanitizeAbsPath(fsEntry.Path, fsEntry.Mode.IsDir())
+	if err != nil {
+		return fmt.Errorf("cannot mutate path in report: %s", err)
+	}
+
+	entry, ok := r.Entries[relPath]
+	if !ok {
+		return fmt.Errorf("cannot mutate path in report: %s not previously added", relPath)
+	}
+	if entry.Mode.IsDir() {
+		return fmt.Errorf("cannot mutate path in report: %s is a directory", relPath)
+	}
+	if entry.Hash == fsEntry.Hash {
+		// Content has not changed, nothing to do.
+		return nil
+	}
+	entry.FinalHash = fsEntry.Hash
+	entry.Size = fsEntry.Size
+	r.Entries[relPath] = entry
+	return nil
+}
+
+func (r *Report) sanitizeAbsPath(path string, isDir bool) (relPath string, err error) {
+	if !strings.HasPrefix(path, r.Root) {
+		return "", fmt.Errorf("%s outside of root %s", path, r.Root)
+	}
+	relPath = filepath.Clean("/" + strings.TrimPrefix(path, r.Root))
+	if isDir {
+		relPath = relPath + "/"
+	}
+	return relPath, nil
+}