From 3c031a2bb451c13ef432626b78baf5ced5771d66 Mon Sep 17 00:00:00 2001 From: James Harris Date: Mon, 18 Nov 2024 11:18:15 +1000 Subject: [PATCH] Implement "blessing" to automatically update expectations on disk. --- internal/cliflags/doc.go | 3 + internal/cliflags/flags.go | 26 +++ internal/loader/builder.go | 2 +- internal/loader/content.go | 13 ++ internal/loader/error.go | 2 +- internal/loader/internal/loadertest/render.go | 2 +- internal/loader/markdownloader/ast.go | 16 +- internal/loader/markdownloader/loader.go | 6 +- internal/runner/bless.go | 200 ++++++++++++++++++ internal/runner/interface.go | 11 +- internal/runner/runner.go | 20 +- internal/runner/runner_test.go | 2 + internal/test/content.go | 11 + run.go | 40 +++- 14 files changed, 331 insertions(+), 23 deletions(-) create mode 100644 internal/cliflags/doc.go create mode 100644 internal/cliflags/flags.go create mode 100644 internal/runner/bless.go diff --git a/internal/cliflags/doc.go b/internal/cliflags/doc.go new file mode 100644 index 0000000..b45fac7 --- /dev/null +++ b/internal/cliflags/doc.go @@ -0,0 +1,3 @@ +// Package flags defines the command-line flags used by Aureus during test +// execution. +package cliflags diff --git a/internal/cliflags/flags.go b/internal/cliflags/flags.go new file mode 100644 index 0000000..53408c5 --- /dev/null +++ b/internal/cliflags/flags.go @@ -0,0 +1,26 @@ +package cliflags + +import ( + "flag" +) + +// Flags is a struct that holds all Aureus command-line flags. +type Flags struct { + Bless bool +} + +// Get returns the Aureus command-line flags. +func Get() Flags { + return flags +} + +var flags Flags + +func init() { + flag.BoolVar( + &flags.Bless, + "aureus.bless", + false, + "replace (on disk) each failing assertion's expected output with its current output", + ) +} diff --git a/internal/loader/builder.go b/internal/loader/builder.go index 786108d..69c75a3 100644 --- a/internal/loader/builder.go +++ b/internal/loader/builder.go @@ -71,7 +71,7 @@ func (b *TestBuilder) addContent(env ContentEnvelope) error { func (b *TestBuilder) addAnonymousContent(env ContentEnvelope) error { emit := func(in, out ContentEnvelope) { name := fmt.Sprintf("anonymous test on line %d", out.Line) - if out.Line == 0 { + if out.IsEntireFile() { name = fmt.Sprintf("anonymous test in %s", path.Base(out.File)) } diff --git a/internal/loader/content.go b/internal/loader/content.go index 3ad07d6..6e1a031 100644 --- a/internal/loader/content.go +++ b/internal/loader/content.go @@ -56,6 +56,12 @@ type ContentEnvelope struct { // the content represents the entire file. Line int + // The half-open range [Begin, End) is the section within the file that + // contains the content, given in bytes. + // + // If the range is [0, 0), the content represents the entire file. + Begin, End int64 + // Skip is a flag that indicates whether this content should be skipped when // running tests. Skip bool @@ -70,6 +76,8 @@ func (e ContentEnvelope) AsTestContent() test.Content { ContentMetaData: test.ContentMetaData{ File: e.File, Line: e.Line, + Begin: e.Begin, + End: e.End, Language: e.Content.Language, Attributes: e.Content.Attributes, }, @@ -77,6 +85,11 @@ func (e ContentEnvelope) AsTestContent() test.Content { } } +// IsEntireFile returns true if the content occupies the entire file. +func (e ContentEnvelope) IsEntireFile() bool { + return e.Begin == 0 && e.End == 0 +} + // SeparateContentByRole separates content into inputs and outputs. func SeparateContentByRole(content []ContentEnvelope) (inputs, outputs []ContentEnvelope) { for _, c := range content { diff --git a/internal/loader/error.go b/internal/loader/error.go index 4c235ac..1e4f8d4 100644 --- a/internal/loader/error.go +++ b/internal/loader/error.go @@ -32,7 +32,7 @@ func location(env ContentEnvelope, qualified bool) string { file = filepath.Base(file) } - if env.Line == 0 { + if env.IsEntireFile() { return file } diff --git a/internal/loader/internal/loadertest/render.go b/internal/loader/internal/loadertest/render.go index 6975f6f..5414a06 100644 --- a/internal/loader/internal/loadertest/render.go +++ b/internal/loader/internal/loadertest/render.go @@ -46,7 +46,7 @@ func renderContent(label string, c test.Content) []byte { w.WriteString(label) w.WriteByte(' ') - if c.Line == 0 { + if c.IsEntireFile() { fmt.Fprintf(&w, "%q", c.File) } else { fmt.Fprintf(&w, `"%s:%d"`, c.File, c.Line) diff --git a/internal/loader/markdownloader/ast.go b/internal/loader/markdownloader/ast.go index 3b45fb2..bbcdc71 100644 --- a/internal/loader/markdownloader/ast.go +++ b/internal/loader/markdownloader/ast.go @@ -11,8 +11,16 @@ func linesOf(n ast.Node, source []byte) string { return string(n.Lines().Value(source)) } -// lineNumberOf returns the first line number of n. -func lineNumberOf(n ast.Node, source []byte) int { - i := n.Lines().At(0).Start - return bytes.Count(source[:i], []byte("\n")) +var newline = []byte("\n") + +// locationOf returns the location of n within source. +func locationOf(n ast.Node, source []byte) (line, begin, end int) { + lines := n.Lines() + count := lines.Len() + + begin = lines.At(0).Start + end = lines.At(count - 1).Stop + line = bytes.Count(source[:begin], newline) + + return line, begin, end } diff --git a/internal/loader/markdownloader/loader.go b/internal/loader/markdownloader/loader.go index bac30b3..2001e87 100644 --- a/internal/loader/markdownloader/loader.go +++ b/internal/loader/markdownloader/loader.go @@ -169,10 +169,14 @@ func loadBlock( return err } + line, begin, end := locationOf(block, source) + return builder.AddContent( loader.ContentEnvelope{ File: filePath, - Line: lineNumberOf(block, source), + Line: line, + Begin: int64(begin), + End: int64(end), Skip: skip, Content: content, }, diff --git a/internal/runner/bless.go b/internal/runner/bless.go new file mode 100644 index 0000000..0641658 --- /dev/null +++ b/internal/runner/bless.go @@ -0,0 +1,200 @@ +package runner + +import ( + "fmt" + "io" + "os" + "regexp" + "strings" + + "github.com/dogmatiq/aureus/internal/test" +) + +// BlessStrategy is a strategy for blessing failed tests. +type BlessStrategy interface { + // bless updates the file containing an assertion's expected output to match + // its actual output. + // + // a is the assertion that failed. r is the file containing the blessed + // output that will replace the current expectation. + bless( + t LoggerT, + a test.Assertion, + r *os.File, + ) +} + +// BlessDisabled is a [BlessStrategy] that explicitly disables blessing of +// failed tests. +// +// It implies that the -aureus.bless flag on the command line is ignored. +type BlessDisabled struct{} + +func (*BlessDisabled) bless(LoggerT, test.Assertion, *os.File) {} + +// BlessAvailable is a [BlessStrategy] that instructs the user that blessing may +// be activated by using the -aureus.bless flag on the command line. +type BlessAvailable struct{} + +func (*BlessAvailable) bless( + t LoggerT, + _ test.Assertion, + _ *os.File, +) { + atoms := strings.Split(t.Name(), "/") + for i, atom := range atoms { + atoms[i] = "^" + regexp.QuoteMeta(atom) + "$" + } + pattern := strings.Join(atoms, "/") + + t.Helper() + t.Log(`To accept the current output as correct, re-run this test with the -aureus.bless flag:`) + t.Log(" ", fmt.Sprintf("go test -aureus.bless -run %q ./...", pattern)) +} + +// BlessEnabled is a [BlessStrategy] that explicitly enables blessing of failed +// tests. +type BlessEnabled struct{} + +func (s *BlessEnabled) bless( + t LoggerT, + a test.Assertion, + r *os.File, +) { + t.Helper() + + if err := edit(a, r); err != nil { + t.Log("Unable to bless output:", err) + } else { + t.Log(`The current output has been blessed. Future runs will consider this output correct.`) + } +} + +func edit(a test.Assertion, r *os.File) error { + // TODO: Tests are loaded using an [fs.FS], so the file system is not + // necessarily the host file system. + // + // Ultimately, we probably need to make it the loader's responsibility to + // bless tests, since it is the loader that knows where the tests came from. + + w, err := os.OpenFile(a.Output.File, os.O_RDWR, 0644) + if err != nil { + return fmt.Errorf("unable to open file containing expected output: %w", err) + } + defer w.Close() + + if a.Output.IsEntireFile() { + n, err := io.Copy(w, r) + if err != nil { + return err + } + return w.Truncate(n) + } + + if err := resize(a, r, w); err != nil { + return fmt.Errorf("unable to resize expected output: %w", err) + } + + if _, err := w.Seek(a.Output.Begin, io.SeekStart); err != nil { + return err + } + + _, err = io.Copy(w, r) + return err +} + +func resize(a test.Assertion, r, w *os.File) error { + sizeExpected := a.Output.End - a.Output.Begin + sizeActual, err := fileSize(r) + if err != nil { + return err + } + + if sizeExpected == sizeActual { + return nil + } + + sizeBefore, err := fileSize(w) + if err != nil { + return err + } + + sizeAfter := sizeBefore - sizeExpected + sizeActual + + op := shrink + if sizeAfter > sizeBefore { + op = grow + } + + return op( + w, + a.Output.End, + sizeBefore, + sizeAfter, + ) +} + +func shrink(w *os.File, pos, before, after int64) error { + delta := after - before + buf := make([]byte, 4096) + + for { + n, err := w.ReadAt(buf, pos) + if err != nil && err != io.EOF { + return err + } + + if n > 0 { + if _, err := w.WriteAt(buf[:n], pos+delta); err != nil { + return err + } + + pos += int64(n) + } + + if err == io.EOF { + return w.Truncate(after) + } + } +} + +func grow(w *os.File, pos, before, after int64) error { + delta := after - before + move := before - pos + 1 + size := min(move, 4096) + buf := make([]byte, size) + + n := move % size + if n == 0 { + n = size + } + + cursor := before - n + + // Move the rest in chunks of the full buffer size. + for cursor >= pos { + _, err := w.ReadAt(buf[:n], cursor) + if err != nil { + return err + } + + if _, err := w.WriteAt(buf[:n], cursor+delta); err != nil { + return err + } + + fmt.Printf(">> moved %d byte from %d to %d (%q)\n", size, cursor, cursor+delta, string(buf[:n])) + + cursor -= n + n = size + } + + return nil +} + +func fileSize(f *os.File) (int64, error) { + stat, err := f.Stat() + if err != nil { + return 0, fmt.Errorf("unable to determine file size of %s: %w", f.Name(), err) + } + return stat.Size(), nil +} diff --git a/internal/runner/interface.go b/internal/runner/interface.go index a2d622e..0fa854a 100644 --- a/internal/runner/interface.go +++ b/internal/runner/interface.go @@ -1,9 +1,16 @@ package runner -// TestingT is a constraint for types that are compatible with [testing.T]. -type TestingT[T any] interface { +// LoggerT is the subset of the [testing.TB] that supports logging only. +type LoggerT interface { Helper() + Name() string Log(...any) +} + +// TestingT is a constraint for types that are compatible with [testing.T]. +type TestingT[T any] interface { + LoggerT + SkipNow() Fail() Failed() bool diff --git a/internal/runner/runner.go b/internal/runner/runner.go index d5e1579..7be2152 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -16,6 +16,7 @@ import ( type Runner[T TestingT[T]] struct { GenerateOutput OutputGenerator[T] TrimSpace bool // TODO: make this a loader concern + BlessStrategy BlessStrategy } // Run makes the assertions described by all documents within a [TestSuite]. @@ -58,7 +59,7 @@ func (r *Runner[T]) assert(t T, a test.Assertion) { if err != nil { t.Log(err) t.Fail() - return // TODO: make stubbed fail panic + return } defer func() { f.Close() @@ -75,20 +76,27 @@ func (r *Runner[T]) assert(t T, a test.Assertion) { if err != nil { t.Log("unable to generate diff:", err) t.Fail() - return // TODO: make stubbed fail panic + return } if diff == nil { logSection(t, "OUTPUT", a.Output.Data, r.TrimSpace, location(a.Output)) - } else { - logSection(t, "OUTPUT DIFF", diff, r.TrimSpace) + return + } + + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Log("unable to rewind output file:", err) t.Fail() - return // TODO: make stubbed fail panic + return } + + logSection(t, "OUTPUT DIFF", diff, r.TrimSpace) + r.BlessStrategy.bless(t, a, f) + t.Fail() } func location(c test.Content) string { - if c.Line == 0 { + if c.IsEntireFile() { return c.File } return fmt.Sprintf("%s:%d", c.File, c.Line) diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index ddfb47e..b57c08b 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -28,6 +28,7 @@ func TestRunner(t *testing.T) { ) error { return prettyPrint(in, out) }, + BlessStrategy: &BlessAvailable{}, } runner.Run(t, tst) @@ -48,6 +49,7 @@ func TestRunner(t *testing.T) { ) error { return prettyPrint(in, out) }, + BlessStrategy: &BlessAvailable{}, } x := &testingT{T: t} diff --git a/internal/test/content.go b/internal/test/content.go index 12c656c..139096d 100644 --- a/internal/test/content.go +++ b/internal/test/content.go @@ -18,6 +18,12 @@ type ContentMetaData struct { // the content represents the entire file. Line int + // The half-open range [Begin, End) is the section within the file that + // contains the content, given in bytes. + // + // If the range is [0, 0), the content represents the entire file. + Begin, End int64 + // Language is the language of the content, if known, e.g. "json", "yaml", // etc. Content with an empty language is treated as plain text. Language string @@ -26,3 +32,8 @@ type ContentMetaData struct { // loader-specific information about the data. Attributes map[string]string } + +// IsEntireFile returns true if the content occupies the entire file. +func (m ContentMetaData) IsEntireFile() bool { + return m.Begin == 0 && m.End == 0 +} diff --git a/run.go b/run.go index a84f895..c5805d5 100644 --- a/run.go +++ b/run.go @@ -1,6 +1,7 @@ package aureus import ( + "github.com/dogmatiq/aureus/internal/cliflags" "github.com/dogmatiq/aureus/internal/loader/fileloader" "github.com/dogmatiq/aureus/internal/loader/markdownloader" "github.com/dogmatiq/aureus/internal/runner" @@ -33,10 +34,16 @@ func Run[T runner.TestingT[T]]( t.Helper() opts := runOptions{ - Dir: "./testdata", - Recursive: true, - TrimSpace: true, + Dir: "./testdata", + Recursive: true, + TrimSpace: true, + BlessStrategy: &runner.BlessAvailable{}, } + + if cliflags.Get().Bless { + Bless(true)(&opts) + } + for _, opt := range options { opt(&opts) } @@ -61,7 +68,8 @@ func Run[T runner.TestingT[T]]( GenerateOutput: func(t T, in runner.Input, out runner.Output) error { return g(t, in, out) }, - TrimSpace: opts.TrimSpace, + TrimSpace: opts.TrimSpace, + BlessStrategy: opts.BlessStrategy, } tests := test.Merge(fileTests, markdownTests) @@ -79,9 +87,10 @@ func Run[T runner.TestingT[T]]( type RunOption func(*runOptions) type runOptions struct { - Dir string - Recursive bool - TrimSpace bool + Dir string + Recursive bool + TrimSpace bool + BlessStrategy runner.BlessStrategy } // FromDir is a [RunOption] that sets the directory to search for tests. By @@ -107,3 +116,20 @@ func TrimSpace(on bool) RunOption { o.TrimSpace = on } } + +// Bless is a [RunOption] that enables or disables "blessing" of failed tests. +// +// If blessing is enabled, the file containing the expected output of each +// failed assertion is replaced with the actual output. +// +// By default blessing is disabled unless the -aureus.bless flag is set on the +// command line. +func Bless(on bool) RunOption { + return func(o *runOptions) { + if on { + o.BlessStrategy = &runner.BlessEnabled{} + } else { + o.BlessStrategy = &runner.BlessDisabled{} + } + } +}