Skip to content

Commit

Permalink
Implement "blessing" to automatically update expectations on disk.
Browse files Browse the repository at this point in the history
  • Loading branch information
jmalloc committed Nov 18, 2024

Verified

This commit was signed with the committer’s verified signature.
jmalloc James Harris
1 parent 5464aae commit 3c031a2
Showing 14 changed files with 331 additions and 23 deletions.
3 changes: 3 additions & 0 deletions internal/cliflags/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
// Package flags defines the command-line flags used by Aureus during test
// execution.
package cliflags
26 changes: 26 additions & 0 deletions internal/cliflags/flags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package cliflags

import (
"flag"
)

// Flags is a struct that holds all Aureus command-line flags.
type Flags struct {
Bless bool
}

// Get returns the Aureus command-line flags.
func Get() Flags {
return flags
}

var flags Flags

func init() {
flag.BoolVar(
&flags.Bless,
"aureus.bless",
false,
"replace (on disk) each failing assertion's expected output with its current output",
)
}
2 changes: 1 addition & 1 deletion internal/loader/builder.go
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ func (b *TestBuilder) addContent(env ContentEnvelope) error {
func (b *TestBuilder) addAnonymousContent(env ContentEnvelope) error {
emit := func(in, out ContentEnvelope) {
name := fmt.Sprintf("anonymous test on line %d", out.Line)
if out.Line == 0 {
if out.IsEntireFile() {
name = fmt.Sprintf("anonymous test in %s", path.Base(out.File))
}

13 changes: 13 additions & 0 deletions internal/loader/content.go
Original file line number Diff line number Diff line change
@@ -56,6 +56,12 @@ type ContentEnvelope struct {
// the content represents the entire file.
Line int

// The half-open range [Begin, End) is the section within the file that
// contains the content, given in bytes.
//
// If the range is [0, 0), the content represents the entire file.
Begin, End int64

// Skip is a flag that indicates whether this content should be skipped when
// running tests.
Skip bool
@@ -70,13 +76,20 @@ func (e ContentEnvelope) AsTestContent() test.Content {
ContentMetaData: test.ContentMetaData{
File: e.File,
Line: e.Line,
Begin: e.Begin,
End: e.End,
Language: e.Content.Language,
Attributes: e.Content.Attributes,
},
Data: e.Content.Data,
}
}

// IsEntireFile returns true if the content occupies the entire file.
func (e ContentEnvelope) IsEntireFile() bool {
return e.Begin == 0 && e.End == 0
}

// SeparateContentByRole separates content into inputs and outputs.
func SeparateContentByRole(content []ContentEnvelope) (inputs, outputs []ContentEnvelope) {
for _, c := range content {
2 changes: 1 addition & 1 deletion internal/loader/error.go
Original file line number Diff line number Diff line change
@@ -32,7 +32,7 @@ func location(env ContentEnvelope, qualified bool) string {
file = filepath.Base(file)
}

if env.Line == 0 {
if env.IsEntireFile() {
return file
}

2 changes: 1 addition & 1 deletion internal/loader/internal/loadertest/render.go
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ func renderContent(label string, c test.Content) []byte {
w.WriteString(label)
w.WriteByte(' ')

if c.Line == 0 {
if c.IsEntireFile() {
fmt.Fprintf(&w, "%q", c.File)
} else {
fmt.Fprintf(&w, `"%s:%d"`, c.File, c.Line)
16 changes: 12 additions & 4 deletions internal/loader/markdownloader/ast.go
Original file line number Diff line number Diff line change
@@ -11,8 +11,16 @@ func linesOf(n ast.Node, source []byte) string {
return string(n.Lines().Value(source))
}

// lineNumberOf returns the first line number of n.
func lineNumberOf(n ast.Node, source []byte) int {
i := n.Lines().At(0).Start
return bytes.Count(source[:i], []byte("\n"))
var newline = []byte("\n")

// locationOf returns the location of n within source.
func locationOf(n ast.Node, source []byte) (line, begin, end int) {
lines := n.Lines()
count := lines.Len()

begin = lines.At(0).Start
end = lines.At(count - 1).Stop
line = bytes.Count(source[:begin], newline)

return line, begin, end
}
6 changes: 5 additions & 1 deletion internal/loader/markdownloader/loader.go
Original file line number Diff line number Diff line change
@@ -169,10 +169,14 @@ func loadBlock(
return err
}

line, begin, end := locationOf(block, source)

return builder.AddContent(
loader.ContentEnvelope{
File: filePath,
Line: lineNumberOf(block, source),
Line: line,
Begin: int64(begin),
End: int64(end),
Skip: skip,
Content: content,
},
200 changes: 200 additions & 0 deletions internal/runner/bless.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
package runner

import (
"fmt"
"io"
"os"
"regexp"
"strings"

"github.com/dogmatiq/aureus/internal/test"
)

// BlessStrategy is a strategy for blessing failed tests.
type BlessStrategy interface {
// bless updates the file containing an assertion's expected output to match
// its actual output.
//
// a is the assertion that failed. r is the file containing the blessed
// output that will replace the current expectation.
bless(
t LoggerT,
a test.Assertion,
r *os.File,
)
}

// BlessDisabled is a [BlessStrategy] that explicitly disables blessing of
// failed tests.
//
// It implies that the -aureus.bless flag on the command line is ignored.
type BlessDisabled struct{}

func (*BlessDisabled) bless(LoggerT, test.Assertion, *os.File) {}

// BlessAvailable is a [BlessStrategy] that instructs the user that blessing may
// be activated by using the -aureus.bless flag on the command line.
type BlessAvailable struct{}

func (*BlessAvailable) bless(
t LoggerT,
_ test.Assertion,
_ *os.File,
) {
atoms := strings.Split(t.Name(), "/")
for i, atom := range atoms {
atoms[i] = "^" + regexp.QuoteMeta(atom) + "$"
}
pattern := strings.Join(atoms, "/")

t.Helper()
t.Log(`To accept the current output as correct, re-run this test with the -aureus.bless flag:`)
t.Log(" ", fmt.Sprintf("go test -aureus.bless -run %q ./...", pattern))
}

// BlessEnabled is a [BlessStrategy] that explicitly enables blessing of failed
// tests.
type BlessEnabled struct{}

func (s *BlessEnabled) bless(
t LoggerT,
a test.Assertion,
r *os.File,
) {
t.Helper()

if err := edit(a, r); err != nil {
t.Log("Unable to bless output:", err)
} else {
t.Log(`The current output has been blessed. Future runs will consider this output correct.`)
}
}

func edit(a test.Assertion, r *os.File) error {
// TODO: Tests are loaded using an [fs.FS], so the file system is not
// necessarily the host file system.
//
// Ultimately, we probably need to make it the loader's responsibility to
// bless tests, since it is the loader that knows where the tests came from.

w, err := os.OpenFile(a.Output.File, os.O_RDWR, 0644)
if err != nil {
return fmt.Errorf("unable to open file containing expected output: %w", err)
}
defer w.Close()

if a.Output.IsEntireFile() {
n, err := io.Copy(w, r)
if err != nil {
return err
}
return w.Truncate(n)
}

if err := resize(a, r, w); err != nil {
return fmt.Errorf("unable to resize expected output: %w", err)
}

if _, err := w.Seek(a.Output.Begin, io.SeekStart); err != nil {
return err
}

_, err = io.Copy(w, r)
return err
}

func resize(a test.Assertion, r, w *os.File) error {
sizeExpected := a.Output.End - a.Output.Begin
sizeActual, err := fileSize(r)
if err != nil {
return err
}

if sizeExpected == sizeActual {
return nil
}

sizeBefore, err := fileSize(w)
if err != nil {
return err
}

sizeAfter := sizeBefore - sizeExpected + sizeActual

op := shrink
if sizeAfter > sizeBefore {
op = grow
}

return op(
w,
a.Output.End,
sizeBefore,
sizeAfter,
)
}

func shrink(w *os.File, pos, before, after int64) error {
delta := after - before
buf := make([]byte, 4096)

for {
n, err := w.ReadAt(buf, pos)
if err != nil && err != io.EOF {
return err
}

if n > 0 {
if _, err := w.WriteAt(buf[:n], pos+delta); err != nil {
return err
}

pos += int64(n)
}

if err == io.EOF {
return w.Truncate(after)
}
}
}

func grow(w *os.File, pos, before, after int64) error {
delta := after - before
move := before - pos + 1
size := min(move, 4096)
buf := make([]byte, size)

n := move % size
if n == 0 {
n = size
}

cursor := before - n

// Move the rest in chunks of the full buffer size.
for cursor >= pos {
_, err := w.ReadAt(buf[:n], cursor)
if err != nil {
return err
}

if _, err := w.WriteAt(buf[:n], cursor+delta); err != nil {
return err
}

fmt.Printf(">> moved %d byte from %d to %d (%q)\n", size, cursor, cursor+delta, string(buf[:n]))

cursor -= n
n = size
}

return nil
}

func fileSize(f *os.File) (int64, error) {
stat, err := f.Stat()
if err != nil {
return 0, fmt.Errorf("unable to determine file size of %s: %w", f.Name(), err)
}
return stat.Size(), nil
}
11 changes: 9 additions & 2 deletions internal/runner/interface.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
package runner

// TestingT is a constraint for types that are compatible with [testing.T].
type TestingT[T any] interface {
// LoggerT is the subset of the [testing.TB] that supports logging only.
type LoggerT interface {
Helper()
Name() string
Log(...any)
}

// TestingT is a constraint for types that are compatible with [testing.T].
type TestingT[T any] interface {
LoggerT

SkipNow()
Fail()
Failed() bool
20 changes: 14 additions & 6 deletions internal/runner/runner.go
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@ import (
type Runner[T TestingT[T]] struct {
GenerateOutput OutputGenerator[T]
TrimSpace bool // TODO: make this a loader concern
BlessStrategy BlessStrategy
}

// Run makes the assertions described by all documents within a [TestSuite].
@@ -58,7 +59,7 @@ func (r *Runner[T]) assert(t T, a test.Assertion) {
if err != nil {
t.Log(err)
t.Fail()
return // TODO: make stubbed fail panic
return
}
defer func() {
f.Close()
@@ -75,20 +76,27 @@ func (r *Runner[T]) assert(t T, a test.Assertion) {
if err != nil {
t.Log("unable to generate diff:", err)
t.Fail()
return // TODO: make stubbed fail panic
return
}

if diff == nil {
logSection(t, "OUTPUT", a.Output.Data, r.TrimSpace, location(a.Output))
} else {
logSection(t, "OUTPUT DIFF", diff, r.TrimSpace)
return
}

if _, err := f.Seek(0, io.SeekStart); err != nil {
t.Log("unable to rewind output file:", err)
t.Fail()
return // TODO: make stubbed fail panic
return
}

logSection(t, "OUTPUT DIFF", diff, r.TrimSpace)
r.BlessStrategy.bless(t, a, f)
t.Fail()
}

func location(c test.Content) string {
if c.Line == 0 {
if c.IsEntireFile() {
return c.File
}
return fmt.Sprintf("%s:%d", c.File, c.Line)
Loading

0 comments on commit 3c031a2

Please sign in to comment.