PIP-1762: Functional testing library (#90)

.gitignore

@@ -30,3 +30,5 @@ mfput.log
 .idea/
 *.iml
 .DS_Store
+
+/jobs-cli

Makefile

@@ -0,0 +1,3 @@
+.PHONY: build
+build:
+	go build -v -o jobs-cli ./cmd/jobs-cli

functional/README.md

@@ -0,0 +1,56 @@
+# Functional Test Framework
+`go test`-like functional testing framework.
+
+## Why use `functional`?
+`functional` is suggested when you want to rapidly develop code in the format
+of a unit test or benchmark test using existing testing tools, but run it
+outside of the `go test` environment.  This is handy for use cases needing data
+inspection and having low tolerance for errors.
+
+`go test` doesn't support running tests programmatically from compiled code; it
+requires the source code, which won't/shouldn't be available in production.
+
+One such use case: runtime health check.  An admin may remotely invoke a health
+check job and watch it run.  `functional` can manage the health check logic
+once the RPC request is received.
+
+Tools like Testify may be used for assertions and mocking.
+
+## Run Tests
+```go
+import (
+	"context"
+	"time"
+
+	"github.com/mailgun/holster/v4/functional"
+)
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Minute)
+	defer cancel()
+
+	tests := []functional.TestFunc{
+		myTest1,
+	}
+	functional.RunSuite(ctx, "My suite", tests)
+}
+
+func myTest1(t *functional.T) {
+	t.Log("Hello World.")
+}
+```
+
+## Testify Assertions
+Testify is compatible with the functional testing framework as-is.
+
+```go
+import (
+	"github.com/mailgun/holster/v4/functional"
+	"github.com/stretchr/testify/require"
+)
+
+func myTest1(t *functional.T) {
+	retval := DoSomething()
+	require.Equal(t, "OK", retval)
+}
+```

functional/b.go

@@ -0,0 +1,139 @@
+package functional
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+)
+
+// Functional benchmark context.
+type B struct {
+	T
+	N int
+
+	// Mean nanoseconds per operation.
+	nsPerOp   float64
+	startTime time.Time
+}
+
+// Functional test code.
+type BenchmarkFunc func(b *B)
+
+type BenchmarkResult struct {
+	Pass bool
+	// Mean nanoseconds per operation.
+	NsPerOp float64
+}
+
+func newB(name string, times int, opts ...FunctionalOption) *B {
+	b := &B{
+		T: T{
+			name:      name,
+			writer:    os.Stdout,
+			errWriter: os.Stderr,
+		},
+		N: times,
+	}
+
+	for _, opt := range opts {
+		opt.Apply(&b.T)
+	}
+
+	return b
+}
+
+func (b *B) Run(name string, fn BenchmarkFunc, opts ...FunctionalOption) BenchmarkResult {
+	return b.RunTimes(name, fn, 1, opts...)
+}
+
+func (b *B) RunTimes(name string, fn BenchmarkFunc, times int, opts ...FunctionalOption) BenchmarkResult {
+	b2 := &B{
+		T: T{
+			name:      joinName(b.name, name),
+			indent:    b.indent + 1,
+			writer:    b.writer,
+			errWriter: b.errWriter,
+		},
+		N: times,
+	}
+
+	b2.invoke(b.T.ctx, fn)
+
+	if !b2.pass {
+		b.pass = false
+	}
+
+	return b.result()
+}
+
+func (b *B) ResetTimer() {
+	b.startTime = time.Now()
+}
+
+func (b *B) invoke(ctx context.Context, fn BenchmarkFunc) {
+	if ctx.Err() != nil {
+		panic(ctx.Err())
+	}
+
+	b.deadline = time.Now().Add(maxTimeout)
+	ctx, cancel := context.WithDeadline(ctx, b.deadline)
+	defer cancel()
+	b.ctx = ctx
+	b.pass = true
+	b.Logf("≈≈≈ RUN   %s", b.name)
+	b.startTime = time.Now()
+
+	func() {
+		defer func() {
+			// Handle panic.
+			if err := recover(); err != nil {
+				errMsg := fmt.Sprintf("%v", err)
+				if errMsg != "" {
+					log.WithField("test", b.name).Error(errMsg)
+				}
+				// TODO: Print stack trace.
+
+				b.pass = false
+			}
+		}()
+
+		fn(b)
+	}()
+
+	endTime := time.Now()
+	elapsed := endTime.Sub(b.startTime)
+	b.nsPerOp = float64(elapsed.Nanoseconds()) / float64(b.N)
+	nsPerOpDur := time.Duration(int64(b.nsPerOp))
+	b.Logf("%s\t%d\t%s ns/op (%s/op)", b.name, b.N, formatFloat(b.nsPerOp), nsPerOpDur.String())
+	if b.pass {
+		b.Logf("⁓⁓⁓ PASS: %s (%s)", b.name, elapsed)
+	} else {
+		b.Logf("⁓⁓⁓ FAIL: %s (%s)", b.name, elapsed)
+	}
+}
+
+func (b *B) result() BenchmarkResult {
+	return BenchmarkResult{
+		Pass:    b.pass,
+		NsPerOp: b.nsPerOp,
+	}
+}
+
+// Format float as human readable string with up to 5 decimal places.
+func formatFloat(d float64) string {
+	str := fmt.Sprintf("%0.5f", d)
+
+	// Strip insignificant zeros from right.
+	var i int
+	for i = len(str) - 1; i > 0; i-- {
+		if str[i] == '.' {
+			return str[0:i]
+		}
+		if str[i] != '0' {
+			return str[0 : i+1]
+		}
+	}
+
+	return str
+}

functional/functional.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2022 Mailgun Technologies Inc
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// `go test`-like functional testing framework.
+// Can be used with Testify require/assert/mock.
+package functional
+
+import (
+	"context"
+	"time"
+)
+
+// Run a test.  Test named after function name.
+func Run(ctx context.Context, fn TestFunc, opts ...FunctionalOption) bool {
+	name := funcName(fn)
+	t := newT(name, opts...)
+	t.invoke(ctx, fn)
+	return t.pass
+}
+
+// Run a test with user-provided name.
+func RunWithName(ctx context.Context, name string, fn TestFunc, opts ...FunctionalOption) bool {
+	t := newT(name, opts...)
+	t.invoke(ctx, fn)
+	return t.pass
+}
+
+// Run a suite of tests as a unit.
+// Generates summary when finished.
+func RunSuite(ctx context.Context, suiteName string, tests []TestFunc, opts ...FunctionalOption) bool {
+	result := map[bool]int{true: 0, false: 0}
+	numTests := len(tests)
+	t := newT(suiteName, opts...)
+	suiteStartTime := time.Now()
+
+	t.invoke(ctx, func(t *T) {
+		for _, test := range tests {
+			testName := funcName(test)
+			pass := t.Run(testName, test)
+			result[pass]++
+		}
+
+		suiteEndTime := time.Now()
+		pass := result[false] == 0
+		passPct := float64(result[true]) / float64(numTests) * 100
+		t.Log()
+		t.Log("Suite test result summary:")
+		t.Logf("    pass: %d (%0.1f%%)", result[true], passPct)
+		t.Logf("    fail: %d", result[false])
+		t.Logf("    elapsed: %s", suiteEndTime.Sub(suiteStartTime))
+
+		if !pass {
+			t.FailNow()
+		}
+	})
+
+	return t.pass
+}
+
+// Run a benchmark test.  Test named after function name.
+func RunBenchmarkTimes(ctx context.Context, fn BenchmarkFunc, times int, opts ...FunctionalOption) BenchmarkResult {
+	name := funcName(fn)
+	b := newB(name, times, opts...)
+	b.invoke(ctx, fn)
+	return b.result()
+}
+
+// Run a benchmark test with user-provided name.
+func RunBenchmarkWithNameTimes(ctx context.Context, name string, fn BenchmarkFunc, times int, opts ...FunctionalOption) BenchmarkResult {
+	b := newB(name, times, opts...)
+	b.invoke(ctx, fn)
+	return b.result()
+}
+
+// Run a suite of benchmark tests as a unit.
+// Run each benchmark n times.
+// Generates summary when finished.
+func RunBenchmarkSuiteTimes(ctx context.Context, suiteName string, times int, tests []BenchmarkFunc, opts ...FunctionalOption) bool {
+	result := map[bool]int{true: 0, false: 0}
+	numTests := len(tests)
+	b := newB(suiteName, 1, opts...)
+	suiteStartTime := time.Now()
+
+	b.invoke(ctx, func(b *B) {
+		for _, test := range tests {
+			testName := funcName(test)
+			bret := b.RunTimes(testName, test, times)
+			result[bret.Pass]++
+		}
+
+		suiteEndTime := time.Now()
+		pass := result[false] == 0
+		passPct := float64(result[true]) / float64(numTests) * 100
+		b.Log()
+		b.Log("Suite benchmark test result summary:")
+		b.Logf("    pass: %d (%0.1f%%)", result[true], passPct)
+		b.Logf("    fail: %d", result[false])
+		b.Logf("    elapsed: %s", suiteEndTime.Sub(suiteStartTime))
+
+		if !pass {
+			b.FailNow()
+		}
+	})
+
+	return b.pass
+}

functional/functional_option.go

@@ -0,0 +1,33 @@
+package functional
+
+import "io"
+
+type FunctionalOption interface {
+	Apply(t *T)
+}
+
+type withWriterOption struct {
+	writer io.Writer
+}
+
+func (o *withWriterOption) Apply(t *T) {
+	t.writer = o.writer
+	t.errWriter = o.writer
+}
+
+// WithWriter sets log output writer.
+func WithWriter(writer io.Writer) FunctionalOption {
+	return &withWriterOption{writer: writer}
+}
+
+type withArgs struct {
+	args []string
+}
+
+func (o *withArgs) Apply(t *T) {
+	t.args = o.args
+}
+
+func WithArgs(args ...string) FunctionalOption {
+	return &withArgs{args: args}
+}