validate: warn if a date or time is in the future, fixes #8

Also add some CLI tests for various data type checks.

CONTRIBUTING.md

@@ -16,3 +16,14 @@ philosophy. There are many features that adif-multitool could add, but they
 should be done in harmony with the tool's philosophy. For new features or
 changes in behavior, please open an issue first to discuss the semantics of the
 feature. For straightforward bug fixes, a pull request by itself is sufficient.
+
+## Add tests
+
+It helps when bug fixes include a test case that fails without the fix.
+New functionality should be covered by automated tests to avoid regressions.
+Commands and many API functions have standard Go unit tests.  There are also
+some tests that exercise the command line, including flag syntax parsing and
+checking stderr, in the [txtar](https://pkg.go.dev/golang.org/x/tools/txtar)
+files [in the `adifmt` package](./adifmt/testdata).  These command language for
+these tests is described in the
+[testscript package](https://pkg.go.dev/github.com/rogpeppe/[email protected]/testscript).

README.md

@@ -645,12 +645,13 @@ specification allows some fields to have values which do not match the
 enumerated options, for example the `SUBMODE` field says “use enumeration values
 for interoperability” but the type is string, allowing any value.  These
 warnings will be printed to standard error with `adifmt validate` but will not
-block the logfile from being printed to standard output.
+block the logfile from being printed to standard output.  Dates and times in the
+future (based on the computer’s current wall clock) will print a warning; there
+is not currently a way to override the current time.
 
 The `--required-fields` option provides a list of fields which must be present in
 a valid record.  Multiple fields may be comma-separated or the option given
-several times.
-For example, checking a contest log might use
+several times.  For example, checking a contest log might use
 `adifmt validate --reqiured-fields qso_date,time_on,call,band,mode,srx_string`
 
 Some but not all validation errors can be corrected with [`adifmt fix`](#fix).

adif/spec/validate.go

@@ -77,7 +77,8 @@ var (
 )
 
 type ValidationContext struct {
-	UnknownEnumValueWarning bool // if true, values not in an enumeration are a warning, otherwise an error
+	UnknownEnumValueWarning bool      // if true, values not in an enumeration are a warning, otherwise an error
+	Now                     time.Time // comparison point for times-in-the-future checks
 	FieldValue              func(name string) string
 }
 
@@ -262,26 +263,47 @@ func ValidateDate(val string, f Field, ctx ValidationContext) Validation {
 	if d.Year() < 1930 {
 		return errorf("%s year before 1930 %q", f.Name, val)
 	}
+	if !ctx.Now.IsZero() && d.After(ctx.Now) {
+		return warningf("%s value %q later than today", f.Name, val)
+	}
 	return valid()
 }
 
 func ValidateTime(val string, f Field, ctx ValidationContext) Validation {
 	if !allNumeric.MatchString(val) {
 		return errorf("%s invalid time %q", f.Name, val)
 	}
+	var dateField, d string
+	if f.Name == TimeOnField.Name {
+		dateField = QsoDateField.Name
+	} else if f.Name == TimeOffField.Name {
+		dateField = QsoDateOffField.Name
+	}
+	if dateField != "" {
+		d = ctx.FieldValue(dateField)
+	}
+	var dtfmt = "20060102"
 	switch len(val) {
 	case 4:
 		_, err := time.Parse("1504", val)
 		if err != nil {
 			return errorf("%s time out of HH:MM range %q", f.Name, val)
 		}
+		dtfmt += "1504"
 	case 6:
 		_, err := time.Parse("150405", val)
 		if err != nil {
 			return errorf("%s time out of HH:MM:SS range %q", f.Name, val)
 		}
+		dtfmt += "150405"
 	default:
-		return errorf("%s not an 4- or 6-digit time %q", f.Name, val)
+		return errorf("%s not a 4- or 6-digit time %q", f.Name, val)
+	}
+	if d != "" && !ctx.Now.IsZero() {
+		t, err := time.ParseInLocation(dtfmt, d+val, time.UTC)
+		if err == nil && t.After(ctx.Now) && t.Truncate(24*time.Hour) == ctx.Now.Truncate(24*time.Hour) {
+			return warningf("%s time %q is later than now, %s=%q", f.Name, val, dateField, d)
+		}
 	}
 	return valid()
 }

adif/spec/validate_test.go

@@ -14,24 +14,27 @@
 
 package spec
 
-import "testing"
+import (
+	"testing"
+	"time"
+)
 
 type validateTest struct {
 	field Field
 	value string
 	want  Validity
 }
 
-var emptyCtx ValidationContext
+var emptyCtx = ValidationContext{FieldValue: func(name string) string { return "" }}
 
 func testValidator(t *testing.T, tc validateTest, ctx ValidationContext, funcname string) {
 	t.Helper()
 	v := TypeValidators[tc.field.Type.Name]
 	if got := v(tc.value, tc.field, ctx); got.Validity != tc.want {
 		if got.Validity == Valid {
-			t.Errorf("%s(%q, %s, ctx) got Valid, want %s", funcname, tc.value, tc.field.Name, tc.want)
+			t.Errorf("%s(%q, %q, ctx) got Valid, want %s", funcname, tc.value, tc.field.Name, tc.want)
 		} else {
-			t.Errorf("%s(%q, %s, ctx) want %s got %s %s", funcname, tc.value, tc.field.Name, tc.want, got.Validity, got.Message)
+			t.Errorf("%s(%q, %q, ctx) want %s got %s %s", funcname, tc.value, tc.field.Name, tc.want, got.Validity, got.Message)
 		}
 	}
 }
@@ -160,8 +163,8 @@ func TestValidateDate(t *testing.T) {
 		{field: QsoDateOffField, value: "20200317", want: Valid},
 		{field: QslrdateField, value: "19991231", want: Valid},
 		{field: QslsdateField, value: "20000229", want: Valid},
-		{field: QrzcomQsoUploadDateField, value: "21000101", want: Valid},
-		{field: LotwQslrdateField, value: "23450607", want: Valid},
+		{field: QrzcomQsoUploadDateField, value: "22000101", want: Valid}, // future date, ctx.Now is zero
+		{field: LotwQslrdateField, value: "23450607", want: Valid},        // future date, ctx.Now is zero
 		{field: QsoDateField, value: "19000101", want: InvalidError},
 		{field: QsoDateField, value: "19800100", want: InvalidError},
 		{field: QsoDateOffField, value: "202012", want: InvalidError},
@@ -201,6 +204,74 @@ func TestValidateTime(t *testing.T) {
 	}
 }
 
+func TestValidateDateRelative(t *testing.T) {
+	ctx := ValidationContext{
+		Now:        time.Date(2023, time.October, 31, 12, 34, 56, 0, time.UTC),
+		FieldValue: emptyCtx.FieldValue}
+	tests := []validateTest{
+		{field: QsoDateField, value: "19991231", want: Valid},
+		{field: QsoDateOffField, value: "20000101", want: Valid},
+		{field: QslrdateField, value: "20231030", want: Valid},
+		{field: QslsdateField, value: "20231031", want: Valid},
+		{field: QrzcomQsoUploadDateField, value: "20231101", want: InvalidWarning}, // future date
+		{field: LotwQslrdateField, value: "23450607", want: InvalidWarning},        // future date
+		{field: QsoDateField, value: "19000101", want: InvalidError},
+		{field: QsoDateField, value: "19800100", want: InvalidError},
+		{field: QsoDateOffField, value: "202012", want: InvalidError},
+		{field: QsoDateOffField, value: "21000229", want: InvalidError},
+		{field: QslrdateField, value: "1031", want: InvalidError},
+		{field: QslsdateField, value: "2001-02-03", want: InvalidError},
+		{field: QrzcomQsoUploadDateField, value: "01/02/2003", want: InvalidError},
+		{field: LotwQslrdateField, value: "01022003", want: InvalidError},
+		{field: LotwQslsdateField, value: "20220431", want: InvalidError},
+	}
+	for _, tc := range tests {
+		testValidator(t, tc, ctx, "ValidateDate")
+	}
+}
+
+func TestValidateTimeRelative(t *testing.T) {
+	now := time.Date(2023, time.October, 31, 12, 34, 56, 0, time.UTC)
+	yesterday := "20231030"
+	today := "20231031"
+	tomorrow := "20231101" // no warning, since the date field will warn
+	tests := []struct {
+		timeValue, dateValue string
+		want                 Validity
+	}{
+		{timeValue: "1516", dateValue: yesterday, want: Valid},
+		{timeValue: "235959", dateValue: yesterday, want: Valid},
+		{timeValue: "1234", dateValue: today, want: Valid},
+		{timeValue: "123456", dateValue: today, want: Valid},
+		{timeValue: "1314", dateValue: today, want: InvalidWarning},
+		{timeValue: "123457", dateValue: today, want: InvalidWarning},
+		{timeValue: "1235", dateValue: today, want: InvalidWarning},
+		{timeValue: "1516", dateValue: today, want: InvalidWarning},
+		{timeValue: "0123", dateValue: tomorrow, want: Valid},
+		{timeValue: "000000", dateValue: tomorrow, want: Valid},
+	}
+	tdFields := []struct{ tf, df Field }{
+		{tf: TimeOnField, df: QsoDateField}, {tf: TimeOffField, df: QsoDateOffField},
+	}
+	for _, tc := range tests {
+		for _, td := range tdFields {
+			ctx := ValidationContext{
+				Now: now,
+				FieldValue: func(name string) string {
+					if name == td.df.Name {
+						return tc.dateValue
+					}
+					if name == td.tf.Name {
+						return tc.timeValue
+					}
+					return ""
+				},
+			}
+			testValidator(t, validateTest{field: td.tf, value: tc.timeValue, want: tc.want}, ctx, "ValidateTime")
+		}
+	}
+}
+
 func TestValidateString(t *testing.T) {
 	tests := []validateTest{
 		{field: ProgramidField, value: "Log & Operate", want: Valid},

adifmt/testdata/validate_datetime_errors.txtar

@@ -0,0 +1,32 @@
+# tests that invalid dates and times are errors
+! adifmt validate -output csv input.csv
+cmp stderr golden.err
+! stdout .
+
+-- input.csv --
+CALL,QSO_DATE,TIME_ON,TIME_OFF,QSLSDATE
+K1A,2012,123,1,201206
+K2B,20221032,0860,123467,9870605
+K3C,2018-10-21,12:34,13:14:15,31/10/2018
+K4D,November 11 2011,11pm,1111am,11111988
+K5E,23450607,0000,0000,19291231
+-- golden.err --
+ERROR on input.csv record 1: QSO_DATE not an 8-digit date "2012"
+ERROR on input.csv record 1: TIME_ON not a 4- or 6-digit time "123"
+ERROR on input.csv record 1: TIME_OFF not a 4- or 6-digit time "1"
+ERROR on input.csv record 1: QSLSDATE not an 8-digit date "201206"
+ERROR on input.csv record 2: QSO_DATE invalid date "20221032"
+ERROR on input.csv record 2: TIME_ON time out of HH:MM range "0860"
+ERROR on input.csv record 2: TIME_OFF time out of HH:MM:SS range "123467"
+ERROR on input.csv record 2: QSLSDATE not an 8-digit date "9870605"
+ERROR on input.csv record 3: QSO_DATE invalid date "2018-10-21"
+ERROR on input.csv record 3: TIME_ON invalid time "12:34"
+ERROR on input.csv record 3: TIME_OFF invalid time "13:14:15"
+ERROR on input.csv record 3: QSLSDATE invalid date "31/10/2018"
+ERROR on input.csv record 4: QSO_DATE invalid date "November 11 2011"
+ERROR on input.csv record 4: TIME_ON invalid time "11pm"
+ERROR on input.csv record 4: TIME_OFF invalid time "1111am"
+ERROR on input.csv record 4: QSLSDATE invalid date "11111988"
+WARNING on input.csv record 5: QSO_DATE value "23450607" later than today
+ERROR on input.csv record 5: QSLSDATE year before 1930 "19291231"
+Error running validate: validate got 17 errors and 1 warnings

adifmt/testdata/validate_enum_errors.txtar

@@ -0,0 +1,18 @@
+# tests that certain unknown enum values are warnings, not errors
+! adifmt validate -output csv input.csv
+cmp stderr golden.err
+! stdout .
+
+-- input.csv --
+CALL,BAND,MODE,CONT,DXCC,STATE
+K1A,11m,AM,NA,291,VT
+K2A,20m,INVALID,NA,1,NY
+K3A,70CM,fm,XY,999,AB
+-- golden.err --
+ERROR on input.csv record 1: BAND unknown value "11m" for enumeration Band
+ERROR on input.csv record 2: MODE unknown value "INVALID" for enumeration Mode
+ERROR on input.csv record 2: STATE value "NY" is not valid for DXCC="1"
+ERROR on input.csv record 3: CONT unknown value "XY" for enumeration Continent
+ERROR on input.csv record 3: DXCC unknown value "999" for enumeration DXCC_Entity_Code
+WARNING on input.csv record 3: STATE has value "AB" but Primary_Administrative_Subdivision doesn't define any values for DXCC="999"
+Error running validate: validate got 5 errors and 1 warnings

adifmt/testdata/validate_enum_warnings.txtar

@@ -0,0 +1,17 @@
+# tests that certain unknown enum values are warnings, not errors
+adifmt validate -output csv input.csv
+cmp stderr golden.err
+cmp stdout input.csv
+
+-- input.csv --
+CALL,MODE,SUBMODE,CONTEST_ID,DXCC,STATE
+K1A,SSB,MSB,ADIF-INVALID-CONTEST,291,CT
+K2A,PSK,PSK123,ARRL-DIGI,,NJ
+3A0DX,CW,,,260,MO
+-- golden.err --
+WARNING on input.csv record 1: SUBMODE value "MSB" is not valid for MODE="SSB"
+WARNING on input.csv record 1: CONTEST_ID unknown value "ADIF-INVALID-CONTEST" for enumeration Contest_ID
+WARNING on input.csv record 2: SUBMODE value "PSK123" is not valid for MODE="PSK"
+WARNING on input.csv record 2: STATE has value "NJ" but DXCC is not set
+WARNING on input.csv record 3: STATE has value "MO" but Primary_Administrative_Subdivision doesn't define any values for DXCC="260"
+validate got 5 warnings

adifmt/testdata/validate_location_errors.txtar

@@ -0,0 +1,26 @@
+# tests that invalid locations are errors
+! adifmt validate -output csv input.csv
+cmp stderr golden.err
+! stdout .
+
+-- input.csv --
+CALL,LAT,LON,GRIDSQUARE
+K1A,12.345,34.567,AB12cd34
+K2B,N12 34.567,W123 45.678,ZY12ab
+K3C,S123 45.678,X23 45.678,AB0CD
+K4D,N12 98.765,E12 34.56789,A01CD23
+K5E,S12 12.34,W0 01.200,MN9876
+K6F,S001 02.340,W000 01.200,oo00
+-- golden.err --
+ERROR on input.csv record 1: LAT invalid location format, make sure to zero-pad "12.345"
+ERROR on input.csv record 1: LON invalid location format, make sure to zero-pad "34.567"
+ERROR on input.csv record 2: LAT invalid location format, make sure to zero-pad "N12 34.567"
+ERROR on input.csv record 3: LON invalid location format, make sure to zero-pad "X23 45.678"
+ERROR on input.csv record 3: GRIDSQUARE odd grid square length "AB0CD"
+ERROR on input.csv record 4: LAT invalid location format, make sure to zero-pad "N12 98.765"
+ERROR on input.csv record 4: LON invalid location format, make sure to zero-pad "E12 34.56789"
+ERROR on input.csv record 4: GRIDSQUARE odd grid square length "A01CD23"
+ERROR on input.csv record 5: LAT invalid location format, make sure to zero-pad "S12 12.34"
+ERROR on input.csv record 5: LON invalid location format, make sure to zero-pad "W0 01.200"
+ERROR on input.csv record 5: GRIDSQUARE non-letter in position 4 "MN9876"
+Error running validate: validate got 11 errors and 0 warnings

adifmt/testdata/validate_numbers_errors.txtar

@@ -0,0 +1,26 @@
+# tests that out-of-range or syntactically invalid numbers are errors
+! adifmt validate -output csv input.csv
+cmp stderr golden.err
+! stdout .
+
+-- input.csv --
+CALL,FREQ,CQZ,ITUZ,K_INDEX
+K1A,7.123.4,0,0,0
+K2A,-14.150,-1,-1,-1
+K3A,29,41,91,10
+K4A,1234567,40,90,9
+K5A,14.3,32.1,FF,4.0
+-- golden.err --
+ERROR on input.csv record 1: FREQ invalid decimal "7.123.4": strconv.ParseFloat: parsing "7.123.4": invalid syntax
+ERROR on input.csv record 1: CQZ value 0 below minimum 1
+ERROR on input.csv record 1: ITUZ value 0 below minimum 1
+ERROR on input.csv record 2: CQZ value -1 below minimum 1
+ERROR on input.csv record 2: ITUZ value -1 below minimum 1
+ERROR on input.csv record 2: K_INDEX value -1 below minimum 0
+ERROR on input.csv record 3: CQZ value 41 above maximum 40
+ERROR on input.csv record 3: ITUZ value 91 above maximum 90
+ERROR on input.csv record 3: K_INDEX value 10 above maximum 9
+ERROR on input.csv record 5: CQZ invalid integer "32.1"
+ERROR on input.csv record 5: ITUZ invalid number "FF"
+ERROR on input.csv record 5: K_INDEX invalid integer "4.0"
+Error running validate: validate got 12 errors and 0 warnings

cmd/validate.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/flwyd/adif-multitool/adif"
 	"github.com/flwyd/adif-multitool/adif/spec"
@@ -36,6 +37,7 @@ func helpValidate() string {
 
 func runValidate(ctx *Context, args []string) error {
 	cctx := ctx.CommandCtx.(*ValidateContext)
+	now := time.Now().UTC() // consistent for the whole log
 	log := os.Stderr
 	var errors, warnings int
 	appFields := make(map[string]adif.DataType)
@@ -48,10 +50,12 @@ func runValidate(ctx *Context, args []string) error {
 		}
 		updateFieldOrder(out, l.FieldOrder)
 		for i, r := range l.Records {
-			vctx := spec.ValidationContext{FieldValue: func(name string) string {
-				f, _ := r.Get(name)
-				return f.Value
-			}}
+			vctx := spec.ValidationContext{
+				Now: now,
+				FieldValue: func(name string) string {
+					f, _ := r.Get(name)
+					return f.Value
+				}}
 			var msgs []string
 			missing := make([]string, 0)
 			for _, x := range cctx.RequiredFields {

cmd/validate_test.go

@@ -22,6 +22,8 @@ import (
 	"github.com/google/go-cmp/cmp"
 )
 
+// Warnings (which are printed to stderr) are tested from adifmt/cli_test.go
+
 func TestValidateEmpty(t *testing.T) {
 	io := adif.NewADIIO()
 	out := &bytes.Buffer{}
@@ -167,5 +169,3 @@ func TestValidateErrors(t *testing.T) {
 		})
 	}
 }
-
-// TODO test warnings (which are printed to stderr)