diff --git a/README.md b/README.md index 3dd4e0a..82b2ccc 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,125 @@ adifmt fix log1.adi \ creates a file named `minimal.csv` with just the date, time, and callsign from each record in the input file `log1.adi`. +## Practical examples + +The following examples transform data into a format expected by a particular +program or ham radio activity. For details on how they work, see documentation +for individual commands or formats below. For contest log examples, see the +[Cabrillo](#cabrillo) section. Contributions of useful pipelines are welcome. + +### Add station and location to a log + +This example uses `edit` to add several fields to a POTA log saved in CSV +format. It then uses `fix` to remove the `:` from the time, transform the +decimal (GPS) latitude and longitude to ADIF sexagesimal format and transforms +`USA` and `CAN` country abbreviations to DXCC entity numbers. `flatten` makes +two copies of each record, one for park `US-0791` and one for park `US-4567`. +`infer` then sets the band from the frequency, grid square (Maidenhead locator) +based on the latitude and longitude, `STATION_CALLSIGN` field to the `OPERATOR` +field, and `SIG` and `SIG_INFO` from the `POTA_REF` field. (POTA doesn't +require the country or latitude/longitude fields; they're included for +illustration.) The input log might look like this: + +```csv +TIME_ON,FREQ,MODE,CALL,STATE,COUNTRY +12:34,7.012,CW,W1AW,CT,USA +12:56,14.234,SSB,VA1XYZ,NS,CAN +``` + +```sh +adiifmt edit mylog.csv \ + --add qso_date=20240704 \ + --add operator=WT0RJ \ + --add my_pota_ref=US-0791,US-4567 \ + --add my_state=DC --add my_country=USA \ + --add my_lat=38.899736 --add my_lon=-77.063331 \ +| adifmt fix \ +| adifmt flatten --fields pota_ref,my_pota_ref \ +| adifmt infer --fields band,my_gridsquare,station_callsign +``` + +### ADIF to SOTA CSV + +This example uses `find` to filter out any records which don’t have `SOTA_REF` +or `MY_SOTA_REF` fields, `edit` to add a `V2` field to each record (required by +the SOTA uploader), `select` to output only the fields expected by the SOTA +uploader and in the right order, `validate` to ensure fields are present and +correctly formatted, and `save --csv-omit-header` to create a file with just +the records, no file header. If your log lacks frequencies, replace the `freq` +field with `band`. (Note that the SOTA uploader now accepts ADIF files, so you +could just use the `find` command and upload directly. This example may be +useful if the data need to be further transformed or imported by a SOTA data +analysis program.) + +```sh +SOTA_CSV_ORDER=version,station_callsign,my_sota_ref,qso_date,time_on,freq,mode,call,sota_ref,comment +adifmt find mylog.adi --if-not 'my_sota_ref=' --or-if-not 'sota_ref=' \ +| adifmt edit --set version=V2 \ +| adifmt select --fields $SOTA_CSV_ORDER \ +| adifmt validate --required-fields station_callsign,qso_date,time_on,freq,mode,call \ +| adifmt save --csv-omit-header --field-order $SOTA_CSV_ORDER sotalog.csv +``` + +The variable assignment syntax for `SOTA_CSV_ORDER` works on Mac and Linux. On +Windows PowerShell assign the variable as `$SOTA_CSV_ORDER = +version,station_callsign,...`. On Windows cmd.exe, assign it as +`set SOTA_CSV_ORDER = version,station_callsign,...` and reference it as +`%SOTA_CSV_ORDER%` rather than the `$` prefix. + +### Filter WARC bands + +This example uses `infer` to set the band from the frequency if the former +isn’t set. It then uses `find` to filter out any contacts made on the +[WARC bands](https://en.wikipedia.org/wiki/WARC_bands): 12, 17, 30, and 60 +meters. Contesting is not allowed on those bands, so this is useful when +preparing a contest submission from a general station log where contacts may +have been made on bands not part of the contest. + +```sh +adifmt infer --fields band mylog.adi \ +| adifmt find --if-not 'band=60m|30m|17m|12m' +``` + +### Set mode from frequency (U.S. band plan) + +This example sets the mode and submode based on the frequency, according to the +U.S. band plan. It assumes that CW and SSB are the only modes in use (no FM, +AM, or digital), but can be extended if there are frequency ranges that you use +exclusively for one mode. `edit --add` will not overwrite the mode if it +already has a value (`edit --set` would force the new value). + +```sh +# US HF SSB (overlaps SSTV & AM) 80m: 3.6:4, 40m: 7.125:7.3, 20m:14.15:14.35, +# 17m:18.11:18.168, 15m:21.2:21.45, 12m:24.93 to 24.99, 10m: 28.3:29 +# 6m 50.1:50.3 is CW/SSB, SSB calling 60.125, assume 60.120+ is SSB +adifmt edit --if 'freq>3.6' --if 'freq<4' \ + --or-if 'freq>7.125' --if 'freq<=7.3' \ + --or-if 'freq>=14.15' --if 'freq<14.35' \ + --or-if 'freq>=18.11' --if 'freq<18.168' \ + --or-if 'freq>=21.2' --if 'freq<21.45' \ + --or-if 'freq>=24.93' --if 'freq<24.99' \ + --or-if 'freq>=28.3' --if 'freq<29' \ + --or-if 'freq>=50.12' --if 'freq<50.3' \ + --add mode=SSB |\ + +# US HF CW (some digital could occur) low end of the band, below FT8 and friends +adifmt edit --if 'freq>3.5' --if 'freq<3.7' \ + --or-if 'freq>7' --if 'freq<7.07' \ + --or-if 'freq>10.1' --if 'freq<10.3' \ + --or-if 'freq>14' --if 'freq<14.07' \ + --or-if 'freq>18.068' --if 'freq<18.1' \ + --or-if 'freq>21' --if 'freq<21.07' \ + --or-if 'freq>24.89' --if 'freq<14.91' \ + --or-if 'freq>28' --if 'freq<28.07' \ + --or-if 'freq>50.1' --if 'freq<50.12' \ + --add mode=CW |\ + +# SSB is usually LSB on 40m and below except 60m, USB on 20m and above +adifmt edit --if mode=SSB --if 'freq<8' --if-not band=60m --add submode=LSB |\ +adifmt edit --if mode=SSB --if 'freq>=14' --or-if band=60m --add submode=USB +``` + ## Features ### Input/Output formats @@ -559,7 +678,7 @@ and a change: ```sh adifmt cat mylog.adi \ - | adifmt edit --if 'mode=SSB' --if 'band>=20m' --add 'submode=USB' \ + | adifmt edit --if 'mode=SSB' --if 'band>=20m' --or-if 'band=60m' --add 'submode=USB' \ | adifmt edit --if 'mode=SSB' --if 'band=40m|80m|160m' --add 'submode=LSB' \ | adifmt save fixed_sideband.adi ``` @@ -613,8 +732,8 @@ contacts on the border of a square as separate: ```sh adifmt flatten --fields VUCC_GRIDS --output tsv \ - | adifmt select --fields VUCC_GRIDS --output tsv \ - | tail +2 | sort | uniq -c + | adifmt select --fields VUCC_GRIDS --output tsv --tsv-omit-header \ + | sort | uniq -c ``` The `flatten` command will turn @@ -762,8 +881,8 @@ find duplicate QSOs by date, band, and mode, use [uniq](https://man7.org/linux/man-pages/man1/uniq.1.html): ```sh -adifmt select --fields call,qso_date,band,mode --output tsv mylog.adi \ - | tail +2 | sort | uniq -d +adifmt select --fields call,qso_date,band,mode --output tsv --tsv-omit-header mylog.adi \ + | sort | uniq -d ``` This is similar to a SQL `SELECT` clause, except it cannot (yet?) transform the @@ -846,6 +965,7 @@ Features I plan to add: the same callsign on the same band with the same mode on the same Zulu day and the same `MY_SIG_INFO` value. * Option for `save` to append records to an existing ADIF file. +* [FLE (fast log entry)](https://df3cb.com/fle/documentation/) format support. * Count the total number of records or the number of distinct values of a field. (The total number of records can currently be counted with `--output=tsv --tsv-omit-header` and piping the output to `wc -l`.) This diff --git a/adifmt/main.go b/adifmt/main.go index 1e6c253..b1dd992 100644 --- a/adifmt/main.go +++ b/adifmt/main.go @@ -146,6 +146,7 @@ func buildContext(fs *flag.FlagSet, prepare func(l *adif.Logfile)) *cmd.Context // General flags fmtopts := "options: " + strings.Join(adif.FormatNames(), ", ") + fs.Var(&ctx.FieldOrder, "field-order", "Comma-separated `field` order for output (repeatable)") fs.Var(&ctx.InputFormat, "input", "input `format` when it cannot be inferred from file extension\n"+fmtopts) fs.Var(&ctx.OutputFormat, "output", diff --git a/adifmt/testdata/sota_csv.txtar b/adifmt/testdata/sota_csv.txtar new file mode 100644 index 0000000..33fe427 --- /dev/null +++ b/adifmt/testdata/sota_csv.txtar @@ -0,0 +1,33 @@ +# ADI to CSV with field order matching the SOTA uploader expectations. +# This pipeline is an example in README.md +exec adifmt find mylog.adi --if-not 'my_sota_ref=' --or-if-not 'sota_ref=' +! stderr . +stdin stdout +exec adifmt edit --set version=V2 +! stderr . +stdin stdout +exec adifmt select --fields version,station_callsign,my_sota_ref,qso_date,time_on,freq,mode,call,sota_ref,comment +! stderr . +stdin stdout +exec adifmt validate --required-fields station_callsign,qso_date,time_on,freq,mode,call +! stderr . +stdin stdout +exec adifmt save --csv-omit-header --field-order version,station_callsign,my_sota_ref,qso_date,time_on,freq,mode,call,sota_ref,comment sotalog.csv +cmp sotalog.csv expected.csv +! stdout . +stderr 'Wrote 3 records to sotalog.csv' + +-- mylog.adi -- +3.1.4 23450607 080910 adifmt (devel) +SOTA activator +20200101 0111 FM 2m 146.52 K1A CT W1AW W1/MB-009 MA Good signal, clear audio +Summit-to-summit +20210202 0222 CW 21.0123 15m W2B CA W1AW 479 559 W6/SN-001 W4C/CM-009 NC +Not a SOTA contact +20220303 0333 SSB 40m 7.200 K3C PA W1AW CT +SOTA chaser +20230404 0444 FT8 14.074 W4D/9H W1AW -6 -10 9H/MA-001 CT +-- expected.csv -- +V2,W1AW,W1/MB-009,20200101,0111,146.52,FM,K1A,,"Good signal, clear audio" +V2,W1AW,W4C/CM-009,20210202,0222,21.0123,CW,W2B,W6/SN-001, +V2,W1AW,,20230404,0444,14.074,FT8,W4D/9H,9H/MA-001, diff --git a/cmd/cat.go b/cmd/cat.go index 01ff5bc..9f2a07f 100644 --- a/cmd/cat.go +++ b/cmd/cat.go @@ -14,15 +14,14 @@ package cmd -import ( - "github.com/flwyd/adif-multitool/adif" -) - var Cat = Command{Name: "cat", Run: runCat, Description: "Concatenate all input files to standard output"} func runCat(ctx *Context, args []string) error { - acc := accumulator{Out: adif.NewLogfile(), Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { diff --git a/cmd/context.go b/cmd/context.go index 14fe0c0..dff75a4 100644 --- a/cmd/context.go +++ b/cmd/context.go @@ -35,6 +35,7 @@ type Context struct { Out io.Writer Locale language.Tag CommandCtx any + FieldOrder FieldList UserdefFields UserdefFieldList SuppressAppHeaders bool Prepare func(*adif.Logfile) diff --git a/cmd/edit.go b/cmd/edit.go index 4092efd..eccd2d0 100644 --- a/cmd/edit.go +++ b/cmd/edit.go @@ -88,18 +88,20 @@ func runEdit(ctx *Context, args []string) error { toTz := cctx.ToZone.Get() adjustTz := fromTz.String() != toTz.String() cond := cctx.Cond.Get() - out := adif.NewLogfile() - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { return err } - updateFieldOrder(out, l.FieldOrder) + updateFieldOrder(acc.Out, l.FieldOrder) for _, r := range l.Records { eval := recordEvalContext{record: r, lang: ctx.Locale} if !cond.Evaluate(eval) { - out.AddRecord(r) // edit condition doesn't match, pass through + acc.Out.AddRecord(r) // edit condition doesn't match, pass through continue } seen := make(map[string]string) @@ -155,14 +157,14 @@ func runEdit(ctx *Context, args []string) error { return fmt.Errorf("could not adjust time zone: %w", err) } } - out.AddRecord(rec) + acc.Out.AddRecord(rec) } } } if err := acc.prepare(); err != nil { return err } - return write(ctx, out) + return write(ctx, acc.Out) } func adjustTimeZone(r *adif.Record, from, to *time.Location) error { diff --git a/cmd/find.go b/cmd/find.go index b87b7c0..da9d55d 100644 --- a/cmd/find.go +++ b/cmd/find.go @@ -14,10 +14,6 @@ package cmd -import ( - "github.com/flwyd/adif-multitool/adif" -) - var Find = Command{Name: "find", Run: runFind, Help: helpFind, Description: "Include only records matching a condition"} @@ -56,23 +52,25 @@ Use quotes so operators are not treated as special shell characters: func runFind(ctx *Context, args []string) error { cctx := ctx.CommandCtx.(*FindContext) cond := cctx.Cond.Get() - out := adif.NewLogfile() - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { return err } - updateFieldOrder(out, l.FieldOrder) + updateFieldOrder(acc.Out, l.FieldOrder) for _, r := range l.Records { eval := recordEvalContext{record: r, lang: ctx.Locale} if cond.Evaluate(eval) { - out.AddRecord(r) + acc.Out.AddRecord(r) } } } if err := acc.prepare(); err != nil { return err } - return write(ctx, out) + return write(ctx, acc.Out) } diff --git a/cmd/fix.go b/cmd/fix.go index fbb9aa9..b73cd85 100644 --- a/cmd/fix.go +++ b/cmd/fix.go @@ -42,32 +42,33 @@ func helpFix() string { } func runFix(ctx *Context, args []string) error { - // TODO add any needed flags - out := adif.NewLogfile() - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { return err } - updateFieldOrder(out, l.FieldOrder) + updateFieldOrder(acc.Out, l.FieldOrder) for _, rec := range l.Records { - out.AddRecord(fixRecord(rec, l)) + acc.Out.AddRecord(fixRecord(rec, l)) } } if err := acc.prepare(); err != nil { return err } // fix again in case userdef fields were added - for _, r := range out.Records { + for _, r := range acc.Out.Records { for _, f := range r.Fields() { - ff := fixField(f, r, out) + ff := fixField(f, r, acc.Out) if f != ff { r.Set(ff) } } } - return write(ctx, out) + return write(ctx, acc.Out) } func fixRecord(r *adif.Record, l *adif.Logfile) *adif.Record { diff --git a/cmd/flatten.go b/cmd/flatten.go index 7398f60..43f81df 100644 --- a/cmd/flatten.go +++ b/cmd/flatten.go @@ -60,8 +60,10 @@ func runFlatten(ctx *Context, args []string) error { delims[n] = d } - out := adif.NewLogfile() - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { @@ -90,14 +92,14 @@ func runFlatten(ctx *Context, args []string) error { } } for _, e := range expn { - out.AddRecord(e) + acc.Out.AddRecord(e) } } } if err := acc.prepare(); err != nil { return err } - return write(ctx, out) + return write(ctx, acc.Out) } var typeDelims = map[spec.DataType]string{ diff --git a/cmd/infer.go b/cmd/infer.go index 72d547a..96ae2a5 100644 --- a/cmd/infer.go +++ b/cmd/infer.go @@ -130,14 +130,16 @@ func runInfer(ctx *Context, args []string) error { return fmt.Errorf("don't know how to infer field %s\n%s", todo[i], helpInfer()) } } - out := adif.NewLogfile() - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { return err } - updateFieldOrder(out, l.FieldOrder) + updateFieldOrder(acc.Out, l.FieldOrder) for _, r := range l.Records { did := make([]string, 0, len(todo)) for _, t := range todo { @@ -157,13 +159,13 @@ func runInfer(ctx *Context, args []string) error { r.SetComment(r.GetComment() + "\n" + c) } } - out.AddRecord(r) + acc.Out.AddRecord(r) } } if err := acc.prepare(); err != nil { return err } - return write(ctx, out) + return write(ctx, acc.Out) } func inferBand(r *adif.Record, name string) bool { diff --git a/cmd/io.go b/cmd/io.go index 6f4d108..cc7ce79 100644 --- a/cmd/io.go +++ b/cmd/io.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/flwyd/adif-multitool/adif" + "golang.org/x/exp/slices" ) func write(ctx *Context, l *adif.Logfile) error { @@ -146,6 +147,17 @@ type accumulator struct { comments []string } +func newAccumulator(c *Context) (*accumulator, error) { + a := accumulator{Out: adif.NewLogfile(), Ctx: c} + a.Out.FieldOrder = slices.Clone(c.FieldOrder) + for _, u := range c.UserdefFields { + if err := a.Out.AddUserdef(u); err != nil { + return nil, err + } + } + return &a, nil +} + func (a *accumulator) read(filename string) (*adif.Logfile, error) { l, err := readFile(a.Ctx, filename) if err != nil { diff --git a/cmd/save.go b/cmd/save.go index 1b4d3ed..b894d0d 100644 --- a/cmd/save.go +++ b/cmd/save.go @@ -26,6 +26,7 @@ import ( "github.com/flwyd/adif-multitool/adif" "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) var Save = Command{Name: "save", Run: runSave, Help: helpSave, @@ -40,7 +41,7 @@ type SaveContext struct { func helpSave() string { return `Unless options are set explicitly, existing files will not be overwritten and -logfiles withoutout any records will not be saved (useful if validate failed). +logfiles without any records will not be saved (useful if validate failed). File name may be a template with {FIELD} placeholders replaced by field values. For example, '{QSO_DATE}_{BAND}.adi' will create a separate file for each @@ -75,6 +76,11 @@ func runSave(ctx *Context, args []string) error { if err != nil { return err } + if len(ctx.FieldOrder) > 0 { + ro := l.FieldOrder + l.FieldOrder = slices.Clone(ctx.FieldOrder) + updateFieldOrder(l, ro) + } for _, u := range ctx.UserdefFields { l.AddUserdef(u) } @@ -130,7 +136,9 @@ func runSave(ctx *Context, args []string) error { return err } } + // TODO use newAccumulator? logs[file] = adif.NewLogfile() + logs[file].FieldOrder = l.FieldOrder for _, f := range l.Header.Fields() { logs[file].Header.Set(f) } diff --git a/cmd/select.go b/cmd/select.go index 30054f9..97b8481 100644 --- a/cmd/select.go +++ b/cmd/select.go @@ -38,9 +38,11 @@ func runSelect(ctx *Context, args []string) error { if len(con.Fields) == 0 { return fmt.Errorf("no fields provided, try %s select -fields CALL,BAND", filepath.Base(os.Args[0])) } - out := adif.NewLogfile() - out.FieldOrder = con.Fields - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } + updateFieldOrder(acc.Out, con.Fields) for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { @@ -54,12 +56,12 @@ func runSelect(ctx *Context, args []string) error { } } if len(fields) > 0 { - out.AddRecord(adif.NewRecord(fields...)) + acc.Out.AddRecord(adif.NewRecord(fields...)) } } } if err := acc.prepare(); err != nil { return err } - return write(ctx, out) + return write(ctx, acc.Out) } diff --git a/cmd/sort.go b/cmd/sort.go index d854e55..772b656 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -53,26 +53,28 @@ func runSort(ctx *Context, args []string) error { comps[i] = spec.ComparatorForField(f, ctx.Locale) } // else resolve dynamically } - out := adif.NewLogfile() - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { return err } for _, r := range l.Records { - out.AddRecord(r) + acc.Out.AddRecord(r) } } if err := acc.prepare(); err != nil { return err } - sort.SliceStable(out.Records, func(i, j int) bool { - a := out.Records[i] - b := out.Records[j] + sort.SliceStable(acc.Out.Records, func(i, j int) bool { + a := acc.Out.Records[i] + b := acc.Out.Records[j] for k, comp := range comps { if comp == nil { - if uf, _ := out.GetUserdef(fields[k]); uf.Type.Indicator() != "" { + if uf, _ := acc.Out.GetUserdef(fields[k]); uf.Type.Indicator() != "" { f := spec.Field{Name: fields[k], Type: spec.DataTypes[uf.Type.Indicator()]} comp = spec.ComparatorForField(f, ctx.Locale) } else { @@ -112,5 +114,5 @@ func runSort(ctx *Context, args []string) error { } return false }) - return write(ctx, out) + return write(ctx, acc.Out) } diff --git a/cmd/validate.go b/cmd/validate.go index fda953e..059a037 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -41,14 +41,16 @@ func runValidate(ctx *Context, args []string) error { log := os.Stderr var errors, warnings int appFields := make(map[string]adif.DataType) - out := adif.NewLogfile() - acc := accumulator{Out: out, Ctx: ctx} + acc, err := newAccumulator(ctx) + if err != nil { + return err + } for _, f := range filesOrStdin(args) { l, err := acc.read(f) if err != nil { return err } - updateFieldOrder(out, l.FieldOrder) + updateFieldOrder(acc.Out, l.FieldOrder) for i, r := range l.Records { vctx := spec.ValidationContext{ Now: now, @@ -114,7 +116,7 @@ func runValidate(ctx *Context, args []string) error { r.SetComment("adif-multitool: validate warnings: " + strings.Join(msgs, "; ")) } } - out.AddRecord(r) + acc.Out.AddRecord(r) } } if errors > 0 { @@ -123,7 +125,7 @@ func runValidate(ctx *Context, args []string) error { if err := acc.prepare(); err != nil { return err } - err := write(ctx, out) + err = write(ctx, acc.Out) if warnings > 0 { fmt.Fprintf(log, "validate got %d warnings\n", warnings) }