From 3baebea7a3e33efe9d0d6c41fffef105e2535ad3 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 2 Jul 2023 15:49:41 -0400 Subject: [PATCH] Add `%N` and `%O` for `strfntime` (#1334) * Add `%N` and `%O` for strfntime * Unit-test mods * artifacts from `make dev` --- docs/src/data-diving-examples.md | 46 +++++++++---------- docs/src/manpage.md | 3 +- docs/src/manpage.txt | 3 +- docs/src/reference-dsl-time.md | 8 +++- docs/src/reference-dsl-time.md.in | 6 ++- docs/src/reference-verbs.md | 39 ++++++++-------- docs/src/two-pass-algorithms.md | 4 +- internal/pkg/bifs/datetime.go | 14 ++++++ man/manpage.txt | 3 +- man/mlr.1 | 5 +- .../strfntime-istanbul/expout | 2 + .../strfntime-sao_paulo/expout | 2 + .../strfntime-utc/expout | 2 + .../strftime-istanbul/expout | 2 + .../strftime-sao_paulo/expout | 2 + .../strftime-utc/expout | 2 + test/input/strfntime-tz.mlr | 2 + test/input/strftime-tz.mlr | 2 + 18 files changed, 96 insertions(+), 51 deletions(-) diff --git a/docs/src/data-diving-examples.md b/docs/src/data-diving-examples.md index 100716ec26..39738f193d 100644 --- a/docs/src/data-diving-examples.md +++ b/docs/src/data-diving-examples.md @@ -160,11 +160,11 @@ CITRUS COUNTY 1332.9 79974.9 483785.1 stats2 -a corr,linreg-ols,r2 -f tiv_2011,tiv_2012 
-tiv_2011_tiv_2012_corr  0.9730497632351701
-tiv_2011_tiv_2012_ols_m 0.9835583980337732
-tiv_2011_tiv_2012_ols_b 433854.6428968301
+tiv_2011_tiv_2012_corr  0.9730497632351692
+tiv_2011_tiv_2012_ols_m 0.9835583980337723
+tiv_2011_tiv_2012_ols_b 433854.6428968317
 tiv_2011_tiv_2012_ols_n 36634
-tiv_2011_tiv_2012_r2    0.9468258417320204
+tiv_2011_tiv_2012_r2    0.9468258417320189
 
@@ -322,7 +322,7 @@ Look at bivariate stats by color and shape. In particular, `u,v` pairwise correl
 
           u_v_corr              w_x_corr
-0.1334180491027861 -0.011319841199852926
+0.1334180491027861 -0.011319841199866178
 
@@ -332,22 +332,22 @@ Look at bivariate stats by color and shape. In particular, `u,v` pairwise correl
 
  color    shape              u_v_corr               w_x_corr
-   red   circle    0.9807984401887242  -0.018565536587084836
-orange   square   0.17685855992752933   -0.07104431573805543
- green   circle   0.05764419437577257   0.011795729888018455
-   red   square    0.0557447712489348 -0.0006801456507506415
-yellow triangle    0.0445727377196281   0.024604310103079844
-yellow   square    0.0437917292729612  -0.044621972016306265
-purple   circle   0.03587354936895115    0.13411339541407613
-  blue   square   0.03241153095761152   -0.05350764811965621
-  blue triangle  0.015356427073158612 -0.0006089997461408209
-orange   circle  0.010518953877704181    -0.1627939732927932
-   red triangle   0.00809782571528054    0.01248662135795501
-purple triangle  0.005155190909099739   -0.04505790925621933
-purple   square  -0.02568027696337717   0.057694296479293694
- green   square -0.025776073450284875 -0.0032651732520739014
-orange triangle -0.030456661186085584   -0.13186999819263814
-yellow   circle  -0.06477331572781515     0.0736944981970553
-  blue   circle   -0.1023476190192966  -0.030528539069839333
- green triangle  -0.10901825107358747   -0.04848782060162855
+   red   circle    0.9807984401887236   -0.01856553658708754
+orange   square   0.17685855992752927   -0.07104431573806054
+ green   circle   0.05764419437577255    0.01179572988801509
+   red   square   0.05574477124893523 -0.0006801456507510942
+yellow triangle   0.04457273771962798   0.024604310103081825
+yellow   square   0.04379172927296089   -0.04462197201631237
+purple   circle   0.03587354936895086     0.1341133954140899
+  blue   square   0.03241153095761164  -0.053507648119643196
+  blue triangle  0.015356427073158766 -0.0006089997461435399
+orange   circle  0.010518953877704048   -0.16279397329279383
+   red triangle   0.00809782571528034   0.012486621357942596
+purple triangle  0.005155190909099334  -0.045057909256220656
+purple   square -0.025680276963377404    0.05769429647930396
+ green   square   -0.0257760734502851  -0.003265173252087127
+orange triangle -0.030456661186085785    -0.1318699981926352
+yellow   circle  -0.06477331572781474    0.07369449819706045
+  blue   circle  -0.10234761901929677  -0.030528539069837757
+ green triangle  -0.10901825107358765   -0.04848782060162929
diff --git a/docs/src/manpage.md b/docs/src/manpage.md index c458367beb..a6220789a2 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -940,6 +940,7 @@ MILLER(1) MILLER(1) Useful for doing a well-formatted check on input data. with the exception that warnings are printed to stderr. Current checks are: + * Data are parseable * If any key is the empty string Options: -h|--help Show this message. @@ -3433,5 +3434,5 @@ MILLER(1) MILLER(1) - 2023-06-25 MILLER(1) + 2023-07-02 MILLER(1) diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 4aad075495..f5f2c30908 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -919,6 +919,7 @@ MILLER(1) MILLER(1) Useful for doing a well-formatted check on input data. with the exception that warnings are printed to stderr. Current checks are: + * Data are parseable * If any key is the empty string Options: -h|--help Show this message. @@ -3412,4 +3413,4 @@ MILLER(1) MILLER(1) - 2023-06-25 MILLER(1) + 2023-07-02 MILLER(1) diff --git a/docs/src/reference-dsl-time.md b/docs/src/reference-dsl-time.md index 680057fb10..d7f0111a34 100644 --- a/docs/src/reference-dsl-time.md +++ b/docs/src/reference-dsl-time.md @@ -246,7 +246,7 @@ Notes: * For `strftime`, this is thanks to [https://github.com/lestrrat-go/strftime](https://github.com/lestrrat-go/strftime), with a Miller-specific modification for fractional seconds. * For `strftime`, this is thanks to [https://github.com/pbnjay/strptime](https://github.com/pbnjay/strptime), with Miller-specific modifications. -Available format strings for `strftime`, taken directly from [https://github.com/lestrrat-go/strftime](https://github.com/lestrrat-go/strftime): +Available format strings for `strftime`, taken directly from [https://github.com/lestrrat-go/strftime](https://github.com/lestrrat-go/strftime) except for `%1..%9`, `%N`, and `%O` which are Miller-specific additions: | Pattern | Description | |---------|-------------| @@ -269,6 +269,8 @@ Available format strings for `strftime`, taken directly from [https://github.com | `%M` | the minute as a decimal number (00-59) | | `%m` | the month as a decimal number (01-12) | | `%n` | a newline | +| `%N` | zero-padded nanoseconds | +| `%O` | non-zero-padded nanoseconds | | `%p` | national representation of either "ante meridiem" (a.m.) or "post meridiem" (p.m.) as appropriate. | | `%R` | equivalent to `%H:%M` | | `%r` | equivalent to `%I:%M:%S %p` | @@ -317,11 +319,15 @@ Examples: mlr -n put 'end { print strftime(0, "%Y-%m-%dT%H:%M:%SZ"); print strftime(0, "%FT%TZ"); + print strfntime(123, "%N"); + print strfntime(123, "%O"); }' 
 1970-01-01T00:00:00Z
 1970-01-01T00:00:00Z
+000000123
+123
 
diff --git a/docs/src/reference-dsl-time.md.in b/docs/src/reference-dsl-time.md.in
index 565715b772..9259abb14a 100644
--- a/docs/src/reference-dsl-time.md.in
+++ b/docs/src/reference-dsl-time.md.in
@@ -178,7 +178,7 @@ Notes:
   * For `strftime`, this is thanks to [https://github.com/lestrrat-go/strftime](https://github.com/lestrrat-go/strftime), with a Miller-specific modification for fractional seconds.
   * For `strftime`, this is thanks to [https://github.com/pbnjay/strptime](https://github.com/pbnjay/strptime), with Miller-specific modifications.
 
-Available format strings for `strftime`, taken directly from [https://github.com/lestrrat-go/strftime](https://github.com/lestrrat-go/strftime):
+Available format strings for `strftime`, taken directly from [https://github.com/lestrrat-go/strftime](https://github.com/lestrrat-go/strftime) except for `%1..%9`, `%N`, and `%O` which are Miller-specific additions:
 
 | Pattern | Description |
 |---------|-------------|
@@ -201,6 +201,8 @@ Available format strings for `strftime`, taken directly from [https://github.com
 | `%M` | the minute as a decimal number (00-59) |
 | `%m` | the month as a decimal number (01-12) |
 | `%n` | a newline |
+| `%N` | zero-padded nanoseconds |
+| `%O` | non-zero-padded nanoseconds |
 | `%p` | national representation of either "ante meridiem" (a.m.) or "post meridiem" (p.m.) as appropriate. |
 | `%R` | equivalent to `%H:%M` |
 | `%r` | equivalent to `%I:%M:%S %p` |
@@ -249,6 +251,8 @@ GENMD-RUN-COMMAND
 mlr -n put 'end {
   print strftime(0, "%Y-%m-%dT%H:%M:%SZ");
   print strftime(0, "%FT%TZ");
+  print strfntime(123, "%N");
+  print strfntime(123, "%O");
 }'
 GENMD-EOF
 
diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md
index e34c9f7882..6607ad2193 100644
--- a/docs/src/reference-verbs.md
+++ b/docs/src/reference-verbs.md
@@ -380,6 +380,7 @@ Consumes records without printing any output,
 Useful for doing a well-formatted check on input data.
 with the exception that warnings are printed to stderr.
 Current checks are:
+* Data are parseable
 * If any key is the empty string
 Options:
 -h|--help Show this message.
@@ -3306,14 +3307,14 @@ fields, optionally categorized by one or more fields.
   data/medium
 
-x_y_cov    0.00004257482082749404
-x_y_corr   0.0005042001844473328
-y_y_cov    0.08461122467974005
+x_y_cov    0.000042574820827444476
+x_y_corr   0.0005042001844467462
+y_y_cov    0.08461122467974003
 y_y_corr   1
-x2_xy_cov  0.041883822817793716
-x2_xy_corr 0.6301743420379936
-x2_y2_cov  -0.0003095372596253918
-x2_y2_corr -0.003424908876111875
+x2_xy_cov  0.04188382281779374
+x2_xy_corr 0.630174342037994
+x2_y2_cov  -0.00030953725962542085
+x2_y2_corr -0.0034249088761121966
 
@@ -3322,12 +3323,12 @@ x2_y2_corr -0.003424908876111875
   data/medium
 
-a   x_y_ols_m             x_y_ols_b          x_y_ols_n x_y_r2                  y_y_ols_m y_y_ols_b                           y_y_ols_n y_y_r2 xy_y2_ols_m        xy_y2_ols_b         xy_y2_ols_n xy_y2_r2
-pan 0.017025512736819345  0.500402892289764  2081      0.00028691820445815624  1         -0.00000000000000002890430283104539 2081      1      0.8781320866715664 0.11908230147563569 2081        0.4174982737731127
-eks 0.04078049236855813   0.4814020796765104 1965      0.0016461239223448218   1         0.00000000000000017862676354313703  1965      1      0.897872861169018  0.1073405443361234  1965        0.4556322386425451
-wye -0.03915349075204785  0.5255096523974457 1966      0.0015051268704373377   1         0.00000000000000004464425401127647  1966      1      0.8538317334220837 0.1267454301662969  1966        0.3899172181859931
-zee 0.0027812364960401333 0.5043070448033061 2047      0.000007751652858787357 1         0.00000000000000004819404567023685  2047      1      0.8524439912011011 0.12401684308018947 2047        0.39356598090006495
-hat -0.018620577041095272 0.5179005397264937 1941      0.00035200366460556604  1         -0.00000000000000003400445761787692 1941      1      0.8412305086345017 0.13557328318623207 1941        0.3687944261732266
+a   x_y_ols_m             x_y_ols_b           x_y_ols_n x_y_r2                  y_y_ols_m y_y_ols_b y_y_ols_n y_y_r2 xy_y2_ols_m        xy_y2_ols_b         xy_y2_ols_n xy_y2_r2
+pan 0.01702551273681908   0.5004028922897639  2081      0.00028691820445814767  1         0         2081      1      0.8781320866715662 0.11908230147563566 2081        0.41749827377311266
+eks 0.0407804923685586    0.48140207967651016 1965      0.0016461239223448587   1         0         1965      1      0.8978728611690183 0.10734054433612333 1965        0.45563223864254526
+wye -0.03915349075204814  0.5255096523974456  1966      0.0015051268704373607   1         0         1966      1      0.8538317334220835 0.1267454301662969  1966        0.38991721818599295
+zee 0.0027812364960399147 0.5043070448033061  2047      0.000007751652858786137 1         0         2047      1      0.8524439912011013 0.12401684308018937 2047        0.39356598090006495
+hat -0.018620577041095078 0.5179005397264935  1941      0.0003520036646055585   1         0         1941      1      0.8412305086345014 0.13557328318623216 1941        0.3687944261732265
Here's an example simple line-fit. The `x` and `y` @@ -3413,11 +3414,11 @@ upsec_count_pca_quality 0.9999590846136102 donesec 92.33051350964094 color purple -upsec_count_pca_m -39.030097447953594 -upsec_count_pca_b 979.9883413064917 +upsec_count_pca_m -39.03009744795354 +upsec_count_pca_b 979.9883413064914 upsec_count_pca_n 21 upsec_count_pca_quality 0.9999908956206317 -donesec 25.108529196302943 +donesec 25.10852919630297 ## step @@ -3645,9 +3646,9 @@ distinct_count 5 5 10000 10000 10000 mode pan wye 1 0.3467901443380824 0.7268028627434533 sum 0 0 50005000 4986.019681679581 5062.057444929905 mean - - 5000.5 0.49860196816795804 0.5062057444929905 -stddev - - 2886.8956799071675 0.29029251511440074 0.2908800864269331 -var - - 8334166.666666667 0.08426974433144457 0.08461122467974005 -skewness - - 0 -0.0006899591185517494 -0.01784976012013298 +stddev - - 2886.8956799071675 0.2902925151144007 0.290880086426933 +var - - 8334166.666666667 0.08426974433144456 0.08461122467974003 +skewness - - 0 -0.0006899591185521965 -0.017849760120133784 minlen 3 3 1 15 13 maxlen 3 3 5 22 22 min eks eks 1 0.00004509679127584487 0.00008818962627266114 diff --git a/docs/src/two-pass-algorithms.md b/docs/src/two-pass-algorithms.md index e475aebf3b..146f3a81e1 100644 --- a/docs/src/two-pass-algorithms.md +++ b/docs/src/two-pass-algorithms.md @@ -598,8 +598,8 @@ hat pan 0.4643355557376876 x_count 10000 x_sum 4986.019681679581 x_mean 0.49860196816795804 -x_var 0.08426974433144457 -x_stddev 0.29029251511440074 +x_var 0.08426974433144456 +x_stddev 0.2902925151144007 
diff --git a/internal/pkg/bifs/datetime.go b/internal/pkg/bifs/datetime.go
index 47e908b167..f94da527d8 100644
--- a/internal/pkg/bifs/datetime.go
+++ b/internal/pkg/bifs/datetime.go
@@ -425,6 +425,18 @@ func init() {
 	appender9 := strftime.AppendFunc(func(b []byte, t time.Time) []byte {
 		return specificationHelper(b, t, "%09d", 1)
 	})
+	appenderN := strftime.AppendFunc(func(b []byte, t time.Time) []byte {
+		nanos := int(t.Nanosecond())
+		s := fmt.Sprintf("%09d", nanos)
+		//return append(b, []byte(s))
+		return append(b, s...)
+	})
+	appenderO := strftime.AppendFunc(func(b []byte, t time.Time) []byte {
+		nanos := int(t.Nanosecond())
+		s := fmt.Sprintf("%d", nanos)
+		//return append(b, []byte(s))
+		return append(b, s...)
+	})
 
 	ss := strftime.NewSpecificationSet()
 	ss.Set('1', appender1)
@@ -436,6 +448,8 @@ func init() {
 	ss.Set('7', appender7)
 	ss.Set('8', appender8)
 	ss.Set('9', appender9)
+	ss.Set('N', appenderN)
+	ss.Set('O', appenderO)
 
 	strftimeExtensions = strftime.WithSpecificationSet(ss)
 }
diff --git a/man/manpage.txt b/man/manpage.txt
index 4aad075495..f5f2c30908 100644
--- a/man/manpage.txt
+++ b/man/manpage.txt
@@ -919,6 +919,7 @@ MILLER(1)                                                            MILLER(1)
        Useful for doing a well-formatted check on input data.
        with the exception that warnings are printed to stderr.
        Current checks are:
+       * Data are parseable
        * If any key is the empty string
        Options:
        -h|--help Show this message.
@@ -3412,4 +3413,4 @@ MILLER(1)                                                            MILLER(1)
 
 
 
-                                  2023-06-25                         MILLER(1)
+                                  2023-07-02                         MILLER(1)
diff --git a/man/mlr.1 b/man/mlr.1
index a0bd1c5310..8b3fdb2e16 100644
--- a/man/mlr.1
+++ b/man/mlr.1
@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2023-06-25
+.\"      Date: 2023-07-02
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2023-06-25" "\ \&" "\ \&"
+.TH "MILLER" "1" "2023-07-02" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1126,6 +1126,7 @@ Consumes records without printing any output,
 Useful for doing a well-formatted check on input data.
 with the exception that warnings are printed to stderr.
 Current checks are:
+* Data are parseable
 * If any key is the empty string
 Options:
 -h|--help Show this message.
diff --git a/test/cases/dsl-local-date-time-functions/strfntime-istanbul/expout b/test/cases/dsl-local-date-time-functions/strfntime-istanbul/expout
index 057ff69bdd..6e6a4997a9 100644
--- a/test/cases/dsl-local-date-time-functions/strfntime-istanbul/expout
+++ b/test/cases/dsl-local-date-time-functions/strfntime-istanbul/expout
@@ -10,3 +10,5 @@ TZ is Asia/Istanbul
 1970-01-01 00:00:00 +0000
 1970-01-01 00:00:00 UTC
 1970-01-01 00:00:00 +0000
+00 000123456
+00 123456
diff --git a/test/cases/dsl-local-date-time-functions/strfntime-sao_paulo/expout b/test/cases/dsl-local-date-time-functions/strfntime-sao_paulo/expout
index 36a591e67f..d871916095 100644
--- a/test/cases/dsl-local-date-time-functions/strfntime-sao_paulo/expout
+++ b/test/cases/dsl-local-date-time-functions/strfntime-sao_paulo/expout
@@ -10,3 +10,5 @@ TZ is America/Sao_Paulo
 1970-01-01 00:00:00 +0000
 1970-01-01 00:00:00 UTC
 1970-01-01 00:00:00 +0000
+00 000123456
+00 123456
diff --git a/test/cases/dsl-local-date-time-functions/strfntime-utc/expout b/test/cases/dsl-local-date-time-functions/strfntime-utc/expout
index 62b9c1fe6f..b6ea47b6fe 100644
--- a/test/cases/dsl-local-date-time-functions/strfntime-utc/expout
+++ b/test/cases/dsl-local-date-time-functions/strfntime-utc/expout
@@ -10,3 +10,5 @@ TZ is UTC
 1970-01-01 00:00:00 +0000
 1970-01-01 00:00:00 UTC
 1970-01-01 00:00:00 +0000
+00 000123456
+00 123456
diff --git a/test/cases/dsl-local-date-time-functions/strftime-istanbul/expout b/test/cases/dsl-local-date-time-functions/strftime-istanbul/expout
index b87bc3d3aa..6dc7ae793d 100644
--- a/test/cases/dsl-local-date-time-functions/strftime-istanbul/expout
+++ b/test/cases/dsl-local-date-time-functions/strftime-istanbul/expout
@@ -10,3 +10,5 @@ TZ is Asia/Istanbul
 1970-01-01 00:00:00 +0000
 1970-01-01 00:00:00 UTC
 1970-01-01 00:00:00 +0000
+00 123456000
+00 123456000
diff --git a/test/cases/dsl-local-date-time-functions/strftime-sao_paulo/expout b/test/cases/dsl-local-date-time-functions/strftime-sao_paulo/expout
index addbf579f8..af29730669 100644
--- a/test/cases/dsl-local-date-time-functions/strftime-sao_paulo/expout
+++ b/test/cases/dsl-local-date-time-functions/strftime-sao_paulo/expout
@@ -10,3 +10,5 @@ TZ is America/Sao_Paulo
 1970-01-01 00:00:00 +0000
 1970-01-01 00:00:00 UTC
 1970-01-01 00:00:00 +0000
+00 123456000
+00 123456000
diff --git a/test/cases/dsl-local-date-time-functions/strftime-utc/expout b/test/cases/dsl-local-date-time-functions/strftime-utc/expout
index d47ca354ff..ba9190df51 100644
--- a/test/cases/dsl-local-date-time-functions/strftime-utc/expout
+++ b/test/cases/dsl-local-date-time-functions/strftime-utc/expout
@@ -10,3 +10,5 @@ TZ is UTC
 1970-01-01 00:00:00 +0000
 1970-01-01 00:00:00 UTC
 1970-01-01 00:00:00 +0000
+00 123456000
+00 123456000
diff --git a/test/input/strfntime-tz.mlr b/test/input/strfntime-tz.mlr
index 45f507617b..0fc2dd72bc 100644
--- a/test/input/strfntime-tz.mlr
+++ b/test/input/strfntime-tz.mlr
@@ -13,5 +13,7 @@ end {
   print strfntime(123456, "%Y-%m-%d %H:%M:%S %z");
   print strfntime(0,      "%Y-%m-%d %H:%M:%S %Z");
   print strfntime(0,      "%Y-%m-%d %H:%M:%S %z");
+  print strfntime(123456, "%S %N");
+  print strfntime(123456, "%S %O");
 
 }
diff --git a/test/input/strftime-tz.mlr b/test/input/strftime-tz.mlr
index f789f2fcc2..fecb46b770 100644
--- a/test/input/strftime-tz.mlr
+++ b/test/input/strftime-tz.mlr
@@ -13,5 +13,7 @@ end {
   print strftime(0.123456, "%Y-%m-%d %H:%M:%S %z");
   print strftime(0.0,      "%Y-%m-%d %H:%M:%S %Z");
   print strftime(0.0,      "%Y-%m-%d %H:%M:%S %z");
+  print strftime(0.123456, "%S %N");
+  print strftime(0.123456, "%S %O");
 
 }