From 3044cf5f1f16ddc3a507e966bb0f19af04edc762 Mon Sep 17 00:00:00 2001 From: arnikola Date: Sun, 1 Sep 2019 21:03:17 -0400 Subject: [PATCH] [Query] Increase performance of temporal functions (#1917) --- docker/grafana/Dockerfile | 1 + .../grafana/m3db_overlaid_dashboard.json | 2 +- .../grafana/temporal_function_comparison.json | 1879 +++++++++++++++++ .../v1/handler/prometheus/native/common.go | 13 +- src/query/block/column.go | 4 + src/query/functions/aggregation/absent.go | 2 - src/query/functions/temporal/aggregation.go | 13 +- .../functions/temporal/aggregation_test.go | 32 +- src/query/functions/temporal/base.go | 207 +- src/query/functions/temporal/base_test.go | 49 +- src/query/functions/temporal/functions.go | 10 +- .../functions/temporal/functions_test.go | 16 +- .../functions/temporal/holt_winters_test.go | 4 +- .../functions/temporal/linear_regression.go | 48 +- .../temporal/linear_regression_test.go | 16 +- src/query/functions/temporal/rate.go | 16 +- src/query/functions/temporal/rate_test.go | 40 +- 17 files changed, 2187 insertions(+), 165 deletions(-) create mode 100644 integrations/grafana/temporal_function_comparison.json diff --git a/docker/grafana/Dockerfile b/docker/grafana/Dockerfile index 642c054e4f..e454d54567 100644 --- a/docker/grafana/Dockerfile +++ b/docker/grafana/Dockerfile @@ -7,6 +7,7 @@ RUN mkdir -p /tmp/grafana_dashboards COPY ./integrations/grafana/m3query_dashboard.json /tmp/grafana_dashboards/m3query_dashboard.json COPY ./integrations/grafana/m3coordinator_dashboard.json /tmp/grafana_dashboards/m3coordinator_dashboard.json COPY ./integrations/grafana/m3db_dashboard.json /tmp/grafana_dashboards/m3db_dashboard.json +COPY ./integrations/grafana/temporal_function_comparison.json /tmp/grafana_dashboards/temporal_function_comparison.json # Need to replace datasource template variable with name of actual data source so auto-import # JustWorksTM. Use a temporary directory to host the dashboards since the default diff --git a/integrations/grafana/m3db_overlaid_dashboard.json b/integrations/grafana/m3db_overlaid_dashboard.json index 71e46b6852..61c47df4da 100644 --- a/integrations/grafana/m3db_overlaid_dashboard.json +++ b/integrations/grafana/m3db_overlaid_dashboard.json @@ -6788,6 +6788,6 @@ }, "timezone": "browser", "title": "M3DB Node Details - mixed", - "uid": "misterquery", + "uid": "mixed", "version": 4 } diff --git a/integrations/grafana/temporal_function_comparison.json b/integrations/grafana/temporal_function_comparison.json new file mode 100644 index 0000000000..89cda44889 --- /dev/null +++ b/integrations/grafana/temporal_function_comparison.json @@ -0,0 +1,1879 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "avg_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "avg_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "avg_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "count_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "count_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "count_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "holt_winters(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range], 0.3, 0.8)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "holt_winters(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range], 0.3, 0.8)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "holt_winters", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "min_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "min_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "min_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "delta(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "delta(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "delta", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "idelta(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "idelta(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "idelta", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "irate(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "irate(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "irate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "max_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "max_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "max_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "increase(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "increase(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "increase", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "sum_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "sum_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "sum_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "predict_linear(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range], 1)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "predict_linear(scrape_duration_seconds{instance=~\"m3db_seed:9004\"}[$range], 1)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "predict_linear", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "stddev_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "stddev_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "stddev_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "rate(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "rate(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "stdvar_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "stdvar_over_time(scrape_duration_seconds{instance=~\"m3db_seed:9004\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "stdvar_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "resets(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "resets(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "resets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "deriv(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "deriv(scrape_duration_seconds{instance=~\"m3db_seed:9004\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "deriv", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "quantile_over_time(0.3, scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "quantile_over_time(0.3, scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "quantile_over_time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "-- Mixed --", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "Prometheus", + "expr": "changes(scrape_duration_seconds{instance=~\"m3db_seed:9004\", role=\"remote\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Prom", + "refId": "B" + }, + { + "datasource": "M3Query - Prometheus", + "expr": "changes(scrape_duration_seconds{instance=~\"m3db_seed:9004\"}[$range])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "M3", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "changes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "1m", + "value": "1m" + }, + "hide": 0, + "label": null, + "name": "range", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "15s", + "30s", + "1m", + "2m", + "5m", + "10m", + "15m", + "1h", + "6h", + "12h", + "24h" + ] + }, + "timezone": "", + "title": "Temporal function comparison", + "uid": "temporal", + "version": 1 +} diff --git a/src/query/api/v1/handler/prometheus/native/common.go b/src/query/api/v1/handler/prometheus/native/common.go index d826e383e1..5c4ea31bcf 100644 --- a/src/query/api/v1/handler/prometheus/native/common.go +++ b/src/query/api/v1/handler/prometheus/native/common.go @@ -94,27 +94,32 @@ func parseParams( if err != nil { return params, xhttp.NewParseError(err, http.StatusBadRequest) } - params.Timeout = t + params.Timeout = t start, err := parseTime(r, startParam, params.Now) if err != nil { return params, xhttp.NewParseError(fmt.Errorf(formatErrStr, startParam, err), http.StatusBadRequest) } - params.Start = start + params.Start = start end, err := parseTime(r, endParam, params.Now) if err != nil { return params, xhttp.NewParseError(fmt.Errorf(formatErrStr, endParam, err), http.StatusBadRequest) } - params.End = end + if start.After(end) { + return params, xhttp.NewParseError(fmt.Errorf("start (%s) must be before end (%s)", + start, end), http.StatusBadRequest) + } + + params.End = end step := fetchOpts.Step if step <= 0 { err := fmt.Errorf("expected postive step size, instead got: %d", step) return params, xhttp.NewParseError(fmt.Errorf(formatErrStr, handler.StepParam, err), http.StatusBadRequest) } - params.Step = fetchOpts.Step + params.Step = fetchOpts.Step query, err := parseQuery(r) if err != nil { return params, xhttp.NewParseError(fmt.Errorf(formatErrStr, queryParam, err), http.StatusBadRequest) diff --git a/src/query/block/column.go b/src/query/block/column.go index 2b6238d18c..590af69941 100644 --- a/src/query/block/column.go +++ b/src/query/block/column.go @@ -226,6 +226,10 @@ func (cb ColumnBlockBuilder) AppendValues(idx int, values []float64) error { } func (cb ColumnBlockBuilder) AddCols(num int) error { + if num < 1 { + return fmt.Errorf("must add more than 0 columns, adding: %d", num) + } + newCols := make([]column, num) cb.block.columns = append(cb.block.columns, newCols...) return nil diff --git a/src/query/functions/aggregation/absent.go b/src/query/functions/aggregation/absent.go index 838ed72ad7..bc61b5fe3a 100644 --- a/src/query/functions/aggregation/absent.go +++ b/src/query/functions/aggregation/absent.go @@ -21,7 +21,6 @@ package aggregation import ( - "fmt" "math" "github.com/m3db/m3/src/query/block" @@ -147,7 +146,6 @@ func (n *absentNode) ProcessBlock( values = step.Values() val float64 = 1 ) - fmt.Println(values) for _, v := range values { if !math.IsNaN(v) { val = math.NaN() diff --git a/src/query/functions/temporal/aggregation.go b/src/query/functions/temporal/aggregation.go index 6fb27b00fe..6ad47844fc 100644 --- a/src/query/functions/temporal/aggregation.go +++ b/src/query/functions/temporal/aggregation.go @@ -74,10 +74,13 @@ type aggProcessor struct { aggFunc aggFunc } -func (a aggProcessor) Init(op baseOp, controller *transform.Controller, opts transform.Options) Processor { +func (a aggProcessor) initialize( + _ time.Duration, + controller *transform.Controller, + opts transform.Options, +) processor { return &aggNode{ controller: controller, - op: op, aggFunc: a.aggFunc, } } @@ -134,12 +137,11 @@ func NewAggOp(args []interface{}, optype string) (transform.Params, error) { } type aggNode struct { - op baseOp controller *transform.Controller aggFunc func([]float64) float64 } -func (a *aggNode) Process(datapoints ts.Datapoints, _ time.Time) float64 { +func (a *aggNode) process(datapoints ts.Datapoints, _ time.Time) float64 { return a.aggFunc(datapoints.Values()) } @@ -211,7 +213,8 @@ func stdvarOverTime(values []float64) float64 { } } - if count == 0 { + // NB: stdvar and stddev are undefined unless there are more than 2 points. + if count < 2 { return math.NaN() } diff --git a/src/query/functions/temporal/aggregation_test.go b/src/query/functions/temporal/aggregation_test.go index f292b837aa..6af94450f5 100644 --- a/src/query/functions/temporal/aggregation_test.go +++ b/src/query/functions/temporal/aggregation_test.go @@ -49,8 +49,8 @@ var testCases = []testCase{ name: "avg_over_time", opType: AvgType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 2.5}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 7}, + {math.NaN(), 1, 1.5, 2, 2.5}, + {5, 5.5, 6, 6.5, 7}, }, afterAllBlocks: [][]float64{ {2, 2, 2, 2, 2}, @@ -61,8 +61,8 @@ var testCases = []testCase{ name: "count_over_time", opType: CountType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 4}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 5}, + {math.NaN(), 1, 2, 3, 4}, + {1, 2, 3, 4, 5}, }, afterAllBlocks: [][]float64{ {5, 5, 5, 5, 5}, @@ -73,8 +73,8 @@ var testCases = []testCase{ name: "min_over_time", opType: MinType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 5}, + {math.NaN(), 1, 1, 1, 1}, + {5, 5, 5, 5, 5}, }, afterAllBlocks: [][]float64{ {0, 0, 0, 0, 0}, @@ -85,8 +85,8 @@ var testCases = []testCase{ name: "max_over_time", opType: MaxType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 4}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 9}, + {math.NaN(), 1, 2, 3, 4}, + {5, 6, 7, 8, 9}, }, afterAllBlocks: [][]float64{ {4, 4, 4, 4, 4}, @@ -97,8 +97,8 @@ var testCases = []testCase{ name: "sum_over_time", opType: SumType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 10}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 35}, + {math.NaN(), 1, 3, 6, 10}, + {5, 11, 18, 26, 35}, }, afterAllBlocks: [][]float64{ {10, 10, 10, 10, 10}, @@ -109,8 +109,8 @@ var testCases = []testCase{ name: "stddev_over_time", opType: StdDevType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1.1180}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1.4142}, + {math.NaN(), math.NaN(), 0.5, 0.81649, 1.1180}, + {math.NaN(), 0.5, 0.81649, 1.11803, 1.4142}, }, afterAllBlocks: [][]float64{ {1.4142, 1.4142, 1.4142, 1.4142, 1.4142}, @@ -121,8 +121,8 @@ var testCases = []testCase{ name: "stdvar_over_time", opType: StdVarType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1.25}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 2}, + {math.NaN(), math.NaN(), 0.25, 0.666666, 1.25}, + {math.NaN(), 0.25, 0.66666, 1.25, 2}, }, afterAllBlocks: [][]float64{ {2, 2, 2, 2, 2}, @@ -133,8 +133,8 @@ var testCases = []testCase{ name: "quantile_over_time", opType: QuantileType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1.6}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 5.8}, + {math.NaN(), 1, 1.2, 1.4, 1.6}, + {5, 5.2, 5.4, 5.6, 5.8}, }, afterAllBlocks: [][]float64{ {0.8, 0.8, 0.8, 0.8, 0.8}, diff --git a/src/query/functions/temporal/base.go b/src/query/functions/temporal/base.go index 92789393d7..a2723aff8b 100644 --- a/src/query/functions/temporal/base.go +++ b/src/query/functions/temporal/base.go @@ -44,13 +44,13 @@ var emptyOp = baseOp{} type baseOp struct { operatorType string duration time.Duration - processorFn MakeProcessor + processorFn makeProcessor } func newBaseOp( duration time.Duration, operatorType string, - processorFn MakeProcessor, + processorFn makeProcessor, ) (baseOp, error) { return baseOp{ operatorType: operatorType, @@ -76,7 +76,7 @@ func (o baseOp) Node( controller: controller, cache: newBlockCache(o, opts), op: o, - processor: o.processorFn.Init(o, controller, opts), + processor: o.processorFn.initialize(o.duration, controller, opts), transformOpts: opts, } } @@ -89,7 +89,7 @@ type baseNode struct { controller controller op baseOp cache *blockCache - processor Processor + processor processor transformOpts transform.Options } @@ -138,7 +138,7 @@ func (c *baseNode) Process( bounds, queryEndBounds) } - c.cache.init(bounds) + c.cache.initialize(bounds) blockDuration := bounds.Duration // Figure out the maximum blocks needed for the temporal function. maxBlocks := int(math.Ceil(float64(c.op.duration) / float64(blockDuration))) @@ -289,8 +289,11 @@ func (c *baseNode) processCompletedBlocks( defer sp.Finish() blocks := make([]block.Block, 0, len(processRequests)) + // NB: valueBuffer gets populated and re-used within the processSingleRequest + // function call. + var valueBuffer ts.Datapoints for _, req := range processRequests { - bl, err := c.processSingleRequest(req) + bl, err := c.processSingleRequest(req, valueBuffer) if err != nil { // cleanup any blocks we opened closeBlocks(blocks) @@ -303,24 +306,95 @@ func (c *baseNode) processCompletedBlocks( return blocks, nil } +// getIndices returns the index of the points on the left and the right of the +// datapoint list given a starting index, as well as a boolean indicating if +// the returned indices are valid. +// +// NB: return values from getIndices should be used as subslice indices rather +// than direct index accesses, as that may cause panics when reaching the end of +// the datapoint list. +func getIndices( + dp []ts.Datapoint, + lBound time.Time, + rBound time.Time, + init int, +) (int, int, bool) { + if init >= len(dp) || init < 0 { + return -1, -1, false + } + + var ( + l, r = init, -1 + leftBound = false + ) + + for i, dp := range dp[init:] { + ts := dp.Timestamp + if !leftBound { + // Trying to set left bound. + if ts.Before(lBound) { + // data point before 0. + continue + } + + leftBound = true + l = i + } + + if ts.Before(rBound) { + continue + } + + r = i + break + } + + if r == -1 { + r = len(dp) + } else { + r = r + init + } + + if leftBound { + l = l + init + } + + return l, r, true +} + +func buildValueBuffer( + current block.UnconsolidatedSeries, + iters []block.UnconsolidatedSeriesIter, +) ts.Datapoints { + l := 0 + for _, dps := range current.Datapoints() { + l += len(dps) + } + + for _, it := range iters { + for _, dps := range it.Current().Datapoints() { + l += len(dps) + } + } + + // NB: sanity check; theoretically this should never happen + // as empty series should not exist when building the value buffer. + if l < 1 { + return ts.Datapoints{} + } + + return make(ts.Datapoints, 0, l) +} + func (c *baseNode) processSingleRequest( request processRequest, + valueBuffer ts.Datapoints, ) (block.Block, error) { seriesIter, err := request.blk.SeriesIter() if err != nil { return nil, err } - depIters := make([]block.UnconsolidatedSeriesIter, len(request.deps)) - for i, blk := range request.deps { - iter, err := blk.SeriesIter() - if err != nil { - return nil, err - } - - depIters[i] = iter - } - var ( meta = request.blk.Meta() bounds = meta.Bounds @@ -328,13 +402,13 @@ func (c *baseNode) processSingleRequest( ) // rename series to exclude their __name__ tag as part of function processing. - resultSeriesMeta := make([]block.SeriesMeta, len(seriesMeta)) - for i, m := range seriesMeta { + resultSeriesMeta := make([]block.SeriesMeta, 0, len(seriesMeta)) + for _, m := range seriesMeta { tags := m.Tags.WithoutName() - resultSeriesMeta[i] = block.SeriesMeta{ + resultSeriesMeta = append(resultSeriesMeta, block.SeriesMeta{ Name: tags.ID(), Tags: tags, - } + }) } builder, err := c.controller.BlockBuilder(request.queryCtx, @@ -348,53 +422,65 @@ func (c *baseNode) processSingleRequest( } aggDuration := c.op.duration - steps := int((aggDuration + bounds.Duration) / bounds.StepSize) - values := make([]ts.Datapoints, 0, steps) - desiredLength := int(math.Ceil(float64(aggDuration) / float64(bounds.StepSize))) + depIters := make([]block.UnconsolidatedSeriesIter, 0, len(request.deps)) + for _, b := range request.deps { + iter, err := b.SeriesIter() + if err != nil { + return nil, err + } + + depIters = append(depIters, iter) + } + for seriesIter.Next() { - values = values[:0] + series := seriesIter.Current() + // First, advance the iterators to ensure they all have this series. for i, iter := range depIters { if !iter.Next() { return nil, fmt.Errorf("incorrect number of series for block: %d", i) } + } + // If valueBuffer is still unset, build it here; if it's been set in a + // previous iteration, reset it for this processing step. + if valueBuffer == nil { + valueBuffer = buildValueBuffer(series, depIters) + } else { + valueBuffer = valueBuffer[:0] + } + + // Write datapoints into value buffer. + for _, iter := range depIters { s := iter.Current() - values = append(values, s.Datapoints()...) + for _, dps := range s.Datapoints() { + valueBuffer = append(valueBuffer, dps...) + } } - series := seriesIter.Current() + var ( + newVal float64 + init = 0 + alignedTime = bounds.Start + start = alignedTime.Add(-1 * aggDuration) + ) + for i := 0; i < series.Len(); i++ { val := series.DatapointsAtStep(i) - values = append(values, val) - newVal := math.NaN() - alignedTime, _ := bounds.TimeForIndex(i) - oldestDatapointTimestamp := alignedTime.Add(-1 * aggDuration) - // Remove the older values from slice as newer values are pushed in. - // TODO: Consider using a rotating slice since this is inefficient - if desiredLength <= len(values) { - values = values[len(values)-desiredLength:] - flattenedValues := make(ts.Datapoints, 0, len(values)) - for _, dps := range values { - var ( - idx int - dp ts.Datapoint - ) - for idx, dp = range dps { - // go until we find datapoints at the oldest timestamp for window - if !dp.Timestamp.Before(oldestDatapointTimestamp) { - break - } - } - - flattenedValues = append(flattenedValues, dps[idx:]...) - } - - newVal = c.processor.Process(flattenedValues, alignedTime) + valueBuffer = append(valueBuffer, val...) + l, r, b := getIndices(valueBuffer, start, alignedTime, init) + if !b { + newVal = c.processor.process(ts.Datapoints{}, alignedTime) + } else { + init = l + newVal = c.processor.process(valueBuffer[l:r], alignedTime) } if err := builder.AppendValue(i, newVal); err != nil { return nil, err } + + start = start.Add(bounds.StepSize) + alignedTime = alignedTime.Add(bounds.StepSize) } } @@ -432,14 +518,19 @@ func (c *baseNode) sweep(processedKeys []bool, maxBlocks int) { } } -// Processor is implemented by the underlying transforms. -type Processor interface { - Process(values ts.Datapoints, evaluationTime time.Time) float64 +// processor is implemented by the underlying transforms. +type processor interface { + process(valueBuffer ts.Datapoints, evaluationTime time.Time) float64 } -// MakeProcessor is a way to create a transform. -type MakeProcessor interface { - Init(op baseOp, controller *transform.Controller, opts transform.Options) Processor +// makeProcessor is a way to create a transform. +type makeProcessor interface { + // initialize initializes the processor. + initialize( + duration time.Duration, + controller *transform.Controller, + opts transform.Options, + ) processor } type processRequest struct { @@ -468,7 +559,7 @@ func newBlockCache(op baseOp, transformOpts transform.Options) *blockCache { } } -func (c *blockCache) init(bounds models.Bounds) { +func (c *blockCache) initialize(bounds models.Bounds) { if c.initialized { return } diff --git a/src/query/functions/temporal/base_test.go b/src/query/functions/temporal/base_test.go index 8308d63d21..9caa508153 100644 --- a/src/query/functions/temporal/base_test.go +++ b/src/query/functions/temporal/base_test.go @@ -39,14 +39,17 @@ import ( "github.com/stretchr/testify/require" ) -type processor struct { -} +type noopProcessor struct{} -func (p processor) Init(op baseOp, controller *transform.Controller, opts transform.Options) Processor { +func (p noopProcessor) initialize( + _ time.Duration, + controller *transform.Controller, + opts transform.Options, +) processor { return &p } -func (p *processor) Process(dps ts.Datapoints, _ time.Time) float64 { +func (p *noopProcessor) process(dps ts.Datapoints, _ time.Time) float64 { vals := dps.Values() sum := 0.0 for _, n := range vals { @@ -56,7 +59,8 @@ func (p *processor) Process(dps ts.Datapoints, _ time.Time) float64 { return sum } -func compareCacheState(t *testing.T, node *baseNode, bounds models.Bounds, state []bool, debugMsg string) { +func compareCacheState(t *testing.T, node *baseNode, + bounds models.Bounds, state []bool, debugMsg string) { actualState := make([]bool, len(state)) for i := range state { _, exists := node.cache.get(bounds.Next(i).Start) @@ -74,7 +78,7 @@ func TestBaseWithB0(t *testing.T) { baseOp := baseOp{ operatorType: "dummy", duration: 5 * time.Minute, - processorFn: processor{}, + processorFn: noopProcessor{}, } node := baseOp.Node(c, transformtest.Options(t, transform.OptionsParams{ @@ -342,7 +346,7 @@ func setup( baseOp := baseOp{ operatorType: "dummy", duration: duration, - processorFn: processor{}, + processorFn: noopProcessor{}, } node := baseOp.Node(c, transformtest.Options(t, transform.OptionsParams{ TimeSpec: transform.TimeSpec{ @@ -514,6 +518,7 @@ func (b *closeSpyBlock) Close() error { func TestSingleProcessRequest(t *testing.T) { values, bounds := test.GenerateValuesAndBounds(nil, nil) + bounds.Start = bounds.Start.Truncate(time.Hour) boundStart := bounds.Start seriesMetas := []block.SeriesMeta{{ @@ -546,7 +551,7 @@ func TestSingleProcessRequest(t *testing.T) { baseOp := baseOp{ operatorType: "dummy", duration: 5 * time.Minute, - processorFn: processor{}, + processorFn: noopProcessor{}, } node := baseOp.Node(c, transformtest.Options(t, transform.OptionsParams{ @@ -563,19 +568,29 @@ func TestSingleProcessRequest(t *testing.T) { deps: []block.UnconsolidatedBlock{block1}, queryCtx: models.NoopQueryContext(), } - bl, err := bNode.processSingleRequest(request) + + bl, err := bNode.processSingleRequest(request, nil) require.NoError(t, err) bNode.propagateNextBlocks([]processRequest{request}, []block.Block{bl}, 1) assert.Len(t, sink.Values, 2, "block processed") - // Current Block: 0 1 2 3 4 5 - // Previous Block: 10 11 12 13 14 15 - // i = 0; prev values [11, 12, 13, 14, 15], current values [0], sum = 50 - // i = 1; prev values [12, 13, 14, 15], current values [0, 1], sum = 40 - assert.Equal(t, sink.Values[0], []float64{50, 40, 30, 20, 10}, - "first series is 10 - 14 which sums to 60, the current block first series is 0-4 which sums to 10, we need 5 values per aggregation") - assert.Equal(t, sink.Values[1], []float64{75, 65, 55, 45, 35}, - "second series is 15 - 19 which sums to 85 and second series is 5-9 which sums to 35") + /* + NB: This test is a little weird to understand; it's simulating a test where + blocks come in out of order. Worked example for expected values below. As + a processing function, it simply adds all values together. + + For series 1: + Previous Block: 10 11 12 13 14, with 10 being outside of the period. + Current Block: 0 1 2 3 4 + + 1st value of processed block uses values: [11, 12, 13, 14], [0] = 50 + 2nd value of processed block uses values: [12, 13, 14], [0, 1] = 40 + 3rd value of processed block uses values: [13, 14], [0, 1, 2] = 30 + 4th value of processed block uses values: [14], [0, 1, 2, 3] = 20 + 5th value of processed block uses values: [0, 1, 2, 3, 4] = 10 + */ + require.Equal(t, sink.Values[0], []float64{50, 40, 30, 20, 10}) + assert.Equal(t, sink.Values[1], []float64{75, 65, 55, 45, 35}) // processSingleRequest renames the series to use their ids; reflect this in our expectation. expectedSeriesMetas := make([]block.SeriesMeta, len(seriesMetas)) diff --git a/src/query/functions/temporal/functions.go b/src/query/functions/temporal/functions.go index 12ec9b7881..2f5a8309f8 100644 --- a/src/query/functions/temporal/functions.go +++ b/src/query/functions/temporal/functions.go @@ -46,9 +46,12 @@ type functionProcessor struct { compFunc comparisonFunc } -func (f functionProcessor) Init(op baseOp, controller *transform.Controller, opts transform.Options) Processor { +func (f functionProcessor) initialize( + _ time.Duration, + controller *transform.Controller, + opts transform.Options, +) processor { return &functionNode{ - op: op, controller: controller, comparisonFunc: f.compFunc, } @@ -80,12 +83,11 @@ func NewFunctionOp(args []interface{}, optype string) (transform.Params, error) } type functionNode struct { - op baseOp controller *transform.Controller comparisonFunc comparisonFunc } -func (f *functionNode) Process(datapoints ts.Datapoints, _ time.Time) float64 { +func (f *functionNode) process(datapoints ts.Datapoints, _ time.Time) float64 { if len(datapoints) == 0 { return math.NaN() } diff --git a/src/query/functions/temporal/functions_test.go b/src/query/functions/temporal/functions_test.go index cece9ea5c8..0a588c527a 100644 --- a/src/query/functions/temporal/functions_test.go +++ b/src/query/functions/temporal/functions_test.go @@ -45,8 +45,8 @@ var testTemporalCases = []testCase{ {6, 4, 4, 2, 5}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 2}, + {math.NaN(), 0, 0, 0, 1}, + {math.NaN(), 1, 1, 2, 2}, }, afterAllBlocks: [][]float64{ {1, 2, 1, 1, 2}, @@ -61,8 +61,8 @@ var testTemporalCases = []testCase{ {6, 4, 4, 2, 5}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 2}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 3}, + {math.NaN(), 0, 1, 1, 2}, + {math.NaN(), 1, 1, 2, 3}, }, afterAllBlocks: [][]float64{ {2, 2, 2, 2, 3}, @@ -109,8 +109,8 @@ var testTemporalCases = []testCase{ {math.NaN(), 4, 4, 2, math.NaN()}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1}, + {math.NaN(), 0, 0, 1, 1}, + {math.NaN(), 0, 0, 1, 1}, }, afterAllBlocks: [][]float64{ {1, 2, 1, 1, 1}, @@ -125,8 +125,8 @@ var testTemporalCases = []testCase{ {math.NaN(), 4, 4, 2, math.NaN()}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 2}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1}, + {math.NaN(), 0, 1, 2, 2}, + {math.NaN(), 0, 0, 1, 1}, }, afterAllBlocks: [][]float64{ {2, 2, 2, 2, 2}, diff --git a/src/query/functions/temporal/holt_winters_test.go b/src/query/functions/temporal/holt_winters_test.go index 32e2fae269..01e8197792 100644 --- a/src/query/functions/temporal/holt_winters_test.go +++ b/src/query/functions/temporal/holt_winters_test.go @@ -40,8 +40,8 @@ var holtWintersTestCases = []testCase{ { name: "holt_winters", afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 4}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 9}, + {math.NaN(), math.NaN(), 2, 3, 4}, + {math.NaN(), 6, 7, 8, 9}, }, afterAllBlocks: [][]float64{ {4, 3.64, 3.1824, -4.8224, 4}, diff --git a/src/query/functions/temporal/linear_regression.go b/src/query/functions/temporal/linear_regression.go index 877647e0c3..13d0230ab0 100644 --- a/src/query/functions/temporal/linear_regression.go +++ b/src/query/functions/temporal/linear_regression.go @@ -30,11 +30,13 @@ import ( ) const ( - // PredictLinearType predicts the value of time series t seconds from now, based on the input series, using simple linear regression. + // PredictLinearType predicts the value of time series t seconds from now, + // based on the input series, using simple linear regression. // PredictLinearType should only be used with gauges. PredictLinearType = "predict_linear" - // DerivType calculates the per-second derivative of the time series, using simple linear regression. + // DerivType calculates the per-second derivative of the time series, + // using simple linear regression. // DerivType should only be used with gauges. DerivType = "deriv" ) @@ -44,9 +46,12 @@ type linearRegressionProcessor struct { isDeriv bool } -func (l linearRegressionProcessor) Init(op baseOp, controller *transform.Controller, opts transform.Options) Processor { +func (l linearRegressionProcessor) initialize( + _ time.Duration, + controller *transform.Controller, + opts transform.Options, +) processor { return &linearRegressionNode{ - op: op, controller: controller, timeSpec: opts.TimeSpec(), fn: l.fn, @@ -56,8 +61,12 @@ func (l linearRegressionProcessor) Init(op baseOp, controller *transform.Control type linearRegFn func(float64, float64) float64 -// NewLinearRegressionOp creates a new base temporal transform for linear regression functions -func NewLinearRegressionOp(args []interface{}, optype string) (transform.Params, error) { +// NewLinearRegressionOp creates a new base temporal transform +// for linear regression functions. +func NewLinearRegressionOp( + args []interface{}, + optype string, +) (transform.Params, error) { var ( fn linearRegFn isDeriv bool @@ -66,12 +75,14 @@ func NewLinearRegressionOp(args []interface{}, optype string) (transform.Params, switch optype { case PredictLinearType: if len(args) != 2 { - return emptyOp, fmt.Errorf("invalid number of args for %s: %d", PredictLinearType, len(args)) + return emptyOp, fmt.Errorf("invalid number of args for %s: %d", + PredictLinearType, len(args)) } duration, ok := args[1].(float64) if !ok { - return emptyOp, fmt.Errorf("unable to cast to scalar argument: %v for %s", args[1], PredictLinearType) + return emptyOp, fmt.Errorf("unable to cast to scalar argument: %v for %s", + args[1], PredictLinearType) } fn = func(slope, intercept float64) float64 { @@ -80,7 +91,8 @@ func NewLinearRegressionOp(args []interface{}, optype string) (transform.Params, case DerivType: if len(args) != 1 { - return emptyOp, fmt.Errorf("invalid number of args for %s: %d", DerivType, len(args)) + return emptyOp, fmt.Errorf("invalid number of args for %s: %d", + DerivType, len(args)) } fn = func(slope, _ float64) float64 { @@ -95,7 +107,8 @@ func NewLinearRegressionOp(args []interface{}, optype string) (transform.Params, duration, ok := args[0].(time.Duration) if !ok { - return emptyOp, fmt.Errorf("unable to cast to scalar argument: %v for %s", args[0], optype) + return emptyOp, fmt.Errorf("unable to cast to scalar argument: %v for %s", + args[0], optype) } l := linearRegressionProcessor{ @@ -107,14 +120,16 @@ func NewLinearRegressionOp(args []interface{}, optype string) (transform.Params, } type linearRegressionNode struct { - op baseOp controller *transform.Controller timeSpec transform.TimeSpec fn linearRegFn isDeriv bool } -func (l linearRegressionNode) Process(dps ts.Datapoints, evaluationTime time.Time) float64 { +func (l linearRegressionNode) process( + dps ts.Datapoints, + evaluationTime time.Time, +) float64 { if dps.Len() < 2 { return math.NaN() } @@ -125,8 +140,13 @@ func (l linearRegressionNode) Process(dps ts.Datapoints, evaluationTime time.Tim // linearRegression performs a least-square linear regression analysis on the // provided datapoints. It returns the slope, and the intercept value at the -// provided time. The algorithm we use comes from https://en.wikipedia.org/wiki/Simple_linear_regression. -func linearRegression(dps ts.Datapoints, interceptTime time.Time, isDeriv bool) (float64, float64) { +// provided time. +// Uses this algorithm: https://en.wikipedia.org/wiki/Simple_linear_regression. +func linearRegression( + dps ts.Datapoints, + interceptTime time.Time, + isDeriv bool, +) (float64, float64) { var ( n float64 sumTimeDiff, sumVals float64 diff --git a/src/query/functions/temporal/linear_regression_test.go b/src/query/functions/temporal/linear_regression_test.go index e3605cc2c4..3d0ba2dcb0 100644 --- a/src/query/functions/temporal/linear_regression_test.go +++ b/src/query/functions/temporal/linear_regression_test.go @@ -41,8 +41,8 @@ var testLinearRegressionCases = []testCase{ name: "predict_linear", opType: PredictLinearType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 5.6666}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 10.6666}, + {math.NaN(), math.NaN(), 3.6666, 4.6666, 5.6666}, + {math.NaN(), 7.6666, 8.6666, 9.6666, 10.6666}, }, afterAllBlocks: [][]float64{ {2, 0.1666, 0.1666, 2, 5.6666}, @@ -53,8 +53,8 @@ var testLinearRegressionCases = []testCase{ name: "deriv", opType: DerivType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 0.0166}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 0.0166}, + {math.NaN(), math.NaN(), 0.0166, 0.0166, 0.0166}, + {math.NaN(), 0.0166, 0.0166, 0.0166, 0.0166}, }, afterAllBlocks: [][]float64{ {0, -0.0083, -0.0083, 0, 0.0166}, @@ -76,8 +76,8 @@ var testLinearRegressionCasesSomeNaNs = []testCase{ name: "predict_linear some NaNs", opType: PredictLinearType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 5.6666}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 10.6666}, + {math.NaN(), math.NaN(), 3.6666, 4.6666, 5.6666}, + {math.NaN(), 7.6666, 8.6666, 9.6666, 10.6666}, }, afterAllBlocks: [][]float64{ {6.6666, 0.6153, 0.8461, 4.6666, 5.6666}, @@ -88,8 +88,8 @@ var testLinearRegressionCasesSomeNaNs = []testCase{ name: "deriv some NaNs", opType: DerivType, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 0.01666}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 0.01666}, + {math.NaN(), math.NaN(), 0.0166, 0.0166, 0.0166}, + {math.NaN(), 0.0166, 0.0166, 0.0166, 0.0166}, }, afterAllBlocks: [][]float64{ {0.0166, -0.0058, -0.0058, 0.0166, 0.0166}, diff --git a/src/query/functions/temporal/rate.go b/src/query/functions/temporal/rate.go index 9711f1a42f..d99807c00c 100644 --- a/src/query/functions/temporal/rate.go +++ b/src/query/functions/temporal/rate.go @@ -53,14 +53,18 @@ type rateProcessor struct { rateFn rateFn } -func (r rateProcessor) Init(op baseOp, controller *transform.Controller, opts transform.Options) Processor { +func (r rateProcessor) initialize( + duration time.Duration, + controller *transform.Controller, + opts transform.Options, +) processor { return &rateNode{ - op: op, controller: controller, timeSpec: opts.TimeSpec(), isRate: r.isRate, isCounter: r.isCounter, rateFn: r.rateFn, + duration: duration, } } @@ -108,15 +112,15 @@ func NewRateOp(args []interface{}, optype string) (transform.Params, error) { type rateFn func(ts.Datapoints, bool, bool, transform.TimeSpec, time.Duration) float64 type rateNode struct { - op baseOp controller *transform.Controller - timeSpec transform.TimeSpec isRate, isCounter bool + duration time.Duration + timeSpec transform.TimeSpec rateFn rateFn } -func (r *rateNode) Process(datapoints ts.Datapoints, _ time.Time) float64 { - return r.rateFn(datapoints, r.isRate, r.isCounter, r.timeSpec, r.op.duration) +func (r *rateNode) process(datapoints ts.Datapoints, _ time.Time) float64 { + return r.rateFn(datapoints, r.isRate, r.isCounter, r.timeSpec, r.duration) } func standardRateFunc( diff --git a/src/query/functions/temporal/rate_test.go b/src/query/functions/temporal/rate_test.go index 58df882316..e7a28f1157 100644 --- a/src/query/functions/temporal/rate_test.go +++ b/src/query/functions/temporal/rate_test.go @@ -53,8 +53,8 @@ var testRateCases = []testRateCase{ {1987036, 1988988, 1990940, 1992892, 1994844}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 37.1333}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 32.5333}, + {math.NaN(), math.NaN(), 37.1333, 37.1333, 37.1333}, + {math.NaN(), 32.5333, 32.5333, 32.5333, 32.5333}, }, afterAllBlocks: [][]float64{ {11312.6333, 37.1333, 37.1333, 37.1333, 37.1333}, @@ -69,8 +69,8 @@ var testRateCases = []testRateCase{ {1987036, 1988988, 1990940, math.NaN(), math.NaN()}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 32.5333}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 32.5333}, + {math.NaN(), math.NaN(), 32.5333, 32.5333, 32.5333}, + {math.NaN(), 32.5333, 32.5333, 32.5333, 32.5333}, }, afterAllBlocks: [][]float64{ {33117.2666, 32.5333, 32.5333, 32.5333, 32.5333}, @@ -101,8 +101,8 @@ var testRateCases = []testRateCase{ {1987036, 1988988, 1990940, 1992892, 1994844}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1165.0844}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 32.5333}, + {math.NaN(), math.NaN(), 0, 0, 1165.0844}, + {math.NaN(), 13.01333, 19.52, 26.0266, 32.5333}, }, afterAllBlocks: [][]float64{ {255709.8666, 259191.4666, 259191.4666, 258099.2, 4573.8666}, @@ -117,8 +117,8 @@ var testRateCases = []testRateCase{ {1987036, 1988988, 1990940, math.NaN(), 1994844}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1310.72}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 32.5333}, + {math.NaN(), math.NaN(), 0, 1310.72, 1310.72}, + {math.NaN(), 13.01333, 19.52, 19.52, 32.5333}, }, afterAllBlocks: [][]float64{ {255709.8666, 259191.4666, 258099.2, 4878.7911, 4878.7911}, @@ -156,8 +156,8 @@ var testDeltaCases = []testRateCase{ {1987036, 1988988, 1990940, 1992892, 1994844}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 2228}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1952}, + {math.NaN(), math.NaN(), 2228, 2228, 2228}, + {math.NaN(), 1952, 1952, 1952, 1952}, }, afterAllBlocks: [][]float64{ {-8912, 2228, 2228, 2228, 2228}, @@ -172,8 +172,8 @@ var testDeltaCases = []testRateCase{ {1987036, 1988988, 1990940, math.NaN(), math.NaN()}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 3904}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 1952}, + {math.NaN(), math.NaN(), 1952, 1952, 3904}, + {math.NaN(), 1952, 1952, 1952, 1952}, }, afterAllBlocks: [][]float64{ {-7808, 1952, 1952, 1952, 3904}, @@ -204,8 +204,8 @@ var testDeltaCases = []testRateCase{ {2299, 2299, 2299, 2787, 2787}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 8912}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 610}, + {math.NaN(), math.NaN(), 4456, 6684, 8912}, + {math.NaN(), 0, 0, 650.6666, 610}, }, afterAllBlocks: [][]float64{ {-2785, -2785, -2785, -2785, 11140}, @@ -220,8 +220,8 @@ var testDeltaCases = []testRateCase{ {2299, 2299, 2299, math.NaN(), 2787}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 6684}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 610}, + {math.NaN(), math.NaN(), 4456, 6684, 6684}, + {math.NaN(), 0, 0, 0, 610}, }, afterAllBlocks: [][]float64{ {-2785, -2785, -2785, 8912, 8912}, @@ -259,8 +259,8 @@ var testIncreaseCases = []testRateCase{ {1987036, 1988988, 1990940, 1992892, 1994844}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 8912}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 9760}, + {math.NaN(), math.NaN(), 4456, 6684, 8912}, + {math.NaN(), 3904, 5856, 7808, 9760}, }, afterAllBlocks: [][]float64{ {8355, 1087957.5, 1087957.5, 1087957.5, 1090742.5}, @@ -275,8 +275,8 @@ var testIncreaseCases = []testRateCase{ {1987036, 1988988, 1990940, math.NaN(), 1994844}, }, afterBlockOne: [][]float64{ - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 10176}, - {math.NaN(), math.NaN(), math.NaN(), math.NaN(), 9760}, + {math.NaN(), math.NaN(), 4456, 10176, 10176}, + {math.NaN(), 3904, 5856, 5856, 9760}, }, afterAllBlocks: [][]float64{ {1099222.5, 2178825, 2175915, 1163592, 1163592},