Merge branch 'master' into large-tiles-aggregation

CHANGELOG.md

@@ -1,5 +1,53 @@
 # Changelog
 
+# 0.15.8
+
+## Misc
+
+- **M3DB**: Pause rollout of background cold flush process by revert until further testing ([6830a8cb4](https://github.com/m3db/m3/commit/6830a8cb4))
+
+# 0.15.7
+
+## Performance
+
+- **M3DB**: Background cold flush process to no longer block data snapshotting or commit log rotation ([#2460](https://github.com/m3db/m3/pull/2460))
+- **M3DB**: Validate individual index entries on decode instead of entire file on open, to further improve bootstrap speed ([#2468](https://github.com/m3db/m3/pull/2468))
+
+## Bug Fixes
+
+- **M3DB**: Strict JSON unmarshal (disallow unknown fields) for raw HTTP/JSON DB node APIs ([#2490](https://github.com/m3db/m3/pull/2490))
+- **M3Query**: Fix to regex selectors with leading wildcard ([#2505](https://github.com/m3db/m3/pull/#2505))
+
+## Documentation
+
+- **All**: Links to M3 meetup recordings ([#2494](https://github.com/m3db/m3/pull/2494))
+
+# 0.15.6
+
+## Features
+
+- **M3DB**: Add per-namespace indexing runtime options to define concurrency weighted to indexing ([#2446](https://github.com/m3db/m3/pull/2446))
+
+## Performance
+
+- **M3DB**: Faster bootstrapping with deferred index checksum and significantly lower memory bump at block rotation ([#2446](https://github.com/m3db/m3/pull/2446))
+- **M3DB**: Faster series aggregate metadata queries by intersecting postings term results with resolved query postings list ([#2441](https://github.com/m3db/m3/pull/2441))
+
+## Bug Fixes
+
+- **M3Query**: Fix for label matching behavior not the same as regular Prometheus for when label value is ".+" or ".*" ([#2479](https://github.com/m3db/m3/pull/2479))
+- **M3Query**: Special case when request from Go process such as Prometheus Go client library for searching series metadata using Go min/max UTC values ([#2487](https://github.com/m3db/m3/pull/2487))
+- **M3Query**: Auto-detect querying for entire retention time range ([#2483](https://github.com/m3db/m3/pull/2483))
+- **M3Query**: Fix for Graphite query for metric with single identifier and no dot separated elements ([#2450](https://github.com/m3db/m3/pull/2450))
+
+## Documentation
+
+- **M3Coordinator**: Add rollup rules example documentation ([#2461](https://github.com/m3db/m3/pull/2461), [#2462](https://github.com/m3db/m3/pull/2462))
+
+## Misc
+
+- **M3DB**: Expose cluster total shards and replicas as metrics ([#2452](https://github.com/m3db/m3/pull/2452))
+
 # 0.15.5
 
 ## Documentation

DEVELOPER.md

@@ -46,7 +46,7 @@ make m3dbnode
 
 ## Running the M3 stack locally
 
-Follow the instructions in `./scripts/development/m3_stack/README.md`
+Follow the instructions in [this README](./scripts/development/m3_stack/README.md).
 
 ## Testing Changes
 

Makefile

@@ -187,7 +187,7 @@ install-tools:
 	GOBIN=$(tools_bin_path) go install github.com/mauricelam/genny
 	GOBIN=$(tools_bin_path) go install github.com/mjibson/esc
 	GOBIN=$(tools_bin_path) go install github.com/pointlander/peg
-	GOBIN=$(tools_bin_path) go install github.com/prateek/gorename
+	GOBIN=$(tools_bin_path) go install github.com/robskillington/gorename
 	GOBIN=$(tools_bin_path) go install github.com/rakyll/statik
 	GOBIN=$(tools_bin_path) go install github.com/garethr/kubeval
 
@@ -251,12 +251,16 @@ docker-integration-test:
 	@echo "--- Running Docker integration test"
 	./scripts/docker-integration-tests/run.sh
 
-
 .PHONY: docker-compatibility-test
 docker-compatibility-test:
 	@echo "--- Running Prometheus compatibility test"
 	./scripts/comparator/run.sh
 
+.PHONY: prom-compat
+prom-compat:
+	@echo "--- Running local Prometheus compatibility test"
+	CI="false" make docker-compatibility-test
+
 .PHONY: site-build
 site-build:
 	@echo "Building site"

README.md

@@ -17,6 +17,8 @@ More information:
 M3 contributors and maintainers have monthly (every four weeks) meetings. Join our M3 meetup group to receive notifications on upcoming meetings: 
 [https://www.meetup.com/M3-Community/](https://www.meetup.com/M3-Community/).
 
+Recordings of past meetups can be found here: [https://vimeo.com/user/120001164/folder/2290331](https://vimeo.com/user/120001164/folder/2290331).
+
 ## Office Hours
 
 Members of the M3 team will hold office hours on the third Thursday of every month from 11-1pm EST. In order to join, make sure to sign up for a slot here: [https://calendly.com/chronosphere-intro/m3-community-office-hours](https://calendly.com/chronosphere-intro/m3-community-office-hours).

docs/m3db/architecture/storage.md

@@ -19,24 +19,24 @@ A fileset has the following files:
 * **Checkpoint file:** Stores a digest of the digests file and written at the succesful completion of a fileset volume being persisted, allows for quickly checking if a volume was completed.
 
 ```
-                                                     ┌─────────────────────┐
-┌─────────────────────┐  ┌─────────────────────┐     │     Index File      │
-│      Info File      │  │   Summaries File    │     │   (sorted by ID)    │
-├─────────────────────┤  │   (sorted by ID)    │     ├─────────────────────┤
-│- Block Start        │  ├─────────────────────┤  ┌─>│- Idx                │
-│- Block Size         │  │- Idx                │  │  │- ID                 │
-│- Entries (Num)      │  │- ID                 │  │  │- Size               │
-│- Major Version      │  │- Index Entry Offset ├──┘  │- Checksum           │
-│- Summaries (Num)    │  └─────────────────────┘     │- Data Entry Offset  ├──┐
-│- BloomFilter (K/M)  │                              │- Encoded Tags       |  |
-│- Snapshot Time      │                              └─────────────────────┘  │
-│- Type (Flush/Snap)  |                                                       |
-│- Snapshot ID        |                                                       │
-│- Volume Index       |                                                       │
-│- Minor Version      |                                                       │
-└─────────────────────┘                                                       │
-                                                                              │
-                         ┌─────────────────────┐  ┌───────────────────────────┘
+                                                     ┌───────────────────────┐
+┌─────────────────────┐  ┌─────────────────────┐     │     Index File        │
+│      Info File      │  │   Summaries File    │     │   (sorted by ID)      │
+├─────────────────────┤  │   (sorted by ID)    │     ├───────────────────────┤
+│- Block Start        │  ├─────────────────────┤  ┌─>│- Idx                  │
+│- Block Size         │  │- Idx                │  │  │- ID                   │
+│- Entries (Num)      │  │- ID                 │  │  │- Size                 │
+│- Major Version      │  │- Index Entry Offset ├──┘  │- Checksum             │
+│- Summaries (Num)    │  └─────────────────────┘     │- Data Entry Offset    ├──┐
+│- BloomFilter (K/M)  │                              │- Encoded Tags         │  │
+│- Snapshot Time      │                              │- Index Entry Checksum │  │
+│- Type (Flush/Snap)  │                              └───────────────────────┘  │
+│- Snapshot ID        │                                                         │
+│- Volume Index       │                                                         │
+│- Minor Version      │                                                         │
+└─────────────────────┘                                                         │
+                                                                                │
+                         ┌─────────────────────┐  ┌─────────────────────────────┘
 ┌─────────────────────┐  │  Bloom Filter File  │  │
 │    Digests File     │  ├─────────────────────┤  │  ┌─────────────────────┐
 ├─────────────────────┤  │- Bitset             │  │  │      Data File      │

go.mod

@@ -82,13 +82,13 @@ require (
 	github.com/pointlander/jetset v1.0.0 // indirect
 	github.com/pointlander/peg v1.0.0
 	github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a // indirect
-	github.com/prateek/gorename v0.0.0-20180424020013-52c7307cddd2
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/common v0.9.1
 	github.com/prometheus/prometheus v1.8.2-0.20200420081721-18254838fbe2
 	github.com/rakyll/statik v0.1.6
 	github.com/remeh/sizedwaitgroup v1.0.0 // indirect
 	github.com/rhysd/go-github-selfupdate v1.2.2 // indirect
+	github.com/robskillington/gorename v0.0.0-20180424020013-52c7307cddd2
 	github.com/russross/blackfriday v2.0.0+incompatible
 	github.com/rveen/ogdl v0.0.0-20200522080342-eeeda1a978e7 // indirect
 	github.com/satori/go.uuid v1.2.0
@@ -130,7 +130,7 @@ require (
 	gopkg.in/validator.v2 v2.0.0-20160201165114-3e4f037f12a1
 	gopkg.in/vmihailenco/msgpack.v2 v2.8.3
 	gopkg.in/yaml.v2 v2.2.8
-	gotest.tools v2.2.0+incompatible // indirect
+	gotest.tools v2.2.0+incompatible
 )
 
 // branch 0.9.3-pool-read-binary-3

go.sum

@@ -604,8 +604,6 @@ github.com/pointlander/peg v1.0.0/go.mod h1:WJTMcgeWYr6fZz4CwHnY1oWZCXew8GWCF93F
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a h1:AA9vgIBDjMHPC2McaGPojgV2dcI78ZC0TLNhYCXEKH8=
 github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a/go.mod h1:lzZQ3Noex5pfAy7mkAeCjcBDteYU85uWWnJ/y6gKU8k=
-github.com/prateek/gorename v0.0.0-20180424020013-52c7307cddd2 h1:/pJs9wFXnmhD12W+dnwoNJXPtLsxa78y+vzC9i/Hs+A=
-github.com/prateek/gorename v0.0.0-20180424020013-52c7307cddd2/go.mod h1:nw9dXugFBQe1pshgrdeRjyYrY+RxNZckdWkiGHX8URE=
 github.com/prometheus/alertmanager v0.20.0/go.mod h1:9g2i48FAyZW6BtbsnvHtMHQXl2aVtrORKwKVCQ+nbrg=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
@@ -641,6 +639,8 @@ github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/rhysd/go-github-selfupdate v1.2.2 h1:G+mNzkc1wEtpmM6sFS/Ghkeq+ad4Yp6EZEHyp//wGEo=
 github.com/rhysd/go-github-selfupdate v1.2.2/go.mod h1:khesvSyKcXDUxeySCedFh621iawCks0dS/QnHPcpCws=
+github.com/robskillington/gorename v0.0.0-20180424020013-52c7307cddd2 h1:t+C9QFlvAI+evRn96lz7eKyzo1CgDx3YVx3N/GJIetk=
+github.com/robskillington/gorename v0.0.0-20180424020013-52c7307cddd2/go.mod h1:CVTJ4xwzb/4H98jrd7NFgNoTAiL63scr2Pl7kqOcQAQ=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

scripts/development/m3_stack/README.md

@@ -17,7 +17,7 @@ The environment variables that let's you configure this setup are:
 
 ## Usage
 
-Use the `start_m3.sh` and `stop_m3.sh` scripts.
+Use the `start_m3.sh` and `stop_m3.sh` scripts. Requires successful run of `make m3dbnode` from project root first.
 
 ## Grafana
 
@@ -56,4 +56,4 @@ Load can easily be increased by modifying the `prometheus.yml` file to reduce th
 
 ## Containers Hanging / Unresponsive
 
-Running the entire stack can be resource intensive. If the containers are unresponsive try increasing the amount of cores and memory that the docker daemon is allowed to use.
+Running the entire stack can be resource intensive. If the containers are unresponsive try increasing the amount of cores and memory that the docker daemon is allowed to use.

scripts/docker-integration-tests/aggregator/test.sh

@@ -151,7 +151,7 @@ function test_aggregated_graphite_metric {
   echo "Read back aggregated averaged metric"
   ATTEMPTS=100 TIMEOUT=1 MAX_TIMEOUT=4 retry_with_backoff read_carbon foo.bar.* 42
 
-  # echo "Finished with carbon metrics"
+  echo "Finished with carbon metrics"
   kill $METRIC_EMIT_PID
   export METRIC_EMIT_PID="-1"
 }
@@ -172,7 +172,7 @@ function prometheus_remote_write {
   local label2_value=${label2_value:-label2}
 
   network_name="aggregator"
-  network=$(docker network ls | fgrep $network_name | tr -s ' ' | cut -f 1 -d ' ')
+  network=$(docker network ls | fgrep $network_name | tr -s ' ' | cut -f 1 -d ' ' | tail -n 1)
   out=$((docker run -it --rm --network $network           \
     $PROMREMOTECLI_IMAGE                                  \
     -u http://m3coordinator01:7202/api/v1/prom/remote/write \

scripts/docker-integration-tests/coordinator_config_rules/test.sh

@@ -43,7 +43,7 @@ function prometheus_remote_write {
   local label2_value=${label2_value:-label2}
 
   network_name="coordinator_config_rules"
-  network=$(docker network ls | fgrep $network_name | tr -s ' ' | cut -f 1 -d ' ')
+  network=$(docker network ls | fgrep $network_name | tr -s ' ' | cut -f 1 -d ' ' | tail -n 1)
   out=$((docker run -it --rm --network $network           \
     $PROMREMOTECLI_IMAGE                                  \
     -u http://coordinator01:7201/api/v1/prom/remote/write \

scripts/docker-integration-tests/prometheus/test.sh

@@ -77,7 +77,8 @@ function prometheus_remote_write {
     fi
   done
 
-  network=$(docker network ls --format '{{.ID}}' | tail -n 1)
+  network_name="prometheus"
+  network=$(docker network ls | fgrep $network_name | tr -s ' ' | cut -f 1 -d ' ' | tail -n 1)
   out=$((docker run -it --rm --network $network           \
     $PROMREMOTECLI_IMAGE                                  \
     -u http://coordinator01:7201/api/v1/prom/remote/write \
@@ -304,6 +305,29 @@ function test_query_restrict_tags {
     '[[ $(curl -s 0.0.0.0:7201/api/v1/query?query=\\{restricted_metrics_type=\"hidden\"\\} | jq -r ".data.result | length") -eq 0 ]]'
 }
 
+function test_series {
+  # Test series search with start/end specified
+  ATTEMPTS=5 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
+    '[[ $(curl -s "0.0.0.0:7201/api/v1/series?match[]=prometheus_remote_storage_succeeded_samples_total&start=0&end=9999999999999.99999" | jq -r ".data | length") -eq 1 ]]'
+
+  # Test series search with no start/end specified
+  ATTEMPTS=5 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
+    '[[ $(curl -s "0.0.0.0:7201/api/v1/series?match[]=prometheus_remote_storage_succeeded_samples_total" | jq -r ".data | length") -eq 1 ]]'
+
+  # Test series search with min/max start time using the Prometheus Go
+  # min/max formatted timestamps, which is sent as part of a Prometheus
+  # remote query.
+  # minTime = time.Unix(math.MinInt64/1000+62135596801, 0).UTC()
+  # maxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC()
+  # minTimeFormatted = minTime.Format(time.RFC3339Nano)
+  # maxTimeFormatted = maxTime.Format(time.RFC3339Nano)
+  # Which:
+  # minTimeFormatted="-292273086-05-16T16:47:06Z"
+  # maxTimeFormatted="292277025-08-18T07:12:54.999999999Z"
+  ATTEMPTS=5 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
+    '[[ $(curl -s "0.0.0.0:7201/api/v1/series?match[]=prometheus_remote_storage_succeeded_samples_total&start=-292273086-05-16T16:47:06Z&end=292277025-08-18T07:12:54.999999999Z" | jq -r ".data | length") -eq 1 ]]'
+}
+
 echo "Running prometheus tests"
 test_prometheus_remote_read
 test_prometheus_remote_write_multi_namespaces
@@ -316,6 +340,7 @@ test_query_limits_applied
 test_query_restrict_metrics_type
 test_query_restrict_tags
 test_prometheus_remote_write_map_tags
+test_series
 
 echo "Running function correctness tests"
 test_correctness

scripts/docker-integration-tests/prometheus_replication/test.sh

@@ -48,7 +48,8 @@ function prometheus_remote_write {
   local expect_status=$6
   local expect_status_err=$7
 
-  network=$(docker network ls --format '{{.ID}}' | tail -n 1)
+  network_name="prometheus_replication"
+  network=$(docker network ls | fgrep $network_name | tr -s ' ' | cut -f 1 -d ' ' | tail -n 1)
   out=$((docker run -it --rm --network $network           \
     $PROMREMOTECLI_IMAGE                                  \
     -u http://coordinator01:7201/api/v1/prom/remote/write \