Merge branch 'master' into linasn/cold-flush-cleanup-panic-fix

* master:
  Start (#3396)
  [query] Graphite fix exponentialMovingAverage seeded constant when num steps used for window size (#3395)
  [query] Fix Graphite exponentialMovingAverage to use correct EMA const and return single series per input series (#3391)
  [DOCS] Add contribution guide for documentation (#3365)
  [dbnode] Skip bootstrapping shards from aggregation (#3394)

CONTRIBUTING.md

@@ -17,3 +17,7 @@ For code changes that affect M3 functionality, or fix bugs, read the [developer
 ## Reporting a Bug
 
 If you have found a bug in M3, [create an issue](https://github.com/m3db/m3/issues/new) and fill in the details asked for in the issue template as much as possible.
+
+## Documentation
+
+For additions and changes to M3 documentation, read the [documentation guide](DOCUMENTATION.md).

DOCUMENTATION.md

@@ -0,0 +1,101 @@
+# Documentation Guide
+
+M3 is a large and complex project, and any help you can offer to explain it better is most welcome. If you have a suggestion for the documentation M3 welcomes it, and documentation pull requests follow the same process as [code contributions](CONTRIBUTING.MD).
+
+The rest of this document explains how to setup the documentation locally and the structure of the content.
+
+## Setup Documentation Locally
+
+### Prerequisites
+
+The M3 documentation uses [Hugo](https://gohugo.io/), a static site generator written in Go. [Find installation instructions in the Hugo documentation](https://gohugo.io/getting-started/installing/).
+
+If you also want to run and serve the M3 documentation, you need the custom theme, [Victor](https://github.com/chronosphereio/victor). Clone or download it into the same directory as the M3 codebase.
+
+To run some tests and test the production build steps (this doesn't run M3 itself), you need [Docker](https://www.docker.com).
+
+## Folder Structure
+
+Hugo hosts the M3 documentation, website, and Open API spec.
+
+### Website
+
+The website is the HTML, CSS, and JavaScript files in the _site/static_ folder. Edit those files to make any changes.
+
+### Open API Docs
+
+Hugo hosts the Open API spec from the _site/static/openapi_ folder. To make changes to the spec, edit the _spec.yml_ file in _src/query/generated/assets_, and a CI job generates the file that Hugo hosts automatically.
+
+### Configuration
+
+The _site/config_ folder has configuration for Hugo, split into three folders:
+
+-   A default  _config.toml_ file in _site/config/\_default_
+-   Overridden configuration for development and production environments in _site/config/development_ and _site/config/production_ respectively.
+
+### Theme Overrides
+
+The _site/layouts_ folder adds a several changes and overridden files to the Victor theme used by the M3 documentation.
+
+### Documentation
+
+The _site/content_ folder contains the documentation files, organized by folders that match the paths for URLs. The _includes_ folder is a special folder not served as part of the documentation and files used by other files.
+
+#### Theme
+
+The M3 documentation uses its own (open source) theme, Victor. You can read all the features that theme provides in [the repository for Victor](https://github.com/chronosphereio/victor).
+
+Victor is a theme based on Hugo modules, [read more in the Hugo docs](https://gohugo.io/hugo-modules/use-modules/) about how to use and update it.
+
+#### Shortcodes
+
+The M3 documentation adds the following extra shortcodes:
+
+-   `{{% apiendpoint %}}` - Combines the values of `Params.api.localCordinator` + `Params.api.apiEndpoint` as defined in the site configuration to output the base API endpoint for M3 running on a localhost.
+-   `{{% docker-version %}}` - Outputs the value of `Params.releases.docker` as defined in the site configuration to output the consistent current Docker release.
+-   `{{% now %}}` - Outputs the current Unix timestamp to allow for up to date timestamps in code examples.
+
+## Running Documentation
+
+As noted in the prerequisites section, if you want to run the documentation locally to see how your edits look with `hugo server`, you need to have the Victor theme in the same parent directory as the M3 codebase, as `hugo server` runs in Hugo's "development" mode (and matches _site/config/development/config.toml_).
+
+This does mean that as you make changes to the theme or documentation, it refreshes automatically in the browser preview. Sometimes Hugo doesn't refresh included files, so you may need to restart the server process.
+
+## Testing Documentation
+
+The M3 documentation has a number of tests that need to pass before a pull request is merged to master and deployed. These include:
+
+-   Link checking with htmltest
+
+To run them you need Docker installed and running, and at the top level of the M3 project, run `make clean install-vendor-m3 docs-test`. If the test is successful you see green text, if not, htmltest tells you which links in which pages you need to fix.
+
+## Building Documentation
+
+There are a couple of different ways to build the documentation depending what you want to do.
+
+- If you want to build only the most up-to-date version of the docs, you can use `hugo` to build.
+- If you want to build all versions of the docs using Hugo run in Docker (this is what CI does to test the documentation). From the top level of the project, run `make docs-build`.
+- If you want to build all versions of the docs with a system-installed version of Hugo (this is what Netlify does to build and serve the documentation). From the top level of the project, run `make site-build`.
+
+## Creating a New Documentation Version
+
+M3 releases versions with some slight changes to documentation for each one which users can access from the drop down menu under the right hand navigation.
+
+Archiving a version of the documentation is a slightly complex process as Hugo doesn't natively support versioning.
+
+1.  Add the new version and path to _site/config/production/config.toml_ in the format:
+
+    ```toml
+    [[params.versions]]
+      version = "{versionNumber}"
+      url = "/v{versionNumber}/docs"
+    ```
+2.  Add the `/v{versionNumber}` path before each `{{ .RelPermalink}}` in the two anchor tags in _site/layouts/partials/navbar.html_
+3.  Make sure the Victor theme is at least version 0.2.11
+4.  Create and tag the commits with the name "docs/v{versionNumber}"
+5.  Switch to a new branch that will add the new version to the master branch
+6.  Add the new version to _site/config/production/config.toml_
+7.  Also add the new version to _site/config/production/config.toml_ in every pre-existing docs version tag
+8.  Switch back to the branch from step 5
+9.  Test everything locally with `make site-build`
+10. Push the new tag to the remote repository

site/content/includes/quickstart-common-steps.md

@@ -1,6 +1,6 @@
 ## Organizing Data with Placements and Namespaces
 
-A time series database (TSDBs) typically consist of one node (or instance) to store metrics data. This setup is simple to use but has issues with scalability over time as the quantity of metrics data written and read increases.
+A time series database (TSDB) typically consist of one node (or instance) to store metrics data. This setup is simple to use but has issues with scalability over time as the quantity of metrics data written and read increases.
 
 As a distributed TSDB, M3 helps solve this problem by spreading metrics data, and demand for that data, across multiple nodes in a cluster. M3 does this by splitting data into segments that match certain criteria (such as above a certain value) across nodes into shards.
 
@@ -189,7 +189,7 @@ curl {{% apiendpoint %}}services/m3db/placement | jq .
 {{% /notice %}}
 
 ### Ready a Namespace
-<!-- TODO: Why?> -->
+
 Once a namespace has finished bootstrapping, you must mark it as ready before receiving traffic by using the _{{% apiendpoint %}}services/m3db/namespace/ready_.
 
 {{< tabs name="ready_namespaces" >}}
@@ -512,10 +512,4 @@ curl -X "POST" -G "{{% apiendpoint %}}query_range" \
 ```
 
 {{% /tab %}}
-{{< /tabs >}}
-<!-- ## Next Steps
-
-This quickstart covered getting a single-node M3DB cluster running, and writing and querying metrics to the cluster. Some next steps are:
-
--   one
--   two -->
+{{< /tabs >}}

site/content/quickstart/_index.md

@@ -1,4 +1,4 @@
 ---
 title: "Quickstart"
 weight: 1
----
+---

site/content/quickstart/docker.md

@@ -1,39 +1,37 @@
 ---
 linkTitle: "Docker"
-title: Creating a Single Node M3DB Cluster with Docker
+title: Creating a Single Node M3 Cluster with Docker
 weight: 1
 ---
 
-<!-- TODO: Fix dates. Cross-platform date generation is a pain, so maybe use Docker locally? See what Netlify supports, or maybe there is a Hugo variable, or create a shortcode -->
+This guide shows how to install and configure M3, create a single-node cluster, and read and write metrics to it.
 
-This guide shows how to install and configure M3DB, create a single-node cluster, and read and write metrics to it.
-<!-- TODO: Does this link actually tell you that anymore? -->
 {{% notice warning %}}
-Deploying a single-node M3DB cluster is a great way to experiment with M3DB and get an idea of what it has to offer, but is not designed for production use. To run M3DB in clustered mode with a separate M3Coordinator, [read the clustered mode guide](/docs/cluster/).
+Deploying a single-node cluster is a great way to experiment with M3 and get an idea of what it has to offer, but isn't designed for production use. To run M3 in clustered mode with a separate M3 Coordinator, [read the clustered mode guides](/docs/cluster/).
 {{% /notice %}}
 
 ## Prerequisites
 
--   **Docker**: You don't need [Docker](https://www.docker.com/get-started) to run M3DB, but it is the simplest and quickest way.
+-   **Docker**: You don't need [Docker](https://www.docker.com/get-started) to run M3DB, but it's the simplest and quickest way.
     -   If you use Docker Desktop, we recommend the following minimum _Resources_ settings.
         -   _CPUs_: 2
         -   _Memory_: 8GB
         -   _Swap_: 1GB
         -   _Disk image size_: 16GB
--   **JQ**: This example uses [jq](https://stedolan.github.io/jq/) to format the output of API calls. It is not essential for using M3DB.
+-   **JQ**: This example uses [jq](https://stedolan.github.io/jq/) to format the output of API calls. It isn't essential for using M3DB.
 -   **curl**: This example uses curl for communicating with M3DB endpoints. You can also use alternatives such as [Wget](https://www.gnu.org/software/wget/) and [HTTPie](https://httpie.org/).
 
 ## Start Docker Container
 
-By default the official M3DB Docker image configures a single M3DB instance as one binary containing:
+By default the official M3 Docker image configures a single instance as one binary containing:
 
 -   An M3DB storage instance for time series storage. It includes an embedded tag-based metrics index and an etcd server for storing the cluster topology and runtime configuration.
--   A coordinator instance for writing and querying tagged metrics, as well as managing cluster topology and runtime configuration.
+-   A M3 Coordinator instance for writing and querying tagged metrics, as well as managing cluster topology and runtime configuration.
 
 The Docker container exposes three ports:
 
 -   `7201` to manage the cluster topology, you make most API calls to this endpoint
--   `7203` for Prometheus to scrape the metrics produced by M3DB and M3Coordinator
+-   `7203` for Prometheus to scrape the metrics produced by M3DB and M3 Coordinator
 
 The command below creates a persistent data directory on the host operating system to maintain durability and persistence between container restarts.
 
@@ -55,7 +53,7 @@ docker run -p 7201:7201 -p 7203:7203 --name m3db -v $(pwd)/m3db_data:/var/lib/m3
 {{< /tabs >}}
 
 {{% notice info %}}
-When running the command above on Docker for Mac, Docker for Windows, and some Linux distributions you may see errors about settings not being at recommended values. Unless you intend to run M3DB in production on macOS or Windows, you can ignore these warnings.
+When running the command above on Docker for Mac, Docker for Windows, and some Linux distributions you may see errors about settings not being at recommended values. Unless you intend to run M3 in production on macOS or Windows, you can ignore these warnings.
 {{% /notice %}}
 
 ## Configuration
@@ -67,7 +65,7 @@ The file groups configuration into `coordinator` or `db` sections that represent
 <!-- TODO: Replicate relevant sections -->
 
 {{% notice tip %}}
-You can find more information on configuring M3DB in the [operational guides section](/docs/operational_guide/).
+You can find more information on configuring M3 in the [operational guides section](/docs/operational_guide/).
 {{% /notice %}}
 
 {{< fileinclude file="quickstart-common-steps.md" >}}

src/dbnode/storage/namespace.go

@@ -1766,6 +1766,7 @@ func (n *dbNamespace) aggregateTiles(
 		processedTileCount int64
 		aggregationSuccess bool
 	)
+
 	defer func() {
 		if aggregationSuccess {
 			return
@@ -1776,7 +1777,15 @@ func (n *dbNamespace) aggregateTiles(
 				zap.Stringer("sourceNs", sourceNs.ID()), zap.Error(err))
 		}
 	}()
+
 	for _, targetShard := range n.OwnedShards() {
+		if !targetShard.IsBootstrapped() {
+			n.log.
+				With(zap.Uint32("shard", targetShard.ID())).
+				Debug("skipping aggregateTiles due to shard not bootstrapped")
+			continue
+		}
+
 		shardProcessedTileCount, err := targetShard.AggregateTiles(
 			ctx, sourceNs, n, targetShard.ID(), onColdFlushNs, opts)
 

src/dbnode/storage/namespace_test.go

@@ -1629,6 +1629,9 @@ func TestNamespaceAggregateTiles(t *testing.T) {
 	targetNs.shards[0] = targetShard0
 	targetNs.shards[1] = targetShard1
 
+	targetShard0.EXPECT().IsBootstrapped().Return(true)
+	targetShard1.EXPECT().IsBootstrapped().Return(true)
+
 	targetShard0.EXPECT().ID().Return(shard0ID)
 	targetShard1.EXPECT().ID().Return(shard1ID)
 
@@ -1646,6 +1649,54 @@ func TestNamespaceAggregateTiles(t *testing.T) {
 	assert.Equal(t, int64(3+2), processedTileCount)
 }
 
+func TestNamespaceAggregateTilesShipBootstrappingShards(t *testing.T) {
+	ctrl := xtest.NewController(t)
+	defer ctrl.Finish()
+
+	ctx := context.NewBackground()
+	defer ctx.Close()
+
+	var (
+		sourceNsID      = ident.StringID("source")
+		targetNsID      = ident.StringID("target")
+		sourceBlockSize = time.Hour
+		targetBlockSize = 2 * time.Hour
+		start           = time.Now().Truncate(targetBlockSize)
+		insOpts         = instrument.NewOptions()
+	)
+
+	opts, err := NewAggregateTilesOptions(start, start.Add(targetBlockSize), time.Second, targetNsID, insOpts)
+	require.NoError(t, err)
+
+	sourceNs, sourceCloser := newTestNamespaceWithIDOpts(t, sourceNsID, namespace.NewOptions())
+	defer sourceCloser()
+	sourceNs.bootstrapState = Bootstrapped
+	sourceRetentionOpts := sourceNs.nopts.RetentionOptions().SetBlockSize(sourceBlockSize)
+	sourceNs.nopts = sourceNs.nopts.SetRetentionOptions(sourceRetentionOpts)
+
+	targetNs, targetCloser := newTestNamespaceWithIDOpts(t, targetNsID, namespace.NewOptions())
+	defer targetCloser()
+	targetNs.bootstrapState = Bootstrapped
+	targetRetentionOpts := targetNs.nopts.RetentionOptions().SetBlockSize(targetBlockSize)
+	targetNs.nopts = targetNs.nopts.SetColdWritesEnabled(true).SetRetentionOptions(targetRetentionOpts)
+
+	targetShard0 := NewMockdatabaseShard(ctrl)
+	targetShard1 := NewMockdatabaseShard(ctrl)
+	targetNs.shards[0] = targetShard0
+	targetNs.shards[1] = targetShard1
+
+	targetShard0.EXPECT().IsBootstrapped().Return(false)
+	targetShard1.EXPECT().IsBootstrapped().Return(false)
+
+	targetShard0.EXPECT().ID().Return(uint32(10))
+	targetShard1.EXPECT().ID().Return(uint32(11))
+
+	processedTileCount, err := targetNs.AggregateTiles(ctx, sourceNs, opts)
+
+	require.NoError(t, err)
+	assert.Zero(t, processedTileCount)
+}
+
 func waitForStats(
 	reporter xmetrics.TestStatsReporter,
 	check func(xmetrics.TestStatsReporter) bool,