Support for numbered filesets

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+# 0.11.0 (pending)
+
+## Migration Disclaimer
+
+Version 0.11.0 of M3 includes further work on supporting writing data at arbitrary times (within the retention period). While most of these changes are transparent to the user in terms of functionality and performance, we had to make a change to the naming format of the files that get persisted to disk (#1720). This change was required to handle multiple fileset volumes per block, which is necessary after introducing "cold flushes" (#1624).
+
+The redesign is **backwards compatible** but not **forwards compatible**. This means that you should be able upgrade your < 0.11.0 clusters to 0.11.0 with no issues, but you will not be able to downgrade without taking some additional steps.
+
+### Troubleshooting and Rolling Back
+
+If you run into any issues with the upgrade or need to downgrade to a previous version for any reason, follow these steps:
+
+1. Stop the node that is having trouble with the upgrade or that you're trying to downgrade.
+2. Remove the filesets that include a volume index in them, e.g. this is a filename with the new volumed format: `fileset-1257890400000000000-0-data.db`, and this is the corresponding filename in the original format: `fileset-1257890400000000000-data.db`.
+3. Modify the `bootstrappers` config in the M3DB YAML file from `filesystem, commitlog, peers, uninitialized_topology` to `filesystem, peers, commitlog, uninitialized_topology`. This will force the node to bootstrap from its peers instead of the local snapshot and commitlog files it has on disk, which is important otherwise when the node restarts, it will think that it has already been bootstrapped.
+4. Turn the node back on.
+
 # 0.10.2
 
 ## Performance

src/cmd/tools/dtest/util/seed/generator_test.go

@@ -128,18 +128,12 @@ func (t *fileInfoExtractor) visit(fPath string, f os.FileInfo, err error) error
 	t.shards[uint32(shardNum)] = struct{}{}
 
 	name := f.Name()
-	first := strings.Index(name, "-")
-	if first == -1 {
-		return fmt.Errorf("unable to find '-' in %v", name)
+	nameSplit := strings.Split(name, "-")
+	if len(nameSplit) < 2 {
+		return fmt.Errorf("unable to parse time from %v", name)
 	}
-	last := strings.LastIndex(name, "-")
-	if last == -1 {
-		return fmt.Errorf("unable to find '-' in %v", name)
-	}
-	if first == last {
-		return fmt.Errorf("found only single '-' in %v", name)
-	}
-	num, parseErr := strconv.ParseInt(name[first+1:last], 10, 64)
+
+	num, parseErr := strconv.ParseInt(nameSplit[1], 10, 64)
 	if parseErr != nil {
 		return err
 	}

src/dbnode/integration/disk_cleanup_helpers.go

@@ -27,11 +27,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/dbnode/persist/fs"
 	"github.com/m3db/m3/src/dbnode/persist/fs/commitlog"
 	"github.com/m3db/m3/src/dbnode/sharding"
 	"github.com/m3db/m3/src/dbnode/storage"
-	"github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/x/ident"
 
 	"github.com/stretchr/testify/require"
@@ -114,7 +114,7 @@ type cleanupTimesFileSet struct {
 
 func (fset *cleanupTimesFileSet) anyExist() bool {
 	for _, t := range fset.times {
-		exists, err := fs.DataFileSetExistsAt(fset.filePathPrefix, fset.namespace, fset.shard, t)
+		exists, err := fs.DataFileSetExists(fset.filePathPrefix, fset.namespace, fset.shard, t, 0)
 		if err != nil {
 			panic(err)
 		}

src/dbnode/integration/disk_cleanup_index_test.go

@@ -26,9 +26,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/dbnode/persist/fs"
 	"github.com/m3db/m3/src/dbnode/retention"
-	"github.com/m3db/m3/src/dbnode/namespace"
 	xclock "github.com/m3db/m3/src/x/clock"
 
 	"github.com/stretchr/testify/require"

src/dbnode/integration/disk_flush_helpers.go

@@ -30,11 +30,11 @@ import (
 
 	"github.com/m3db/m3/src/dbnode/encoding"
 	"github.com/m3db/m3/src/dbnode/integration/generate"
+	ns "github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/dbnode/persist"
 	"github.com/m3db/m3/src/dbnode/persist/fs"
 	"github.com/m3db/m3/src/dbnode/sharding"
 	"github.com/m3db/m3/src/dbnode/storage"
-	ns "github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/x/ident"
 	"github.com/m3db/m3/src/x/ident/testutil"
 	xtime "github.com/m3db/m3/src/x/time"
@@ -122,8 +122,8 @@ func waitUntilDataFilesFlushed(
 		for timestamp, seriesList := range testData {
 			for _, series := range seriesList {
 				shard := shardSet.Lookup(series.ID)
-				exists, err := fs.DataFileSetExistsAt(
-					filePathPrefix, namespace, shard, timestamp.ToTime())
+				exists, err := fs.DataFileSetExists(
+					filePathPrefix, namespace, shard, timestamp.ToTime(), 0)
 				if err != nil {
 					panic(err)
 				}