[dbnode] Make caching after block retrieval a configuration option

src/cmd/services/m3dbnode/config/config.go

@@ -261,6 +261,10 @@ type BlockRetrievePolicy struct {
 	// FetchConcurrency is the concurrency to fetch blocks from disk. For
 	// spinning disks it is highly recommended to set this value to 1.
 	FetchConcurrency int `yaml:"fetchConcurrency" validate:"min=0"`
+
+	// CacheBlocksOnRetrieve globally enables/disables callbacks used to cache blocks fetched
+	// from disk.
+	CacheBlocksOnRetrieve *bool `yaml:"cacheBlocksOnRetrieve"`
 }
 
 // CommitLogPolicy is the commit log policy.

src/cmd/tools/dtest/docker/harness/resources/harness.go

@@ -27,7 +27,9 @@ import (
 	"github.com/m3db/m3/src/query/generated/proto/admin"
 	xerrors "github.com/m3db/m3/src/x/errors"
 	"github.com/m3db/m3/src/x/instrument"
-	dockertest "github.com/ory/dockertest"
+
+	protobuftypes "github.com/gogo/protobuf/types"
+	"github.com/ory/dockertest"
 	"go.uber.org/zap"
 )
 
@@ -150,13 +152,14 @@ func SetupSingleM3DBNode() (DockerResources, error) {
 		coldWriteNamespace = admin.NamespaceAddRequest{
 			Name: ColdWriteNsName,
 			Options: &namespace.NamespaceOptions{
-				BootstrapEnabled:  true,
-				FlushEnabled:      true,
-				WritesToCommitLog: true,
-				CleanupEnabled:    true,
-				SnapshotEnabled:   true,
-				RepairEnabled:     true,
-				ColdWritesEnabled: true,
+				BootstrapEnabled:      true,
+				FlushEnabled:          true,
+				WritesToCommitLog:     true,
+				CleanupEnabled:        true,
+				SnapshotEnabled:       true,
+				RepairEnabled:         true,
+				ColdWritesEnabled:     true,
+				CacheBlocksOnRetrieve: &protobuftypes.BoolValue{Value: true},
 				RetentionOptions: &namespace.RetentionOptions{
 					RetentionPeriodNanos:                     int64(4 * time.Hour),
 					BlockSizeNanos:                           int64(time.Hour),

src/dbnode/generated/proto/namespace/namespace.proto

@@ -21,17 +21,18 @@ message IndexOptions {
 }
 
 message NamespaceOptions {
-    bool bootstrapEnabled                  = 1;
-    bool flushEnabled                      = 2;
-    bool writesToCommitLog                 = 3;
-    bool cleanupEnabled                    = 4;
-    bool repairEnabled                     = 5;
-    RetentionOptions retentionOptions      = 6;
-    bool snapshotEnabled                   = 7;
-    IndexOptions indexOptions              = 8;
-    SchemaOptions schemaOptions            = 9;
-    bool coldWritesEnabled                 = 10;
-    NamespaceRuntimeOptions runtimeOptions = 11;
+    bool bootstrapEnabled                           = 1;
+    bool flushEnabled                               = 2;
+    bool writesToCommitLog                          = 3;
+    bool cleanupEnabled                             = 4;
+    bool repairEnabled                              = 5;
+    RetentionOptions retentionOptions               = 6;
+    bool snapshotEnabled                            = 7;
+    IndexOptions indexOptions                       = 8;
+    SchemaOptions schemaOptions                     = 9;
+    bool coldWritesEnabled                          = 10;
+    NamespaceRuntimeOptions runtimeOptions          = 11;
+    google.protobuf.BoolValue cacheBlocksOnRetrieve = 12;
 }
 
 message Registry {

src/dbnode/namespace/config.go

@@ -48,15 +48,16 @@ func (m *MapConfiguration) Map() (Map, error) {
 
 // MetadataConfiguration is the configuration for a single namespace
 type MetadataConfiguration struct {
-	ID                string                  `yaml:"id" validate:"nonzero"`
-	BootstrapEnabled  *bool                   `yaml:"bootstrapEnabled"`
-	FlushEnabled      *bool                   `yaml:"flushEnabled"`
-	WritesToCommitLog *bool                   `yaml:"writesToCommitLog"`
-	CleanupEnabled    *bool                   `yaml:"cleanupEnabled"`
-	RepairEnabled     *bool                   `yaml:"repairEnabled"`
-	ColdWritesEnabled *bool                   `yaml:"coldWritesEnabled"`
-	Retention         retention.Configuration `yaml:"retention" validate:"nonzero"`
-	Index             IndexConfiguration      `yaml:"index"`
+	ID                    string                  `yaml:"id" validate:"nonzero"`
+	BootstrapEnabled      *bool                   `yaml:"bootstrapEnabled"`
+	FlushEnabled          *bool                   `yaml:"flushEnabled"`
+	WritesToCommitLog     *bool                   `yaml:"writesToCommitLog"`
+	CleanupEnabled        *bool                   `yaml:"cleanupEnabled"`
+	RepairEnabled         *bool                   `yaml:"repairEnabled"`
+	ColdWritesEnabled     *bool                   `yaml:"coldWritesEnabled"`
+	CacheBlocksOnRetrieve *bool                   `yaml:"cacheBlocksOnRetrieve"`
+	Retention             retention.Configuration `yaml:"retention" validate:"nonzero"`
+	Index                 IndexConfiguration      `yaml:"index"`
 }
 
 // Metadata returns a Metadata corresponding to the receiver struct
@@ -84,6 +85,9 @@ func (mc *MetadataConfiguration) Metadata() (Metadata, error) {
 	if v := mc.ColdWritesEnabled; v != nil {
 		opts = opts.SetColdWritesEnabled(*v)
 	}
+	if v := mc.CacheBlocksOnRetrieve; v != nil {
+		opts = opts.SetCacheBlocksOnRetrieve(*v)
+	}
 	return NewMetadata(ident.StringID(mc.ID), opts)
 }