Cortex single-process (lite) mode.

.gitignore

@@ -5,6 +5,7 @@ cmd/ingester/ingester
 cmd/querier/querier
 cmd/ruler/ruler
 cmd/table-manager/table-manager
+cmd/lite/lite
 .uptodate
 .pkg
 pkg/ingester/client/cortex.pb.go

BUILD.bazel

@@ -0,0 +1,3 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_prefix")
+
+go_prefix("github.com/weaveworks/cortex")

Makefile

@@ -119,7 +119,8 @@ clean:
 # and https://github.com/bazelbuild/rules_go/issues/423.  If you ever regenerate
 # the BUILD files, watch out for the rules in vendor/golang.org/x/crypto/curve25519
 update-gazelle:
-	gazelle -go_prefix github.com/weaveworks/cortex -external vendored
+	gazelle -go_prefix github.com/weaveworks/cortex -external vendored \
+		-build_file_name BUILD.bazel
 
 bazel:
 	bazel build //cmd/...

WORKSPACE

@@ -1,7 +1,7 @@
 git_repository(
     name = "io_bazel_rules_go",
     remote = "https://github.com/bazelbuild/rules_go.git",
-    tag = "0.4.3",
+    commit = "9bd1d8969862baac16540d44d2cf87f6a44e7734",
 )
 load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
 

cmd/ingester/main.go

@@ -26,11 +26,13 @@ func main() {
 		}
 		chunkStoreConfig chunk.StoreConfig
 		storageConfig    chunk.StorageClientConfig
+		schemaConfig     chunk.SchemaConfig
 		ingesterConfig   ingester.Config
 	)
 	// Ingester needs to know our gRPC listen port.
 	ingesterConfig.ListenPort = &serverConfig.GRPCListenPort
-	util.RegisterFlags(&serverConfig, &chunkStoreConfig, &storageConfig, &ingesterConfig)
+	util.RegisterFlags(&serverConfig, &chunkStoreConfig, &storageConfig,
+		&schemaConfig, &ingesterConfig)
 	flag.Parse()
 
 	server, err := server.New(serverConfig)
@@ -39,18 +41,18 @@ func main() {
 	}
 	defer server.Shutdown()
 
-	storageClient, err := chunk.NewStorageClient(storageConfig)
+	storageClient, err := chunk.NewStorageClient(storageConfig, schemaConfig)
 	if err != nil {
 		log.Fatalf("Error initializing storage client: %v", err)
 	}
 
-	chunkStore, err := chunk.NewStore(chunkStoreConfig, storageClient)
+	chunkStore, err := chunk.NewStore(chunkStoreConfig, schemaConfig, storageClient)
 	if err != nil {
 		log.Fatal(err)
 	}
 	defer chunkStore.Stop()
 
-	ingester, err := ingester.New(ingesterConfig, chunkStore)
+	ingester, err := ingester.New(ingesterConfig, schemaConfig, chunkStore)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/lite/BUILD.bazel

@@ -0,0 +1,32 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/chunk:go_default_library",
+        "//pkg/distributor:go_default_library",
+        "//pkg/ingester:go_default_library",
+        "//pkg/ingester/client:go_default_library",
+        "//pkg/querier:go_default_library",
+        "//pkg/ring:go_default_library",
+        "//pkg/util:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/github.com/prometheus/common/log:go_default_library",
+        "//vendor/github.com/prometheus/common/route:go_default_library",
+        "//vendor/github.com/prometheus/prometheus/promql:go_default_library",
+        "//vendor/github.com/prometheus/prometheus/web/api/v1:go_default_library",
+        "//vendor/github.com/weaveworks/common/middleware:go_default_library",
+        "//vendor/github.com/weaveworks/common/server:go_default_library",
+        "//vendor/github.com/weaveworks/common/user:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "lite",
+    library = ":go_default_library",
+    visibility = ["//visibility:public"],
+)

cmd/lite/Dockerfile

@@ -0,0 +1,4 @@
+FROM       quay.io/prometheus/busybox:latest
+COPY       lite /bin/cortex
+EXPOSE     80
+ENTRYPOINT [ "/bin/cortex" ]

cmd/lite/main.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/common/route"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/web/api/v1"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+
+	"github.com/weaveworks/common/middleware"
+	"github.com/weaveworks/common/server"
+	"github.com/weaveworks/common/user"
+	"github.com/weaveworks/cortex/pkg/chunk"
+	"github.com/weaveworks/cortex/pkg/distributor"
+	"github.com/weaveworks/cortex/pkg/ingester"
+	"github.com/weaveworks/cortex/pkg/ingester/client"
+	"github.com/weaveworks/cortex/pkg/querier"
+	"github.com/weaveworks/cortex/pkg/ring"
+	"github.com/weaveworks/cortex/pkg/util"
+)
+
+func main() {
+	var (
+		serverConfig = server.Config{
+			MetricsNamespace: "cortex",
+			GRPCMiddleware: []grpc.UnaryServerInterceptor{
+				middleware.ServerUserHeaderInterceptor,
+			},
+		}
+
+		chunkStoreConfig  chunk.StoreConfig
+		distributorConfig distributor.Config
+		ingesterConfig    ingester.Config
+		schemaConfig      chunk.SchemaConfig
+		storageConfig     chunk.StorageClientConfig
+		unauthenticated   bool
+	)
+	// Ingester needs to know our gRPC listen port.
+	ingesterConfig.ListenPort = &serverConfig.GRPCListenPort
+	util.RegisterFlags(&serverConfig, &chunkStoreConfig, &distributorConfig,
+		&ingesterConfig, &schemaConfig, &storageConfig)
+	flag.BoolVar(&unauthenticated, "unauthenticated", false, "Set to true to disable multitenancy.")
+	flag.Parse()
+
+	server, err := server.New(serverConfig)
+	if err != nil {
+		log.Fatalf("Error initializing server: %v", err)
+	}
+	defer server.Shutdown()
+
+	storageClient, err := chunk.NewStorageClient(storageConfig, schemaConfig)
+	if err != nil {
+		log.Fatalf("Error initializing storage client: %v", err)
+	}
+
+	chunkStore, err := chunk.NewStore(chunkStoreConfig, schemaConfig, storageClient)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer chunkStore.Stop()
+
+	r, err := ring.New(ingesterConfig.RingConfig)
+	if err != nil {
+		log.Fatalf("Error initializing ring: %v", err)
+	}
+	defer r.Stop()
+	ingesterConfig.KVClient = r.KVClient
+
+	dist, err := distributor.New(distributorConfig, r)
+	if err != nil {
+		log.Fatalf("Error initializing distributor: %v", err)
+	}
+	defer dist.Stop()
+	prometheus.MustRegister(dist)
+
+	ingester, err := ingester.New(ingesterConfig, schemaConfig, chunkStore)
+	if err != nil {
+		log.Fatal(err)
+	}
+	prometheus.MustRegister(ingester)
+	defer ingester.Shutdown()
+
+	tableClient, err := chunk.NewDynamoDBTableClient(storageConfig.AWSStorageConfig.DynamoDBConfig)
+	if err != nil {
+		log.Fatalf("Error initializing DynamoDB table client: %v", err)
+	}
+
+	tableManager, err := chunk.NewTableManager(schemaConfig, tableClient)
+	if err != nil {
+		log.Fatalf("Error initializing DynamoDB table manager: %v", err)
+	}
+	tableManager.Start()
+	defer tableManager.Stop()
+
+	queryable := querier.NewQueryable(dist, chunkStore)
+	engine := promql.NewEngine(queryable, nil)
+	api := v1.NewAPI(engine, querier.DummyStorage{Queryable: queryable},
+		querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{})
+	promRouter := route.New(func(r *http.Request) (context.Context, error) {
+		return r.Context(), nil
+	}).WithPrefix("/api/prom/api/v1")
+	api.Register(promRouter)
+
+	activeMiddleware := middleware.AuthenticateUser
+	if unauthenticated {
+		activeMiddleware = middleware.Func(func(next http.Handler) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				ctx := user.InjectOrgID(r.Context(), "0")
+				next.ServeHTTP(w, r.WithContext(ctx))
+			})
+		})
+	}
+
+	subrouter := server.HTTP.PathPrefix("/api/prom").Subrouter()
+	subrouter.PathPrefix("/api/v1").Handler(activeMiddleware.Wrap(promRouter))
+	subrouter.Path("/read").Handler(activeMiddleware.Wrap(http.HandlerFunc(queryable.Q.RemoteReadHandler)))
+	subrouter.Path("/validate_expr").Handler(activeMiddleware.Wrap(http.HandlerFunc(dist.ValidateExprHandler)))
+	subrouter.Path("/user_stats").Handler(activeMiddleware.Wrap(http.HandlerFunc(dist.UserStatsHandler)))
+
+	client.RegisterIngesterServer(server.GRPC, ingester)
+	server.HTTP.Handle("/ready", http.HandlerFunc(ingester.ReadinessHandler))
+	server.HTTP.Handle("/flush", http.HandlerFunc(ingester.FlushHandler))
+	server.HTTP.Handle("/ring", r)
+	server.HTTP.Handle("/api/prom/push", activeMiddleware.Wrap(http.HandlerFunc(dist.PushHandler)))
+	server.Run()
+}

cmd/querier/BUILD → cmd/querier/BUILD.bazel

@@ -14,7 +14,6 @@ go_library(
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/prometheus/common/route:go_default_library",
         "//vendor/github.com/prometheus/prometheus/promql:go_default_library",
-        "//vendor/github.com/prometheus/prometheus/retrieval:go_default_library",
         "//vendor/github.com/prometheus/prometheus/web/api/v1:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",

cmd/querier/main.go

@@ -11,7 +11,6 @@ import (
 	"github.com/prometheus/common/log"
 	"github.com/prometheus/common/route"
 	"github.com/prometheus/prometheus/promql"
-	"github.com/prometheus/prometheus/retrieval"
 	"github.com/prometheus/prometheus/web/api/v1"
 
 	"github.com/weaveworks/common/middleware"
@@ -23,14 +22,6 @@ import (
 	"github.com/weaveworks/cortex/pkg/util"
 )
 
-type dummyTargetRetriever struct{}
-
-func (r dummyTargetRetriever) Targets() []*retrieval.Target { return nil }
-
-type dummyAlertmanagerRetriever struct{}
-
-func (r dummyAlertmanagerRetriever) Alertmanagers() []string { return nil }
-
 func main() {
 	var (
 		serverConfig = server.Config{
@@ -43,8 +34,10 @@ func main() {
 		distributorConfig distributor.Config
 		chunkStoreConfig  chunk.StoreConfig
 		storageConfig     chunk.StorageClientConfig
+		schemaConfig      chunk.SchemaConfig
 	)
-	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &chunkStoreConfig, &storageConfig)
+	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig,
+		&chunkStoreConfig, &storageConfig, &schemaConfig)
 	flag.Parse()
 
 	r, err := ring.New(ringConfig)
@@ -67,20 +60,21 @@ func main() {
 	defer server.Shutdown()
 	server.HTTP.Handle("/ring", r)
 
-	storageClient, err := chunk.NewStorageClient(storageConfig)
+	storageClient, err := chunk.NewStorageClient(storageConfig, schemaConfig)
 	if err != nil {
 		log.Fatalf("Error initializing storage client: %v", err)
 	}
 
-	chunkStore, err := chunk.NewStore(chunkStoreConfig, storageClient)
+	chunkStore, err := chunk.NewStore(chunkStoreConfig, schemaConfig, storageClient)
 	if err != nil {
 		log.Fatal(err)
 	}
 	defer chunkStore.Stop()
 
 	queryable := querier.NewQueryable(dist, chunkStore)
 	engine := promql.NewEngine(queryable, nil)
-	api := v1.NewAPI(engine, querier.DummyStorage{Queryable: queryable}, dummyTargetRetriever{}, dummyAlertmanagerRetriever{})
+	api := v1.NewAPI(engine, querier.DummyStorage{Queryable: queryable},
+		querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{})
 	promRouter := route.New(func(r *http.Request) (context.Context, error) {
 		return r.Context(), nil
 	}).WithPrefix("/api/prom/api/v1")

cmd/ruler/main.go

@@ -29,16 +29,18 @@ func main() {
 		rulerConfig       ruler.Config
 		chunkStoreConfig  chunk.StoreConfig
 		storageConfig     chunk.StorageClientConfig
+		schemaConfig      chunk.SchemaConfig
 	)
-	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &rulerConfig, &chunkStoreConfig, &storageConfig)
+	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig,
+		&rulerConfig, &chunkStoreConfig, &storageConfig, &schemaConfig)
 	flag.Parse()
 
-	storageClient, err := chunk.NewStorageClient(storageConfig)
+	storageClient, err := chunk.NewStorageClient(storageConfig, schemaConfig)
 	if err != nil {
 		log.Fatalf("Error initializing storage client: %v", err)
 	}
 
-	chunkStore, err := chunk.NewStore(chunkStoreConfig, storageClient)
+	chunkStore, err := chunk.NewStore(chunkStoreConfig, schemaConfig, storageClient)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/table-manager/main.go

@@ -20,18 +20,19 @@ func main() {
 				middleware.ServerUserHeaderInterceptor,
 			},
 		}
-		dynamoDBConfig     = chunk.DynamoDBConfig{}
-		tableManagerConfig = chunk.TableManagerConfig{}
+
+		dynamoDBConfig = chunk.DynamoDBConfig{}
+		schemaConfig   chunk.SchemaConfig
 	)
-	util.RegisterFlags(&serverConfig, &dynamoDBConfig, &tableManagerConfig)
+	util.RegisterFlags(&serverConfig, &dynamoDBConfig, &schemaConfig)
 	flag.Parse()
 
 	tableClient, err := chunk.NewDynamoDBTableClient(dynamoDBConfig)
 	if err != nil {
 		log.Fatalf("Error initializing DynamoDB table client: %v", err)
 	}
 
-	tableManager, err := chunk.NewTableManager(tableManagerConfig, tableClient)
+	tableManager, err := chunk.NewTableManager(schemaConfig, tableClient)
 	if err != nil {
 		log.Fatalf("Error initializing DynamoDB table manager: %v", err)
 	}

pkg/chunk/BUILD → pkg/chunk/BUILD.bazel

@@ -8,12 +8,14 @@ go_library(
         "chunk.go",
         "chunk_cache.go",
         "chunk_store.go",
+        "dynamodb_table_client.go",
         "inmemory_storage_client.go",
         "memcache_client.go",
         "schema.go",
         "schema_config.go",
         "schema_util.go",
         "storage_client.go",
+        "table_client.go",
         "table_manager.go",
     ],
     visibility = ["//visibility:public"],
@@ -41,6 +43,7 @@ go_library(
         "//vendor/github.com/weaveworks/common/mtime:go_default_library",
         "//vendor/github.com/weaveworks/common/user:go_default_library",
         "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/golang.org/x/time/rate:go_default_library",
     ],
 )
 
@@ -65,6 +68,8 @@ go_test(
         "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/dynamodb:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/s3:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/s3/s3iface:go_default_library",
         "//vendor/github.com/bradfitz/gomemcache/memcache:go_default_library",
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/prometheus/common/model:go_default_library",