From e9cb649dcd8605cccd166f8c74c4265d9d82ecc2 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Mon, 3 Apr 2023 13:39:17 -0400 Subject: [PATCH] cmd/server: configure hashring replication factor using a singleton picker --- cmd/spicedb/main.go | 13 ++------ internal/testserver/cluster.go | 10 +++--- pkg/balancer/hashring.go | 31 +++++++++++++------ pkg/cmd/serve.go | 2 ++ pkg/cmd/server/server.go | 42 ++++++++++++++++++-------- pkg/cmd/server/zz_generated.options.go | 8 +++++ 6 files changed, 70 insertions(+), 36 deletions(-) diff --git a/cmd/spicedb/main.go b/cmd/spicedb/main.go index 60c721837a..0abf6113ee 100644 --- a/cmd/spicedb/main.go +++ b/cmd/spicedb/main.go @@ -4,7 +4,6 @@ import ( "errors" "os" - "github.com/cespare/xxhash/v2" "github.com/rs/zerolog" "github.com/sercand/kuberesolver/v3" "github.com/spf13/cobra" @@ -14,15 +13,11 @@ import ( log "github.com/authzed/spicedb/internal/logging" consistentbalancer "github.com/authzed/spicedb/pkg/balancer" "github.com/authzed/spicedb/pkg/cmd" + "github.com/authzed/spicedb/pkg/cmd/server" cmdutil "github.com/authzed/spicedb/pkg/cmd/server" "github.com/authzed/spicedb/pkg/cmd/testserver" ) -const ( - hashringReplicationFactor = 20 - backendsPerKey = 1 -) - var errParsing = errors.New("parsing error") func main() { @@ -30,11 +25,7 @@ func main() { kuberesolver.RegisterInCluster() // Enable consistent hashring gRPC load balancer - balancer.Register(consistentbalancer.NewConsistentHashringBuilder( - xxhash.Sum64, - hashringReplicationFactor, - backendsPerKey, - )) + balancer.Register(consistentbalancer.NewConsistentHashringBuilder(server.ConsistentHashringPicker)) log.SetGlobalLogger(zerolog.New(os.Stdout)) diff --git a/internal/testserver/cluster.go b/internal/testserver/cluster.go index 1791fe3661..ea289a34da 100644 --- a/internal/testserver/cluster.go +++ b/internal/testserver/cluster.go @@ -59,7 +59,9 @@ var testResolverBuilder = &SafeManualResolverBuilder{} func init() { // register hashring balancer - balancer.Register(hashbalancer.NewConsistentHashringBuilder(xxhash.Sum64, 20, 1)) + balancer.Register(hashbalancer.NewConsistentHashringBuilder( + hashbalancer.NewConsistentHashringPickerBuilder(xxhash.Sum64, 1500, 1)), + ) // Register a manual resolver.Builder that we can feed addresses for tests // Registration is not thread safe, so we register a single resolver.Builder @@ -124,7 +126,7 @@ type SafeManualResolver struct { // ResolveNow implements the resolver.Resolver interface // It sends the static list of addresses to the underlying resolver.ClientConn -func (r *SafeManualResolver) ResolveNow(options resolver.ResolveNowOptions) { +func (r *SafeManualResolver) ResolveNow(_ resolver.ResolveNowOptions) { if r.cc == nil { return } @@ -139,11 +141,11 @@ func (r *SafeManualResolver) Close() {} // TestClusterWithDispatch creates a cluster with `size` nodes // The cluster has a real dispatch stack that uses bufconn grpc connections func TestClusterWithDispatch(t testing.TB, size uint, ds datastore.Datastore) ([]*grpc.ClientConn, func()) { - return TestClusterWithDispatchAndCacheConfig(t, size, ds, true) + return TestClusterWithDispatchAndCacheConfig(t, size, ds) } // TestClusterWithDispatchAndCacheConfig creates a cluster with `size` nodes and with cache toggled. -func TestClusterWithDispatchAndCacheConfig(t testing.TB, size uint, ds datastore.Datastore, cacheEnabled bool) ([]*grpc.ClientConn, func()) { +func TestClusterWithDispatchAndCacheConfig(t testing.TB, size uint, ds datastore.Datastore) ([]*grpc.ClientConn, func()) { // each cluster gets a unique prefix since grpc resolution is process-global prefix := getPrefix(t) diff --git a/pkg/balancer/hashring.go b/pkg/balancer/hashring.go index 11cdbf0148..91fd702ae9 100644 --- a/pkg/balancer/hashring.go +++ b/pkg/balancer/hashring.go @@ -30,17 +30,25 @@ const ( var logger = grpclog.Component("consistenthashring") // NewConsistentHashringBuilder creates a new balancer.Builder that -// will create a consistent hashring balancer with the given config. +// will create a consistent hashring balancer with the picker builder. // Before making a connection, register it with grpc with: // `balancer.Register(consistent.NewConsistentHashringBuilder(hasher, factor, spread))` -func NewConsistentHashringBuilder(hasher consistent.HasherFunc, replicationFactor uint16, spread uint8) balancer.Builder { +func NewConsistentHashringBuilder(pickerBuilder base.PickerBuilder) balancer.Builder { return base.NewBalancerBuilder( BalancerName, - &consistentHashringPickerBuilder{hasher: hasher, replicationFactor: replicationFactor, spread: spread}, + pickerBuilder, base.Config{HealthCheck: true}, ) } +// NewConsistentHashringPickerBuilder creates a new picker builder +// that will create consistent hashrings according to the supplied +// config. If the ReplicationFactor is changed, that new parameter +// will be used when the next picker is created. +func NewConsistentHashringPickerBuilder(hasher consistent.HasherFunc, initialReplicationFactor uint16, spread uint8) *ConsistentHashringPickerBuilder { + return &ConsistentHashringPickerBuilder{hasher: hasher, ReplicationFactor: initialReplicationFactor, spread: spread} +} + type subConnMember struct { balancer.SubConn key string @@ -54,19 +62,22 @@ func (s subConnMember) Key() string { var _ consistent.Member = &subConnMember{} -type consistentHashringPickerBuilder struct { - hasher consistent.HasherFunc - replicationFactor uint16 - spread uint8 +// ConsistentHashringPickerBuilder is an implementation of base.PickerBuilder and +// is used to build pickers based on updates to the node architecture. +type ConsistentHashringPickerBuilder struct { + ReplicationFactor uint16 + + hasher consistent.HasherFunc + spread uint8 } -func (b *consistentHashringPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker { +func (b *ConsistentHashringPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker { logger.Infof("consistentHashringPicker: Build called with info: %v", info) if len(info.ReadySCs) == 0 { return base.NewErrPicker(balancer.ErrNoSubConnAvailable) } - hashring := consistent.MustNewHashring(b.hasher, b.replicationFactor) + hashring := consistent.MustNewHashring(b.hasher, b.ReplicationFactor) for sc, scInfo := range info.ReadySCs { if err := hashring.Add(subConnMember{ SubConn: sc, @@ -106,3 +117,5 @@ func (p *consistentHashringPicker) Pick(info balancer.PickInfo) (balancer.PickRe SubConn: chosen.SubConn, }, nil } + +var _ base.PickerBuilder = &ConsistentHashringPickerBuilder{} diff --git a/pkg/cmd/serve.go b/pkg/cmd/serve.go index 72eee86e73..f80ce5c4dc 100644 --- a/pkg/cmd/serve.go +++ b/pkg/cmd/serve.go @@ -106,6 +106,8 @@ func RegisterServeFlags(cmd *cobra.Command, config *server.Config) error { cmd.Flags().Uint16Var(&config.DispatchConcurrencyLimits.LookupSubjects, "dispatch-lookup-subjects-concurrency-limit", 0, "maximum number of parallel goroutines to create for each lookup subjects request or subrequest. defaults to --dispatch-concurrency-limit") cmd.Flags().Uint16Var(&config.DispatchConcurrencyLimits.ReachableResources, "dispatch-reachable-resources-concurrency-limit", 0, "maximum number of parallel goroutines to create for each reachable resources request or subrequest. defaults to --dispatch-concurrency-limit") + cmd.Flags().Uint16Var(&config.DispatchHashringReplicationFactor, "dispatch-hashring-replication-factor", 100, "set the replication factor of the consistent hasher used for the dispatcher") + // Flags for configuring API behavior cmd.Flags().BoolVar(&config.DisableV1SchemaAPI, "disable-v1-schema-api", false, "disables the V1 schema API") cmd.Flags().BoolVar(&config.DisableVersionResponse, "disable-version-response", false, "disables version response support in the API") diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index ce1c8fced2..81b2d46576 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -11,6 +11,7 @@ import ( "time" "github.com/authzed/grpcutil" + "github.com/cespare/xxhash/v2" grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/hashicorp/go-multierror" @@ -35,11 +36,23 @@ import ( v1svc "github.com/authzed/spicedb/internal/services/v1" "github.com/authzed/spicedb/internal/telemetry" "github.com/authzed/spicedb/pkg/balancer" + consistentbalancer "github.com/authzed/spicedb/pkg/balancer" datastorecfg "github.com/authzed/spicedb/pkg/cmd/datastore" "github.com/authzed/spicedb/pkg/cmd/util" "github.com/authzed/spicedb/pkg/datastore" ) +const ( + hashringReplicationFactor = 100 + backendsPerKey = 1 +) + +var ConsistentHashringPicker = consistentbalancer.NewConsistentHashringPickerBuilder( + xxhash.Sum64, + hashringReplicationFactor, + backendsPerKey, +) + //go:generate go run github.com/ecordell/optgen -output zz_generated.options.go . Config type Config struct { // API config @@ -67,18 +80,19 @@ type Config struct { SchemaPrefixesRequired bool // Dispatch options - DispatchServer util.GRPCServerConfig - DispatchMaxDepth uint32 - GlobalDispatchConcurrencyLimit uint16 - DispatchConcurrencyLimits graph.ConcurrencyLimits - DispatchUpstreamAddr string - DispatchUpstreamCAPath string - DispatchUpstreamTimeout time.Duration - DispatchClientMetricsEnabled bool - DispatchClientMetricsPrefix string - DispatchClusterMetricsEnabled bool - DispatchClusterMetricsPrefix string - Dispatcher dispatch.Dispatcher + DispatchServer util.GRPCServerConfig + DispatchMaxDepth uint32 + GlobalDispatchConcurrencyLimit uint16 + DispatchConcurrencyLimits graph.ConcurrencyLimits + DispatchUpstreamAddr string + DispatchUpstreamCAPath string + DispatchUpstreamTimeout time.Duration + DispatchClientMetricsEnabled bool + DispatchClientMetricsPrefix string + DispatchClusterMetricsEnabled bool + DispatchClusterMetricsPrefix string + Dispatcher dispatch.Dispatcher + DispatchHashringReplicationFactor uint16 DispatchCacheConfig CacheConfig ClusterDispatchCacheConfig CacheConfig @@ -238,6 +252,10 @@ func (c *Config) Complete(ctx context.Context) (RunnableServer, error) { } closeables.AddWithError(dispatcher.Close) + // Set this value to take effect the next time the replicas are updated + // Applies to ALL running servers. + ConsistentHashringPicker.ReplicationFactor = c.DispatchHashringReplicationFactor + if len(c.DispatchUnaryMiddleware) == 0 && len(c.DispatchStreamingMiddleware) == 0 { if c.GRPCAuthFunc == nil { c.DispatchUnaryMiddleware, c.DispatchStreamingMiddleware = DefaultDispatchMiddleware(log.Logger, auth.MustRequirePresharedKey(c.PresharedKey), ds) diff --git a/pkg/cmd/server/zz_generated.options.go b/pkg/cmd/server/zz_generated.options.go index a3e4dd33e2..f4ea3dbf04 100644 --- a/pkg/cmd/server/zz_generated.options.go +++ b/pkg/cmd/server/zz_generated.options.go @@ -52,6 +52,7 @@ func (c *Config) ToOption() ConfigOption { to.DispatchClusterMetricsEnabled = c.DispatchClusterMetricsEnabled to.DispatchClusterMetricsPrefix = c.DispatchClusterMetricsPrefix to.Dispatcher = c.Dispatcher + to.DispatchHashringReplicationFactor = c.DispatchHashringReplicationFactor to.DispatchCacheConfig = c.DispatchCacheConfig to.ClusterDispatchCacheConfig = c.ClusterDispatchCacheConfig to.DisableV1SchemaAPI = c.DisableV1SchemaAPI @@ -275,6 +276,13 @@ func WithDispatcher(dispatcher dispatch.Dispatcher) ConfigOption { } } +// WithDispatchHashringReplicationFactor returns an option that can set DispatchHashringReplicationFactor on a Config +func WithDispatchHashringReplicationFactor(dispatchHashringReplicationFactor uint16) ConfigOption { + return func(c *Config) { + c.DispatchHashringReplicationFactor = dispatchHashringReplicationFactor + } +} + // WithDispatchCacheConfig returns an option that can set DispatchCacheConfig on a Config func WithDispatchCacheConfig(dispatchCacheConfig CacheConfig) ConfigOption { return func(c *Config) {