From 69a104713fa02b589d5ac59bbd64b427e9c55927 Mon Sep 17 00:00:00 2001 From: Jayant Shrivastava Date: Mon, 10 Oct 2022 10:48:08 -0400 Subject: [PATCH] changefeedccl: use numcpu >> 2 workers for event consumers Previously, a default value of 8 was used for the kvevent parallel consumer. The reason for this was that we observed performance improvements in a 15 node 32 VCPU cluster when we increased this parameter to 8. After 8, the improvements were much smaller. The issue with a default of 8 is that that on smaller machines, 8 workers can be too much overhead, especially since the work is CPU intensive. This change updates the default to be runtime.NumCPU() >> 2 workers, which aligns with using 8 workers on 32 VCPU machines. Fixes https://github.com/cockroachdb/cockroach/issues/89589 Epic: none Release note: None --- pkg/ccl/changefeedccl/changefeedbase/settings.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/ccl/changefeedccl/changefeedbase/settings.go b/pkg/ccl/changefeedccl/changefeedbase/settings.go index 0a364eba8c35..77a8ddcc9660 100644 --- a/pkg/ccl/changefeedccl/changefeedbase/settings.go +++ b/pkg/ccl/changefeedccl/changefeedbase/settings.go @@ -10,6 +10,7 @@ package changefeedbase import ( "encoding/json" + "runtime" "time" "github.com/cockroachdb/cockroach/pkg/settings" @@ -239,7 +240,13 @@ var EventConsumerWorkers = settings.RegisterIntSetting( settings.TenantWritable, "changefeed.event_consumer_workers", "the number of workers to use when processing events; 0 or 1 disables", - int64(util.ConstantWithMetamorphicTestRange("changefeed.consumer_max_workers", 8, 0, 32)), + func() int64 { + idealNumber := runtime.NumCPU() >> 2 + if idealNumber < 1 { + return 1 + } + return int64(idealNumber) + }(), settings.NonNegativeInt, ).WithPublic()