From d73a2958a1bb2b8bd31e524e079b962626649392 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 6 Nov 2024 09:55:33 +0800 Subject: [PATCH] [improve](report) split agent batch tasks automaticlly (#43257) to avoid the binary message size exceeds the thrift max message size limitation --- .../src/main/java/org/apache/doris/common/Config.java | 8 ++++++++ .../main/java/org/apache/doris/master/ReportHandler.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8aedeb091f346a..92eedf4ab71f40 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1119,6 +1119,14 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int report_queue_size = 100; + // if the number of report task in FE exceed max_report_task_num_per_rpc, then split it to multiple rpc + @ConfField(mutable = true, masterOnly = true, description = { + "重新发送 agent task 时，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。", + "The maximum number of batched tasks per RPC assigned to each BE when resending agent tasks, " + + "the default value is 10000." + }) + public static int report_resend_batch_task_num_per_rpc = 10000; + /** * If set to true, metric collector will be run as a daemon timer to collect metrics at fix interval */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 9ead1a56ff893e..7ebc403575ee35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -524,7 +524,7 @@ private static void taskReport(long backendId, Map> running List diffTasks = AgentTaskQueue.getDiffTasks(backendId, runningTasks); - AgentBatchTask batchTask = new AgentBatchTask(); + AgentBatchTask batchTask = new AgentBatchTask(Config.report_resend_batch_task_num_per_rpc); long taskReportTime = System.currentTimeMillis(); for (AgentTask task : diffTasks) { // these tasks no need to do diff