From 3638daa8a6098dd8fc81ace7f2cb87c803ad5202 Mon Sep 17 00:00:00 2001 From: LeslieKid <3530611790@qq.com> Date: Tue, 19 Nov 2024 23:59:13 +0800 Subject: [PATCH 1/5] add user guide for compaction offload. --- .../cluster_deployment/with_meta.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/content/en/docs/user-guide/cluster_deployment/with_meta.md b/content/en/docs/user-guide/cluster_deployment/with_meta.md index 1141a465..c6494c72 100644 --- a/content/en/docs/user-guide/cluster_deployment/with_meta.md +++ b/content/en/docs/user-guide/cluster_deployment/with_meta.md @@ -187,9 +187,27 @@ timeout = "5s" server_addrs = ['http://{HoraeMetaAddr}:2379'] ``` +### Compaction Offload + +Compaction offload is also supported in `WithMeta` mode. To enable compaction offload, the compaction mode of HoraeDB Server should be configured as remote compaction: + +```toml +[analytic.compaction_mode] +compaction_mode = "Offload" +node_picker = "Remote" +``` + +A Compaction Server, responsible for executing the compaction task, is also needed. The configuration of Compaction Server is similar to the HoraeDB Server, except that the node type (default `HoraeDB`) should be set to `CompactionServer`: + +```toml +[cluster_deployment] +mode = "WithMeta" +node_type = "CompactionServer" +``` + ### Complete Config of HoraeDB -With all the parts of the configurations mentioned above, a runnable complete config for HoraeDB can be made. In order to make the HoraeDB cluster runnable, we can decide to adopt RocksDB-based WAL and local-disk-based Object Storage: +With all the parts of the configurations mentioned above, a runnable complete config for HoraeDB can be made. In order to make the HoraeDB cluster runnable, we can decide to adopt RocksDB-based WAL and local-disk-based Object Storage without compaction offload: ```toml [server] From ada4582dd052a51a5ce8f227f0353f9503f981e3 Mon Sep 17 00:00:00 2001 From: LeslieKid <3530611790@qq.com> Date: Thu, 21 Nov 2024 01:20:15 +0800 Subject: [PATCH 2/5] address review comments. --- .../docs/user-guide/cluster_deployment/with_meta.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/content/en/docs/user-guide/cluster_deployment/with_meta.md b/content/en/docs/user-guide/cluster_deployment/with_meta.md index c6494c72..e809beec 100644 --- a/content/en/docs/user-guide/cluster_deployment/with_meta.md +++ b/content/en/docs/user-guide/cluster_deployment/with_meta.md @@ -189,12 +189,19 @@ server_addrs = ['http://{HoraeMetaAddr}:2379'] ### Compaction Offload -Compaction offload is also supported in `WithMeta` mode. To enable compaction offload, the compaction mode of HoraeDB Server should be configured as remote compaction: +Compaction offload is also supported. To enable compaction offload, the corresponding compaction mode with node picker and endpoint should be configured. + +- `node_picker`: There are two types of node picker -- `Local` and `Remote`(WIP). + - When the `Local` is setted, the local compaction task would be offloaded to the specific remote compaction server, which decided by `endpoint`. +- `endpoint`: The endpoint, in the form `addr:port`, indicating the _grpc port_ of the remote compaction server. + +Here is an example for it: ```toml [analytic.compaction_mode] compaction_mode = "Offload" -node_picker = "Remote" +node_picker = "Local" +endpoint = "{RemoteCompactionServerAddr}:{RemoteCompactionServerGrpcPort}" ``` A Compaction Server, responsible for executing the compaction task, is also needed. The configuration of Compaction Server is similar to the HoraeDB Server, except that the node type (default `HoraeDB`) should be set to `CompactionServer`: From fd0c99aea6b6d50c1bf5a1bca9f3fcec2b5802a4 Mon Sep 17 00:00:00 2001 From: LeslieKid <3530611790@qq.com> Date: Thu, 21 Nov 2024 01:39:50 +0800 Subject: [PATCH 3/5] fix fmt. --- content/en/docs/user-guide/cluster_deployment/with_meta.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/user-guide/cluster_deployment/with_meta.md b/content/en/docs/user-guide/cluster_deployment/with_meta.md index e809beec..7068849a 100644 --- a/content/en/docs/user-guide/cluster_deployment/with_meta.md +++ b/content/en/docs/user-guide/cluster_deployment/with_meta.md @@ -191,7 +191,7 @@ server_addrs = ['http://{HoraeMetaAddr}:2379'] Compaction offload is also supported. To enable compaction offload, the corresponding compaction mode with node picker and endpoint should be configured. -- `node_picker`: There are two types of node picker -- `Local` and `Remote`(WIP). +- `node_picker`: There are two types of node picker -- `Local` and `Remote`(WIP). - When the `Local` is setted, the local compaction task would be offloaded to the specific remote compaction server, which decided by `endpoint`. - `endpoint`: The endpoint, in the form `addr:port`, indicating the _grpc port_ of the remote compaction server. From b0bb97fc143da60455d54a613ea7b0736604a8e7 Mon Sep 17 00:00:00 2001 From: LeslieKid <3530611790@qq.com> Date: Fri, 22 Nov 2024 14:41:23 +0800 Subject: [PATCH 4/5] address review comments. --- .../en/docs/user-guide/cluster_deployment/with_meta.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/content/en/docs/user-guide/cluster_deployment/with_meta.md b/content/en/docs/user-guide/cluster_deployment/with_meta.md index 7068849a..2a73e414 100644 --- a/content/en/docs/user-guide/cluster_deployment/with_meta.md +++ b/content/en/docs/user-guide/cluster_deployment/with_meta.md @@ -204,13 +204,7 @@ node_picker = "Local" endpoint = "{RemoteCompactionServerAddr}:{RemoteCompactionServerGrpcPort}" ``` -A Compaction Server, responsible for executing the compaction task, is also needed. The configuration of Compaction Server is similar to the HoraeDB Server, except that the node type (default `HoraeDB`) should be set to `CompactionServer`: - -```toml -[cluster_deployment] -mode = "WithMeta" -node_type = "CompactionServer" -``` +A Compaction Server, responsible for executing the compaction task, is also needed. The configuration of Compaction Server is similar to the HoraeDB Server. ### Complete Config of HoraeDB From 161709d75d58da06d5e90ee6c57188aeedcd9d4d Mon Sep 17 00:00:00 2001 From: Jiacai Liu Date: Fri, 22 Nov 2024 14:52:59 +0800 Subject: [PATCH 5/5] Update content/en/docs/user-guide/cluster_deployment/with_meta.md --- content/en/docs/user-guide/cluster_deployment/with_meta.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/user-guide/cluster_deployment/with_meta.md b/content/en/docs/user-guide/cluster_deployment/with_meta.md index 2a73e414..1023604c 100644 --- a/content/en/docs/user-guide/cluster_deployment/with_meta.md +++ b/content/en/docs/user-guide/cluster_deployment/with_meta.md @@ -204,7 +204,7 @@ node_picker = "Local" endpoint = "{RemoteCompactionServerAddr}:{RemoteCompactionServerGrpcPort}" ``` -A Compaction Server, responsible for executing the compaction task, is also needed. The configuration of Compaction Server is similar to the HoraeDB Server. +A Compaction Server, responsible for executing the compaction task, is also needed. Currently `horaedb-server` will act as this role, in the future we can move it to an independent service. ### Complete Config of HoraeDB