From 5957f85f6552fcc14a3ce908c36b726a4e32e0a7 Mon Sep 17 00:00:00 2001 From: Wine93 Date: Thu, 15 Jun 2023 19:58:38 +0800 Subject: [PATCH] Fix(deploy): wait all chunkserver online time (#114). Signed-off-by: Wine93 --- cli/command/deploy.go | 10 +++++++++- cli/command/migrate.go | 1 + cli/command/scale_out.go | 5 ++++- internal/common/common.go | 9 +++++---- .../task/scripts/shell/wait_chunkserver.sh | 18 +++++++++--------- internal/task/task/common/create_pool.go | 7 ++++--- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/cli/command/deploy.go b/cli/command/deploy.go index fda7c1d71..4b40b2834 100644 --- a/cli/command/deploy.go +++ b/cli/command/deploy.go @@ -200,13 +200,20 @@ func precheckBeforeDeploy(curveadm *cli.CurveAdm, return nil } +func calcNumOfChunkserver(curveadm *cli.CurveAdm, dcs []*topology.DeployConfig) int { + services := curveadm.FilterDeployConfigByRole(dcs, topology.ROLE_CHUNKSERVER) + return len(services) +} + func genDeployPlaybook(curveadm *cli.CurveAdm, dcs []*topology.DeployConfig, options deployOptions) (*playbook.Playbook, error) { + var steps []int kind := dcs[0].GetKind() - steps := CURVEFS_DEPLOY_STEPS if kind == topology.KIND_CURVEBS { steps = CURVEBS_DEPLOY_STEPS + } else { + steps = CURVEFS_DEPLOY_STEPS } steps = skipDeploySteps(steps, options) @@ -230,6 +237,7 @@ func genDeployPlaybook(curveadm *cli.CurveAdm, options[comm.KEY_CREATE_POOL_TYPE] = comm.POOL_TYPE_PHYSICAL } else if step == CREATE_LOGICAL_POOL { options[comm.KEY_CREATE_POOL_TYPE] = comm.POOL_TYPE_LOGICAL + options[comm.KEY_NUMBER_OF_CHUNKSERVER] = calcNumOfChunkserver(curveadm, dcs) } pb.AddStep(&playbook.PlaybookStep{ diff --git a/cli/command/migrate.go b/cli/command/migrate.go index 2a1b1ddf5..a393fa36a 100644 --- a/cli/command/migrate.go +++ b/cli/command/migrate.go @@ -214,6 +214,7 @@ func genMigratePlaybook(curveadm *cli.CurveAdm, options[comm.KEY_CREATE_POOL_TYPE] = comm.POOL_TYPE_LOGICAL options[comm.KEY_MIGRATE_SERVERS] = migrates options[comm.KEY_NEW_TOPOLOGY_DATA] = data + options[comm.KEY_NUMBER_OF_CHUNKSERVER] = calcNumOfChunkserver(curveadm, dcs) case playbook.UPDATE_TOPOLOGY: options[comm.KEY_NEW_TOPOLOGY_DATA] = data } diff --git a/cli/command/scale_out.go b/cli/command/scale_out.go index e2d272afe..e11172f15 100644 --- a/cli/command/scale_out.go +++ b/cli/command/scale_out.go @@ -23,6 +23,8 @@ package command import ( + "time" + "github.com/fatih/color" "github.com/opencurve/curveadm/cli/cli" comm "github.com/opencurve/curveadm/internal/common" @@ -33,7 +35,6 @@ import ( cliutil "github.com/opencurve/curveadm/internal/utils" utils "github.com/opencurve/curveadm/internal/utils" "github.com/spf13/cobra" - "time" ) var ( @@ -334,6 +335,8 @@ func genScaleOutPlaybook(curveadm *cli.CurveAdm, options[comm.KEY_CREATE_POOL_TYPE] = comm.POOL_TYPE_LOGICAL options[comm.KEY_SCALE_OUT_CLUSTER] = dcs2scaleOut options[comm.KEY_NEW_TOPOLOGY_DATA] = data + options[comm.KEY_NUMBER_OF_CHUNKSERVER] = calcNumOfChunkserver(curveadm, dcs) + + calcNumOfChunkserver(curveadm, dcs2scaleOut) case playbook.UPDATE_TOPOLOGY: options[comm.KEY_NEW_TOPOLOGY_DATA] = data } diff --git a/internal/common/common.go b/internal/common/common.go index c897f0cd7..8ea729852 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -41,10 +41,11 @@ var ( // task options const ( // common - KEY_ALL_DEPLOY_CONFIGS = "ALL_DEPLOY_CONFIGS" - KEY_CREATE_POOL_TYPE = "POOL_TYPE" - POOL_TYPE_LOGICAL = "logicalpool" - POOL_TYPE_PHYSICAL = "physicalpool" + KEY_ALL_DEPLOY_CONFIGS = "ALL_DEPLOY_CONFIGS" + KEY_CREATE_POOL_TYPE = "POOL_TYPE" + POOL_TYPE_LOGICAL = "logicalpool" + POOL_TYPE_PHYSICAL = "physicalpool" + KEY_NUMBER_OF_CHUNKSERVER = "NUMBER_OF_CHUNKSERVER" // format KEY_ALL_FORMAT_STATUS = "ALL_FORMAT_STATUS" diff --git a/internal/task/scripts/shell/wait_chunkserver.sh b/internal/task/scripts/shell/wait_chunkserver.sh index e62f91fd0..303783d72 100644 --- a/internal/task/scripts/shell/wait_chunkserver.sh +++ b/internal/task/scripts/shell/wait_chunkserver.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash -# Usage: no args, just run it in bash +# Usage: wait_chunkserver 3 # Created Date: 2022-03-09 # Author: aspirer -# FIXME(P0): wait not works, return -12 +g_total="$1" + wait=0 while ((wait<60)) do - status=$(curve_ops_tool chunkserver-status |grep "offline") - total=$(echo ${status} | grep -c "total num = 0") - offline=$(echo ${status} | grep -c "offline = 0") - if [ ${total} -eq 0 ] && [ ${offline} -eq 1 ]; then - echo "CURVEADM_SUCCESS" + online=$(curve_ops_tool chunkserver-status | sed -nr 's/.*online = ([0-9]+).*/\1/p') + if [[ $online -eq $g_total ]]; then exit 0 fi + sleep 0.5s - wait=$(expr ${wait} + 1) + wait=$((wait+1)) done -echo "CURVEADM_TIMEOUT" + +echo "wait all chunkserver online timeout, total=$g_total, online=$online" exit 1 diff --git a/internal/task/task/common/create_pool.go b/internal/task/task/common/create_pool.go index 62a85062f..dcbf5a00a 100644 --- a/internal/task/task/common/create_pool.go +++ b/internal/task/task/common/create_pool.go @@ -187,8 +187,8 @@ func NewCreateTopologyTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (* // new task pooltype := curveadm.MemStorage().Get(comm.KEY_CREATE_POOL_TYPE).(string) - name := utils.Choose(pooltype == comm.POOL_TYPE_LOGICAL, "Create Logical Pool", - "Create Physical Pool") + name := utils.Choose(pooltype == comm.POOL_TYPE_LOGICAL, + "Create Logical Pool", "Create Physical Pool") subname := fmt.Sprintf("host=%s role=%s containerId=%s", dc.GetHost(), dc.GetRole(), tui.TrimContainerId(containerId)) t := task.NewTask(name, subname, hc.GetSSHConfig()) @@ -244,6 +244,7 @@ func NewCreateTopologyTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (* if dc.GetKind() == topology.KIND_CURVEBS && pooltype == comm.POOL_TYPE_LOGICAL { waitChunkserversScript := scripts.WAIT_CHUNKSERVERS waitChunkserversScriptPath := fmt.Sprintf("%s/wait_chunkservers.sh", layout.ToolsBinDir) + nchunkserver := curveadm.MemStorage().Get(comm.KEY_NUMBER_OF_CHUNKSERVER).(int) t.AddStep(&step.InstallFile{ // install wait_chunkservers script ContainerId: &containerId, ContainerDestPath: waitChunkserversScriptPath, @@ -252,7 +253,7 @@ func NewCreateTopologyTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (* }) t.AddStep(&step.ContainerExec{ // wait all chunkservers online before create logical pool ContainerId: &containerId, - Command: fmt.Sprintf("bash %s", waitChunkserversScriptPath), + Command: fmt.Sprintf("bash %s %d", waitChunkserversScriptPath, nchunkserver), Success: &success, Out: &out, ExecOptions: curveadm.ExecOptions(),