diff --git a/managed/devops/roles/configure-ybc-server/tasks/download-ybc-package.yml b/managed/devops/roles/configure-ybc-server/tasks/download-ybc-package.yml index 3c4c95d58cd4..2c5b207e304f 100644 --- a/managed/devops/roles/configure-ybc-server/tasks/download-ybc-package.yml +++ b/managed/devops/roles/configure-ybc-server/tasks/download-ybc-package.yml @@ -13,6 +13,7 @@ group: "{{ user_name }}" tags: - ybc-install + - reinstall-ybc - name: Download ybc package | Move unpacked tarball to "{{ tmp_ybc_package_location }}" copy: @@ -21,6 +22,7 @@ remote_src: True tags: - ybc-install + - reinstall-ybc - name: Download ybc package | Create release ybc directory file: @@ -31,6 +33,7 @@ group: "{{ user_name }}" tags: - ybc-install + - reinstall-ybc - name: Download ybc package | Clean up temporary ybc remote downloads file: @@ -38,6 +41,7 @@ state: absent tags: - ybc-install + - reinstall-ybc - name: Download ybc package | Unpack package "{{ tmp_ybc_package_location }}" unarchive: @@ -48,6 +52,7 @@ group: "{{ user_name }}" tags: - ybc-install + - reinstall-ybc - name: Download ybc package | Remove "{{ tmp_ybc_package_location }}" file: @@ -55,3 +60,4 @@ state: absent tags: - ybc-install + - reinstall-ybc diff --git a/managed/devops/roles/configure-ybc-server/tasks/main.yml b/managed/devops/roles/configure-ybc-server/tasks/main.yml index 472557f455da..e9eed95336f3 100644 --- a/managed/devops/roles/configure-ybc-server/tasks/main.yml +++ b/managed/devops/roles/configure-ybc-server/tasks/main.yml @@ -13,7 +13,9 @@ become: yes become_method: sudo become_user: "{{ user_name }}" - tags: ybc-install + tags: + - ybc-install + - reinstall-ybc - name: Create ybc symlink folders file: @@ -24,6 +26,7 @@ group: "{{ user_name }}" tags: - ybc-install + - reinstall-ybc - name: Install | Create symlink to ybc package folder file: @@ -35,6 +38,7 @@ state: link tags: - ybc-install + - reinstall-ybc - block: - set_fact: @@ -50,6 +54,7 @@ group: "{{ user_name }}" tags: - ybc-install + - reinstall-ybc - name: Setup | Symlink {{ yb_process_type }} logs directory file: @@ -62,6 +67,7 @@ force: yes tags: - ybc-install + - reinstall-ybc - name: Setup | Create ybc config directory file: @@ -72,6 +78,7 @@ group: "{{ user_name }}" tags: - ybc-install + - reinstall-ybc - name: Configure | Create ybc gflags set_fact: diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/AbstractTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/AbstractTaskBase.java index 70c17dfc3e75..65b9cd5abfac 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/AbstractTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/AbstractTaskBase.java @@ -253,7 +253,7 @@ protected SubTaskGroup createSubTaskGroup(String name) { } protected SubTaskGroup createSubTaskGroup(String name, boolean ignoreErrors) { - return createSubTaskGroup(name, SubTaskGroupType.Invalid); + return createSubTaskGroup(name, SubTaskGroupType.Invalid, ignoreErrors); } protected SubTaskGroup createSubTaskGroup(String name, SubTaskGroupType subTaskGroupType) { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/KubernetesUpgradeTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/KubernetesUpgradeTaskBase.java index 1fbbc852645e..d988486b9aaf 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/KubernetesUpgradeTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/KubernetesUpgradeTaskBase.java @@ -382,7 +382,6 @@ public void createUpgradeTask( if (enableYbc) { Set primaryTservers = new HashSet<>(universe.getTServersInPrimaryCluster()); installYbcOnThePods( - universe.getName(), primaryTservers, false, ybcSoftwareVersion, @@ -429,11 +428,7 @@ public void createUpgradeTask( Set replicaTservers = new HashSet(universe.getNodesInCluster(asyncCluster.uuid)); installYbcOnThePods( - universe.getName(), - replicaTservers, - true, - ybcSoftwareVersion, - asyncCluster.userIntent.ybcFlags); + replicaTservers, true, ybcSoftwareVersion, asyncCluster.userIntent.ybcFlags); performYbcAction(replicaTservers, true, "stop"); createWaitForYbcServerTask(replicaTservers); } @@ -524,7 +519,6 @@ public void createNonRollingGflagUpgradeTask( Set primaryTservers = new HashSet(universe.getTServersInPrimaryCluster()); installYbcOnThePods( - universe.getName(), primaryTservers, false, ybcSoftwareVersion, @@ -565,7 +559,6 @@ public void createNonRollingGflagUpgradeTask( universe.getNodesInCluster( universe.getUniverseDetails().getReadOnlyClusters().get(0).uuid)); installYbcOnThePods( - universe.getName(), replicaTservers, true, ybcSoftwareVersion, diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java b/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java index 110d94b5c4cb..be78bbbafc04 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java @@ -679,7 +679,7 @@ private void waitForSubTasks(boolean abortOnFailure) { anyEx = (anyEx != null) ? anyEx : e.getCause(); removeCompletedSubTask(iter, runnableSubTask, e.getCause()); // Call parent task abort if abortOnFailure set. - if (abortOnFailure) { + if (abortOnFailure && !ignoreErrors) { runnableTask.setAbortTime(Instant.now()); runnableTask.cancelWaiterIfAborted(); } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateBackup.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateBackup.java index 1560d8727dc0..9aefd26ece0d 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateBackup.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateBackup.java @@ -122,26 +122,29 @@ public void run() { getRunnableTask().reset(); if (isFirstTry()) { - if (ybcBackup - && universe.isYbcEnabled() - && !universe - .getUniverseDetails() - .getYbcSoftwareVersion() - .equals(ybcManager.getStableYbcVersion())) { - - if (universe + if (ybcBackup && universe.isYbcEnabled()) { + if (!universe .getUniverseDetails() - .getPrimaryCluster() - .userIntent - .providerType - .equals(Common.CloudType.kubernetes)) { - createUpgradeYbcTaskOnK8s( - params().getUniverseUUID(), ybcManager.getStableYbcVersion()) - .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + .getYbcSoftwareVersion() + .equals(ybcManager.getStableYbcVersion())) { + if (universe + .getUniverseDetails() + .getPrimaryCluster() + .userIntent + .providerType + .equals(Common.CloudType.kubernetes)) { + createUpgradeYbcTaskOnK8s( + params().getUniverseUUID(), ybcManager.getStableYbcVersion()) + .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + } else { + createUpgradeYbcTask( + params().getUniverseUUID(), ybcManager.getStableYbcVersion(), true) + .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + } } else { - createUpgradeYbcTask( - params().getUniverseUUID(), ybcManager.getStableYbcVersion(), true) - .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + // Try re-install ybc if ping check fails + // Skip upgrade case, since upgrade will anyway re-configure it + handleUnavailableYbcServers(universe, ybcManager); } } } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java index 2418b833d330..4d4eeff34ac6 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java @@ -220,14 +220,12 @@ public void run() { // Install YBC on the pods if (taskParams().isEnableYbc()) { installYbcOnThePods( - universe.getName(), tserversAdded, false, taskParams().getYbcSoftwareVersion(), taskParams().getPrimaryCluster().userIntent.ybcFlags); if (readClusters.size() == 1) { installYbcOnThePods( - universe.getName(), readOnlyTserversAdded, true, taskParams().getYbcSoftwareVersion(), diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java index 808e211bac57..989172eb401e 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java @@ -450,7 +450,6 @@ private boolean editCluster( if (universe.isYbcEnabled()) { installYbcOnThePods( - universe.getName(), tserversToAdd, isReadOnlyCluster, ybcManager.getStableYbcVersion(), diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java index 1361706e629c..e8b6c7a755e7 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java @@ -1380,71 +1380,50 @@ public KubernetesCommandExecutor createKubernetesExecutorTask( } public void installYbcOnThePods( - String universeName, Set servers, boolean isReadOnlyCluster, String ybcSoftwareVersion, Map ybcGflags) { SubTaskGroup ybcUpload = - createSubTaskGroup( - KubernetesCommandExecutor.CommandType.COPY_PACKAGE.getSubTaskGroupName()); - createKubernetesYbcExecutorTask( - ybcUpload, - universeName, - KubernetesCommandExecutor.CommandType.COPY_PACKAGE, - servers, - isReadOnlyCluster, - ybcSoftwareVersion, - ybcGflags); + createSubTaskGroup(KubernetesCommandExecutor.CommandType.COPY_PACKAGE.getSubTaskGroupName()) + .setSubTaskGroupType(SubTaskGroupType.ConfigureUniverse); + createKubernetesYbcCopyPackageTask( + ybcUpload, servers, isReadOnlyCluster, ybcSoftwareVersion, ybcGflags); getRunnableTask().addSubTaskGroup(ybcUpload); } public void performYbcAction( Set servers, boolean isReadOnlyCluster, String command) { SubTaskGroup ybcAction = - createSubTaskGroup(KubernetesCommandExecutor.CommandType.YBC_ACTION.getSubTaskGroupName()); - createKubernetesYbcExecutorTask( - ybcAction, - KubernetesCommandExecutor.CommandType.YBC_ACTION, - servers, - isReadOnlyCluster, - command); + createSubTaskGroup(KubernetesCommandExecutor.CommandType.YBC_ACTION.getSubTaskGroupName()) + .setSubTaskGroupType(SubTaskGroupType.StartingNodeProcesses); + createKubernetesYbcActionTask(ybcAction, servers, isReadOnlyCluster, command); getRunnableTask().addSubTaskGroup(ybcAction); } // Create Kubernetes Executor task for copying YBC package and conf file to the pod - public void createKubernetesYbcExecutorTask( + public void createKubernetesYbcCopyPackageTask( SubTaskGroup subTaskGroup, - String universeName, - KubernetesCommandExecutor.CommandType commandType, Set servers, boolean isReadOnlyCluster, String ybcSoftwareVersion, Map ybcGflags) { for (NodeDetails node : servers) { - KubernetesCommandExecutor.Params params = new KubernetesCommandExecutor.Params(); Cluster primaryCluster = taskParams().getPrimaryCluster(); Universe universe = Universe.getOrBadRequest(taskParams().getUniverseUUID()); if (primaryCluster == null) { primaryCluster = universe.getUniverseDetails().getPrimaryCluster(); } - params.commandType = commandType; - params.setUniverseUUID(taskParams().getUniverseUUID()); - params.ybcServerName = node.nodeName; - params.setYbcSoftwareVersion(ybcSoftwareVersion); - params.ybcGflags = ybcGflags; List readOnlyClusters = taskParams().getReadOnlyClusters(); if (isReadOnlyCluster && readOnlyClusters.size() == 0) { readOnlyClusters = universe.getUniverseDetails().getReadOnlyClusters(); } - params.providerUUID = + UUID providerUUID = isReadOnlyCluster ? UUID.fromString(readOnlyClusters.get(0).userIntent.provider) : UUID.fromString(primaryCluster.userIntent.provider); - KubernetesCommandExecutor task = createTask(KubernetesCommandExecutor.class); - task.initialize(params); - task.setUserTaskUUID(getUserTaskUUID()); - subTaskGroup.addSubTask(task); + createKubernetesYbcCopyPackageSubTask( + subTaskGroup, node, providerUUID, ybcSoftwareVersion, ybcGflags); } } @@ -1512,14 +1491,12 @@ public void createKubernetesYbcExecutorTask( } // Create Kubernetes Executor task for perform ybc - public void createKubernetesYbcExecutorTask( + public void createKubernetesYbcActionTask( SubTaskGroup subTaskGroup, - KubernetesCommandExecutor.CommandType commandType, Set servers, boolean isReadOnlyCluster, String command) { for (NodeDetails node : servers) { - KubernetesCommandExecutor.Params params = new KubernetesCommandExecutor.Params(); Cluster primaryCluster = taskParams().getPrimaryCluster(); List readOnlyClusters = taskParams().getReadOnlyClusters(); Universe universe = Universe.getOrBadRequest(taskParams().getUniverseUUID()); @@ -1529,19 +1506,12 @@ public void createKubernetesYbcExecutorTask( if (isReadOnlyCluster && readOnlyClusters.size() == 0) { readOnlyClusters = universe.getUniverseDetails().getReadOnlyClusters(); } - params.commandType = commandType; - params.setUniverseUUID(taskParams().getUniverseUUID()); - params.ybcServerName = node.nodeName; - params.isReadOnlyCluster = isReadOnlyCluster; - params.providerUUID = + UUID providerUUID = isReadOnlyCluster ? UUID.fromString(readOnlyClusters.get(0).userIntent.provider) : UUID.fromString(primaryCluster.userIntent.provider); - params.command = command; - KubernetesCommandExecutor task = createTask(KubernetesCommandExecutor.class); - task.initialize(params); - task.setUserTaskUUID(getUserTaskUUID()); - subTaskGroup.addSubTask(task); + createKubernetesYbcActionSubTask( + subTaskGroup, node, providerUUID, isReadOnlyCluster, command); } } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java index a630dac06344..532d9ce48a0d 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java @@ -117,7 +117,6 @@ public void run() { // Install YBC on the RR tservers and wait for its completion if (universe.isYbcEnabled()) { installYbcOnThePods( - universe.getName(), tserversAdded, true, ybcManager.getStableYbcVersion(), diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreBackup.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreBackup.java index f6d63ea70028..32de3d3bcd82 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreBackup.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreBackup.java @@ -53,25 +53,29 @@ public void run() { if (isFirstTry()) { backupHelper.validateRestoreOverwrites(taskParams().backupStorageInfoList, universe); - if (universe.isYbcEnabled() - && !universe - .getUniverseDetails() - .getYbcSoftwareVersion() - .equals(ybcManager.getStableYbcVersion())) { - - if (universe + if (universe.isYbcEnabled()) { + if (!universe .getUniverseDetails() - .getPrimaryCluster() - .userIntent - .providerType - .equals(Common.CloudType.kubernetes)) { - createUpgradeYbcTaskOnK8s( - taskParams().getUniverseUUID(), ybcManager.getStableYbcVersion()) - .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + .getYbcSoftwareVersion() + .equals(ybcManager.getStableYbcVersion())) { + if (universe + .getUniverseDetails() + .getPrimaryCluster() + .userIntent + .providerType + .equals(Common.CloudType.kubernetes)) { + createUpgradeYbcTaskOnK8s( + taskParams().getUniverseUUID(), ybcManager.getStableYbcVersion()) + .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + } else { + createUpgradeYbcTask( + taskParams().getUniverseUUID(), ybcManager.getStableYbcVersion(), true) + .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + } } else { - createUpgradeYbcTask( - taskParams().getUniverseUUID(), ybcManager.getStableYbcVersion(), true) - .setSubTaskGroupType(SubTaskGroupType.UpgradingYbc); + // Try re-install ybc if ping check fails + // Skip upgrade case, since upgrade will anyway re-configure it + handleUnavailableYbcServers(universe, ybcManager); } } } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java index 5cc6953ff054..5850a8e3cb5c 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java @@ -172,6 +172,7 @@ import com.yugabyte.yw.common.backuprestore.BackupUtil; import com.yugabyte.yw.common.backuprestore.ybc.YbcBackupNodeRetriever; import com.yugabyte.yw.common.backuprestore.ybc.YbcBackupUtil; +import com.yugabyte.yw.common.backuprestore.ybc.YbcManager; import com.yugabyte.yw.common.config.CustomerConfKeys; import com.yugabyte.yw.common.config.GlobalConfKeys; import com.yugabyte.yw.common.config.UniverseConfKeys; @@ -192,10 +193,13 @@ import com.yugabyte.yw.forms.TableInfoForm.NamespaceInfoResp; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster; +import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.ClusterType; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.SoftwareUpgradeState; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; import com.yugabyte.yw.forms.UniverseTaskParams; import com.yugabyte.yw.forms.UpgradeTaskParams; +import com.yugabyte.yw.forms.UpgradeTaskParams.UpgradeTaskSubType; +import com.yugabyte.yw.forms.UpgradeTaskParams.UpgradeTaskType; import com.yugabyte.yw.forms.XClusterConfigCreateFormData; import com.yugabyte.yw.forms.XClusterConfigTaskParams; import com.yugabyte.yw.metrics.MetricQueryHelper; @@ -1909,9 +1913,15 @@ public SubTaskGroup createUnivManageAlertDefinitionsTask(boolean active) { return subTaskGroup; } - /** Create a task to ping yb-controller servers on each node */ public SubTaskGroup createWaitForYbcServerTask(Collection nodeDetailsSet) { - SubTaskGroup subTaskGroup = createSubTaskGroup("WaitForYbcServer"); + return createWaitForYbcServerTask( + nodeDetailsSet, false /* ignoreErrors */, 20 /* numRetries */); + } + + /** Create a task to ping yb-controller servers on each node */ + public SubTaskGroup createWaitForYbcServerTask( + Collection nodeDetailsSet, boolean ignoreErrors, int numRetries) { + SubTaskGroup subTaskGroup = createSubTaskGroup("WaitForYbcServer", ignoreErrors); WaitForYbcServer task = createTask(WaitForYbcServer.class); WaitForYbcServer.Params params = new WaitForYbcServer.Params(); params.setUniverseUUID(taskParams().getUniverseUUID()); @@ -1920,6 +1930,7 @@ public SubTaskGroup createWaitForYbcServerTask(Collection nodeDetai nodeDetailsSet == null ? null : nodeDetailsSet.stream().map(node -> node.nodeName).collect(Collectors.toSet()); + params.numRetries = numRetries; task.initialize(params); subTaskGroup.addSubTask(task); getRunnableTask().addSubTaskGroup(subTaskGroup); @@ -4197,9 +4208,9 @@ public SubTaskGroup createTableBackupTasksYbc( BackupTableParams backupParams, int parallelDBBackups) { SubTaskGroup subTaskGroup = createSubTaskGroup("BackupTableYbc"); Universe universe = Universe.getOrBadRequest(backupParams.getUniverseUUID()); - YbcBackupNodeRetriever nodeRetriever = new YbcBackupNodeRetriever(universe, parallelDBBackups); + YbcBackupNodeRetriever nodeRetriever = + new YbcBackupNodeRetriever(universe, parallelDBBackups, backupParams.backupDBStates); Duration scheduleRetention = ScheduleUtil.getScheduleRetention(backupParams.baseBackupUUID); - nodeRetriever.initializeNodePoolForBackups(backupParams.backupDBStates); Backup previousBackup = (!backupParams.baseBackupUUID.equals(backupParams.backupUuid)) ? Backup.getLastSuccessfulBackupInChain( @@ -4394,6 +4405,144 @@ public SubTaskGroup createUpgradeYbcTaskOnK8s(UUID universeUUID, String ybcSoftw return subTaskGroup; } + /** + * Create subtask for copying YBC package on K8s pod and add to subtask group. + * + * @param subTaskGroup + * @param node + * @param providerUUID + * @param ybcSoftwareVersion + * @param ybcGflags + */ + public void createKubernetesYbcCopyPackageSubTask( + SubTaskGroup subTaskGroup, + NodeDetails node, + UUID providerUUID, + String ybcSoftwareVersion, + Map ybcGflags) { + KubernetesCommandExecutor.Params params = new KubernetesCommandExecutor.Params(); + params.commandType = KubernetesCommandExecutor.CommandType.COPY_PACKAGE; + params.setUniverseUUID(taskParams().getUniverseUUID()); + params.ybcServerName = node.nodeName; + params.setYbcSoftwareVersion(ybcSoftwareVersion); + params.ybcGflags = ybcGflags; + params.providerUUID = providerUUID; + KubernetesCommandExecutor task = createTask(KubernetesCommandExecutor.class); + task.initialize(params); + task.setUserTaskUUID(getUserTaskUUID()); + subTaskGroup.addSubTask(task); + } + + /** + * Create subtask for running YBC action and add to subtask group. + * + * @param subTaskGroup + * @param node + * @param providerUUID + * @param isReadOnlyCluster + * @param command + */ + public void createKubernetesYbcActionSubTask( + SubTaskGroup subTaskGroup, + NodeDetails node, + UUID providerUUID, + boolean isReadOnlyCluster, + String command) { + KubernetesCommandExecutor.Params params = new KubernetesCommandExecutor.Params(); + params.commandType = KubernetesCommandExecutor.CommandType.YBC_ACTION; + params.setUniverseUUID(taskParams().getUniverseUUID()); + params.ybcServerName = node.nodeName; + params.isReadOnlyCluster = isReadOnlyCluster; + params.providerUUID = providerUUID; + params.command = command; + KubernetesCommandExecutor task = createTask(KubernetesCommandExecutor.class); + task.initialize(params); + task.setUserTaskUUID(getUserTaskUUID()); + subTaskGroup.addSubTask(task); + } + + public void handleUnavailableYbcServers(Universe universe, YbcManager ybcManager) { + String cert = universe.getCertificateNodetoNode(); + int ybcPort = universe.getUniverseDetails().communicationPorts.ybControllerrRpcPort; + Map ybcGflags = + universe.getUniverseDetails().getPrimaryCluster().userIntent.ybcFlags; + String ybcSoftwareVersion = ybcManager.getStableYbcVersion(); + Consumer paramsCustomizer = params -> {}; + + String configureSubTaskDescription = + String.format("ConfigureYbcServer on Universe %s", universe.getUniverseUUID()); + SubTaskGroup configureYbcGroup = + createSubTaskGroup( + configureSubTaskDescription, + SubTaskGroupType.ConfigureUniverse, + true /* ignoreErrors */); + SubTaskGroup stopYbcActionGroup = + createSubTaskGroup( + "StopYbcAction", SubTaskGroupType.StoppingNodeProcesses, true /* ignoreErrors */); + SubTaskGroup startYbcActionGroup = + createSubTaskGroup( + "StartYbcAction", SubTaskGroupType.StartingNodeProcesses, true /* ignoreErrors */); + + List reinstallNodes = new ArrayList<>(); + for (Cluster cluster : universe.getUniverseDetails().clusters) { + boolean isK8s = cluster.userIntent.providerType == CloudType.kubernetes; + universe.getTserversInCluster(cluster.uuid).stream() + .filter(NodeDetails::isConsideredRunning) + .filter(node -> !ybcManager.ybcPingCheck(node.cloudInfo.private_ip, cert, ybcPort)) + .forEach( + node -> { + if (isK8s) { + createKubernetesYbcCopyPackageSubTask( + configureYbcGroup, + node, + UUID.fromString(cluster.userIntent.provider), + ybcSoftwareVersion, + ybcGflags); + createKubernetesYbcActionSubTask( + stopYbcActionGroup, + node, + UUID.fromString(cluster.userIntent.provider), + cluster.clusterType == ClusterType.ASYNC, + "stop" /* command */); + } else { + AnsibleConfigureServers.Params params = + ybcManager.getAnsibleConfigureYbcServerTaskParams( + universe, + node, + ybcGflags, + UpgradeTaskType.YbcGFlags, + UpgradeTaskSubType.YbcGflagsUpdate); + AnsibleConfigureServers task = createTask(AnsibleConfigureServers.class); + task.initialize(params); + task.setUserTaskUUID(getUserTaskUUID()); + configureYbcGroup.addSubTask(task); + stopYbcActionGroup.addSubTask( + getServerControlTask( + node, + ServerType.CONTROLLER, + "stop" /* command */, + 0 /* sleepAfterCmdMillis */, + paramsCustomizer)); + startYbcActionGroup.addSubTask( + getServerControlTask( + node, + ServerType.CONTROLLER, + "start" /* command */, + 0 /* sleepAfterCmdMillis */, + paramsCustomizer)); + } + reinstallNodes.add(node); + }); + } + if (reinstallNodes.size() > 0) { + getRunnableTask().addSubTaskGroup(configureYbcGroup); + getRunnableTask().addSubTaskGroup(stopYbcActionGroup); + getRunnableTask().addSubTaskGroup(startYbcActionGroup); + createWaitForYbcServerTask(reinstallNodes, true /* ignoreErrors */, 10 /* numRetries */) + .setSubTaskGroupType(SubTaskGroupType.StartingNodeProcesses); + } + } + /** * Creates a task to install xxhash on the DB nodes from third-party packages. * diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java index 78a5dc9dd543..632c34ca4f15 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java @@ -294,7 +294,6 @@ private void createUpgradeTask( new HashSet( universe.getNodesInCluster(taskParams().getReadOnlyClusters().get(0).uuid)); installYbcOnThePods( - universe.getName(), replicaTservers, true, ybcSoftwareVersion, @@ -306,7 +305,6 @@ private void createUpgradeTask( new HashSet( universe.getNodesInCluster(taskParams().getPrimaryCluster().uuid)); installYbcOnThePods( - universe.getName(), primaryTservers, false, ybcSoftwareVersion, diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java index ee6256121912..c838850c63b7 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java @@ -70,11 +70,7 @@ public void run() { new HashSet<>(universe.getRunningTserversInPrimaryCluster()); installYbcOnThePods( - universe.getName(), - nodeDetailSet, - false, - universeDetails.getYbcSoftwareVersion(), - ybcGflagsMap); + nodeDetailSet, false, universeDetails.getYbcSoftwareVersion(), ybcGflagsMap); performYbcAction(nodeDetailSet, false, "stop"); createWaitForYbcServerTask(nodeDetailSet) .setSubTaskGroupType(SubTaskGroupType.ConfigureUniverse); @@ -83,11 +79,7 @@ public void run() { if (!readOnlyClusters.isEmpty()) { nodeDetailSet = universeDetails.getTserverNodesInCluster(readOnlyClusters.get(0).uuid); installYbcOnThePods( - universe.getName(), - nodeDetailSet, - true, - universeDetails.getYbcSoftwareVersion(), - ybcGflagsMap); + nodeDetailSet, true, universeDetails.getYbcSoftwareVersion(), ybcGflagsMap); performYbcAction(nodeDetailSet, true, "stop"); createWaitForYbcServerTask(nodeDetailSet) .setSubTaskGroupType(SubTaskGroupType.ConfigureUniverse); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/BackupTableYbc.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/BackupTableYbc.java index b510b3f86a23..1578c7db1e89 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/BackupTableYbc.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/BackupTableYbc.java @@ -98,6 +98,9 @@ public void run() { taskParams().getKeyspace(), taskParams().backupParamsIdentifier); + // Initialise node-pool for backups + taskParams().nodeRetriever.initializeNodePoolForBackups(); + // Wait on node-ip if (StringUtils.isBlank(taskParams().nodeIp)) { taskParams().nodeIp = taskParams().nodeRetriever.getNodeIpForBackup(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallYbcSoftwareOnK8s.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallYbcSoftwareOnK8s.java index d4295d5d0e01..331b791682e9 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallYbcSoftwareOnK8s.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallYbcSoftwareOnK8s.java @@ -39,7 +39,6 @@ public void run() { allTservers.addAll(primaryTservers); /* This calls kubectl cp, it is idempotent */ installYbcOnThePods( - universe.getName(), primaryTservers, false, taskParams().getYbcSoftwareVersion(), @@ -52,7 +51,6 @@ public void run() { universe.getUniverseDetails().getReadOnlyClusters().get(0).uuid)); allTservers.addAll(replicaTservers); installYbcOnThePods( - universe.getName(), replicaTservers, true, taskParams().getYbcSoftwareVersion(), diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/WaitForYbcServer.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/WaitForYbcServer.java index dbada073e797..8ef61c798e50 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/WaitForYbcServer.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/WaitForYbcServer.java @@ -33,9 +33,8 @@ protected WaitForYbcServer(BaseTaskDependencies baseTaskDependencies, YbcManager } public static class Params extends UniverseDefinitionTaskParams { - // The universe UUID must be stored in universeUUID field. - // The xCluster info object to persist. public Set nodeNameList = null; + public int numRetries = 20; } protected Params taskParams() { @@ -52,6 +51,6 @@ public void run() { .map(nodeName -> universe.getNode(nodeName)) .collect(Collectors.toSet()); - ybcManager.waitForYbc(universe, nodeDetailsSet); + ybcManager.waitForYbc(universe, nodeDetailsSet, taskParams().numRetries); } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java index 194a769d57e8..f3b8f8981c23 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java @@ -1395,6 +1395,10 @@ private List getConfigureSubCommand( } break; case YbcGFlags: + subcommand.add("--package"); + subcommand.add(ybServerPackage); + subcommand.add("--ybc_package"); + subcommand.add(ybcPackage); subcommand.add("--ybc_flags"); subcommand.add(Json.stringify(Json.toJson(ybcFlags))); subcommand.add("--configure_ybc"); @@ -1402,6 +1406,8 @@ private List getConfigureSubCommand( subcommand.add(ybcDir); subcommand.add("--tags"); subcommand.add("override_ybc_gflags"); + subcommand.add("--tags"); + subcommand.add("reinstall-ybc"); break; default: break; diff --git a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetriever.java b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetriever.java index 43ed9eb829f4..bf0405f79a43 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetriever.java +++ b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetriever.java @@ -12,6 +12,7 @@ import java.util.UUID; import java.util.concurrent.CancellationException; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -27,14 +28,22 @@ public class YbcBackupNodeRetriever { private final LinkedBlockingQueue universeTserverIPs; private final Universe universe; private final YbcManager ybcManager; + private final AtomicBoolean poolInitialized; + private final Map backupDBStates; - public YbcBackupNodeRetriever(Universe universe, int parallelism) { + public YbcBackupNodeRetriever( + Universe universe, int parallelism, Map backupDBStates) { this.universeTserverIPs = new LinkedBlockingQueue<>(parallelism); this.universe = universe; this.ybcManager = StaticInjectorHolder.injector().instanceOf(YbcManager.class); + this.poolInitialized = new AtomicBoolean(false); + this.backupDBStates = backupDBStates; } - public void initializeNodePoolForBackups(Map backupDBStates) { + public synchronized void initializeNodePoolForBackups() { + if (poolInitialized.get()) { + return; + } Set nodeIPsAlreadyAssigned = backupDBStates.entrySet().stream() .filter( @@ -59,6 +68,7 @@ public void initializeNodePoolForBackups(Map backupDB nodeIPsAlreadyAssigned.size() + universeTserverIPs.size()); } } + poolInitialized.set(true); } public String getNodeIpForBackup() { diff --git a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcManager.java b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcManager.java index 17a279eefae1..ff8af1f75e63 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/ybc/YbcManager.java @@ -112,7 +112,7 @@ public class YbcManager { private static final int WAIT_EACH_ATTEMPT_MS = 5000; private static final int WAIT_EACH_SHORT_ATTEMPT_MS = 2000; - private static final int MAX_RETRIES = 10; + private static final int MAX_SHORT_RETRIES = 10; private static final int MAX_NUM_RETRIES = 20; private static final long INITIAL_SLEEP_TIME_IN_MS = 1000L; private static final long INCREMENTAL_SLEEP_TIME_IN_MS = 2000L; @@ -554,7 +554,7 @@ public void deleteYbcBackupTask(String taskID, YbcClient ybcClient) { BackupServiceTaskDeleteRequest.newBuilder().setTaskId(taskID).build(); BackupServiceTaskDeleteResponse taskDeleteResponse = null; int numRetries = 0; - while (numRetries < MAX_RETRIES) { + while (numRetries < MAX_SHORT_RETRIES) { taskDeleteResponse = ybcClient.backupServiceTaskDelete(taskDeleteRequest); if (!taskDeleteResponse.getStatus().getCode().equals(ControllerStatus.IN_PROGRESS)) { break; @@ -609,7 +609,7 @@ public String downloadSuccessMarker( BackupServiceTaskResultRequest.newBuilder().setTaskId(taskID).build(); BackupServiceTaskResultResponse downloadSuccessMarkerResultResponse = null; int numRetries = 0; - while (numRetries < MAX_RETRIES) { + while (numRetries < MAX_SHORT_RETRIES) { downloadSuccessMarkerResultResponse = ybcClient.backupServiceTaskResult(downloadSuccessMarkerResultRequest); if (!(downloadSuccessMarkerResultResponse @@ -1087,6 +1087,10 @@ public void performActionOnYbcK8sNode( } public void waitForYbc(Universe universe, Set nodeDetailsSet) { + waitForYbc(universe, nodeDetailsSet, MAX_NUM_RETRIES); + } + + public void waitForYbc(Universe universe, Set nodeDetailsSet, int numRetries) { String certFile = universe.getCertificateNodetoNode(); int ybcPort = universe.getUniverseDetails().communicationPorts.ybControllerrRpcPort; String errMsg = ""; @@ -1122,14 +1126,14 @@ public void waitForYbc(Universe universe, Set nodeDetailsSet) { "Node IP: {} Ping not complete. Sleeping for {} millis", nodeIp, waitTimeInMillis); Duration duration = Duration.ofMillis(waitTimeInMillis); Thread.sleep(duration.toMillis()); - if (numTries <= MAX_NUM_RETRIES) { + if (numTries <= numRetries) { LOG.info("Node IP: {} Ping not complete. Continuing", nodeIp); continue; } } - if (numTries > MAX_NUM_RETRIES) { + if (numTries > numRetries) { LOG.info("Node IP: {} Ping failed. Exceeded max retries", nodeIp); - errMsg = String.format("Exceeded max retries: %s", MAX_NUM_RETRIES); + errMsg = String.format("Exceeded max retries: %s", numRetries); isYbcConfigured = false; break; } else if (pingResp.getSequence() != seqNum) { @@ -1204,7 +1208,7 @@ public AnsibleConfigureServers.Params getAnsibleConfigureYbcServerTaskParams( params.enableYEDIS = userIntent.enableYEDIS; params.setEnableYbc(universe.getUniverseDetails().isEnableYbc()); - params.setYbcSoftwareVersion(universe.getUniverseDetails().getYbcSoftwareVersion()); + params.setYbcSoftwareVersion(getStableYbcVersion()); params.installYbc = universe.getUniverseDetails().installYbc; params.setYbcInstalled(universe.getUniverseDetails().isYbcInstalled()); params.ybcGflags = gflags; diff --git a/managed/src/test/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetrieverTest.java b/managed/src/test/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetrieverTest.java index 3390eacf33da..f3f1301e9a1e 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetrieverTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/backuprestore/ybc/YbcBackupNodeRetrieverTest.java @@ -190,8 +190,9 @@ public void testNodesInPoolNewTask() throws InterruptedException { subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())).thenReturn(true); - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 3); - ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 3, subTasksMap); + ybcBackupNodeRetriever.initializeNodePoolForBackups(); // Verify 3 polls are successful. String node_ip1 = ybcBackupNodeRetriever.getNodeIpForBackup(); String node_ip2 = ybcBackupNodeRetriever.getNodeIpForBackup(); @@ -218,8 +219,9 @@ public void testNodesInPoolResumedTaskWithRunningBackup() throws InterruptedExce subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())).thenReturn(true); - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 3); - ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 3, subTasksMap); + ybcBackupNodeRetriever.initializeNodePoolForBackups(); // Verify 2 polls are successful and node-ips are not equal to 127.0.0.1 String node_ip2 = ybcBackupNodeRetriever.getNodeIpForBackup(); assertTrue(StringUtils.isNotBlank(node_ip2)); @@ -251,8 +253,9 @@ public void testNodesInPoolResumedTaskWithoutRunningBackup() throws InterruptedE subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())).thenReturn(true); - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 3); - ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 3, subTasksMap); + ybcBackupNodeRetriever.initializeNodePoolForBackups(); // Verify 3 polls are successful. String node_ip1 = ybcBackupNodeRetriever.getNodeIpForBackup(); String node_ip2 = ybcBackupNodeRetriever.getNodeIpForBackup(); @@ -268,9 +271,10 @@ public void testNodesInPoolResumedTaskWithoutRunningBackup() throws InterruptedE @Test public void testPoolSizeOne() throws InterruptedException { Map subTasksMap = new HashMap<>(); - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 1); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 1, subTasksMap); when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())).thenReturn(true); - ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap); + ybcBackupNodeRetriever.initializeNodePoolForBackups(); // Verify 1 poll is successful. String node_ip1 = ybcBackupNodeRetriever.getNodeIpForBackup(); assertTrue(StringUtils.isNotBlank(node_ip1)); @@ -289,8 +293,9 @@ public void testPoolWithFewUnhealthyNodes() throws InterruptedException { when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())) .thenReturn(false) .thenReturn(true); - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 3); - ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 3, subTasksMap); + ybcBackupNodeRetriever.initializeNodePoolForBackups(); // Verify 2 polls are successful. String node_ip1 = ybcBackupNodeRetriever.getNodeIpForBackup(); @@ -311,20 +316,21 @@ public void testAllUnhealthyNodes() { subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())).thenReturn(false); - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 3); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 3, subTasksMap); assertThrows( - RuntimeException.class, - () -> ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap)); + RuntimeException.class, () -> ybcBackupNodeRetriever.initializeNodePoolForBackups()); } @Test public void testNodePreferenceOneNodeMasterLeader() { when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())).thenReturn(true); // Verify parallelism 1 get master leader node - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 1); Map subTasksMap = new HashMap<>(); subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); - ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 1, subTasksMap); + ybcBackupNodeRetriever.initializeNodePoolForBackups(); String node_ip1 = ybcBackupNodeRetriever.getNodeIpForBackup(); String leaderIP = mockUniverse.getMasterLeaderNode().cloudInfo.private_ip; assertTrue(node_ip1.equals(leaderIP)); @@ -334,12 +340,13 @@ public void testNodePreferenceOneNodeMasterLeader() { public void testNodePreferenceOrderMultipleIPs() { when(mockYbcManager.ybcPingCheck(anyString(), eq(null), anyInt())).thenReturn(true); ArgumentCaptor nodeIPCaptor = ArgumentCaptor.forClass(String.class); - YbcBackupNodeRetriever ybcBackupNodeRetriever = new YbcBackupNodeRetriever(mockUniverse, 3); Map subTasksMap = new HashMap<>(); subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); subTasksMap.put(UUID.randomUUID(), new ParallelBackupState()); - ybcBackupNodeRetriever.initializeNodePoolForBackups(subTasksMap); + YbcBackupNodeRetriever ybcBackupNodeRetriever = + new YbcBackupNodeRetriever(mockUniverse, 3, subTasksMap); + ybcBackupNodeRetriever.initializeNodePoolForBackups(); verify(mockYbcManager, times(3)).ybcPingCheck(nodeIPCaptor.capture(), eq(null), anyInt()); List capturedIPs = nodeIPCaptor.getAllValues(); NodeDetails masterLeaderNode = mockUniverse.getMasterLeaderNode();