Skip to content

Commit

Permalink
[PLAT-12933] [k8s] Ability to roll N nodes at a time during upgrades …
Browse files Browse the repository at this point in the history 
…for multi-AZ(region) universes

Summary:
Propogated ability to roll N nodes to k8s universes (see main logic here https://phorge.dev.yugabyte.com/D35955)
Pods are split inside each az according to either maxRollBatchSize provided in task params or alternatively according to runtime config.
For the case of POD_DELETE we just spawn multiple simultaneous tasks whereas for other commands multiple pods are updated using partition feature

Test Plan:
1) create k8s universe
2) run restart via API with provided maxRollBatchSize (2,2) - verify pods are started in pairs
3) run gflags upgrade while setting runtime conf value to 2 - verify pods are updated in pairs, verify gflags are applied to all of them

Reviewers: sneelakantan, anijhawan, #yba-api-review

Reviewed By: anijhawan, #yba-api-review

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D37345
  • Loading branch information
@yorq
yorq committed Sep 3, 2024
1 parent eba9b49 commit f44c92e
Show file tree
Hide file tree
Showing 13 changed files with 665 additions and 175 deletions.
40 changes: 30 additions & 10 deletions managed/src/main/java/com/yugabyte/yw/commissioner/KubernetesUpgradeTaskBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.yugabyte.yw.common.operator.OperatorStatusUpdater;
import com.yugabyte.yw.common.operator.OperatorStatusUpdater.UniverseState;
import com.yugabyte.yw.common.operator.OperatorStatusUpdaterFactory;
import com.yugabyte.yw.forms.RollMaxBatchSize;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster;
import com.yugabyte.yw.forms.UpgradeTaskParams;
Expand Down Expand Up @@ -72,8 +73,17 @@ protected void createPrecheckTasks(Universe universe) {
.filter(n -> n.state != NodeDetails.NodeState.Live)
.findFirst();
if (nonLive.isEmpty()) {
List<MastersAndTservers> split = nodesToBeRestarted.splitToSingle();
createCheckNodesAreSafeToTakeDownTask(split, getTargetSoftwareVersion(), false);
RollMaxBatchSize rollMaxBatchSize = getCurrentRollBatchSize(universe);
// Use only primary nodes
MastersAndTservers forCluster =
nodesToBeRestarted.getForCluster(
universe.getUniverseDetails().getPrimaryCluster().uuid);

if (!forCluster.isEmpty()) {
List<MastersAndTservers> split =
UpgradeTaskBase.split(universe, forCluster, rollMaxBatchSize);
createCheckNodesAreSafeToTakeDownTask(split, getTargetSoftwareVersion(), true);
}
}
}
}
Expand Down Expand Up @@ -128,6 +138,10 @@ private List<NodeDetails> fetchTServerNodes(Universe universe, UpgradeOption upg

public abstract SubTaskGroupType getTaskSubGroupType();

private RollMaxBatchSize getCurrentRollBatchSize(Universe universe) {
return getCurrentRollBatchSize(universe, taskParams().rollMaxBatchSize);
}

// Wrapper that takes care of common pre and post upgrade tasks and user has
// flexibility to manipulate subTaskGroupQueue through the lambda passed in parameter
public void runUpgrade(Runnable upgradeLambda) {
Expand Down Expand Up @@ -323,7 +337,6 @@ public void createUpgradeTask(
null,
ServerType.MASTER,
softwareVersion,
taskParams().sleepAfterMasterRestartMillis,
universeOverrides,
azOverrides,
isMasterChanged,
Expand All @@ -333,7 +346,9 @@ public void createUpgradeTask(
commandType,
enableYbc,
ybcSoftwareVersion,
/* addDelayAfterStartup */ true);
PodUpgradeParams.builder()
.delayAfterStartup(taskParams().sleepAfterMasterRestartMillis)
.build());
}

if (isTServerChanged) {
Expand All @@ -348,7 +363,6 @@ public void createUpgradeTask(
null,
ServerType.TSERVER,
softwareVersion,
taskParams().sleepAfterTServerRestartMillis,
universeOverrides,
azOverrides,
false, // master change is false since it has already been upgraded.
Expand All @@ -358,7 +372,10 @@ public void createUpgradeTask(
commandType,
enableYbc,
ybcSoftwareVersion,
/* addDelayAfterStartup */ true);
PodUpgradeParams.builder()
.delayAfterStartup(taskParams().sleepAfterTServerRestartMillis)
.rollMaxBatchSize(getCurrentRollBatchSize(universe))
.build());

if (enableYbc) {
Set<NodeDetails> primaryTservers = new HashSet<>(universe.getTServersInPrimaryCluster());
Expand Down Expand Up @@ -392,7 +409,6 @@ public void createUpgradeTask(
null,
ServerType.TSERVER,
softwareVersion,
taskParams().sleepAfterTServerRestartMillis,
universeOverrides,
azOverrides,
false, // master change is false since it has already been upgraded.
Expand All @@ -402,7 +418,10 @@ public void createUpgradeTask(
commandType,
enableYbc,
ybcSoftwareVersion,
/* addDelayAfterStartup */ true);
PodUpgradeParams.builder()
.delayAfterStartup(taskParams().sleepAfterTServerRestartMillis)
.rollMaxBatchSize(getCurrentRollBatchSize(universe))
.build());

if (enableYbc) {
Set<NodeDetails> replicaTservers =
Expand All @@ -428,7 +447,6 @@ public void createUpgradeTask(
null,
ServerType.MASTER,
softwareVersion,
taskParams().sleepAfterMasterRestartMillis,
universeOverrides,
azOverrides,
isMasterChanged,
Expand All @@ -438,7 +456,9 @@ public void createUpgradeTask(
commandType,
enableYbc,
ybcSoftwareVersion,
true);
PodUpgradeParams.builder()
.delayAfterStartup(taskParams().sleepAfterMasterRestartMillis)
.build());
}
}

Expand Down
31 changes: 2 additions & 29 deletions managed/src/main/java/com/yugabyte/yw/commissioner/UpgradeTaskBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import com.yugabyte.yw.commissioner.tasks.upgrade.SoftwareUpgradeYB;
import com.yugabyte.yw.common.PlacementInfoUtil;
import com.yugabyte.yw.common.PlatformServiceException;
import com.yugabyte.yw.common.config.UniverseConfKeys;
import com.yugabyte.yw.common.gflags.GFlagsUtil;
import com.yugabyte.yw.common.kms.util.EncryptionAtRestUtil;
import com.yugabyte.yw.forms.RollMaxBatchSize;
Expand Down Expand Up @@ -194,36 +193,10 @@ protected void createPrecheckTasks(Universe universe) {
}

private RollMaxBatchSize getCurrentRollBatchSize(Universe universe) {
RollMaxBatchSize rollMaxBatchSize = new RollMaxBatchSize();
if (taskParams().rollMaxBatchSize != null
&& confGetter.getConfForScope(universe, UniverseConfKeys.upgradeBatchRollEnabled)) {
rollMaxBatchSize = taskParams().rollMaxBatchSize;
} else {
RollMaxBatchSize max = getMaxNodesToRoll(universe);
int percent =
confGetter.getConfForScope(universe, UniverseConfKeys.upgradeBatchRollAutoPercent);
int number =
confGetter.getConfForScope(universe, UniverseConfKeys.upgradeBatchRollAutoNumber);
int numberToSet = 0;
if (percent > 0) {
numberToSet = max.getPrimaryBatchSize() * percent / 100;
} else if (number > 1) {
numberToSet = Math.min(number, max.getPrimaryBatchSize());
}
if (numberToSet > 1) {
rollMaxBatchSize.setPrimaryBatchSize(numberToSet);
rollMaxBatchSize.setReadReplicaBatchSize(numberToSet);
}
}
if (getTaskCache() != null && getTaskCache().get(SPLIT_FALLBACK) != null) {
RollMaxBatchSize fallback = getTaskCache().get(SPLIT_FALLBACK, RollMaxBatchSize.class);
// Setting this only for primary cluster, RR still can be rolled with any speed.
rollMaxBatchSize.setPrimaryBatchSize(fallback.getPrimaryBatchSize());
}
return rollMaxBatchSize;
return getCurrentRollBatchSize(universe, taskParams().rollMaxBatchSize);
}

private List<MastersAndTservers> split(
public static List<MastersAndTservers> split(
Universe universe, MastersAndTservers forCluster, RollMaxBatchSize rollMaxBatchSize) {
List<MastersAndTservers> result = new ArrayList<>();
forCluster.mastersList.forEach(
Expand Down
9 changes: 3 additions & 6 deletions managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,6 @@ private boolean editCluster(
curPlacement,
ServerType.MASTER,
newIntent.ybSoftwareVersion,
DEFAULT_WAIT_TIME_MS,
universeOverrides,
azOverrides,
true,
Expand All @@ -505,7 +504,7 @@ private boolean editCluster(
KubernetesCommandExecutor.CommandType.HELM_UPGRADE,
universe.isYbcEnabled(),
universe.getUniverseDetails().getYbcSoftwareVersion(),
/* addDelayAfterStartup */ false);
PodUpgradeParams.DEFAULT);

upgradePodsTask(
universe.getName(),
Expand All @@ -514,7 +513,6 @@ private boolean editCluster(
curPlacement,
ServerType.TSERVER,
newIntent.ybSoftwareVersion,
DEFAULT_WAIT_TIME_MS,
universeOverrides,
azOverrides,
false,
Expand All @@ -524,7 +522,7 @@ private boolean editCluster(
KubernetesCommandExecutor.CommandType.HELM_UPGRADE,
universe.isYbcEnabled(),
universe.getUniverseDetails().getYbcSoftwareVersion(),
/* addDelayAfterStartup */ false);
PodUpgradeParams.DEFAULT);
} else if (instanceTypeChanged) {
upgradePodsTask(
universe.getName(),
Expand All @@ -533,7 +531,6 @@ private boolean editCluster(
curPlacement,
ServerType.TSERVER,
newIntent.ybSoftwareVersion,
DEFAULT_WAIT_TIME_MS,
universeOverrides,
azOverrides,
false,
Expand All @@ -543,7 +540,7 @@ private boolean editCluster(
KubernetesCommandExecutor.CommandType.HELM_UPGRADE,
universe.isYbcEnabled(),
universe.getUniverseDetails().getYbcSoftwareVersion(),
/* addDelayAfterStartup */ false);
PodUpgradeParams.DEFAULT);
} else if (masterAddressesChanged) {
// Update master_addresses flag on Master
// and tserver_master_addrs flag on tserver without restart.
Expand Down
Loading

0 comments on commit f44c92e

Please sign in to comment.