Skip to content

Commit

Permalink
[Platform][Kubernetes] Allow specifying namespace in the AZ config (#…
Browse files Browse the repository at this point in the history
…6807)

These set of changes introduces a new key KUBENAMESPACE in the AZ
config. Useful for the situations where the platform software does not
have access to create, list or delete namespaces in the cluster.

When KUBENAMESPACE is present in the AZ config, the deployment in that
AZ in done in that namespace only. Platform does not try to create
namespace when creating the universe. It does not delete the namespace
when destroying the universe.

Makes changes to the kubectl, helm commands from
KubernetesManager.java to use namespace as well as nodePrefix. The
value of nodePrefix and namespace can be same if the AZ does not have
KUBENAMESPACE.

DestroyKubernetesUniverse now always deletes the volumes before
deleting the namespace and uses release label selector to make sure
the volumes being deleted are of the universe being
destroyed. Deleting a failed universe was deleting volumes of an
existing one.

Use kubectl apply for the pull secret. This makes sure that we don't
fail if the namespace already has a pull secret in it. This will
just update the existing pull secret.

[Platform UI] Add option to specify namespace in the AZ modal
- Adds the given value as KUBENAMESPACE in the zone config.

Fixes #6551
Fixes #6702

Test plan:

The following tests have been done on OpenShift, where the Kubernetes
ServiceAccount does not have access to create namespaces.

- Created a single AZ provider with KUBENAMESPACE set.
- Created a universe with it, tried GFlags update, database backup,
  universe edit operations, these pass as expected.
- Destroyed the same universe.
- Performed same set of operations with a multi AZ provider and a
  universe.

Signed-off-by: Bhavin Gandhi <[email protected]>
  • Loading branch information
bhavin192 authored Feb 8, 2021
1 parent 72fe704 commit ab0adaa
Show file tree
Hide file tree
Showing 19 changed files with 920 additions and 343 deletions.
5 changes: 5 additions & 0 deletions managed/devops/bin/cluster_health.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,11 @@ def __init__(self, node_fqdn, config_map):
self.pod_name = node_fqdn.split('.')[0]
# The pod names are yb-master-n/yb-tserver-n where n is the pod number
# and yb-master/yb-tserver are the container names.

# TODO(bhavin192): need to change in case of multiple releases
# in one namespace. Something like find the word 'master' in
# the name.

self.container = self.pod_name.rsplit('-', 1)[0]
self.config = config_map[self.namespace]

Expand Down
5 changes: 5 additions & 0 deletions managed/devops/bin/yb_backup.py
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,11 @@ def __init__(self, server_fqdn, config_map):
self.pod_name = server_fqdn.split('.')[0]
# The pod names are yb-master-n/yb-tserver-n where n is the pod number
# and yb-master/yb-tserver are the container names.

# TODO(bhavin192): need to change in case of multiple releases
# in one namespace. Something like find the word 'master' in
# the name.

self.container = self.pod_name.rsplit('-', 1)[0]
self.env_config = os.environ.copy()
self.env_config["KUBECONFIG"] = config_map[self.namespace]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ public void run() {
KubernetesCommandExecutor.CommandType.HELM_DELETE.getSubTaskGroupName(), executor);
helmDeletes.setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.RemovingUnusedServers);

SubTaskGroup volumeDeletes = new SubTaskGroup(
KubernetesCommandExecutor.CommandType.VOLUME_DELETE.getSubTaskGroupName(), executor);
volumeDeletes.setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.RemovingUnusedServers);

SubTaskGroup namespaceDeletes = new SubTaskGroup(
KubernetesCommandExecutor.CommandType.NAMESPACE_DELETE.getSubTaskGroupName(), executor);
namespaceDeletes.setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.RemovingUnusedServers);
Expand All @@ -79,23 +83,44 @@ public void run() {

Map<String, String> config = entry.getValue();

if (runHelmDelete) {
String namespace = config.get("KUBENAMESPACE");

if (runHelmDelete || namespace != null) {
// Delete the helm deployments.
helmDeletes.addTask(createDestroyKubernetesTask(
universe.getUniverseDetails().nodePrefix, azName, config,
KubernetesCommandExecutor.CommandType.HELM_DELETE,
providerUUID));
}

// Delete the namespaces of the deployments.
namespaceDeletes.addTask(createDestroyKubernetesTask(
// Delete the PVCs created for this AZ.
volumeDeletes.addTask(createDestroyKubernetesTask(
universe.getUniverseDetails().nodePrefix, azName, config,
KubernetesCommandExecutor.CommandType.NAMESPACE_DELETE,
KubernetesCommandExecutor.CommandType.VOLUME_DELETE,
providerUUID));

// TODO(bhavin192): delete the pull secret as well? As of now,
// we depend on the fact that, deleting the namespace will
// delete the pull secret. That won't be the case with
// providers which have KUBENAMESPACE paramter in the AZ
// config. How to find the pull secret name? Should we delete
// it when we have multiple releases in one namespace?. It is
// possible that same provider is creating multiple releases
// in one namespace. Tracked here:
// https://github.com/yugabyte/yugabyte-db/issues/7012

// Delete the namespaces of the deployments only if those were
// created by us.
if (namespace == null) {
namespaceDeletes.addTask(createDestroyKubernetesTask(
universe.getUniverseDetails().nodePrefix, azName, config,
KubernetesCommandExecutor.CommandType.NAMESPACE_DELETE,
providerUUID));
}
}

subTaskGroupQueue.add(helmDeletes);
subTaskGroupQueue.add(volumeDeletes);
subTaskGroupQueue.add(namespaceDeletes);

// Create tasks to remove the universe entry from the Universe table.
Expand Down Expand Up @@ -133,6 +158,10 @@ public void run() {
}
if (config != null) {
params.config = config;
// This assumes that the config is az config. It is true in this
// particular case, all callers just pass az config.
// params.namespace remains null if config is not passed.
params.namespace = PlacementInfoUtil.getKubernetesNamespace(nodePrefix, az, config);
}
params.universeUUID = taskParams().universeUUID;
KubernetesCommandExecutor task = new KubernetesCommandExecutor();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,9 +154,14 @@ public void createPodsTask(KubernetesPlacement newPlacement, String masterAddres
KubernetesCheckNumPod.CommandType.WAIT_FOR_PODS, azCode, config,
podsToWaitFor));
} else {
// Create the namespaces of the deployment.
createNamespaces.addTask(createKubernetesExecutorTask(
KubernetesCommandExecutor.CommandType.CREATE_NAMESPACE, azCode, config));
// Don't create the namespace if user has provided
// KUBENAMESPACE value, as we might not have access to list or
// create namespaces in such cases.
if (config.get("KUBENAMESPACE") == null) {
// Create the namespaces of the deployment.
createNamespaces.addTask(createKubernetesExecutorTask(
KubernetesCommandExecutor.CommandType.CREATE_NAMESPACE, azCode, config));
}

// Apply the necessary pull secret to each namespace.
applySecrets.addTask(createKubernetesExecutorTask(
Expand Down Expand Up @@ -241,6 +246,8 @@ public void upgradePodsTask(KubernetesPlacement newPlacement, String masterAddre
createSingleKubernetesExecutorTaskForServerType(CommandType.HELM_UPGRADE,
tempPI, azCode, masterAddresses, softwareVersion, serverType, config,
masterPartition, tserverPartition);
// TODO(bhavin192): might need to account for multiple
// releases in one namespace.
String podName = String.format("%s-%d", sType, partition);
createKubernetesWaitForPodTask(KubernetesWaitForPod.CommandType.WAIT_FOR_POD,
podName, azCode, config);
Expand Down Expand Up @@ -318,9 +325,13 @@ public void deletePodsTask(KubernetesPlacement currPlacement, String masterAddre
volumeDeletes.addTask(createKubernetesExecutorTask(
KubernetesCommandExecutor.CommandType.VOLUME_DELETE, azCode, config));

// Delete the namespaces of the deployments.
namespaceDeletes.addTask(createKubernetesExecutorTask(
KubernetesCommandExecutor.CommandType.NAMESPACE_DELETE, azCode, config));
// If the namespace is configured at the AZ, we don't delete
// it, as it is not created by us.
if (config.get("KUBENAMESPACE") == null) {
// Delete the namespaces of the deployments.
namespaceDeletes.addTask(createKubernetesExecutorTask(
KubernetesCommandExecutor.CommandType.NAMESPACE_DELETE, azCode, config));
}
}
}
subTaskGroupQueue.add(helmDeletes);
Expand Down Expand Up @@ -431,6 +442,10 @@ public KubernetesCommandExecutor createKubernetesExecutorTaskForServerType(
}
if (config != null) {
params.config = config;
// This assumes that the config is az config.
// params.namespace remains null if config is not passed.
params.namespace = PlacementInfoUtil.getKubernetesNamespace(taskParams().nodePrefix,
az, config);
}
params.masterPartition = masterPartition;
params.tserverPartition = tserverPartition;
Expand Down Expand Up @@ -486,6 +501,10 @@ public void createSingleKubernetesExecutorTaskForServerType(
}
if (config != null) {
params.config = config;
// This assumes that the config is az config.
// params.namespace remains null if config is not passed.
params.namespace = PlacementInfoUtil.getKubernetesNamespace(taskParams().nodePrefix,
az, config);
}
params.masterPartition = masterPartition;
params.tserverPartition = tserverPartition;
Expand All @@ -509,11 +528,16 @@ public void createKubernetesWaitForPodTask(KubernetesWaitForPod.CommandType comm
params.providerUUID = UUID.fromString(primary.userIntent.provider);
params.commandType = commandType;
params.nodePrefix = taskParams().nodePrefix;

if (az != null) {
params.nodePrefix = String.format("%s-%s", params.nodePrefix, az);
}
if (config != null) {
params.config = config;
// This assumes that the config is az config.
// params.namespace remains null if config is not passed.
params.namespace = PlacementInfoUtil.getKubernetesNamespace(taskParams().nodePrefix,
az, config);
}
params.universeUUID = taskParams().universeUUID;
params.podName = podName;
Expand All @@ -537,6 +561,10 @@ public KubernetesCheckNumPod createKubernetesCheckPodNumTask(
}
if (config != null) {
params.config = config;
// This assumes that the config is az config.
// params.namespace remains null if config is not passed.
params.namespace = PlacementInfoUtil.getKubernetesNamespace(taskParams().nodePrefix,
az, config);
}
params.universeUUID = taskParams().universeUUID;
params.podNum = numPods;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public static class Params extends AbstractTaskParams {
// We use the nodePrefix as Helm Chart's release name,
// so we would need that for any sort helm operations.
public String nodePrefix;
public String namespace;
public int podNum = 0;
public Map<String, String> config = null;
}
Expand Down Expand Up @@ -106,7 +107,7 @@ private boolean waitForPods() {
if (taskParams().config == null) {
config = Provider.get(taskParams().providerUUID).getConfig();
}
ShellResponse podResponse = kubernetesManager.getPodInfos(config, taskParams().nodePrefix);
ShellResponse podResponse = kubernetesManager.getPodInfos(config, taskParams().nodePrefix, taskParams().namespace);
JsonNode podInfos = parseShellResponseAsJson(podResponse);
if (podInfos.path("items").size() == taskParams().podNum) {
return true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ public static class Params extends UniverseTaskParams {
// We use the nodePrefix as Helm Chart's release name,
// so we would need that for any sort helm operations.
public String nodePrefix;
public String namespace;
public String ybSoftwareVersion = null;
public boolean enableNodeToNodeEncrypt = false;
public boolean enableClientToNodeEncrypt = false;
Expand Down Expand Up @@ -164,42 +165,48 @@ public void run() {
if (config == null) {
config = Provider.get(taskParams().providerUUID).getConfig();
}
if (taskParams().commandType != CommandType.POD_INFO && taskParams().namespace == null) {
throw new IllegalArgumentException("namespace can be null only in case of POD_INFO");
}

// TODO: add checks for the shell process handler return values.
ShellResponse response = null;
switch (taskParams().commandType) {
case CREATE_NAMESPACE:
response = kubernetesManager.createNamespace(config, taskParams().nodePrefix);
response = kubernetesManager.createNamespace(config, taskParams().namespace);
break;
case APPLY_SECRET:
String pullSecret = this.getPullSecret();
if (pullSecret != null) {
response = kubernetesManager.applySecret(config, taskParams().nodePrefix, pullSecret);
response = kubernetesManager.applySecret(config, taskParams().namespace, pullSecret);
}
break;
case HELM_INSTALL:
overridesFile = this.generateHelmOverride();
response = kubernetesManager.helmInstall(config, taskParams().providerUUID, taskParams().nodePrefix, overridesFile);
response = kubernetesManager.helmInstall(config, taskParams().providerUUID, taskParams().nodePrefix,taskParams().namespace, overridesFile);
flag = true;
break;
case HELM_UPGRADE:
overridesFile = this.generateHelmOverride();
response = kubernetesManager.helmUpgrade(config, taskParams().nodePrefix, overridesFile);
response = kubernetesManager.helmUpgrade(config, taskParams().nodePrefix, taskParams().namespace, overridesFile);
flag = true;
break;
case UPDATE_NUM_NODES:
int numNodes = this.getNumNodes();
if (numNodes > 0) {
response = kubernetesManager.updateNumNodes(config, taskParams().nodePrefix, numNodes);
// TODO(bhavin192): we might also need nodePrefix later, so
// that we can have multiple releases in one namespace.
response = kubernetesManager.updateNumNodes(config, taskParams().namespace, numNodes);
}
break;
case HELM_DELETE:
kubernetesManager.helmDelete(config, taskParams().nodePrefix);
kubernetesManager.helmDelete(config, taskParams().nodePrefix, taskParams().namespace);
break;
case VOLUME_DELETE:
kubernetesManager.deleteStorage(config, taskParams().nodePrefix);
kubernetesManager.deleteStorage(config, taskParams().nodePrefix, taskParams().namespace);
break;
case NAMESPACE_DELETE:
kubernetesManager.deleteNamespace(config, taskParams().nodePrefix);
kubernetesManager.deleteNamespace(config, taskParams().namespace);
break;
case POD_INFO:
processNodeInfo();
Expand All @@ -216,7 +223,7 @@ public void run() {
private ShellResponse getPodError(Map<String, String> config) {
ShellResponse response = new ShellResponse();
response.code = -1;
ShellResponse podResponse = kubernetesManager.getPodInfos(config, taskParams().nodePrefix);
ShellResponse podResponse = kubernetesManager.getPodInfos(config, taskParams().nodePrefix, taskParams().namespace);
JsonNode podInfos = parseShellResponseAsJson(podResponse);
boolean flag = false;
for (JsonNode podInfo: podInfos.path("items")) {
Expand Down Expand Up @@ -263,16 +270,23 @@ private Map<String, String> getClusterIpForLoadBalancer() {
String regionName = AvailabilityZone.get(azUUID).region.code;
Map<String, String> config = entry.getValue();

String namespace = taskParams().nodePrefix;

// TODO(bhavin192): we seem to be iterating over all the AZs
// here, and still selecting services for only one AZ governed
// by the taskParams().nodePrefix. Is it even required to
// iterate in that case?
ShellResponse svcResponse =
kubernetesManager.getServices(config, namespace);
kubernetesManager.getServices(config, taskParams().nodePrefix, taskParams().namespace);
JsonNode svcInfos = parseShellResponseAsJson(svcResponse);

for (JsonNode svcInfo: svcInfos.path("items")) {
JsonNode serviceMetadata = svcInfo.path("metadata");
JsonNode serviceSpec = svcInfo.path("spec");
String serviceType = serviceSpec.path("type").asText();
// TODO(bhavin192): this will need an update when we have
// multiple releases in one namespace, the generated service
// name will be different and not just yb-masters, yb-tservers
// etc. Values file will still have yb-masters, yb-tservers
// etc.
serviceToIP.put(serviceMetadata.path("name").asText(),
serviceSpec.path("clusterIP").asText());
}
Expand All @@ -296,11 +310,12 @@ private void processNodeInfo() {
String regionName = AvailabilityZone.get(azUUID).region.code;
Map<String, String> config = entry.getValue();

String namespace = isMultiAz ?
String nodePrefix = isMultiAz ?
String.format("%s-%s", taskParams().nodePrefix, azName) : taskParams().nodePrefix;
String namespace = PlacementInfoUtil.getKubernetesNamespace(isMultiAz, taskParams().nodePrefix, azName, config);

ShellResponse podResponse =
kubernetesManager.getPodInfos(config, namespace);
kubernetesManager.getPodInfos(config, nodePrefix, namespace);
JsonNode podInfos = parseShellResponseAsJson(podResponse);

for (JsonNode podInfo: podInfos.path("items")) {
Expand All @@ -317,6 +332,12 @@ private void processNodeInfo() {
String podName = isMultiAz ?
String.format("%s_%s", podSpec.path("hostname").asText(), azName) :
podSpec.path("hostname").asText();
String podNamespace = podInfo.path("metadata").path("namespace").asText();
if (podNamespace.isEmpty() || podNamespace == null) {
throw new IllegalArgumentException("metadata.namespace of pod " + podName
+ " is empty. This shouldn't happen");
}
pod.put("namespace", podNamespace);
pods.set(podName, pod);
}
}
Expand All @@ -331,13 +352,8 @@ private void processNodeInfo() {
NodeDetails nodeDetail = defaultNode.clone();
Map.Entry<String, JsonNode> pod = iter.next();
String hostname = pod.getKey();

// The namespace of the deployment in multi-az is constructed by appending
// the zone to the universe name.
String namespace = isMultiAz ?
PlacementInfoUtil.getKubernetesNamespace(taskParams().nodePrefix, hostname.split("_")[1]) :
taskParams().nodePrefix;
JsonNode podVals = pod.getValue();
String namespace = podVals.get("namespace").asText();
UUID azUUID = UUID.fromString(podVals.get("az_uuid").asText());
String domain = azToDomain.get(azUUID);
if (hostname.contains("master")) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,14 @@ public static class Params extends AbstractTaskParams {
public UUID providerUUID;
public CommandType commandType;
public UUID universeUUID;
// TODO(bhavin192): nodePrefix can be removed as we are not doing
// any sort of Helm operation here. Or we might want to use it for
// some sort of label based selection.

// We use the nodePrefix as Helm Chart's release name,
// so we would need that for any sort helm operations.
public String nodePrefix;
public String namespace;
public String podName = null;
public Map<String, String> config = null;
}
Expand Down Expand Up @@ -118,7 +123,7 @@ private String waitForPod() {
if (taskParams().config == null) {
config = Provider.get(taskParams().providerUUID).getConfig();
}
ShellResponse podResponse = kubernetesManager.getPodStatus(config, taskParams().nodePrefix,
ShellResponse podResponse = kubernetesManager.getPodStatus(config, taskParams().namespace,
taskParams().podName);
JsonNode podInfo = parseShellResponseAsJson(podResponse);
JsonNode statusNode = podInfo.path("status");
Expand Down
Loading

0 comments on commit ab0adaa

Please sign in to comment.