Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added wait for service external Ip assignment #11325

Merged
merged 14 commits into from
Sep 18, 2019
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,6 @@
"loc.messages.InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary",
"loc.messages.AllContainersNotInReadyState": "All the containers are not in a ready state.",
"loc.messages.CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s",
"loc.messages.KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features."
"loc.messages.KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features.",
"loc.messages.CouldNotDetermineServiceStatus": "Could not determine the service's status due to the error: %s"
}
5 changes: 5 additions & 0 deletions Tasks/KubernetesManifestV0/src/models/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ export class KubernetesWorkload {
public static cronjob: string = 'cronjob';
}

export class DiscoveryAndLoadBalancerResource {
public static service: string = 'service';
public static ingress: string = 'ingress';
}

export const deploymentTypes: string[] = ['deployment', 'replicaset', 'daemonset', 'pod', 'statefulset'];
export const workloadTypes: string[] = ['deployment', 'replicaset', 'daemonset', 'pod', 'statefulset', 'job', 'cronjob'];
export const workloadTypesWithRolloutStatus: string[] = ['deployment', 'daemonset', 'statefulset'];
Expand Down
53 changes: 52 additions & 1 deletion Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,13 @@ export async function deploy(kubectl: Kubectl, manifestFilePaths: string[], depl
const deployedManifestFiles = deployManifests(inputManifestFiles, kubectl, isCanaryDeploymentStrategy(deploymentStrategy));

// check manifest stability
const resourceTypes: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, models.deploymentTypes);
const resourceTypes: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, models.deploymentTypes.concat([constants.DiscoveryAndLoadBalancerResource.service]) );
await checkManifestStability(kubectl, resourceTypes);

// print ingress resources
const ingressResources: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, [constants.DiscoveryAndLoadBalancerResource.ingress]);
getIngressResources(kubectl, ingressResources);

// annotate resources
const allPods = JSON.parse((kubectl.getAllPods()).stdout);
annotateResources(deployedManifestFiles, kubectl, resourceTypes, allPods);
Expand Down Expand Up @@ -92,6 +96,18 @@ async function checkManifestStability(kubectl: Kubectl, resources: Resource[]):
tl.warning(tl.loc('CouldNotDeterminePodStatus', JSON.stringify(ex)));
}
}
if (isEqual(resource.type, constants.DiscoveryAndLoadBalancerResource.service, StringComparer.OrdinalIgnoreCase)) {
try {
const service = getService(kubectl, resource.name);
const spec = service.spec;
const status = service.status;
if (isEqual(spec.type, "LoadBalancer", StringComparer.OrdinalIgnoreCase) && !isLoadBalancerIPAssigned(status)) {
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
await waitForServiceExternalIPAssignment(kubectl, resource.name);
}
} catch (ex) {
tl.warning(tl.loc('CouldNotDetermineServiceStatus', JSON.stringify(ex)));
}
}
}
utils.checkForErrors(rolloutStatusResults);
}
Expand Down Expand Up @@ -263,6 +279,41 @@ function isPodReady(podStatus: any): boolean {
return allContainersAreReady;
}

function getService(kubectl: Kubectl, serviceName) {
const serviceResult = kubectl.getResource(constants.DiscoveryAndLoadBalancerResource.service, serviceName);
utils.checkForErrors([serviceResult]);
return JSON.parse(serviceResult.stdout)
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
}

async function waitForServiceExternalIPAssignment(kubectl: Kubectl, serviceName: string): Promise<void> {
const sleepTimeout = 10 * 1000; // 10 seconds
const iterations = 18; // 18 * 10 seconds timeout = 3 minutes max timeout

for (let i = 0; i < iterations; i++) {
await sleep(sleepTimeout);
tl.debug(`Waiting for service ${serviceName} IP assignment`);
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
let status = getService(kubectl, serviceName).status;
if (isLoadBalancerIPAssigned(status)) {
return;
}
}
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
tl.warning(`wait for service/${serviceName} external IP assignment timedout`);
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
}

function isLoadBalancerIPAssigned(status: any) {
if (status.loadBalancer && status.loadBalancer.ingress && status.loadBalancer.ingress.length > 0) {
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
return true;
}
return false;
}

function getIngressResources(kubectl: Kubectl, ingressResources: Resource[]) {
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
const numberOfResources = ingressResources.length;
for (let i = 0; i < numberOfResources; i++) {
anuragc617 marked this conversation as resolved.
Show resolved Hide resolved
kubectl.getResource(constants.DiscoveryAndLoadBalancerResource.ingress, ingressResources[i].name);
}
}

function sleep(timeout: number) {
return new Promise(resolve => setTimeout(resolve, timeout));
}
5 changes: 3 additions & 2 deletions Tasks/KubernetesManifestV0/task.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"author": "Microsoft Corporation",
"version": {
"Major": 0,
"Minor": 158,
"Minor": 159,
"Patch": 0
},
"demands": [],
Expand Down Expand Up @@ -333,6 +333,7 @@
"InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary",
"AllContainersNotInReadyState": "All the containers are not in a ready state.",
"CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s",
"KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features."
"KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features.",
"CouldNotDetermineServiceStatus": "Could not determine the service's status due to the error: %s"
}
}
5 changes: 3 additions & 2 deletions Tasks/KubernetesManifestV0/task.loc.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"author": "Microsoft Corporation",
"version": {
"Major": 0,
"Minor": 158,
"Minor": 159,
"Patch": 0
},
"demands": [],
Expand Down Expand Up @@ -333,6 +333,7 @@
"InvalidPromotetActionDeploymentStrategy": "ms-resource:loc.messages.InvalidPromotetActionDeploymentStrategy",
"AllContainersNotInReadyState": "ms-resource:loc.messages.AllContainersNotInReadyState",
"CouldNotDeterminePodStatus": "ms-resource:loc.messages.CouldNotDeterminePodStatus",
"KubectlShouldBeUpgraded": "ms-resource:loc.messages.KubectlShouldBeUpgraded"
"KubectlShouldBeUpgraded": "ms-resource:loc.messages.KubectlShouldBeUpgraded",
"CouldNotDetermineServiceStatus": "ms-resource:loc.messages.CouldNotDetermineServiceStatus"
}
}