From 827236e9c126522690212b0e522fb95f19dd810a Mon Sep 17 00:00:00 2001 From: Deepak Sattiraju Date: Tue, 18 Jun 2019 19:31:47 +0530 Subject: [PATCH 1/3] [KubernetesManifest] Check manifest stability for Pods --- .../resources.resjson/en-US/resources.resjson | 3 +- .../src/utils/DeploymentHelper.ts | 60 +++++++++++++++++++ Tasks/KubernetesManifestV0/task.json | 5 +- Tasks/KubernetesManifestV0/task.loc.json | 5 +- 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson index 5007602a2f3a..bd25f4e5114d 100644 --- a/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson @@ -66,5 +66,6 @@ "loc.messages.NullInputObjectMetadata": "Input object metadata is null.", "loc.messages.CanaryDeploymentAlreadyExistErrorMessage": "Canary deployment already exists. Rejecting this deployment.", "loc.messages.InvalidRejectActionDeploymentStrategy": "Reject action works only with strategy: canary", - "loc.messages.InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary" + "loc.messages.InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary", + "loc.messages.AllContainersNotInReadyState": "All the containers are not in a ready state" } \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts index 9b7edd437282..017354c28934 100644 --- a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts +++ b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts @@ -13,6 +13,7 @@ import * as fileHelper from '../utils/FileHelper'; import * as utils from '../utils/utilities'; import { IExecSyncResult } from 'azure-pipelines-task-lib/toolrunner'; import { Kubectl, Resource } from 'kubernetes-common-v2/kubectl-object-model'; +import { isEqual, StringComparer } from './StringComparison'; export function deploy(kubectl: Kubectl, manifestFilePaths: string[], deploymentStrategy: string) { @@ -65,6 +66,9 @@ function checkManifestStability(kubectl: Kubectl, resourceTypes: Resource[]) { if (models.recognizedWorkloadTypesWithRolloutStatus.indexOf(resource.type.toLowerCase()) >= 0) { rolloutStatusResults.push(kubectl.checkRolloutStatus(resource.type, resource.name)); } + if (isEqual(resource.type, constants.KubernetesWorkload.Pod, StringComparer.OrdinalIgnoreCase)) { + checkPodStatus(kubectl, resource.name); + } }); utils.checkForErrors(rolloutStatusResults); } @@ -137,3 +141,59 @@ function updateImagePullSecretsInManifestFiles(filePaths: string[], imagePullSec function isCanaryDeploymentStrategy(deploymentStrategy: string): boolean { return deploymentStrategy != null && deploymentStrategy.toUpperCase() == canaryDeploymentHelper.CANARY_DEPLOYMENT_STRATEGY.toUpperCase(); } + +function checkPodStatus(kubectl: Kubectl, podName: string) { + const startTime = new Date(); + const timeOut = 5 * 60 * 1000; // Timeout 5 min + let currentTime = new Date(); + let podStatus; + while (currentTime.getTime() - startTime.getTime() < timeOut) { + tl.debug('Polling for pod status'); + podStatus = getPodStatus(kubectl, podName); + if (podStatus.status && podStatus.phase) { + if (podStatus.phase !== 'Pending') { + break; + } + } + currentTime = new Date(); + } + podStatus = getPodStatus(kubectl, podName); + switch (podStatus.phase) { + case 'Succeeded': + case 'Running': + if (checkContainerStatuses(podStatus)) { + console.log(`pod/${podName} is successfully rolled out`); + } + break; + case 'Pending': + if (!checkContainerStatuses(podStatus)) { + tl.warning(`pod/${podName} rollout status check timedout`); + } + break; + case 'Failed': + tl.error(`pod/${podName} rollout failed`); + break; + default: + tl.warning(`pod/${podName} rollout status: ${podStatus.status.phase}`); + } +} + +function getPodStatus(kubectl: Kubectl, podName: string): any { + const podResult = kubectl.getResource('pod', podName); + utils.checkForErrors([podResult]); + return JSON.parse(podResult.stdout).status; +} + +function checkContainerStatuses(podStatus: any): boolean { + let allReady = true; + podStatus.containerStatuses.forEach(container => { + if (container.ready === false) { + console.log(`'${container.name}' status: ${JSON.stringify(container.state)}`); + allReady = false; + } + }); + if (!allReady) { + tl.warning(tl.loc('AllContainersNotInReadyState')); + } + return allReady; +} diff --git a/Tasks/KubernetesManifestV0/task.json b/Tasks/KubernetesManifestV0/task.json index 085cd4839875..50ff2cd7c2a4 100644 --- a/Tasks/KubernetesManifestV0/task.json +++ b/Tasks/KubernetesManifestV0/task.json @@ -14,7 +14,7 @@ "version": { "Major": 0, "Minor": 154, - "Patch": 2 + "Patch": 3 }, "demands": [], "groups": [], @@ -310,6 +310,7 @@ "NullInputObjectMetadata": "Input object metadata is null.", "CanaryDeploymentAlreadyExistErrorMessage": "Canary deployment already exists. Rejecting this deployment.", "InvalidRejectActionDeploymentStrategy": "Reject action works only with strategy: canary", - "InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary" + "InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary", + "AllContainersNotInReadyState": "All the containers are not in a ready state" } } \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/task.loc.json b/Tasks/KubernetesManifestV0/task.loc.json index 7dbe277b4357..9762a2c8ae8f 100644 --- a/Tasks/KubernetesManifestV0/task.loc.json +++ b/Tasks/KubernetesManifestV0/task.loc.json @@ -14,7 +14,7 @@ "version": { "Major": 0, "Minor": 154, - "Patch": 2 + "Patch": 3 }, "demands": [], "groups": [], @@ -310,6 +310,7 @@ "NullInputObjectMetadata": "ms-resource:loc.messages.NullInputObjectMetadata", "CanaryDeploymentAlreadyExistErrorMessage": "ms-resource:loc.messages.CanaryDeploymentAlreadyExistErrorMessage", "InvalidRejectActionDeploymentStrategy": "ms-resource:loc.messages.InvalidRejectActionDeploymentStrategy", - "InvalidPromotetActionDeploymentStrategy": "ms-resource:loc.messages.InvalidPromotetActionDeploymentStrategy" + "InvalidPromotetActionDeploymentStrategy": "ms-resource:loc.messages.InvalidPromotetActionDeploymentStrategy", + "AllContainersNotInReadyState": "ms-resource:loc.messages.AllContainersNotInReadyState" } } \ No newline at end of file From d7b60ad323a0f1220e70f865bfe0ab708d86ffc5 Mon Sep 17 00:00:00 2001 From: Deepak Sattiraju Date: Wed, 19 Jun 2019 13:03:32 +0530 Subject: [PATCH 2/3] renaming function --- Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts index 017354c28934..4dc1297aa273 100644 --- a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts +++ b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts @@ -161,12 +161,12 @@ function checkPodStatus(kubectl: Kubectl, podName: string) { switch (podStatus.phase) { case 'Succeeded': case 'Running': - if (checkContainerStatuses(podStatus)) { + if (checkIfAllContainersAreInReadyState(podStatus)) { console.log(`pod/${podName} is successfully rolled out`); } break; case 'Pending': - if (!checkContainerStatuses(podStatus)) { + if (!checkIfAllContainersAreInReadyState(podStatus)) { tl.warning(`pod/${podName} rollout status check timedout`); } break; @@ -184,7 +184,7 @@ function getPodStatus(kubectl: Kubectl, podName: string): any { return JSON.parse(podResult.stdout).status; } -function checkContainerStatuses(podStatus: any): boolean { +function checkIfAllContainersAreInReadyState(podStatus: any): boolean { let allReady = true; podStatus.containerStatuses.forEach(container => { if (container.ready === false) { From 04f5acdb9a86b69d102bdc0aaadce9c4d5f3e0ed Mon Sep 17 00:00:00 2001 From: Deepak Sattiraju Date: Mon, 24 Jun 2019 15:50:40 +0530 Subject: [PATCH 3/3] Resolving PR comments --- .../resources.resjson/en-US/resources.resjson | 3 +- .../src/utils/DeploymentHelper.ts | 34 +++++++++++-------- Tasks/KubernetesManifestV0/task.json | 3 +- Tasks/KubernetesManifestV0/task.loc.json | 3 +- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson index bd25f4e5114d..7aa5500b2c6e 100644 --- a/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson @@ -67,5 +67,6 @@ "loc.messages.CanaryDeploymentAlreadyExistErrorMessage": "Canary deployment already exists. Rejecting this deployment.", "loc.messages.InvalidRejectActionDeploymentStrategy": "Reject action works only with strategy: canary", "loc.messages.InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary", - "loc.messages.AllContainersNotInReadyState": "All the containers are not in a ready state" + "loc.messages.AllContainersNotInReadyState": "All the containers are not in a ready state", + "loc.messages.CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s" } \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts index 4dc1297aa273..77f162f5955d 100644 --- a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts +++ b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts @@ -41,7 +41,7 @@ function getManifestFiles(manifestFilePaths: string[]): string[] { const files: string[] = utils.getManifestFiles(manifestFilePaths); if (files == null || files.length === 0) { - throw (tl.loc('ManifestFileNotFound')); + throw (tl.loc('ManifestFileNotFound', manifestFilePaths)); } return files; @@ -67,7 +67,11 @@ function checkManifestStability(kubectl: Kubectl, resourceTypes: Resource[]) { rolloutStatusResults.push(kubectl.checkRolloutStatus(resource.type, resource.name)); } if (isEqual(resource.type, constants.KubernetesWorkload.Pod, StringComparer.OrdinalIgnoreCase)) { - checkPodStatus(kubectl, resource.name); + try { + checkPodStatus(kubectl, resource.name); + } catch (ex) { + tl.warning(tl.loc('CouldNotDeterminePodStatus', JSON.stringify(ex))); + } } }); utils.checkForErrors(rolloutStatusResults); @@ -148,12 +152,10 @@ function checkPodStatus(kubectl: Kubectl, podName: string) { let currentTime = new Date(); let podStatus; while (currentTime.getTime() - startTime.getTime() < timeOut) { - tl.debug('Polling for pod status'); + tl.debug(`Polling for pod status: ${podName}`); podStatus = getPodStatus(kubectl, podName); - if (podStatus.status && podStatus.phase) { - if (podStatus.phase !== 'Pending') { + if (podStatus.phase && podStatus.phase !== 'Pending') { break; - } } currentTime = new Date(); } @@ -161,12 +163,12 @@ function checkPodStatus(kubectl: Kubectl, podName: string) { switch (podStatus.phase) { case 'Succeeded': case 'Running': - if (checkIfAllContainersAreInReadyState(podStatus)) { + if (isPodReady(podStatus)) { console.log(`pod/${podName} is successfully rolled out`); } break; case 'Pending': - if (!checkIfAllContainersAreInReadyState(podStatus)) { + if (!isPodReady(podStatus)) { tl.warning(`pod/${podName} rollout status check timedout`); } break; @@ -174,26 +176,28 @@ function checkPodStatus(kubectl: Kubectl, podName: string) { tl.error(`pod/${podName} rollout failed`); break; default: - tl.warning(`pod/${podName} rollout status: ${podStatus.status.phase}`); + tl.warning(`pod/${podName} rollout status: ${podStatus.phase}`); } } function getPodStatus(kubectl: Kubectl, podName: string): any { const podResult = kubectl.getResource('pod', podName); utils.checkForErrors([podResult]); - return JSON.parse(podResult.stdout).status; + const podStatus = JSON.parse(podResult.stdout).status; + tl.debug(`Pod Status: ${JSON.stringify(podStatus)}`); + return podStatus; } -function checkIfAllContainersAreInReadyState(podStatus: any): boolean { - let allReady = true; +function isPodReady(podStatus: any): boolean { + let allContainersAreReady = true; podStatus.containerStatuses.forEach(container => { if (container.ready === false) { console.log(`'${container.name}' status: ${JSON.stringify(container.state)}`); - allReady = false; + allContainersAreReady = false; } }); - if (!allReady) { + if (!allContainersAreReady) { tl.warning(tl.loc('AllContainersNotInReadyState')); } - return allReady; + return allContainersAreReady; } diff --git a/Tasks/KubernetesManifestV0/task.json b/Tasks/KubernetesManifestV0/task.json index bdcbcbe5b66a..159a4d45c685 100644 --- a/Tasks/KubernetesManifestV0/task.json +++ b/Tasks/KubernetesManifestV0/task.json @@ -311,6 +311,7 @@ "CanaryDeploymentAlreadyExistErrorMessage": "Canary deployment already exists. Rejecting this deployment.", "InvalidRejectActionDeploymentStrategy": "Reject action works only with strategy: canary", "InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary", - "AllContainersNotInReadyState": "All the containers are not in a ready state" + "AllContainersNotInReadyState": "All the containers are not in a ready state", + "CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s" } } \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/task.loc.json b/Tasks/KubernetesManifestV0/task.loc.json index 1506ddd6172d..a0a48a97e3a6 100644 --- a/Tasks/KubernetesManifestV0/task.loc.json +++ b/Tasks/KubernetesManifestV0/task.loc.json @@ -311,6 +311,7 @@ "CanaryDeploymentAlreadyExistErrorMessage": "ms-resource:loc.messages.CanaryDeploymentAlreadyExistErrorMessage", "InvalidRejectActionDeploymentStrategy": "ms-resource:loc.messages.InvalidRejectActionDeploymentStrategy", "InvalidPromotetActionDeploymentStrategy": "ms-resource:loc.messages.InvalidPromotetActionDeploymentStrategy", - "AllContainersNotInReadyState": "ms-resource:loc.messages.AllContainersNotInReadyState" + "AllContainersNotInReadyState": "ms-resource:loc.messages.AllContainersNotInReadyState", + "CouldNotDeterminePodStatus": "ms-resource:loc.messages.CouldNotDeterminePodStatus" } } \ No newline at end of file