From 5b90186cb617e4fb028b8ff8c27b383d186ad4f7 Mon Sep 17 00:00:00 2001 From: anuragc617 <44864882+anuragc617@users.noreply.github.com> Date: Wed, 18 Sep 2019 23:31:14 +0530 Subject: [PATCH] added wait for service external Ip assignment (#11325) * service wait for ip * updated task.json * code refactoring * addressed review comments * addressing review comments * some more review comments * added test for service IP --- .../resources.resjson/en-US/resources.resjson | 6 ++- Tasks/KubernetesManifestV0/Tests/L0.ts | 1 + Tasks/KubernetesManifestV0/Tests/TestSetup.ts | 7 +++ .../Tests/manifests/deployment.yaml | 20 +++++++- .../src/models/constants.ts | 11 ++++ .../src/utils/DeploymentHelper.ts | 51 ++++++++++++++++++- Tasks/KubernetesManifestV0/task.json | 6 ++- Tasks/KubernetesManifestV0/task.loc.json | 6 ++- 8 files changed, 103 insertions(+), 5 deletions(-) diff --git a/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson index 69dae446fe55..e7d545466248 100644 --- a/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/KubernetesManifestV0/Strings/resources.resjson/en-US/resources.resjson @@ -74,5 +74,9 @@ "loc.messages.InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary", "loc.messages.AllContainersNotInReadyState": "All the containers are not in a ready state.", "loc.messages.CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s", - "loc.messages.KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features." + "loc.messages.KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features.", + "loc.messages.CouldNotDetermineServiceStatus": "Could not determine the service %s status due to the error: %s", + "loc.messages.waitForServiceIpAssignment": "Waiting for service %s external IP assignment", + "loc.messages.waitForServiceIpAssignmentTimedOut": "Wait for service %s external IP assignment timed out", + "loc.messages.ServiceExternalIP": "service %s external IP is %s" } \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/Tests/L0.ts b/Tasks/KubernetesManifestV0/Tests/L0.ts index a58d10cd25a6..16a1e04ac015 100644 --- a/Tasks/KubernetesManifestV0/Tests/L0.ts +++ b/Tasks/KubernetesManifestV0/Tests/L0.ts @@ -45,6 +45,7 @@ describe('Kubernetes Manifests Suite', function () { process.env[shared.TestEnvVars.imagePullSecrets] = 'test-key1\ntest-key2'; tr.run(); assert(tr.succeeded, 'task should have succeeded'); + assert(tr.stdout.indexOf('nginx-service 104.211.243.77') != -1, 'nginx-service external IP is 104.211.243.77') done(); }); diff --git a/Tasks/KubernetesManifestV0/Tests/TestSetup.ts b/Tasks/KubernetesManifestV0/Tests/TestSetup.ts index 508720f75aa9..b9ec77116d0d 100644 --- a/Tasks/KubernetesManifestV0/Tests/TestSetup.ts +++ b/Tasks/KubernetesManifestV0/Tests/TestSetup.ts @@ -247,6 +247,11 @@ a.exec[`${kubectlPath} scale ${process.env[shared.TestEnvVars.kind]}/${process.e stdout: 'created secret' } +a.exec[`${kubectlPath} get service/nginx-service -o json --namespace testnamespace`] = { + 'code': 0, + 'stdout': '{\r\n "apiVersion": "v1",\r\n "kind": "Service",\r\n "metadata": {\r\n "annotations": {\r\n "azure-pipelines/jobName": "Agent phase",\r\n "azure-pipelines/org": "https://codedev.ms/anchauh/",\r\n "azure-pipelines/pipeline": "aksCd-153 - 64 - CD",\r\n "azure-pipelines/pipelineId": "40",\r\n "azure-pipelines/project": "nginx",\r\n "azure-pipelines/run": "41",\r\n "azure-pipelines/runuri": "https://codedev.ms/anchauh/nginx/_releaseProgress?releaseId=41",\r\n "kubectl.kubernetes.io/last-applied-configuration": "{\\"apiVersion\\":\\"v1\\",\\"kind\\":\\"Service\\",\\"metadata\\":{\\"annotations\\":{},\\"labels\\":{\\"app\\":\\"nginx\\"},\\"name\\":\\"nginx-service\\",\\"namespace\\":\\"testnamespace\\"},\\"spec\\":{\\"ports\\":[{\\"name\\":\\"http\\",\\"port\\":80,\\"protocol\\":\\"TCP\\",\\"targetPort\\":\\"http\\"}],\\"selector\\":{\\"app\\":\\"nginx\\"},\\"type\\":\\"LoadBalancer\\"}}\\n"\r\n },\r\n "creationTimestamp": "2019-09-11T10:09:09Z",\r\n "labels": {\r\n "app": "nginx"\r\n },\r\n "name": "nginx-service",\r\n "namespace": "testnamespace",\r\n "resourceVersion": "8754335",\r\n "selfLink": "/api/v1/namespaces/testnamespace/services/nginx-service",\r\n "uid": "31f02713-d47c-11e9-9448-16b93c17a2b4"\r\n },\r\n "spec": {\r\n "clusterIP": "10.0.157.189",\r\n "externalTrafficPolicy": "Cluster",\r\n "ports": [\r\n {\r\n "name": "http",\r\n "nodePort": 32112,\r\n "port": 80,\r\n "protocol": "TCP",\r\n "targetPort": "http"\r\n }\r\n ],\r\n "selector": {\r\n "app": "nginx"\r\n },\r\n "sessionAffinity": "***",\r\n "type": "LoadBalancer"\r\n },\r\n "status": {\r\n "loadBalancer": {\r\n "ingress": [\r\n {\r\n "ip": "104.211.243.77"\r\n }\r\n ]\r\n }\r\n }\r\n }' +} + const pipelineAnnotations: string = [ `azure-pipelines/run=${buildNumber}`, `azure-pipelines/pipeline="${definitionName}"`, @@ -282,6 +287,7 @@ if (process.env[shared.TestEnvVars.arguments]) { }; } + tr.setAnswers(a); tr.registerMock('azure-pipelines-task-lib/toolrunner', require('azure-pipelines-task-lib/mock-toolrunner')); @@ -324,6 +330,7 @@ tr.registerMock('../utils/FileHelper', { }); if (newFilePaths.length === 0) { + console.log(shared.ManifestFilesPath); newFilePaths.push(shared.ManifestFilesPath); } return newFilePaths; diff --git a/Tasks/KubernetesManifestV0/Tests/manifests/deployment.yaml b/Tasks/KubernetesManifestV0/Tests/manifests/deployment.yaml index 41df31909c65..4a24b4e9f65e 100644 --- a/Tasks/KubernetesManifestV0/Tests/manifests/deployment.yaml +++ b/Tasks/KubernetesManifestV0/Tests/manifests/deployment.yaml @@ -20,4 +20,22 @@ spec: ports: - containerPort: 80 imagePullSecrets: - - name: key1 \ No newline at end of file + - name: key1 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: nginx-service + labels: + app: nginx +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: nginx \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/src/models/constants.ts b/Tasks/KubernetesManifestV0/src/models/constants.ts index edb0fa071f2f..6b25c1816c38 100644 --- a/Tasks/KubernetesManifestV0/src/models/constants.ts +++ b/Tasks/KubernetesManifestV0/src/models/constants.ts @@ -13,6 +13,17 @@ export class KubernetesWorkload { public static cronjob: string = 'cronjob'; } +export class DiscoveryAndLoadBalancerResource { + public static service: string = 'service'; + public static ingress: string = 'ingress'; +} + +export class ServiceTypes { + public static loadBalancer: string = 'LoadBalancer'; + public static nodePort: string = 'NodePort'; + public static clusterIP: string = 'ClusterIP' +} + export const deploymentTypes: string[] = ['deployment', 'replicaset', 'daemonset', 'pod', 'statefulset']; export const workloadTypes: string[] = ['deployment', 'replicaset', 'daemonset', 'pod', 'statefulset', 'job', 'cronjob']; export const workloadTypesWithRolloutStatus: string[] = ['deployment', 'daemonset', 'statefulset']; diff --git a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts index 177ce52ae477..8236072e57bc 100644 --- a/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts +++ b/Tasks/KubernetesManifestV0/src/utils/DeploymentHelper.ts @@ -34,9 +34,15 @@ export async function deploy(kubectl: Kubectl, manifestFilePaths: string[], depl const deployedManifestFiles = deployManifests(inputManifestFiles, kubectl, isCanaryDeploymentStrategy(deploymentStrategy)); // check manifest stability - const resourceTypes: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, models.deploymentTypes); + const resourceTypes: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, models.deploymentTypes.concat([constants.DiscoveryAndLoadBalancerResource.service])); await checkManifestStability(kubectl, resourceTypes); + // print ingress resources + const ingressResources: Resource[] = KubernetesObjectUtility.getResources(deployedManifestFiles, [constants.DiscoveryAndLoadBalancerResource.ingress]); + ingressResources.forEach(ingressResource => { + kubectl.getResource(constants.DiscoveryAndLoadBalancerResource.ingress, ingressResource.name); + }); + // annotate resources const allPods = JSON.parse((kubectl.getAllPods()).stdout); annotateResources(deployedManifestFiles, kubectl, resourceTypes, allPods); @@ -92,6 +98,21 @@ async function checkManifestStability(kubectl: Kubectl, resources: Resource[]): tl.warning(tl.loc('CouldNotDeterminePodStatus', JSON.stringify(ex))); } } + if (isEqual(resource.type, constants.DiscoveryAndLoadBalancerResource.service, StringComparer.OrdinalIgnoreCase)) { + try { + const service = getService(kubectl, resource.name); + const spec = service.spec; + const status = service.status; + if (isEqual(spec.type, constants.ServiceTypes.loadBalancer, StringComparer.OrdinalIgnoreCase)) { + if(!isLoadBalancerIPAssigned(status)) { + await waitForServiceExternalIPAssignment(kubectl, resource.name); + } + console.log(tl.loc('ServiceExternalIP', resource.name, status.loadBalancer.ingress[0].ip)); + } + } catch (ex) { + tl.warning(tl.loc('CouldNotDetermineServiceStatus', resource.name, JSON.stringify(ex))); + } + } } utils.checkForErrors(rolloutStatusResults); } @@ -263,6 +284,34 @@ function isPodReady(podStatus: any): boolean { return allContainersAreReady; } +function getService(kubectl: Kubectl, serviceName) { + const serviceResult = kubectl.getResource(constants.DiscoveryAndLoadBalancerResource.service, serviceName); + utils.checkForErrors([serviceResult]); + return JSON.parse(serviceResult.stdout); +} + +async function waitForServiceExternalIPAssignment(kubectl: Kubectl, serviceName: string): Promise { + const sleepTimeout = 10 * 1000; // 10 seconds + const iterations = 18; // 18 * 10 seconds timeout = 3 minutes max timeout + + for (let i = 0; i < iterations; i++) { + console.log(tl.loc('waitForServiceIpAssignment', serviceName)); + await sleep(sleepTimeout); + let status = getService(kubectl, serviceName).status; + if (isLoadBalancerIPAssigned(status)) { + return; + } + } + tl.warning(tl.loc('waitForServiceIpAssignmentTimedOut', serviceName)); +} + +function isLoadBalancerIPAssigned(status: any) { + if (status && status.loadBalancer && status.loadBalancer.ingress && status.loadBalancer.ingress.length > 0) { + return true; + } + return false; +} + function sleep(timeout: number) { return new Promise(resolve => setTimeout(resolve, timeout)); } \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/task.json b/Tasks/KubernetesManifestV0/task.json index c686c66a038d..efc9d8098b6a 100644 --- a/Tasks/KubernetesManifestV0/task.json +++ b/Tasks/KubernetesManifestV0/task.json @@ -333,6 +333,10 @@ "InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary", "AllContainersNotInReadyState": "All the containers are not in a ready state.", "CouldNotDeterminePodStatus": "Could not determine the pod's status due to the error: %s", - "KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features." + "KubectlShouldBeUpgraded": "kubectl client version equal to v1.14 or higher is required to use kustomize features.", + "CouldNotDetermineServiceStatus": "Could not determine the service %s status due to the error: %s", + "waitForServiceIpAssignment": "Waiting for service %s external IP assignment", + "waitForServiceIpAssignmentTimedOut": "Wait for service %s external IP assignment timed out", + "ServiceExternalIP": "service %s external IP is %s" } } \ No newline at end of file diff --git a/Tasks/KubernetesManifestV0/task.loc.json b/Tasks/KubernetesManifestV0/task.loc.json index ff27f5467ec3..82bdb9de9e84 100644 --- a/Tasks/KubernetesManifestV0/task.loc.json +++ b/Tasks/KubernetesManifestV0/task.loc.json @@ -333,6 +333,10 @@ "InvalidPromotetActionDeploymentStrategy": "ms-resource:loc.messages.InvalidPromotetActionDeploymentStrategy", "AllContainersNotInReadyState": "ms-resource:loc.messages.AllContainersNotInReadyState", "CouldNotDeterminePodStatus": "ms-resource:loc.messages.CouldNotDeterminePodStatus", - "KubectlShouldBeUpgraded": "ms-resource:loc.messages.KubectlShouldBeUpgraded" + "KubectlShouldBeUpgraded": "ms-resource:loc.messages.KubectlShouldBeUpgraded", + "CouldNotDetermineServiceStatus": "ms-resource:loc.messages.CouldNotDetermineServiceStatus", + "waitForServiceIpAssignment": "ms-resource:loc.messages.waitForServiceIpAssignment", + "waitForServiceIpAssignmentTimedOut": "ms-resource:loc.messages.waitForServiceIpAssignmentTimedOut", + "ServiceExternalIP": "ms-resource:loc.messages.ServiceExternalIP" } } \ No newline at end of file