Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added SMI canary deployment strategy to KubernetesManifest task #11218

Merged
merged 8 commits into from
Oct 22, 2019
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@
"loc.input.help.namespace": "Sets the namespace for the commands by using the –namespace flag. If the namespace is not provided, the commands will run in the default namespace.",
"loc.input.label.strategy": "Strategy",
"loc.input.help.strategy": "Deployment strategy to be used",
"loc.input.label.trafficSplitMethod": "Traffic split method",
"loc.input.help.trafficSplitMethod": "Traffic split method to be used",
"loc.input.label.percentage": "Percentage",
"loc.input.help.percentage": "Percentage of traffic redirect to canary deployment",
"loc.input.label.baselineAndCanaryReplicas": "Baseline and canary replicas",
"loc.input.help.baselineAndCanaryReplicas": "Baseline and canary replicas count",
"loc.input.label.manifests": "Manifests",
"loc.input.help.manifests": "Manifests to deploy",
"loc.input.label.containers": "Containers",
Expand Down Expand Up @@ -69,7 +74,6 @@
"loc.messages.NullInputObject": "Input object is null.",
"loc.messages.ArgumentsInputNotSupplied": "Arguments are not supplied.",
"loc.messages.NullInputObjectMetadata": "Input object metadata is null.",
"loc.messages.CanaryDeploymentAlreadyExistErrorMessage": "Canary deployment already exists. Rejecting this deployment.",
"loc.messages.InvalidRejectActionDeploymentStrategy": "Reject action works only with strategy: canary",
"loc.messages.InvalidPromotetActionDeploymentStrategy": "Promote action works only with strategy: canary",
"loc.messages.AllContainersNotInReadyState": "All the containers are not in a ready state.",
Expand All @@ -78,5 +82,7 @@
"loc.messages.CouldNotDetermineServiceStatus": "Could not determine the service %s status due to the error: %s",
"loc.messages.waitForServiceIpAssignment": "Waiting for service %s external IP assignment",
"loc.messages.waitForServiceIpAssignmentTimedOut": "Wait for service %s external IP assignment timed out",
"loc.messages.ServiceExternalIP": "service %s external IP is %s"
"loc.messages.ServiceExternalIP": "service %s external IP is %s",
"loc.messages.UnableToCreateTrafficSplitManifestFile": "Unable to create TrafficSplit manifest file.",
"loc.messages.StableSpecSelectorNotExist": "Resource %s not deployed using SMI canary deployment."
}
6 changes: 4 additions & 2 deletions Tasks/KubernetesManifestV0/Tests/L0.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ describe('Kubernetes Manifests Suite', function () {
delete process.env[shared.TestEnvVars.namespace];
delete process.env[shared.TestEnvVars.dockerComposeFile];
delete process.env[shared.TestEnvVars.releaseName];
delete process.env[shared.TestEnvVars.baselineAndCanaryReplicas];
delete process.env[shared.TestEnvVars.trafficSplitMethod];
delete process.env.RemoveNamespaceFromEndpoint;
});

Expand All @@ -54,13 +56,13 @@ describe('Kubernetes Manifests Suite', function () {
const tr: ttm.MockTestRunner = new ttm.MockTestRunner(tp);
process.env[shared.TestEnvVars.action] = shared.Actions.deploy;
process.env[shared.TestEnvVars.strategy] = shared.Strategy.canary;
process.env[shared.TestEnvVars.trafficSplitMethod] = shared.TrafficSplitMethod.pod;
process.env[shared.TestEnvVars.percentage] = '30';
process.env[shared.TestEnvVars.isStableDeploymentPresent] = 'true';
process.env[shared.TestEnvVars.isCanaryDeploymentPresent] = 'false';
process.env[shared.TestEnvVars.isBaselineDeploymentPresent] = 'false';
tr.run();
assert(tr.succeeded, 'task should have succeeded');
assert(tr.stderr.indexOf('"nginx-deployment-canary" not found') != -1, 'Canary deployment is not present');
assert(tr.stdout.indexOf('nginx-deployment-canary created') != -1, 'Canary deployment is created');
assert(tr.stdout.indexOf('nginx-deployment-baseline created') != -1, 'Baseline deployment is created');
assert(tr.stdout.indexOf('deployment "nginx-deployment-canary" successfully rolled out') != -1, 'Canary deployment is successfully rolled out');
Expand All @@ -80,7 +82,7 @@ describe('Kubernetes Manifests Suite', function () {
process.env[shared.TestEnvVars.isCanaryDeploymentPresent] = 'true';
process.env[shared.TestEnvVars.isBaselineDeploymentPresent] = 'true';
tr.run();
assert(tr.failed, 'task should have failed');
assert(tr.succeeded, 'task should have succeeded');
done();
});

Expand Down
5 changes: 4 additions & 1 deletion Tasks/KubernetesManifestV0/Tests/TestSetup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ tr.setInput('secretName', process.env[shared.TestEnvVars.secretName] || '');
tr.setInput('secretType', process.env[shared.TestEnvVars.secretType] || '');
tr.setInput('dockerComposeFile', process.env[shared.TestEnvVars.dockerComposeFile] || '');
tr.setInput('kustomizationPath', process.env[shared.TestEnvVars.kustomizationPath] || '');
tr.setInput('baselineAndCanaryReplicas', process.env[shared.TestEnvVars.baselineAndCanaryReplicas] || '0');
tr.setInput('trafficSplitMethod', process.env[shared.TestEnvVars.trafficSplitMethod]);

process.env.SYSTEM_DEFAULTWORKINGDIRECTORY = testnamespaceWorkingDirectory;
process.env.SYSTEM_TEAMFOUNDATIONCOLLECTIONURI = teamFoundationCollectionUri;
Expand Down Expand Up @@ -340,7 +342,8 @@ tr.registerMock('../utils/FileHelper', {
},
getNewUserDirPath: fh.getNewUserDirPath,
ensureDirExists: fh.ensureDirExists,
assertFileExists: fh.assertFileExists
assertFileExists: fh.assertFileExists,
writeManifestToFile: fh.writeManifestToFile
});

tr.registerMock('uuid/v4', function () {
Expand Down
9 changes: 8 additions & 1 deletion Tasks/KubernetesManifestV0/Tests/TestShared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ export let TestEnvVars = {
endpointAuthorizationType: "__endpointAuthorizationType__",
isStableDeploymentPresent: "__isStableDeploymentPresent__",
isCanaryDeploymentPresent: "__isCanaryDeploymentPresent__",
isBaselineDeploymentPresent: "__isBaselineDeploymentPresent__"
isBaselineDeploymentPresent: "__isBaselineDeploymentPresent__",
baselineAndCanaryReplicas: "__baselineAndCanaryReplicas__",
trafficSplitMethod: "__trafficSplitMethod__"
};

export let OperatingSystems = {
Expand Down Expand Up @@ -64,6 +66,11 @@ export let Strategy = {
none: "none"
};

export let TrafficSplitMethod = {
pod: "pod",
smi: "smi"
};

export const ManifestFilesPath = path.join(__dirname, 'manifests', 'deployment.yaml');
export const CanaryManifestFilesPath = path.join(__dirname, 'manifests', 'deployment-canary.yaml');
export const BaselineManifestFilesPath = path.join(__dirname, 'manifests', 'deployment-baseline.yaml');
Expand Down
38 changes: 27 additions & 11 deletions Tasks/KubernetesManifestV0/src/actions/promote.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,42 @@ import * as tl from 'azure-pipelines-task-lib/task';

import * as deploymentHelper from '../utils/DeploymentHelper';
import * as canaryDeploymentHelper from '../utils/CanaryDeploymentHelper';
import * as SMICanaryDeploymentHelper from '../utils/SMICanaryDeploymentHelper';
import * as utils from '../utils/utilities';
import * as TaskInputParameters from '../models/TaskInputParameters';

import { Kubectl } from 'kubernetes-common-v2/kubectl-object-model';

export async function promote(ignoreSslErrors?: boolean) {

const kubectl = new Kubectl(await utils.getKubectl(), TaskInputParameters.namespace, ignoreSslErrors);

if (canaryDeploymentHelper.isCanaryDeploymentStrategy()) {
tl.debug('Deploying input manifests');
await deploymentHelper.deploy(kubectl, TaskInputParameters.manifests, 'None');
tl.debug('Deployment strategy selected is Canary. Deleting canary and baseline workloads.');
try {
canaryDeploymentHelper.deleteCanaryDeployment(kubectl, TaskInputParameters.manifests);
} catch (ex) {
tl.warning('Exception occurred while deleting canary and baseline workloads. Exception: ' + ex);
}
} else {
if (!canaryDeploymentHelper.isCanaryDeploymentStrategy()) {
tl.debug('Strategy is not canary deployment. Invalid request.');
throw (tl.loc('InvalidPromotetActionDeploymentStrategy'));
}

let includeServices = false;
if (canaryDeploymentHelper.isSMICanaryStrategy()) {
includeServices = true;
// In case of SMI traffic split strategy when deployment is promoted, first we will redirect traffic to
// Canary deployment, then update stable deployment and then redirect traffic to stable deployment
tl.debug('Redirecting traffic to canary deployment');
SMICanaryDeploymentHelper.redirectTrafficToCanaryDeployment(kubectl, TaskInputParameters.manifests);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just want to confirm, Redirecting entire traffic to canary during promote, how are we making sure Canary pods can handle that much traffic?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In SMI, first you will redirect traffic to other deploymnt(canary) before updating stable. Generally customer will increase the canary pods before updating stable one.


tl.debug('Deploying input manifests with SMI canary strategy');
await deploymentHelper.deploy(kubectl, TaskInputParameters.manifests, 'None');

tl.debug('Redirecting traffic to stable deployment');
SMICanaryDeploymentHelper.redirectTrafficToStableDeployment(kubectl, TaskInputParameters.manifests);
} else {
tl.debug('Deploying input manifests');
await deploymentHelper.deploy(kubectl, TaskInputParameters.manifests, 'None');
}

tl.debug('Deployment strategy selected is Canary. Deleting canary and baseline workloads.');
try {
canaryDeploymentHelper.deleteCanaryDeployment(kubectl, TaskInputParameters.manifests, includeServices);
} catch (ex) {
tl.warning('Exception occurred while deleting canary and baseline workloads. Exception: ' + ex);
}
}
16 changes: 12 additions & 4 deletions Tasks/KubernetesManifestV0/src/actions/reject.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,26 @@
'use strict';
import * as tl from 'azure-pipelines-task-lib/task';
import * as canaryDeploymentHelper from '../utils/CanaryDeploymentHelper';
import * as SMICanaryDeploymentHelper from '../utils/SMICanaryDeploymentHelper';
import { Kubectl } from 'kubernetes-common-v2/kubectl-object-model';
import * as utils from '../utils/utilities';
import * as TaskInputParameters from '../models/TaskInputParameters';

export async function reject(ignoreSslErrors?: boolean) {
const kubectl = new Kubectl(await utils.getKubectl(), TaskInputParameters.namespace, ignoreSslErrors);

if (canaryDeploymentHelper.isCanaryDeploymentStrategy()) {
tl.debug('Deployment strategy selected is Canary. Deleting baseline and canary workloads.');
canaryDeploymentHelper.deleteCanaryDeployment(kubectl, TaskInputParameters.manifests);
} else {
if (!canaryDeploymentHelper.isCanaryDeploymentStrategy()) {
tl.debug('Strategy is not canary deployment. Invalid request.');
throw (tl.loc('InvalidRejectActionDeploymentStrategy'));
}

let includeServices = false;
if (canaryDeploymentHelper.isSMICanaryStrategy()) {
tl.debug('Reject deployment with SMI canary strategy');
includeServices = true;
SMICanaryDeploymentHelper.redirectTrafficToStableDeployment(kubectl, TaskInputParameters.manifests);
}

tl.debug('Deployment strategy selected is Canary. Deleting baseline and canary workloads.');
canaryDeploymentHelper.deleteCanaryDeployment(kubectl, TaskInputParameters.manifests, includeServices);
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ export const imagePullSecrets: string[] = tl.getDelimitedInput('imagePullSecrets
export const manifests = tl.getDelimitedInput('manifests', '\n');
export const canaryPercentage: string = tl.getInput('percentage');
export const deploymentStrategy: string = tl.getInput('strategy', false);
export const trafficSplitMethod: string = tl.getInput('trafficSplitMethod', false);
export const baselineAndCanaryReplicas: string = tl.getInput('baselineAndCanaryReplicas', true);
export const args: string = tl.getInput('arguments', false);
export const secretArguments: string = tl.getInput('secretArguments', false) || '';
export const secretType: string = tl.getInput('secretType', false);
Expand Down
Loading