-
Notifications
You must be signed in to change notification settings - Fork 4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(aws-ec2): AutoScalingGroup rolling updates #595
Changes from all commits
2864b98
a6851f5
f5e724d
81e2012
9011802
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -65,6 +65,63 @@ export interface AutoScalingGroupProps { | |
* @default true | ||
*/ | ||
allowAllOutbound?: boolean; | ||
|
||
/** | ||
* What to do when an AutoScalingGroup's instance configuration is changed | ||
* | ||
* This is applied when any of the settings on the ASG are changed that | ||
* affect how the instances should be created (VPC, instance type, startup | ||
* scripts, etc.). It indicates how the existing instances should be | ||
* replaced with new instances matching the new config. By default, nothing | ||
* is done and only new instances are launched with the new config. | ||
* | ||
* @default UpdateType.None | ||
*/ | ||
updateType?: UpdateType; | ||
|
||
/** | ||
* Configuration for rolling updates | ||
* | ||
* Only used if updateType == UpdateType.RollingUpdate. | ||
*/ | ||
rollingUpdateConfiguration?: RollingUpdateConfiguration; | ||
|
||
/** | ||
* Configuration for replacing updates. | ||
* | ||
* Only used if updateType == UpdateType.ReplacingUpdate. Specifies how | ||
* many instances must signal success for the update to succeed. | ||
*/ | ||
replacingUpdateMinSuccessfulInstancesPercent?: number; | ||
|
||
/** | ||
* If the ASG has scheduled actions, don't reset unchanged group sizes | ||
* | ||
* Only used if the ASG has scheduled actions (which may scale your ASG up | ||
* or down regardless of cdk deployments). If true, the size of the group | ||
* will only be reset if it has been changed in the CDK app. If false, the | ||
* sizes will always be changed back to what they were in the CDK app | ||
* on deployment. | ||
* | ||
* @default true | ||
*/ | ||
ignoreUnmodifiedSizeProperties?: boolean; | ||
|
||
/** | ||
* How many ResourceSignal calls CloudFormation expects before the resource is considered created | ||
* | ||
* @default 1 | ||
*/ | ||
resourceSignalCount?: number; | ||
|
||
/** | ||
* The length of time to wait for the resourceSignalCount | ||
* | ||
* The maximum value is 43200 (12 hours). | ||
* | ||
* @default 300 (5 minutes) | ||
*/ | ||
resourceSignalTimeoutSec?: number; | ||
} | ||
|
||
/** | ||
|
@@ -136,6 +193,10 @@ export class AutoScalingGroup extends cdk.Construct implements ec2.IClassicLoadB | |
const maxSize = props.maxSize || 1; | ||
const desiredCapacity = props.desiredCapacity || 1; | ||
|
||
if (desiredCapacity < minSize || desiredCapacity > maxSize) { | ||
throw new Error(`Should have minSize (${minSize}) <= desiredCapacity (${desiredCapacity}) <= maxSize (${maxSize})`); | ||
} | ||
|
||
const asgProps: cloudformation.AutoScalingGroupResourceProps = { | ||
minSize: minSize.toString(), | ||
maxSize: maxSize.toString(), | ||
|
@@ -162,6 +223,8 @@ export class AutoScalingGroup extends cdk.Construct implements ec2.IClassicLoadB | |
|
||
this.autoScalingGroup = new cloudformation.AutoScalingGroupResource(this, 'ASG', asgProps); | ||
this.osType = machineImage.os.type; | ||
|
||
this.applyUpdatePolicies(props); | ||
} | ||
|
||
public attachToClassicLB(loadBalancer: ec2.ClassicLoadBalancer): void { | ||
|
@@ -186,4 +249,213 @@ export class AutoScalingGroup extends cdk.Construct implements ec2.IClassicLoadB | |
public addToRolePolicy(statement: cdk.PolicyStatement) { | ||
this.role.addToPolicy(statement); | ||
} | ||
|
||
/** | ||
* Apply CloudFormation update policies for the AutoScalingGroup | ||
*/ | ||
private applyUpdatePolicies(props: AutoScalingGroupProps) { | ||
if (props.updateType === UpdateType.ReplacingUpdate) { | ||
this.asgUpdatePolicy.autoScalingReplacingUpdate = { willReplace: true }; | ||
|
||
if (props.replacingUpdateMinSuccessfulInstancesPercent !== undefined) { | ||
// Yes, this goes on CreationPolicy, not as a process parameter to ReplacingUpdate. | ||
// It's a little confusing, but the docs seem to explicitly state it will only be used | ||
// during the update? | ||
// | ||
// https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-attribute-creationpolicy.html | ||
this.asgCreationPolicy.autoScalingCreationPolicy = { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. there is another creation policy that we use to prevent false positives when
Should we support this variation in this PR, or another PR? I am actually running into this issue trying to port our bastion pattern that requires some There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep, just added that to this PR as well: |
||
minSuccessfulInstancesPercent: validatePercentage(props.replacingUpdateMinSuccessfulInstancesPercent) | ||
}; | ||
} | ||
} else if (props.updateType === UpdateType.RollingUpdate) { | ||
this.asgUpdatePolicy.autoScalingRollingUpdate = renderRollingUpdateConfig(props.rollingUpdateConfiguration); | ||
} | ||
|
||
// undefined is treated as 'true' | ||
if (props.ignoreUnmodifiedSizeProperties !== false) { | ||
this.asgUpdatePolicy.autoScalingScheduledAction = { ignoreUnmodifiedGroupSizeProperties: true }; | ||
} | ||
|
||
if (props.resourceSignalCount !== undefined || props.resourceSignalTimeoutSec !== undefined) { | ||
this.asgCreationPolicy.resourceSignal = { | ||
count: props.resourceSignalCount, | ||
timeout: props.resourceSignalTimeoutSec !== undefined ? renderIsoDuration(props.resourceSignalTimeoutSec) : undefined, | ||
}; | ||
} | ||
} | ||
|
||
/** | ||
* Create and return the ASG update policy | ||
*/ | ||
private get asgUpdatePolicy() { | ||
if (this.autoScalingGroup.options.updatePolicy === undefined) { | ||
this.autoScalingGroup.options.updatePolicy = {}; | ||
} | ||
return this.autoScalingGroup.options.updatePolicy; | ||
} | ||
|
||
/** | ||
* Create and return the ASG creation policy | ||
*/ | ||
private get asgCreationPolicy() { | ||
if (this.autoScalingGroup.options.creationPolicy === undefined) { | ||
this.autoScalingGroup.options.creationPolicy = {}; | ||
} | ||
return this.autoScalingGroup.options.creationPolicy; | ||
} | ||
} | ||
|
||
/** | ||
* The type of update to perform on instances in this AutoScalingGroup | ||
*/ | ||
export enum UpdateType { | ||
/** | ||
* Don't do anything | ||
*/ | ||
None = 'None', | ||
|
||
/** | ||
* Replace the entire AutoScalingGroup | ||
* | ||
* Builds a new AutoScalingGroup first, then delete the old one. | ||
*/ | ||
ReplacingUpdate = 'Replace', | ||
|
||
/** | ||
* Replace the instances in the AutoScalingGroup. | ||
*/ | ||
RollingUpdate = 'RollingUpdate', | ||
} | ||
|
||
/** | ||
* Additional settings when a rolling update is selected | ||
*/ | ||
export interface RollingUpdateConfiguration { | ||
/** | ||
* The maximum number of instances that AWS CloudFormation updates at once. | ||
* | ||
* @default 1 | ||
*/ | ||
maxBatchSize?: number; | ||
|
||
/** | ||
* The minimum number of instances that must be in service before more instances are replaced. | ||
* | ||
* This number affects the speed of the replacement. | ||
* | ||
* @default 0 | ||
*/ | ||
minInstancesInService?: number; | ||
|
||
/** | ||
* The percentage of instances that must signal success for an update to succeed. | ||
* | ||
* If an instance doesn't send a signal within the time specified in the | ||
* pauseTime property, AWS CloudFormation assumes that the instance wasn't | ||
* updated. | ||
* | ||
* This number affects the success of the replacement. | ||
* | ||
* If you specify this property, you must also enable the | ||
* waitOnResourceSignals and pauseTime properties. | ||
* | ||
* @default 100 | ||
*/ | ||
minSuccessfulInstancesPercent?: number; | ||
|
||
/** | ||
* The pause time after making a change to a batch of instances. | ||
* | ||
* This is intended to give those instances time to start software applications. | ||
* | ||
* Specify PauseTime in the ISO8601 duration format (in the format | ||
* PT#H#M#S, where each # is the number of hours, minutes, and seconds, | ||
* respectively). The maximum PauseTime is one hour (PT1H). | ||
* | ||
* @default 300 if the waitOnResourceSignals property is true, otherwise 0 | ||
*/ | ||
pauseTimeSec?: number; | ||
|
||
/** | ||
* Specifies whether the Auto Scaling group waits on signals from new instances during an update. | ||
* | ||
* AWS CloudFormation must receive a signal from each new instance within | ||
* the specified PauseTime before continuing the update. | ||
* | ||
* To have instances wait for an Elastic Load Balancing health check before | ||
* they signal success, add a health-check verification by using the | ||
* cfn-init helper script. For an example, see the verify_instance_health | ||
* command in the Auto Scaling rolling updates sample template. | ||
* | ||
* @default true if you specified the minSuccessfulInstancesPercent property, false otherwise | ||
*/ | ||
waitOnResourceSignals?: boolean; | ||
|
||
/** | ||
* Specifies the Auto Scaling processes to suspend during a stack update. | ||
* | ||
* Suspending processes prevents Auto Scaling from interfering with a stack | ||
* update. | ||
* | ||
* @default HealthCheck, ReplaceUnhealthy, AZRebalance, AlarmNotification, ScheduledActions. | ||
*/ | ||
suspendProcesses?: ScalingProcess[]; | ||
} | ||
|
||
export enum ScalingProcess { | ||
Launch = 'Launch', | ||
Terminate = 'Terminate', | ||
HealthCheck = 'HealthCheck', | ||
ReplaceUnhealthy = 'ReplaceUnhealthy', | ||
AZRebalance = 'AZRebalance', | ||
AlarmNotification = 'AlarmNotification', | ||
ScheduledActions = 'ScheduledActions', | ||
AddToLoadBalancer = 'AddToLoadBalancer' | ||
} | ||
|
||
/** | ||
* Render the rolling update configuration into the appropriate object | ||
*/ | ||
function renderRollingUpdateConfig(config: RollingUpdateConfiguration = {}): cdk.AutoScalingRollingUpdate { | ||
const waitOnResourceSignals = config.minSuccessfulInstancesPercent !== undefined ? true : false; | ||
const pauseTimeSec = config.pauseTimeSec !== undefined ? config.pauseTimeSec : (waitOnResourceSignals ? 300 : 0); | ||
|
||
return { | ||
maxBatchSize: config.maxBatchSize, | ||
minInstancesInService: config.minInstancesInService, | ||
minSuccessfulInstancesPercent: validatePercentage(config.minSuccessfulInstancesPercent), | ||
waitOnResourceSignals, | ||
pauseTime: renderIsoDuration(pauseTimeSec), | ||
suspendProcesses: config.suspendProcesses !== undefined ? config.suspendProcesses : | ||
// Recommended list of processes to suspend from here: | ||
// https://aws.amazon.com/premiumsupport/knowledge-center/auto-scaling-group-rolling-updates/ | ||
[ScalingProcess.HealthCheck, ScalingProcess.ReplaceUnhealthy, ScalingProcess.AZRebalance, | ||
ScalingProcess.AlarmNotification, ScalingProcess.ScheduledActions], | ||
}; | ||
} | ||
|
||
/** | ||
* Render a number of seconds to a PTnX string. | ||
*/ | ||
function renderIsoDuration(seconds: number): string { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Did not test, but I fully expect the validator on the other end to be strict about it. |
||
const ret: string[] = []; | ||
|
||
if (seconds >= 3600) { | ||
ret.push(`${Math.floor(seconds / 3600)}H`); | ||
seconds %= 3600; | ||
} | ||
if (seconds >= 60) { | ||
ret.push(`${Math.floor(seconds / 60)}M`); | ||
seconds %= 60; | ||
} | ||
if (seconds > 0) { | ||
ret.push(`${seconds}S`); | ||
} | ||
|
||
return 'PT' + ret.join(''); | ||
} | ||
|
||
function validatePercentage(x?: number): number | undefined { | ||
if (x === undefined || (0 <= x && x <= 100)) { return x; } | ||
throw new Error(`Expected: a percentage 0..100, got: ${x}`); | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Like!