feat(cli): automatically roll back stacks if necessary

packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js

@@ -1,15 +1,17 @@
 const cdk = require('aws-cdk-lib');
 const lambda = require('aws-cdk-lib/aws-lambda');
+const sqs = require('aws-cdk-lib/aws-sqs');
 const cr = require('aws-cdk-lib/custom-resources');
 
 /**
  * This stack will be deployed in multiple phases, to achieve a very specific effect
  *
- * It contains resources r1 and r2, where r1 gets deployed first.
+ * It contains resources r1 and r2, and a queue q, where r1 gets deployed first.
  *
  * - PHASE = 1: both resources deploy regularly.
  * - PHASE = 2a: r1 gets updated, r2 will fail to update
  * - PHASE = 2b: r1 gets updated, r2 will fail to update, and r1 will fail its rollback.
+ * - PHASE = 3: q gets replaced w.r.t. phases 1 and 2
  *
  * To exercise this app:
  *
@@ -22,7 +24,7 @@ const cr = require('aws-cdk-lib/custom-resources');
  * # This will start a rollback that will fail because r1 fails its rollabck
  *
  * env PHASE=2b npx cdk rollback --force
- * # This will retry the rollabck and skip r1
+ * # This will retry the rollback and skip r1
  * ```
  */
 class RollbacktestStack extends cdk.Stack {
@@ -31,6 +33,7 @@ class RollbacktestStack extends cdk.Stack {
 
     let r1props = {};
     let r2props = {};
+    let fifo = false;
 
     const phase = process.env.PHASE;
     switch (phase) {
@@ -46,6 +49,9 @@ class RollbacktestStack extends cdk.Stack {
         r1props.FailRollback = true;
         r2props.FailUpdate = true;
         break;
+      case '3':
+        fifo = true;
+        break;
     }
 
     const fn = new lambda.Function(this, 'Fun', {
@@ -76,6 +82,10 @@ class RollbacktestStack extends cdk.Stack {
       properties: r2props,
     });
     r2.node.addDependency(r1);
+
+    new sqs.Queue(this, 'Queue', {
+      fifo,
+    });
   }
 }
 

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts

@@ -2412,6 +2412,103 @@ integTest(
   }),
 );
 
+integTest(
+  'automatic rollback if paused and change contains a replacement',
+  withSpecificFixture('rollback-test-app', async (fixture) => {
+    let phase = '1';
+
+    // Should succeed
+    await fixture.cdkDeploy('test-rollback', {
+      options: ['--no-rollback'],
+      modEnv: { PHASE: phase },
+      verbose: false,
+    });
+    try {
+      phase = '2a';
+
+      // Should fail
+      const deployOutput = await fixture.cdkDeploy('test-rollback', {
+        options: ['--no-rollback'],
+        modEnv: { PHASE: phase },
+        verbose: false,
+        allowErrExit: true,
+      });
+      expect(deployOutput).toContain('UPDATE_FAILED');
+
+      // Do a deployment with a replacement and --force: this will roll back first and then deploy normally
+      phase = '3';
+      await fixture.cdkDeploy('test-rollback', {
+        options: ['--no-rollback', '--force'],
+        modEnv: { PHASE: phase },
+        verbose: false,
+      });
+    } finally {
+      await fixture.cdkDestroy('test-rollback');
+    }
+  }),
+);
+
+integTest(
+  'automatic rollback if paused and --no-rollback is removed from flags',
+  withSpecificFixture('rollback-test-app', async (fixture) => {
+    let phase = '1';
+
+    // Should succeed
+    await fixture.cdkDeploy('test-rollback', {
+      options: ['--no-rollback'],
+      modEnv: { PHASE: phase },
+      verbose: false,
+    });
+    try {
+      phase = '2a';
+
+      // Should fail
+      const deployOutput = await fixture.cdkDeploy('test-rollback', {
+        options: ['--no-rollback'],
+        modEnv: { PHASE: phase },
+        verbose: false,
+        allowErrExit: true,
+      });
+      expect(deployOutput).toContain('UPDATE_FAILED');
+
+      // Do a deployment removing --no-rollback: this will roll back first and then deploy normally
+      phase = '1';
+      await fixture.cdkDeploy('test-rollback', {
+        options: ['--force'],
+        modEnv: { PHASE: phase },
+        verbose: false,
+      });
+    } finally {
+      await fixture.cdkDestroy('test-rollback');
+    }
+  }),
+);
+
+integTest(
+  'automatic rollback if replacement and --no-rollback is removed from flags',
+  withSpecificFixture('rollback-test-app', async (fixture) => {
+    let phase = '1';
+
+    // Should succeed
+    await fixture.cdkDeploy('test-rollback', {
+      options: ['--no-rollback'],
+      modEnv: { PHASE: phase },
+      verbose: false,
+    });
+    try {
+      // Do a deployment with a replacement and removing --no-rollback: this will do a regular rollback deploy
+      phase = '3';
+      await fixture.cdkDeploy('test-rollback', {
+        options: ['--force'],
+        modEnv: { PHASE: phase },
+        verbose: false,
+      });
+    } finally {
+      await fixture.cdkDestroy('test-rollback');
+    }
+  }),
+);
+
 integTest(
   'test cdk rollback --force',
   withSpecificFixture('rollback-test-app', async (fixture) => {

packages/@aws-cdk/cloudformation-diff/lib/format.ts

@@ -160,7 +160,7 @@ export class Formatter {
     const resourceType = diff.isRemoval ? diff.oldResourceType : diff.newResourceType;
 
     // eslint-disable-next-line max-len
-    this.print(`${this.formatResourcePrefix(diff)} ${this.formatValue(resourceType, chalk.cyan)} ${this.formatLogicalId(logicalId)} ${this.formatImpact(diff.changeImpact)}`);
+    this.print(`${this.formatResourcePrefix(diff)} ${this.formatValue(resourceType, chalk.cyan)} ${this.formatLogicalId(logicalId)} ${this.formatImpact(diff.changeImpact)}`.trimEnd());
 
     if (diff.isUpdate) {
       const differenceCount = diff.differenceCount;

packages/aws-cdk/lib/api/bootstrap/bootstrap-environment.ts

@@ -8,7 +8,7 @@ import { warning } from '../../logging';
 import { loadStructuredFile, serializeStructure } from '../../serialize';
 import { rootDir } from '../../util/directories';
 import { ISDK, Mode, SdkProvider } from '../aws-auth';
-import { DeployStackResult } from '../deploy-stack';
+import { RegularDeployStackResult } from '../deploy-stack';
 
 /* eslint-disable max-len */
 
@@ -21,7 +21,7 @@ export class Bootstrapper {
   constructor(private readonly source: BootstrapSource) {
   }
 
-  public bootstrapEnvironment(environment: cxapi.Environment, sdkProvider: SdkProvider, options: BootstrapEnvironmentOptions = {}): Promise<DeployStackResult> {
+  public bootstrapEnvironment(environment: cxapi.Environment, sdkProvider: SdkProvider, options: BootstrapEnvironmentOptions = {}): Promise<RegularDeployStackResult> {
     switch (this.source.source) {
       case 'legacy':
         return this.legacyBootstrap(environment, sdkProvider, options);
@@ -41,7 +41,7 @@ export class Bootstrapper {
    * Deploy legacy bootstrap stack
    *
    */
-  private async legacyBootstrap(environment: cxapi.Environment, sdkProvider: SdkProvider, options: BootstrapEnvironmentOptions = {}): Promise<DeployStackResult> {
+  private async legacyBootstrap(environment: cxapi.Environment, sdkProvider: SdkProvider, options: BootstrapEnvironmentOptions = {}): Promise<RegularDeployStackResult> {
     const params = options.parameters ?? {};
 
     if (params.trustedAccounts?.length) {
@@ -71,7 +71,7 @@ export class Bootstrapper {
   private async modernBootstrap(
     environment: cxapi.Environment,
     sdkProvider: SdkProvider,
-    options: BootstrapEnvironmentOptions = {}): Promise<DeployStackResult> {
+    options: BootstrapEnvironmentOptions = {}): Promise<RegularDeployStackResult> {
 
     const params = options.parameters ?? {};
 
@@ -291,7 +291,7 @@ export class Bootstrapper {
   private async customBootstrap(
     environment: cxapi.Environment,
     sdkProvider: SdkProvider,
-    options: BootstrapEnvironmentOptions = {}): Promise<DeployStackResult> {
+    options: BootstrapEnvironmentOptions = {}): Promise<RegularDeployStackResult> {
 
     // Look at the template, decide whether it's most likely a legacy or modern bootstrap
     // template, and use the right bootstrapper for that.

packages/aws-cdk/lib/api/bootstrap/deploy-bootstrap.ts

@@ -6,7 +6,7 @@ import * as fs from 'fs-extra';
 import { BOOTSTRAP_VERSION_OUTPUT, BootstrapEnvironmentOptions, BOOTSTRAP_VERSION_RESOURCE, BOOTSTRAP_VARIANT_PARAMETER, DEFAULT_BOOTSTRAP_VARIANT } from './bootstrap-props';
 import * as logging from '../../logging';
 import { Mode, SdkProvider, ISDK } from '../aws-auth';
-import { deployStack, DeployStackResult } from '../deploy-stack';
+import { deployStack, RegularDeployStackResult } from '../deploy-stack';
 import { NoBootstrapStackEnvironmentResources } from '../environment-resources';
 import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info';
 
@@ -63,14 +63,15 @@ export class BootstrapStack {
     template: any,
     parameters: Record<string, string | undefined>,
     options: Omit<BootstrapEnvironmentOptions, 'parameters'>,
-  ): Promise<DeployStackResult> {
+  ): Promise<RegularDeployStackResult> {
     if (this.currentToolkitInfo.found && !options.force) {
       // Safety checks
       const abortResponse = {
+        type: 'did-deploy-stack',
         noOp: true,
         outputs: {},
         stackArn: this.currentToolkitInfo.bootstrapStack.stackId,
-      };
+      } satisfies RegularDeployStackResult;
 
       // Validate that the bootstrap stack we're trying to replace is from the same variant as the one we're trying to deploy
       const currentVariant = this.currentToolkitInfo.variant;
@@ -110,7 +111,7 @@ export class BootstrapStack {
 
     const assembly = builder.buildAssembly();
 
-    return deployStack({
+    const ret = await deployStack({
       stack: assembly.getStackByName(this.toolkitStackName),
       resolvedEnvironment: this.resolvedEnvironment,
       sdk: this.sdk,
@@ -124,6 +125,12 @@ export class BootstrapStack {
       // Obviously we can't need a bootstrap stack to deploy a bootstrap stack
       envResources: new NoBootstrapStackEnvironmentResources(this.resolvedEnvironment, this.sdk),
     });
+
+    if (ret.type !== 'did-deploy-stack') {
+      throw new Error(`Unexpected deployStack result. This should not happen: ${JSON.stringify(ret)}`);
+    }
+
+    return ret;
   }
 }
 

packages/aws-cdk/lib/api/deploy-stack.ts

@@ -20,7 +20,17 @@ import { AssetManifestBuilder } from '../util/asset-manifest-builder';
 import { determineAllowCrossAccountAssetPublishing } from './util/checks';
 import { publishAssets } from '../util/asset-publishing';
 
-export interface DeployStackResult {
+export type DeployStackResult =
+  // Regular result
+  | RegularDeployStackResult
+  // The stack is currently in a failpaused state, and needs to be rolled back before the deployment
+  | { type: 'failpaused-need-rollback-first'; reason: 'not-norollback' | 'replacement' }
+  // The upcoming change has a replacement, which requires deploying without --no-rollback.
+  | { type: 'replacement-requires-norollback' }
+  ;
+
+export interface RegularDeployStackResult {
+  readonly type: 'did-deploy-stack';
   readonly noOp: boolean;
   readonly outputs: { [name: string]: string };
   readonly stackArn: string;
@@ -279,6 +289,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
       print(`\n ${ICON} %s\n`, chalk.bold('hotswap deployment skipped - no changes were detected (use --force to override)'));
     }
     return {
+      type: 'did-deploy-stack',
       noOp: true,
       outputs: cloudFormationStack.outputs,
       stackArn: cloudFormationStack.stackId,
@@ -326,7 +337,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
       print('Falling back to doing a full deployment');
       options.sdk.appendCustomUserAgent('cdk-hotswap/fallback');
     } else {
-      return { noOp: true, stackArn: cloudFormationStack.stackId, outputs: cloudFormationStack.outputs };
+      return { type: 'did-deploy-stack', noOp: true, stackArn: cloudFormationStack.stackId, outputs: cloudFormationStack.outputs };
     }
   }
 
@@ -408,12 +419,26 @@ class FullCloudFormationDeployment {
         ].join('\n'));
       }
 
-      return { noOp: true, outputs: this.cloudFormationStack.outputs, stackArn: changeSetDescription.StackId! };
+      return { type: 'did-deploy-stack', noOp: true, outputs: this.cloudFormationStack.outputs, stackArn: changeSetDescription.StackId! };
     }
 
     if (!execute) {
       print('Changeset %s created and waiting in review for manual execution (--no-execute)', changeSetDescription.ChangeSetId);
-      return { noOp: false, outputs: this.cloudFormationStack.outputs, stackArn: changeSetDescription.StackId! };
+      return { type: 'did-deploy-stack', noOp: false, outputs: this.cloudFormationStack.outputs, stackArn: changeSetDescription.StackId! };
+    }
+
+    // If there are replacements in the changeset, check the rollback flag and stack status
+    const replacement = hasReplacement(changeSetDescription);
+    const isPausedFailState = this.cloudFormationStack.stackStatus.isRollbackable;
+    const rollback = this.options.rollback ?? true;
+    if (isPausedFailState && replacement) {
+      return { type: 'failpaused-need-rollback-first', reason: 'replacement' };
+    }
+    if (isPausedFailState && !rollback) {
+      return { type: 'failpaused-need-rollback-first', reason: 'not-norollback' };
+    }
+    if (!rollback && replacement) {
+      return { type: 'replacement-requires-norollback' };
     }
 
     return this.executeChangeSet(changeSetDescription);
@@ -439,7 +464,7 @@ class FullCloudFormationDeployment {
     return waitForChangeSet(this.cfn, this.stackName, changeSetName, { fetchAll: willExecute });
   }
 
-  private async executeChangeSet(changeSet: CloudFormation.DescribeChangeSetOutput): Promise<DeployStackResult> {
+  private async executeChangeSet(changeSet: CloudFormation.DescribeChangeSetOutput): Promise<RegularDeployStackResult> {
     debug('Initiating execution of changeset %s on stack %s', changeSet.ChangeSetId, this.stackName);
 
     await this.cfn.executeChangeSet({
@@ -478,7 +503,7 @@ class FullCloudFormationDeployment {
     }
   }
 
-  private async directDeployment(): Promise<DeployStackResult> {
+  private async directDeployment(): Promise<RegularDeployStackResult> {
     print('%s: %s stack...', chalk.bold(this.stackName), this.update ? 'updating' : 'creating');
 
     const startTime = new Date();
@@ -496,7 +521,7 @@ class FullCloudFormationDeployment {
       } catch (err: any) {
         if (err.message === 'No updates are to be performed.') {
           debug('No updates are to be performed for stack %s', this.stackName);
-          return { noOp: true, outputs: this.cloudFormationStack.outputs, stackArn: this.cloudFormationStack.stackId };
+          return { type: 'did-deploy-stack', noOp: true, outputs: this.cloudFormationStack.outputs, stackArn: this.cloudFormationStack.stackId };
         }
         throw err;
       }
@@ -518,7 +543,7 @@ class FullCloudFormationDeployment {
     }
   }
 
-  private async monitorDeployment(startTime: Date, expectedChanges: number | undefined): Promise<DeployStackResult> {
+  private async monitorDeployment(startTime: Date, expectedChanges: number | undefined): Promise<RegularDeployStackResult> {
     const monitor = this.options.quiet ? undefined : StackActivityMonitor.withDefaultPrinter(this.cfn, this.stackName, this.stackArtifact, {
       resourcesTotal: expectedChanges,
       progress: this.options.progress,
@@ -539,7 +564,7 @@ class FullCloudFormationDeployment {
       await monitor?.stop();
     }
     debug('Stack %s has completed updating', this.stackName);
-    return { noOp: false, outputs: finalState.outputs, stackArn: finalState.stackId };
+    return { type: 'did-deploy-stack', noOp: false, outputs: finalState.outputs, stackArn: finalState.stackId };
   }
 
   /**
@@ -718,3 +743,10 @@ function suffixWithErrors(msg: string, errors?: string[]) {
 function arrayEquals(a: any[], b: any[]): boolean {
   return a.every(item => b.includes(item)) && b.every(item => a.includes(item));
 }
+
+function hasReplacement(cs: AWS.CloudFormation.DescribeChangeSetOutput) {
+  return (cs.Changes ?? []).some(c => {
+    const a = c.ResourceChange?.PolicyAction;
+    return a === 'ReplaceAndDelete' || a === 'ReplaceAndRetain' || a === 'ReplaceAndSnapshot';
+  });
+}

packages/aws-cdk/lib/api/deployments.ts

@@ -8,7 +8,7 @@ import { debug, warning, error } from '../logging';
 import { Mode } from './aws-auth/credentials';
 import { ISDK } from './aws-auth/sdk';
 import { CredentialsOptions, SdkForEnvironment, SdkProvider } from './aws-auth/sdk-provider';
-import { deployStack, DeployStackResult, destroyStack, DeploymentMethod } from './deploy-stack';
+import { deployStack, destroyStack, DeploymentMethod, DeployStackResult } from './deploy-stack';
 import { EnvironmentResources, EnvironmentResourcesRegistry } from './environment-resources';
 import { HotswapMode, HotswapPropertyOverrides } from './hotswap/common';
 import { loadCurrentTemplateWithNestedStacks, loadCurrentTemplate, RootTemplateWithNestedStacks } from './nested-stack-helpers';