feat(pipelines): changes needed to support other engines

packages/@aws-cdk/pipelines/lib/blueprint/index.ts

@@ -1,6 +1,6 @@
 export * from './asset-type';
 export * from './blueprint';
-export * from './blueprint-queries';
+export * from '../helpers-internal/blueprint-queries';
 export * from './file-set';
 export * from './script-step';
 export * from './stack-deployment';

packages/@aws-cdk/pipelines/lib/blueprint/stack-deployment.ts

@@ -1,4 +1,5 @@
 import * as path from 'path';
+import { parse as parseUrl } from 'url';
 import * as cxapi from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
 import { AssetManifestReader, DockerImageManifestEntry, FileManifestEntry } from '../private/asset-manifest';
@@ -17,7 +18,8 @@ export interface StackDeploymentProps {
   readonly tags?: Record<string, string>;
   readonly customCloudAssembly?: IFileSet;
   readonly absoluteTemplatePath: string;
-  readonly requiredAssets?: StackAsset[];
+  readonly assets?: StackAsset[];
+  readonly templateS3Uri?: string;
 }
 
 export class StackDeployment {
@@ -38,7 +40,8 @@ export class StackDeployment {
       absoluteTemplatePath: path.join(stackArtifact.assembly.directory, stackArtifact.templateFile),
       assumeRoleArn: stackArtifact.assumeRoleArn,
       executionRoleArn: stackArtifact.cloudFormationExecutionRoleArn,
-      requiredAssets: extractStackAssets(stackArtifact),
+      assets: extractStackAssets(stackArtifact),
+      templateS3Uri: stackArtifact.stackTemplateAssetObjectUrl,
     });
   }
 
@@ -53,9 +56,23 @@ export class StackDeployment {
   public readonly customCloudAssembly?: FileSet;
   public readonly absoluteTemplatePath: string;
   public readonly requiredAssets: StackAsset[];
-
   public readonly dependsOnStacks: StackDeployment[] = [];
 
+  /**
+   * The asset that represents the CloudFormation template for this stack.
+   */
+  public readonly templateAsset?: StackAsset;
+
+  /**
+   * The S3 URL which points to the template asset location in the publishing
+   * bucket.
+   *
+   * This is `undefined` if the stack template is not published.
+   *
+   * @example https://bucket.s3.amazonaws.com/object/key
+   */
+  public readonly templateUrl?: string;
+
   constructor(props: StackDeploymentProps) {
     this.stackArtifactId = props.stackArtifactId;
     this.stackHierarchicalId = props.stackHierarchicalId;
@@ -67,7 +84,17 @@ export class StackDeployment {
     this.stackName = props.stackName;
     this.customCloudAssembly = props.customCloudAssembly?.primaryOutput;
     this.absoluteTemplatePath = props.absoluteTemplatePath;
-    this.requiredAssets = props.requiredAssets ?? [];
+    this.templateUrl = props.templateS3Uri ? s3UrlFromUri(props.templateS3Uri) : undefined;
+
+    this.requiredAssets = new Array<StackAsset>();
+
+    for (const asset of props.assets ?? []) {
+      if (asset.isTemplate) {
+        this.templateAsset = asset;
+      } else {
+        this.requiredAssets.push(asset);
+      }
+    }
   }
 
   public relativeTemplatePath(root: string) {
@@ -107,6 +134,11 @@ export interface StackAsset {
    * Type of asset to publish
    */
   readonly assetType: AssetType;
+
+  /**
+   * Does this asset represent the template.
+   */
+  readonly isTemplate?: boolean;
 }
 
 function extractStackAssets(stackArtifact: cxapi.CloudFormationStackArtifact): StackAsset[] {
@@ -118,14 +150,12 @@ function extractStackAssets(stackArtifact: cxapi.CloudFormationStackArtifact): S
 
     for (const entry of manifest.entries) {
       let assetType: AssetType;
+      let isTemplate;
+
       if (entry instanceof DockerImageManifestEntry) {
         assetType = AssetType.DOCKER_IMAGE;
       } else if (entry instanceof FileManifestEntry) {
-        // Don't publishg the template for this stack
-        if (entry.source.packaging === 'file' && entry.source.path === stackArtifact.templateFile) {
-          continue;
-        }
-
+        isTemplate = entry.source.packaging === 'file' && entry.source.path === stackArtifact.templateFile;
         assetType = AssetType.FILE;
       } else {
         throw new Error(`Unrecognized asset type: ${entry.type}`);
@@ -136,9 +166,15 @@ function extractStackAssets(stackArtifact: cxapi.CloudFormationStackArtifact): S
         assetId: entry.id.assetId,
         assetSelector: entry.id.toString(),
         assetType,
+        isTemplate,
       });
     }
   }
 
   return ret;
+}
+
+function s3UrlFromUri(uri: string) {
+  const url = parseUrl(uri);
+  return `https://${url.hostname}.s3.amazonaws.com${url.path}`;
 }

packages/@aws-cdk/pipelines/lib/blueprint/step.ts

@@ -5,6 +5,7 @@ export abstract class Step implements IFileSet {
   public abstract readonly primaryOutput?: FileSet;
 
   public readonly requiredFileSets: FileSet[] = [];
+  public readonly isSource: boolean = false;
 
   constructor(public readonly id: string) {
   }

packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-engine.ts

@@ -8,14 +8,13 @@ import { Aws, Stack } from '@aws-cdk/core';
 import * as cxapi from '@aws-cdk/cx-api';
 import { Construct, Node } from 'constructs';
 import { AssetType, BlueprintQueries, ManualApprovalStep, ScriptStep, StackAsset, StackDeployment, Step } from '../blueprint';
+import { GraphNode, GraphNodeCollection, isGraph, AGraphNode, PipelineStructure } from '../helpers-internal';
 import { BuildDeploymentOptions, IDeploymentEngine } from '../main/engine';
 import { appOf, assemblyBuilderOf, embeddedAsmPath } from '../private/construct-internals';
 import { toPosixPath } from '../private/fs';
-import { GraphNode, GraphNodeCollection, isGraph } from '../private/graph';
 import { enumerate, flatten, maybeSuffix } from '../private/javascript';
 import { writeTemplateConfiguration } from '../private/template-configuration';
 import { CodeBuildFactory, mergeBuildEnvironments, stackVariableNamespace } from './_codebuild-factory';
-import { AGraphNode, PipelineStructure } from './_pipeline-structure';
 import { ArtifactMap } from './artifact-map';
 import { CodeBuildStep } from './codebuild-step';
 import { CodePipelineActionFactoryResult, ICodePipelineActionFactory } from './codepipeline-action-factory';

packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-source.ts

@@ -22,6 +22,9 @@ export abstract class CodePipelineSource extends Step implements ICodePipelineAc
     return new GitHubSource(repoString, props);
   }
 
+  // tells `PipelineGraph` to hoist a "Source" step
+  public readonly isSource = true;
+
   public abstract produce(options: CodePipelineActionOptions): CodePipelineActionFactoryResult;
 }
 

packages/@aws-cdk/pipelines/lib/blueprint/blueprint-queries.ts → packages/@aws-cdk/pipelines/lib/helpers-internal/blueprint-queries.ts

@@ -1,7 +1,4 @@
-import { Blueprint } from './blueprint';
-import { ScriptStep, StackOutputReference } from './script-step';
-import { StackDeployment } from './stack-deployment';
-import { Step } from './step';
+import { Step, ScriptStep, StackOutputReference, Blueprint, StackDeployment } from '../blueprint';
 
 /**
  * Answer some questions about a pipeline blueprint

packages/@aws-cdk/pipelines/lib/private/graph/index.ts → packages/@aws-cdk/pipelines/lib/helpers-internal/graph.ts

@@ -1,7 +1,7 @@
 /**
  * A library for nested graphs
  */
-import { addAll, extract, flatMap } from '../../private/javascript';
+import { addAll, extract, flatMap } from '../private/javascript';
 import { topoSort } from './toposort';
 
 export interface GraphNodeProps<A> {
@@ -21,6 +21,23 @@ export class GraphNode<A> {
     this.data = props.data;
   }
 
+  /**
+   * A graph-wide unique identifier for this node. Rendered by joining the IDs
+   * of all ancestors with hyphens.
+   */
+  public get uniqueId(): string {
+    return this.ancestorPath(this.root).map(x => x.id).join('-');
+  }
+
+  /**
+   * The union of all dependencies of this node and the dependencies of all
+   * parent graphs.
+   */
+  public get allDeps(): GraphNode<A>[] {
+    const fromParent = this.parentGraph?.allDeps ?? [];
+    return [...this.dependencies, ...fromParent];
+  }
+
   public dependOn(...dependencies: GraphNode<A>[]) {
     if (dependencies.includes(this)) {
       throw new Error(`Cannot add dependency on self: ${this}`);

packages/@aws-cdk/pipelines/lib/helpers-internal/index.ts

@@ -0,0 +1,3 @@
+export * from './pipeline-structure';
+export * from './graph';
+export * from './blueprint-queries';

packages/@aws-cdk/pipelines/lib/codepipeline/_pipeline-structure.ts → packages/@aws-cdk/pipelines/lib/helpers-internal/pipeline-structure.ts

@@ -1,9 +1,28 @@
 import { AssetType, Blueprint, BlueprintQueries, FileSet, ScriptStep, StackAsset, StackDeployment, StageDeployment, Step, Wave } from '../blueprint';
-import { DependencyBuilders, Graph, GraphNode, GraphNodeCollection } from '../private/graph';
-import { CodePipelineSource } from './codepipeline-source';
+import { DependencyBuilders, Graph, GraphNode, GraphNodeCollection } from './graph';
 
 export interface PipelineStructureProps {
+  /**
+   * Add a self-mutation step.
+   *
+   * @default false
+   */
   readonly selfMutation?: boolean;
+
+  /**
+   * Publishes the template asset to S3.
+   *
+   * @default false
+   */
+  readonly publishTemplate?: boolean;
+
+  /**
+   * Add a "prepare" step for each stack which can be used to create the change
+   * set. If this is disbled, only the "execute" step will be included.
+   *
+   * @default true
+   */
+  readonly prepareStep?: boolean;
 }
 
 /**
@@ -21,11 +40,17 @@ export class PipelineStructure {
   private readonly synthNode: AGraphNode;
   private readonly selfMutateNode?: AGraphNode;
   private readonly stackOutputDependencies = new DependencyBuilders<StackDeployment, any>();
+  private readonly publishTemplate: boolean;
+  private readonly prepareStep: boolean;
+
   private lastPreparationNode: AGraphNode;
   private _fileAssetCtr = 0;
   private _dockerAssetCtr = 0;
 
   constructor(public readonly blueprint: Blueprint, props: PipelineStructureProps = {}) {
+    this.publishTemplate = props.publishTemplate ?? false;
+    this.prepareStep = props.prepareStep ?? true;
+
     this.queries = new BlueprintQueries(blueprint);
 
     this.synthNode = this.addBuildStep(blueprint.synthStep);
@@ -90,28 +115,47 @@ export class PipelineStructure {
 
     for (const stack of stage.stacks) {
       const stackGraph: AGraph = Graph.of(this.simpleStackName(stack.stackName, stage.stageName), { type: 'stack-group', stack });
-      const prepareNode: AGraphNode = GraphNode.of('Prepare', { type: 'prepare', stack });
+      const prepareNode: AGraphNode | undefined = this.prepareStep ? GraphNode.of('Prepare', { type: 'prepare', stack }) : undefined;
       const deployNode: AGraphNode = GraphNode.of('Deploy', {
         type: 'execute',
         stack,
         captureOutputs: this.queries.stackOutputsReferenced(stack).length > 0,
       });
 
       retGraph.add(stackGraph);
-      stackGraph.add(prepareNode, deployNode);
-      deployNode.dependOn(prepareNode);
+
+      stackGraph.add(deployNode);
+      let firstDeployNode;
+      if (prepareNode) {
+        stackGraph.add(prepareNode);
+        deployNode.dependOn(prepareNode);
+        firstDeployNode = prepareNode;
+      } else {
+        firstDeployNode = deployNode;
+      }
+
       stackGraphs.set(stack, stackGraph);
 
       // Depend on Cloud Assembly
       const cloudAssembly = stack.customCloudAssembly?.primaryOutput ?? this.cloudAssemblyFileSet;
-      prepareNode.dependOn(this.addAndRecurse(cloudAssembly.producer, retGraph));
+
+      firstDeployNode.dependOn(this.addAndRecurse(cloudAssembly.producer, retGraph));
+
+      // add the template asset
+      if (this.publishTemplate) {
+        if (!stack.templateAsset) {
+          throw new Error(`"publishTemplate" is enabled, but stack ${stack.stackArtifactId} does not have a template asset`);
+        }
+
+        firstDeployNode.dependOn(this.publishAsset(stack.templateAsset));
+      }
 
       // Depend on Assets
       // FIXME: Custom Cloud Assembly currently doesn't actually help separating
       // out templates from assets!!!
       for (const asset of stack.requiredAssets) {
         const assetNode = this.publishAsset(asset);
-        prepareNode.dependOn(assetNode);
+        firstDeployNode.dependOn(assetNode);
       }
 
       // Add stack output synchronization point
@@ -122,7 +166,15 @@ export class PipelineStructure {
 
     for (const stack of stage.stacks) {
       for (const dep of stack.dependsOnStacks) {
-        stackGraphs.get(stack)?.dependOn(stackGraphs.get(dep)!);
+        const stackNode = stackGraphs.get(stack);
+        const depNode = stackGraphs.get(dep);
+        if (!stackNode) {
+          throw new Error(`cannot find node for ${stack.stackName}`);
+        }
+        if (!depNode) {
+          throw new Error(`cannot find node for ${dep.stackName}`);
+        }
+        stackNode.dependOn(depNode);
       }
     }
 
@@ -160,7 +212,7 @@ export class PipelineStructure {
 
     // If the step is a source step, change the parent to a special "Source" stage
     // (CodePipeline wants it that way)
-    if (step instanceof CodePipelineSource) {
+    if (step.isSource) {
       parent = this.topLevelGraph('Source');
     }
 

packages/@aws-cdk/pipelines/lib/private/graph/toposort.ts → packages/@aws-cdk/pipelines/lib/helpers-internal/toposort.ts

@@ -1,4 +1,4 @@
-import { GraphNode } from './index';
+import { GraphNode } from './graph';
 
 export function printDependencyMap<A>(dependencies: Map<GraphNode<A>, Set<GraphNode<A>>>) {
   const lines = ['---'];

packages/@aws-cdk/pipelines/test/blueprint/codepipeline/pipeline-structure.test.ts

@@ -1,8 +1,7 @@
 /* eslint-disable import/no-extraneous-dependencies */
 import '@aws-cdk/assert-internal/jest';
 import * as cdkp from '../../../lib';
-import { PipelineStructure } from '../../../lib/codepipeline/_pipeline-structure';
-import { Graph, GraphNode } from '../../../lib/private/graph';
+import { Graph, GraphNode, PipelineStructure } from '../../../lib/helpers-internal';
 import { flatten } from '../../../lib/private/javascript';
 import { AppWithOutput, OneStackApp } from '../test-app';
 import { TestApp } from '../testutil';

packages/@aws-cdk/pipelines/test/blueprint/graph/dependencies.test.ts

@@ -1,4 +1,4 @@
-import { Graph, GraphNode } from '../../../lib/private/graph';
+import { Graph, GraphNode } from '../../../lib/helpers-internal';
 
 class PlainNode extends GraphNode<any> { }