chore: experimental cdk migrate command (#25859)

cdk migrate command.

See readme for additional description of command. Default input path is a file named "template.txt" in the parent directory for now. Default output path is the current directory. If you have any issues with the actually generated code, please create an issue on https://github.com/iph/noctilucent

Please refer to noctilucent or command help supported languages to see what languages are supported.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

package.json

@@ -33,7 +33,8 @@
     "patch-package": "^6.5.1",
     "semver": "^7.5.2",
     "standard-version": "^9.5.0",
-    "typescript": "~4.9.5"
+    "ts-node": "^10.9.1",
+    "typescript": "~5.1.3"
   },
   "resolutions": {
     "colors": "1.4.0",

packages/@aws-cdk-testing/cli-integ/lib/corking.ts

@@ -19,11 +19,12 @@ export class MemoryStream extends stream.Writable {
     this.parts.splice(0, this.parts.length);
   }
 
-  public async flushTo(strm: NodeJS.WritableStream) {
+  public async flushTo(strm: NodeJS.WritableStream): Promise<void> {
     const flushed = strm.write(this.buffer());
     if (!flushed) {
       return new Promise(ok => strm.once('drain', ok));
     }
+    return;
   }
 
   public toString() {

packages/@aws-cdk-testing/framework-integ/test/aws-lambda-nodejs/tsconfig-custom.json

@@ -4,7 +4,6 @@
     "inlineSourceMap": true,
     "inlineSources": true,
     "alwaysStrict": true,
-    "charset": "utf8",
     "declaration": true,
     "emitDecoratorMetadata": true,
     "experimentalDecorators": true,

packages/@aws-cdk-testing/framework-integ/test/aws-lambda-nodejs/tsconfig.json

@@ -4,7 +4,6 @@
     "inlineSourceMap": true,
     "inlineSources": true,
     "alwaysStrict": true,
-    "charset": "utf8",
     "declaration": true,
     "experimentalDecorators": true,
     "incremental": true,

packages/@aws-cdk/cli-lib-alpha/generate.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euo pipefail
+
+commit=${CODEBUILD_RESOLVED_SOURCE_VERSION:-}
+# CODEBUILD_RESOLVED_SOURCE_VERSION is not defined (i.e. local build or CodePipeline build),
+# use the HEAD commit hash
+if [ -z "${commit}" ]; then
+  commit="$(git rev-parse --verify HEAD)"
+fi
+
+cat > build-info.json <<HERE
+{
+  "comment": "Generated at $(date -u +"%Y-%m-%dT%H:%M:%SZ") by generate.sh",
+  "commit": "${commit:0:7}"
+}
+HERE

packages/@aws-cdk/cli-lib-alpha/package.json

@@ -52,7 +52,7 @@
     "build+test+package": "yarn build+test && yarn package",
     "bundle": "esbuild --bundle lib/index.ts --target=node14 --platform=node --external:fsevents --minify-whitespace --outfile=lib/main.js",
     "compat": "cdk-compat",
-    "gen": "../../../packages/aws-cdk/generate.sh",
+    "gen": "./generate.sh",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "pkglint": "pkglint -f",

packages/aws-cdk-lib/aws-lambda-nodejs/lib/bundling.ts

@@ -216,7 +216,6 @@ export class Bundling implements cdk.BundlingOptions {
       ...this.props.metafile ? [`--metafile=${pathJoin(options.outputDir, 'index.meta.json')}`] : [],
       ...this.props.banner ? [`--banner:js=${JSON.stringify(this.props.banner)}`] : [],
       ...this.props.footer ? [`--footer:js=${JSON.stringify(this.props.footer)}`] : [],
-      ...this.props.charset ? [`--charset=${this.props.charset}`] : [],
       ...this.props.mainFields ? [`--main-fields=${this.props.mainFields.join(',')}`] : [],
       ...this.props.inject ? this.props.inject.map(i => `--inject:${i}`) : [],
       ...this.props.esbuildArgs ? [toCliArgs(this.props.esbuildArgs)] : [],

packages/aws-cdk-lib/aws-lambda-nodejs/lib/util.ts

@@ -149,6 +149,7 @@ export function getTsconfigCompilerOptions(tsconfigPath: string): string {
   const compilerOptions = extractTsConfig(tsconfigPath);
   const excludedCompilerOptions = [
     'composite',
+    'charset',
     'noEmit',
     'tsBuildInfoFile',
   ];

packages/aws-cdk-lib/aws-lambda-nodejs/test/bundling.test.ts

@@ -233,7 +233,7 @@ test('esbuild bundling with esbuild options', () => {
           defineInstructions,
           '--log-level=silent --keep-names --tsconfig=/asset-input/lib/custom-tsconfig.ts',
           '--metafile=/asset-output/index.meta.json --banner:js="/* comments */" --footer:js="/* comments */"',
-          '--charset=utf8 --main-fields=module,main --inject:./my-shim.js',
+          '--main-fields=module,main --inject:./my-shim.js',
           '--log-limit="0" --resolve-extensions=".ts,.js" --splitting --keep-names',
         ].join(' '),
       ],

packages/aws-cdk-lib/aws-lambda-nodejs/test/util.test.ts

@@ -186,7 +186,6 @@ describe('getTsconfigCompilerOptions', () => {
     const compilerOptions = getTsconfigCompilerOptions(tsconfig);
     expect(compilerOptions).toEqual([
       '--alwaysStrict',
-      '--charset utf8',
       '--declaration',
       '--declarationMap false',
       '--experimentalDecorators',
@@ -219,7 +218,6 @@ describe('getTsconfigCompilerOptions', () => {
     const compilerOptions = getTsconfigCompilerOptions(tsconfig);
     expect(compilerOptions).toEqual([
       '--alwaysStrict',
-      '--charset utf8',
       '--declaration',
       '--declarationMap false',
       '--experimentalDecorators',

packages/aws-cdk-lib/package.json

@@ -32,7 +32,7 @@
     "stripDeprecated": true,
     "compressAssembly": true,
     "pre": [
-      "npx ts-node -P tsconfig.dev.json region-info/build-tools/generate-static-data.ts",
+      "ts-node -P tsconfig.dev.json region-info/build-tools/generate-static-data.ts",
       "(cp -f $(node -p 'require.resolve(\"aws-sdk/apis/metadata.json\")') custom-resources/lib/aws-custom-resource/sdk-api-metadata.json && rm -rf custom-resources/test/aws-custom-resource/cdk.out)",
       "(rm -rf core/test/fs/fixtures && cd core/test/fs && tar -xzf fixtures.tar.gz)",
       "(rm -rf assets/test/fs/fixtures && cd assets/test/fs && tar -xzvf fixtures.tar.gz)",
@@ -162,7 +162,7 @@
     "ts-mock-imports": "^1.3.8",
     "ts-node": "^10.9.1",
     "sinon": "^9.2.4",
-    "typescript": "~5.0.4",
+    "typescript": "~5.1.3",
     "typescript-json-schema": "^0.56.0"
   },
   "peerDependencies": {

packages/aws-cdk-migration/package.json

@@ -35,7 +35,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "glob": "^7.2.3",
-    "typescript": "~4.9.5"
+    "typescript": "~5.1.3"
   },
   "devDependencies": {
     "@types/glob": "^7.2.0",

packages/aws-cdk/.eslintrc.js

@@ -1,5 +1,6 @@
 const baseConfig = require('@aws-cdk/cdk-build-tools/config/eslintrc');
 baseConfig.ignorePatterns.push('lib/init-templates/**/typescript/**/*.ts');
 baseConfig.ignorePatterns.push('test/integ/cli/sam_cdk_integ_app/**/*.ts');
+baseConfig.ignorePatterns.push('vendor/noctilucent/**/*.ts');
 baseConfig.parserOptions.project = __dirname + '/tsconfig.json';
 module.exports = baseConfig;

packages/aws-cdk/.gitignore

@@ -39,3 +39,6 @@ test/integ/cli/*.d.ts
 .DS_Store
 
 junit.xml
+
+# Exclude the noctilucent WASM package
+lib/vendor/noctilucent/

packages/aws-cdk/.npmignore

@@ -28,3 +28,6 @@ tsconfig.json
 # exclude cdk artifacts
 **/cdk.out
 junit.xml
+
+# exclude noctilucent source
+/vendor/noctilucent/

packages/aws-cdk/generate.sh

@@ -13,4 +13,32 @@ cat > build-info.json <<HERE
   "comment": "Generated at $(date -u +"%Y-%m-%dT%H:%M:%SZ") by generate.sh",
   "commit": "${commit:0:7}"
 }
-HERE
+HERE
+
+# Build noctilucent package
+(
+  # Check out the submodule if it's not there already
+  if [ ! -f "vendor/noctilucent/Cargo.toml" ]; then
+    git -C ./vendor clone https://github.com/iph/noctilucent.git
+  fi
+
+  # update the package to the pinned commit hash
+  git -C vendor/noctilucent reset --hard HEAD
+  git -C vendor/noctilucent fetch && git -C vendor/noctilucent checkout 6da7c9fade55f8443bba7b8fdfcd4ebfe5208fb1
+
+  # Install wasm-pack if it's not there already
+  if ! command -v wasm-pack >/dev/null 2>/dev/null; then
+    echo "installing wasm-pack, this may take a while..."
+    cargo install wasm-pack
+  fi
+
+  pkgroot=$(cd $(dirname -- "$0") && pwd)
+
+  cd vendor/noctilucent
+  wasm-pack build --target nodejs                                               \
+    --out-dir="${pkgroot}/lib/vendor/noctilucent"                               \
+    --out-name=index                                                            
+
+  cd ../../lib/vendor/noctilucent
+  rm package.json
+)

packages/aws-cdk/lib/cli.ts

@@ -22,6 +22,7 @@ import { CdkToolkit, AssetBuildTime } from '../lib/cdk-toolkit';
 import { realHandler as context } from '../lib/commands/context';
 import { realHandler as docs } from '../lib/commands/docs';
 import { realHandler as doctor } from '../lib/commands/doctor';
+import { MIGRATE_SUPPORTED_LANGUAGES, cliMigrate } from '../lib/commands/migrate';
 import { RequireApproval } from '../lib/diff';
 import { availableInitLanguages, cliInit, printAvailableTemplates } from '../lib/init';
 import { data, debug, error, print, setLogLevel, setCI } from '../lib/logging';
@@ -269,6 +270,12 @@ async function parseCommandLineArguments(args: string[]) {
       .option('list', { type: 'boolean', desc: 'List the available templates' })
       .option('generate-only', { type: 'boolean', default: false, desc: 'If true, only generates project files, without executing additional operations such as setting up a git repo, installing dependencies or compiling the project' }),
     )
+    .command('migrate', false /* hidden from "cdk --help" */, (yargs: Argv) => yargs
+      .option('language', { type: 'string', alias: 'l', desc: 'The language to be used for the new project (default can be configured in ~/.cdk.json)', choices: MIGRATE_SUPPORTED_LANGUAGES })
+      .option('input-path', { type: 'string', alias: 'inputpath', desc: 'The path to the CloudFormation template to migrate' })
+      .option('output-path', { type: 'string', alias: 'outputpath', desc: 'The output path for the migrated cdk code' })
+      .option('generate-only', { type: 'boolean', default: false, desc: 'If true, only generates project files, without executing additional operations such as setting up a git repo, installing dependencies or compiling the project' }),
+    )
     .command('context', 'Manage cached context values', (yargs: Argv) => yargs
       .option('reset', { alias: 'e', desc: 'The context key (or its index) to reset', type: 'string', requiresArg: true })
       .option('force', { alias: 'f', desc: 'Ignore missing key error', type: 'boolean', default: false })
@@ -649,14 +656,16 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
         } else {
           return cliInit(args.TEMPLATE, language, undefined, args.generateOnly);
         }
+      case 'migrate':
+        const migrateLanguage = configuration.settings.get(['language']);
+        return cliMigrate(args.inputpath, migrateLanguage, args.generateOnly, args.outputpath);
       case 'version':
         return data(version.DISPLAY_VERSION);
 
       default:
         throw new Error('Unknown command: ' + command);
     }
   }
-
 }
 
 /**

packages/aws-cdk/lib/commands/migrate.ts

@@ -0,0 +1,71 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { initializeProject, availableInitTemplates } from '../../lib/init';
+import { warning } from '../logging';
+import * as nocti from '../vendor/noctilucent';
+
+/** The list of languages supported by the built-in noctilucent binary. */
+export const MIGRATE_SUPPORTED_LANGUAGES: readonly string[] = nocti.supported_languages();
+
+export async function cliMigrate(
+  inputpath: string = process.cwd() + '/../template.txt',
+  language = MIGRATE_SUPPORTED_LANGUAGES[0],
+  generateOnly = false,
+  outputpath = process.cwd(),
+) {
+  warning('This is an experimental feature. We make no guarantees about the outcome or stability of the functionality.');
+  const type = 'default'; // "default" is the default type (and maps to 'app')
+  const template = (await availableInitTemplates()).find(t => t.hasName(type!));
+  if (!template) {
+    throw new Error(`couldn't find template for ${type} app type, this should never happen`);
+  }
+
+  if (!MIGRATE_SUPPORTED_LANGUAGES.includes(language)) {
+    throw new Error(`Unsupported language for cdk migrate: ${language}. Supported languages are: ${MIGRATE_SUPPORTED_LANGUAGES.join(', ')}`);
+  }
+
+  await initializeProject(template, language, true, generateOnly, outputpath);
+  const template_file = fs.readFileSync(inputpath, 'utf8');
+  const generated_app = nocti.transmute(template_file, language);
+
+  // clear out the init'd bin/lib files to replace with our own
+  delete_files(outputpath + '/lib/');
+
+  // we hardcode everything to be called noctstack still so this works for now.
+  // Will change this to be much smarter once we can change stack name in noct
+  const bin_app = `#!/usr/bin/env node
+  import 'source-map-support/register';
+  import * as cdk from 'aws-cdk-lib';
+  import { NoctStack } from '../lib/generated_stack';
+
+  const app = new cdk.App();
+  new NoctStack(app, 'NoctStack', {
+    /* If you don't specify 'env', this stack will be environment-agnostic.
+     * Account/Region-dependent features and context lookups will not work,
+     * but a single synthesized template can be deployed anywhere. */
+
+    /* Uncomment the next line to specialize this stack for the AWS Account
+     * and Region that are implied by the current CLI configuration. */
+    // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
+
+    /* Uncomment the next line if you know exactly what Account and Region you
+     * want to deploy the stack to. */
+    // env: { account: '123456789012', region: 'us-east-1' },
+
+    /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
+  });`;
+  const myname = path.basename(outputpath);
+  fs.writeFileSync(outputpath + '/lib/' + 'generated_stack.ts', generated_app);
+  fs.writeFileSync(outputpath + '/bin/' + `${myname}.ts`, bin_app);
+}
+
+function delete_files(filepath: string) {
+  fs.readdir(filepath, (err, files) => {
+    if (err) throw err;
+    for (const file of files) {
+      fs.unlink(filepath + file, (cause) => {
+        if (cause) throw cause;
+      });
+    }
+  });
+}

packages/aws-cdk/lib/init-templates/app/typescript/package.json

@@ -17,7 +17,7 @@
     "ts-jest": "^29.1.0",
     "aws-cdk": "%cdk-version%",
     "ts-node": "^10.9.1",
-    "typescript": "~5.0.4"
+    "typescript": "~5.1.3"
   },
   "dependencies": {
     "aws-cdk-lib": "%cdk-version%",

packages/aws-cdk/lib/init-templates/lib/typescript/package.json

@@ -15,7 +15,7 @@
     "constructs": "%constructs-version%",
     "jest": "^29.5.0",
     "ts-jest": "^29.1.0",
-    "typescript": "~5.0.4"
+    "typescript": "~5.1.3"
   },
   "peerDependencies": {
     "aws-cdk-lib": "%cdk-version%",

packages/aws-cdk/lib/init-templates/sample-app/typescript/package.json

@@ -17,7 +17,7 @@
     "ts-jest": "^29.1.0",
     "aws-cdk": "%cdk-version%",
     "ts-node": "^10.9.1",
-    "typescript": "~5.0.4"
+    "typescript": "~5.1.3"
   },
   "dependencies": {
     "aws-cdk-lib": "%cdk-version%",

packages/aws-cdk/lib/init.ts

@@ -247,7 +247,7 @@ export async function printAvailableTemplates(language?: string) {
   }
 }
 
-async function initializeProject(template: InitTemplate, language: string, canUseNetwork: boolean, generateOnly: boolean, workDir: string) {
+export async function initializeProject(template: InitTemplate, language: string, canUseNetwork: boolean, generateOnly: boolean, workDir: string) {
   await assertIsEmptyDirectory(workDir);
   print(`Applying project template ${chalk.green(template.name)} for ${chalk.blue(language)}`);
   await template.install(language, workDir);

packages/aws-cdk/package.json

@@ -35,7 +35,8 @@
       },
       "resources": {
         "../../node_modules/vm2/lib/bridge.js": "lib/bridge.js",
-        "../../node_modules/vm2/lib/setup-sandbox.js": "lib/setup-sandbox.js"
+        "../../node_modules/vm2/lib/setup-sandbox.js": "lib/setup-sandbox.js",
+        "lib/vendor/noctilucent/index_bg.wasm": "lib/index_bg.wasm"
       },
       "allowedLicenses": [
         "Apache-2.0",

packages/aws-cdk/tsconfig.json

@@ -2,7 +2,11 @@
     "compilerOptions": {
         "target": "ES2020",
         "module": "commonjs",
-        "lib": ["es2019", "es2020", "dom"],
+        "lib": [
+            "es2019",
+            "es2020",
+            "dom"
+        ],
         "strict": true,
         "alwaysStrict": true,
         "skipLibCheck": true,
@@ -25,7 +29,8 @@
     ],
     "exclude": [
         "lib/init-templates/**/typescript/*/*.ts",
-        "test/integ/cli/sam_cdk_integ_app/**/*"
+        "test/integ/cli/sam_cdk_integ_app/**/*",
+        "vendor/**/*",
     ],
     "references": [
         {

packages/aws-cdk/vendor/README.md

@@ -0,0 +1,13 @@
+## Vendored-in dependencies
+
+The dependencies in this directory are checked out using the `gen` script. 
+This will fetch and clone the noctilucent crate and generate the wasm code if 
+that has not been done already, ensuring the dependencies are adequately 
+checked out.
+
+In order to update the notcilucent crate, run the ./generate.sh script. If you wish
+to update to a different noctilucent commit hash instead of the one provided, modify
+the hash in the generate.sh script and then rerun ./generate.sh
+
+The `THIRD_PARTY_LICENSES` file might need updating accordingly, which can be
+automatically done by running `yarn pkglint`.

packages/awslint/package.json

@@ -30,7 +30,7 @@
     "@types/jest": "^29.5.1",
     "@types/yargs": "^15.0.15",
     "@aws-cdk/pkglint": "0.0.0",
-    "typescript": "~4.9.5",
+    "typescript": "~5.1.3",
     "@typescript-eslint/eslint-plugin": "^4.33.0",
     "@typescript-eslint/parser": "^4.33.0",
     "eslint": "^7.32.0",