From 6cf55c6d687c043f6b85f4079fdd9eda451b1f86 Mon Sep 17 00:00:00 2001 From: AllanFly120 Date: Tue, 10 Dec 2019 17:04:37 -0800 Subject: [PATCH] chore: add yarn codegen script calling Gradle script (#545) * chore: add generate-clients script generating clients from models path * feat: clean the codegen input and output folder after generating * fix: update lerna commands use use '--include-dependencies' in replace of '--include-filtered-dependencies' --- package.json | 9 +- scripts/copyModels.js | 31 ------- scripts/generate-clients/code-gen.js | 65 ++++++++++++++ scripts/generate-clients/copy-to-clients.js | 89 +++++++++++++++++++ scripts/generate-clients/index.js | 30 +++++++ scripts/rebuildClients.js | 98 --------------------- scripts/utils/ModelFinder.js | 66 -------------- scripts/utils/constants.js | 3 - yarn.lock | 32 +++++++ 9 files changed, 221 insertions(+), 202 deletions(-) delete mode 100644 scripts/copyModels.js create mode 100644 scripts/generate-clients/code-gen.js create mode 100644 scripts/generate-clients/copy-to-clients.js create mode 100644 scripts/generate-clients/index.js delete mode 100644 scripts/rebuildClients.js delete mode 100644 scripts/utils/ModelFinder.js delete mode 100644 scripts/utils/constants.js diff --git a/package.json b/package.json index 3b99a99ddb6e4..2f394edc94808 100644 --- a/package.json +++ b/package.json @@ -5,14 +5,13 @@ "description": "AWS SDK for JavaScript from the future", "main": "index.js", "scripts": { + "generate-clients": "node ./scripts/generate-clients", "bootstrap": "yarn", "clean": "yarn clear-build-cache && yarn clear-build-info && lerna clean", "clear-build-cache": "rimraf ./packages/*/build/* ./clients/*/*/build/*", "clear-build-info": "rimraf ./packages/*/*.tsbuildinfo ./clients/*/*/*.tsbuildinfo", - "copy-models": "node ./scripts/copyModels.js", - "update-clients": "node ./packages/package-generator/build/cli.js import-all --matching './models/*/*/service-2.json'", - "build:crypto-dependencies": "lerna run --scope '@aws-sdk/types' --scope '@aws-sdk/util-utf8-browser' --scope '@aws-sdk/util-locate-window' --scope '@aws-sdk/hash-node' --include-filtered-dependencies pretest", - "build:smithy-client": "lerna run --scope '@aws-sdk/client-rds-data' --include-filtered-dependencies pretest", + "build:crypto-dependencies": "lerna run --scope '@aws-sdk/types' --scope '@aws-sdk/util-utf8-browser' --scope '@aws-sdk/util-locate-window' --scope '@aws-sdk/hash-node' --include-dependencies pretest", + "build:smithy-client": "lerna run --scope '@aws-sdk/client-rds-data' --include-dependencies pretest", "pretest": "yarn build:crypto-dependencies && yarn build:smithy-client", "test": "jest --coverage --passWithNoTests", "pretest-all": "lerna run pretest", @@ -36,9 +35,11 @@ "devDependencies": { "@commitlint/cli": "^8.1.0", "@commitlint/config-conventional": "^8.1.0", + "@types/fs-extra": "^8.0.1", "@types/jest": "^24.0.12", "codecov": "^3.4.0", "cucumber": "0.5.x", + "fs-extra": "^8.1.0", "generate-changelog": "^1.7.1", "husky": "^3.0.0", "jest": "^24.7.1", diff --git a/scripts/copyModels.js b/scripts/copyModels.js deleted file mode 100644 index a392c4d9a373b..0000000000000 --- a/scripts/copyModels.js +++ /dev/null @@ -1,31 +0,0 @@ -const fs = require("fs"); -const glob = require("glob"); -const path = require("path"); - -const modelsDir = require("yargs") - .alias("models", "m") - .string("m") - .demandOption("m", "A models directory must be specified") - .coerce("m", path.resolve).argv.models; - -const targetDir = path.join(path.dirname(__dirname), "serviceModels"); -ensureDirectoryExists(targetDir); - -glob(path.join(modelsDir, "*", "*", "*.json"), (err, matches) => { - for (const match of matches) { - const relativePath = path.relative(modelsDir, match); - - let targetSubdir = targetDir; - for (const dir of path.dirname(relativePath).split(path.sep)) { - ensureDirectoryExists((targetSubdir = path.join(targetSubdir, dir))); - } - - fs.copyFileSync(match, path.join(targetDir, relativePath)); - } -}); - -function ensureDirectoryExists(target) { - if (!fs.existsSync(target)) { - fs.mkdirSync(target); - } -} diff --git a/scripts/generate-clients/code-gen.js b/scripts/generate-clients/code-gen.js new file mode 100644 index 0000000000000..256c0756d68d8 --- /dev/null +++ b/scripts/generate-clients/code-gen.js @@ -0,0 +1,65 @@ +const path = require("path"); +const { copyFileSync, emptyDirSync } = require("fs-extra"); +const { readdirSync, lstatSync } = require("fs"); +const { spawn } = require("child_process"); + +const CODE_GEN_INPUT_DIR = path.normalize( + path.join(__dirname, "..", "..", "codegen", "sdk-codegen", "aws-models") +); +const CODE_GEN_ROOT = path.normalize( + path.join(__dirname, "..", "..", "codegen") +); + +async function generateClients(models) { + console.info("models directory: ", models); + if (models === CODE_GEN_INPUT_DIR) { + // console.log("skipping copying models to codegen directory"); + throw new Error( + `models directory cannot be the same as ${CODE_GEN_INPUT_DIR}` + ); + } else { + console.log(`clearing code gen input folder...`); + emptyDirSync(CODE_GEN_INPUT_DIR); + console.log(`copying models from ${models} to ${CODE_GEN_INPUT_DIR}...`); + // copySync(models, CODE_GEN_INPUT_DIR, { + // overwrite: true + // }); + for (const modelFileName of readdirSync(models)) { + const modelPath = path.join(models, modelFileName); + if (!lstatSync(modelPath).isFile()) continue; + console.log(`copying model ${modelFileName}...`); + copyFileSync(modelPath, path.join(CODE_GEN_INPUT_DIR, modelFileName), { + overwrite: true + }); + } + } + await spawnProcess( + "./gradlew", + [":sdk-codegen:clean", ":sdk-codegen:build"], + { + cwd: CODE_GEN_ROOT + } + ); +} + +const spawnProcess = (command, args = [], options = {}) => + new Promise((resolve, reject) => { + try { + const ls = spawn(command, args, options); + ls.stdout.on("data", data => { + console.log(data.toString()); + }); + ls.stderr.on("data", data => { + console.error(`stderr: ${data.toString()}`); + }); + + ls.on("close", code => { + console.log(`child process exited with code ${code}`); + resolve(); + }); + } catch (e) { + reject(e); + } + }); + +module.exports = { generateClients, CODE_GEN_INPUT_DIR }; diff --git a/scripts/generate-clients/copy-to-clients.js b/scripts/generate-clients/copy-to-clients.js new file mode 100644 index 0000000000000..10f61c6236ab4 --- /dev/null +++ b/scripts/generate-clients/copy-to-clients.js @@ -0,0 +1,89 @@ +const path = require("path"); +const { copySync, ensureDirSync } = require("fs-extra"); +const { + readdirSync, + lstatSync, + readFileSync, + existsSync, + writeFileSync +} = require("fs"); + +const CODE_GEN_OUTPUT_DIR = path.normalize( + path.join( + __dirname, + "..", + "..", + "codegen", + "sdk-codegen", + "build", + "smithyprojections", + "sdk-codegen" + ) +); + +const unOverridables = [ + "package.json", + "tsconfig.es.json", + "tsconfig.json", + "tsconfig.test.json" +]; + +async function copyToClients(clientsDir) { + for (const modelName of readdirSync(CODE_GEN_OUTPUT_DIR)) { + if (modelName === "source") continue; + const artifactPath = path.join( + CODE_GEN_OUTPUT_DIR, + modelName, + "typescript-codegen" + ); + const packageManifestPath = path.join(artifactPath, "package.json"); + if (!existsSync(packageManifestPath)) { + console.error(`${modelName} generates empty client, skip.`); + continue; + } + const packageManifest = JSON.parse( + readFileSync(packageManifestPath).toString() + ); + const packageName = packageManifest.name.replace("@aws-sdk/", ""); + console.log(`copying ${packageName} from ${artifactPath} to ${clientsDir}`); + const destPath = path.join(clientsDir, packageName); + for (const packageSub of readdirSync(artifactPath)) { + const packageSubPath = path.join(artifactPath, packageSub); + const destSubPath = path.join(destPath, packageSub); + if (unOverridables.indexOf(packageSub) >= 0) { + if (!existsSync(destSubPath)) + copySync(packageSubPath, destSubPath, { overwrite: true }); + else if (packageSub === "package.json") { + /** + * Copy package.json content in detail. + * Basically merge the generated package.json and dest package.json + * but prefer the values from dest when they contain the same key + * */ + const destManifest = JSON.parse(readFileSync(destSubPath).toString()); + const updatedManifest = { + ...packageManifest, + ...destManifest, + scripts: { + ...packageManifest.scripts, + ...destManifest.scripts + }, + dependencies: { + ...packageManifest.dependencies, + ...destManifest.dependencies + }, + devDependencies: { + ...packageManifest.devDependencies, + ...destManifest.devDependencies + } + }; + writeFileSync(destSubPath, JSON.stringify(updatedManifest, null, 2)); + } + } else { + if (lstatSync(packageSubPath).isDirectory()) ensureDirSync(destSubPath); + copySync(packageSubPath, destSubPath, { overwrite: true }); + } + } + } +} + +module.exports = { copyToClients, CODE_GEN_OUTPUT_DIR }; diff --git a/scripts/generate-clients/index.js b/scripts/generate-clients/index.js new file mode 100644 index 0000000000000..ce800c1d5d2f4 --- /dev/null +++ b/scripts/generate-clients/index.js @@ -0,0 +1,30 @@ +const yargs = require("yargs"); +const path = require("path"); +const { emptyDirSync } = require("fs-extra"); +const { generateClients, CODE_GEN_INPUT_DIR } = require("./code-gen"); +const { copyToClients, CODE_GEN_OUTPUT_DIR } = require("./copy-to-clients"); + +const CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "clients")); + +const { models, output: clientsDir } = yargs + .alias("m", "models") + .string("m") + .describe("m", "the directory of models") + .required("m") + .alias("o", "output") + .string("o") + .describe("o", "the output directory for built clients") + .default("o", CLIENTS_DIR) + .help().argv; + +(async () => { + try { + await generateClients(models); + await copyToClients(clientsDir); + emptyDirSync(CODE_GEN_INPUT_DIR); + emptyDirSync(CODE_GEN_OUTPUT_DIR); + } catch (e) { + console.log(e); + process.exit(1); + } +})(); diff --git a/scripts/rebuildClients.js b/scripts/rebuildClients.js deleted file mode 100644 index 6136692ca0573..0000000000000 --- a/scripts/rebuildClients.js +++ /dev/null @@ -1,98 +0,0 @@ -const yargs = require("yargs"); -const path = require("path"); -const fs = require("fs"); -const execSync = require("child_process").execSync; -const ServiceModelFinder = require("./utils/ModelFinder").ServiceModelFinder; -const clientNameRegex = require("./utils/constants").clientNameRegex; - -/** - * This script will scan packages directory, recognize service client packages and re-generate them from models - * If this command fails with cannot find model \'clientModuleIdentifier\', please run npm bootstrap && npm test first. - */ -const models = yargs - .alias("m", "models") - .string("m") - .default("m", path.join("..", "..", "models")) - .describe("m", "the directory of models") - .help() - .coerce("m", directory => { - return path.normalize(path.join(__filename, path.normalize(directory))); - }).argv.models; - -console.info("models directory: ", models); - -const existingServiceClients = grabExistingClients(); -console.info( - "existing service clients: ", - existingServiceClients.map(item => path.parse(item).base) -); -console.info("generating service clients..."); -const serviceModelFinder = new ServiceModelFinder(models); -for (const serviceClient of existingServiceClients) { - const clientName = path.parse(serviceClient).base; - const models = serviceModelFinder.findModelsForService(clientName); - const [_, serviceId, runtime] = clientNameRegex.exec(clientName); - console.info(`generating ${runtime} client from model at ${models.service}`); - generateClient(models, runtime); -} -console.log("done!"); - -function grabExistingClients() { - const packagesDir = path.join(path.dirname(__dirname), "packages"); - const clientPackagesPaths = []; - for (const package of fs.readdirSync(packagesDir)) { - const packageDir = path.join(packagesDir, package); - if (fs.statSync(packageDir).isDirectory() && isClientPackage(packageDir)) { - clientPackagesPaths.push(packageDir); - } - } - return clientPackagesPaths; -} - -function isClientPackage(directory) { - const baseName = path.parse(directory).base; - if (!clientNameRegex.test(baseName)) return false; - const clientFiles = [ - { base: "commands", isFile: false }, - { base: "model", isFile: false }, - { base: "types", isFile: false }, - { base: `(\\w+)Client\\.ts`, isFile: true }, - { base: `(\\w+)Configuration\\.ts`, isFile: true }, - { base: `index\\.ts`, isFile: true } - ]; - try { - const files = fs.readdirSync(directory); - for (const clientFilePattern of clientFiles) { - const matchedFile = arrayFindPattern(files, clientFilePattern.base); - if ( - !matchedFile && - fs.statSync(path.join(directory, matchedFile)).isFile() !== - clientFilePattern.isFile - ) { - return false; - } - } - } catch (e) { - return false; - } - return true; -} - -function arrayFindPattern(array, pattern) { - return array.find(item => { - const matched = new RegExp(pattern).exec(item); - //RegExp.exec() returns null if no matched - return Boolean(matched); - }); -} - -function generateClient(models, runtime) { - const command = `node packages/package-generator/build/cli.js client --m ${ - models.service - } ${models.smoke ? `--s ${models.smoke}` : ""} --r ${runtime}`; - const packageRoot = path.dirname(__dirname); - const log = execSync(command, { cwd: packageRoot }); - console.info(log.toString()); -} - -module.exports.clientNameRegex = clientNameRegex; diff --git a/scripts/utils/ModelFinder.js b/scripts/utils/ModelFinder.js deleted file mode 100644 index 943140aab2708..0000000000000 --- a/scripts/utils/ModelFinder.js +++ /dev/null @@ -1,66 +0,0 @@ -const fs = require("fs"); -const path = require("path"); -const clientNameRegex = require("./constants").clientNameRegex; -const clientModuleIdentifier = require("../../packages/package-generator/build/clientModuleIdentifier") - .clientModuleIdentifier; - -class ServiceModelFinder { - constructor(models) { - this.latestModels = this.fetchLatestModels(models); - this.loadedModelCache = new Map(); - } - - fetchLatestModels(modelsDir) { - const serviceModelDirs = []; - for (const modelName of fs.readdirSync(modelsDir)) { - const modelDir = path.join(modelsDir, modelName); - if (!fs.statSync(modelDir).isDirectory()) continue; - const versions = fs - .readdirSync(modelDir) - .filter(version => - fs.statSync(path.join(modelDir, version)).isDirectory() - ); - if (!versions || versions.length === 0) { - throw new Error(`No api version found in ${modelDir}`); - } - const latestVersion = versions.sort().reverse()[0]; - const versionDir = path.join(modelDir, latestVersion); - const serviceModels = fs.readdirSync(versionDir); - if (serviceModels.find(modelName => modelName === "service-2.json")) { - serviceModelDirs.push(versionDir); - } - } - return serviceModelDirs; - } - - /** - * Fetch the directory of model and smoke test for given service name. - * @param {string} service service client package name. like: client-sqs-node - * @returns {object} looks like {service: string, smoke: string}; - */ - findModelsForService(packageName) { - const [_, service, runtime] = clientNameRegex.exec(packageName); - if (this.loadedModelCache.has(`client-${service}`)) { - return this.loadedModelCache.get(`client-${service}`); - } - for (const latestModel of this.latestModels.slice( - this.loadedModelCache.size - )) { - const modelDir = path.join(latestModel, "service-2.json"); - const smokeDir = path.join(latestModel, "smoke.json"); - const { metadata } = JSON.parse(fs.readFileSync(modelDir).toString()); - const universalClientName = clientModuleIdentifier(metadata); - const loadedModel = { service: modelDir }; - if (fs.existsSync(smokeDir)) { - loadedModel.smoke = smokeDir; - } - this.loadedModelCache.set(universalClientName, loadedModel); - if (universalClientName === `client-${service}`) { - return loadedModel; - } - } - throw new Error(`No model found for ${packageName}`); - } -} - -module.exports.ServiceModelFinder = ServiceModelFinder; diff --git a/scripts/utils/constants.js b/scripts/utils/constants.js deleted file mode 100644 index 19e160ec81bf5..0000000000000 --- a/scripts/utils/constants.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - clientNameRegex: /^client-(\w+-?\w+)-(node|browser|universal)$/ -}; diff --git a/yarn.lock b/yarn.lock index 7454a95052645..86d91e0f41c57 100644 --- a/yarn.lock +++ b/yarn.lock @@ -55,6 +55,15 @@ dependencies: tslib "^1.9.3" +"@aws-sdk/apply-body-checksum-middleware@^0.1.0-preview.8": + version "0.1.0-preview.8" + resolved "https://registry.yarnpkg.com/@aws-sdk/apply-body-checksum-middleware/-/apply-body-checksum-middleware-0.1.0-preview.8.tgz#19a37c810c7f5f00cb5f4d194ad167b5b5b0aa81" + integrity sha512-UoK128Kum26NYuOfnG1vVjfFfYBy7m8bIkBYlORwrGYLHawUwaxan8MTCrk4P/XSlYm0NsyAicAVDvAeGWSZXA== + dependencies: + "@aws-sdk/is-array-buffer" "^0.1.0-preview.3" + "@aws-sdk/types" "^0.1.0-preview.7" + tslib "^1.8.0" + "@aws-sdk/client-cognito-identity-browser@^0.1.0-preview.10": version "0.1.0-preview.10" resolved "https://registry.yarnpkg.com/@aws-sdk/client-cognito-identity-browser/-/client-cognito-identity-browser-0.1.0-preview.10.tgz#148bd70a282da20e1f2b5e75fe290c08f31a535c" @@ -123,6 +132,14 @@ "@aws-sdk/xml-body-parser" "^0.1.0-preview.9" tslib "^1.8.0" +"@aws-sdk/location-constraint-middleware@^0.1.0-preview.7": + version "0.1.0-preview.7" + resolved "https://registry.yarnpkg.com/@aws-sdk/location-constraint-middleware/-/location-constraint-middleware-0.1.0-preview.7.tgz#0ae88632dc37187b95242e8a0370b345745b3a48" + integrity sha512-K7EZ77oSVqyrTDZKJ8OQ7m1/EAEoRFUknE0HsiTRImhRHqxPhR+sP6brWU+gdywgYbDVaXJox30GUMNAOqhs9Q== + dependencies: + "@aws-sdk/types" "^0.1.0-preview.7" + tslib "^1.8.0" + "@aws-sdk/middleware-serializer@^0.1.0-preview.7": version "0.1.0-preview.7" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-serializer/-/middleware-serializer-0.1.0-preview.7.tgz#7d4ed3d16b47f12f7ed0762ff9594c0f2da9c96c" @@ -160,6 +177,14 @@ "@aws-sdk/types" "^0.1.0-preview.7" tslib "^1.8.0" +"@aws-sdk/ssec-middleware@^0.1.0-preview.8": + version "0.1.0-preview.8" + resolved "https://registry.yarnpkg.com/@aws-sdk/ssec-middleware/-/ssec-middleware-0.1.0-preview.8.tgz#d82764bbb1fa4397711ee74f4625be20a86c52f5" + integrity sha512-vDxnda94Wp2M1RXQXPJYedDwNt9sAN2j+N6QvDpwOZ2G2qEVLOvkPPTEz2GwzU6dTQY3HSd/Psomz+9OeamWtA== + dependencies: + "@aws-sdk/types" "^0.1.0-preview.7" + tslib "^1.8.0" + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.5.5": version "7.5.5" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.5.5.tgz#bc0782f6d69f7b7d49531219699b988f669a8f9d" @@ -1464,6 +1489,13 @@ dependencies: "@types/node" "*" +"@types/fs-extra@^8.0.1": + version "8.0.1" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-8.0.1.tgz#a2378d6e7e8afea1564e44aafa2e207dadf77686" + integrity sha512-J00cVDALmi/hJOYsunyT52Hva5TnJeKP5yd1r+mH/ZU0mbYZflR0Z5kw5kITtKTRYMhm1JMClOFYdHnQszEvqw== + dependencies: + "@types/node" "*" + "@types/glob@*", "@types/glob@^7.1.1": version "7.1.1" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"