diff --git a/package.json b/package.json index e819b34f933c..cac7e517d554 100644 --- a/package.json +++ b/package.json @@ -5,14 +5,13 @@ "description": "AWS SDK for JavaScript from the future", "main": "index.js", "scripts": { + "generate-clients": "node ./scripts/generate-clients", "bootstrap": "yarn", "clean": "yarn clear-build-cache && yarn clear-build-info && lerna clean", "clear-build-cache": "rimraf ./packages/*/build/* ./clients/*/*/build/*", "clear-build-info": "rimraf ./packages/*/*.tsbuildinfo ./clients/*/*/*.tsbuildinfo", - "copy-models": "node ./scripts/copyModels.js", - "update-clients": "node ./packages/package-generator/build/cli.js import-all --matching './models/*/*/service-2.json'", - "build:crypto-dependencies": "lerna run --scope '@aws-sdk/types' --scope '@aws-sdk/util-utf8-browser' --scope '@aws-sdk/util-locate-window' --scope '@aws-sdk/hash-node' --include-filtered-dependencies pretest", - "build:smithy-client": "lerna run --scope '@aws-sdk/client-rds-data' --include-filtered-dependencies pretest", + "build:crypto-dependencies": "lerna run --scope '@aws-sdk/types' --scope '@aws-sdk/util-utf8-browser' --scope '@aws-sdk/util-locate-window' --scope '@aws-sdk/hash-node' --include-dependencies pretest", + "build:smithy-client": "lerna run --scope '@aws-sdk/client-rds-data' --include-dependencies pretest", "pretest": "yarn build:crypto-dependencies && yarn build:smithy-client", "test": "jest --coverage --passWithNoTests", "pretest-all": "lerna run pretest", @@ -36,9 +35,11 @@ "devDependencies": { "@commitlint/cli": "^8.1.0", "@commitlint/config-conventional": "^8.1.0", + "@types/fs-extra": "^8.0.1", "@types/jest": "^24.0.12", "codecov": "^3.4.0", "cucumber": "0.5.x", + "fs-extra": "^8.1.0", "generate-changelog": "^1.7.1", "husky": "^3.0.0", "jest": "^24.7.1", diff --git a/scripts/copyModels.js b/scripts/copyModels.js deleted file mode 100644 index 26b3ce956abb..000000000000 --- a/scripts/copyModels.js +++ /dev/null @@ -1,35 +0,0 @@ -const fs = require('fs'); -const glob = require('glob'); -const path = require('path'); - -const modelsDir = require('yargs') - .alias('models', 'm') - .string('m') - .demandOption('m', 'A models directory must be specified') - .coerce('m', path.resolve) - .argv - .models; - -const targetDir = path.join(path.dirname(__dirname), 'serviceModels'); -ensureDirectoryExists(targetDir); - -glob(path.join(modelsDir, '*', '*', '*.json'), (err, matches) => { - for (const match of matches) { - const relativePath = path.relative(modelsDir, match); - - let targetSubdir = targetDir; - for (const dir of path.dirname(relativePath).split(path.sep)) { - ensureDirectoryExists( - targetSubdir = path.join(targetSubdir, dir) - ); - } - - fs.copyFileSync(match, path.join(targetDir, relativePath)); - } -}) - -function ensureDirectoryExists(target) { - if (!fs.existsSync(target)) { - fs.mkdirSync(target); - } -} diff --git a/scripts/generate-clients/code-gen.js b/scripts/generate-clients/code-gen.js new file mode 100644 index 000000000000..256c0756d68d --- /dev/null +++ b/scripts/generate-clients/code-gen.js @@ -0,0 +1,65 @@ +const path = require("path"); +const { copyFileSync, emptyDirSync } = require("fs-extra"); +const { readdirSync, lstatSync } = require("fs"); +const { spawn } = require("child_process"); + +const CODE_GEN_INPUT_DIR = path.normalize( + path.join(__dirname, "..", "..", "codegen", "sdk-codegen", "aws-models") +); +const CODE_GEN_ROOT = path.normalize( + path.join(__dirname, "..", "..", "codegen") +); + +async function generateClients(models) { + console.info("models directory: ", models); + if (models === CODE_GEN_INPUT_DIR) { + // console.log("skipping copying models to codegen directory"); + throw new Error( + `models directory cannot be the same as ${CODE_GEN_INPUT_DIR}` + ); + } else { + console.log(`clearing code gen input folder...`); + emptyDirSync(CODE_GEN_INPUT_DIR); + console.log(`copying models from ${models} to ${CODE_GEN_INPUT_DIR}...`); + // copySync(models, CODE_GEN_INPUT_DIR, { + // overwrite: true + // }); + for (const modelFileName of readdirSync(models)) { + const modelPath = path.join(models, modelFileName); + if (!lstatSync(modelPath).isFile()) continue; + console.log(`copying model ${modelFileName}...`); + copyFileSync(modelPath, path.join(CODE_GEN_INPUT_DIR, modelFileName), { + overwrite: true + }); + } + } + await spawnProcess( + "./gradlew", + [":sdk-codegen:clean", ":sdk-codegen:build"], + { + cwd: CODE_GEN_ROOT + } + ); +} + +const spawnProcess = (command, args = [], options = {}) => + new Promise((resolve, reject) => { + try { + const ls = spawn(command, args, options); + ls.stdout.on("data", data => { + console.log(data.toString()); + }); + ls.stderr.on("data", data => { + console.error(`stderr: ${data.toString()}`); + }); + + ls.on("close", code => { + console.log(`child process exited with code ${code}`); + resolve(); + }); + } catch (e) { + reject(e); + } + }); + +module.exports = { generateClients, CODE_GEN_INPUT_DIR }; diff --git a/scripts/generate-clients/copy-to-clients.js b/scripts/generate-clients/copy-to-clients.js new file mode 100644 index 000000000000..10f61c6236ab --- /dev/null +++ b/scripts/generate-clients/copy-to-clients.js @@ -0,0 +1,89 @@ +const path = require("path"); +const { copySync, ensureDirSync } = require("fs-extra"); +const { + readdirSync, + lstatSync, + readFileSync, + existsSync, + writeFileSync +} = require("fs"); + +const CODE_GEN_OUTPUT_DIR = path.normalize( + path.join( + __dirname, + "..", + "..", + "codegen", + "sdk-codegen", + "build", + "smithyprojections", + "sdk-codegen" + ) +); + +const unOverridables = [ + "package.json", + "tsconfig.es.json", + "tsconfig.json", + "tsconfig.test.json" +]; + +async function copyToClients(clientsDir) { + for (const modelName of readdirSync(CODE_GEN_OUTPUT_DIR)) { + if (modelName === "source") continue; + const artifactPath = path.join( + CODE_GEN_OUTPUT_DIR, + modelName, + "typescript-codegen" + ); + const packageManifestPath = path.join(artifactPath, "package.json"); + if (!existsSync(packageManifestPath)) { + console.error(`${modelName} generates empty client, skip.`); + continue; + } + const packageManifest = JSON.parse( + readFileSync(packageManifestPath).toString() + ); + const packageName = packageManifest.name.replace("@aws-sdk/", ""); + console.log(`copying ${packageName} from ${artifactPath} to ${clientsDir}`); + const destPath = path.join(clientsDir, packageName); + for (const packageSub of readdirSync(artifactPath)) { + const packageSubPath = path.join(artifactPath, packageSub); + const destSubPath = path.join(destPath, packageSub); + if (unOverridables.indexOf(packageSub) >= 0) { + if (!existsSync(destSubPath)) + copySync(packageSubPath, destSubPath, { overwrite: true }); + else if (packageSub === "package.json") { + /** + * Copy package.json content in detail. + * Basically merge the generated package.json and dest package.json + * but prefer the values from dest when they contain the same key + * */ + const destManifest = JSON.parse(readFileSync(destSubPath).toString()); + const updatedManifest = { + ...packageManifest, + ...destManifest, + scripts: { + ...packageManifest.scripts, + ...destManifest.scripts + }, + dependencies: { + ...packageManifest.dependencies, + ...destManifest.dependencies + }, + devDependencies: { + ...packageManifest.devDependencies, + ...destManifest.devDependencies + } + }; + writeFileSync(destSubPath, JSON.stringify(updatedManifest, null, 2)); + } + } else { + if (lstatSync(packageSubPath).isDirectory()) ensureDirSync(destSubPath); + copySync(packageSubPath, destSubPath, { overwrite: true }); + } + } + } +} + +module.exports = { copyToClients, CODE_GEN_OUTPUT_DIR }; diff --git a/scripts/generate-clients/index.js b/scripts/generate-clients/index.js new file mode 100644 index 000000000000..ce800c1d5d2f --- /dev/null +++ b/scripts/generate-clients/index.js @@ -0,0 +1,30 @@ +const yargs = require("yargs"); +const path = require("path"); +const { emptyDirSync } = require("fs-extra"); +const { generateClients, CODE_GEN_INPUT_DIR } = require("./code-gen"); +const { copyToClients, CODE_GEN_OUTPUT_DIR } = require("./copy-to-clients"); + +const CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "clients")); + +const { models, output: clientsDir } = yargs + .alias("m", "models") + .string("m") + .describe("m", "the directory of models") + .required("m") + .alias("o", "output") + .string("o") + .describe("o", "the output directory for built clients") + .default("o", CLIENTS_DIR) + .help().argv; + +(async () => { + try { + await generateClients(models); + await copyToClients(clientsDir); + emptyDirSync(CODE_GEN_INPUT_DIR); + emptyDirSync(CODE_GEN_OUTPUT_DIR); + } catch (e) { + console.log(e); + process.exit(1); + } +})(); diff --git a/scripts/rebuildClients.js b/scripts/rebuildClients.js deleted file mode 100644 index 54c677123231..000000000000 --- a/scripts/rebuildClients.js +++ /dev/null @@ -1,95 +0,0 @@ -const yargs = require('yargs'); -const path = require('path'); -const fs = require('fs'); -const execSync = require('child_process').execSync; -const ServiceModelFinder = require('./utils/ModelFinder').ServiceModelFinder; -const clientNameRegex = require('./utils/constants').clientNameRegex; - -/** - * This script will scan packages directory, recognize service client packages and re-generate them from models - * If this command fails with cannot find model \'clientModuleIdentifier\', please run npm bootstrap && npm test first. - */ -const models = yargs - .alias('m', 'models') - .string('m') - .default('m', path.join('..', '..', 'models')) - .describe('m', 'the directory of models') - .help() - .coerce('m', (directory) => { - return path.normalize(path.join(__filename, path.normalize(directory))); - }) - .argv - .models; - -console.info('models directory: ', models); - -const existingServiceClients = grabExistingClients(); -console.info('existing service clients: ', existingServiceClients.map(item => path.parse(item).base)); -console.info('generating service clients...'); -const serviceModelFinder = new ServiceModelFinder(models); -for (const serviceClient of existingServiceClients) { - const clientName = path.parse(serviceClient).base; - const models = serviceModelFinder.findModelsForService(clientName); - const [_, serviceId, runtime] = clientNameRegex.exec(clientName); - console.info(`generating ${runtime} client from model at ${models.service}`) - generateClient(models, runtime); -} -console.log('done!'); - -function grabExistingClients() { - const packagesDir = path.join(path.dirname(__dirname), 'packages'); - const clientPackagesPaths = []; - for (const package of fs.readdirSync(packagesDir)) { - const packageDir = path.join(packagesDir, package); - if (fs.statSync(packageDir).isDirectory() && isClientPackage(packageDir)) { - clientPackagesPaths.push(packageDir); - } - } - return clientPackagesPaths; -} - -function isClientPackage(directory) { - const baseName = path.parse(directory).base; - if (!clientNameRegex.test(baseName)) return false; - const clientFiles = [ - {base: 'commands', isFile: false}, - {base: 'model', isFile: false}, - {base: 'types', isFile: false}, - {base: `(\\w+)Client\\.ts`, isFile: true}, - {base: `(\\w+)Configuration\\.ts`, isFile: true}, - {base: `index\\.ts`, isFile: true}, - ] - try { - const files = fs.readdirSync(directory); - for (const clientFilePattern of clientFiles) { - const matchedFile = arrayFindPattern(files, clientFilePattern.base); - if ( - !matchedFile && - fs.statSync(path.join(directory, matchedFile)).isFile() !== clientFilePattern.isFile - ) { - return false; - } - - } - } catch(e) { - return false; - } - return true; -} - -function arrayFindPattern(array, pattern) { - return array.find((item) => { - const matched = new RegExp(pattern).exec(item); - //RegExp.exec() returns null if no matched - return Boolean(matched) - }) -} - -function generateClient(models, runtime) { - const command = `node packages/package-generator/build/cli.js client --m ${models.service} ${models.smoke ? `--s ${models.smoke}` : ''} --r ${runtime}`; - const packageRoot = path.dirname(__dirname); - const log = execSync(command, {cwd: packageRoot}); - console.info(log.toString()); -} - -module.exports.clientNameRegex = clientNameRegex; diff --git a/scripts/utils/ModelFinder.js b/scripts/utils/ModelFinder.js deleted file mode 100644 index 8a1e26f179c9..000000000000 --- a/scripts/utils/ModelFinder.js +++ /dev/null @@ -1,61 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const clientNameRegex = require('./constants').clientNameRegex; -const clientModuleIdentifier = require('../../packages/package-generator/build/clientModuleIdentifier').clientModuleIdentifier; - -class ServiceModelFinder { - constructor(models) { - this.latestModels = this.fetchLatestModels(models); - this.loadedModelCache = new Map(); - } - - fetchLatestModels(modelsDir) { - const serviceModelDirs = []; - for (const modelName of fs.readdirSync(modelsDir)) { - const modelDir = path.join(modelsDir, modelName); - if (!fs.statSync(modelDir).isDirectory()) continue; - const versions = fs.readdirSync(modelDir).filter( - version => fs.statSync(path.join(modelDir, version)).isDirectory() - ); - if (!versions || versions.length === 0) { - throw new Error(`No api version found in ${modelDir}`); - } - const latestVersion = versions.sort().reverse()[0]; - const versionDir = path.join(modelDir, latestVersion); - const serviceModels = fs.readdirSync(versionDir); - if (serviceModels.find(modelName => modelName === 'service-2.json')) { - serviceModelDirs.push(versionDir); - } - } - return serviceModelDirs; - } - - /** - * Fetch the directory of model and smoke test for given service name. - * @param {string} service service client package name. like: client-sqs-node - * @returns {object} looks like {service: string, smoke: string}; - */ - findModelsForService(packageName) { - const [_, service, runtime] = clientNameRegex.exec(packageName); - if (this.loadedModelCache.has(`client-${service}`)) { - return this.loadedModelCache.get(`client-${service}`); - } - for (const latestModel of this.latestModels.slice(this.loadedModelCache.size)) { - const modelDir = path.join(latestModel, 'service-2.json'); - const smokeDir = path.join(latestModel, 'smoke.json'); - const {metadata} = JSON.parse(fs.readFileSync(modelDir).toString()); - const universalClientName = clientModuleIdentifier(metadata); - const loadedModel = {service: modelDir}; - if (fs.existsSync(smokeDir)) { - loadedModel.smoke = smokeDir; - } - this.loadedModelCache.set(universalClientName, loadedModel); - if (universalClientName === `client-${service}`) { - return loadedModel; - } - } - throw new Error(`No model found for ${packageName}`); - } -} - -module.exports.ServiceModelFinder = ServiceModelFinder; \ No newline at end of file diff --git a/scripts/utils/constants.js b/scripts/utils/constants.js deleted file mode 100644 index c953781078da..000000000000 --- a/scripts/utils/constants.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - clientNameRegex: /^client-(\w+-?\w+)-(node|browser|universal)$/ -} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 354f02b978bd..1fc0ca55aa86 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1464,6 +1464,13 @@ dependencies: "@types/node" "*" +"@types/fs-extra@^8.0.1": + version "8.0.1" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-8.0.1.tgz#a2378d6e7e8afea1564e44aafa2e207dadf77686" + integrity sha512-J00cVDALmi/hJOYsunyT52Hva5TnJeKP5yd1r+mH/ZU0mbYZflR0Z5kw5kITtKTRYMhm1JMClOFYdHnQszEvqw== + dependencies: + "@types/node" "*" + "@types/glob@*", "@types/glob@^7.1.1": version "7.1.1" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"