From 5416bfdf7dda04bbd5d42c951284c9494e55115d Mon Sep 17 00:00:00 2001 From: TJ Higgins Date: Fri, 17 May 2019 16:26:05 -0400 Subject: [PATCH 1/6] Add architect push cmd. Added Listr for user feedback on cmds. --- src/commands/install.ts | 68 +++++++++++++++++++------------ src/commands/push.ts | 77 +++++++++++++++++++++++++++++++++++ src/common/protoc-executor.ts | 37 ++++++++++------- src/common/service-config.ts | 56 +++++++++++++++++++++++-- 4 files changed, 194 insertions(+), 44 deletions(-) create mode 100644 src/commands/push.ts diff --git a/src/commands/install.ts b/src/commands/install.ts index fd87f1be7..88b5bfaab 100644 --- a/src/commands/install.ts +++ b/src/commands/install.ts @@ -1,5 +1,6 @@ import { flags } from '@oclif/command'; import chalk from 'chalk'; +import * as Listr from 'listr'; import * as path from 'path'; import Command from '../base'; @@ -8,6 +9,7 @@ import ServiceConfig from '../common/service-config'; const _info = chalk.blue; const _error = chalk.red; +const _success = chalk.green; export default class Install extends Command { static description = 'Install dependencies of the current service'; @@ -27,39 +29,51 @@ export default class Install extends Command { static args = []; async run() { - try { - const { flags } = this.parse(Install); - let process_path = process.cwd(); - if (flags.prefix) { - process_path = path.isAbsolute(flags.prefix) ? - flags.prefix : - path.join(process_path, flags.prefix); - } - this.installDependencies(process_path); - } catch (error) { - this.error(_error(error.message)); + const { flags } = this.parse(Install); + let process_path = process.cwd(); + if (flags.prefix) { + process_path = path.isAbsolute(flags.prefix) ? + flags.prefix : + path.join(process_path, flags.prefix); } + + const tasks = new Listr(await this.getTasks(process_path, flags.recursive), { concurrent: 3 }); + await tasks.run(); + this.log(_success('Installed')); } - installDependencies(service_path: string) { - const { flags } = this.parse(Install); - const service_config = ServiceConfig.loadFromPath(service_path); - this.log(`Installing dependencies for ${_info(service_config.name)}`); + async getTasks(root_service_path: string, recursive: boolean): Promise { + const tasks: Listr.ListrTask[] = []; + const dependencies = await ServiceConfig.getDependencies(root_service_path, recursive); + dependencies.forEach(dependency => { + const sub_tasks: Listr.ListrTask[] = []; - // Install all dependencies - Object.keys(service_config.dependencies).forEach((dependency_name: string) => { - if (service_config.dependencies.hasOwnProperty(dependency_name)) { - const dependency_identifier = service_config.dependencies[dependency_name]; - const dependency_path = ServiceConfig.parsePathFromDependencyIdentifier(dependency_identifier, service_path); - ProtocExecutor.execute(dependency_path, service_path, service_config.language); - if (flags.recursive) { - this.installDependencies(dependency_path); - } + if (dependency.service_config.proto) { + sub_tasks.push({ + title: _info(dependency.service_config.name), + task: () => { + return ProtocExecutor.execute(dependency.service_path, dependency.service_path, dependency.service_config.language); + } + }); } + + dependency.dependencies.forEach(sub_dependency => { + sub_tasks.push({ + title: _info(sub_dependency.service_config.name), + task: () => { + return ProtocExecutor.execute(sub_dependency.service_path, dependency.service_path, dependency.service_config.language); + } + }); + }); + + tasks.push({ + title: `Installing dependencies for ${_info(dependency.service_config.name)}`, + task: () => { + return new Listr(sub_tasks, { concurrent: 2 }); + } + }); }); - if (service_config.proto) { - ProtocExecutor.execute(service_path, service_path, service_config.language); - } + return tasks; } } diff --git a/src/commands/push.ts b/src/commands/push.ts new file mode 100644 index 000000000..38284ccae --- /dev/null +++ b/src/commands/push.ts @@ -0,0 +1,77 @@ +import { flags } from '@oclif/command'; +import chalk from 'chalk'; +import { execSync } from 'child_process'; +import * as Listr from 'listr'; +import * as path from 'path'; +import * as url from 'url'; + +import Command from '../base'; +import ServiceConfig from '../common/service-config'; + +import Build from './build'; + +const _info = chalk.blue; +const _error = chalk.red; +const _success = chalk.green; + +export default class Push extends Command { + static description = 'Push service(s) to a registry'; + + static flags = { + help: flags.help({ char: 'h' }), + tag: flags.string({ + char: 't', + required: false, + description: 'Name and optionally a tag in the ‘name:tag’ format' + }), + recursive: flags.boolean({ + char: 'r', + default: false, + description: 'Whether or not to build images for the cited dependencies' + }) + }; + + static args = [ + { + name: 'context', + description: 'Path to the service to build' + } + ]; + + async run() { + const { args, flags } = this.parse(Push); + if (flags.recursive && flags.tag) { + this.error(_error('Cannot specify tag for recursive pushes')); + } + + let root_service_path = process.cwd(); + if (args.context) { + root_service_path = path.resolve(args.context); + } + + const tasks = new Listr(await this.getTasks(root_service_path, flags.recursive, flags.tag), { concurrent: 2 }); + await tasks.run(); + this.log(_success('Pushed')); + } + + async getTasks(service_path: string, recursive: boolean, tag?: string): Promise { + const dependencies = await ServiceConfig.getDependencies(service_path, recursive); + const tasks: Listr.ListrTask[] = []; + dependencies.forEach(dependency => { + tasks.push({ + title: _info(`Pushing docker image for ${dependency.service_config.name}`), + task: async () => { + await this.pushImage(dependency.service_path, dependency.service_config, tag); + } + }); + }); + return tasks; + } + + async pushImage(service_path: string, service_config: ServiceConfig, tag?: string) { + await Build.run([service_path]); + const tag_name = tag || `architect-${service_config.name}`; + const repository_name = url.resolve('localhost:8081/', tag_name); + execSync(`docker push ${repository_name}`); + } +} diff --git a/src/common/protoc-executor.ts b/src/common/protoc-executor.ts index f66138531..4407a8e3d 100644 --- a/src/common/protoc-executor.ts +++ b/src/common/protoc-executor.ts @@ -1,5 +1,5 @@ -import {execSync} from 'child_process'; -import {copyFileSync, existsSync, mkdirSync, realpathSync, writeFileSync} from 'fs'; +import { exec, execSync } from 'child_process'; +import { copyFileSync, existsSync, mkdirSync, realpathSync, writeFileSync } from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -14,7 +14,7 @@ namespace ProtocExecutor { } }; - export const execute = (dependency_path: string, target_path: string, target_language: SUPPORTED_LANGUAGES): void => { + export const execute = async (dependency_path: string, target_path: string, target_language: SUPPORTED_LANGUAGES): Promise => { const dependency_config = ServiceConfig.loadFromPath(dependency_path); if (!dependency_config.proto) { throw new Error(`${dependency_config.name} has no .proto file configured.`); @@ -43,17 +43,26 @@ namespace ProtocExecutor { const mount_dirname = '/opt/protoc'; const mounted_proto_path = path.posix.join(mount_dirname, ServiceConfig.convertServiceNameToFolderName(dependency_config.name), dependency_config.proto); - execSync([ - 'docker', 'run', - '-v', `${target_path}:/defs`, - '-v', `${tmpRoot}:${mount_dirname}`, - '--user', process.platform === 'win32' ? '1000:1000' : '$(id -u):$(id -g)', // TODO figure out correct user for windows - 'architectio/protoc-all', - '-f', `${mounted_proto_path}`, - '-i', mount_dirname, - '-l', target_language, - '-o', MANAGED_PATHS.DEPENDENCY_STUBS_DIRECTORY - ].join(' '), {stdio: 'ignore'}); + + // execSync caused the output logs to hang + await new Promise(resolve => { + const thread = exec([ + 'docker', 'run', + '-v', `${target_path}:/defs`, + '-v', `${tmpRoot}:${mount_dirname}`, + '--user', process.platform === 'win32' ? '1000:1000' : '$(id -u):$(id -g)', // TODO figure out correct user for windows + 'architectio/protoc-all', + '-f', `${mounted_proto_path}`, + '-i', mount_dirname, + '-l', target_language, + '-o', MANAGED_PATHS.DEPENDENCY_STUBS_DIRECTORY + ].join(' ')); + + thread.on('close', () => { + resolve(); + }); + }); + execSync(`rm -rf ${tmpDir}`); _postHooks(stub_directory, target_language); diff --git a/src/common/service-config.ts b/src/common/service-config.ts index 58477814b..b7fb29e55 100644 --- a/src/common/service-config.ts +++ b/src/common/service-config.ts @@ -3,7 +3,14 @@ import * as path from 'path'; import MANAGED_PATHS from './managed-paths'; import SUPPORTED_LANGUAGES from './supported-languages'; -import {SemvarValidator} from './validation-utils'; +import { SemvarValidator } from './validation-utils'; + +interface ServiceDependency { + service_path: string; + service_config: ServiceConfig; + + dependencies: ServiceDependency[]; +} export default class ServiceConfig { static _require(path: string) { @@ -43,6 +50,49 @@ export default class ServiceConfig { .setLanguage(configJSON.language); } + static async getDependencies(root_service_path: string, recursive: boolean): Promise { + const service_dependencies: ServiceDependency[] = []; + const services_map: any = {}; + + const root_service_config = ServiceConfig.loadFromPath(root_service_path!); + services_map[root_service_path] = { + service_path: root_service_path!, + service_config: root_service_config, + dependencies: [] + }; + const queue = [services_map[root_service_path]]; + + while (queue.length > 0) { + const service_dependency = queue.shift()!; + const service_config = service_dependency.service_config; + service_dependencies.push(service_dependency); + + if (recursive) { + const dependency_names = Object.keys(service_config.dependencies); + for (let dependency_name of dependency_names) { + const dependency_path = ServiceConfig.parsePathFromDependencyIdentifier( + service_config.dependencies[dependency_name], + service_dependency.service_path + ); + + // Handle circular deps + if (!services_map[dependency_path]) { + const dependency_config = ServiceConfig.loadFromPath(dependency_path!); + services_map[dependency_path] = { + service_path: dependency_path, + service_config: dependency_config, + dependencies: [] + }; + queue.push(services_map[dependency_path]); + } + service_dependency.dependencies.push(services_map[dependency_path]); + } + } + } + + return service_dependencies; + } + static convertServiceNameToFolderName(service_name: string): string { return service_name.replace('-', '_'); } @@ -53,7 +103,7 @@ export default class ServiceConfig { keywords: string[]; author: string; license: string; - dependencies: {[s: string]: string}; + dependencies: { [s: string]: string }; proto?: string; main: string; language: SUPPORTED_LANGUAGES; @@ -117,7 +167,7 @@ export default class ServiceConfig { return this; } - setDependencies(dependencies: {[s: string]: string}) { + setDependencies(dependencies: { [s: string]: string }) { this.dependencies = dependencies; return this; } From bb5fd82472d682db02441013646fb00291654705 Mon Sep 17 00:00:00 2001 From: TJ Higgins Date: Sat, 18 May 2019 16:04:48 -0400 Subject: [PATCH 2/6] Add AppConfig to base Command --- .gitignore | 1 + package-lock.json | 49 ++++++++++++++++++++++++++++++++++++++++ package.json | 4 ++++ src/app-config.ts | 52 +++++++++++++++++++++++++++++++++++++++++++ src/base.ts | 12 ++++++++-- src/commands/login.ts | 6 ++--- src/commands/push.ts | 2 +- 7 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 src/app-config.ts diff --git a/.gitignore b/.gitignore index 2b1ffedf5..a26b75de6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ node_modules .idea architect_services oclif.manifest.json +*.env diff --git a/package-lock.json b/package-lock.json index 5a2fd2a29..4257ac095 100644 --- a/package-lock.json +++ b/package-lock.json @@ -458,6 +458,14 @@ "@types/node": "*" } }, + "@types/dotenv": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@types/dotenv/-/dotenv-6.1.1.tgz", + "integrity": "sha512-ftQl3DtBvqHl9L16tpqqzA4YzCSXZfi7g8cQceTz5rOlYtk/IZbFjAv3mLOQlNIgOaylCQWQoBdDQHPgEBJPHg==", + "requires": { + "@types/node": "*" + } + }, "@types/express": { "version": "4.16.1", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.16.1.tgz", @@ -520,6 +528,11 @@ "@types/through": "*" } }, + "@types/joi": { + "version": "14.3.3", + "resolved": "https://registry.npmjs.org/@types/joi/-/joi-14.3.3.tgz", + "integrity": "sha512-6gAT/UkIzYb7zZulAbcof3lFxpiD5EI6xBeTvkL1wYN12pnFQ+y/+xl9BvnVgxkmaIDN89xWhGZLD9CvuOtZ9g==" + }, "@types/keytar": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@types/keytar/-/keytar-4.4.0.tgz", @@ -1942,6 +1955,11 @@ "sentence-case": "^1.1.2" } }, + "dotenv": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.0.0.tgz", + "integrity": "sha512-30xVGqjLjiUOArT4+M5q9sYdvuR4riM6yK9wMcas9Vbp6zZa+ocC9dp6QoftuhTPhFAiLK/0C5Ni2nou/Bk8lg==" + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -2791,6 +2809,11 @@ "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", "dev": true }, + "hoek": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-6.1.3.tgz", + "integrity": "sha512-YXXAAhmF9zpQbC7LEcREFtXfGq5K1fmd+4PHkBq8NUqmzW3G+Dq10bI/i0KucLRwss3YYFQ0fSfoxBZYiGUqtQ==" + }, "hosted-git-info": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", @@ -3205,6 +3228,14 @@ "buffer-alloc": "^1.2.0" } }, + "isemail": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/isemail/-/isemail-3.2.0.tgz", + "integrity": "sha512-zKqkK+O+dGqevc93KNsbZ/TqTUFd46MwWjYOoMrjIMZ51eU7DtQG3Wmd9SQQT7i7RVnuTPEiYEWHU3MSbxC1Tg==", + "requires": { + "punycode": "2.x.x" + } + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -3260,6 +3291,16 @@ "is-object": "^1.0.1" } }, + "joi": { + "version": "14.3.1", + "resolved": "https://registry.npmjs.org/joi/-/joi-14.3.1.tgz", + "integrity": "sha512-LQDdM+pkOrpAn4Lp+neNIFV3axv1Vna3j38bisbQhETPMANYRbFJFUyOZcOClYvM/hppMhGWuKSFEK9vjrB+bQ==", + "requires": { + "hoek": "6.x.x", + "isemail": "3.x.x", + "topo": "3.x.x" + } + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -7162,6 +7203,14 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "topo": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/topo/-/topo-3.0.3.tgz", + "integrity": "sha512-IgpPtvD4kjrJ7CRA3ov2FhWQADwv+Tdqbsf1ZnPUSAtCJ9e1Z44MmoSGDXGk4IppoZA7jd/QRkNddlLJWlUZsQ==", + "requires": { + "hoek": "6.x.x" + } + }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", diff --git a/package.json b/package.json index 7a15529bc..f67a305c8 100644 --- a/package.json +++ b/package.json @@ -15,14 +15,18 @@ "@oclif/errors": "^1.2.2", "@oclif/plugin-help": "^2.1.2", "@types/auth0": "^2.9.11", + "@types/dotenv": "^6.1.1", "@types/fs-extra": "^7.0.0", + "@types/joi": "^14.3.3", "@types/keytar": "^4.4.0", "@types/listr": "^0.14.0", "@types/request": "^2.48.1", "auth0": "^2.17.0", "chalk": "^2.4.1", + "dotenv": "^8.0.0", "fs-extra": "^8.0.1", "inquirer": "^6.2.0", + "joi": "^14.3.1", "keytar": "^4.6.0", "listr": "^0.14.3", "oclif": "^1.12.5", diff --git a/src/app-config.ts b/src/app-config.ts new file mode 100644 index 000000000..d684c6170 --- /dev/null +++ b/src/app-config.ts @@ -0,0 +1,52 @@ +import * as dotenv from 'dotenv'; +import * as Joi from 'joi'; + +const CONFIG_SCHEMA: Joi.ObjectSchema = Joi.object({ + OAUTH_DOMAIN: Joi + .string() + .hostname() + .default('architect.auth0.com'), + OAUTH_CLIENT_ID: Joi + .string() + .default('X9U08B2hg6QEmRUIFKZoCSqgNBmAM6aU'), + DEFAULT_REGISTRY_HOST: Joi + .string() + .uri() + .default('registry.architect.io'), + API_HOST: Joi + .string() + .uri() + .default('https://api.architect.io') +}); + +const validate_config = ( + input_config: { [key: string]: any }, +): { [key: string]: string } => { + const { error, value: validatedEnvConfig } = Joi.validate( + input_config, + CONFIG_SCHEMA, + { stripUnknown: true }, + ); + if (error) { + throw new Error(`Config validation error: ${error.message}`); + } + return validatedEnvConfig; +}; + +export class AppConfig { + readonly oauth_domain: string; + readonly oauth_client_id: string; + readonly default_registry_host: string; + readonly api_host: string; + + constructor() { + // Load environment params from a .env file if found + dotenv.config(); + + const app_env = validate_config(process.env); + this.oauth_domain = app_env.OAUTH_DOMAIN; + this.oauth_client_id = app_env.OAUTH_CLIENT_ID; + this.default_registry_host = app_env.DEFAULT_REGISTRY_HOST; + this.api_host = app_env.API_HOST; + } +} diff --git a/src/base.ts b/src/base.ts index 75d4fa7a7..2f8b6e736 100644 --- a/src/base.ts +++ b/src/base.ts @@ -3,11 +3,15 @@ import * as keytar from 'keytar'; import * as request from 'request'; import * as url from 'url'; +import { AppConfig } from './app-config'; + export default abstract class extends Command { + app_config!: AppConfig; architect!: ArchitectClient; async init() { - this.architect = new ArchitectClient(); + this.app_config = new AppConfig(); + this.architect = new ArchitectClient(this.app_config.api_host); } styled_json(obj: object) { @@ -17,7 +21,11 @@ export default abstract class extends Command { } class ArchitectClient { - private readonly domain = 'https://api.architect.io'; + private readonly domain: string; + + constructor(domain: string) { + this.domain = domain; + } async get(path: string) { return this.request('GET', path); diff --git a/src/commands/login.ts b/src/commands/login.ts index 2b8edc9ae..13f1ce834 100644 --- a/src/commands/login.ts +++ b/src/commands/login.ts @@ -53,8 +53,8 @@ export default class Login extends Command { async login(username: string, password: string) { const auth0 = new AuthenticationClient({ - domain: 'architect.auth0.com', - clientId: 'X9U08B2hg6QEmRUIFKZoCSqgNBmAM6aU' + domain: this.app_config.oauth_domain, + clientId: this.app_config.oauth_client_id }); auth0.passwordGrant({ @@ -66,7 +66,7 @@ export default class Login extends Command { this.log(_error(err.message)); } else { const auth = JSON.stringify(authResult); - const registry_domain = 'registry.architect.io'; + const registry_domain = this.app_config.default_registry_host; const res = spawnSync('docker', ['login', registry_domain, '-u', username, '--password-stdin'], { input: auth }); if (res.status === 0) { await keytar.setPassword('architect.io', username, auth); diff --git a/src/commands/push.ts b/src/commands/push.ts index 38284ccae..15c86af09 100644 --- a/src/commands/push.ts +++ b/src/commands/push.ts @@ -71,7 +71,7 @@ export default class Push extends Command { async pushImage(service_path: string, service_config: ServiceConfig, tag?: string) { await Build.run([service_path]); const tag_name = tag || `architect-${service_config.name}`; - const repository_name = url.resolve('localhost:8081/', tag_name); + const repository_name = url.resolve(`${this.app_config.default_registry_host}/`, tag_name); execSync(`docker push ${repository_name}`); } } From 3a4700bbf8f5256715eae9902e6ee633692ccd68 Mon Sep 17 00:00:00 2001 From: TJ Higgins Date: Fri, 24 May 2019 07:22:50 +0800 Subject: [PATCH 3/6] Expose getTasks methods on commands for chaining/subtasks. Added getUser to architect client. --- src/base.ts | 21 ++++++++- src/commands/build.ts | 91 ++++++++++++++++++++---------------- src/commands/install.ts | 31 ++++++------ src/commands/push.ts | 38 ++++++++++----- src/commands/start.ts | 71 +++++++++++++++------------- src/common/service-config.ts | 2 +- test/commands/build.test.ts | 18 +++---- 7 files changed, 157 insertions(+), 115 deletions(-) diff --git a/src/base.ts b/src/base.ts index 2f8b6e736..b35b32229 100644 --- a/src/base.ts +++ b/src/base.ts @@ -20,6 +20,16 @@ export default abstract class extends Command { } } +class UserEntity { + readonly access_token: string; + readonly username: string; + + constructor(partial: { [key: string]: string }) { + this.access_token = partial.access_token; + this.username = partial['https://architect.io/username']; + } +} + class ArchitectClient { private readonly domain: string; @@ -27,6 +37,14 @@ class ArchitectClient { this.domain = domain; } + async getUser() { + const credentials = await keytar.findCredentials('architect.io'); + if (credentials.length === 0) { + throw Error('denied: `architect login` required'); + } + return new UserEntity(JSON.parse(credentials[0].password)); + } + async get(path: string) { return this.request('GET', path); } @@ -48,7 +66,8 @@ class ArchitectClient { if (credentials.length === 0) { throw Error('denied: `architect login` required'); } - const access_token = JSON.parse(credentials[0].password).access_token; + const user = await this.getUser(); + const access_token = user.access_token; const options = { url: url.resolve(this.domain, path), diff --git a/src/commands/build.ts b/src/commands/build.ts index 4c98aa23c..96d39c648 100644 --- a/src/commands/build.ts +++ b/src/commands/build.ts @@ -1,6 +1,7 @@ import { flags } from '@oclif/command'; import chalk from 'chalk'; -import { execSync } from 'child_process'; +import { exec } from 'child_process'; +import * as Listr from 'listr'; import * as path from 'path'; import Command from '../base'; @@ -11,7 +12,6 @@ import Install from './install'; const _info = chalk.blue; const _error = chalk.red; -const _success = chalk.green; export default class Build extends Command { static description = `Create an ${MANAGED_PATHS.ARCHITECT_JSON} file for a service`; @@ -37,53 +37,62 @@ export default class Build extends Command { } ]; - async run() { - const { args } = this.parse(Build); - let root_service_path = process.cwd(); - if (args.context) { - root_service_path = path.resolve(args.context); - } + static async getTasks(service_path: string, tag?: string, recursive?: boolean): Promise { + const dependencies = await ServiceConfig.getDependencies(service_path, recursive); + const tasks: Listr.ListrTask[] = []; - try { - await this.buildImage(root_service_path); - } catch (err) { - this.error(_error(err.message)); - } + dependencies.forEach(dependency => { + tasks.push({ + title: `Building docker image for ${_info(dependency.service_config.name)}`, + task: async () => { + const install_tasks = await Install.getTasks(dependency.service_path); + const build_task = { + title: 'Building', + task: async () => { + await Build.buildImage(dependency.service_path, dependency.service_config, tag); + } + }; + return new Listr(install_tasks.concat([build_task])); + } + }); + }); + return tasks; } - async buildImage(service_path: string) { - const { flags } = this.parse(Build); - const service_config = ServiceConfig.loadFromPath(service_path); + static async buildImage(service_path: string, service_config: ServiceConfig, tag?: string) { + const dockerfile_path = path.join(__dirname, '../../Dockerfile'); + const tag_name = tag || `architect-${service_config.name}`; + + // execSync caused the output logs to hang + await new Promise(resolve => { + const thread = exec([ + 'docker', 'build', + '--compress', + '--build-arg', `SERVICE_LANGUAGE=${service_config.language}`, + '-t', tag_name, + '-f', dockerfile_path, + '--label', `architect.json='${JSON.stringify(service_config)}'`, + service_path + ].join(' ')); + thread.on('close', () => { + resolve(); + }); + }); + } + + async run() { + const { args, flags } = this.parse(Build); if (flags.recursive && flags.tag) { - this.error(_error('Cannot override tag for recursive builds')); + this.error(_error('Cannot specify tag for recursive builds')); } - if (flags.recursive) { - const dependency_names = Object.keys(service_config.dependencies); - for (let dependency_name of dependency_names) { - const dependency_path = ServiceConfig.parsePathFromDependencyIdentifier( - service_config.dependencies[dependency_name], - service_path - ); - await this.buildImage(dependency_path); - } + let root_service_path = process.cwd(); + if (args.context) { + root_service_path = path.resolve(args.context); } - this.log(_info(`Building docker image for ${service_config.name}`)); - const dockerfile_path = path.join(__dirname, '../../Dockerfile'); - await Install.run(['--prefix', service_path]); - const tag_name = flags.tag || `architect-${service_config.name}`; - - execSync([ - 'docker', 'build', - '--compress', - '--build-arg', `SERVICE_LANGUAGE=${service_config.language}`, - '-t', tag_name, - '-f', dockerfile_path, - '--label', `architect.json='${JSON.stringify(service_config)}'`, - service_path - ].join(' ')); - this.log(_success(`Successfully built image for ${service_config.name}`)); + const tasks = new Listr(await Build.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2 }); + await tasks.run(); } } diff --git a/src/commands/install.ts b/src/commands/install.ts index 88b5bfaab..a23da0918 100644 --- a/src/commands/install.ts +++ b/src/commands/install.ts @@ -8,8 +8,6 @@ import ProtocExecutor from '../common/protoc-executor'; import ServiceConfig from '../common/service-config'; const _info = chalk.blue; -const _error = chalk.red; -const _success = chalk.green; export default class Install extends Command { static description = 'Install dependencies of the current service'; @@ -28,21 +26,7 @@ export default class Install extends Command { static args = []; - async run() { - const { flags } = this.parse(Install); - let process_path = process.cwd(); - if (flags.prefix) { - process_path = path.isAbsolute(flags.prefix) ? - flags.prefix : - path.join(process_path, flags.prefix); - } - - const tasks = new Listr(await this.getTasks(process_path, flags.recursive), { concurrent: 3 }); - await tasks.run(); - this.log(_success('Installed')); - } - - async getTasks(root_service_path: string, recursive: boolean): Promise { + static async getTasks(root_service_path: string, recursive?: boolean): Promise { const tasks: Listr.ListrTask[] = []; const dependencies = await ServiceConfig.getDependencies(root_service_path, recursive); dependencies.forEach(dependency => { @@ -76,4 +60,17 @@ export default class Install extends Command { return tasks; } + + async run() { + const { flags } = this.parse(Install); + let process_path = process.cwd(); + if (flags.prefix) { + process_path = path.isAbsolute(flags.prefix) ? + flags.prefix : + path.join(process_path, flags.prefix); + } + + const tasks = new Listr(await Install.getTasks(process_path, flags.recursive), { concurrent: 3 }); + await tasks.run(); + } } diff --git a/src/commands/push.ts b/src/commands/push.ts index 15c86af09..1033a42df 100644 --- a/src/commands/push.ts +++ b/src/commands/push.ts @@ -1,6 +1,6 @@ import { flags } from '@oclif/command'; import chalk from 'chalk'; -import { execSync } from 'child_process'; +import { exec, execSync } from 'child_process'; import * as Listr from 'listr'; import * as path from 'path'; import * as url from 'url'; @@ -12,7 +12,6 @@ import Build from './build'; const _info = chalk.blue; const _error = chalk.red; -const _success = chalk.green; export default class Push extends Command { static description = 'Push service(s) to a registry'; @@ -49,29 +48,44 @@ export default class Push extends Command { root_service_path = path.resolve(args.context); } - const tasks = new Listr(await this.getTasks(root_service_path, flags.recursive, flags.tag), { concurrent: 2 }); + const tasks = new Listr(await this.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2 }); await tasks.run(); - this.log(_success('Pushed')); } - async getTasks(service_path: string, recursive: boolean, tag?: string): Promise { - const dependencies = await ServiceConfig.getDependencies(service_path, recursive); + async getTasks(root_service_path: string, tag?: string, recursive?: boolean): Promise { + const dependencies = await ServiceConfig.getDependencies(root_service_path, recursive); const tasks: Listr.ListrTask[] = []; dependencies.forEach(dependency => { tasks.push({ - title: _info(`Pushing docker image for ${dependency.service_config.name}`), + title: `Pushing docker image for ${_info(dependency.service_config.name)}`, task: async () => { - await this.pushImage(dependency.service_path, dependency.service_config, tag); + const build_tasks = await Build.getTasks(dependency.service_path, tag); + const push_task = { + title: 'Pushing', + task: async () => { + await this.pushImage(dependency.service_config, tag); + } + }; + return new Listr(build_tasks.concat([push_task])); } }); }); return tasks; } - async pushImage(service_path: string, service_config: ServiceConfig, tag?: string) { - await Build.run([service_path]); + async pushImage(service_config: ServiceConfig, tag?: string) { const tag_name = tag || `architect-${service_config.name}`; - const repository_name = url.resolve(`${this.app_config.default_registry_host}/`, tag_name); - execSync(`docker push ${repository_name}`); + + const user = await this.architect.getUser(); + const repository_name = url.resolve(`${this.app_config.default_registry_host}/`, `${user.username}/${tag_name}`); + execSync(`docker tag ${tag_name} ${repository_name}`); + // execSync caused the output logs to hang + await new Promise(resolve => { + const thread = exec(`docker push ${repository_name}`); + + thread.on('close', () => { + resolve(); + }); + }); } } diff --git a/src/commands/start.ts b/src/commands/start.ts index 5fd8b3b68..c4c35c4d1 100644 --- a/src/commands/start.ts +++ b/src/commands/start.ts @@ -1,6 +1,7 @@ import { flags } from '@oclif/command'; import chalk from 'chalk'; import { ChildProcess, spawn } from 'child_process'; +import * as Listr from 'listr'; import * as path from 'path'; import * as readline from 'readline'; @@ -17,24 +18,53 @@ export default class Start extends Command { static description = 'Start the service locally'; static flags = { - help: flags.help({ char: 'h' }), - config_path: flags.string({ - char: 'c', - description: 'Path to a config file containing locations of ' + - 'each service in the application' - }) + help: flags.help({ char: 'h' }) }; + static args = [ + { + name: 'context', + description: 'Path to the service to build' + } + ]; + deployment_config: DeploymentConfig = {}; async run() { + const { args } = this.parse(Start); // Ensures that the python launcher doesn't buffer output and cause hanging process.env.PYTHONUNBUFFERED = 'true'; process.env.PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION = 'python'; - const service_path = process.cwd(); - await this.startService(service_path, true); - this.exit(); + let service_path = process.cwd(); + if (args.context) { + service_path = path.resolve(args.context); + } + + const tasks = new Listr(await this.getTasks(service_path), { renderer: 'verbose' }); + await tasks.run(); + } + + async getTasks(root_service_path: string): Promise { + const recursive = true; + const dependencies = await ServiceConfig.getDependencies(root_service_path, recursive); + dependencies.reverse(); + const tasks: Listr.ListrTask[] = []; + dependencies.forEach(dependency => { + tasks.push({ + title: `Deploying ${_info(dependency.service_config.name)}`, + task: async () => { + const isServiceRunning = await this.isServiceRunning(dependency.service_config.name); + if (isServiceRunning) { + this.log(`${_info(dependency.service_config.name)} already deployed`); + return; + } + const is_root_service = root_service_path === dependency.service_path; + await this.executeLauncher(dependency.service_path, dependency.service_config, is_root_service); + } + }); + }); + return tasks; } async isServiceRunning(service_name: string) { @@ -69,29 +99,6 @@ export default class Start extends Command { }; } - async startService( - service_path: string, - is_root_service = false - ): Promise { - const service_config = ServiceConfig.loadFromPath(service_path); - const dependency_names = Object.keys(service_config.dependencies); - for (let dependency_name of dependency_names) { - const dependency_path = ServiceConfig.parsePathFromDependencyIdentifier( - service_config.dependencies[dependency_name] - ); - await this.startService(dependency_path); - } - - const isServiceRunning = await this.isServiceRunning(service_config.name); - if (isServiceRunning) { - this.log(`${_info(service_config.name)} already deployed`); - return; - } - - this.log(`Deploying ${_info(service_config.name)}`); - await this.executeLauncher(service_path, service_config, is_root_service); - } - async executeLauncher( service_path: string, service_config: ServiceConfig, diff --git a/src/common/service-config.ts b/src/common/service-config.ts index b7fb29e55..fedfa2ab9 100644 --- a/src/common/service-config.ts +++ b/src/common/service-config.ts @@ -50,7 +50,7 @@ export default class ServiceConfig { .setLanguage(configJSON.language); } - static async getDependencies(root_service_path: string, recursive: boolean): Promise { + static async getDependencies(root_service_path: string, recursive?: boolean): Promise { const service_dependencies: ServiceDependency[] = []; const services_map: any = {}; diff --git a/test/commands/build.test.ts b/test/commands/build.test.ts index 2dd21b5b5..9876f4f1a 100644 --- a/test/commands/build.test.ts +++ b/test/commands/build.test.ts @@ -1,16 +1,15 @@ -import {expect, test} from '@oclif/test'; -import {execSync} from 'child_process'; +import { expect, test } from '@oclif/test'; +import { execSync } from 'child_process'; describe('build', () => { test .stdout() - .timeout(7000) + .timeout(10000) .command(['build', './test/calculator-example/addition-service/']) .it('builds docker image', ctx => { - const {stdout} = ctx; + const { stdout } = ctx; expect(stdout).to.contain('Building docker image for addition-service'); expect(stdout).to.contain('Installing dependencies for addition-service'); - expect(stdout).to.contain('Successfully built image for addition-service'); const docker_images = execSync('docker images | grep architect-addition-service'); expect(docker_images.toString()).to.contain('architect-addition-service'); @@ -18,13 +17,12 @@ describe('build', () => { test .stdout() - .timeout(7000) + .timeout(10000) .command(['build', '--tag', 'tag-override', './test/calculator-example/addition-service/']) .it('allows tag overrides', ctx => { - const {stdout} = ctx; + const { stdout } = ctx; expect(stdout).to.contain('Building docker image for addition-service'); expect(stdout).to.contain('Installing dependencies for addition-service'); - expect(stdout).to.contain('Successfully built image for addition-service'); const docker_images = execSync('docker images | grep tag-override'); expect(docker_images.toString()).to.contain('tag-override'); @@ -35,13 +33,11 @@ describe('build', () => { .timeout(15000) .command(['build', '--recursive', './test/calculator-example/python-subtraction-service/']) .it('builds images recursively', ctx => { - const {stdout} = ctx; + const { stdout } = ctx; expect(stdout).to.contain('Building docker image for subtraction-service'); expect(stdout).to.contain('Installing dependencies for subtraction-service'); - expect(stdout).to.contain('Successfully built image for subtraction-service'); expect(stdout).to.contain('Building docker image for addition-service'); expect(stdout).to.contain('Installing dependencies for addition-service'); - expect(stdout).to.contain('Successfully built image for addition-service'); const docker_images = execSync('docker images | grep architect-'); expect(docker_images.toString()).to.contain('architect-subtraction-service'); From e42a6e56500901539ca2da7656c2b7017c6f05c9 Mon Sep 17 00:00:00 2001 From: TJ Higgins Date: Mon, 3 Jun 2019 08:27:44 -0400 Subject: [PATCH 4/6] Cleanup code by switching from promise exec to execa --- package-lock.json | 55 ++++++++++++++++++++++++++++++----- package.json | 2 ++ src/commands/build.ts | 27 +++++++---------- src/commands/push.ts | 13 ++------- src/common/protoc-executor.ts | 34 +++++++++------------- 5 files changed, 75 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4257ac095..0e1c9764f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -466,6 +466,14 @@ "@types/node": "*" } }, + "@types/execa": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@types/execa/-/execa-0.9.0.tgz", + "integrity": "sha512-mgfd93RhzjYBUHHV532turHC2j4l/qxsF/PbfDmprHDEUHmNZGlDn1CEsulGK3AfsPdhkWzZQT/S/k0UGhLGsA==", + "requires": { + "@types/node": "*" + } + }, "@types/express": { "version": "4.16.1", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.16.1.tgz", @@ -2092,18 +2100,36 @@ "dev": true }, "execa": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", - "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==", - "dev": true, + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", "requires": { "cross-spawn": "^6.0.0", - "get-stream": "^3.0.0", + "get-stream": "^4.0.0", "is-stream": "^1.1.0", "npm-run-path": "^2.0.0", "p-finally": "^1.0.0", "signal-exit": "^3.0.0", "strip-eof": "^1.0.0" + }, + "dependencies": { + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "requires": { + "pump": "^3.0.0" + } + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } } }, "expand-brackets": { @@ -4201,7 +4227,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", - "dev": true, "requires": { "path-key": "^2.0.0" } @@ -6015,6 +6040,21 @@ "ms": "^2.1.1" } }, + "execa": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", + "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==", + "dev": true, + "requires": { + "cross-spawn": "^6.0.0", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, "fs-extra": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-6.0.1.tgz", @@ -6938,8 +6978,7 @@ "strip-eof": { "version": "1.0.0", "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", - "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", - "dev": true + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" }, "strip-indent": { "version": "1.0.1", diff --git a/package.json b/package.json index f67a305c8..9056b8f0d 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@oclif/plugin-help": "^2.1.2", "@types/auth0": "^2.9.11", "@types/dotenv": "^6.1.1", + "@types/execa": "^0.9.0", "@types/fs-extra": "^7.0.0", "@types/joi": "^14.3.3", "@types/keytar": "^4.4.0", @@ -24,6 +25,7 @@ "auth0": "^2.17.0", "chalk": "^2.4.1", "dotenv": "^8.0.0", + "execa": "^1.0.0", "fs-extra": "^8.0.1", "inquirer": "^6.2.0", "joi": "^14.3.1", diff --git a/src/commands/build.ts b/src/commands/build.ts index 96d39c648..f51aa3657 100644 --- a/src/commands/build.ts +++ b/src/commands/build.ts @@ -1,6 +1,6 @@ import { flags } from '@oclif/command'; import chalk from 'chalk'; -import { exec } from 'child_process'; +import * as execa from 'execa'; import * as Listr from 'listr'; import * as path from 'path'; @@ -63,22 +63,15 @@ export default class Build extends Command { const dockerfile_path = path.join(__dirname, '../../Dockerfile'); const tag_name = tag || `architect-${service_config.name}`; - // execSync caused the output logs to hang - await new Promise(resolve => { - const thread = exec([ - 'docker', 'build', - '--compress', - '--build-arg', `SERVICE_LANGUAGE=${service_config.language}`, - '-t', tag_name, - '-f', dockerfile_path, - '--label', `architect.json='${JSON.stringify(service_config)}'`, - service_path - ].join(' ')); - - thread.on('close', () => { - resolve(); - }); - }); + await execa.shell([ + 'docker', 'build', + '--compress', + '--build-arg', `SERVICE_LANGUAGE=${service_config.language}`, + '-t', tag_name, + '-f', dockerfile_path, + '--label', `architect.json='${JSON.stringify(service_config)}'`, + service_path + ].join(' ')); } async run() { diff --git a/src/commands/push.ts b/src/commands/push.ts index 1033a42df..45feaf3e3 100644 --- a/src/commands/push.ts +++ b/src/commands/push.ts @@ -1,6 +1,6 @@ import { flags } from '@oclif/command'; import chalk from 'chalk'; -import { exec, execSync } from 'child_process'; +import * as execa from 'execa'; import * as Listr from 'listr'; import * as path from 'path'; import * as url from 'url'; @@ -78,14 +78,7 @@ export default class Push extends Command { const user = await this.architect.getUser(); const repository_name = url.resolve(`${this.app_config.default_registry_host}/`, `${user.username}/${tag_name}`); - execSync(`docker tag ${tag_name} ${repository_name}`); - // execSync caused the output logs to hang - await new Promise(resolve => { - const thread = exec(`docker push ${repository_name}`); - - thread.on('close', () => { - resolve(); - }); - }); + await execa.shell(`docker tag ${tag_name} ${repository_name}`); + await execa.shell(`docker push ${repository_name}`); } } diff --git a/src/common/protoc-executor.ts b/src/common/protoc-executor.ts index 4407a8e3d..e233cf755 100644 --- a/src/common/protoc-executor.ts +++ b/src/common/protoc-executor.ts @@ -1,4 +1,4 @@ -import { exec, execSync } from 'child_process'; +import * as execa from 'execa'; import { copyFileSync, existsSync, mkdirSync, realpathSync, writeFileSync } from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -44,26 +44,18 @@ namespace ProtocExecutor { const mount_dirname = '/opt/protoc'; const mounted_proto_path = path.posix.join(mount_dirname, ServiceConfig.convertServiceNameToFolderName(dependency_config.name), dependency_config.proto); - // execSync caused the output logs to hang - await new Promise(resolve => { - const thread = exec([ - 'docker', 'run', - '-v', `${target_path}:/defs`, - '-v', `${tmpRoot}:${mount_dirname}`, - '--user', process.platform === 'win32' ? '1000:1000' : '$(id -u):$(id -g)', // TODO figure out correct user for windows - 'architectio/protoc-all', - '-f', `${mounted_proto_path}`, - '-i', mount_dirname, - '-l', target_language, - '-o', MANAGED_PATHS.DEPENDENCY_STUBS_DIRECTORY - ].join(' ')); - - thread.on('close', () => { - resolve(); - }); - }); - - execSync(`rm -rf ${tmpDir}`); + await execa.shell([ + 'docker', 'run', + '-v', `${target_path}:/defs`, + '-v', `${tmpRoot}:${mount_dirname}`, + '--user', process.platform === 'win32' ? '1000:1000' : '$(id -u):$(id -g)', // TODO figure out correct user for windows + 'architectio/protoc-all', + '-f', `${mounted_proto_path}`, + '-i', mount_dirname, + '-l', target_language, + '-o', MANAGED_PATHS.DEPENDENCY_STUBS_DIRECTORY + ].join(' ')); + await execa.shell(`rm -rf ${tmpDir}`); _postHooks(stub_directory, target_language); }; From d87bac3a7a831a6229c5a1fdcbc6b4ca8e195423 Mon Sep 17 00:00:00 2001 From: TJ Higgins Date: Mon, 3 Jun 2019 10:32:52 -0400 Subject: [PATCH 5/6] Fix race condition with concurrent installs of service --- src/commands/build.ts | 7 ++++++- src/commands/install.ts | 11 ++++++++--- src/commands/push.ts | 7 ++++++- src/common/protoc-executor.ts | 30 +++++++++++++++--------------- src/common/service-config.ts | 6 ++++-- test/commands/build.test.ts | 8 ++++---- test/commands/install.test.ts | 11 ++++++----- 7 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/commands/build.ts b/src/commands/build.ts index f51aa3657..3e4120f8a 100644 --- a/src/commands/build.ts +++ b/src/commands/build.ts @@ -27,6 +27,10 @@ export default class Build extends Command { char: 'r', default: false, description: 'Whether or not to build images for the cited dependencies' + }), + verbose: flags.boolean({ + char: 'v', + description: 'Verbose log output' }) }; @@ -85,7 +89,8 @@ export default class Build extends Command { root_service_path = path.resolve(args.context); } - const tasks = new Listr(await Build.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2 }); + const renderer = flags.verbose ? 'verbose' : 'default'; + const tasks = new Listr(await Build.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2, renderer }); await tasks.run(); } } diff --git a/src/commands/install.ts b/src/commands/install.ts index a23da0918..b5f4ec133 100644 --- a/src/commands/install.ts +++ b/src/commands/install.ts @@ -16,11 +16,15 @@ export default class Install extends Command { help: flags.help({ char: 'h' }), prefix: flags.string({ char: 'p', - description: 'Path prefix indicating where the install command should execute from.' + description: 'Path prefix indicating where the install command should execute from' }), recursive: flags.boolean({ char: 'r', - description: 'Generate architect dependency files for all services in the dependency tree.' + description: 'Generate architect dependency files for all services in the dependency tree' + }), + verbose: flags.boolean({ + char: 'v', + description: 'Verbose log output' }) }; @@ -70,7 +74,8 @@ export default class Install extends Command { path.join(process_path, flags.prefix); } - const tasks = new Listr(await Install.getTasks(process_path, flags.recursive), { concurrent: 3 }); + const renderer = flags.verbose ? 'verbose' : 'default'; + const tasks = new Listr(await Install.getTasks(process_path, flags.recursive), { concurrent: 3, renderer }); await tasks.run(); } } diff --git a/src/commands/push.ts b/src/commands/push.ts index 45feaf3e3..c0087b03d 100644 --- a/src/commands/push.ts +++ b/src/commands/push.ts @@ -27,6 +27,10 @@ export default class Push extends Command { char: 'r', default: false, description: 'Whether or not to build images for the cited dependencies' + }), + verbose: flags.boolean({ + char: 'v', + description: 'Verbose log output' }) }; @@ -48,7 +52,8 @@ export default class Push extends Command { root_service_path = path.resolve(args.context); } - const tasks = new Listr(await this.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2 }); + const renderer = flags.verbose ? 'verbose' : 'default'; + const tasks = new Listr(await this.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2, renderer }); await tasks.run(); } diff --git a/src/common/protoc-executor.ts b/src/common/protoc-executor.ts index e233cf755..57ca00f51 100644 --- a/src/common/protoc-executor.ts +++ b/src/common/protoc-executor.ts @@ -19,35 +19,35 @@ namespace ProtocExecutor { if (!dependency_config.proto) { throw new Error(`${dependency_config.name} has no .proto file configured.`); } + const dependency_folder = ServiceConfig.convertServiceNameToFolderName(dependency_config.name); + const target_config = ServiceConfig.loadFromPath(target_path); + // Prevent race conditions when building the same service concurrently for different targets + const namespace = `${ServiceConfig.convertServiceNameToFolderName(target_config.name)}__${dependency_folder}`; // Make the folder to store dependency stubs - const stubs_directory = path.join(target_path, MANAGED_PATHS.DEPENDENCY_STUBS_DIRECTORY); - if (!existsSync(stubs_directory)) { - mkdirSync(stubs_directory); - } - - const stub_directory = path.join(stubs_directory, ServiceConfig.convertServiceNameToFolderName(dependency_config.name)); + const stub_directory = path.join(target_path, MANAGED_PATHS.DEPENDENCY_STUBS_DIRECTORY, dependency_folder); if (!existsSync(stub_directory)) { - mkdirSync(stub_directory); + mkdirSync(stub_directory, { recursive: true }); } - const tmpRoot = realpathSync(os.tmpdir()); - const tmpDir = path.join(tmpRoot, ServiceConfig.convertServiceNameToFolderName(dependency_config.name)); - if (!existsSync(tmpDir)) { - mkdirSync(tmpDir); + const tmp_root = realpathSync(os.tmpdir()); + const tmp_dir = path.join(tmp_root, namespace); + const tmp_dependency_dir = path.join(tmp_dir, dependency_folder); + if (!existsSync(tmp_dependency_dir)) { + mkdirSync(tmp_dependency_dir, { recursive: true }); } copyFileSync( path.join(dependency_path, dependency_config.proto), - path.join(tmpDir, dependency_config.proto) + path.join(tmp_dependency_dir, dependency_config.proto) ); const mount_dirname = '/opt/protoc'; - const mounted_proto_path = path.posix.join(mount_dirname, ServiceConfig.convertServiceNameToFolderName(dependency_config.name), dependency_config.proto); + const mounted_proto_path = path.posix.join(mount_dirname, dependency_folder, dependency_config.proto); await execa.shell([ 'docker', 'run', '-v', `${target_path}:/defs`, - '-v', `${tmpRoot}:${mount_dirname}`, + '-v', `${tmp_dir}:${mount_dirname}`, '--user', process.platform === 'win32' ? '1000:1000' : '$(id -u):$(id -g)', // TODO figure out correct user for windows 'architectio/protoc-all', '-f', `${mounted_proto_path}`, @@ -55,7 +55,7 @@ namespace ProtocExecutor { '-l', target_language, '-o', MANAGED_PATHS.DEPENDENCY_STUBS_DIRECTORY ].join(' ')); - await execa.shell(`rm -rf ${tmpDir}`); + await execa.shell(`rm -rf ${tmp_dir}`); _postHooks(stub_directory, target_language); }; diff --git a/src/common/service-config.ts b/src/common/service-config.ts index fedfa2ab9..5fec48f8e 100644 --- a/src/common/service-config.ts +++ b/src/common/service-config.ts @@ -67,7 +67,7 @@ export default class ServiceConfig { const service_config = service_dependency.service_config; service_dependencies.push(service_dependency); - if (recursive) { + if (root_service_path === service_dependency.service_path || recursive) { const dependency_names = Object.keys(service_config.dependencies); for (let dependency_name of dependency_names) { const dependency_path = ServiceConfig.parsePathFromDependencyIdentifier( @@ -83,7 +83,9 @@ export default class ServiceConfig { service_config: dependency_config, dependencies: [] }; - queue.push(services_map[dependency_path]); + if (recursive) { + queue.push(services_map[dependency_path]); + } } service_dependency.dependencies.push(services_map[dependency_path]); } diff --git a/test/commands/build.test.ts b/test/commands/build.test.ts index 9876f4f1a..55775d31b 100644 --- a/test/commands/build.test.ts +++ b/test/commands/build.test.ts @@ -5,7 +5,7 @@ describe('build', () => { test .stdout() .timeout(10000) - .command(['build', './test/calculator-example/addition-service/']) + .command(['build', './test/calculator-example/addition-service/', '--verbose']) .it('builds docker image', ctx => { const { stdout } = ctx; expect(stdout).to.contain('Building docker image for addition-service'); @@ -18,7 +18,7 @@ describe('build', () => { test .stdout() .timeout(10000) - .command(['build', '--tag', 'tag-override', './test/calculator-example/addition-service/']) + .command(['build', '--tag', 'tag-override', './test/calculator-example/addition-service/', '--verbose']) .it('allows tag overrides', ctx => { const { stdout } = ctx; expect(stdout).to.contain('Building docker image for addition-service'); @@ -30,8 +30,8 @@ describe('build', () => { test .stdout() - .timeout(15000) - .command(['build', '--recursive', './test/calculator-example/python-subtraction-service/']) + .timeout(20000) + .command(['build', '--recursive', './test/calculator-example/python-subtraction-service/', '--verbose']) .it('builds images recursively', ctx => { const { stdout } = ctx; expect(stdout).to.contain('Building docker image for subtraction-service'); diff --git a/test/commands/install.test.ts b/test/commands/install.test.ts index 2fc343ad0..cd019756b 100644 --- a/test/commands/install.test.ts +++ b/test/commands/install.test.ts @@ -1,11 +1,11 @@ -import {expect, test} from '@oclif/test'; +import { expect, test } from '@oclif/test'; describe('install', () => { test .stdout() - .command(['install', '--prefix', './test/calculator-example/addition-service/']) + .command(['install', '--prefix', './test/calculator-example/addition-service/', '--verbose']) .it('installs dependency stubs', ctx => { - const {stdout} = ctx; + const { stdout } = ctx; expect(stdout).to.contain('Installing dependencies for addition-service'); }); @@ -15,10 +15,11 @@ describe('install', () => { .command([ 'install', '--recursive', - '--prefix', './test/calculator-example/test-script/' + '--prefix', './test/calculator-example/test-script/', + '--verbose' ]) .it('installs dependencies recursively', ctx => { - const {stdout} = ctx; + const { stdout } = ctx; expect(stdout).to.contain('Installing dependencies for test-service'); expect(stdout).to.contain('Installing dependencies for division-service'); expect(stdout).to.contain('Installing dependencies for subtraction-service'); From 730f641971ae992ab2a0feaaadc754f5fa0d7b42 Mon Sep 17 00:00:00 2001 From: TJ Higgins Date: Mon, 3 Jun 2019 22:02:10 -0400 Subject: [PATCH 6/6] Move static tasks method to base ArchitectCommand --- src/base.ts | 45 +++++++++++++++++++++++++++++++---- src/commands/build.ts | 52 ++++++++++++++++++++--------------------- src/commands/install.ts | 31 ++++++++++++------------ src/commands/push.ts | 20 ++++++++-------- src/commands/start.ts | 16 ++++++------- tslint.json | 3 ++- 6 files changed, 102 insertions(+), 65 deletions(-) diff --git a/src/base.ts b/src/base.ts index b35b32229..3d112fcc7 100644 --- a/src/base.ts +++ b/src/base.ts @@ -1,17 +1,50 @@ import Command from '@oclif/command'; +import * as Config from '@oclif/config'; import * as keytar from 'keytar'; +import * as Listr from 'listr'; import * as request from 'request'; import * as url from 'url'; import { AppConfig } from './app-config'; -export default abstract class extends Command { +export default abstract class ArchitectCommand extends Command { + static async tasks(this: any, argv?: string[], opts?: Config.LoadOptions): Promise { + if (!argv) argv = process.argv.slice(2); + const config = await Config.load(opts || module.parent && module.parent.parent && module.parent.parent.filename || __dirname); + let cmd = new this(argv, config); + return cmd._tasks(argv); + } + protected static app_config: AppConfig; + protected static architect: ArchitectClient; + app_config!: AppConfig; architect!: ArchitectClient; async init() { - this.app_config = new AppConfig(); - this.architect = new ArchitectClient(this.app_config.api_host); + if (!ArchitectCommand.app_config) { + ArchitectCommand.app_config = new AppConfig(); + ArchitectCommand.architect = new ArchitectClient(ArchitectCommand.app_config.api_host); + } + this.app_config = ArchitectCommand.app_config; + this.architect = ArchitectCommand.architect; + } + + async tasks(): Promise { throw Error('Not implemented'); } + + async _tasks(): Promise { + let err: Error | undefined; + try { + // remove redirected env var to allow subsessions to run autoupdated client + delete process.env[this.config.scopedEnvVarKey('REDIRECTED')]; + + await this.init(); + return await this.tasks(); + } catch (e) { + err = e; + await this.catch(e); + } finally { + await this.finally(err); + } } styled_json(obj: object) { @@ -42,7 +75,11 @@ class ArchitectClient { if (credentials.length === 0) { throw Error('denied: `architect login` required'); } - return new UserEntity(JSON.parse(credentials[0].password)); + const user = new UserEntity(JSON.parse(credentials[0].password)); + if (!user.username) { + throw Error('denied: `architect login` required'); + } + return user; } async get(path: string) { diff --git a/src/commands/build.ts b/src/commands/build.ts index 3e4120f8a..2cb481def 100644 --- a/src/commands/build.ts +++ b/src/commands/build.ts @@ -41,28 +41,6 @@ export default class Build extends Command { } ]; - static async getTasks(service_path: string, tag?: string, recursive?: boolean): Promise { - const dependencies = await ServiceConfig.getDependencies(service_path, recursive); - const tasks: Listr.ListrTask[] = []; - - dependencies.forEach(dependency => { - tasks.push({ - title: `Building docker image for ${_info(dependency.service_config.name)}`, - task: async () => { - const install_tasks = await Install.getTasks(dependency.service_path); - const build_task = { - title: 'Building', - task: async () => { - await Build.buildImage(dependency.service_path, dependency.service_config, tag); - } - }; - return new Listr(install_tasks.concat([build_task])); - } - }); - }); - return tasks; - } - static async buildImage(service_path: string, service_config: ServiceConfig, tag?: string) { const dockerfile_path = path.join(__dirname, '../../Dockerfile'); const tag_name = tag || `architect-${service_config.name}`; @@ -79,18 +57,40 @@ export default class Build extends Command { } async run() { - const { args, flags } = this.parse(Build); + const { flags } = this.parse(Build); if (flags.recursive && flags.tag) { this.error(_error('Cannot specify tag for recursive builds')); } + const renderer = flags.verbose ? 'verbose' : 'default'; + const tasks = new Listr(await this.tasks(), { concurrent: 2, renderer }); + await tasks.run(); + } + async tasks(): Promise { + const { args, flags } = this.parse(Build); let root_service_path = process.cwd(); if (args.context) { root_service_path = path.resolve(args.context); } - const renderer = flags.verbose ? 'verbose' : 'default'; - const tasks = new Listr(await Build.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2, renderer }); - await tasks.run(); + const dependencies = await ServiceConfig.getDependencies(root_service_path, flags.recursive); + const tasks: Listr.ListrTask[] = []; + + dependencies.forEach(dependency => { + tasks.push({ + title: `Building docker image for ${_info(dependency.service_config.name)}`, + task: async () => { + const install_tasks = await Install.tasks(['-p', dependency.service_path]); + const build_task = { + title: 'Building', + task: async () => { + await Build.buildImage(dependency.service_path, dependency.service_config, flags.tag); + } + }; + return new Listr(install_tasks.concat([build_task])); + } + }); + }); + return tasks; } } diff --git a/src/commands/install.ts b/src/commands/install.ts index b5f4ec133..b7d6fb17b 100644 --- a/src/commands/install.ts +++ b/src/commands/install.ts @@ -30,9 +30,22 @@ export default class Install extends Command { static args = []; - static async getTasks(root_service_path: string, recursive?: boolean): Promise { + async run() { + const { flags } = this.parse(Install); + const renderer = flags.verbose ? 'verbose' : 'default'; + const tasks = new Listr(await this.tasks(), { concurrent: 3, renderer }); + await tasks.run(); + } + + async tasks(): Promise { + const { flags } = this.parse(Install); + let root_service_path = process.cwd(); + if (flags.prefix) { + root_service_path = path.isAbsolute(flags.prefix) ? flags.prefix : path.join(root_service_path, flags.prefix); + } + const tasks: Listr.ListrTask[] = []; - const dependencies = await ServiceConfig.getDependencies(root_service_path, recursive); + const dependencies = await ServiceConfig.getDependencies(root_service_path, flags.recursive); dependencies.forEach(dependency => { const sub_tasks: Listr.ListrTask[] = []; @@ -64,18 +77,4 @@ export default class Install extends Command { return tasks; } - - async run() { - const { flags } = this.parse(Install); - let process_path = process.cwd(); - if (flags.prefix) { - process_path = path.isAbsolute(flags.prefix) ? - flags.prefix : - path.join(process_path, flags.prefix); - } - - const renderer = flags.verbose ? 'verbose' : 'default'; - const tasks = new Listr(await Install.getTasks(process_path, flags.recursive), { concurrent: 3, renderer }); - await tasks.run(); - } } diff --git a/src/commands/push.ts b/src/commands/push.ts index c0087b03d..1042de363 100644 --- a/src/commands/push.ts +++ b/src/commands/push.ts @@ -42,33 +42,33 @@ export default class Push extends Command { ]; async run() { - const { args, flags } = this.parse(Push); + const { flags } = this.parse(Push); if (flags.recursive && flags.tag) { this.error(_error('Cannot specify tag for recursive pushes')); } + const renderer = flags.verbose ? 'verbose' : 'default'; + const tasks = new Listr(await this.tasks(), { concurrent: 2, renderer }); + await tasks.run(); + } + async tasks(): Promise { + const { args, flags } = this.parse(Push); let root_service_path = process.cwd(); if (args.context) { root_service_path = path.resolve(args.context); } - const renderer = flags.verbose ? 'verbose' : 'default'; - const tasks = new Listr(await this.getTasks(root_service_path, flags.tag, flags.recursive), { concurrent: 2, renderer }); - await tasks.run(); - } - - async getTasks(root_service_path: string, tag?: string, recursive?: boolean): Promise { - const dependencies = await ServiceConfig.getDependencies(root_service_path, recursive); + const dependencies = await ServiceConfig.getDependencies(root_service_path, flags.recursive); const tasks: Listr.ListrTask[] = []; dependencies.forEach(dependency => { tasks.push({ title: `Pushing docker image for ${_info(dependency.service_config.name)}`, task: async () => { - const build_tasks = await Build.getTasks(dependency.service_path, tag); + const build_tasks = await Build.tasks([dependency.service_path, '-t', flags.tag || '']); const push_task = { title: 'Pushing', task: async () => { - await this.pushImage(dependency.service_config, tag); + await this.pushImage(dependency.service_config, flags.tag); } }; return new Listr(build_tasks.concat([push_task])); diff --git a/src/commands/start.ts b/src/commands/start.ts index c4c35c4d1..f24643178 100644 --- a/src/commands/start.ts +++ b/src/commands/start.ts @@ -31,21 +31,21 @@ export default class Start extends Command { deployment_config: DeploymentConfig = {}; async run() { - const { args } = this.parse(Start); // Ensures that the python launcher doesn't buffer output and cause hanging process.env.PYTHONUNBUFFERED = 'true'; process.env.PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION = 'python'; - let service_path = process.cwd(); - if (args.context) { - service_path = path.resolve(args.context); - } - - const tasks = new Listr(await this.getTasks(service_path), { renderer: 'verbose' }); + const tasks = new Listr(await this.tasks(), { renderer: 'verbose' }); await tasks.run(); } - async getTasks(root_service_path: string): Promise { + async tasks(): Promise { + const { args } = this.parse(Start); + let root_service_path = process.cwd(); + if (args.context) { + root_service_path = path.resolve(args.context); + } + const recursive = true; const dependencies = await ServiceConfig.getDependencies(root_service_path, recursive); dependencies.reverse(); diff --git a/tslint.json b/tslint.json index 236d4bf42..d59a116fc 100644 --- a/tslint.json +++ b/tslint.json @@ -8,6 +8,7 @@ "rules": { "semicolon": [true, "always"], "ter-indent": [false], - "object-curly-spacing": ["always"] + "object-curly-spacing": ["always"], + "quotemark": [true, "single"] } }