diff --git a/.gitignore b/.gitignore index e57c16df4..1052816e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ node_modules/ -runner/ .terraform/ .cml/ .DS_Store diff --git a/bin/cml.js b/bin/cml.js index 9f41d3f65..027aa45c4 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -28,6 +28,24 @@ const setupOpts = (opts) => { if (process.env[oldName]) process.env[newName] = process.env[oldName]; } + const legacyEnvironmentPrefixes = { + CML_CI: 'CML_REPO', + CML_PUBLISH: 'CML_ASSET', + CML_RERUN_WORKFLOW: 'CML_WORKFLOW', + CML_SEND_COMMENT: 'CML_COMMENT', + CML_SEND_GITHUB_CHECK: 'CML_CHECK', + CML_TENSORBOARD_DEV: 'CML_TENSORBOARD' + }; + + for (const [oldPrefix, newPrefix] of Object.entries( + legacyEnvironmentPrefixes + )) { + for (const key in process.env) { + if (key.startsWith(`${oldPrefix}_`)) + process.env[key.replace(oldPrefix, newPrefix)] = process.env[key]; + } + } + const { markdownfile } = opts; opts.markdownFile = markdownfile; opts.cmlCommand = opts._[0]; @@ -80,22 +98,46 @@ const handleError = (message, error) => { }; (async () => { + setupLogger({ log: 'debug' }); try { await yargs .env('CML') .options({ log: { type: 'string', - description: 'Maximum log level', + description: 'Logging verbosity', choices: ['error', 'warn', 'info', 'debug'], - default: 'info' + default: 'info', + group: 'Global Options:' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + defaultDescription: 'infer from the environment', + description: 'Git provider where the repository is hosted', + group: 'Global Options:' + }, + repo: { + type: 'string', + defaultDescription: 'infer from the environment', + description: 'Repository URL or slug', + group: 'Global Options:' + }, + token: { + type: 'string', + defaultDescription: 'infer from the environment', + description: 'Personal access token', + group: 'Global Options:' } }) + .global('version', false) + .group('help', 'Global Options:') .fail(handleError) .middleware(setupOpts) .middleware(setupLogger) .middleware(setupTelemetry) - .commandDir('./cml', { exclude: /\.test\.js$/ }) + .commandDir('./cml') + .commandDir('./legacy/commands') .command( '$0 ', false, @@ -110,9 +152,11 @@ const handleError = (message, error) => { const { telemetryEvent } = yargs.parsed.argv; await send({ event: telemetryEvent }); } catch (err) { - const { telemetryEvent } = yargs.parsed.argv; - const event = { ...telemetryEvent, error: err.message }; - await send({ event }); + if (yargs.parsed.argv) { + const { telemetryEvent } = yargs.parsed.argv; + const event = { ...telemetryEvent, error: err.message }; + await send({ event }); + } winston.error({ err }); process.exit(1); } diff --git a/bin/cml.test.js b/bin/cml.test.js index a0310b68d..612fed3ab 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -8,23 +8,26 @@ describe('command-line interface tests', () => { "cml.js Commands: - cml.js ci Fixes specific CI setups - cml.js pr Create a pull request with the - specified files - cml.js publish Upload an image to build a report - cml.js rerun-workflow Reruns a workflow given the jobId or - workflow Id - cml.js runner Launch and register a self-hosted - runner - cml.js send-comment Comment on a commit - cml.js send-github-check Create a check report - cml.js tensorboard-dev Get a tensorboard link + cml.js check Manage CI checks + cml.js comment Manage comments + cml.js pr Manage pull requests + cml.js runner Manage self-hosted (cloud & on-premise) CI runners + cml.js tensorboard Manage tensorboard.dev connections + cml.js workflow Manage CI workflows + cml.js ci Prepare Git repository for CML operations + + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]" + --version Show version number [boolean]" `); }); }); diff --git a/bin/cml/asset.js b/bin/cml/asset.js new file mode 100644 index 000000000..8b3950c7a --- /dev/null +++ b/bin/cml/asset.js @@ -0,0 +1,8 @@ +exports.command = 'asset'; +exports.description = false; +exports.builder = (yargs) => + yargs + .commandDir('./asset', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/asset/publish.js b/bin/cml/asset/publish.js new file mode 100644 index 000000000..208deb2cd --- /dev/null +++ b/bin/cml/asset/publish.js @@ -0,0 +1,73 @@ +const fs = require('fs').promises; +const kebabcaseKeys = require('kebabcase-keys'); +const winston = require('winston'); + +const { CML } = require('../../../src/cml'); + +exports.command = 'publish '; +exports.description = 'Publish an asset'; + +exports.handler = async (opts) => { + if (opts.gitlabUploads) { + winston.warn( + '--gitlab-uploads will be deprecated soon, use --native instead' + ); + opts.native = true; + } + + const { file, repo, native, asset: path } = opts; + const cml = new CML({ ...opts, repo: native ? repo : 'cml' }); + const output = await cml.publish({ ...opts, path }); + + if (!file) console.log(output); + else await fs.writeFile(file, output); +}; + +exports.builder = (yargs) => yargs.env('CML_ASSET').options(exports.options); + +exports.options = kebabcaseKeys({ + url: { + type: 'string', + description: 'Self-Hosted URL', + hidden: true + }, + md: { + type: 'boolean', + description: 'Output in markdown format [title || name](url)' + }, + title: { + type: 'string', + alias: 't', + description: 'Markdown title [title](url) or ![](url title)' + }, + native: { + type: 'boolean', + description: + "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" + }, + gitlabUploads: { + type: 'boolean', + hidden: true + }, + rmWatermark: { + type: 'boolean', + description: 'Avoid CML watermark.' + }, + mimeType: { + type: 'string', + defaultDescription: 'infer from the file contents', + description: 'MIME type' + }, + file: { + type: 'string', + alias: 'f', + description: + 'Append the output to the given file or create it if does not exist', + hidden: true + }, + repo: { + type: 'string', + description: + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + } +}); diff --git a/bin/cml/publish.test.js b/bin/cml/asset/publish.test.js similarity index 74% rename from bin/cml/publish.test.js rename to bin/cml/asset/publish.test.js index bc1b19dc1..a96fb947e 100644 --- a/bin/cml/publish.test.js +++ b/bin/cml/asset/publish.test.js @@ -1,5 +1,5 @@ const fs = require('fs'); -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); describe('CML e2e', () => { test('cml publish --help', async () => { @@ -8,26 +8,25 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js publish - Upload an image to build a report + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Specifies the repo to be used. If not specified is extracted + from the CI ENV. [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified is - extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --md Output in markdown format [title || name](url). [boolean] - -t, --title Markdown title [title](url) or ![](url title). [string] + --md Output in markdown format [title || name](url) [boolean] + -t, --title Markdown title [title](url) or ![](url title) [string] --native Uses driver's native capabilities to upload assets instead - of CML's storage. Not available on GitHub. [boolean] + of CML's storage; not available on GitHub [boolean] --rm-watermark Avoid CML watermark. [boolean] - --mime-type Specifies the mime-type. If not set guess it from the - content. [string]" + --mime-type MIME type [string] [default: infer from the file contents]" `); }); diff --git a/bin/cml/check.js b/bin/cml/check.js new file mode 100644 index 000000000..afeed503f --- /dev/null +++ b/bin/cml/check.js @@ -0,0 +1,8 @@ +exports.command = 'check'; +exports.description = 'Manage CI checks'; +exports.builder = (yargs) => + yargs + .commandDir('./check', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/check/create.js b/bin/cml/check/create.js new file mode 100755 index 000000000..43637d7b3 --- /dev/null +++ b/bin/cml/check/create.js @@ -0,0 +1,51 @@ +const fs = require('fs').promises; +const kebabcaseKeys = require('kebabcase-keys'); + +exports.command = 'create '; +exports.description = 'Create a check report'; + +exports.handler = async (opts) => { + const { cml, markdownfile } = opts; + const report = await fs.readFile(markdownfile, 'utf-8'); + await cml.checkCreate({ ...opts, report }); +}; + +exports.builder = (yargs) => yargs.env('CML_CHECK').options(exports.options); + +exports.options = kebabcaseKeys({ + token: { + type: 'string', + description: + "GITHUB_TOKEN or Github App token. Personal access token won't work" + }, + commitSha: { + type: 'string', + alias: 'head-sha', + defaultDescription: 'HEAD', + description: 'Commit SHA linked to this comment' + }, + conclusion: { + type: 'string', + choices: [ + 'success', + 'failure', + 'neutral', + 'cancelled', + 'skipped', + 'timed_out' + ], + default: 'success', + description: 'Conclusion status of the check' + }, + status: { + type: 'string', + choices: ['queued', 'in_progress', 'completed'], + default: 'completed', + description: 'Status of the check' + }, + title: { + type: 'string', + default: 'CML Report', + description: 'Title of the check' + } +}); diff --git a/bin/cml/send-github-check.test.js b/bin/cml/check/create.test.js similarity index 61% rename from bin/cml/send-github-check.test.js rename to bin/cml/check/create.test.js index 67760ed8e..28d61c365 100644 --- a/bin/cml/send-github-check.test.js +++ b/bin/cml/check/create.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); const fs = require('fs').promises; describe('CML e2e', () => { @@ -36,29 +36,27 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js send-github-check - Create a check report + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token GITHUB_TOKEN or Github App token. Personal access token won't work + [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token GITHUB_TOKEN or Github App token. Personal access - token won't work [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --commit-sha, --head-sha Commit SHA linked to this comment. Defaults to HEAD. - [string] - --conclusion Sets the conclusion status of the check. + --commit-sha, --head-sha Commit SHA linked to this comment + [string] [default: HEAD] + --conclusion Conclusion status of the check [string] [choices: \\"success\\", \\"failure\\", \\"neutral\\", \\"cancelled\\", \\"skipped\\", \\"timed_out\\"] [default: \\"success\\"] - --status Sets the status of the check. + --status Status of the check [string] [choices: \\"queued\\", \\"in_progress\\", \\"completed\\"] [default: \\"completed\\"] - --title Sets title of the check. - [string] [default: \\"CML Report\\"]" + --title Title of the check [string] [default: \\"CML Report\\"]" `); }); }); diff --git a/bin/cml/ci.js b/bin/cml/ci.js deleted file mode 100644 index 64d957762..000000000 --- a/bin/cml/ci.js +++ /dev/null @@ -1,34 +0,0 @@ -const kebabcaseKeys = require('kebabcase-keys'); - -const { GIT_USER_NAME, GIT_USER_EMAIL, repoOptions } = require('../../src/cml'); - -exports.command = 'ci'; -exports.description = 'Fixes specific CI setups'; - -exports.handler = async (opts) => { - const { cml, telemetryEvent: event } = opts; - await cml.ci(opts); - await cml.telemetrySend({ event }); -}; - -exports.builder = (yargs) => - yargs.env('CML_CI').options( - kebabcaseKeys({ - ...repoOptions, - unshallow: { - type: 'boolean', - description: - 'Fetch as much as possible, converting a shallow repository to a complete one.' - }, - userEmail: { - type: 'string', - default: GIT_USER_EMAIL, - description: 'Set Git user email.' - }, - userName: { - type: 'string', - default: GIT_USER_NAME, - description: 'Set Git user name.' - } - }) - ); diff --git a/bin/cml/ci.test.js b/bin/cml/ci.test.js deleted file mode 100644 index 121a55af8..000000000 --- a/bin/cml/ci.test.js +++ /dev/null @@ -1,29 +0,0 @@ -const { exec } = require('../../src/utils'); - -describe('CML e2e', () => { - test('cml-ci --help', async () => { - const output = await exec(`echo none | node ./bin/cml.js ci --help`); - - expect(output).toMatchInlineSnapshot(` - "cml.js ci - - Fixes specific CI setups - - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is extracted - from the CI ENV. [string] - --token Personal access token to be used. If not specified is extracted - from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --unshallow Fetch as much as possible, converting a shallow repository to a - complete one. [boolean] - --user-email Set Git user email. [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Set Git user name. [string] [default: \\"Olivaw[bot]\\"]" - `); - }); -}); diff --git a/bin/cml/comment.js b/bin/cml/comment.js new file mode 100644 index 000000000..15d4b5d8e --- /dev/null +++ b/bin/cml/comment.js @@ -0,0 +1,8 @@ +exports.command = 'comment'; +exports.description = 'Manage comments'; +exports.builder = (yargs) => + yargs + .commandDir('./comment', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/comment/create.js b/bin/cml/comment/create.js new file mode 100644 index 000000000..ff6bb15fe --- /dev/null +++ b/bin/cml/comment/create.js @@ -0,0 +1,54 @@ +const kebabcaseKeys = require('kebabcase-keys'); + +exports.command = 'create '; +exports.description = 'Create a comment'; + +exports.handler = async (opts) => { + const { cml } = opts; + console.log(await cml.commentCreate(opts)); +}; + +exports.builder = (yargs) => yargs.env('CML_COMMENT').options(exports.options); + +exports.options = kebabcaseKeys({ + pr: { + type: 'boolean', + description: + 'Post to an existing PR/MR associated with the specified commit' + }, + commitSha: { + type: 'string', + alias: 'head-sha', + default: 'HEAD', + description: 'Commit SHA linked to this comment' + }, + publish: { + type: 'boolean', + description: 'Upload any local images found in the Markdown report' + }, + watch: { + type: 'boolean', + description: 'Watch for changes and automatically update the comment' + }, + triggerFile: { + type: 'string', + description: 'File used to trigger the watcher', + hidden: true + }, + native: { + type: 'boolean', + description: + "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" + }, + update: { + type: 'boolean', + description: + 'Update the last CML comment (if any) instead of creating a new one', + hidden: true + }, + rmWatermark: { + type: 'boolean', + description: + 'Avoid watermark; CML needs a watermark to be able to distinguish CML comments from others' + } +}); diff --git a/bin/cml/send-comment.test.js b/bin/cml/comment/create.test.js similarity index 55% rename from bin/cml/send-comment.test.js rename to bin/cml/comment/create.test.js index 872cff17b..8ed29a39a 100644 --- a/bin/cml/send-comment.test.js +++ b/bin/cml/comment/create.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); const fs = require('fs').promises; describe('Comment integration tests', () => { @@ -16,35 +16,30 @@ describe('Comment integration tests', () => { expect(output).toMatchInlineSnapshot(` "cml.js send-comment - Comment on a commit + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified - is extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] --pr Post to an existing PR/MR associated with the specified commit [boolean] --commit-sha, --head-sha Commit SHA linked to this comment [string] [default: \\"HEAD\\"] - --publish Upload local files and images linked from the - Markdown report [boolean] + --publish Upload any local images found in the Markdown report + [boolean] --watch Watch for changes and automatically update the - report [boolean] + comment [boolean] --native Uses driver's native capabilities to upload assets - instead of CML's storage. Not available on GitHub. + instead of CML's storage; not available on GitHub [boolean] - --update Update the last CML comment (if any) instead of - creating a new one [boolean] - --rm-watermark Avoid watermark. CML needs a watermark to be able to - distinguish CML reports from other comments in order - to provide extra functionality. [boolean]" + --rm-watermark Avoid watermark; CML needs a watermark to be able to + distinguish CML comments from others [boolean]" `); }); diff --git a/bin/cml/comment/update.js b/bin/cml/comment/update.js new file mode 100644 index 000000000..adfdb6483 --- /dev/null +++ b/bin/cml/comment/update.js @@ -0,0 +1,10 @@ +const { builder, handler } = require('./create'); + +exports.command = 'update '; +exports.description = 'Update a comment'; + +exports.handler = async (opts) => { + await handler({ ...opts, update: true }); +}; + +exports.builder = builder; diff --git a/bin/cml/pr.js b/bin/cml/pr.js old mode 100755 new mode 100644 index c051d32a8..65930cbfb --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -1,79 +1,20 @@ -const kebabcaseKeys = require('kebabcase-keys'); - -const { - GIT_REMOTE, - GIT_USER_NAME, - GIT_USER_EMAIL, - repoOptions -} = require('../../src/cml'); +const { options, handler } = require('./pr/create'); exports.command = 'pr '; -exports.description = 'Create a pull request with the specified files'; - -exports.handler = async (opts) => { - const { cml, globpath: globs } = opts; - const link = await cml.prCreate({ ...opts, globs }); - console.log(link); -}; - +exports.description = 'Manage pull requests'; +exports.handler = handler; exports.builder = (yargs) => - yargs.env('CML_PR').options( - kebabcaseKeys({ - ...repoOptions, - md: { - type: 'boolean', - description: 'Output in markdown format [](url).' - }, - skipCI: { - type: 'boolean', - description: 'Force skip CI for the created commit (if any)' - }, - merge: { - type: 'boolean', - alias: 'auto-merge', - conflicts: ['rebase', 'squash'], - description: 'Try to merge the pull request upon creation.' - }, - rebase: { - type: 'boolean', - conflicts: ['merge', 'squash'], - description: 'Try to rebase-merge the pull request upon creation.' - }, - squash: { - type: 'boolean', - conflicts: ['merge', 'rebase'], - description: 'Try to squash-merge the pull request upon creation.' - }, - branch: { - type: 'string', - description: 'Branch name for the pull request.' - }, - title: { - type: 'string', - description: 'Pull request title.' - }, - body: { - type: 'string', - description: 'Pull request description.' - }, - message: { - type: 'string', - description: 'Commit message.' - }, - remote: { - type: 'string', - default: GIT_REMOTE, - description: 'Sets git remote.' - }, - userEmail: { - type: 'string', - default: GIT_USER_EMAIL, - description: 'Sets git user email.' - }, - userName: { - type: 'string', - default: GIT_USER_NAME, - description: 'Sets git user name.' - } - }) - ); + yargs + .commandDir('./pr', { exclude: /\.test\.js$/ }) + .recommendCommands() + .env('CML_PR') + .options( + Object.fromEntries( + Object.entries(options).map(([key, value]) => [ + key, + { ...value, hidden: true, global: false } + ]) + ) + ) + .check(({ globpath }) => globpath) + .strict(); diff --git a/bin/cml/pr.test.js b/bin/cml/pr.test.js deleted file mode 100644 index b8ae60c40..000000000 --- a/bin/cml/pr.test.js +++ /dev/null @@ -1,40 +0,0 @@ -const { exec } = require('../../src/utils'); - -describe('CML e2e', () => { - test('cml-pr --help', async () => { - const output = await exec(`echo none | node ./bin/cml.js pr --help`); - - expect(output).toMatchInlineSnapshot(` - "cml.js pr - - Create a pull request with the specified files - - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified is - extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --md Output in markdown format [](url). [boolean] - --skip-ci Force skip CI for the created commit (if any) [boolean] - --merge, --auto-merge Try to merge the pull request upon creation. [boolean] - --rebase Try to rebase-merge the pull request upon creation. - [boolean] - --squash Try to squash-merge the pull request upon creation. - [boolean] - --branch Branch name for the pull request. [string] - --title Pull request title. [string] - --body Pull request description. [string] - --message Commit message. [string] - --remote Sets git remote. [string] [default: \\"origin\\"] - --user-email Sets git user email. - [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Sets git user name. [string] [default: \\"Olivaw[bot]\\"]" - `); - }); -}); diff --git a/bin/cml/pr/create.js b/bin/cml/pr/create.js new file mode 100755 index 000000000..3e8274104 --- /dev/null +++ b/bin/cml/pr/create.js @@ -0,0 +1,76 @@ +const kebabcaseKeys = require('kebabcase-keys'); + +const { + GIT_REMOTE, + GIT_USER_NAME, + GIT_USER_EMAIL +} = require('../../../src/cml'); + +exports.command = 'create '; +exports.description = 'Create a pull request with the specified files'; + +exports.handler = async (opts) => { + const { cml, globpath: globs } = opts; + const link = await cml.prCreate({ ...opts, globs }); + console.log(link); +}; + +exports.builder = (yargs) => yargs.env('CML_PR').options(exports.options); + +exports.options = kebabcaseKeys({ + md: { + type: 'boolean', + description: 'Output in markdown format [](url)' + }, + skipCI: { + type: 'boolean', + description: 'Force skip CI for the created commit (if any)' + }, + merge: { + type: 'boolean', + alias: 'auto-merge', + conflicts: ['rebase', 'squash'], + description: 'Try to merge the pull request upon creation' + }, + rebase: { + type: 'boolean', + conflicts: ['merge', 'squash'], + description: 'Try to rebase-merge the pull request upon creation' + }, + squash: { + type: 'boolean', + conflicts: ['merge', 'rebase'], + description: 'Try to squash-merge the pull request upon creation' + }, + branch: { + type: 'string', + description: 'Pull request branch name' + }, + title: { + type: 'string', + description: 'Pull request title' + }, + body: { + type: 'string', + description: 'Pull request description' + }, + message: { + type: 'string', + description: 'Commit message' + }, + remote: { + type: 'string', + default: GIT_REMOTE, + description: 'Git remote' + }, + userEmail: { + type: 'string', + default: GIT_USER_EMAIL, + description: 'Git user email' + }, + userName: { + type: 'string', + default: GIT_USER_NAME, + description: 'Git user name' + } +}); diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js new file mode 100644 index 000000000..86a7fd446 --- /dev/null +++ b/bin/cml/pr/create.test.js @@ -0,0 +1,27 @@ +const { exec } = require('../../../src/utils'); + +describe('CML e2e', () => { + test('cml-pr --help', async () => { + const output = await exec(`echo none | node ./bin/cml.js pr --help`); + + expect(output).toMatchInlineSnapshot(` + "cml.js pr + + Manage pull requests + + Commands: + cml.js pr create Create a pull request with the specified + files + + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean]" + `); + }); +}); diff --git a/bin/cml/publish.js b/bin/cml/publish.js deleted file mode 100644 index 148bed459..000000000 --- a/bin/cml/publish.js +++ /dev/null @@ -1,75 +0,0 @@ -const fs = require('fs').promises; -const kebabcaseKeys = require('kebabcase-keys'); -const winston = require('winston'); - -const { CML, repoOptions } = require('../../src/cml'); - -exports.command = 'publish '; -exports.description = 'Upload an image to build a report'; - -exports.handler = async (opts) => { - if (opts.gitlabUploads) { - winston.warn( - '--gitlab-uploads will be deprecated soon. Use --native instead.' - ); - opts.native = true; - } - - const { file, repo, native, asset: path } = opts; - const cml = new CML({ ...opts, repo: native ? repo : 'cml' }); - const output = await cml.publish({ ...opts, path }); - - if (!file) console.log(output); - else await fs.writeFile(file, output); -}; - -exports.builder = (yargs) => - yargs.env('CML_PUBLISH').options( - kebabcaseKeys({ - ...repoOptions, - url: { - type: 'string', - description: 'Self-Hosted URL', - hidden: true - }, - md: { - type: 'boolean', - description: 'Output in markdown format [title || name](url).' - }, - title: { - type: 'string', - alias: 't', - description: 'Markdown title [title](url) or ![](url title).' - }, - native: { - type: 'boolean', - description: - "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." - }, - gitlabUploads: { - type: 'boolean', - hidden: true - }, - rmWatermark: { - type: 'boolean', - description: 'Avoid CML watermark.' - }, - mimeType: { - type: 'string', - description: - 'Specifies the mime-type. If not set guess it from the content.' - }, - file: { - type: 'string', - alias: 'f', - description: - 'Append the output to the given file. Create it if does not exist.', - hidden: true - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - } - }) - ); diff --git a/bin/cml/repo.js b/bin/cml/repo.js new file mode 100644 index 000000000..cbf7a8b87 --- /dev/null +++ b/bin/cml/repo.js @@ -0,0 +1,8 @@ +exports.command = 'repo'; +exports.description = false; +exports.builder = (yargs) => + yargs + .commandDir('./repo', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/repo/prepare.js b/bin/cml/repo/prepare.js new file mode 100644 index 000000000..b3218f788 --- /dev/null +++ b/bin/cml/repo/prepare.js @@ -0,0 +1,31 @@ +const kebabcaseKeys = require('kebabcase-keys'); + +const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../../src/cml'); + +exports.command = 'prepare'; +exports.description = 'Prepare the cloned repository'; + +exports.handler = async (opts) => { + const { cml } = opts; + await cml.ci(opts); +}; + +exports.builder = (yargs) => yargs.env('CML_REPO').options(exports.options); + +exports.options = kebabcaseKeys({ + unshallow: { + type: 'boolean', + description: + 'Fetch as much as possible, converting a shallow repository to a complete one' + }, + userEmail: { + type: 'string', + default: GIT_USER_EMAIL, + description: 'Git user email' + }, + userName: { + type: 'string', + default: GIT_USER_NAME, + description: 'Git user name' + } +}); diff --git a/bin/cml/repo/prepare.test.js b/bin/cml/repo/prepare.test.js new file mode 100644 index 000000000..02fbc1615 --- /dev/null +++ b/bin/cml/repo/prepare.test.js @@ -0,0 +1,29 @@ +const { exec } = require('../../../src/utils'); + +describe('CML e2e', () => { + test('cml-ci --help', async () => { + const output = await exec(`echo none | node ./bin/cml.js ci --help`); + + expect(output).toMatchInlineSnapshot(` + "cml.js ci + + Prepare Git repository for CML operations + + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] + + Options: + --unshallow Fetch as much as possible, converting a shallow repository to a + complete one [boolean] + --user-email Git user email [string] [default: \\"olivaw@iterative.ai\\"] + --user-name Git user name [string] [default: \\"Olivaw[bot]\\"]" + `); + }); +}); diff --git a/bin/cml/rerun-workflow.js b/bin/cml/rerun-workflow.js deleted file mode 100644 index eaa19207d..000000000 --- a/bin/cml/rerun-workflow.js +++ /dev/null @@ -1,22 +0,0 @@ -const kebabcaseKeys = require('kebabcase-keys'); - -exports.command = 'rerun-workflow'; -exports.description = 'Reruns a workflow given the jobId or workflow Id'; - -const { repoOptions } = require('../../src/cml'); - -exports.handler = async (opts) => { - const { cml } = opts; - await cml.pipelineRerun(opts); -}; - -exports.builder = (yargs) => - yargs.env('CML_CI').options( - kebabcaseKeys({ - ...repoOptions, - id: { - type: 'string', - description: 'Specifies the run Id to be rerun.' - } - }) - ); diff --git a/bin/cml/rerun-workflow.test.js b/bin/cml/rerun-workflow.test.js deleted file mode 100644 index f52bf1389..000000000 --- a/bin/cml/rerun-workflow.test.js +++ /dev/null @@ -1,28 +0,0 @@ -const { exec } = require('../../src/utils'); - -describe('CML e2e', () => { - test('cml-ci --help', async () => { - const output = await exec( - `echo none | node ./bin/cml.js rerun-workflow --help` - ); - - expect(output).toMatchInlineSnapshot(` - "cml.js rerun-workflow - - Reruns a workflow given the jobId or workflow Id - - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is extracted from - the CI ENV. [string] - --token Personal access token to be used. If not specified is extracted - from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --id Specifies the run Id to be rerun. [string]" - `); - }); -}); diff --git a/bin/cml/runner.js b/bin/cml/runner.js old mode 100755 new mode 100644 index 6c7420767..f294d4369 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -1,589 +1,20 @@ -const { join } = require('path'); -const { homedir } = require('os'); -const fs = require('fs').promises; -const net = require('net'); -const kebabcaseKeys = require('kebabcase-keys'); -const timestring = require('timestring'); -const winston = require('winston'); - -const { randid, sleep } = require('../../src/utils'); -const tf = require('../../src/terraform'); -const { repoOptions } = require('../../src/cml'); - -let cml; -let RUNNER; -let RUNNER_SHUTTING_DOWN = false; -let RUNNER_TIMER = 0; -const RUNNER_JOBS_RUNNING = []; -const GH_5_MIN_TIMEOUT = (35 * 24 * 60 - 5) * 60 * 1000; - -const { RUNNER_NAME } = process.env; - -const shutdown = async (opts) => { - if (RUNNER_SHUTTING_DOWN) return; - RUNNER_SHUTTING_DOWN = true; - - const { error, cloud } = opts; - const { - name, - workdir = '', - tfResource, - noRetry, - reason, - destroyDelay - } = opts; - const tfPath = workdir; - - const unregisterRunner = async () => { - if (!RUNNER) return; - - try { - winston.info(`Unregistering runner ${name}...`); - await cml.unregisterRunner({ name }); - RUNNER && RUNNER.kill('SIGINT'); - winston.info('\tSuccess'); - } catch (err) { - winston.error(`\tFailed: ${err.message}`); - } - }; - - const retryWorkflows = async () => { - try { - if (!noRetry && RUNNER_JOBS_RUNNING.length > 0) { - winston.info(`Still pending jobs, retrying workflow...`); - - await Promise.all( - RUNNER_JOBS_RUNNING.map( - async (job) => - await cml.pipelineRerun({ id: job.pipeline, jobId: job.id }) - ) - ); - } - } catch (err) { - winston.error(err); - } - }; - - const destroyTerraform = async () => { - if (!tfResource) return; - - winston.info(`Waiting ${destroyDelay} seconds to destroy`); - await sleep(destroyDelay); - - try { - winston.debug(await tf.destroy({ dir: tfPath })); - } catch (err) { - winston.error(`\tFailed destroying terraform: ${err.message}`); - } - }; - - if (!cloud) { - try { - await unregisterRunner(); - await retryWorkflows(); - } catch (err) { - winston.error(`Error connecting the SCM: ${err.message}`); - } - } - - await destroyTerraform(); - - if (error) throw error; - - winston.info('runner status', { reason, status: 'terminated' }); - process.exit(0); -}; - -const runCloud = async (opts) => { - const runTerraform = async (opts) => { - winston.info('Terraform apply...'); - - const { token, repo, driver } = cml; - const { - tpiVersion, - labels, - idleTimeout, - name, - cmlVersion, - single, - dockerVolumes, - cloud, - cloudRegion: region, - cloudType: type, - cloudPermissionSet: permissionSet, - cloudMetadata: metadata, - cloudGpu: gpu, - cloudHddSize: hddSize, - cloudSshPrivate: sshPrivate, - cloudSpot: spot, - cloudSpotPrice: spotPrice, - cloudStartupScript: startupScript, - cloudAwsSecurityGroup: awsSecurityGroup, - cloudAwsSubnet: awsSubnet, - workdir - } = opts; - - await tf.checkMinVersion(); - - if (gpu === 'tesla') - winston.warn( - 'GPU model "tesla" has been deprecated; please use "v100" instead.' - ); - - const tfPath = workdir; - const tfMainPath = join(tfPath, 'main.tf.json'); - - const tpl = tf.iterativeCmlRunnerTpl({ - tpiVersion, - repo, - token, - driver, - labels, - cmlVersion, - idleTimeout, - name, - single, - cloud, - region, - type, - permissionSet, - metadata, - gpu: gpu === 'tesla' ? 'v100' : gpu, - hddSize, - sshPrivate, - spot, - spotPrice, - startupScript, - awsSecurityGroup, - awsSubnet, - dockerVolumes - }); - - await fs.writeFile(tfMainPath, JSON.stringify(tpl)); - - await tf.init({ dir: tfPath }); - await tf.apply({ dir: tfPath }); - - const tfStatePath = join(tfPath, 'terraform.tfstate'); - const tfstate = await tf.loadTfState({ path: tfStatePath }); - - return tfstate; - }; - - winston.info('Deploying cloud runner plan...'); - const tfstate = await runTerraform(opts); - const { resources } = tfstate; - for (const resource of resources) { - if (resource.type.startsWith('iterative_')) { - for (const { attributes } of resource.instances) { - const nonSensitiveValues = { - awsSecurityGroup: attributes.aws_security_group, - awsSubnetId: attributes.aws_subnet_id, - cloud: attributes.cloud, - driver: attributes.driver, - id: attributes.id, - idleTimeout: attributes.idle_timeout, - image: attributes.image, - instanceGpu: attributes.instance_gpu, - instanceHddSize: attributes.instance_hdd_size, - instanceIp: attributes.instance_ip, - instanceLaunchTime: attributes.instance_launch_time, - instanceType: attributes.instance_type, - instancePermissionSet: attributes.instance_permission_set, - labels: attributes.labels, - cmlVersion: attributes.cml_version, - metadata: attributes.metadata, - name: attributes.name, - region: attributes.region, - repo: attributes.repo, - single: attributes.single, - spot: attributes.spot, - spotPrice: attributes.spot_price, - timeouts: attributes.timeouts - }; - winston.info(JSON.stringify(nonSensitiveValues)); - } - } - } -}; - -const runLocal = async (opts) => { - winston.info(`Launching ${cml.driver} runner`); - const { - workdir, - name, - labels, - single, - idleTimeout, - noRetry, - dockerVolumes, - tfResource, - tpiVersion - } = opts; - - if (tfResource) { - await tf.checkMinVersion(); - - const tfPath = workdir; - await fs.mkdir(tfPath, { recursive: true }); - const tfMainPath = join(tfPath, 'main.tf.json'); - const tpl = tf.iterativeProviderTpl({ tpiVersion }); - await fs.writeFile(tfMainPath, JSON.stringify(tpl)); - - await tf.init({ dir: tfPath }); - await tf.apply({ dir: tfPath }); - - const path = join(tfPath, 'terraform.tfstate'); - const tfstate = await tf.loadTfState({ path }); - tfstate.resources = [ - JSON.parse(Buffer.from(tfResource, 'base64').toString('utf-8')) - ]; - await tf.saveTfState({ tfstate, path }); - } - - if (process.platform === 'linux') { - const acpiSock = net.connect('/var/run/acpid.socket'); - acpiSock.on('connect', () => { - winston.info('Connected to acpid service.'); - }); - acpiSock.on('error', (err) => { - winston.warn( - `Error connecting to ACPI socket: ${err.message}. The acpid.service helps with instance termination detection.` - ); - }); - acpiSock.on('data', (buf) => { - const data = buf.toString().toLowerCase(); - if (data.includes('power') && data.includes('button')) { - shutdown({ ...opts, reason: 'ACPI shutdown' }); - } - }); - } - - const dataHandler = async (data) => { - const logs = await cml.parseRunnerLog({ data, name }); - for (const log of logs) { - winston.info('runner status', log); - - if (log.status === 'job_started') { - const { job: id, pipeline, date } = log; - RUNNER_JOBS_RUNNING.push({ id, pipeline, date }); - } - - if (log.status === 'job_ended') { - RUNNER_JOBS_RUNNING.pop(); - if (single) await shutdown({ ...opts, reason: 'single job' }); - } - } - }; - - const proc = await cml.startRunner({ - workdir, - name, - labels, - single, - idleTimeout, - dockerVolumes - }); - - proc.stderr.on('data', dataHandler); - proc.stdout.on('data', dataHandler); - proc.on('disconnect', () => - shutdown({ ...opts, error: new Error('runner proccess lost') }) - ); - proc.on('close', (exit) => { - const reason = `runner closed with exit code ${exit}`; - if (exit === 0) shutdown({ ...opts, reason }); - else shutdown({ ...opts, error: new Error(reason) }); - }); - - RUNNER = proc; - if (idleTimeout > 0) { - const watcher = setInterval(async () => { - const idle = RUNNER_JOBS_RUNNING.length === 0; - - if (RUNNER_TIMER >= idleTimeout) { - shutdown({ ...opts, reason: `timeout:${idleTimeout}` }); - clearInterval(watcher); - } - - RUNNER_TIMER = idle ? RUNNER_TIMER + 1 : 0; - }, 1000); - } - - if (!noRetry) { - if (cml.driver === 'github') { - const watcherSeventyTwo = setInterval(() => { - RUNNER_JOBS_RUNNING.forEach((job) => { - if ( - new Date().getTime() - new Date(job.date).getTime() > - GH_5_MIN_TIMEOUT - ) { - shutdown({ ...opts, reason: 'timeout:35days' }); - clearInterval(watcherSeventyTwo); - } - }); - }, 60 * 1000); - } - } -}; - -const run = async (opts) => { - opts.workdir = opts.workdir || `${homedir()}/.cml/${opts.name}`; - - process.on('unhandledRejection', (reason) => - shutdown({ ...opts, error: new Error(reason) }) - ); - process.on('uncaughtException', (error) => shutdown({ ...opts, error })); - - ['SIGTERM', 'SIGINT', 'SIGQUIT'].forEach((signal) => { - process.on(signal, () => shutdown({ ...opts, reason: signal })); - }); - - const { - driver, - workdir, - cloud, - labels, - name, - reuse, - reuseIdle, - dockerVolumes - } = opts; - - await cml.repoTokenCheck(); - - const runners = await cml.runners(); - const runner = await cml.runnerByName({ name, runners }); - if (runner) { - if (!reuse) - throw new Error( - `Runner name ${name} is already in use. Please change the name or terminate the existing runner.` - ); - winston.info(`Reusing existing runner named ${name}...`); - return; - } - - if ( - reuse && - (await cml.runnersByLabels({ labels, runners })).find( - (runner) => runner.online - ) - ) { - winston.info( - `Reusing existing online runners with the ${labels} labels...` - ); - return; - } - - if (reuseIdle) { - if (driver === 'bitbucket') { - throw new Error( - 'cml runner flag --reuse-idle is unsupported by bitbucket' - ); - } - winston.info( - `Checking for existing idle runner matching labels: ${labels}.` - ); - const currentRunners = await cml.runnersByLabels({ labels, runners }); - const availableRunner = currentRunners.find( - (runner) => runner.online && !runner.busy - ); - if (availableRunner) { - winston.info('Found matching idle runner.', availableRunner); - return; - } - } - - if (dockerVolumes.length && cml.driver !== 'gitlab') - winston.warn('Parameters --docker-volumes is only supported in gitlab'); - - if (driver === 'github') - winston.warn( - 'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.' - ); - - if (RUNNER_NAME) - winston.warn( - 'ignoring RUNNER_NAME environment variable, use CML_RUNNER_NAME or --name instead' - ); - - winston.info(`Preparing workdir ${workdir}...`); - await fs.mkdir(workdir, { recursive: true }); - await fs.chmod(workdir, '766'); - - if (cloud) await runCloud(opts); - else await runLocal(opts); -}; +const { options, handler } = require('./runner/launch'); exports.command = 'runner'; -exports.description = 'Launch and register a self-hosted runner'; - -exports.handler = async (opts) => { - ({ cml } = opts); - try { - await run(opts); - } catch (error) { - await shutdown({ ...opts, error }); - } -}; - +exports.description = 'Manage self-hosted (cloud & on-premise) CI runners'; +exports.handler = handler; exports.builder = (yargs) => - yargs.env('CML_RUNNER').options( - kebabcaseKeys({ - ...repoOptions, - labels: { - type: 'string', - default: 'cml', - description: - 'One or more user-defined labels for this runner (delimited with commas)' - }, - idleTimeout: { - type: 'string', - default: '5 minutes', - coerce: (val) => - /^-?\d+$/.test(val) ? parseInt(val) : timestring(val), - description: - 'Time to wait for jobs before shutting down (e.g. "5min"). Use "never" to disable' - }, - name: { - type: 'string', - default: `cml-${randid()}`, - defaultDescription: 'cml-{ID}', - description: 'Name displayed in the repository once registered' - }, - noRetry: { - type: 'boolean', - description: - 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' - }, - single: { - type: 'boolean', - conflicts: ['reuse', 'reuseIdle'], - description: 'Exit after running a single job' - }, - reuse: { - type: 'boolean', - conflicts: ['single', 'reuseIdle'], - description: - "Don't launch a new runner if an existing one has the same name or overlapping labels" - }, - reuseIdle: { - type: 'boolean', - conflicts: ['reuse', 'single'], - description: - 'Only creates a new runner if the matching labels dont exist or are already busy.' - }, - workdir: { - type: 'string', - hidden: true, - alias: 'path', - description: 'Runner working directory' - }, - dockerVolumes: { - type: 'array', - default: [], - description: 'Docker volumes. This feature is only supported in GitLab' - }, - cloud: { - type: 'string', - choices: ['aws', 'azure', 'gcp', 'kubernetes'], - description: 'Cloud to deploy the runner' - }, - cloudRegion: { - type: 'string', - default: 'us-west', - description: - 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' - }, - cloudType: { - type: 'string', - description: - 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' - }, - cloudPermissionSet: { - type: 'string', - default: '', - description: - 'Specifies the instance profile in AWS or instance service account in GCP' - }, - cloudMetadata: { - type: 'array', - string: true, - default: [], - coerce: (items) => { - const keyValuePairs = items.map((item) => [ - ...item.split(/=(.+)/), - null - ]); - return Object.fromEntries(keyValuePairs); - }, - description: - 'Key Value pairs to associate cml-runner instance on the provider i.e. tags/labels "key=value"' - }, - cloudGpu: { - type: 'string', - description: - 'GPU type. Choices: k80, v100, or native types e.g. nvidia-tesla-t4', - coerce: (val) => (val === 'nogpu' ? undefined : val) - }, - cloudHddSize: { - type: 'number', - description: 'HDD size in GB' - }, - cloudSshPrivate: { - type: 'string', - description: - 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' - }, - cloudSpot: { - type: 'boolean', - description: 'Request a spot instance' - }, - cloudSpotPrice: { - type: 'number', - default: -1, - description: - 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' - }, - cloudStartupScript: { - type: 'string', - description: - 'Run the provided Base64-encoded Linux shell script during the instance initialization' - }, - cloudAwsSecurityGroup: { - type: 'string', - default: '', - description: 'Specifies the security group in AWS' - }, - cloudAwsSubnet: { - type: 'string', - default: '', - description: 'Specifies the subnet to use within AWS', - alias: 'cloud-aws-subnet-id' - }, - tpiVersion: { - type: 'string', - default: '>= 0.9.10', - description: - 'Pin the iterative/iterative terraform provider to a specific version. i.e. "= 0.10.4" See: https://www.terraform.io/language/expressions/version-constraints', - hidden: true - }, - cmlVersion: { - type: 'string', - default: require('../../package.json').version, - description: 'CML version to load on TPI instance', - hidden: true - }, - tfResource: { - hidden: true, - alias: 'tf_resource' - }, - destroyDelay: { - type: 'number', - default: 10, - hidden: true, - description: - 'Seconds to wait for collecting logs on failure (https://github.com/iterative/cml/issues/413)' - } - }) - ); + yargs + .commandDir('./runner', { exclude: /\.test\.js$/ }) + .recommendCommands() + .env('CML_RUNNER') + .options( + Object.fromEntries( + Object.entries(options).map(([key, value]) => [ + key, + { ...value, hidden: true, global: false } + ]) + ) + ) + .check(() => process.argv.some((arg) => arg.startsWith('-'))) + .strict(); diff --git a/bin/cml/runner.test.js b/bin/cml/runner.test.js deleted file mode 100644 index 57b88db59..000000000 --- a/bin/cml/runner.test.js +++ /dev/null @@ -1,214 +0,0 @@ -jest.setTimeout(2000000); - -const isIp = require('is-ip'); -const { CML } = require('../..//src/cml'); -const { exec, sshConnection, randid, sleep } = require('../../src/utils'); - -const IDLE_TIMEOUT = 15; -const { - TEST_GITHUB_TOKEN, - TEST_GITHUB_REPO, - TEST_GITLAB_TOKEN, - TEST_GITLAB_REPO, - SSH_PRIVATE -} = process.env; - -const launchRunner = async (opts) => { - const { cloud, type, repo, token, privateKey, name } = opts; - const command = `node ./bin/cml.js runner --cloud ${cloud} --cloud-type ${type} --repo ${repo} --token ${token} --cloud-ssh-private="${privateKey}" --name ${name} --cloud-spot true --idle-timeout ${IDLE_TIMEOUT}`; - - const output = await exec(command); - const state = JSON.parse(output.split(/\n/).pop()); - - return state; -}; - -const testRunner = async (opts) => { - const { repo, token, name, privateKey } = opts; - const { instanceIp: host } = await launchRunner(opts); - expect(isIp(host)).toBe(true); - - const sshOpts = { host, username: 'ubuntu', privateKey }; - const cml = new CML({ repo, token }); - - let runner = await cml.runnerByName({ name }); - expect(runner).not.toBe(undefined); - await sshConnection(sshOpts); - - await sleep(IDLE_TIMEOUT + 60); - - runner = await cml.runnerByName({ name }); - expect(runner).toBe(undefined); - - let sshErr; - try { - await sshConnection(sshOpts); - } catch (err) { - sshErr = err; - } - expect(sshErr).not.toBe(undefined); -}; - -describe('CML e2e', () => { - test('cml-runner --help', async () => { - const output = await exec(`echo none | node ./bin/cml.js runner --help`); - - expect(output).toMatchInlineSnapshot(` - "cml.js runner - - Launch and register a self-hosted runner - - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If - not specified is extracted from the - CI ENV. [string] - --token Personal access token to be used. If - not specified is extracted from ENV - REPO_TOKEN. [string] - --driver If not specify it infers it from the - ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --labels One or more user-defined labels for - this runner (delimited with commas) - [string] [default: \\"cml\\"] - --idle-timeout Time to wait for jobs before - shutting down (e.g. \\"5min\\"). Use - \\"never\\" to disable - [string] [default: \\"5 minutes\\"] - --name Name displayed in the repository - once registered - [string] [default: cml-{ID}] - --no-retry Do not restart workflow terminated - due to instance disposal or GitHub - Actions timeout [boolean] - --single Exit after running a single job - [boolean] - --reuse Don't launch a new runner if an - existing one has the same name or - overlapping labels [boolean] - --reuse-idle Only creates a new runner if the - matching labels dont exist or are - already busy. [boolean] - --docker-volumes Docker volumes. This feature is only - supported in GitLab - [array] [default: []] - --cloud Cloud to deploy the runner - [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"] - --cloud-region Region where the instance is - deployed. Choices: [us-east, - us-west, eu-west, eu-north]. Also - accepts native cloud regions - [string] [default: \\"us-west\\"] - --cloud-type Instance type. Choices: [m, l, xl]. - Also supports native types like i.e. - t2.micro [string] - --cloud-permission-set Specifies the instance profile in - AWS or instance service account in - GCP [string] [default: \\"\\"] - --cloud-metadata Key Value pairs to associate - cml-runner instance on the provider - i.e. tags/labels \\"key=value\\" - [array] [default: []] - --cloud-gpu GPU type. Choices: k80, v100, or - native types e.g. nvidia-tesla-t4 - [string] - --cloud-hdd-size HDD size in GB [number] - --cloud-ssh-private Custom private RSA SSH key. If not - provided an automatically generated - throwaway key will be used [string] - --cloud-spot Request a spot instance [boolean] - --cloud-spot-price Maximum spot instance bidding price - in USD. Defaults to the current spot - bidding price [number] [default: -1] - --cloud-startup-script Run the provided Base64-encoded - Linux shell script during the - instance initialization [string] - --cloud-aws-security-group Specifies the security group in AWS - [string] [default: \\"\\"] - --cloud-aws-subnet, Specifies the subnet to use within - --cloud-aws-subnet-id AWS [string] [default: \\"\\"]" - `); - }); - - test.skip('cml-runner GL/AWS', async () => { - const opts = { - repo: TEST_GITLAB_REPO, - token: TEST_GITLAB_TOKEN, - privateKey: SSH_PRIVATE, - cloud: 'aws', - type: 't2.micro', - name: `cml-test-${randid()}` - }; - - await testRunner(opts); - }); - - test.skip('cml-runner GH/AWS', async () => { - const opts = { - repo: TEST_GITHUB_REPO, - token: TEST_GITHUB_TOKEN, - privateKey: SSH_PRIVATE, - cloud: 'aws', - type: 't2.micro', - name: `cml-test-${randid()}` - }; - - await testRunner(opts); - }); - - test.skip('cml-runner GL/Azure', async () => { - const opts = { - repo: TEST_GITLAB_REPO, - token: TEST_GITLAB_TOKEN, - privateKey: SSH_PRIVATE, - cloud: 'azure', - type: 'm', - name: `cml-test-${randid()}` - }; - - await testRunner(opts); - }); - - test.skip('cml-runner GH/Azure', async () => { - const opts = { - repo: TEST_GITHUB_REPO, - token: TEST_GITHUB_TOKEN, - privateKey: SSH_PRIVATE, - cloud: 'azure', - type: 'm', - name: `cml-test-${randid()}` - }; - - await testRunner(opts); - }); - - test.skip('cml-runner GL/GCP', async () => { - const opts = { - repo: TEST_GITLAB_REPO, - token: TEST_GITLAB_TOKEN, - privateKey: SSH_PRIVATE, - cloud: 'gcp', - type: 'm', - name: `cml-test-${randid()}` - }; - - await testRunner(opts); - }); - - test.skip('cml-runner GH/GCP', async () => { - const opts = { - repo: TEST_GITHUB_REPO, - token: TEST_GITHUB_TOKEN, - privateKey: SSH_PRIVATE, - cloud: 'gcp', - type: 'm', - name: `cml-test-${randid()}` - }; - - await testRunner(opts); - }); -}); diff --git a/bin/cml/runner/launch.js b/bin/cml/runner/launch.js new file mode 100755 index 000000000..669db00b2 --- /dev/null +++ b/bin/cml/runner/launch.js @@ -0,0 +1,582 @@ +const { join } = require('path'); +const { homedir } = require('os'); +const fs = require('fs').promises; +const net = require('net'); +const kebabcaseKeys = require('kebabcase-keys'); +const timestring = require('timestring'); +const winston = require('winston'); + +const { randid, sleep } = require('../../../src/utils'); +const tf = require('../../../src/terraform'); + +let cml; +let RUNNER; +let RUNNER_SHUTTING_DOWN = false; +let RUNNER_TIMER = 0; +const RUNNER_JOBS_RUNNING = []; +const GH_5_MIN_TIMEOUT = (35 * 24 * 60 - 5) * 60 * 1000; + +const { RUNNER_NAME } = process.env; + +const shutdown = async (opts) => { + if (RUNNER_SHUTTING_DOWN) return; + RUNNER_SHUTTING_DOWN = true; + + const { error, cloud } = opts; + const { + name, + workdir = '', + tfResource, + noRetry, + reason, + destroyDelay + } = opts; + const tfPath = workdir; + + const unregisterRunner = async () => { + if (!RUNNER) return; + + try { + winston.info(`Unregistering runner ${name}...`); + await cml.unregisterRunner({ name }); + RUNNER && RUNNER.kill('SIGINT'); + winston.info('\tSuccess'); + } catch (err) { + winston.error(`\tFailed: ${err.message}`); + } + }; + + const retryWorkflows = async () => { + try { + if (!noRetry && RUNNER_JOBS_RUNNING.length > 0) { + winston.info(`Still pending jobs, retrying workflow...`); + + await Promise.all( + RUNNER_JOBS_RUNNING.map( + async (job) => + await cml.pipelineRerun({ id: job.pipeline, jobId: job.id }) + ) + ); + } + } catch (err) { + winston.error(err); + } + }; + + const destroyTerraform = async () => { + if (!tfResource) return; + + winston.info(`Waiting ${destroyDelay} seconds to destroy`); + await sleep(destroyDelay); + + try { + winston.debug(await tf.destroy({ dir: tfPath })); + } catch (err) { + winston.error(`\tFailed destroying terraform: ${err.message}`); + } + }; + + if (!cloud) { + try { + await unregisterRunner(); + await retryWorkflows(); + } catch (err) { + winston.error(`Error connecting the SCM: ${err.message}`); + } + } + + await destroyTerraform(); + + if (error) throw error; + + winston.info('runner status', { reason, status: 'terminated' }); + process.exit(0); +}; + +const runCloud = async (opts) => { + const runTerraform = async (opts) => { + winston.info('Terraform apply...'); + + const { token, repo, driver } = cml; + const { + tpiVersion, + labels, + idleTimeout, + name, + cmlVersion, + single, + dockerVolumes, + cloud, + cloudRegion: region, + cloudType: type, + cloudPermissionSet: permissionSet, + cloudMetadata: metadata, + cloudGpu: gpu, + cloudHddSize: hddSize, + cloudSshPrivate: sshPrivate, + cloudSpot: spot, + cloudSpotPrice: spotPrice, + cloudStartupScript: startupScript, + cloudAwsSecurityGroup: awsSecurityGroup, + cloudAwsSubnet: awsSubnet, + workdir + } = opts; + + await tf.checkMinVersion(); + + if (gpu === 'tesla') + winston.warn( + 'GPU model "tesla" has been deprecated; please use "v100" instead.' + ); + + const tfPath = workdir; + const tfMainPath = join(tfPath, 'main.tf.json'); + + const tpl = tf.iterativeCmlRunnerTpl({ + tpiVersion, + repo, + token, + driver, + labels, + cmlVersion, + idleTimeout, + name, + single, + cloud, + region, + type, + permissionSet, + metadata, + gpu: gpu === 'tesla' ? 'v100' : gpu, + hddSize, + sshPrivate, + spot, + spotPrice, + startupScript, + awsSecurityGroup, + awsSubnet, + dockerVolumes + }); + + await fs.writeFile(tfMainPath, JSON.stringify(tpl)); + + await tf.init({ dir: tfPath }); + await tf.apply({ dir: tfPath }); + + const tfStatePath = join(tfPath, 'terraform.tfstate'); + const tfstate = await tf.loadTfState({ path: tfStatePath }); + + return tfstate; + }; + + winston.info('Deploying cloud runner plan...'); + const tfstate = await runTerraform(opts); + const { resources } = tfstate; + for (const resource of resources) { + if (resource.type.startsWith('iterative_')) { + for (const { attributes } of resource.instances) { + const nonSensitiveValues = { + awsSecurityGroup: attributes.aws_security_group, + awsSubnetId: attributes.aws_subnet_id, + cloud: attributes.cloud, + driver: attributes.driver, + id: attributes.id, + idleTimeout: attributes.idle_timeout, + image: attributes.image, + instanceGpu: attributes.instance_gpu, + instanceHddSize: attributes.instance_hdd_size, + instanceIp: attributes.instance_ip, + instanceLaunchTime: attributes.instance_launch_time, + instanceType: attributes.instance_type, + instancePermissionSet: attributes.instance_permission_set, + labels: attributes.labels, + cmlVersion: attributes.cml_version, + metadata: attributes.metadata, + name: attributes.name, + region: attributes.region, + repo: attributes.repo, + single: attributes.single, + spot: attributes.spot, + spotPrice: attributes.spot_price, + timeouts: attributes.timeouts + }; + winston.info(JSON.stringify(nonSensitiveValues)); + } + } + } +}; + +const runLocal = async (opts) => { + winston.info(`Launching ${cml.driver} runner`); + const { + workdir, + name, + labels, + single, + idleTimeout, + noRetry, + dockerVolumes, + tfResource, + tpiVersion + } = opts; + + if (tfResource) { + await tf.checkMinVersion(); + + const tfPath = workdir; + await fs.mkdir(tfPath, { recursive: true }); + const tfMainPath = join(tfPath, 'main.tf.json'); + const tpl = tf.iterativeProviderTpl({ tpiVersion }); + await fs.writeFile(tfMainPath, JSON.stringify(tpl)); + + await tf.init({ dir: tfPath }); + await tf.apply({ dir: tfPath }); + + const path = join(tfPath, 'terraform.tfstate'); + const tfstate = await tf.loadTfState({ path }); + tfstate.resources = [ + JSON.parse(Buffer.from(tfResource, 'base64').toString('utf-8')) + ]; + await tf.saveTfState({ tfstate, path }); + } + + if (process.platform === 'linux') { + const acpiSock = net.connect('/var/run/acpid.socket'); + acpiSock.on('connect', () => { + winston.info('Connected to acpid service.'); + }); + acpiSock.on('error', (err) => { + winston.warn( + `Error connecting to ACPI socket: ${err.message}. The acpid.service helps with instance termination detection.` + ); + }); + acpiSock.on('data', (buf) => { + const data = buf.toString().toLowerCase(); + if (data.includes('power') && data.includes('button')) { + shutdown({ ...opts, reason: 'ACPI shutdown' }); + } + }); + } + + const dataHandler = async (data) => { + const logs = await cml.parseRunnerLog({ data, name }); + for (const log of logs) { + winston.info('runner status', log); + + if (log.status === 'job_started') { + const { job: id, pipeline, date } = log; + RUNNER_JOBS_RUNNING.push({ id, pipeline, date }); + } + + if (log.status === 'job_ended') { + RUNNER_JOBS_RUNNING.pop(); + if (single) await shutdown({ ...opts, reason: 'single job' }); + } + } + }; + + const proc = await cml.startRunner({ + workdir, + name, + labels, + single, + idleTimeout, + dockerVolumes + }); + + proc.stderr.on('data', dataHandler); + proc.stdout.on('data', dataHandler); + proc.on('disconnect', () => + shutdown({ ...opts, error: new Error('runner proccess lost') }) + ); + proc.on('close', (exit) => { + const reason = `runner closed with exit code ${exit}`; + if (exit === 0) shutdown({ ...opts, reason }); + else shutdown({ ...opts, error: new Error(reason) }); + }); + + RUNNER = proc; + if (idleTimeout > 0) { + const watcher = setInterval(async () => { + const idle = RUNNER_JOBS_RUNNING.length === 0; + + if (RUNNER_TIMER >= idleTimeout) { + shutdown({ ...opts, reason: `timeout:${idleTimeout}` }); + clearInterval(watcher); + } + + RUNNER_TIMER = idle ? RUNNER_TIMER + 1 : 0; + }, 1000); + } + + if (!noRetry) { + if (cml.driver === 'github') { + const watcherSeventyTwo = setInterval(() => { + RUNNER_JOBS_RUNNING.forEach((job) => { + if ( + new Date().getTime() - new Date(job.date).getTime() > + GH_5_MIN_TIMEOUT + ) { + shutdown({ ...opts, reason: 'timeout:35days' }); + clearInterval(watcherSeventyTwo); + } + }); + }, 60 * 1000); + } + } +}; + +const run = async (opts) => { + opts.workdir = opts.workdir || `${homedir()}/.cml/${opts.name}`; + + process.on('unhandledRejection', (reason) => + shutdown({ ...opts, error: new Error(reason) }) + ); + process.on('uncaughtException', (error) => shutdown({ ...opts, error })); + + ['SIGTERM', 'SIGINT', 'SIGQUIT'].forEach((signal) => { + process.on(signal, () => shutdown({ ...opts, reason: signal })); + }); + + const { + driver, + workdir, + cloud, + labels, + name, + reuse, + reuseIdle, + dockerVolumes + } = opts; + + await cml.repoTokenCheck(); + + const runners = await cml.runners(); + const runner = await cml.runnerByName({ name, runners }); + if (runner) { + if (!reuse) + throw new Error( + `Runner name ${name} is already in use. Please change the name or terminate the existing runner.` + ); + winston.info(`Reusing existing runner named ${name}...`); + return; + } + + if ( + reuse && + (await cml.runnersByLabels({ labels, runners })).find( + (runner) => runner.online + ) + ) { + winston.info( + `Reusing existing online runners with the ${labels} labels...` + ); + return; + } + + if (reuseIdle) { + if (driver === 'bitbucket') { + throw new Error( + 'cml runner flag --reuse-idle is unsupported by bitbucket' + ); + } + winston.info( + `Checking for existing idle runner matching labels: ${labels}.` + ); + const currentRunners = await cml.runnersByLabels({ labels, runners }); + const availableRunner = currentRunners.find( + (runner) => runner.online && !runner.busy + ); + if (availableRunner) { + winston.info('Found matching idle runner.', availableRunner); + return; + } + } + + if (dockerVolumes.length && cml.driver !== 'gitlab') + winston.warn('Parameters --docker-volumes is only supported in gitlab'); + + if (driver === 'github') + winston.warn( + 'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.' + ); + + if (RUNNER_NAME) + winston.warn( + 'ignoring RUNNER_NAME environment variable, use CML_RUNNER_NAME or --name instead' + ); + + winston.info(`Preparing workdir ${workdir}...`); + await fs.mkdir(workdir, { recursive: true }); + await fs.chmod(workdir, '766'); + + if (cloud) await runCloud(opts); + else await runLocal(opts); +}; + +exports.command = 'launch'; +exports.description = 'Launch and register a self-hosted runner'; + +exports.handler = async (opts) => { + ({ cml } = opts); + try { + await run(opts); + } catch (error) { + await shutdown({ ...opts, error }); + } +}; + +exports.builder = (yargs) => yargs.env('CML_RUNNER').options(exports.options); + +exports.options = kebabcaseKeys({ + labels: { + type: 'string', + default: 'cml', + description: + 'One or more user-defined labels for this runner (delimited with commas)' + }, + idleTimeout: { + type: 'string', + default: '5 minutes', + coerce: (val) => (/^-?\d+$/.test(val) ? parseInt(val) : timestring(val)), + description: + 'Time to wait for jobs before shutting down (e.g. "5min"). Use "never" to disable' + }, + name: { + type: 'string', + default: `cml-${randid()}`, + defaultDescription: 'cml-{ID}', + description: 'Name displayed in the repository once registered' + }, + noRetry: { + type: 'boolean', + description: + 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' + }, + single: { + type: 'boolean', + conflicts: ['reuse', 'reuseIdle'], + description: 'Exit after running a single job' + }, + reuse: { + type: 'boolean', + conflicts: ['single', 'reuseIdle'], + description: + "Don't launch a new runner if an existing one has the same name or overlapping labels" + }, + reuseIdle: { + type: 'boolean', + conflicts: ['reuse', 'single'], + description: + "Creates a new runner only if the matching labels don't exist or are already busy" + }, + workdir: { + type: 'string', + hidden: true, + alias: 'path', + description: 'Runner working directory' + }, + dockerVolumes: { + type: 'array', + default: [], + description: 'Docker volumes, only supported in GitLab' + }, + cloud: { + type: 'string', + choices: ['aws', 'azure', 'gcp', 'kubernetes'], + description: 'Cloud to deploy the runner' + }, + cloudRegion: { + type: 'string', + default: 'us-west', + description: + 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' + }, + cloudType: { + type: 'string', + description: + 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' + }, + cloudPermissionSet: { + type: 'string', + default: '', + description: + 'Specifies the instance profile in AWS or instance service account in GCP' + }, + cloudMetadata: { + type: 'array', + string: true, + default: [], + coerce: (items) => { + const keyValuePairs = items.map((item) => [...item.split(/=(.+)/), null]); + return Object.fromEntries(keyValuePairs); + }, + description: + 'Key Value pairs to associate cml-runner instance on the provider i.e. tags/labels "key=value"' + }, + cloudGpu: { + type: 'string', + description: + 'GPU type. Choices: k80, v100, or native types e.g. nvidia-tesla-t4', + coerce: (val) => (val === 'nogpu' ? undefined : val) + }, + cloudHddSize: { + type: 'number', + description: 'HDD size in GB' + }, + cloudSshPrivate: { + type: 'string', + description: + 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' + }, + cloudSpot: { + type: 'boolean', + description: 'Request a spot instance' + }, + cloudSpotPrice: { + type: 'number', + default: -1, + description: + 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' + }, + cloudStartupScript: { + type: 'string', + description: + 'Run the provided Base64-encoded Linux shell script during the instance initialization' + }, + cloudAwsSecurityGroup: { + type: 'string', + default: '', + description: 'Specifies the security group in AWS' + }, + cloudAwsSubnet: { + type: 'string', + default: '', + description: 'Specifies the subnet to use within AWS', + alias: 'cloud-aws-subnet-id' + }, + tpiVersion: { + type: 'string', + default: '>= 0.9.10', + description: + 'Pin the iterative/iterative terraform provider to a specific version. i.e. "= 0.10.4" See: https://www.terraform.io/language/expressions/version-constraints', + hidden: true + }, + cmlVersion: { + type: 'string', + default: require('../../../package.json').version, + description: 'CML version to load on TPI instance', + hidden: true + }, + tfResource: { + hidden: true, + alias: 'tf_resource' + }, + destroyDelay: { + type: 'number', + default: 10, + hidden: true, + description: + 'Seconds to wait for collecting logs on failure (https://github.com/iterative/cml/issues/413)' + } +}); diff --git a/bin/cml/runner/launch.test.js b/bin/cml/runner/launch.test.js new file mode 100644 index 000000000..12a970b18 --- /dev/null +++ b/bin/cml/runner/launch.test.js @@ -0,0 +1,153 @@ +jest.setTimeout(2000000); + +const isIp = require('is-ip'); +const { CML } = require('../../../src/cml'); +const { exec, sshConnection, randid, sleep } = require('../../../src/utils'); + +const IDLE_TIMEOUT = 15; +const { + TEST_GITHUB_TOKEN, + TEST_GITHUB_REPO, + TEST_GITLAB_TOKEN, + TEST_GITLAB_REPO, + SSH_PRIVATE +} = process.env; + +const launchRunner = async (opts) => { + const { cloud, type, repo, token, privateKey, name } = opts; + const command = `node ./bin/cml.js runner --cloud ${cloud} --cloud-type ${type} --repo ${repo} --token ${token} --cloud-ssh-private="${privateKey}" --name ${name} --cloud-spot true --idle-timeout ${IDLE_TIMEOUT}`; + + const output = await exec(command); + const state = JSON.parse(output.split(/\n/).pop()); + + return state; +}; + +const testRunner = async (opts) => { + const { repo, token, name, privateKey } = opts; + const { instanceIp: host } = await launchRunner(opts); + expect(isIp(host)).toBe(true); + + const sshOpts = { host, username: 'ubuntu', privateKey }; + const cml = new CML({ repo, token }); + + let runner = await cml.runnerByName({ name }); + expect(runner).not.toBe(undefined); + await sshConnection(sshOpts); + + await sleep(IDLE_TIMEOUT + 60); + + runner = await cml.runnerByName({ name }); + expect(runner).toBe(undefined); + + let sshErr; + try { + await sshConnection(sshOpts); + } catch (err) { + sshErr = err; + } + expect(sshErr).not.toBe(undefined); +}; + +describe('CML e2e', () => { + test('cml-runner --help', async () => { + const output = await exec(`echo none | node ./bin/cml.js runner --help`); + + expect(output).toMatchInlineSnapshot(` + "cml.js runner + + Manage self-hosted (cloud & on-premise) CI runners + + Commands: + cml.js runner launch Launch and register a self-hosted runner + + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean]" + `); + }); + + test.skip('cml-runner GL/AWS', async () => { + const opts = { + repo: TEST_GITLAB_REPO, + token: TEST_GITLAB_TOKEN, + privateKey: SSH_PRIVATE, + cloud: 'aws', + type: 't2.micro', + name: `cml-test-${randid()}` + }; + + await testRunner(opts); + }); + + test.skip('cml-runner GH/AWS', async () => { + const opts = { + repo: TEST_GITHUB_REPO, + token: TEST_GITHUB_TOKEN, + privateKey: SSH_PRIVATE, + cloud: 'aws', + type: 't2.micro', + name: `cml-test-${randid()}` + }; + + await testRunner(opts); + }); + + test.skip('cml-runner GL/Azure', async () => { + const opts = { + repo: TEST_GITLAB_REPO, + token: TEST_GITLAB_TOKEN, + privateKey: SSH_PRIVATE, + cloud: 'azure', + type: 'm', + name: `cml-test-${randid()}` + }; + + await testRunner(opts); + }); + + test.skip('cml-runner GH/Azure', async () => { + const opts = { + repo: TEST_GITHUB_REPO, + token: TEST_GITHUB_TOKEN, + privateKey: SSH_PRIVATE, + cloud: 'azure', + type: 'm', + name: `cml-test-${randid()}` + }; + + await testRunner(opts); + }); + + test.skip('cml-runner GL/GCP', async () => { + const opts = { + repo: TEST_GITLAB_REPO, + token: TEST_GITLAB_TOKEN, + privateKey: SSH_PRIVATE, + cloud: 'gcp', + type: 'm', + name: `cml-test-${randid()}` + }; + + await testRunner(opts); + }); + + test.skip('cml-runner GH/GCP', async () => { + const opts = { + repo: TEST_GITHUB_REPO, + token: TEST_GITHUB_TOKEN, + privateKey: SSH_PRIVATE, + cloud: 'gcp', + type: 'm', + name: `cml-test-${randid()}` + }; + + await testRunner(opts); + }); +}); diff --git a/bin/cml/send-comment.js b/bin/cml/send-comment.js deleted file mode 100644 index af11e0027..000000000 --- a/bin/cml/send-comment.js +++ /dev/null @@ -1,58 +0,0 @@ -const kebabcaseKeys = require('kebabcase-keys'); - -const { repoOptions } = require('../../src/cml'); - -exports.command = 'send-comment '; -exports.description = 'Comment on a commit'; - -exports.handler = async (opts) => { - const { cml } = opts; - console.log(await cml.commentCreate(opts)); -}; - -exports.builder = (yargs) => - yargs.env('CML_SEND_COMMENT').options( - kebabcaseKeys({ - ...repoOptions, - pr: { - type: 'boolean', - description: - 'Post to an existing PR/MR associated with the specified commit' - }, - commitSha: { - type: 'string', - alias: 'head-sha', - default: 'HEAD', - description: 'Commit SHA linked to this comment' - }, - publish: { - type: 'boolean', - description: - 'Upload local files and images linked from the Markdown report' - }, - watch: { - type: 'boolean', - description: 'Watch for changes and automatically update the report' - }, - triggerFile: { - type: 'string', - description: 'File used to trigger the watcher', - hidden: true - }, - native: { - type: 'boolean', - description: - "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." - }, - update: { - type: 'boolean', - description: - 'Update the last CML comment (if any) instead of creating a new one' - }, - rmWatermark: { - type: 'boolean', - description: - 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' - } - }) - ); diff --git a/bin/cml/send-github-check.js b/bin/cml/send-github-check.js deleted file mode 100755 index a2f4bc8b7..000000000 --- a/bin/cml/send-github-check.js +++ /dev/null @@ -1,54 +0,0 @@ -const fs = require('fs').promises; -const kebabcaseKeys = require('kebabcase-keys'); - -const { repoOptions } = require('../../src/cml'); - -exports.command = 'send-github-check '; -exports.description = 'Create a check report'; - -exports.handler = async (opts) => { - const { cml, markdownfile } = opts; - const report = await fs.readFile(markdownfile, 'utf-8'); - await cml.checkCreate({ ...opts, report }); -}; - -exports.builder = (yargs) => - yargs.env('CML_SEND_GITHUB_CHECK').options( - kebabcaseKeys({ - ...repoOptions, - token: { - type: 'string', - description: - "GITHUB_TOKEN or Github App token. Personal access token won't work" - }, - commitSha: { - type: 'string', - alias: 'head-sha', - description: 'Commit SHA linked to this comment. Defaults to HEAD.' - }, - conclusion: { - type: 'string', - choices: [ - 'success', - 'failure', - 'neutral', - 'cancelled', - 'skipped', - 'timed_out' - ], - default: 'success', - description: 'Sets the conclusion status of the check.' - }, - status: { - type: 'string', - choices: ['queued', 'in_progress', 'completed'], - default: 'completed', - description: 'Sets the status of the check.' - }, - title: { - type: 'string', - default: 'CML Report', - description: 'Sets title of the check.' - } - }) - ); diff --git a/bin/cml/tensorboard.js b/bin/cml/tensorboard.js new file mode 100644 index 000000000..851141bfc --- /dev/null +++ b/bin/cml/tensorboard.js @@ -0,0 +1,13 @@ +exports.command = 'tensorboard'; +exports.description = 'Manage tensorboard.dev connections'; +exports.builder = (yargs) => + yargs + .options({ + driver: { hidden: true }, + repo: { hidden: true }, + token: { hidden: true } + }) + .commandDir('./tensorboard', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/tensorboard-dev.js b/bin/cml/tensorboard/connect.js similarity index 63% rename from bin/cml/tensorboard-dev.js rename to bin/cml/tensorboard/connect.js index 0cdb2d9a2..809d50809 100644 --- a/bin/cml/tensorboard-dev.js +++ b/bin/cml/tensorboard/connect.js @@ -4,7 +4,7 @@ const { spawn } = require('child_process'); const { homedir } = require('os'); const tempy = require('tempy'); -const { exec, watermarkUri, sleep } = require('../../src/utils'); +const { exec, watermarkUri, sleep } = require('../../../src/utils'); const tbLink = async (opts = {}) => { const { stdout, stderror, title, name, rmWatermark, md, timeout = 60 } = opts; @@ -72,8 +72,8 @@ const launchAndWaitLink = async (opts = {}) => { }; exports.tbLink = tbLink; -exports.command = 'tensorboard-dev'; -exports.description = 'Get a tensorboard link'; +exports.command = 'connect'; +exports.description = 'Connect to tensorboard.dev and get a link'; exports.handler = async (opts) => { const { file, credentials, name, description } = opts; @@ -95,48 +95,47 @@ exports.handler = async (opts) => { }; exports.builder = (yargs) => - yargs.env('CML_TENSORBOARD_DEV').options( - kebabcaseKeys({ - credentials: { - type: 'string', - alias: 'c', - required: true, - description: - 'TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable CML_TENSORBOARD_DEV_CREDENTIALS.' - }, - logdir: { - type: 'string', - description: 'Directory containing the logs to process.' - }, - name: { - type: 'string', - description: 'Tensorboard experiment title. Max 100 characters.' - }, - description: { - type: 'string', - description: - 'Tensorboard experiment description. Markdown format. Max 600 characters.' - }, - md: { - type: 'boolean', - description: 'Output as markdown [title || name](url).' - }, - title: { - type: 'string', - alias: 't', - description: - 'Markdown title, if not specified, param name will be used.' - }, - file: { - type: 'string', - alias: 'f', - description: - 'Append the output to the given file. Create it if does not exist.', - hidden: true - }, - rmWatermark: { - type: 'boolean', - description: 'Avoid CML watermark.' - } - }) - ); + yargs.env('CML_TENSORBOARD').options(exports.options); + +exports.options = kebabcaseKeys({ + credentials: { + type: 'string', + alias: 'c', + required: true, + description: + 'TensorBoard credentials as JSON, usually found at ~/.config/tensorboard/credentials/uploader-creds.json' + }, + logdir: { + type: 'string', + description: 'Directory containing the logs to process' + }, + name: { + type: 'string', + description: 'Tensorboard experiment title; max 100 characters' + }, + description: { + type: 'string', + description: + 'Tensorboard experiment description in Markdown format; max 600 characters' + }, + md: { + type: 'boolean', + description: 'Output as markdown [title || name](url)' + }, + title: { + type: 'string', + alias: 't', + description: 'Markdown title, if not specified, param name will be used' + }, + file: { + type: 'string', + alias: 'f', + description: + 'Append the output to the given file or create it if does not exist', + hidden: true + }, + rmWatermark: { + type: 'boolean', + description: 'Avoid CML watermark' + } +}); diff --git a/bin/cml/tensorboard-dev.test.js b/bin/cml/tensorboard/connect.test.js similarity index 73% rename from bin/cml/tensorboard-dev.test.js rename to bin/cml/tensorboard/connect.test.js index 7b94bf2a2..fd9fa83b5 100644 --- a/bin/cml/tensorboard-dev.test.js +++ b/bin/cml/tensorboard/connect.test.js @@ -1,7 +1,7 @@ const fs = require('fs').promises; const tempy = require('tempy'); -const { exec, isProcRunning, sleep } = require('../../src/utils'); -const { tbLink } = require('./tensorboard-dev'); +const { exec, isProcRunning, sleep } = require('../../../src/utils'); +const { tbLink } = require('./connect'); const CREDENTIALS = '{"refresh_token": "1//03FiVnGk2xhnNCgYIARAAGAMSNwF-L9IrPH8FOOVWEYUihFDToqxyLArxfnbKFmxEfhzys_KYVVzBisYlAy225w4HaX3ais5TV_Q", "token_uri": "https://oauth2.googleapis.com/token", "client_id": "373649185512-8v619h5kft38l4456nm2dj4ubeqsrvh6.apps.googleusercontent.com", "client_secret": "pOyAuU2yq2arsM98Bw5hwYtr", "scopes": ["openid", "https://www.googleapis.com/auth/userinfo.email"], "type": "authorized_user"}'; @@ -57,25 +57,30 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js tensorboard-dev - Get a tensorboard link + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - -c, --credentials TB credentials as json. Usually found at - ~/.config/tensorboard/credentials/uploader-creds.json. If - not specified will look for the json at the env variable - CML_TENSORBOARD_DEV_CREDENTIALS. [string] [required] - --logdir Directory containing the logs to process. [string] - --name Tensorboard experiment title. Max 100 characters. [string] - --description Tensorboard experiment description. Markdown format. Max - 600 characters. [string] - --md Output as markdown [title || name](url). [boolean] - -t, --title Markdown title, if not specified, param name will be used. + -c, --credentials TensorBoard credentials as JSON, usually found at + ~/.config/tensorboard/credentials/uploader-creds.json + [string] [required] + --logdir Directory containing the logs to process [string] + --name Tensorboard experiment title; max 100 characters [string] + --description Tensorboard experiment description in Markdown format; max + 600 characters [string] + --md Output as markdown [title || name](url) [boolean] + -t, --title Markdown title, if not specified, param name will be used [string] - --rm-watermark Avoid CML watermark. [boolean]" + --rm-watermark Avoid CML watermark [boolean]" `); }); diff --git a/bin/cml/workflow.js b/bin/cml/workflow.js new file mode 100644 index 000000000..d58fa5909 --- /dev/null +++ b/bin/cml/workflow.js @@ -0,0 +1,8 @@ +exports.command = 'workflow'; +exports.description = 'Manage CI workflows'; +exports.builder = (yargs) => + yargs + .commandDir('./workflow', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/workflow/rerun.js b/bin/cml/workflow/rerun.js new file mode 100644 index 000000000..18c5f0936 --- /dev/null +++ b/bin/cml/workflow/rerun.js @@ -0,0 +1,18 @@ +const kebabcaseKeys = require('kebabcase-keys'); + +exports.command = 'rerun'; +exports.description = 'Rerun a workflow given the jobId or workflowId'; + +exports.handler = async (opts) => { + const { cml } = opts; + await cml.pipelineRerun(opts); +}; + +exports.builder = (yargs) => yargs.env('CML_WORKFLOW').options(exports.options); + +exports.options = kebabcaseKeys({ + id: { + type: 'string', + description: 'Run identifier to be rerun' + } +}); diff --git a/bin/cml/workflow/rerun.test.js b/bin/cml/workflow/rerun.test.js new file mode 100644 index 000000000..17a335b39 --- /dev/null +++ b/bin/cml/workflow/rerun.test.js @@ -0,0 +1,26 @@ +const { exec } = require('../../../src/utils'); + +describe('CML e2e', () => { + test('cml-ci --help', async () => { + const output = await exec( + `echo none | node ./bin/cml.js rerun-workflow --help` + ); + + expect(output).toMatchInlineSnapshot(` + "cml.js rerun-workflow + + Global Options: + --log Logging verbosity + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Git provider where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] + + Options: + --id Run identifier to be rerun [string]" + `); + }); +}); diff --git a/bin/legacy/commands/ci.js b/bin/legacy/commands/ci.js new file mode 100644 index 000000000..348043615 --- /dev/null +++ b/bin/legacy/commands/ci.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../../cml/repo/prepare'); + +exports.command = 'ci'; +exports.description = 'Prepare Git repository for CML operations'; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/commands/publish.js b/bin/legacy/commands/publish.js new file mode 100644 index 000000000..c9f33ab97 --- /dev/null +++ b/bin/legacy/commands/publish.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../../cml/asset/publish'); + +exports.command = 'publish '; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/commands/rerun-workflow.js b/bin/legacy/commands/rerun-workflow.js new file mode 100644 index 000000000..56aa257fd --- /dev/null +++ b/bin/legacy/commands/rerun-workflow.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../../cml/workflow/rerun'); + +exports.command = 'rerun-workflow'; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/commands/send-comment.js b/bin/legacy/commands/send-comment.js new file mode 100644 index 000000000..0eae9b26b --- /dev/null +++ b/bin/legacy/commands/send-comment.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../../cml/comment/create'); + +exports.command = 'send-comment '; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/commands/send-github-check.js b/bin/legacy/commands/send-github-check.js new file mode 100644 index 000000000..61f0483a3 --- /dev/null +++ b/bin/legacy/commands/send-github-check.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../../cml/check/create'); + +exports.command = 'send-github-check '; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/commands/tensorboard-dev.js b/bin/legacy/commands/tensorboard-dev.js new file mode 100644 index 000000000..a59c018db --- /dev/null +++ b/bin/legacy/commands/tensorboard-dev.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../../cml/tensorboard/connect'); + +exports.command = 'tensorboard-dev'; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy.js b/bin/legacy/link.js similarity index 100% rename from bin/legacy.js rename to bin/legacy/link.js diff --git a/bin/legacy.test.js b/bin/legacy/link.test.js similarity index 84% rename from bin/legacy.test.js rename to bin/legacy/link.test.js index 7b6171f45..58dfd09bb 100644 --- a/bin/legacy.test.js +++ b/bin/legacy/link.test.js @@ -1,5 +1,5 @@ -const { bin } = require('../package.json'); -const { exec } = require('../src/utils'); +const { bin } = require('../../package.json'); +const { exec } = require('../../src/utils'); const commands = Object.keys(bin) .filter((command) => command.startsWith('cml-')) diff --git a/package-lock.json b/package-lock.json index ec6248340..5e46aa034 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,13 +49,12 @@ }, "bin": { "cml": "bin/cml.js", - "cml-cloud-runner-entrypoint": "bin/legacy.js", - "cml-pr": "bin/legacy.js", - "cml-publish": "bin/legacy.js", - "cml-runner": "bin/legacy.js", - "cml-send-comment": "bin/legacy.js", - "cml-send-github-check": "bin/legacy.js", - "cml-tensorboard-dev": "bin/legacy.js" + "cml-pr": "bin/legacy/link.js", + "cml-publish": "bin/legacy/link.js", + "cml-runner": "bin/legacy/link.js", + "cml-send-comment": "bin/legacy/link.js", + "cml-send-github-check": "bin/legacy/link.js", + "cml-tensorboard-dev": "bin/legacy/link.js" }, "devDependencies": { "eslint": "^8.1.0", diff --git a/package.json b/package.json index c4536cba2..d529ba5a2 100644 --- a/package.json +++ b/package.json @@ -34,13 +34,12 @@ }, "bin": { "cml": "bin/cml.js", - "cml-send-github-check": "bin/legacy.js", - "cml-send-comment": "bin/legacy.js", - "cml-publish": "bin/legacy.js", - "cml-tensorboard-dev": "bin/legacy.js", - "cml-runner": "bin/legacy.js", - "cml-cloud-runner-entrypoint": "bin/legacy.js", - "cml-pr": "bin/legacy.js" + "cml-send-github-check": "bin/legacy/link.js", + "cml-send-comment": "bin/legacy/link.js", + "cml-publish": "bin/legacy/link.js", + "cml-tensorboard-dev": "bin/legacy/link.js", + "cml-runner": "bin/legacy/link.js", + "cml-pr": "bin/legacy/link.js" }, "scripts": { "lintfix": "eslint --fix ./ && prettier --write '**/*.{js,json,md,yaml,yml}'", diff --git a/src/cml.js b/src/cml.js index ba07d4a78..505c1d749 100755 --- a/src/cml.js +++ b/src/cml.js @@ -229,20 +229,26 @@ class CML { } if (watch) { - let lock; + let first = true; + let lock = false; watcher.add(triggerFile || markdownFile); watcher.on('all', async (event, path) => { if (lock) return; lock = true; try { winston.info(`watcher event: ${event} ${path}`); - await this.commentCreate({ ...opts, update: true, watch: false }); + await this.commentCreate({ + ...opts, + update: update || !first, + watch: false + }); if (event !== 'unlink' && path === triggerFile) { await fs.unlink(triggerFile); } } catch (err) { winston.warn(err); } + first = false; lock = false; }); winston.info('watching for file changes...'); @@ -576,29 +582,10 @@ Automated commits for ${this.repo}/commit/${sha} created by CML. } } -const repoOptions = { - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' - } -}; - module.exports = { CML, default: CML, GIT_USER_EMAIL, GIT_USER_NAME, - GIT_REMOTE, - repoOptions + GIT_REMOTE };