diff --git a/bin/cml-runner.js b/bin/cml-runner.js index 2978605c6..3c39817a3 100755 --- a/bin/cml-runner.js +++ b/bin/cml-runner.js @@ -323,7 +323,9 @@ const run = async (opts) => { // if (name !== NAME) { await cml.repoTokenCheck(); - if (await cml.runnerByName({ name })) { + const runners = await cml.runners(); + const runner = await cml.runnerByName({ name, runners }); + if (runner) { if (!reuse) throw new Error( `Runner name ${name} is already in use. Please change the name or terminate the other runner.` @@ -334,7 +336,9 @@ const run = async (opts) => { if ( reuse && - (await cml.runnersByLabels({ labels })).find((runner) => runner.online) + (await cml.runnersByLabels({ labels, runners })).find( + (runner) => runner.online + ) ) { console.log(`Reusing existing online runners with the ${labels} labels...`); process.exit(0); diff --git a/package-lock.json b/package-lock.json index fd0db40eb..66402b757 100644 --- a/package-lock.json +++ b/package-lock.json @@ -662,6 +662,11 @@ "universal-user-agent": "^6.0.0" } }, + "@octokit/openapi-types": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-8.2.1.tgz", + "integrity": "sha512-BJz6kWuL3n+y+qM8Pv+UGbSxH6wxKf/SBs5yzGufMHwDefsa+Iq7ZGy1BINMD2z9SkXlIzk1qiu988rMuGXEMg==" + }, "@octokit/plugin-paginate-rest": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.3.2.tgz", @@ -670,6 +675,11 @@ "@octokit/types": "^5.3.0" } }, + "@octokit/plugin-request-log": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz", + "integrity": "sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA==" + }, "@octokit/plugin-rest-endpoint-methods": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.1.4.tgz", @@ -679,6 +689,25 @@ "deprecation": "^2.3.1" } }, + "@octokit/plugin-throttling": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.5.1.tgz", + "integrity": "sha512-d2jh3/RZo98DRw2J0jFxhKz7nrTGalGdkfRtxM+pI5k1wRb4BKBjiuE9cuZnhZyj+zLC1EcIptj7K+28LJZ3eA==", + "requires": { + "@octokit/types": "^6.0.1", + "bottleneck": "^2.15.3" + }, + "dependencies": { + "@octokit/types": { + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.18.1.tgz", + "integrity": "sha512-5YsddjO1U+xC8ZYKV8yZYebW55PCc7qiEEeZ+wZRr6qyclynzfyD65KZ5FdtIeP0/cANyFaD7hV69qElf1nMsQ==", + "requires": { + "@octokit/openapi-types": "^8.2.1" + } + } + } + }, "@octokit/request": { "version": "5.4.7", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.7.tgz", @@ -704,6 +733,114 @@ "once": "^1.4.0" } }, + "@octokit/rest": { + "version": "18.6.7", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-18.6.7.tgz", + "integrity": "sha512-Kn6WrI2ZvmAztdx+HEaf88RuJn+LK72S8g6OpciE4kbZddAN84fu4fiPGxcEu052WmqKVnA/cnQsbNlrYC6rqQ==", + "requires": { + "@octokit/core": "^3.5.0", + "@octokit/plugin-paginate-rest": "^2.6.2", + "@octokit/plugin-request-log": "^1.0.2", + "@octokit/plugin-rest-endpoint-methods": "5.4.1" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.5.tgz", + "integrity": "sha512-BpGYsPgJt05M7/L/5FoE1PiAbdxXFZkX/3kDYcsvd1v6UhlnE5e96dTDr0ezX/EFwciQxf3cNV0loipsURU+WA==", + "requires": { + "@octokit/types": "^6.0.3" + } + }, + "@octokit/core": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.5.1.tgz", + "integrity": "sha512-omncwpLVxMP+GLpLPgeGJBF6IWJFjXDS5flY5VbppePYX9XehevbDykRH9PdCdvqt9TS5AOTiDide7h0qrkHjw==", + "requires": { + "@octokit/auth-token": "^2.4.4", + "@octokit/graphql": "^4.5.8", + "@octokit/request": "^5.6.0", + "@octokit/request-error": "^2.0.5", + "@octokit/types": "^6.0.3", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.6.4.tgz", + "integrity": "sha512-SWTdXsVheRmlotWNjKzPOb6Js6tjSqA2a8z9+glDJng0Aqjzti8MEWOtuT8ZSu6wHnci7LZNuarE87+WJBG4vg==", + "requires": { + "@octokit/request": "^5.6.0", + "@octokit/types": "^6.0.3", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/plugin-paginate-rest": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.14.0.tgz", + "integrity": "sha512-S2uEu2uHeI7Vf+Lvj8tv3O5/5TCAa8GHS0dUQN7gdM7vKA6ZHAbR6HkAVm5yMb1mbedLEbxOuQ+Fa0SQ7tCDLA==", + "requires": { + "@octokit/types": "^6.18.0" + } + }, + "@octokit/plugin-rest-endpoint-methods": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.4.1.tgz", + "integrity": "sha512-Nx0g7I5ayAYghsLJP4Q1Ch2W9jYYM0FlWWWZocUro8rNxVwuZXGfFd7Rcqi9XDWepSXjg1WByiNJnZza2hIOvQ==", + "requires": { + "@octokit/types": "^6.18.1", + "deprecation": "^2.3.1" + } + }, + "@octokit/request": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.0.tgz", + "integrity": "sha512-4cPp/N+NqmaGQwbh3vUsYqokQIzt7VjsgTYVXiwpUP2pxd5YiZB2XuTedbb0SPtv9XS7nzAKjAuQxmY8/aZkiA==", + "requires": { + "@octokit/endpoint": "^6.0.1", + "@octokit/request-error": "^2.1.0", + "@octokit/types": "^6.16.1", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.1", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "requires": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.18.1.tgz", + "integrity": "sha512-5YsddjO1U+xC8ZYKV8yZYebW55PCc7qiEEeZ+wZRr6qyclynzfyD65KZ5FdtIeP0/cANyFaD7hV69qElf1nMsQ==", + "requires": { + "@octokit/openapi-types": "^8.2.1" + } + }, + "before-after-hook": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", + "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + } + } + }, "@octokit/types": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-5.4.1.tgz", @@ -1205,6 +1342,11 @@ "file-uri-to-path": "1.0.0" } }, + "bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", diff --git a/package.json b/package.json index 745dd4e62..80b014042 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,8 @@ "dependencies": { "@actions/core": "^1.2.5", "@actions/github": "^4.0.0", + "@octokit/plugin-throttling": "^3.5.1", + "@octokit/rest": "^18.6.7", "form-data": "^3.0.0", "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", diff --git a/src/cml.js b/src/cml.js index b3ba264d0..0ed5f47a7 100644 --- a/src/cml.js +++ b/src/cml.js @@ -217,19 +217,23 @@ class CML { return await getDriver(this).unregisterRunner({ runnerId, ...opts }); } - async getRunners(opts = {}) { - return await getDriver(this).getRunners(opts); + async runners(opts = {}) { + return await getDriver(this).runners(opts); } async runnerByName(opts = {}) { - const { name } = opts; - const runners = await this.getRunners(opts); + let { name, runners } = opts; + + if (!runners) runners = await this.runners(opts); + return runners.find((runner) => runner.name === name); } async runnersByLabels(opts = {}) { - const { labels } = opts; - const runners = await this.getRunners(opts); + let { labels, runners } = opts; + + if (!runners) runners = await this.runners(opts); + return runners.filter((runner) => labels.split(',').every((label) => runner.labels.includes(label)) ); diff --git a/src/drivers/bitbucket_cloud.js b/src/drivers/bitbucket_cloud.js index cbb8b139d..eee4e900f 100644 --- a/src/drivers/bitbucket_cloud.js +++ b/src/drivers/bitbucket_cloud.js @@ -105,8 +105,8 @@ class BitbucketCloud { throw new Error('Bitbucket Cloud does not support unregisterRunner!'); } - async getRunners(opts = {}) { - throw new Error('Bitbucket Cloud does not support getRunners!'); + async runners(opts = {}) { + throw new Error('Bitbucket Cloud does not support runners!'); } async prCreate(opts = {}) { diff --git a/src/drivers/github.js b/src/drivers/github.js index b14f4e90a..4419f3b79 100644 --- a/src/drivers/github.js +++ b/src/drivers/github.js @@ -4,6 +4,8 @@ const { resolve } = require('path'); const fs = require('fs').promises; const github = require('@actions/github'); +const { Octokit } = require('@octokit/rest'); +const { throttling } = require('@octokit/plugin-throttling'); const tar = require('tar'); const { download, exec } = require('../utils'); @@ -42,7 +44,19 @@ const ownerRepo = (opts) => { const octokit = (token, repo) => { if (!token) throw new Error('token not found'); - const octokitOptions = {}; + const throttleHandler = (retryAfter, options) => { + if (options.request.retryCount <= 5) { + console.log(`Retrying after ${retryAfter} seconds!`); + return true; + } + }; + const octokitOptions = { + auth: token, + throttle: { + onRateLimit: throttleHandler, + onAbuseLimit: throttleHandler + } + }; if (!repo.includes('github.com')) { // GitHub Enterprise, use the: repo URL host + '/api/v3' - as baseURL @@ -51,7 +65,8 @@ const octokit = (token, repo) => { octokitOptions.baseUrl = `https://${host}/api/v3`; } - return github.getOctokit(token, octokitOptions); + const MyOctokit = Octokit.plugin(throttling); + return new MyOctokit(octokitOptions); }; class Github { @@ -220,19 +235,21 @@ class Github { } } - async getRunners(opts = {}) { + async runners(opts = {}) { const { owner, repo } = ownerRepo({ uri: this.repo }); const { paginate, actions } = octokit(this.token, this.repo); let runners; if (typeof repo === 'undefined') { runners = await paginate(actions.listSelfHostedRunnersForOrg, { - org: owner + org: owner, + per_page: 100 }); } else { runners = await paginate(actions.listSelfHostedRunnersForRepo, { owner, - repo + repo, + per_page: 100 }); } diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index a7654e3a7..755ed0f4f 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -172,7 +172,7 @@ class Gitlab { } } - async getRunners(opts = {}) { + async runners(opts = {}) { const endpoint = `/runners?per_page=100`; const runners = await this.request({ endpoint, method: 'GET' }); return await Promise.all(