shutdown redone

.eslintrc.js

@@ -4,26 +4,21 @@ module.exports = {
     commonjs: true,
     es6: true
   },
-  extends: [
-    'standard',
-    'prettier'
-  ],
+  extends: ['standard', 'prettier'],
   globals: {
     Atomics: 'readonly',
     SharedArrayBuffer: 'readonly'
   },
   parserOptions: {
     ecmaVersion: 2018
   },
-  ignorePatterns: ['assets/',  'dist/', 'node_modules/'],
+  ignorePatterns: ['assets/', 'dist/', 'node_modules/'],
   rules: {
-    'camelcase': [1, {'properties': 'never'}],
+    camelcase: [1, { properties: 'never' }],
     'prettier/prettier': 'error'
   },
-  plugins: [
-    'prettier'
-  ],
+  plugins: ['prettier'],
   env: {
     jest: true
   }
-}
+};

.github/workflows/gitlab.yml

@@ -1,8 +1,5 @@
 name: GitLab
-on:
-  workflow_dispatch:
-  pull_request:
-  push:
+on: pull_request
 jobs:
   test:
     runs-on: ubuntu-latest

.github/workflows/test-deploy.yml

@@ -1,7 +1,7 @@
 name: Test & Deploy
 on:
   schedule:
-    - cron: 0 0 * * *
+    - cron: 0 1 * * 6 # Sat 01:00
   release:
     types: [published]
   pull_request_target:
@@ -46,7 +46,7 @@ jobs:
           TEST_GITLAB_SHA: c4c13286e78dc252dd2611f31a755f10d343fbd4
           TEST_BBCLOUD_TOKEN: ${{ secrets.TEST_BBCLOUD_TOKEN }}
           TEST_BBCLOUD_REPO: https://bitbucket.org/iterative-ai/cml-qa-tests-dummy
-          TEST_BBCLOUD_SHA: 9bb9131ce0af294fe1c6eedca1f2bce3983e80bd
+          TEST_BBCLOUD_SHA: b511535a89f76d3d311b1c15e3e712b15c0b94e3
   packages:
     needs: [lint, test]
     runs-on: ubuntu-latest

.restyled.yaml

@@ -1,8 +1,4 @@
 restylers:
   - name: prettier
     arguments: ['--write']
-    include:
-      - 'src/**/*.js'
-      - 'bin/**/*.js'
-      - 'assets/*.json'
-      - './**/*.{yaml,yml,md,json}'
+    include: ['**/*.{js,json,md,yaml,yml}']

Dockerfile

@@ -1,8 +1,7 @@
 ARG BASE_IMAGE=ubuntu:20.04
 FROM ${BASE_IMAGE}
 
-# TODO: consider using iterative.ai or something else
-LABEL maintainer="dvcorg <[email protected]>"
+LABEL maintainer="CML <[email protected]>"
 
 # CONFIGURE NON-INTERACTIVE APT
 ENV DEBIAN_FRONTEND=noninteractive

bin/cml-runner.js

@@ -50,6 +50,8 @@ const shutdown = async (opts) => {
   if (error) console.error(error);
 
   const unregisterRunner = async () => {
+    if (!RUNNER) return;
+
     try {
       console.log(`Unregistering runner ${name}...`);
       await cml.unregisterRunner({ name });
@@ -60,7 +62,23 @@ const shutdown = async (opts) => {
     }
   };
 
-  const shutdownDockerMachine = async () => {
+  const retryWorkflows = async () => {
+    try {
+      if (!noRetry && RUNNER_JOBS_RUNNING.length) {
+        await Promise.all(
+          RUNNER_JOBS_RUNNING.map(
+            async (job) => await cml.pipelineRestart({ jobId: job.id })
+          )
+        );
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  };
+
+  const destroyDockerMachine = async () => {
+    if (!DOCKER_MACHINE) return;
+
     console.log('docker-machine destroy...');
     console.log(
       'Docker machine is deprecated and will be removed!! Check how to deploy using our tf provider.'
@@ -74,6 +92,8 @@ const shutdown = async (opts) => {
   };
 
   const destroyTerraform = async () => {
+    if (!tfResource) return;
+
     try {
       console.log(await tf.destroy({ dir: tfPath }));
     } catch (err) {
@@ -85,38 +105,20 @@ const shutdown = async (opts) => {
   if (cloud) {
     await destroyTerraform();
   } else {
-    await sleep(RUNNER_DESTROY_DELAY);
+    await unregisterRunner();
+    await retryWorkflows();
 
-    try {
-      if (!noRetry && RUNNER_JOBS_RUNNING.length) {
-        await Promise.all(
-          RUNNER_JOBS_RUNNING.map(
-            async (job) => await cml.pipelineRestart({ jobId: job.id })
-          )
-        );
-      }
-    } catch (err) {
-      console.log(err);
-    }
-
-    RUNNER && (await unregisterRunner());
-
-    if (!tfResource) {
-      console.log(`\tNo TF resource found`);
-    } else {
-      await destroyTerraform();
-    }
+    if (DOCKER_MACHINE || tfResource) await sleep(RUNNER_DESTROY_DELAY);
+    await destroyDockerMachine();
+    await destroyTerraform();
 
-    DOCKER_MACHINE && (await shutdownDockerMachine());
+    RUNNER && RUNNER.kill('SIGINT');
   }
 
-  RUNNER && RUNNER.kill('SIGINT');
   process.exit(error ? 1 : 0);
 };
 
 const runCloud = async (opts) => {
-  const { cloudSshPrivateVisible } = opts;
-
   const runTerraform = async (opts) => {
     console.log('Terraform apply...');
 
@@ -185,21 +187,31 @@ const runCloud = async (opts) => {
   console.log('Deploying cloud runner plan...');
   const tfstate = await runTerraform(opts);
   const { resources } = tfstate;
-  for (let i = 0; i < resources.length; i++) {
-    const resource = resources[i];
-
+  for (const resource of resources) {
     if (resource.type.startsWith('iterative_')) {
-      const { instances } = resource;
-
-      for (let j = 0; j < instances.length; j++) {
-        const instance = instances[j];
-
-        if (!cloudSshPrivateVisible) {
-          instance.attributes.ssh_private = '[MASKED]';
-        }
-
-        instance.attributes.token = '[MASKED]';
-        console.log(JSON.stringify(instance));
+      for (const { attributes } of resource.instances) {
+        const nonSensitiveValues = {
+          awsSecurityGroup: attributes.aws_security_group,
+          cloud: attributes.cloud,
+          driver: attributes.driver,
+          id: attributes.id,
+          idleTimeout: attributes.idle_timeout,
+          image: attributes.image,
+          instanceGpu: attributes.instance_gpu,
+          instanceHddSize: attributes.instance_hdd_size,
+          instanceIp: attributes.instance_ip,
+          instanceLaunchTime: attributes.instance_launch_time,
+          instanceType: attributes.instance_type,
+          labels: attributes.labels,
+          name: attributes.name,
+          region: attributes.region,
+          repo: attributes.repo,
+          single: attributes.single,
+          spot: attributes.spot,
+          spotPrice: attributes.spot_price,
+          timeouts: attributes.timeouts
+        };
+        console.log(JSON.stringify(nonSensitiveValues));
       }
     }
   }
@@ -293,7 +305,7 @@ const run = async (opts) => {
 
   cml = new CML({ driver, repo, token });
 
-  await tf.checkMinVersion();
+  if (cloud || tfResource) await tf.checkMinVersion();
 
   // prepare tf
   if (tfResource) {
@@ -325,8 +337,11 @@ const run = async (opts) => {
     process.exit(0);
   }
 
-  if (reuse && (await cml.runnersByLabels({ labels })).length > 0) {
-    console.log(`Reusing existing runners with the ${labels} labels...`);
+  if (
+    reuse &&
+    (await cml.runnersByLabels({ labels })).find((runner) => runner.online)
+  ) {
+    console.log(`Reusing existing online runners with the ${labels} labels...`);
     process.exit(0);
   }
 
@@ -411,11 +426,7 @@ const opts = yargs
     'cloud-ssh-private',
     'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used'
   )
-  .boolean('cloud-ssh-private-visible')
-  .describe(
-    'cloud-ssh-private-visible',
-    'Show the private SSH key in the output with the rest of the instance properties (not recommended)'
-  )
+  .coerce('cloud-ssh-private', (val) => val.replace(/\n/g, '\\n'))
   .boolean('cloud-spot')
   .describe('cloud-spot', 'Request a spot instance')
   .default('cloud-spot-price', '-1')

bin/cml-send-comment.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+const print = console.log;
 console.log = console.error;
 
 const fs = require('fs').promises;
@@ -11,7 +12,7 @@ const run = async (opts) => {
   const path = opts._[0];
   const report = await fs.readFile(path, 'utf-8');
   const cml = new CML(opts);
-  await cml.commentCreate({ ...opts, report });
+  print(await cml.commentCreate({ ...opts, report }));
 };
 
 const opts = yargs
@@ -23,6 +24,11 @@ const opts = yargs
     'Commit SHA linked to this comment. Defaults to HEAD.'
   )
   .alias('commit-sha', 'head-sha')
+  .boolean('update')
+  .describe(
+    'update',
+    'Update the last CML comment (if any) instead of creating a new one'
+  )
   .boolean('rm-watermark')
   .describe(
     'rm-watermark',
@@ -36,10 +42,10 @@ const opts = yargs
   .default('token')
   .describe(
     'token',
-    'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.'
+    'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.'
   )
   .default('driver')
-  .choices('driver', ['github', 'gitlab'])
+  .choices('driver', ['github', 'gitlab', 'bitbucket'])
   .describe('driver', 'If not specify it infers it from the ENV.')
   .help('h')
   .demand(1).argv;

bin/cml-send-comment.test.js

@@ -21,15 +21,17 @@ describe('Comment integration tests', () => {
       Options:
         --version                 Show version number                        [boolean]
         --commit-sha, --head-sha  Commit SHA linked to this comment. Defaults to HEAD.
+        --update                  Update the last CML comment (if any) instead of
+                                  creating a new one                         [boolean]
         --rm-watermark            Avoid watermark. CML needs a watermark to be able to
                                   distinguish CML reports from other comments in order
                                   to provide extra functionality.            [boolean]
         --repo                    Specifies the repo to be used. If not specified is
                                   extracted from the CI ENV.
         --token                   Personal access token to be used. If not specified
-                                  in extracted from ENV REPO_TOKEN.
+                                  is extracted from ENV REPO_TOKEN.
         --driver                  If not specify it infers it from the ENV.
-                                                         [choices: \\"github\\", \\"gitlab\\"]
+                                            [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]
         -h                        Show help                                  [boolean]"
     `);
   });

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dvcorg/cml",
-  "version": "0.4.5",
+  "version": "0.4.7",
   "author": {
     "name": "DVC",
     "url": "http://cml.dev"
@@ -39,7 +39,7 @@
     "cml-pr": "bin/cml-pr.js"
   },
   "scripts": {
-    "lintfix": "eslint --fix ./",
+    "lintfix": "eslint --fix ./ && prettier --write '**/*.{js,json,md,yaml,yml}'",
     "lint": "eslint ./",
     "test": "jest --passWithNoTests",
     "do_snapshots": "jest --updateSnapshot"
@@ -51,9 +51,10 @@
   },
   "lint-staged": {
     "*.js": [
-      "eslint --fix"
+      "eslint --fix",
+      "prettier --write"
     ],
-    "*.{md,yaml,yml}": [
+    "*.{json,md,yaml,yml}": [
       "prettier --write"
     ]
   },