diff --git a/.dependancron b/.dependancron
index a6e7bcb30..c945ef144 100644
--- a/.dependancron
+++ b/.dependancron
@@ -1 +1 @@
-2.0.17
+2.0.18
diff --git a/README.md b/README.md
index 8894e1ae2..c3e71c1a2 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,9 @@
+[](https://github.com/iterative/setup-cml)
+[](https://www.npmjs.com/package/@dvcorg/cml)
+
**What is CML?** Continuous Machine Learning (CML) is an open-source library for
implementing continuous integration & delivery (CI/CD) in machine learning
projects. Use it to automate parts of your development workflow, including model
@@ -23,40 +26,39 @@ We built CML with these principles in mind:
plots in each Git Pull Request. Rigorous engineering practices help your team
make informed, data-driven decisions.
- **No additional services.** Build your own ML platform using just GitHub or
- GitLab and your favorite cloud services: AWS, Azure, GCP. No databases,
+ GitLab and your favourite cloud services: AWS, Azure, GCP. No databases,
services or complex setup needed.
-_⁉️ Need help? Just want to chat about continuous integration for ML?
-[Visit our Discord channel!](https://discord.gg/bzA6uY7)_
+:question: Need help? Just want to chat about continuous integration for ML?
+[Visit our Discord channel!](https://discord.gg/bzA6uY7)
-🌟🌟🌟 Check out our
+:play_or_pause_button: Check out our
[YouTube video series](https://www.youtube.com/playlist?list=PL7WG7YrwYcnDBDuCkFbcyjnZQrdskFsBz)
-for hands-on MLOps tutorials using CML! 🌟🌟🌟
+for hands-on MLOps tutorials using CML!
## Table of contents
1. [Usage](#usage)
-2. [Getting started](#getting-started)
+2. [Getting started (tutorial)](#getting-started)
3. [Using CML with DVC](#using-cml-with-dvc)
4. [Using self-hosted runners](#using-self-hosted-runners)
5. [Install CML as a package](#install-cml-as-a-package)
-6. [Examples](#a-library-of-cml-projects)
+6. [Example Projects](#see-also)
## Usage
You'll need a GitHub or GitLab account to begin. Users may wish to familiarize
themselves with [Github Actions](https://help.github.com/en/actions) or
-[GitLab CI/CD](https://about.gitlab.com/stages-devops-lifecycle/continuous-integration/).
+[GitLab CI/CD](https://about.gitlab.com/stages-devops-lifecycle/continuous-integration).
Here, will discuss the GitHub use case.
-⚠️ **GitLab users!** Please see our
-[docs about configuring CML with GitLab](https://github.com/iterative/cml/wiki/CML-with-GitLab).
-
-🪣 **Bitbucket Cloud users** We support you, too-
-[see our docs here](https://github.com/iterative/cml/wiki/CML-with-Bitbucket-Cloud).🪣
-_Bitbucket Server support estimated to arrive by January 2021._
-
-The key file in any CML project is `.github/workflows/cml.yaml`.
+- **GitLab users**: Please see our
+ [docs about configuring CML with GitLab](https://github.com/iterative/cml/wiki/CML-with-GitLab).
+- **Bitbucket Cloud users**: Please see our
+ [docs on CML with Bitbucket Cloud](https://github.com/iterative/cml/wiki/CML-with-Bitbucket-Cloud).
+ _Bitbucket Server support estimated to arrive by May 2021._
+- **GitHub Actions users**: The key file in any CML project is
+ `.github/workflows/cml.yaml`:
```yaml
name: your-workflow-name
@@ -64,34 +66,47 @@ on: [push]
jobs:
run:
runs-on: [ubuntu-latest]
- container: docker://dvcorg/cml-py3:latest
+ # optionally use a convenient Ubuntu LTS + CUDA + DVC + CML image
+ # container: docker://dvcorg/cml-py3:latest
steps:
- uses: actions/checkout@v2
- - name: 'Train my model'
- env:
- repo_token: ${{ secrets.GITHUB_TOKEN }}
+ # may need to setup NodeJS & Python3 on e.g. self-hosted
+ # - uses: actions/setup-node@v2
+ # with:
+ # node-version: '12'
+ # - uses: actions/setup-python@v2
+ # with:
+ # python-version: '3.x'
+ - uses: iterative/setup-cml@v1
+ - name: Train model
run: |
-
# Your ML workflow goes here
pip install -r requirements.txt
python train.py
-
- # Write your CML report
+ - name: Write CML report
+ env:
+ repo_token: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ # Post reports as comments in GitHub PRs
cat results.txt >> report.md
cml-send-comment report.md
```
+We helpfully provide CML and other useful libraries pre-installed on our
+[custom Docker images](https://github.com/iterative/cml/blob/master/docker/Dockerfile).
+In the above example, uncommenting the field
+`container: docker://dvcorg/cml-py3:latest` will make the GitHub Actions runner
+pull the CML Docker image. The image already has NodeJS, Python 3, DVC and CML
+set up on an Ubuntu LTS base with CUDA libraries and
+[Terraform](https://www.terraform.io) installed for convenience.
+
### CML Functions
-CML provides a number of helper functions to help package outputs from ML
-workflows, such as numeric data and data vizualizations about model performance,
-into a CML report. The library comes pre-installed on our
-[custom Docker images](https://github.com/iterative/cml/blob/master/docker/Dockerfile).
-In the above example, note the field `container: docker://dvcorg/cml-py3:latest`
-specifies the CML Docker image with Python 3 will be pulled by the GitHub
-Actions runner.
+CML provides a number of helper functions to help package the outputs of ML
+workflows (including numeric data and visualizations about model performance)
+into a CML report.
-Below is a list of CML functions for writing markdown reports and delivering
+Below is a table of CML functions for writing markdown reports and delivering
those reports to your CI system (GitHub Actions or GitLab CI).
| Function | Description | Inputs |
@@ -105,38 +120,40 @@ those reports to your CI system (GitHub Actions or GitLab CI).
CML reports are written in
[GitHub Flavored Markdown](https://github.github.com/gfm/). That means they can
-contain images, tables, formatted text, HTML blocks, code snippets and more -
+contain images, tables, formatted text, HTML blocks, code snippets and more —
really, what you put in a CML report is up to you. Some examples:
-📝 **Text**. Write to your report using whatever method you prefer. For example,
-copy the contents of a text file containing the results of ML model training:
+:spiral_notepad: **Text** Write to your report using whatever method you prefer.
+For example, copy the contents of a text file containing the results of ML model
+training:
```bash
cat results.txt >> report.md
```
-🖼️ **Images** Display images using the markdown or HTML. Note that if an image
-is an output of your ML workflow (i.e., it is produced by your workflow), you
-will need to use the `cml-publish` function to include it a CML report. For
-example, if `graph.png` is the output of my workflow `python train.py`, run:
+:framed_picture: **Images** Display images using the markdown or HTML. Note that
+if an image is an output of your ML workflow (i.e., it is produced by your
+workflow), you will need to use the `cml-publish` function to include it a CML
+report. For example, if `graph.png` is output by `python train.py`, run:
```bash
cml-publish graph.png --md >> report.md
```
-## Getting started
+## Getting Started
1. Fork our
- [example project repository](https://github.com/iterative/example_cml). ⚠️
- Note that if you are using GitLab,
- [you will need to create a Personal Access Token](https://github.com/iterative/cml/wiki/CML-with-GitLab#variables)
- for this example to work.
+ [example project repository](https://github.com/iterative/example_cml).
+
+> :warning: Note that if you are using GitLab,
+> [you will need to create a Personal Access Token](https://github.com/iterative/cml/wiki/CML-with-GitLab#variables)
+> for this example to work.
-The following steps can all be done in the GitHub browser interface. However, to
-follow along the commands, we recommend cloning your fork to your local
-workstation:
+> :warning: The following steps can all be done in the GitHub browser interface.
+> However, to follow along with the commands, we recommend cloning your fork to
+> your local workstation:
```bash
git clone https://github.com//example_cml
@@ -151,10 +168,11 @@ on: [push]
jobs:
run:
runs-on: [ubuntu-latest]
- container: docker://dvcorg/cml-py3:latest
steps:
- uses: actions/checkout@v2
- - name: 'Train my model'
+ - uses: actions/setup-python@v2
+ - uses: iterative/setup-cml@v1
+ - name: Train model
env:
repo_token: ${{ secrets.GITHUB_TOKEN }}
run: |
@@ -166,9 +184,9 @@ jobs:
cml-send-comment report.md
```
-4. In your text editor of choice, edit line 16 of `train.py` to `depth = 5`.
+3. In your text editor of choice, edit line 16 of `train.py` to `depth = 5`.
-5. Commit and push the changes:
+4. Commit and push the changes:
```bash
git checkout -b experiment
@@ -176,34 +194,38 @@ git add . && git commit -m "modify forest depth"
git push origin experiment
```
-6. In GitHub, open up a Pull Request to compare the `experiment` branch to
+5. In GitHub, open up a Pull Request to compare the `experiment` branch to
`master`.
Shortly, you should see a comment from `github-actions` appear in the Pull
-Request with your CML report. This is a result of the function
-`cml-send-comment` in your workflow.
+Request with your CML report. This is a result of the `cml-send-comment`
+function in your workflow.
-This is the gist of the CML workflow: when you push changes to your GitHub
-repository, the workflow in your `.github/workflows/cml.yaml` file gets run and
-a report generated. CML functions let you display relevant results from the
-workflow, like model performance metrics and vizualizations, in GitHub checks
-and comments. What kind of workflow you want to run, and want to put in your CML
-report, is up to you.
+This is the outline of the CML workflow:
+
+- you push changes to your GitHub repository,
+- the workflow in your `.github/workflows/cml.yaml` file gets run, and
+- a report is generated and posted to GitHub.
+
+CML functions let you display relevant results from the workflow — such as model
+performance metrics and visualizations — in GitHub checks and comments. What
+kind of workflow you want to run, and want to put in your CML report, is up to
+you.
## Using CML with DVC
-In many ML projects, data isn't stored in a Git repository and needs to be
+In many ML projects, data isn't stored in a Git repository, but needs to be
downloaded from external sources. [DVC](https://dvc.org) is a common way to
bring data to your CML runner. DVC also lets you visualize how metrics differ
between commits to make reports like this:
-The `.github/workflows/cml.yaml` file to create this report is:
+The `.github/workflows/cml.yaml` file used to create this report is:
```yaml
name: model-training
@@ -214,8 +236,7 @@ jobs:
container: docker://dvcorg/cml-py3:latest
steps:
- uses: actions/checkout@v2
- - name: 'Train my model'
- shell: bash
+ - name: Train model
env:
repo_token: ${{ secrets.GITHUB_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -234,26 +255,27 @@ jobs:
dvc metrics diff master --show-md >> report.md
# Publish confusion matrix diff
- echo -e "## Plots\n### Class confusions" >> report.md
+ echo "## Plots" >> report.md
+ echo "### Class confusions" >> report.md
dvc plots diff --target classes.csv --template confusion -x actual -y predicted --show-vega master > vega.json
vl2png vega.json -s 1.5 | cml-publish --md >> report.md
# Publish regularization function diff
- echo "### Effects of regularization\n" >> report.md
+ echo "### Effects of regularization" >> report.md
dvc plots diff --target estimators.csv -x Regularization --show-vega master > vega.json
vl2png vega.json -s 1.5 | cml-publish --md >> report.md
cml-send-comment report.md
```
-If you're using DVC with cloud storage, take note of environmental variables for
-your storage format.
+> :warning: If you're using DVC with cloud storage, take note of environment
+> variables for your storage format.
-### Environmental variables for supported cloud providers
+### Environment variables for supported cloud providers
- S3 and S3 compatible storage (Minio, DigitalOcean Spaces, IBM Cloud Object Storage...)
+ S3 and S3-compatible storage (Minio, DigitalOcean Spaces, IBM Cloud Object Storage...)
```yaml
@@ -264,7 +286,7 @@ env:
AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }}
```
-> :point_right: AWS_SESSION_TOKEN is optional.
+> :point_right: `AWS_SESSION_TOKEN` is optional.
@@ -302,9 +324,10 @@ env:
Google Storage
-> :warning: Normally, GOOGLE_APPLICATION_CREDENTIALS points to the path of the
-> json file that contains the credentials. However in the action this variable
-> CONTAINS the content of the file. Copy that json and add it as a secret.
+> :warning: Normally, `GOOGLE_APPLICATION_CREDENTIALS` is the **path** of the
+> `json` file containing the credentials. However in the action this secret
+> variable is the **contents** of the file. Copy the `json` contents and add it
+> as a secret.
```yaml
env:
@@ -320,9 +343,9 @@ env:
> :warning: After configuring your
> [Google Drive credentials](https://dvc.org/doc/command-reference/remote/add)
-> you will find a json file at
-> `your_project_path/.dvc/tmp/gdrive-user-credentials.json`. Copy that json and
-> add it as a secret.
+> you will find a `json` file at
+> `your_project_path/.dvc/tmp/gdrive-user-credentials.json`. Copy its contents
+> and add it as a secret variable.
```yaml
env:
@@ -335,81 +358,82 @@ env:
GitHub Actions are run on GitHub-hosted runners by default. However, there are
many great reasons to use your own runners: to take advantage of GPUs; to
-orchestrate your team's shared computing resources, or to train in the cloud.
+orchestrate your team's shared computing resources, or to access on-premise
+data.
-☝️ **Tip!** Check out the
-[official GitHub documentation](https://help.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners)
-to get started setting up your self-hosted runner.
+> :point_up: **Tip!** Check out the
+> [official GitHub documentation](https://help.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners)
+> to get started setting up your own self-hosted runner.
### Allocating cloud resources with CML
-When a workflow requires computational resources (such as GPUs) CML can
+When a workflow requires computational resources (such as GPUs), CML can
automatically allocate cloud instances using `cml-runner`. You can spin up
instances on your AWS or Azure account (GCP support is forthcoming!).
For example, the following workflow deploys a `t2.micro` instance on AWS EC2 and
trains a model on the instance. After the job runs, the instance automatically
-shuts down. You might notice that this workflow is quite similar to the
-[basic use case](#usage) highlighted in the beginning of the docs- that's
-because it is! What's new is that we've added `cml-runner`, plus a few
-environmental variables for passing your cloud service credentials to the
+shuts down.
+
+You might notice that this workflow is quite similar to the
+[basic use case](#usage) above. The only addition is `cml-runner` and a few
+environment variables for passing your cloud service credentials to the
workflow.
```yaml
-name: "Train-in-the-cloud"
+name: Train-in-the-cloud
on: [push]
-
jobs:
deploy-runner:
runs-on: [ubuntu-latest]
steps:
- uses: iterative/setup-cml@v1
- uses: actions/checkout@v2
- - name: "Deploy runner on EC2"
- shell: bash
+ - name: Deploy runner on EC2
env:
repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
cml-runner \
- --cloud aws \
- --cloud-region us-west \
- --cloud-type=t2.micro \
- --labels=cml-runner
- name: model-training
- needs: deploy-runner
- runs-on: [self-hosted,cml-runner]
+ --cloud aws \
+ --cloud-region us-west \
+ --cloud-type=t2.micro \
+ --labels=cml-runner
+ model-training:
+ needs: [deploy-runner]
+ runs-on: [self-hosted, cml-runner]
container: docker://dvcorg/cml-py3:latest
steps:
- - uses: actions/checkout@v2
- - name: "Train my model"
- env:
- repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- run: |
- pip install -r requirements.txt
- python train.py
- # Publish report with CML
- cat metrics.txt > report.md
- cml-send-comment report.md
+ - uses: actions/checkout@v2
+ - name: Train model
+ env:
+ repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+ run: |
+ pip install -r requirements.txt
+ python train.py
+
+ cat metrics.txt > report.md
+ cml-send-comment report.md
```
-In the above workflow, the step `deploy-runner` launches an EC2 `t2-micro`
-instance in the `us-west` region. The next step, `model-training`, runs on the
-newly launched instance.
+In the workflow above, the `deploy-runner` step launches an EC2 `t2-micro`
+instance in the `us-west` region. The `model-training` step then runs on the
+newly-launched instance.
-**Note that you can use any container with this workflow!** While you must have
-CML and its dependencies setup to use CML functions like `cml-send-comment` from
-your instance, you can create your favorite training environment in the cloud by
-pulling the Docker container of your choice.
+> :tada: **Note that you can use any container with this workflow!** While you
+> must [have CML and its dependencies set up](#install-cml-as-a-package) to use
+> functions such `cml-send-comment` from your instance, you can create your
+> favourite training environment in the cloud by pulling the Docker container of
+> your choice.
We like the CML container (`docker://dvcorg/cml-py3`) because it comes loaded
with Python, CUDA, `git`, `node` and other essentials for full-stack data
-science. But we don't mind if you do it your way :)
+science.
### Arguments
-The function `cml-runner` accepts the following arguments:
+The `cml-runner` function accepts the following arguments:
```
Usage: cml-runner.js
@@ -462,30 +486,31 @@ Options:
-h Show help [boolean]
```
-### Environmental variables
+### Environment variables
-You will need to
-[create a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line)
-with repository read/write access and workflow privileges. In the example
-workflow, this token is stored as `PERSONAL_ACCESS_TOKEN`.
+> :warning: You will need to
+> [create a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line)
+> with repository read/write access and workflow privileges. In the example
+> workflow, this token is stored as `PERSONAL_ACCESS_TOKEN`.
Note that you will also need to provide access credentials for your cloud
compute resources as secrets. In the above example, `AWS_ACCESS_KEY_ID` and
`AWS_SECRET_ACCESS_KEY` are required to deploy EC2 instances.
Please see our docs about
-[environmental variables needed to authenticate with supported cloud services](#environmental-variables-for-supported-cloud-providers).
+[environment variables needed to authenticate with supported cloud services](#environment-variables-for-supported-cloud-providers).
-### Using on-premise machines as self-hosted runners
+### On-premise (local) runners
-You can also use the new `cml-runner` function to set up a local self-hosted
-runner. On your local machine or on-premise GPU cluster, you'll install CML as a
-package and then run:
+This means using on-premise machines as self-hosted runners. The `cml-runner`
+function is used to set up a local self-hosted runner. On your local machine or
+on-premise GPU cluster, [install CML as a package](#install-cml-as-a-package)
+and then run:
-```yaml
+```bash
cml-runner \
--repo $your_project_repository_url \
- --token=$personal_access_token \
+ --token=$PERSONAL_ACCESS_TOKEN \
--labels tf \
--idle-timeout 180
```
@@ -494,8 +519,9 @@ Now your machine will be listening for workflows from your project repository.
## Install CML as a package
-In the above examples, CML is pre-installed in a custom Docker image, which is
-pulled by a CI runner. You can also install CML as a package:
+In the examples above, CML is installed by the `setup-cml` action, or comes
+pre-installed in a custom Docker image pulled by a CI runner. You can also
+install CML as a package:
```bash
npm i -g @dvcorg/cml
@@ -506,26 +532,28 @@ CLI commands:
```bash
sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev \
- librsvg2-dev libfontconfig-dev
+ librsvg2-dev libfontconfig-dev
npm install -g vega-cli vega-lite
```
-CML and Vega-Lite package installation require `npm` command from Node package.
-Below you can find how to install Node.
+CML and Vega-Lite package installation require the NodeJS package manager
+(`npm`) which ships with NodeJS. Installation instructions are below.
-### Install Node in GitHub
+### Install NodeJS in GitHub
-In GitHub there is a special action for NPM installation:
+This is probably not necessary when using GitHub's default containers or one of
+CML's Docker containers. Self-hosted runners may need to use a set up action to
+install NodeJS:
```bash
-uses: actions/setup-node@v1
+uses: actions/setup-node@v2
with:
node-version: '12'
```
-### Install Node in GitLab
+### Install NodeJS in GitLab
-GitLab requires direct installation of the NMP package:
+GitLab requires direct installation of NodeJS:
```bash
curl -sL https://deb.nodesource.com/setup_12.x | bash
@@ -533,9 +561,9 @@ apt-get update
apt-get install -y nodejs
```
-## A library of CML projects
+## See Also
-Here are some example projects using CML.
+These are some example projects using CML.
- [Basic CML project](https://github.com/iterative/cml_base_case)
- [CML with DVC to pull data](https://github.com/iterative/cml_dvc_case)
diff --git a/bin/cml-pr.js b/bin/cml-pr.js
new file mode 100644
index 000000000..04c4121f9
--- /dev/null
+++ b/bin/cml-pr.js
@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+
+const print = console.log;
+console.log = console.error;
+
+const yargs = require('yargs');
+const decamelize = require('decamelize-keys');
+
+const CML = require('../src/cml');
+
+const run = async (opts) => {
+ const globs = opts._.length ? opts._ : undefined;
+ const cml = new CML(opts);
+ print(await cml.pr_create({ ...opts, globs }));
+};
+
+const opts = decamelize(
+ yargs
+ .usage('Usage: $0 ')
+ .describe('md', 'Output in markdown format [](url).')
+ .boolean('md')
+ .default('repo')
+ .describe(
+ 'repo',
+ 'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+ )
+ .default('token')
+ .describe(
+ 'token',
+ 'Personal access token to be used. If not specified in extracted from ENV repo_token.'
+ )
+ .default('driver')
+ .choices('driver', ['github', 'gitlab'])
+ .describe('driver', 'If not specify it infers it from the ENV.')
+ .help('h').argv
+);
+
+run(opts).catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
diff --git a/bin/cml-publish.js b/bin/cml-publish.js
index a86cc864e..59f1c738b 100644
--- a/bin/cml-publish.js
+++ b/bin/cml-publish.js
@@ -71,7 +71,7 @@ const argv = yargs
.default('token')
.describe(
'token',
- 'Personal access token to be used. If not specified in extracted from ENV repo_token or GITLAB_TOKEN.'
+ 'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.'
)
.default('driver')
.choices('driver', ['github', 'gitlab'])
diff --git a/bin/cml-publish.test.js b/bin/cml-publish.test.js
index e050e1802..c5c023fc3 100644
--- a/bin/cml-publish.test.js
+++ b/bin/cml-publish.test.js
@@ -25,8 +25,9 @@ describe('CML e2e', () => {
exist.
--repo Specifies the repo to be used. If not specified is extracted
from the CI ENV.
- --token Personal access token to be used. If not specified in
- extracted from ENV repo_token or GITLAB_TOKEN.
+ --token Personal access token to be used. If not specified,
+ extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN,
+ or BITBUCKET_TOKEN.
--driver If not specify it infers it from the ENV.
[choices: \\"github\\", \\"gitlab\\"]
-h Show help [boolean]"
diff --git a/bin/cml-runner.js b/bin/cml-runner.js
index ea901e7ad..1a86ffce4 100755
--- a/bin/cml-runner.js
+++ b/bin/cml-runner.js
@@ -25,7 +25,7 @@ const {
RUNNER_REUSE = false,
RUNNER_DRIVER,
RUNNER_REPO,
- repo_token
+ REPO_TOKEN
} = process.env;
let cml;
@@ -350,7 +350,7 @@ const opts = decamelize(
'repo',
'Repository to be used for registering the runner. If not specified, it will be inferred from the environment'
)
- .default('token', repo_token)
+ .default('token', REPO_TOKEN)
.describe(
'token',
'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment'
diff --git a/bin/cml-send-comment.js b/bin/cml-send-comment.js
index 60aaf0874..179ce6842 100644
--- a/bin/cml-send-comment.js
+++ b/bin/cml-send-comment.js
@@ -47,7 +47,7 @@ const argv = yargs
.default('token')
.describe(
'token',
- 'Personal access token to be used. If not specified in extracted from ENV repo_token.'
+ 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.'
)
.default('driver')
.choices('driver', ['github', 'gitlab'])
diff --git a/bin/cml-send-comment.test.js b/bin/cml-send-comment.test.js
index 91599218f..b596b1ad8 100644
--- a/bin/cml-send-comment.test.js
+++ b/bin/cml-send-comment.test.js
@@ -29,7 +29,7 @@ describe('Comment integration tests', () => {
--repo Specifies the repo to be used. If not specified is extracted
from the CI ENV.
--token Personal access token to be used. If not specified in
- extracted from ENV repo_token.
+ extracted from ENV REPO_TOKEN.
--driver If not specify it infers it from the ENV.
[choices: \\"github\\", \\"gitlab\\"]
-h Show help [boolean]"
diff --git a/bin/cml-send-github-check.js b/bin/cml-send-github-check.js
index 50e187b8c..f35b0ecf7 100644
--- a/bin/cml-send-github-check.js
+++ b/bin/cml-send-github-check.js
@@ -43,7 +43,7 @@ const argv = yargs
.default('token')
.describe(
'token',
- 'Personal access token to be used. If not specified in extracted from ENV repo_token.'
+ 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.'
)
.help('h')
.demand(1).argv;
diff --git a/bin/cml-send-github-check.test.js b/bin/cml-send-github-check.test.js
index bbbcc0ccd..ef33b3c89 100644
--- a/bin/cml-send-github-check.test.js
+++ b/bin/cml-send-github-check.test.js
@@ -45,7 +45,7 @@ describe('CML e2e', () => {
--repo Specifies the repo to be used. If not specified is extracted
from the CI ENV.
--token Personal access token to be used. If not specified in extracted
- from ENV repo_token.
+ from ENV REPO_TOKEN.
-h Show help [boolean]
--conclusion[choices: \\"success\\", \\"failure\\", \\"neutral\\", \\"cancelled\\", \\"skipped\\",
\\"timed_out\\"] [default: Sets the conclusion status of the check.]"
diff --git a/package-lock.json b/package-lock.json
index 0ea68cec9..588a38765 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "@dvcorg/cml",
- "version": "0.3.6",
+ "version": "0.3.7",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@@ -585,6 +585,42 @@
"@types/yargs": "^13.0.0"
}
},
+ "@kwsites/file-exists": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz",
+ "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==",
+ "requires": {
+ "debug": "^4.1.1"
+ }
+ },
+ "@kwsites/promise-deferred": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz",
+ "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="
+ },
+ "@nodelib/fs.scandir": {
+ "version": "2.1.4",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz",
+ "integrity": "sha512-33g3pMJk3bg5nXbL/+CY6I2eJDzZAni49PfJnL5fghPTggPvBd/pFNSgJsdAgWptuFu7qq/ERvOYFlhvsLTCKA==",
+ "requires": {
+ "@nodelib/fs.stat": "2.0.4",
+ "run-parallel": "^1.1.9"
+ }
+ },
+ "@nodelib/fs.stat": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.4.tgz",
+ "integrity": "sha512-IYlHJA0clt2+Vg7bccq+TzRdJvv19c2INqBSsoOLp1je7xjtr7J26+WXR72MCdvU9q1qTzIWDfhMf+DRvQJK4Q=="
+ },
+ "@nodelib/fs.walk": {
+ "version": "1.2.6",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.6.tgz",
+ "integrity": "sha512-8Broas6vTtW4GIXTAHDoE32hnN2M5ykgCpWGbuXHQ15vEMqr23pB76e/GZcYsZCHALv50ktd24qhEyKr6wBtow==",
+ "requires": {
+ "@nodelib/fs.scandir": "2.1.4",
+ "fastq": "^1.6.0"
+ }
+ },
"@octokit/auth-token": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.2.tgz",
@@ -954,6 +990,11 @@
"is-string": "^1.0.5"
}
},
+ "array-union": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
+ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw=="
+ },
"array-unique": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
@@ -1714,6 +1755,14 @@
}
}
},
+ "debug": {
+ "version": "4.3.1",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
+ "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
+ "requires": {
+ "ms": "2.1.2"
+ }
+ },
"decamelize": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
@@ -1841,6 +1890,21 @@
"integrity": "sha512-Dj6Wk3tWyTE+Fo1rW8v0Xhwk80um6yFYKbuAxc9c3EZxIHFDYwbi34Uk42u1CdnIiVorvt4RmlSDjIPyzGC2ew==",
"dev": true
},
+ "dir-glob": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
+ "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
+ "requires": {
+ "path-type": "^4.0.0"
+ },
+ "dependencies": {
+ "path-type": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
+ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="
+ }
+ }
+ },
"doctrine": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -2539,6 +2603,66 @@
"integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==",
"dev": true
},
+ "fast-glob": {
+ "version": "3.2.5",
+ "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.5.tgz",
+ "integrity": "sha512-2DtFcgT68wiTTiwZ2hNdJfcHNke9XOfnwmBRWXhmeKM8rF0TGwmC/Qto3S7RoZKp5cilZbxzO5iTNTQsJ+EeDg==",
+ "requires": {
+ "@nodelib/fs.stat": "^2.0.2",
+ "@nodelib/fs.walk": "^1.2.3",
+ "glob-parent": "^5.1.0",
+ "merge2": "^1.3.0",
+ "micromatch": "^4.0.2",
+ "picomatch": "^2.2.1"
+ },
+ "dependencies": {
+ "braces": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+ "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+ "requires": {
+ "fill-range": "^7.0.1"
+ }
+ },
+ "fill-range": {
+ "version": "7.0.1",
+ "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+ "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+ "requires": {
+ "to-regex-range": "^5.0.1"
+ }
+ },
+ "is-number": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="
+ },
+ "micromatch": {
+ "version": "4.0.4",
+ "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz",
+ "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==",
+ "requires": {
+ "braces": "^3.0.1",
+ "picomatch": "^2.2.3"
+ },
+ "dependencies": {
+ "picomatch": {
+ "version": "2.2.3",
+ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.3.tgz",
+ "integrity": "sha512-KpELjfwcCDUb9PeigTs2mBJzXUPzAuP2oPcA989He8Rte0+YUAjw1JVedDhuTKPkHjSYzMN3npC9luThGYEKdg=="
+ }
+ }
+ },
+ "to-regex-range": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+ "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+ "requires": {
+ "is-number": "^7.0.0"
+ }
+ }
+ }
+ },
"fast-json-stable-stringify": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -2551,6 +2675,14 @@
"integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
"dev": true
},
+ "fastq": {
+ "version": "1.11.0",
+ "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.11.0.tgz",
+ "integrity": "sha512-7Eczs8gIPDrVzT+EksYBcupqMyxSHXXrHOLRRxU2/DicV8789MRBRR8+Hc2uWzUupOs4YS4JzBmBxjjCVBxD/g==",
+ "requires": {
+ "reusify": "^1.0.4"
+ }
+ },
"fb-watchman": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.1.tgz",
@@ -3357,7 +3489,6 @@
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
"integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
- "dev": true,
"requires": {
"is-glob": "^4.0.1"
}
@@ -3371,6 +3502,26 @@
"type-fest": "^0.8.1"
}
},
+ "globby": {
+ "version": "11.0.3",
+ "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.3.tgz",
+ "integrity": "sha512-ffdmosjA807y7+lA1NM0jELARVmYul/715xiILEjo3hBLPTcirgQNnXECn5g3mtR8TOLCVbkfua1Hpen25/Xcg==",
+ "requires": {
+ "array-union": "^2.1.0",
+ "dir-glob": "^3.0.1",
+ "fast-glob": "^3.1.1",
+ "ignore": "^5.1.4",
+ "merge2": "^1.3.0",
+ "slash": "^3.0.0"
+ },
+ "dependencies": {
+ "ignore": {
+ "version": "5.1.8",
+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
+ "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw=="
+ }
+ }
+ },
"graceful-fs": {
"version": "4.2.4",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz",
@@ -3921,8 +4072,7 @@
"is-extglob": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
- "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
- "dev": true
+ "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI="
},
"is-generator-fn": {
"version": "2.1.0",
@@ -3934,7 +4084,6 @@
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
"integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
- "dev": true,
"requires": {
"is-extglob": "^2.1.1"
}
@@ -5462,6 +5611,11 @@
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
"dev": true
},
+ "merge2": {
+ "version": "1.4.1",
+ "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+ "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="
+ },
"micromatch": {
"version": "3.1.10",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
@@ -6056,8 +6210,7 @@
"picomatch": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz",
- "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
- "dev": true
+ "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg=="
},
"pify": {
"version": "2.3.0",
@@ -6239,6 +6392,11 @@
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
"dev": true
},
+ "queue-microtask": {
+ "version": "1.2.3",
+ "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
+ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="
+ },
"react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@@ -6498,6 +6656,11 @@
"integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==",
"dev": true
},
+ "reusify": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw=="
+ },
"rsvp": {
"version": "4.8.5",
"resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
@@ -6510,6 +6673,14 @@
"integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==",
"dev": true
},
+ "run-parallel": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
+ "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+ "requires": {
+ "queue-microtask": "^1.2.2"
+ }
+ },
"rxjs": {
"version": "6.5.5",
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz",
@@ -6711,6 +6882,16 @@
"integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
"dev": true
},
+ "simple-git": {
+ "version": "2.38.0",
+ "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.38.0.tgz",
+ "integrity": "sha512-CORjrfirWMEGbJAxaXDH/PjZVOeATeG2bkafM9DsLVcFkbF9sXQGIIpEI6FeyXpvUsFK69T/pa4+4FKY9TUJMQ==",
+ "requires": {
+ "@kwsites/file-exists": "^1.1.1",
+ "@kwsites/promise-deferred": "^1.1.1",
+ "debug": "^4.3.1"
+ }
+ },
"sisteransi": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
@@ -6720,8 +6901,7 @@
"slash": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
- "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
- "dev": true
+ "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="
},
"slice-ansi": {
"version": "2.1.0",
diff --git a/package.json b/package.json
index 3def2629e..2664457a7 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "@dvcorg/cml",
- "version": "0.3.6",
+ "version": "0.3.7",
"author": {
"name": "DVC",
"url": "http://cml.dev"
@@ -35,7 +35,8 @@
"cml-publish": "bin/cml-publish.js",
"cml-tensorboard-dev": "bin/cml-tensorboard-dev.js",
"cml-runner": "bin/cml-runner.js",
- "cml-cloud-runner-entrypoint": "bin/cml-runner.js"
+ "cml-cloud-runner-entrypoint": "bin/cml-runner.js",
+ "test-cml-pr": "bin/cml-pr.js"
},
"scripts": {
"lintfix": "eslint --fix ./",
@@ -64,12 +65,14 @@
"form-data": "^3.0.0",
"fs-extra": "^9.0.1",
"git-url-parse": "^11.4.0",
+ "globby": "^11.0.3",
"is-svg": "^4.2.2",
"js-base64": "^3.5.2",
"node-fetch": "^2.6.0",
"node-forge": "^0.10.0",
"node-ssh": "^11.0.0",
"semver": "^7.3.4",
+ "simple-git": "^2.38.0",
"strip-url-auth": "^1.0.1",
"tar": "^6.1.0",
"tempy": "^0.6.0",
diff --git a/src/cml.js b/src/cml.js
index 9fe359f65..8412158c9 100644
--- a/src/cml.js
+++ b/src/cml.js
@@ -1,6 +1,8 @@
const { execSync } = require('child_process');
const git_url_parse = require('git-url-parse');
const strip_auth = require('strip-url-auth');
+const git = require('simple-git/promise')('./');
+const globby = require('globby');
const Gitlab = require('./drivers/gitlab');
const Github = require('./drivers/github');
@@ -48,12 +50,15 @@ const get_driver = (opts) => {
const infer_token = () => {
const {
+ REPO_TOKEN,
repo_token,
GITHUB_TOKEN,
GITLAB_TOKEN,
BITBUCKET_TOKEN
} = process.env;
- return repo_token || GITHUB_TOKEN || GITLAB_TOKEN || BITBUCKET_TOKEN;
+ return (
+ REPO_TOKEN || repo_token || GITHUB_TOKEN || GITLAB_TOKEN || BITBUCKET_TOKEN
+ );
};
class CML {
@@ -229,11 +234,94 @@ class CML {
await this.runner_token();
} catch (err) {
throw new Error(
- 'repo_token does not have enough permissions to access workflow API'
+ 'REPO_TOKEN does not have enough permissions to access workflow API'
);
}
}
+ async pr_create(opts = {}) {
+ const { globs = ['dvc.lock', '.gitignore'], md } = opts;
+
+ const { files } = await git.status();
+ if (!files.length) {
+ console.log('No files changed. Nothing to do.');
+ return;
+ }
+
+ const driver = get_driver(this);
+ const paths = (await globby(globs)).filter((path) =>
+ files.map((item) => item.path).includes(path)
+ );
+
+ const render_pr = (url) => {
+ if (md)
+ return `[CML's ${
+ this.driver === 'gitlab' ? 'Merge' : 'Pull'
+ } Request](${url})`;
+ return url;
+ };
+
+ const sha = await exec(`git rev-parse HEAD`);
+ const sha_short = sha.substr(0, 7);
+ let target = await exec(`git branch --show-current`);
+ if (!target) {
+ if (this.driver === 'gitlab') {
+ target = await exec('echo $CI_BUILD_REF_NAME');
+ }
+ }
+ const source = `${target}-cmlpr-${sha_short}`;
+
+ await exec(`git fetch origin`);
+
+ const branch_exists = (await exec(`git branch -r`)).includes(source);
+ if (branch_exists) {
+ const prs = await driver.prs();
+ const { url } =
+ prs.find((pr) => pr.source === source && pr.target === target) || {};
+
+ if (url) return render_pr(url);
+ } else {
+ try {
+ await exec(`git config --local user.email "david@iterative.ai"`);
+ await exec(`git config --local user.name "cml-bot"`);
+ await exec('git config advice.addIgnoredFile false');
+
+ if (this.driver !== 'github') {
+ const repo = new URL(this.repo);
+ repo.password = this.token;
+ repo.username = driver.user_name;
+
+ await exec(`git remote rm origin`);
+ await exec(`git remote add origin "${repo.toString()}.git"`);
+ }
+
+ await exec(`git checkout -B ${target} ${sha}`);
+ await exec(`git checkout -b ${source}`);
+ await exec(`git add ${paths.join(' ')}`);
+ await exec(`git commit -m "CML [skip ci]"`);
+ await exec(`git push --set-upstream origin ${source}`);
+ await exec(`git checkout -B ${target} ${sha}`);
+ } catch (err) {
+ await exec(`git checkout -B ${target} ${sha}`);
+ throw err;
+ }
+ }
+
+ const title = `CML commits ${target} ${sha_short}`;
+ const description = `
+ Automated commits for ${this.repo}/commit/${sha} created by CML.
+ `;
+
+ const url = await driver.pr_create({
+ source,
+ target,
+ title,
+ description
+ });
+
+ return render_pr(url);
+ }
+
log_error(e) {
console.error(e.message);
}
diff --git a/src/cml.test.js b/src/cml.test.js
index dc24de45f..c44f74ea0 100644
--- a/src/cml.test.js
+++ b/src/cml.test.js
@@ -13,7 +13,7 @@ describe('Github tests', () => {
jest.resetModules();
process.env = {};
- process.env.repo_token = TOKEN;
+ process.env.REPO_TOKEN = TOKEN;
});
afterAll(() => {
@@ -91,7 +91,7 @@ describe('Gitlab tests', () => {
jest.resetModules();
process.env = {};
- process.env.repo_token = TOKEN;
+ process.env.REPO_TOKEN = TOKEN;
});
afterAll(() => {
diff --git a/src/drivers/github.js b/src/drivers/github.js
index a0be2f233..9330dbca0 100644
--- a/src/drivers/github.js
+++ b/src/drivers/github.js
@@ -231,6 +231,58 @@ class Github {
)
.map((runner) => ({ id: runner.id, name: runner.name }));
}
+
+ async pr_create(opts = {}) {
+ const { source: head, target: base, title, description: body } = opts;
+ const { owner, repo } = owner_repo({ uri: this.repo });
+ const { pulls } = octokit(this.token, this.repo);
+
+ const {
+ data: { url }
+ } = await pulls.create({
+ owner,
+ repo,
+ head,
+ base,
+ title,
+ body
+ });
+
+ return url;
+ }
+
+ async prs(opts = {}) {
+ const { state = 'open' } = opts;
+ const { owner, repo } = owner_repo({ uri: this.repo });
+ const { pulls } = octokit(this.token, this.repo);
+
+ const { data: prs } = await pulls.list({
+ owner,
+ repo,
+ state
+ });
+
+ return prs.map((pr) => {
+ const {
+ url,
+ head: { ref: source },
+ base: { ref: target }
+ } = pr;
+ return {
+ url,
+ source,
+ target
+ };
+ });
+ }
+
+ get user_email() {
+ return 'action@github.com';
+ }
+
+ get user_name() {
+ return 'GitHub Action';
+ }
}
module.exports = Github;
diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js
index c7bbf1188..68d59fe08 100644
--- a/src/drivers/gitlab.js
+++ b/src/drivers/gitlab.js
@@ -8,7 +8,7 @@ const { resolve } = require('path');
const { fetch_upload_data, download, exec } = require('../utils');
-const { IN_DOCKER } = process.env;
+const { IN_DOCKER, GITLAB_USER_EMAIL, GITLAB_USER_NAME } = process.env;
const API_VER = 'v4';
class Gitlab {
constructor(opts = {}) {
@@ -180,6 +180,39 @@ class Gitlab {
return runners.map((runner) => ({ id: runner.id, name: runner.name }));
}
+ async pr_create(opts = {}) {
+ const { project_path } = this;
+ const { source, target, title, description } = opts;
+
+ const endpoint = `/projects/${project_path}/merge_requests`;
+ const body = new URLSearchParams();
+ body.append('source_branch', source);
+ body.append('target_branch', target);
+ body.append('title', title);
+ body.append('description', description);
+
+ const { web_url } = await this.request({ endpoint, method: 'POST', body });
+
+ return web_url;
+ }
+
+ async prs(opts = {}) {
+ const { project_path } = this;
+ const { state = 'opened' } = opts;
+
+ const endpoint = `/projects/${project_path}/merge_requests?state=${state}`;
+ const prs = await this.request({ endpoint, method: 'GET' });
+
+ return prs.map((pr) => {
+ const { web_url: url, source_branch: source, target_branch: target } = pr;
+ return {
+ url,
+ source,
+ target
+ };
+ });
+ }
+
async request(opts = {}) {
const { token } = this;
const { endpoint, method = 'GET', body, raw } = opts;
@@ -198,6 +231,14 @@ class Gitlab {
return await response.json();
}
+
+ get user_email() {
+ return GITLAB_USER_EMAIL;
+ }
+
+ get user_name() {
+ return GITLAB_USER_NAME;
+ }
}
module.exports = Gitlab;