-
Notifications
You must be signed in to change notification settings - Fork 341
236 lines (235 loc) · 7.99 KB
/
test-deploy.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
name: Test & Deploy
on:
schedule:
- cron: '0 8 * * 1' # M H d m w (Mondays at 8:00)
push:
tags: ['v*']
pull_request_target:
workflow_dispatch:
jobs:
authorize:
environment:
${{ (github.event_name == 'pull_request_target' &&
github.event.pull_request.head.repo.full_name != github.repository) &&
'external' || 'internal' }}
runs-on: ubuntu-latest
steps:
- run: echo ✓
lint:
needs: authorize
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- uses: actions/setup-node@v2
- run: npm ci
- run: npm run lint
test:
needs: authorize
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- uses: actions/setup-node@v2
- uses: actions/setup-python@v2
with:
python-version: 3.9
- run: pip install tensorboard
- run: npm ci
- run: npm run test
env:
GITHUB_TOKEN: ${{ github.token }}
TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
TEST_GITHUB_REPO: https://github.com/iterative/cml_qa_tests_dummy
TEST_GITHUB_SHA: 0cd16da26e35f8e5d57b2549a97e22618abf08f6
TEST_GITLAB_TOKEN: ${{ secrets.TEST_GITLAB_TOKEN }}
TEST_GITLAB_REPO: https://gitlab.com/iterative.ai/cml_qa_tests_dummy
TEST_GITLAB_SHA: f8b8b49a253243830ef59a7f090eb887157b2b67
TEST_BBCLOUD_TOKEN: ${{ secrets.TEST_BBCLOUD_TOKEN }}
TEST_BBCLOUD_REPO: https://bitbucket.org/iterative-ai/cml-qa-tests-dummy
TEST_BBCLOUD_SHA: b511535a89f76d3d311b1c15e3e712b15c0b94e3
test-os:
needs: authorize
name: test-${{ matrix.system }}
strategy:
matrix:
system: [ubuntu, macos, windows]
runs-on: ${{ matrix.system }}-latest
steps:
- uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- if: matrix.system == 'windows'
uses: actions/setup-node@v2
- name: install
shell: bash
run: |
# https://github.com/npm/npm/issues/18503#issuecomment-347579469
npm pack && npm install -g --no-save ./*cml*.tgz
for cmd in '' runner publish pr; do
cml $cmd --version
done
- if: matrix.system != 'windows'
run: |
for cmd in runner publish pr; do
cml-$cmd --version
done
packages:
needs: [lint, test, test-os]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- uses: actions/setup-node@v2
with:
registry-url: https://registry.npmjs.org
- run: npm install
- run:
npm ${{ github.event_name == 'push' && 'publish' || 'publish
--dry-run' }}
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- name: build
run: |
cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc
npx --yes pkg package.json
rm assets/magic.mgc
for cmd in '' runner publish pr; do
build/cml-linux $cmd --version
done
release:
if: github.event_name == 'push'
needs: packages
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- id: build
name: build
run: |
echo ::set-output name=tag::${GITHUB_REF#refs/tags/}
npm install
cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc
npx --yes pkg package.json
rm assets/magic.mgc
- uses: softprops/action-gh-release@v1
with:
name: CML ${{ steps.build.outputs.tag }}
draft: true
generate_release_notes: true
files: |
build/cml-linux
build/cml-macos
env:
GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
images:
runs-on: ubuntu-latest
needs: packages
strategy:
matrix:
dvc: [1, 2]
base: [0, 1]
gpu: [false, true]
include:
- base: 0
ubuntu: 18.04
python: 2.7
cuda: 10.1
cudnn: 7
- base: 1
ubuntu: 20.04
python: 3.8
cuda: 11.2.1
cudnn: 8
- latest: true # update the values below after introducing a new major version
base: 1
dvc: 2
steps:
- uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
fetch-depth: 0
- name: Metadata
id: metadata
run: |
latest_tag=$(git describe --tags | cut -d- -f1)
cml_version=${latest_tag##v}
dvc_version=$(python3 -c '
from distutils.version import StrictVersion as Ver
from urllib.request import urlopen
from json import load
data = load(urlopen("https://pypi.org/pypi/dvc/json"))
ver_pre = "${{ matrix.dvc }}".rstrip(".") + "."
print(
max(
(i.strip() for i in data["releases"] if i.startswith(ver_pre)),
default="${{ matrix.dvc }}",
key=Ver
)
)')
echo ::set-output name=cache_tag::${cml_version}-${dvc_version}-${{ matrix.base }}-${{ matrix.gpu }}
echo ::set-output name=cml_version::$cml_version
tag=${cml_version//.*/}-dvc${{ matrix.dvc }}-base${{ matrix.base }}
if [[ ${{ matrix.gpu }} == true ]]; then
echo ::set-output name=base::nvidia/cuda:${{ matrix.cuda }}-cudnn${{ matrix.cudnn }}-runtime-ubuntu${{ matrix.ubuntu }}
tag=${tag}-gpu
else
echo ::set-output name=base::ubuntu:${{ matrix.ubuntu }}
fi
TAGS="$(
for registry in docker.io/{dvcorg,iterativeai} ghcr.io/iterative; do
if [[ "${{ matrix.latest }}" == "true" ]]; then
if [[ "${{ matrix.gpu }}" == "true" ]]; then
echo "${registry}/cml:latest-gpu"
else
echo "${registry}/cml:latest"
fi
fi
echo "${registry}/cml:${tag}"
done | head -c-1
)"
echo ::set-output name=tags::"${TAGS//$'\n'/'%0A'}"
- uses: docker/setup-buildx-action@v1
- uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key:
${{ runner.os }}-buildx-${{ steps.metadata.outputs.cache_tag }}-${{
github.sha }}
restore-keys:
${{ runner.os }}-buildx-${{ steps.metadata.outputs.cache_tag }}-
- uses: docker/login-action@v1
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- uses: docker/build-push-action@v2
with:
push:
${{ github.event_name == 'push' || github.event_name == 'schedule'
|| github.event_name == 'workflow_dispatch' }}
context: ./
file: ./Dockerfile
tags: |
${{ steps.metadata.outputs.tags }}
build-args: |
CML_VERSION=${{ steps.metadata.outputs.cml_version }}
DVC_VERSION=${{ matrix.dvc }}
PYTHON_VERSION=${{ matrix.python }}
BASE_IMAGE=${{ steps.metadata.outputs.base }}
pull: true
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
- name: Move cache
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache