-
Notifications
You must be signed in to change notification settings - Fork 1.3k
/
Copy pathbuild-test-matrix.js
298 lines (276 loc) · 10 KB
/
build-test-matrix.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
// Small script used to calculate the matrix of tests that are going to be
// performed for a CI run.
//
// This is invoked by the `determine` step and is written in JS because I
// couldn't figure out how to write it in bash.
const fs = require('fs');
const { spawn } = require('node:child_process');
// Number of generic buckets to shard crates into. Note that we additionally add
// single-crate buckets for our biggest crates.
const GENERIC_BUCKETS = 3;
// Crates which are their own buckets. These are the very slowest to
// compile-and-test crates.
const SINGLE_CRATE_BUCKETS = ["wasmtime", "wasmtime-cli", "wasmtime-wasi"];
const ubuntu = 'ubuntu-24.04';
const windows = 'windows-2022';
const macos = 'macos-14';
// This is the small, fast-to-execute matrix we use for PRs before they enter
// the merge queue. Same schema as `FULL_MATRIX`.
const FAST_MATRIX = [
{
"os": ubuntu,
"name": "Test Linux x86_64",
"filter": "linux-x64",
"isa": "x64",
},
];
// This is the full, unsharded, and unfiltered matrix of what we test on
// CI. This includes a number of platforms and a number of cross-compiled
// targets that are emulated with QEMU. This must be kept tightly in sync with
// the `test` step in `main.yml`.
//
// The supported keys here are:
//
// * `os` - the github-actions name of the runner os
//
// * `name` - the human-readable name of the job
//
// * `filter` - a string which if `prtest:$filter` is in the commit messages
// it'll force running this test suite on PR CI.
//
// * `isa` - changes to `cranelift/codegen/src/$isa` will automatically run this
// test suite.
//
// * `target` - used for cross-compiles if present. Effectively Cargo's
// `--target` option for all its operations.
//
// * `gcc_package`, `gcc`, `qemu`, `qemu_target` - configuration for building
// QEMU and installing cross compilers to execute a cross-compiled test suite
// on CI.
//
// * `rust` - the Rust version to install, and if unset this'll be set to
// `default`
const FULL_MATRIX = [
...FAST_MATRIX,
{
"os": ubuntu,
"name": "Test MSRV on Linux x86_64",
"filter": "linux-x64",
"isa": "x64",
"rust": "msrv",
},
{
"os": ubuntu,
"name": "Test Linux x86_64 with MPK",
"filter": "linux-x64",
"isa": "x64"
},
{
"os": macos,
"name": "Test macOS x86_64",
"filter": "macos-x64",
"target": "x86_64-apple-darwin",
},
{
"os": macos,
"name": "Test macOS arm64",
"filter": "macos-arm64",
"target": "aarch64-apple-darwin",
},
{
"os": windows,
"name": "Test Windows MSVC x86_64",
"filter": "windows-x64",
},
{
"os": windows,
"target": "x86_64-pc-windows-gnu",
"name": "Test Windows MinGW x86_64",
"filter": "mingw-x64"
},
{
"os": ubuntu,
"target": "aarch64-unknown-linux-gnu",
"gcc_package": "gcc-aarch64-linux-gnu",
"gcc": "aarch64-linux-gnu-gcc",
"qemu": "qemu-aarch64 -L /usr/aarch64-linux-gnu",
"qemu_target": "aarch64-linux-user",
"name": "Test Linux arm64",
"filter": "linux-arm64",
"isa": "aarch64",
},
{
"os": ubuntu,
"target": "s390x-unknown-linux-gnu",
"gcc_package": "gcc-s390x-linux-gnu",
"gcc": "s390x-linux-gnu-gcc",
"qemu": "qemu-s390x -L /usr/s390x-linux-gnu",
"qemu_target": "s390x-linux-user",
"name": "Test Linux s390x",
"filter": "linux-s390x",
"isa": "s390x"
},
{
"os": ubuntu,
"target": "riscv64gc-unknown-linux-gnu",
"gcc_package": "gcc-riscv64-linux-gnu",
"gcc": "riscv64-linux-gnu-gcc",
"qemu": "qemu-riscv64 -cpu rv64,v=true,vlen=256,vext_spec=v1.0,zfa=true,zfh=true,zba=true,zbb=true,zbc=true,zbs=true,zbkb=true,zcb=true,zicond=true -L /usr/riscv64-linux-gnu",
"qemu_target": "riscv64-linux-user",
"name": "Test Linux riscv64",
"filter": "linux-riscv64",
"isa": "riscv64",
},
{
"name": "Tests on i686-unknown-linux-gnu",
"os": ubuntu,
"target": "i686-unknown-linux-gnu",
"gcc_package": "gcc-i686-linux-gnu",
"gcc": "i686-linux-gnu-gcc",
},
{
"name": "Tests on armv7-unknown-linux-gnueabihf",
"os": ubuntu,
"target": "armv7-unknown-linux-gnueabihf",
"gcc_package": "gcc-arm-linux-gnueabihf",
"gcc": "arm-linux-gnueabihf-gcc",
"qemu": "qemu-arm -L /usr/arm-linux-gnueabihf -E LD_LIBRARY_PATH=/usr/arm-linux-gnueabihf/lib",
"qemu_target": "arm-linux-user",
},
];
/// Get the workspace's full list of member crates.
async function getWorkspaceMembers() {
// Spawn a `cargo metadata` subprocess, accumulate its JSON output from
// `stdout`, and wait for it to exit.
const child = spawn("cargo", ["metadata"], { encoding: "utf8" });
let data = "";
child.stdout.on("data", chunk => data += chunk);
await new Promise((resolve, reject) => {
child.on("close", resolve);
child.on("error", reject);
});
// Get the names of the crates in the workspace from the JSON metadata by
// building a package-id to name map and then translating the package-ids
// listed as workspace members.
const metadata = JSON.parse(data);
const id_to_name = {};
for (const pkg of metadata.packages) {
id_to_name[pkg.id] = pkg.name;
}
return metadata.workspace_members.map(m => id_to_name[m]);
}
/// For each given target configuration, shard the workspace's crates into
/// buckets across that config.
///
/// This is essentially a `flat_map` where each config that logically tests all
/// crates in the workspace is mapped to N sharded configs that each test only a
/// subset of crates in the workspace. Each sharded config's subset of crates to
/// test are disjoint from all its siblings, and the union of all these siblings'
/// crates to test is the full workspace members set.
///
/// With some poetic license around a `crates_to_test` key that doesn't actually
/// exist, logically each element of the input `configs` list gets transformed
/// like this:
///
/// { os: "ubuntu-latest", isa: "x64", ..., crates: "all" }
///
/// ==>
///
/// [
/// { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime"] },
/// { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-cli"] },
/// { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-wasi"] },
/// { os: "ubuntu-latest", isa: "x64", ..., crates: ["cranelift", "cranelift-codegen", ...] },
/// { os: "ubuntu-latest", isa: "x64", ..., crates: ["wasmtime-slab", "cranelift-entity", ...] },
/// { os: "ubuntu-latest", isa: "x64", ..., crates: ["cranelift-environ", "wasmtime-cli-flags", ...] },
/// ...
/// ]
///
/// Note that `crates: "all"` is implicit in the input and omitted. Similarly,
/// `crates: [...]` in each output config is actually implemented via adding a
/// `bucket` key, which contains the CLI flags we must pass to `cargo` to run
/// tests for just this config's subset of crates.
async function shard(configs) {
const members = await getWorkspaceMembers();
// Divide the workspace crates into N disjoint subsets. Crates that are
// particularly expensive to compile and test form their own singleton subset.
const buckets = Array.from({ length: GENERIC_BUCKETS }, _ => new Set());
let i = 0;
for (const crate of members) {
if (SINGLE_CRATE_BUCKETS.indexOf(crate) != -1) continue;
buckets[i].add(crate);
i = (i + 1) % GENERIC_BUCKETS;
}
for (crate of SINGLE_CRATE_BUCKETS) {
buckets.push(new Set([crate]));
}
// For each config, expand it into N configs, one for each disjoint set we
// created above.
const sharded = [];
for (const config of configs) {
for (const bucket of buckets) {
sharded.push(Object.assign(
{},
config,
{
name: `${config.name} (${Array.from(bucket).join(', ')})`,
// We run tests via `cargo test --workspace`, so exclude crates that
// aren't in this bucket, rather than naming only the crates that are
// in this bucket.
bucket: members
.map(c => bucket.has(c) ? `--package ${c}` : `--exclude ${c}`)
.join(" "),
}
));
}
}
return sharded;
}
async function main() {
// Our first argument is a file that is a giant json blob which contains at
// least all the messages for all of the commits that were a part of this PR.
// This is used to test if any commit message includes a string.
const commits = fs.readFileSync(process.argv[2]).toString();
// The second argument is a file that contains the names of all files modified
// for a PR, used for file-based filters.
const names = fs.readFileSync(process.argv[3]).toString();
for (let config of FULL_MATRIX) {
if (config.rust === undefined) {
config.rust = 'default';
}
}
// If the optional third argument to this script is `true` then that means all
// tests are being run and no filtering should happen.
if (process.argv[4] == 'true') {
console.log(JSON.stringify(await shard(FULL_MATRIX), undefined, 2));
return;
}
// When we aren't running the full CI matrix, filter configs down to just the
// relevant bits based on files changed in this commit or if the commit asks
// for a certain config to run.
const filtered = FULL_MATRIX.filter(config => {
// If an ISA-specific test was modified, then include that ISA config.
if (config.isa && names.includes(`cranelift/codegen/src/isa/${config.isa}`)) {
return true;
}
// If any runtest was modified, include all ISA configs as runtests can
// target any backend.
if (names.includes(`cranelift/filetests/filetests/runtests`)) {
if (config.isa !== undefined)
return true;
}
// If the commit explicitly asks for this test config, then include it.
if (config.filter && commits.includes(`prtest:${config.filter}`)) {
return true;
}
return false;
});
// If at least one test is being run via our filters then run those tests.
if (filtered.length > 0) {
console.log(JSON.stringify(await shard(filtered), undefined, 2));
return;
}
// Otherwise if nothing else is being run, run the fast subset of the matrix.
console.log(JSON.stringify(await shard(FAST_MATRIX), undefined, 2));
}
main()