From bd45a8755af510c987340b9b9d9f51b9cc353e64 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 17 Oct 2024 12:53:14 +0300 Subject: [PATCH 1/2] feat(checkout): Add --test option For bisection we are likely expecting particular test result to fail or pass. This option will require watch to wait for test result and return code will depend on it. Return codes: 0 - test passed 1 - test failed 2 - early steps failed (kernel build, infrastructure issue,etc) We add also safeguards, in case by some reasons test didn't run, so watch will timeout, if all other necessary steps are completed. Signed-off-by: Denys Fedoryshchenko --- kci-dev/subcommands/checkout.py | 47 ++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/kci-dev/subcommands/checkout.py b/kci-dev/subcommands/checkout.py index 307761d..6786e1d 100644 --- a/kci-dev/subcommands/checkout.py +++ b/kci-dev/subcommands/checkout.py @@ -5,6 +5,7 @@ import json import re import subprocess +import sys import time import click @@ -105,7 +106,7 @@ def check_node(node): return "FAIL" -def watch_jobs(baseurl, token, treeid, jobfilter): +def watch_jobs(baseurl, token, treeid, jobfilter, test): # we need to add to jobfilter "checkout" node jobfilter = list(jobfilter) jobfilter.append("checkout") @@ -123,7 +124,12 @@ def watch_jobs(baseurl, token, treeid, jobfilter): # Tricky part in watch is that we might have one item in jobfilter (job, test), # but it might spawn multiple nodes with same name + test_result = None + jobs_done_ts = None for node in nodes: + if node["name"] == test: + test_result = node["result"] + break if node["name"] in jobfilter: result = check_node(node) if result == "DONE": @@ -137,13 +143,35 @@ def watch_jobs(baseurl, token, treeid, jobfilter): if isinstance(joblist, list) and node["name"] in joblist: joblist.remove(node["name"]) color = "red" + if test: + # if we have a test, and prior job failed, we should indicate that + click.secho( + f"Job {node['name']} failed, test can't be executed", + fg="red", + ) + sys.exit(2) click.secho( f"Node {node['_id']} job {node['name']} State {node['state']} Result {node['result']}", fg=color, ) if len(joblist) == 0 and inprogress == 0: click.secho("All jobs completed", fg="green") - return + if not test: + return + else: + if not jobs_done_ts: + jobs_done_ts = time.time() + # if all jobs done, usually test results must be available + # max within 60s. Safeguard in case of test node is not available + if not test_result and time.time() - jobs_done_ts < 60: + continue + + if test_result == "pass": + click.secho(f"Test {test} passed", fg="green") + sys.exit(0) + else: + click.secho(f"Test {test} failed", fg="red") + sys.exit(1) click.echo(f"\rRefresh in 30s...", nl=False) time.sleep(30) @@ -200,8 +228,14 @@ def retrieve_tot_commit(repourl, branch): help="Platform filter to trigger", multiple=True, ) +@click.option( + "--test", + help="Return code based on the test result", +) @click.pass_context -def checkout(ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree, watch): +def checkout( + ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree, watch, test +): cfg = ctx.obj.get("CFG") instance = ctx.obj.get("INSTANCE") url = api_connection(cfg[instance]["pipeline"]) @@ -213,6 +247,9 @@ def checkout(ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree, if watch and not jobfilter: click.secho("No job filter defined. Can't watch for a job(s)!", fg="red") return + if test and not watch: + click.secho("Test option only works with watch option", fg="red") + return if not commit and not tipoftree: click.secho("No commit or tree/branch latest commit defined", fg="red") return @@ -242,8 +279,10 @@ def checkout(ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree, click.secho("No treeid returned. Can't watch for a job(s)!", fg="red") return click.secho(f"Watching for jobs on treeid: {treeid}", fg="green") + if test: + click.secho(f"Watching for test result: {test}", fg="green") # watch for jobs - watch_jobs(apiurl, token, treeid, jobfilter) + watch_jobs(apiurl, token, treeid, jobfilter, test) if __name__ == "__main__": From 2beb801c3764b3f9da96bda4f8f28e71e2d14113 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 17 Oct 2024 13:03:52 +0300 Subject: [PATCH 2/2] feat(checkout.md): Add documentation for --test option Add documentation for --test added to checkout subcommand. Signed-off-by: Denys Fedoryshchenko --- docs/checkout.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/checkout.md b/docs/checkout.md index 1a0203a..c874f2e 100644 --- a/docs/checkout.md +++ b/docs/checkout.md @@ -153,3 +153,20 @@ Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Resu ``` The command will keep watching the progress of the test until all jobs are done. You can also stop the watching by pressing `Ctrl+C` or command will stop after all jobs are done(or failed). + +### --test + +Together with --watch option, you can use --test option to wait for particular test results. Return code of kci-dev will depend on the test result: + +- `pass` - return code 0 (test passed) +- `fail` - return code 1 (test failed) +- `error` - return code 2 (prior steps failed, such as compilation, test setup, etc, or infrastructure error) + +For example: +```sh +kci-dev.py checkout --giturl https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git --branch master --tipoftree --jobfilter baseline-nfs-arm64-qualcomm --jobfilter kbuild-gcc-12-arm64-chromeos-qualcomm --platformfilter sc7180-trogdor-kingoftown --watch --test crit +``` + +This command will wait for the test results of the test with the name `crit`. It will follow first jobs, such as `checkout`, `kbuild-gcc-12-arm64-chromeos-qualcomm`, `baseline-nfs-arm64-qualcomm` and when they are complete will wait until timeout for the test `crit` to finish. If the test `crit` will pass, the command will return 0, if it will fail, the command will return 1, if any of the jobs will fail or timeout, the command will return 2. + +This command can be used for regression bisection, where you can test if the test `crit` pass or fail on the specific commit.