Merge pull request #34 from nuclearcat/add-test-option

Add test option

docs/checkout.md

@@ -153,3 +153,20 @@ Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Resu
 ```
 
 The command will keep watching the progress of the test until all jobs are done. You can also stop the watching by pressing `Ctrl+C` or command will stop after all jobs are done(or failed).
+
+### --test
+
+Together with --watch option, you can use --test option to wait for particular test results. Return code of kci-dev will depend on the test result:
+
+- `pass` - return code 0 (test passed)
+- `fail` - return code 1 (test failed)
+- `error` - return code 2 (prior steps failed, such as compilation, test setup, etc, or infrastructure error)
+
+For example:
+```sh
+kci-dev.py checkout --giturl https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git --branch master --tipoftree --jobfilter baseline-nfs-arm64-qualcomm --jobfilter kbuild-gcc-12-arm64-chromeos-qualcomm --platformfilter sc7180-trogdor-kingoftown --watch --test crit
+```
+
+This command will wait for the test results of the test with the name `crit`. It will follow first jobs, such as `checkout`, `kbuild-gcc-12-arm64-chromeos-qualcomm`, `baseline-nfs-arm64-qualcomm` and when they are complete will wait until timeout for the test `crit` to finish. If the test `crit` will pass, the command will return 0, if it will fail, the command will return 1, if any of the jobs will fail or timeout, the command will return 2.
+
+This command can be used for regression bisection, where you can test if the test `crit` pass or fail on the specific commit.

kci-dev/subcommands/checkout.py

@@ -5,6 +5,7 @@
 import json
 import re
 import subprocess
+import sys
 import time
 
 import click
@@ -105,7 +106,7 @@ def check_node(node):
             return "FAIL"
 
 
-def watch_jobs(baseurl, token, treeid, jobfilter):
+def watch_jobs(baseurl, token, treeid, jobfilter, test):
     # we need to add to jobfilter "checkout" node
     jobfilter = list(jobfilter)
     jobfilter.append("checkout")
@@ -123,7 +124,12 @@ def watch_jobs(baseurl, token, treeid, jobfilter):
 
         # Tricky part in watch is that we might have one item in jobfilter (job, test),
         # but it might spawn multiple nodes with same name
+        test_result = None
+        jobs_done_ts = None
         for node in nodes:
+            if node["name"] == test:
+                test_result = node["result"]
+                break
             if node["name"] in jobfilter:
                 result = check_node(node)
                 if result == "DONE":
@@ -137,13 +143,35 @@ def watch_jobs(baseurl, token, treeid, jobfilter):
                     if isinstance(joblist, list) and node["name"] in joblist:
                         joblist.remove(node["name"])
                     color = "red"
+                    if test:
+                        # if we have a test, and prior job failed, we should indicate that
+                        click.secho(
+                            f"Job {node['name']} failed, test can't be executed",
+                            fg="red",
+                        )
+                        sys.exit(2)
                 click.secho(
                     f"Node {node['_id']} job {node['name']} State {node['state']} Result {node['result']}",
                     fg=color,
                 )
         if len(joblist) == 0 and inprogress == 0:
             click.secho("All jobs completed", fg="green")
-            return
+            if not test:
+                return
+            else:
+                if not jobs_done_ts:
+                    jobs_done_ts = time.time()
+                # if all jobs done, usually test results must be available
+                # max within 60s. Safeguard in case of test node is not available
+                if not test_result and time.time() - jobs_done_ts < 60:
+                    continue
+
+                if test_result == "pass":
+                    click.secho(f"Test {test} passed", fg="green")
+                    sys.exit(0)
+                else:
+                    click.secho(f"Test {test} failed", fg="red")
+                    sys.exit(1)
 
         click.echo(f"\rRefresh in 30s...", nl=False)
         time.sleep(30)
@@ -200,8 +228,14 @@ def retrieve_tot_commit(repourl, branch):
     help="Platform filter to trigger",
     multiple=True,
 )
+@click.option(
+    "--test",
+    help="Return code based on the test result",
+)
 @click.pass_context
-def checkout(ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree, watch):
+def checkout(
+    ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree, watch, test
+):
     cfg = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
     url = api_connection(cfg[instance]["pipeline"])
@@ -213,6 +247,9 @@ def checkout(ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree,
     if watch and not jobfilter:
         click.secho("No job filter defined. Can't watch for a job(s)!", fg="red")
         return
+    if test and not watch:
+        click.secho("Test option only works with watch option", fg="red")
+        return
     if not commit and not tipoftree:
         click.secho("No commit or tree/branch latest commit defined", fg="red")
         return
@@ -242,8 +279,10 @@ def checkout(ctx, giturl, branch, commit, jobfilter, platformfilter, tipoftree,
             click.secho("No treeid returned. Can't watch for a job(s)!", fg="red")
             return
         click.secho(f"Watching for jobs on treeid: {treeid}", fg="green")
+        if test:
+            click.secho(f"Watching for test result: {test}", fg="green")
         # watch for jobs
-        watch_jobs(apiurl, token, treeid, jobfilter)
+        watch_jobs(apiurl, token, treeid, jobfilter, test)
 
 
 if __name__ == "__main__":