Improve compatibility with unstable builds

grizzly/common/report.py

@@ -52,6 +52,7 @@ class Report:
         "path",
         "prefix",
         "stack",
+        "unstable",
     )
 
     def __init__(
@@ -60,6 +61,7 @@ def __init__(
         target_binary: Path,
         is_hang: bool = False,
         size_limit: int = MAX_LOG_SIZE,
+        unstable: bool = False,
     ) -> None:
         self._crash_info: Optional[CrashInfo] = None
         self._logs = self._select_logs(log_path)
@@ -69,6 +71,8 @@ def __init__(
         self._target_binary = target_binary
         self.is_hang = is_hang
         self.path = log_path
+        # if a build crashes before the initial testcase is served it is unstable
+        self.unstable = unstable
         # tail files in log_path if needed
         if size_limit < 1:
             LOG.warning("No limit set on report log size!")

grizzly/common/reporter.py

@@ -259,6 +259,9 @@ def _submit_report(
         if report.is_hang:
             self.add_extra_metadata("is_hang", True)
 
+        if report.unstable:
+            self.add_extra_metadata("unstable_build", True)
+
         # dump test cases and the contained files to working directory
         test_case_meta = []
         for test_number, test_case in enumerate(reversed(test_cases)):

grizzly/common/runner.py

@@ -248,7 +248,7 @@ def initial(self) -> bool:
         """
         return self._tests_run < 2
 
-    def post_launch(self, delay: int = -1) -> None:
+    def post_launch(self, delay: int = 0) -> None:
         """Perform actions after launching browser before loading test cases.
 
         Args:
@@ -257,41 +257,38 @@ def post_launch(self, delay: int = -1) -> None:
         Returns:
             None
         """
-        if delay >= 0 and not self.startup_failure:
-            with TestCase("post_launch_delay.html", "None") as content:
-                content.add_from_file(
-                    Path(__file__).parent / "post_launch_delay.html",
-                    file_name=content.entry_point,
-                    copy=True,
-                )
-                srv_map = ServerMap()
-                srv_map.set_redirect("grz_start", content.entry_point, required=False)
-                srv_map.set_redirect("grz_continue", "grz_start", required=True)
-                # temporarily override server timeout
-                org_timeout = self._server.timeout
-                # add time buffer to redirect delay
-                # in practice this should take a few seconds (~10s)
-                # in extreme cases ~40s (slow build + debugger)
-                self._server.timeout = delay + 180
-                if delay > 0:
-                    LOG.info("Browser launched, continuing in %ds...", delay)
-                # serve prompt page
-                server_status, _ = self._server.serve_path(
-                    content.root,
-                    continue_cb=self._target.monitor.is_healthy,
-                    server_map=srv_map,
-                )
-                # restore server timeout
-                self._server.timeout = org_timeout
-                if server_status != Served.ALL:
-                    self.startup_failure = True
-                    if server_status == Served.TIMEOUT:
-                        # this should never happen with a correctly functioning build
-                        LOG.warning("Target hung after launch")
-
-        if self.startup_failure:
-            # TODO: we need a better way to handle delayed startup failures
-            LOG.warning("Post launch check failed!")
+        assert delay >= 0
+        with TestCase("post_launch_delay.html", "None") as content:
+            content.add_from_file(
+                Path(__file__).parent / "post_launch_delay.html",
+                file_name=content.entry_point,
+                copy=True,
+            )
+            srv_map = ServerMap()
+            srv_map.set_redirect("grz_start", content.entry_point, required=False)
+            srv_map.set_redirect("grz_continue", "grz_start", required=True)
+            # temporarily override server timeout
+            org_timeout = self._server.timeout
+            # add time buffer to redirect delay
+            # in practice this should take a few seconds (~10s)
+            # in extreme cases ~40s (slow build + debugger)
+            self._server.timeout = delay + 180
+            if delay > 0:
+                LOG.info("Browser launched, continuing in %ds...", delay)
+            # serve prompt page
+            server_status, _ = self._server.serve_path(
+                content.root,
+                continue_cb=self._target.monitor.is_healthy,
+                server_map=srv_map,
+            )
+            # restore server timeout
+            self._server.timeout = org_timeout
+            if server_status != Served.ALL:
+                self.startup_failure = True
+                if server_status == Served.TIMEOUT:
+                    # this should never happen with a correctly functioning build
+                    LOG.warning("Target hung after launch")
+                LOG.warning("Post launch check failed!")
 
     def run(
         self,

grizzly/common/test_reporter.py

@@ -177,7 +177,9 @@ def test_fuzzmanager_reporter_01(mocker, tmp_path, tests, frequent, force, sig_c
         test_cases.append(fake_test)
     reporter = FuzzManagerReporter("fake-tool")
     reporter.submit(
-        test_cases, Report(log_path, Path("bin"), is_hang=True), force=force
+        test_cases,
+        Report(log_path, Path("bin"), is_hang=True, unstable=True),
+        force=force,
     )
     assert not log_path.is_dir()
     assert fake_collector.call_args == ({"tool": "fake-tool"},)

grizzly/common/test_runner.py

@@ -44,7 +44,7 @@ def test_runner_01(mocker, coverage, scheme):
         testcase.add_from_bytes(b"", testcase.entry_point)
         serv_files = {"a.bin": testcase.root / "a.bin"}
         server.serve_path.return_value = (Served.ALL, serv_files)
-        result = runner.run([], serv_map, testcase, coverage=coverage)
+        result = runner.run(set(), serv_map, testcase, coverage=coverage)
         assert testcase.https == (scheme == "https")
     assert runner.initial
     assert runner._tests_run == 1
@@ -81,7 +81,7 @@ def test_runner_02(mocker):
     runner = Runner(server, target, relaunch=1)
     assert runner._relaunch == 1
     smap = ServerMap()
-    result = runner.run([], smap, testcase)
+    result = runner.run(set(), smap, testcase)
     assert runner.initial
     assert result.attempted
     assert target.close.call_count == 1
@@ -100,7 +100,7 @@ def test_runner_02(mocker):
     target.monitor.is_idle.return_value = True
     runner = Runner(server, target, relaunch=1)
     assert runner._relaunch == 1
-    result = runner.run([], ServerMap(), testcase)
+    result = runner.run(set(), ServerMap(), testcase)
     assert result.attempted
     assert target.close.call_count == 1
     assert target.monitor.is_healthy.call_count > 0
@@ -111,7 +111,7 @@ def test_runner_02(mocker):
     target.monitor.is_healthy.return_value = False
     for _ in range(2):
         smap = ServerMap()
-        result = runner.run([], smap, testcase)
+        result = runner.run(set(), smap, testcase)
         assert result.attempted
         assert target.close.call_count == 0
         assert target.monitor.is_healthy.call_count == 0
@@ -154,7 +154,7 @@ def test_runner_03(mocker, srv_result, served):
     target.check_result.return_value = Result.NONE
     test = mocker.Mock(spec_set=TestCase, entry_point="x", required=["x"])
     runner = Runner(server, target)
-    result = runner.run([], ServerMap(), test)
+    result = runner.run(set(), ServerMap(), test)
     assert runner.initial
     assert runner.startup_failure
     assert result
@@ -224,7 +224,7 @@ def test_runner_05(mocker, served, attempted, target_result, status):
     testcase = mocker.Mock(spec_set=TestCase, entry_point="a.bin", required=["a.bin"])
     runner = Runner(server, target)
     runner.launch("http://a/")
-    result = runner.run([], ServerMap(), testcase)
+    result = runner.run(set(), ServerMap(), testcase)
     assert result.attempted == attempted
     assert result.status == status
     assert not result.timeout
@@ -242,7 +242,7 @@ def test_runner_06(mocker):
     runner = Runner(server, target, idle_threshold=0.01, idle_delay=0.01, relaunch=10)
     assert runner._idle is not None
     result = runner.run(
-        [],
+        set(),
         ServerMap(),
         mocker.Mock(spec_set=TestCase, entry_point="a.bin", required=tuple(serv_files)),
     )
@@ -357,7 +357,7 @@ def test_runner_10(mocker, tmp_path):
             "test/inc_file3.txt": inc3,
         }
         server.serve_path.return_value = (Served.ALL, serv_files)
-        result = runner.run([], smap, test)
+        result = runner.run(set(), smap, test)
         assert result.attempted
         assert result.status == Result.NONE
         assert "inc_file.bin" in test
@@ -386,7 +386,7 @@ def test_runner_11(mocker):
                 "extra.js": test.root / "extra.js",
             },
         )
-        result = runner.run([], ServerMap(), test)
+        result = runner.run(set(), ServerMap(), test)
         assert result.attempted
         assert result.status == Result.NONE
         assert "test.html" in test
@@ -401,8 +401,6 @@ def test_runner_11(mocker):
         (10, (Served.ALL, None), False),
         # continue immediately
         (0, (Served.ALL, None), False),
-        # skip post launch delay page
-        (-1, None, False),
         # startup failure
         (0, (Served.NONE, None), True),
         # target hang while loading content
@@ -412,10 +410,7 @@ def test_runner_11(mocker):
 def test_runner_12(mocker, delay, srv_result, startup_failure):
     """test Runner.post_launch()"""
     srv_timeout = 1
-    server = mocker.Mock(
-        spec_set=Sapphire,
-        timeout=srv_timeout,
-    )
+    server = mocker.Mock(spec_set=Sapphire, timeout=srv_timeout)
     server.serve_path.return_value = srv_result
     runner = Runner(server, mocker.Mock(spec_set=Target, launch_timeout=30))
     runner.launch("http://a/")