diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index ae35d0525c..2600baf0e6 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -1,8 +1,8 @@ name: End-to-End Test on: pull_request: - schedule: - - cron: '0 0 * * 0' # Run every Sunday at 00:00 UTC + # schedule: + # - cron: '0 0 * * 0' # Run every Sunday at 00:00 UTC jobs: bug_reproduction: timeout-minutes: 1440 @@ -26,4 +26,4 @@ jobs: mkdir -m 777 -p profile/data - name: Run bug reproduction run: | - pytest -m "local" \ No newline at end of file + pytest -m "regression" diff --git a/pytest.ini b/pytest.ini index 13bc2a8e86..97481b7cf1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] markers = - local: mark a test to run on a local machine \ No newline at end of file + local: mark a test to run on a local machine + regression: mark a test to run e2e regressions on a pull request diff --git a/test/test_bug_reproduction.py b/test/test_bug_reproduction.py index 1edb124a18..41ccfa2bff 100644 --- a/test/test_bug_reproduction.py +++ b/test/test_bug_reproduction.py @@ -2,6 +2,7 @@ import os import pathlib import queue +import random from typing import Dict, List, Tuple import unittest @@ -118,6 +119,7 @@ class TestBugReproduction(unittest.TestCase): def __init__(self, methodName: str = "runTest") -> None: super().__init__(methodName) + # TODO: make _num_workers a command line argument self._num_workers = 2 # Number of workers to run the test def test_all_bugs(self): @@ -146,6 +148,40 @@ def test_all_bugs(self): self.assertFalse(errors, f'Test failed with {errors}') +@pytest.mark.regression +class TestRegression(unittest.TestCase): + + def __init__(self, methodName: str = "runTest") -> None: + super().__init__(methodName) + + self._num_workers = 1 # Number of workers to run the test + + def test_all_bugs(self): + manager = multiprocessing.Manager() + workqueue = multiprocessing.Queue() + + operator, bugs = random.choice(list(all_bugs.items())) + bug_id, bug_config = random.choice(list(bugs.items())) + workqueue.put((operator, bug_id, bug_config)) + + reproduction_results: Dict[str, bool] = manager.dict() # workers write reproduction results + # to this dict. Bug ID -> if success + processes: List[multiprocessing.Process] = [] + for i in range(self._num_workers): + p = multiprocessing.Process(target=run_worker, args=(workqueue, i, reproduction_results)) + p.start() + processes.append(p) + + for p in processes: + p.join() + + errors = [] + for bug_id, if_reproduced in reproduction_results.items(): + if not if_reproduced: + errors.append(bug_id) + + self.assertFalse(errors, f'Test failed with {errors}') + if __name__ == '__main__': unittest.main() \ No newline at end of file