From 991ccd233f783015574c1871237385762640f2dc Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 18 Oct 2024 11:44:01 -0500 Subject: [PATCH 1/4] Add gen-collect-only CI check --- .github/workflows/run-tests.yml | 21 ++++++++++++++++--- Makefile | 5 ++++- .../gen_helpers/gen_base/gen_runner.py | 4 ++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 83926c47c5..7b9cf51ab3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -24,7 +24,7 @@ on: - cron: '0 0 * * *' jobs: - table_of_contents: + table-of-contents: runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] steps: - name: Checkout repository @@ -71,7 +71,7 @@ jobs: pyspec-tests: runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs] - needs: [lint,codespell,table_of_contents] + needs: [lint, codespell, table-of-contents] strategy: matrix: version: ["phase0", "altair", "bellatrix", "capella", "deneb", "electra", "whisk", "eip7594"] @@ -104,9 +104,24 @@ jobs: - name: Install pyspec requirements run: make install_test - name: test-${{ matrix.version }} - run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}} + run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{ env.spec_test_preset_type }} - uses: actions/upload-artifact@v4 if: always() with: name: test-reports-${{ matrix.version }} path: tests/core/pyspec/test-reports + + gen-collect-only: + runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12.4' + cache: '' + - name: Install pyspec requirements + run: make install_test + - name: Run generators with --collect-only + run: make generate_tests collect_only=true diff --git a/Makefile b/Makefile index 39d493645e..be83d1a302 100644 --- a/Makefile +++ b/Makefile @@ -159,6 +159,9 @@ lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) +# If set to true, it will not run generator tests. +collect_only ?= false + # Runs a generator, identified by param 1 define run_generator # Started! @@ -177,7 +180,7 @@ define run_generator . venv/bin/activate; \ pip3 install ../../../dist/eth2spec-*.whl; \ pip3 install 'eth2spec[generator]'; \ - python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \ + python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(collect_only)),--collect-only); \ echo "generator $(1) finished" endef diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 85ccec017a..aa60d6d92b 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -248,6 +248,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"Collected test at: {case_dir}") diagnostics_obj.collected_test_count += 1 + # Bail here if we do not want to generate. + if collect_only: + continue + is_skip, diagnostics_obj = should_skip_case_dir(case_dir, args.force, diagnostics_obj) if is_skip: continue From 57bd0ae2bfa5bd10c9cf3fc9d87c79068d84bf9a Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 18 Oct 2024 12:15:59 -0500 Subject: [PATCH 2/4] Remove --collect-only & rebrand to modcheck --- .github/workflows/run-tests.yml | 6 ++-- Makefile | 4 +-- .../gen_helpers/gen_base/gen_runner.py | 36 ++++++++----------- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7b9cf51ab3..0fe661eb98 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -111,7 +111,7 @@ jobs: name: test-reports-${{ matrix.version }} path: tests/core/pyspec/test-reports - gen-collect-only: + gen-modcheck: runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] steps: - name: Checkout repository @@ -123,5 +123,5 @@ jobs: cache: '' - name: Install pyspec requirements run: make install_test - - name: Run generators with --collect-only - run: make generate_tests collect_only=true + - name: Run generators with --modcheck + run: make generate_tests modcheck=true diff --git a/Makefile b/Makefile index be83d1a302..abb4ca21b1 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ lint_generators: pyspec flake8 --config $(LINTER_CONFIG_FILE) # If set to true, it will not run generator tests. -collect_only ?= false +modcheck ?= false # Runs a generator, identified by param 1 define run_generator @@ -180,7 +180,7 @@ define run_generator . venv/bin/activate; \ pip3 install ../../../dist/eth2spec-*.whl; \ pip3 install 'eth2spec[generator]'; \ - python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(collect_only)),--collect-only); \ + python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(modcheck)),--modcheck); \ echo "generator $(1) finished" endef diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index aa60d6d92b..7597475b22 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -187,14 +187,18 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): help="specify forks to run with. Allows all if no fork names are specified.", ) parser.add_argument( - "-c", - "--collect-only", + "--modcheck", action="store_true", default=False, - help="if set only print tests to generate, do not actually run the test and dump the target data", + help="check generator modules, do not run any tests.", ) - args = parser.parse_args() + + # Bail here if we are checking modules. + if args.modcheck: + print("Doing modcheck, returning early.") + return + output_dir = args.output_dir if not args.force: file_mode = "x" @@ -222,8 +226,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if len(presets) != 0: print(f"Filtering test-generator runs to only include forks: {', '.join(forks)}") - collect_only = args.collect_only - diagnostics_obj = Diagnostics() provider_start = time.time() @@ -231,9 +233,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): all_test_case_params = [] for tprov in test_providers: - if not collect_only: - # runs anything that we don't want to repeat for every test case. - tprov.prepare() + # Runs anything that we don't want to repeat for every test case. + tprov.prepare() for test_case in tprov.make_cases(): # If preset list is assigned, filter by presets. @@ -248,10 +249,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"Collected test at: {case_dir}") diagnostics_obj.collected_test_count += 1 - # Bail here if we do not want to generate. - if collect_only: - continue - is_skip, diagnostics_obj = should_skip_case_dir(case_dir, args.force, diagnostics_obj) if is_skip: continue @@ -273,14 +270,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): provider_end = time.time() span = round(provider_end - provider_start, 2) - if collect_only: - print(f"Collected {diagnostics_obj.collected_test_count} tests in total") - else: - summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" - summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" - if span > TIME_THRESHOLD_TO_PRINT: - summary_message += f" in {span} seconds" - print(summary_message) + summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" + summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" + if span > TIME_THRESHOLD_TO_PRINT: + summary_message += f" in {span} seconds" + print(summary_message) diagnostics_output = { "collected_test_count": diagnostics_obj.collected_test_count, From c280e56957fb165c6cbbb13949983335e32f90b4 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 18 Oct 2024 12:26:52 -0500 Subject: [PATCH 3/4] Remove print statement --- tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 7597475b22..b6883cc34d 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -196,7 +196,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): # Bail here if we are checking modules. if args.modcheck: - print("Doing modcheck, returning early.") return output_dir = args.output_dir From 82093ddaacc61eab9f44bedda757650e998996c2 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:11:37 -0500 Subject: [PATCH 4/4] Revert some changes --- .github/workflows/run-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0fe661eb98..b8c2fa6e14 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -24,7 +24,7 @@ on: - cron: '0 0 * * *' jobs: - table-of-contents: + table_of_contents: runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] steps: - name: Checkout repository @@ -71,7 +71,7 @@ jobs: pyspec-tests: runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs] - needs: [lint, codespell, table-of-contents] + needs: [lint,codespell,table_of_contents] strategy: matrix: version: ["phase0", "altair", "bellatrix", "capella", "deneb", "electra", "whisk", "eip7594"] @@ -104,7 +104,7 @@ jobs: - name: Install pyspec requirements run: make install_test - name: test-${{ matrix.version }} - run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{ env.spec_test_preset_type }} + run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}} - uses: actions/upload-artifact@v4 if: always() with: