Combine specs and test-generators

.circleci/config.yml

@@ -1,5 +1,4 @@
-# Python CircleCI 2.0 configuration file
-version: 2
+version: 2.1
 jobs:
   build:
     docker:
@@ -8,34 +7,80 @@ jobs:
 
     steps:
       - checkout
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
-      - run:
-          name: install dependencies
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install -r requirements.txt
       - run:
-          name: build phase0 spec
-          command: make build/phase0
+          name: Build pyspec
+          command: make pyspec
 
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
+      - run:
+          name: Run py-tests
+          command: make test
 
       - run:
-          name: run tests
-          command: |
-            . venv/bin/activate
-            pytest tests
+          name: Generate YAML tests
+          command: make gen_yaml_tests
 
       - store_artifacts:
           path: test-reports
           destination: test-reports
+
+      - run:
+          name: Save YAML tests for deployment
+          command: |
+            mkdir /tmp/workspace
+            cp -r yaml_tests /tmp/workspace/
+            git log -1 >> /tmp/workspace/latest_commit_message
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - yaml_tests
+            - latest_commit_message
+  commit:
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - add_ssh_keys:
+          fingerprints:
+            - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa"
+      - run:
+          name: Checkout test repository
+          command: |
+            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
+            git clone [email protected]:ethereum/eth2.0-tests.git
+      - run:
+          name: Commit and push generated YAML tests
+          command: |
+            cd eth2.0-tests
+            git config user.name 'eth2TestGenBot'
+            git config user.email '[email protected]'
+            for filename in /tmp/workspace/yaml_tests/*; do
+              rm -rf $(basename $filename)
+              cp -r $filename .
+            done
+            git add .
+            if git diff --cached --exit-code >& /dev/null; then
+              echo "No changes to commit"
+            else
+              echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message
+              cat /tmp/workspace/latest_commit_message >> commit_message
+              git commit -F commit_message
+              git push origin master
+            fi
+workflows:
+  version: 2.1
+
+  build_and_commit:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only: /.*/
+      - commit:
+          requires:
+            - build
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              ignore: /.*/

.gitignore

@@ -1,7 +1,15 @@
 *.pyc
 /__pycache__
-/venv
+venv
+.venvs
+.venv
 /.pytest_cache
 
 build/
 output/
+
+yaml_tests/
+.pytest_cache
+
+# Dynamically built from Markdown spec
+test_libs/pyspec/pyspec/phase0/spec.py

Makefile

@@ -1,29 +1,73 @@
 SPEC_DIR = ./specs
 SCRIPT_DIR = ./scripts
-BUILD_DIR = ./build
-UTILS_DIR = ./utils
+TEST_LIBS_DIR = ./test_libs
+PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
+PY_TEST_DIR = ./py_tests
+YAML_TEST_DIR = ./yaml_tests
+GENERATOR_DIR = ./test_generators
 
+# Collect a list of generator names
+GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/)))
+# Map this list of generator paths to a list of test output paths
+YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS))
+GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
 
-.PHONY: clean all test
+PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/pyspec/phase0/spec.py
+PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS)
 
 
-all: $(BUILD_DIR)/phase0
+.PHONY: clean all test gen_yaml_tests pyspec phase0
 
+all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
 clean:
-	rm -rf $(BUILD_DIR)
+	rm -rf $(YAML_TEST_DIR)
+	rm -rf $(GENERATOR_VENVS)
+	rm -rf $(PY_TEST_DIR)/venv $(PY_TEST_DIR)/.pytest_cache
+	rm -rf $(PY_SPEC_ALL_TARGETS)
 
+# "make gen_yaml_tests" to run generators
+gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
 # runs a limited set of tests against a minimal config
-# run pytest with `-m` option to full suite
-test:
-	pytest -m minimal_config tests/
+test: $(PY_SPEC_ALL_TARGETS)
+	cd $(PY_TEST_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; pytest -m minimal_config .
 
+# "make pyspec" to create the pyspec for all phases.
+pyspec: $(PY_SPEC_ALL_TARGETS)
 
-$(BUILD_DIR)/phase0:
+# "make phase0" to create pyspec for phase0
+phase0: $(PY_SPEC_PHASE_0_TARGETS)
+
+
+$(PY_SPEC_DIR)/pyspec/phase0/spec.py:
+	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@
+
+
+CURRENT_DIR = ${CURDIR}
+
+# The function that builds a set of suite files, by calling a generator for the given type (param 1)
+define build_yaml_tests
+	$(info running generator $(1))
+	# Create the output
+	mkdir -p $(YAML_TEST_DIR)$(1)
+
+	# 1) Create a virtual environment
+	# 2) Activate the venv, this is where dependencies are installed for the generator
+	# 3) Install all the necessary requirements
+	# 4) Run the generator. The generator is assumed to have an "main.py" file.
+	# 5) We output to the tests dir (generator program should accept a "-o <filepath>" argument.
+	cd $(GENERATOR_DIR)$(1); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1)
+
+	$(info generator $(1) finished)
+endef
+
+# The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary)
+$(YAML_TEST_DIR):
+	$(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS})
 	mkdir -p $@
-	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py
-	mkdir -p $@/utils
-	cp $(UTILS_DIR)/phase0/* $@/utils
-	cp $(UTILS_DIR)/phase0/state_transition.py $@
-	touch $@/__init__.py $@/utils/__init__.py
+
+# For any target within the tests dir, build it using the build_yaml_tests function.
+# (creation of output dir is a dependency)
+$(YAML_TEST_DIR)%: $(YAML_TEST_DIR)
+	$(call build_yaml_tests,$*)

README.md

@@ -6,7 +6,8 @@ To learn more about sharding and eth2.0/Serenity, see the [sharding FAQ](https:/
 
 This repo hosts the current eth2.0 specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed upon changes to spec can be made through pull requests.
 
-# Specs
+
+## Specs
 
 Core specifications for eth2.0 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are:
 * [Phase 0 -- The Beacon Chain](specs/core/0_beacon-chain.md)
@@ -18,10 +19,20 @@ Accompanying documents can be found in [specs](specs) and include
 * [General test format](specs/test-format.md)
 * [Honest validator implementation doc](specs/validator/0_beacon-chain-validator.md)
 
-## Design goals
+
+### Design goals
+
 The following are the broad design goals for Ethereum 2.0:
 * to minimize complexity, even at the cost of some losses in efficiency
 * to remain live through major network partitions and when very large portions of nodes go offline
 * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available
 * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time
 * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain)
+
+
+## For spec contributors
+
+Documentation on the different components used during spec writing can be found here:
+* [YAML Test Generators](test_generators/README.md)
+* [Executable Python Spec](test_libs/pyspec/README.md)
+

py_tests/README.md

@@ -0,0 +1,30 @@
+# ETH 2.0 py-tests
+
+These tests are not intended for client-consumption.
+These tests are sanity tests, to verify if the spec itself is consistent.
+
+There are ideas to port these tests to the YAML test suite,
+ but we are still looking for inputs on how this should work.
+
+## How to run tests
+
+### Automated
+
+Run `make test` from the root of the spec repository.
+
+### Manual
+
+From within the py_tests folder:
+
+Install dependencies:
+```bash
+python3 -m venv venv
+. venv/bin/activate
+pip3 install -r requirements.txt
+```
+Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement.
+
+Run the tests:
+```
+pytest -m minimal_config .
+```

tests/phase0/block_processing/test_process_attestation.py → py_tests/phase0/block_processing/test_process_attestation.py

@@ -1,18 +1,17 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import pyspec.phase0.spec as spec
 
-from build.phase0.state_transition import (
+from pyspec.phase0.state_transition import (
     state_transition,
 )
-from build.phase0.spec import (
-    ZERO_HASH,
+from pyspec.phase0.spec import (
     get_current_epoch,
     process_attestation,
     slot_to_epoch,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_empty_block_for_next_slot,
     get_valid_attestation,
 )

tests/phase0/block_processing/test_process_block_header.py → py_tests/phase0/block_processing/test_process_block_header.py

@@ -2,13 +2,13 @@
 import pytest
 
 
-from build.phase0.spec import (
+from pyspec.phase0.spec import (
     get_beacon_proposer_index,
     cache_state,
     advance_slot,
     process_block_header,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_empty_block_for_next_slot,
 )
 

tests/phase0/block_processing/test_process_deposit.py → py_tests/phase0/block_processing/test_process_deposit.py

@@ -1,14 +1,14 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import pyspec.phase0.spec as spec
 
-from build.phase0.spec import (
+from pyspec.phase0.spec import (
     get_balance,
     ZERO_HASH,
     process_deposit,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_deposit,
     privkeys,
     pubkeys,

tests/phase0/block_processing/test_process_proposer_slashing.py → py_tests/phase0/block_processing/test_process_proposer_slashing.py

@@ -1,13 +1,13 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
-from build.phase0.spec import (
+import pyspec.phase0.spec as spec
+from pyspec.phase0.spec import (
     get_balance,
     get_current_epoch,
     process_proposer_slashing,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     get_valid_proposer_slashing,
 )
 

tests/phase0/block_processing/test_voluntary_exit.py → py_tests/phase0/block_processing/test_voluntary_exit.py

@@ -1,14 +1,14 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import pyspec.phase0.spec as spec
 
-from build.phase0.spec import (
+from pyspec.phase0.spec import (
     get_active_validator_indices,
     get_current_epoch,
     process_voluntary_exit,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_voluntary_exit,
     pubkey_to_privkey,
 )

tests/phase0/conftest.py → py_tests/phase0/conftest.py

@@ -1,8 +1,8 @@
 import pytest
 
-from build.phase0 import spec
+from pyspec.phase0 import spec
 
-from tests.phase0.helpers import (
+from .helpers import (
     create_genesis_state,
 )