Combine specs and test-generators

.circleci/config.yml

@@ -1,5 +1,4 @@
-# Python CircleCI 2.0 configuration file
-version: 2
+version: 2.1
 jobs:
   build:
     docker:
@@ -8,34 +7,83 @@ jobs:
 
     steps:
       - checkout
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
-      - run:
-          name: install dependencies
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install -r requirements.txt
       - run:
-          name: build phase0 spec
-          command: make build/phase0
+          name: Build pyspec
+          command: make pyspec
 
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
+      - run:
+          name: Run py-tests
+          command: make test
 
       - run:
-          name: run tests
-          command: |
-            . venv/bin/activate
-            pytest tests
+          name: Generate YAML tests
+          command: make gen_yaml_tests
 
-      - store_artifacts:
-          path: test-reports
-          destination: test-reports
+# TODO in future PR (after #851): decide on CI triggering of yaml tests building,
+#  and destination of output (new yaml tests LFS-configured repository)
+#
+#      - store_artifacts:
+#          path: test-reports
+#          destination: test-reports
+#
+#      - run:
+#          name: Save YAML tests for deployment
+#          command: |
+#            mkdir /tmp/workspace
+#            cp -r yaml_tests /tmp/workspace/
+#            git log -1 >> /tmp/workspace/latest_commit_message
+#      - persist_to_workspace:
+#          root: /tmp/workspace
+#          paths:
+#            - yaml_tests
+#            - latest_commit_message
+#  commit:
+#    docker:
+#      - image: circleci/python:3.6
+#    steps:
+#      - attach_workspace:
+#          at: /tmp/workspace
+#      - add_ssh_keys:
+#          fingerprints:
+#            - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa"
+#      - run:
+#          name: Checkout test repository
+#          command: |
+#            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
+#            git clone [email protected]:ethereum/eth2.0-tests.git
+#      - run:
+#          name: Commit and push generated YAML tests
+#          command: |
+#            cd eth2.0-tests
+#            git config user.name 'eth2TestGenBot'
+#            git config user.email '[email protected]'
+#            for filename in /tmp/workspace/yaml_tests/*; do
+#              rm -rf $(basename $filename)
+#              cp -r $filename .
+#            done
+#            git add .
+#            if git diff --cached --exit-code >& /dev/null; then
+#              echo "No changes to commit"
+#            else
+#              echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message
+#              cat /tmp/workspace/latest_commit_message >> commit_message
+#              git commit -F commit_message
+#              git push origin master
+#            fi
+#workflows:
+#  version: 2.1
+#
+#  build_and_commit:
+#    jobs:
+#      - build:
+#          filters:
+#            tags:
+#              only: /.*/
+#      - commit:
+#          requires:
+#            - build
+#          filters:
+#            tags:
+#              only: /.*/
+#            branches:
+#              ignore: /.*/

.gitignore

@@ -1,7 +1,15 @@
 *.pyc
 /__pycache__
-/venv
+venv
+.venvs
+.venv
 /.pytest_cache
 
 build/
 output/
+
+yaml_tests/
+.pytest_cache
+
+# Dynamically built from Markdown spec
+test_libs/pyspec/eth2spec/phase0/spec.py

Makefile

@@ -1,29 +1,74 @@
 SPEC_DIR = ./specs
 SCRIPT_DIR = ./scripts
-BUILD_DIR = ./build
-UTILS_DIR = ./utils
+TEST_LIBS_DIR = ./test_libs
+PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
+PY_TEST_DIR = ./py_tests
+YAML_TEST_DIR = ./yaml_tests
+GENERATOR_DIR = ./test_generators
+CONFIGS_DIR = ./configs
 
+# Collect a list of generator names
+GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/)))
+# Map this list of generator paths to a list of test output paths
+YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS))
+GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
 
-.PHONY: clean all test
+PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
+PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS)
 
 
-all: $(BUILD_DIR)/phase0
+.PHONY: clean all test gen_yaml_tests pyspec phase0
 
+all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
 clean:
-	rm -rf $(BUILD_DIR)
+	rm -rf $(YAML_TEST_DIR)
+	rm -rf $(GENERATOR_VENVS)
+	rm -rf $(PY_TEST_DIR)/venv $(PY_TEST_DIR)/.pytest_cache
+	rm -rf $(PY_SPEC_ALL_TARGETS)
 
+# "make gen_yaml_tests" to run generators
+gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
 # runs a limited set of tests against a minimal config
-# run pytest with `-m` option to full suite
-test: all
-	pytest -m minimal_config tests/
+test: $(PY_SPEC_ALL_TARGETS)
+	cd $(PY_TEST_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; pytest -m minimal_config .
 
+# "make pyspec" to create the pyspec for all phases.
+pyspec: $(PY_SPEC_ALL_TARGETS)
 
-$(BUILD_DIR)/phase0: $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/*.py $(UTILS_DIR)/phase0/*.py
+# "make phase0" to create pyspec for phase0
+phase0: $(PY_SPEC_PHASE_0_TARGETS)
+
+
+$(PY_SPEC_DIR)/eth2spec/phase0/spec.py:
+	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@
+
+
+CURRENT_DIR = ${CURDIR}
+
+# The function that builds a set of suite files, by calling a generator for the given type (param 1)
+define build_yaml_tests
+	$(info running generator $(1))
+	# Create the output
+	mkdir -p $(YAML_TEST_DIR)$(1)
+
+	# 1) Create a virtual environment
+	# 2) Activate the venv, this is where dependencies are installed for the generator
+	# 3) Install all the necessary requirements
+	# 4) Run the generator. The generator is assumed to have an "main.py" file.
+	# 5) We output to the tests dir (generator program should accept a "-o <filepath>" argument.
+	cd $(GENERATOR_DIR)$(1); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR)
+
+	$(info generator $(1) finished)
+endef
+
+# The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary)
+$(YAML_TEST_DIR):
+	$(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS})
 	mkdir -p $@
-	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py
-	mkdir -p $@/utils
-	cp $(UTILS_DIR)/phase0/* $@/utils
-	cp $(UTILS_DIR)/phase0/state_transition.py $@
-	touch $@/__init__.py $@/utils/__init__.py
+
+# For any target within the tests dir, build it using the build_yaml_tests function.
+# (creation of output dir is a dependency)
+$(YAML_TEST_DIR)%: $(YAML_TEST_DIR)
+	$(call build_yaml_tests,$*)

README.md

@@ -6,7 +6,8 @@ To learn more about sharding and eth2.0/Serenity, see the [sharding FAQ](https:/
 
 This repo hosts the current eth2.0 specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed upon changes to spec can be made through pull requests.
 
-# Specs
+
+## Specs
 
 Core specifications for eth2.0 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are:
 * [Phase 0 -- The Beacon Chain](specs/core/0_beacon-chain.md)
@@ -21,10 +22,21 @@ Accompanying documents can be found in [specs](specs) and include
 * [Merkle proof formats](specs/light_client/merkle_proofs.md)
 * [Light client syncing protocol](specs/light_client/sync_protocol.md)
 
-## Design goals
+
+### Design goals
+
 The following are the broad design goals for Ethereum 2.0:
 * to minimize complexity, even at the cost of some losses in efficiency
 * to remain live through major network partitions and when very large portions of nodes go offline
 * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available
 * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time
 * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain)
+
+
+## For spec contributors
+
+Documentation on the different components used during spec writing can be found here:
+* [YAML Test Generators](test_generators/README.md)
+* [Executable Python Spec](test_libs/eth2spec/README.md)
+* [Py-tests](py_tests/README.md)
+

configs/constant_presets/mainnet.yaml

@@ -22,7 +22,7 @@ SHUFFLE_ROUND_COUNT: 90
 # Deposit contract
 # ---------------------------------------------------------------
 # **TBD**
-DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123567890123456789012357890
+DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
 # 2**5 ` (= 32)
 DEPOSIT_CONTRACT_TREE_DEPTH: 32
 

configs/constant_presets/minimal.yaml

@@ -16,13 +16,13 @@ MAX_ATTESTATION_PARTICIPANTS: 4096
 # 2**2 ` (= 4)
 MAX_EXIT_DEQUEUES_PER_EPOCH: 4
 # See issue 563
-SHUFFLE_ROUND_COUNT: 90
+SHUFFLE_ROUND_COUNT: 10
 
 
 # Deposit contract
 # ---------------------------------------------------------------
 # **TBD**
-DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123567890123456789012357890
+DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
 # 2**5 ` (= 32)
 DEPOSIT_CONTRACT_TREE_DEPTH: 32
 
@@ -62,8 +62,8 @@ SLOTS_PER_EPOCH: 8
 MIN_SEED_LOOKAHEAD: 1
 # 2**2 ` (= 4) epochs 25.6 minutes
 ACTIVATION_EXIT_DELAY: 4
-# 2**4 ` (= 16) epochs ~1.7 hours
-EPOCHS_PER_ETH1_VOTING_PERIOD: 16
+# [customized] higher frequency new deposits from eth1 for testing
+EPOCHS_PER_ETH1_VOTING_PERIOD: 2
 # [customized] smaller state
 SLOTS_PER_HISTORICAL_ROOT: 64
 # 2**8 ` (= 256) epochs ~27 hours

configs/fork_timelines/testing.yaml

@@ -0,0 +1,6 @@
+# Testing fork timeline
+
+# Equal to GENESIS_EPOCH
+phase0: 536870912
+
+# No other forks considered in testing yet (to be implemented)

py_tests/README.md

@@ -0,0 +1,30 @@
+# ETH 2.0 py-tests
+
+These tests are not intended for client-consumption.
+These tests are sanity tests, to verify if the spec itself is consistent.
+
+There are ideas to port these tests to the YAML test suite,
+ but we are still looking for inputs on how this should work.
+
+## How to run tests
+
+### Automated
+
+Run `make test` from the root of the spec repository.
+
+### Manual
+
+From within the py_tests folder:
+
+Install dependencies:
+```bash
+python3 -m venv venv
+. venv/bin/activate
+pip3 install -r requirements.txt
+```
+Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement.
+
+Run the tests:
+```
+pytest -m minimal_config .
+```

tests/phase0/block_processing/test_process_attestation.py → py_tests/phase0/block_processing/test_process_attestation.py

@@ -1,17 +1,17 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import eth2spec.phase0.spec as spec
 
-from build.phase0.state_transition import (
+from eth2spec.phase0.state_transition import (
     state_transition,
 )
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_current_epoch,
     process_attestation,
     slot_to_epoch,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_empty_block_for_next_slot,
     get_valid_attestation,
 )

tests/phase0/block_processing/test_process_attester_slashing.py → py_tests/phase0/block_processing/test_process_attester_slashing.py

@@ -1,13 +1,13 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
-from build.phase0.spec import (
+import eth2spec.phase0.spec as spec
+from eth2spec.phase0.spec import (
     get_balance,
     get_beacon_proposer_index,
     process_attester_slashing,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     get_valid_attester_slashing,
 )
 

tests/phase0/block_processing/test_process_block_header.py → py_tests/phase0/block_processing/test_process_block_header.py

@@ -2,13 +2,13 @@
 import pytest
 
 
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_beacon_proposer_index,
     cache_state,
     advance_slot,
     process_block_header,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_empty_block_for_next_slot,
 )