SSZ signature from EF are always opaque blobs (security issue - #555)

Enable
- Attestation
- Beaconstate (minimal only)
- Deposit
- DepositData
- ProposerSlashing

Updates #518

tests/official/debug_ssz.nim

@@ -0,0 +1,75 @@
+# beacon_chain
+# Copyright (c) 2018-Present Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  # Standard library
+  os, unittest, strutils, streams,
+  # Status libraries
+  stint, stew/bitseqs,
+  # Third-party
+  yaml,
+  # Beacon chain internals
+  ../../beacon_chain/spec/[datatypes, digest, crypto],
+  ../../beacon_chain/ssz
+
+# Config
+# ---------------------------------------------
+
+const
+  FixturesDir = currentSourcePath.rsplit(DirSep, 1)[0] / "fixtures"
+  SSZDir = FixturesDir/"tests-v0.9.1"/const_preset/"phase0"/"ssz_static"
+  UnitTestDir = SSZDir/"Attestation"/"ssz_lengthy"/"case_12"
+
+type
+  SSZHashTreeRoot = object
+    # The test files have the values at the "root"
+    # so we **must** use "root" as a field name
+    root: string
+    # Some have a signing_root field
+    signing_root: string
+
+# Make signing root optional
+setDefaultValue(SSZHashTreeRoot, signing_root, "")
+
+# Parsing + Test
+# ---------------------------------------------
+
+type Skip = enum
+  SkipNone
+  SkipHashTreeRoot
+  SkipSigningRoot
+
+proc checkSSZ(T: typedesc, dir: string, expectedHash: SSZHashTreeRoot, skip = SkipNone) =
+  # Deserialize into a ref object to not fill Nim stack
+  var deserialized: ref T
+  new deserialized
+  deserialized[] = SSZ.loadFile(dir/"serialized.ssz", T)
+
+  echo "\n\nObject: ", T
+  echo "---------------------------------------"
+  echo deserialized[]
+
+  if not(skip == SkipHashTreeRoot):
+    check: expectedHash.root == "0x" & toLowerASCII($deserialized.hashTreeRoot())
+  if expectedHash.signing_root != "" and not(skip == SkipSigningRoot):
+    check: expectedHash.signing_root == "0x" & toLowerASCII($deserialized[].signingRoot())
+
+proc loadExpectedHashTreeRoot(dir: string): SSZHashTreeRoot =
+  var s = openFileStream(dir/"roots.yaml")
+  yaml.load(s, result)
+  s.close()
+
+# Manual checks
+# ---------------------------------------------
+
+# Compile with -d:ssz_testing for consensus objects
+# as they are always an Opaque Blob even if they might seem like a valid BLS signature
+
+echo "Current preset: ", const_preset
+
+let hash = loadExpectedHashTreeRoot(UnitTestDir)
+checkSSZ(Attestation, UnitTestDir, hash)

tests/official/test_fixture_ssz_consensus_objects.nim

@@ -43,28 +43,29 @@ setDefaultValue(SSZHashTreeRoot, signing_root, "")
 # Checking the values against the yaml file is TODO (require more flexible Yaml parser)
 const Unsupported = toHashSet([
     "AggregateAndProof",    # Type for signature aggregation - not implemented
-    "Attestation",          # RangeError on deserialization
-    # "AttestationData",
-    "AttesterSlashing",     # RangeError on deserialization
-    "BeaconBlock",          # RangeError on deserialization
-    "BeaconBlockBody",      # RangeError on deserialization
-    # "BeaconBlockHeader",  # HashTreeRoot KO - SigningRook OK
-    "BeaconState",          # HashTreeRoot KO
-    # "Checkpoint",
-    "Deposit",              # HashTreeRoot KO
-    "DepositData",          # HashTreeRoot KO - SigningRoot KO
-    # "Eth1Data",
-    # "Fork",
-    # "HistoricalBatch",    # OK
-    "IndexedAttestation",   # RangeError on deserialization
-    # "PendingAttestation", # OK
-    "ProposerSlashing",     # HashTreeRoot KO
+    # "Attestation",        #
+    # "AttestationData",    #
+    "AttesterSlashing",     # HashTreeRoot KO
+    "BeaconBlock",          # HashTreeRoot KO
+    "BeaconBlockBody",      # HashTreeRoot KO
+    # "BeaconBlockHeader",  #
+    # "BeaconState",        # OK on minimal, KO on mainnet
+    # "Checkpoint",         #
+    # "Deposit",            #
+    # "DepositData",        #
+    # "Eth1Data",           #
+    # "Fork",               #
+    # "HistoricalBatch",    #
+    "IndexedAttestation",   # HashTreeRoot KO
+    # "PendingAttestation", # OK on minimal, KO on mainnet
+    # "ProposerSlashing",   #
     "Validator",            # HashTreeRoot KO
-    # "VoluntaryExit"       # hashTreeRoot KO - SigningRoot OK
+    # "VoluntaryExit"       #
   ])
 
 const UnsupportedMainnet = toHashSet([
     "PendingAttestation",   # HashTreeRoot KO
+    "BeaconState",
   ])
 
 type Skip = enum
@@ -78,6 +79,7 @@ proc checkSSZ(T: typedesc, dir: string, expectedHash: SSZHashTreeRoot, skip = Sk
   new deserialized
   deserialized[] = SSZ.loadFile(dir/"serialized.ssz", T)
 
+  # echo "Dir: ", dir
   if not(skip == SkipHashTreeRoot):
     check: expectedHash.root == "0x" & toLowerASCII($deserialized.hashTreeRoot())
   if expectedHash.signing_root != "" and not(skip == SkipSigningRoot):
@@ -108,10 +110,6 @@ proc runSSZtests() =
           discard
         continue
 
-    let signingRootOnly = &"                     ↶↶↶ {sszType} - Skipping HashTreeRoot and testing SigningRoot only"
-    if sszType == "BeaconBlockHeader" or sszType == "VoluntaryExit":
-      echo signingRootOnly
-
     test &"  Testing    {sszType:20}   consensus object ✓✓✓":
       let path = SSZDir/sszType
       for pathKind, sszTestKind in walkDir(path, relative = true):
@@ -128,7 +126,7 @@ proc runSSZtests() =
           of "AttesterSlashing": checkSSZ(AttesterSlashing, path, hash)
           of "BeaconBlock": checkSSZ(BeaconBlock, path, hash)
           of "BeaconBlockBody": checkSSZ(BeaconBlockBody, path, hash)
-          of "BeaconBlockHeader": checkSSZ(BeaconBlockHeader, path, hash, SkipHashTreeRoot) # TODO
+          of "BeaconBlockHeader": checkSSZ(BeaconBlockHeader, path, hash)
           of "BeaconState": checkSSZ(BeaconState, path, hash)
           of "Checkpoint": checkSSZ(Checkpoint, path, hash)
           of "Deposit": checkSSZ(Deposit, path, hash)
@@ -140,7 +138,7 @@ proc runSSZtests() =
           of "PendingAttestation": checkSSZ(PendingAttestation, path, hash)
           of "ProposerSlashing": checkSSZ(ProposerSlashing, path, hash)
           of "Validator": checkSSZ(VoluntaryExit, path, hash)
-          of "VoluntaryExit": checkSSZ(VoluntaryExit, path, hash, SkipHashTreeRoot) # TODO
+          of "VoluntaryExit": checkSSZ(VoluntaryExit, path, hash)
           else:
             raise newException(ValueError, "Unsupported test: " & sszType)
 

tests/official/test_fixture_ssz_consensus_objects.nim.cfg

@@ -0,0 +1 @@
+-d:ssz_testing