-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2 from status-im/move-ssz-code-ii
Copy ssz code from nimbus-eth2 and adjust to stand on its own
- Loading branch information
Showing
13 changed files
with
3,003 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,250 @@ | ||
| # ssz_serialization | ||
| # Copyright (c) 2018-2021 Status Research & Development GmbH | ||
| # Licensed and distributed under either of | ||
| # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). | ||
| # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). | ||
| # at your option. This file may not be copied, modified, or distributed except according to those terms. | ||
|
|
||
| {.push raises: [Defect].} | ||
| {.pragma: raisesssz, raises: [Defect, MalformedSszError, SszSizeMismatchError].} | ||
|
|
||
| ## SSZ serialization for core SSZ types, as specified in: | ||
| # https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#serialization | ||
|
|
||
| import | ||
| std/typetraits, | ||
| stew/[endians2, leb128, objects], | ||
| serialization, serialization/testing/tracing, | ||
| ./ssz_serialization/[codec, bitseqs, types] | ||
|
|
||
| export | ||
| serialization, codec, types, bitseqs | ||
|
|
||
| type | ||
| SszReader* = object | ||
| stream: InputStream | ||
|
|
||
| SszWriter* = object | ||
| stream: OutputStream | ||
|
|
||
| SizePrefixed*[T] = distinct T | ||
| SszMaxSizeExceeded* = object of SerializationError | ||
|
|
||
| VarSizedWriterCtx = object | ||
| fixedParts: WriteCursor | ||
| offset: int | ||
|
|
||
| FixedSizedWriterCtx = object | ||
|
|
||
| serializationFormat SSZ | ||
|
|
||
| SSZ.setReader SszReader | ||
| SSZ.setWriter SszWriter, PreferredOutput = seq[byte] | ||
|
|
||
| template sizePrefixed*[TT](x: TT): untyped = | ||
| type T = TT | ||
| SizePrefixed[T](x) | ||
|
|
||
| proc init*(T: type SszReader, | ||
| stream: InputStream): T = | ||
| T(stream: stream) | ||
|
|
||
| proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) {.raises: [Defect, IOError].} = | ||
| mixin toSszType | ||
|
|
||
| when x is byte: | ||
| s.write x | ||
| elif x is bool: | ||
| s.write byte(ord(x)) | ||
| elif x is UintN: | ||
| when cpuEndian == bigEndian: | ||
| s.write toBytesLE(x) | ||
| else: | ||
| s.writeMemCopy x | ||
| elif x is array: | ||
| when x[0] is byte: | ||
| trs "APPENDING FIXED SIZE BYTES", x | ||
| s.write x | ||
| else: | ||
| for elem in x: | ||
| trs "WRITING FIXED SIZE ARRAY ELEMENT" | ||
| s.writeFixedSized toSszType(elem) | ||
| elif x is tuple|object: | ||
| enumInstanceSerializedFields(x, fieldName, field): | ||
| trs "WRITING FIXED SIZE FIELD", fieldName | ||
| s.writeFixedSized toSszType(field) | ||
| else: | ||
| unsupported x.type | ||
|
|
||
| template writeOffset(cursor: var WriteCursor, offset: int) = | ||
| write cursor, toBytesLE(uint32 offset) | ||
|
|
||
| template supports*(_: type SSZ, T: type): bool = | ||
| mixin toSszType | ||
| anonConst compiles(fixedPortionSize toSszType(declval T)) | ||
|
|
||
| func init*(T: type SszWriter, stream: OutputStream): T = | ||
| result.stream = stream | ||
|
|
||
| proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe, raises: [Defect, IOError].} | ||
|
|
||
| proc beginRecord*(w: var SszWriter, TT: type): auto = | ||
| type T = TT | ||
| when isFixedSize(T): | ||
| FixedSizedWriterCtx() | ||
| else: | ||
| const offset = when T is array|HashArray: len(T) * offsetSize | ||
| else: fixedPortionSize(T) | ||
| VarSizedWriterCtx(offset: offset, | ||
| fixedParts: w.stream.delayFixedSizeWrite(offset)) | ||
|
|
||
| template writeField*(w: var SszWriter, | ||
| ctx: var auto, | ||
| fieldName: string, | ||
| field: auto) = | ||
| mixin toSszType | ||
| when ctx is FixedSizedWriterCtx: | ||
| writeFixedSized(w.stream, toSszType(field)) | ||
| else: | ||
| type FieldType = type toSszType(field) | ||
|
|
||
| when isFixedSize(FieldType): | ||
| writeFixedSized(ctx.fixedParts, toSszType(field)) | ||
| else: | ||
| trs "WRITING OFFSET ", ctx.offset, " FOR ", fieldName | ||
| writeOffset(ctx.fixedParts, ctx.offset) | ||
| let initPos = w.stream.pos | ||
| trs "WRITING VAR SIZE VALUE OF TYPE ", name(FieldType) | ||
| when FieldType is BitList: | ||
| trs "BIT SEQ ", bytes(field) | ||
| writeVarSizeType(w, toSszType(field)) | ||
| ctx.offset += w.stream.pos - initPos | ||
|
|
||
| template endRecord*(w: var SszWriter, ctx: var auto) = | ||
| when ctx is VarSizedWriterCtx: | ||
| finalize ctx.fixedParts | ||
|
|
||
| proc writeSeq[T](w: var SszWriter, value: seq[T]) | ||
| {.raises: [Defect, IOError].} = | ||
| # Please note that `writeSeq` exists in order to reduce the code bloat | ||
| # produced from generic instantiations of the unique `List[N, T]` types. | ||
| when isFixedSize(T): | ||
| trs "WRITING LIST WITH FIXED SIZE ELEMENTS" | ||
| for elem in value: | ||
| w.stream.writeFixedSized toSszType(elem) | ||
| trs "DONE" | ||
| else: | ||
| trs "WRITING LIST WITH VAR SIZE ELEMENTS" | ||
| var offset = value.len * offsetSize | ||
| var cursor = w.stream.delayFixedSizeWrite offset | ||
| for elem in value: | ||
| cursor.writeFixedSized uint32(offset) | ||
| let initPos = w.stream.pos | ||
| w.writeVarSizeType toSszType(elem) | ||
| offset += w.stream.pos - initPos | ||
| finalize cursor | ||
| trs "DONE" | ||
|
|
||
| proc writeVarSizeType(w: var SszWriter, value: auto) {.raises: [Defect, IOError].} = | ||
| trs "STARTING VAR SIZE TYPE" | ||
|
|
||
| when value is HashArray|HashList: | ||
| writeVarSizeType(w, value.data) | ||
| elif value is SingleMemberUnion: | ||
| doAssert value.selector == 0'u8 | ||
| w.writeValue 0'u8 | ||
| w.writeValue value.value | ||
| elif value is List: | ||
| # We reduce code bloat by forwarding all `List` types to a general `seq[T]` proc. | ||
| writeSeq(w, asSeq value) | ||
| elif value is BitList: | ||
| # ATTENTION! We can reuse `writeSeq` only as long as our BitList type is implemented | ||
| # to internally match the binary representation of SSZ BitLists in memory. | ||
| writeSeq(w, bytes value) | ||
| elif value is object|tuple|array: | ||
| trs "WRITING OBJECT OR ARRAY" | ||
| var ctx = beginRecord(w, type value) | ||
| enumerateSubFields(value, field): | ||
| writeField w, ctx, astToStr(field), field | ||
| endRecord w, ctx | ||
| else: | ||
| unsupported type(value) | ||
|
|
||
| proc writeValue*(w: var SszWriter, x: auto) {.gcsafe, raises: [Defect, IOError].} = | ||
| mixin toSszType | ||
| type T = type toSszType(x) | ||
|
|
||
| when isFixedSize(T): | ||
| w.stream.writeFixedSized toSszType(x) | ||
| else: | ||
| w.writeVarSizeType toSszType(x) | ||
|
|
||
| func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} | ||
|
|
||
| func sszSizeForVarSizeList[T](value: openArray[T]): int = | ||
| result = len(value) * offsetSize | ||
| for elem in value: | ||
| result += sszSize(toSszType elem) | ||
|
|
||
| func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} = | ||
| mixin toSszType | ||
| type T = type toSszType(value) | ||
|
|
||
| when isFixedSize(T): | ||
| anonConst fixedPortionSize(T) | ||
|
|
||
| elif T is array|List|HashList|HashArray: | ||
| type E = ElemType(T) | ||
| when isFixedSize(E): | ||
| len(value) * anonConst(fixedPortionSize(E)) | ||
| elif T is HashArray: | ||
| sszSizeForVarSizeList(value.data) | ||
| elif T is array: | ||
| sszSizeForVarSizeList(value) | ||
| else: | ||
| sszSizeForVarSizeList(asSeq value) | ||
|
|
||
| elif T is BitList: | ||
| return len(bytes(value)) | ||
|
|
||
| elif T is SingleMemberUnion: | ||
| sszSize(toSszType value.value) + 1 | ||
|
|
||
| elif T is object|tuple: | ||
| result = anonConst fixedPortionSize(T) | ||
| enumInstanceSerializedFields(value, _{.used.}, field): | ||
| type FieldType = type toSszType(field) | ||
| when not isFixedSize(FieldType): | ||
| result += sszSize(toSszType field) | ||
|
|
||
| else: | ||
| unsupported T | ||
|
|
||
| proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} = | ||
| var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64)) | ||
| let initPos = w.stream.pos | ||
| w.writeValue T(x) | ||
| let length = toBytes(uint64(w.stream.pos - initPos), Leb128) | ||
| cursor.finalWrite length.toOpenArray() | ||
|
|
||
| proc readValue*(r: var SszReader, val: var auto) {. | ||
| raises: [Defect, MalformedSszError, SszSizeMismatchError, IOError].} = | ||
| mixin readSszBytes | ||
| type T = type val | ||
| when isFixedSize(T): | ||
| const minimalSize = fixedPortionSize(T) | ||
| if r.stream.readable(minimalSize): | ||
| readSszBytes(r.stream.read(minimalSize), val) | ||
| else: | ||
| raise newException(MalformedSszError, "SSZ input of insufficient size") | ||
| else: | ||
| # TODO(zah) Read the fixed portion first and precisely measure the | ||
| # size of the dynamic portion to consume the right number of bytes. | ||
| readSszBytes(r.stream.read(r.stream.len.get), val) | ||
|
|
||
| proc readSszBytes*[T](data: openArray[byte], val: var T) {. | ||
| raises: [Defect, MalformedSszError, SszSizeMismatchError].} = | ||
| # Overload `readSszBytes` to perform custom operations on T after | ||
| # deserialization | ||
| mixin readSszValue | ||
| readSszValue(data, val) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.