From a0732100c76d8777ff30877991aafc5e8b8df930 Mon Sep 17 00:00:00 2001 From: Jian Xiao Date: Mon, 24 Jun 2024 22:33:08 +0000 Subject: [PATCH 1/4] [1/N] Chunk encoding optimization: Support encoding format with gnark --- encoding/serialization.go | 36 +++++++++++++++++++++++++++ encoding/serialization_test.go | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 encoding/serialization_test.go diff --git a/encoding/serialization.go b/encoding/serialization.go index 830d29a77..bccc58a78 100644 --- a/encoding/serialization.go +++ b/encoding/serialization.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "encoding/json" + "errors" "fmt" "github.com/consensys/gnark-crypto/ecc/bn254" @@ -22,6 +23,41 @@ func (c *Frame) Deserialize(data []byte) (*Frame, error) { return c, err } +func (c *Frame) SerializeGnark() ([]byte, error) { + coded := make([]byte, 0) + // This is compressed format with just 32 bytes. + proofBytes := c.Proof.Bytes() + coded = append(coded, proofBytes[:]...) + for _, coeff := range c.Coeffs { + coded = append(coded, coeff.Marshal()...) + } + return coded, nil +} + +func (c *Frame) DeserializeGnark(data []byte) (*Frame, error) { + var f Frame + buf := data + err := f.Proof.Unmarshal(buf[:32]) + if err != nil { + return nil, err + } + buf = buf[32:] + if len(buf)%32 != 0 { + return nil, errors.New("invalid chunk length") + } + f.Coeffs = make([]Symbol, len(buf)/32) + i := 0 + for len(buf) > 0 { + if len(buf) < 32 { + return nil, errors.New("invalid chunk length") + } + f.Coeffs[i].Unmarshal(buf[:32]) + i++ + buf = buf[32:] + } + return &f, nil +} + func (f *Frame) Encode() ([]byte, error) { var buf bytes.Buffer enc := gob.NewEncoder(&buf) diff --git a/encoding/serialization_test.go b/encoding/serialization_test.go new file mode 100644 index 000000000..967bdb0e9 --- /dev/null +++ b/encoding/serialization_test.go @@ -0,0 +1,45 @@ +package encoding_test + +import ( + "testing" + + "github.com/Layr-Labs/eigenda/encoding" + "github.com/consensys/gnark-crypto/ecc/bn254/fp" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/stretchr/testify/assert" +) + +func TestSerDeserGnark(t *testing.T) { + var XCoord, YCoord fp.Element + _, err := XCoord.SetString("21661178944771197726808973281966770251114553549453983978976194544185382599016") + assert.NoError(t, err) + _, err = YCoord.SetString("9207254729396071334325696286939045899948985698134704137261649190717970615186") + assert.NoError(t, err) + + var f encoding.Frame + f.Proof = encoding.Proof{ + X: XCoord, + Y: YCoord, + } + for i := 0; i < 3; i++ { + f.Coeffs = append(f.Coeffs, fr.NewElement(uint64(i))) + } + + gnark, err := f.SerializeGnark() + assert.Nil(t, err) + // The gob encoding via f.Serialize() will generate 318 bytes + // whereas gnark only 128 bytes + assert.Equal(t, 128, len(gnark)) + gob, err := f.Serialize() + assert.Nil(t, err) + assert.Equal(t, 318, len(gob)) + + // Verify the deserialization can get back original data + c, err := new(encoding.Frame).DeserializeGnark(gnark) + assert.Nil(t, err) + assert.True(t, f.Proof.Equal(&c.Proof)) + assert.Equal(t, len(f.Coeffs), len(c.Coeffs)) + for i := 0; i < len(f.Coeffs); i++ { + assert.True(t, f.Coeffs[i].Equal(&c.Coeffs[i])) + } +} From 0b04de255118dd8ceed99d126c9b5bf70254c36d Mon Sep 17 00:00:00 2001 From: Jian Xiao Date: Mon, 24 Jun 2024 22:40:55 +0000 Subject: [PATCH 2/4] more coeffs --- encoding/serialization_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/encoding/serialization_test.go b/encoding/serialization_test.go index 967bdb0e9..ffc3d9895 100644 --- a/encoding/serialization_test.go +++ b/encoding/serialization_test.go @@ -16,23 +16,25 @@ func TestSerDeserGnark(t *testing.T) { _, err = YCoord.SetString("9207254729396071334325696286939045899948985698134704137261649190717970615186") assert.NoError(t, err) + numCoeffs := 64 var f encoding.Frame f.Proof = encoding.Proof{ X: XCoord, Y: YCoord, } - for i := 0; i < 3; i++ { + for i := 0; i < numCoeffs; i++ { f.Coeffs = append(f.Coeffs, fr.NewElement(uint64(i))) } gnark, err := f.SerializeGnark() assert.Nil(t, err) - // The gob encoding via f.Serialize() will generate 318 bytes - // whereas gnark only 128 bytes - assert.Equal(t, 128, len(gnark)) + // The gnark encoding via f.Serialize() will generate less bytes + // than gob. + assert.Equal(t, 32*(1+numCoeffs), len(gnark)) gob, err := f.Serialize() assert.Nil(t, err) - assert.Equal(t, 318, len(gob)) + // 2080 with gnark v.s. 2574 with gob + assert.Equal(t, 2574, len(gob)) // Verify the deserialization can get back original data c, err := new(encoding.Frame).DeserializeGnark(gnark) From a78bed7976c3cf36b7e7b9c8fece8b1ad05998c5 Mon Sep 17 00:00:00 2001 From: Jian Xiao Date: Mon, 24 Jun 2024 22:58:29 +0000 Subject: [PATCH 3/4] memory --- encoding/serialization.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encoding/serialization.go b/encoding/serialization.go index bccc58a78..9978e476a 100644 --- a/encoding/serialization.go +++ b/encoding/serialization.go @@ -24,7 +24,7 @@ func (c *Frame) Deserialize(data []byte) (*Frame, error) { } func (c *Frame) SerializeGnark() ([]byte, error) { - coded := make([]byte, 0) + coded := make([]byte, 0, 32*(1+len(c.Coeffs))) // This is compressed format with just 32 bytes. proofBytes := c.Proof.Bytes() coded = append(coded, proofBytes[:]...) From 5191a210e6b694d5018a705f7105b998f448952e Mon Sep 17 00:00:00 2001 From: Jian Xiao Date: Tue, 25 Jun 2024 16:39:27 +0000 Subject: [PATCH 4/4] constant --- encoding/serialization.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/encoding/serialization.go b/encoding/serialization.go index 9978e476a..bd9daed58 100644 --- a/encoding/serialization.go +++ b/encoding/serialization.go @@ -24,7 +24,7 @@ func (c *Frame) Deserialize(data []byte) (*Frame, error) { } func (c *Frame) SerializeGnark() ([]byte, error) { - coded := make([]byte, 0, 32*(1+len(c.Coeffs))) + coded := make([]byte, 0, bn254.SizeOfG1AffineCompressed+BYTES_PER_SYMBOL*len(c.Coeffs)) // This is compressed format with just 32 bytes. proofBytes := c.Proof.Bytes() coded = append(coded, proofBytes[:]...) @@ -37,23 +37,23 @@ func (c *Frame) SerializeGnark() ([]byte, error) { func (c *Frame) DeserializeGnark(data []byte) (*Frame, error) { var f Frame buf := data - err := f.Proof.Unmarshal(buf[:32]) + err := f.Proof.Unmarshal(buf[:bn254.SizeOfG1AffineCompressed]) if err != nil { return nil, err } - buf = buf[32:] - if len(buf)%32 != 0 { + buf = buf[bn254.SizeOfG1AffineCompressed:] + if len(buf)%BYTES_PER_SYMBOL != 0 { return nil, errors.New("invalid chunk length") } - f.Coeffs = make([]Symbol, len(buf)/32) + f.Coeffs = make([]Symbol, len(buf)/BYTES_PER_SYMBOL) i := 0 for len(buf) > 0 { - if len(buf) < 32 { + if len(buf) < BYTES_PER_SYMBOL { return nil, errors.New("invalid chunk length") } - f.Coeffs[i].Unmarshal(buf[:32]) + f.Coeffs[i].Unmarshal(buf[:BYTES_PER_SYMBOL]) i++ - buf = buf[32:] + buf = buf[BYTES_PER_SYMBOL:] } return &f, nil }