diff --git a/pkg/encoding/encoder/decode.go b/pkg/encoding/encoder/decode.go index 7eefa707cc..2d13ccd655 100644 --- a/pkg/encoding/encoder/decode.go +++ b/pkg/encoding/encoder/decode.go @@ -15,12 +15,7 @@ import ( // maxInputSize is the upper bound of the original data size. This is needed because // the frames and indices don't encode the length of the original data. If maxInputSize // is smaller than the original input size, decoded data will be trimmed to fit the maxInputSize. -func (g *Encoder) Decode(frames []Frame, indices []uint64, maxInputSize uint64) ([]byte, error) { - numSys := GetNumSys(maxInputSize, g.ChunkLen) - - if uint64(len(frames)) < numSys { - return nil, errors.New("number of frame must be sufficient") - } +func (g *Encoder) Decode(frames []Frame, indices []uint64) ([]bls.Fr, error) { samples := make([]*bls.Fr, g.NumEvaluations()) // copy evals based on frame coeffs into samples @@ -70,6 +65,22 @@ func (g *Encoder) Decode(frames []Frame, indices []uint64, maxInputSize uint64) return nil, err } + return reconstructedPoly, nil +} + +func (g *Encoder) DecodeBytes(frames []Frame, indices []uint64, maxInputSize uint64) ([]byte, error) { + + numSys := GetNumSys(maxInputSize, g.ChunkLen) + + if uint64(len(frames)) < numSys { + return nil, errors.New("number of frame must be sufficient") + } + + reconstructedPoly, err := g.Decode(frames, indices) + if err != nil { + return nil, err + } + data := ToByteArray(reconstructedPoly, maxInputSize) return data, nil diff --git a/pkg/encoding/encoder/encode_test.go b/pkg/encoding/encoder/encode_test.go index c6e060f90a..3bda4d1732 100644 --- a/pkg/encoding/encoder/encode_test.go +++ b/pkg/encoding/encoder/encode_test.go @@ -25,7 +25,7 @@ func TestEncodeDecode_InvertsWhenSamplingAllFrames(t *testing.T) { // sample some frames samples, indices := sampleFrames(frames, uint64(len(frames))) - data, err := enc.Decode(samples, indices, uint64(len(GETTYSBURG_ADDRESS_BYTES))) + data, err := enc.DecodeBytes(samples, indices, uint64(len(GETTYSBURG_ADDRESS_BYTES))) require.Nil(t, err) require.NotNil(t, data) @@ -47,7 +47,7 @@ func TestEncodeDecode_InvertsWhenSamplingMissingFrame(t *testing.T) { // sample some frames samples, indices := sampleFrames(frames, uint64(len(frames)-1)) - data, err := enc.Decode(samples, indices, uint64(len(GETTYSBURG_ADDRESS_BYTES))) + data, err := enc.DecodeBytes(samples, indices, uint64(len(GETTYSBURG_ADDRESS_BYTES))) require.Nil(t, err) require.NotNil(t, data) @@ -71,7 +71,7 @@ func TestEncodeDecode_ErrorsWhenNotEnoughSampledFrames(t *testing.T) { // sample some frames samples, indices := sampleFrames(frames, uint64(len(frames)-2)) - data, err := enc.Decode(samples, indices, uint64(len(GETTYSBURG_ADDRESS_BYTES))) + data, err := enc.DecodeBytes(samples, indices, uint64(len(GETTYSBURG_ADDRESS_BYTES))) require.Nil(t, data) require.NotNil(t, err) diff --git a/pkg/encoding/encoder/encoder_fuzz_test.go b/pkg/encoding/encoder/encoder_fuzz_test.go index 3ee377c432..cfff319330 100644 --- a/pkg/encoding/encoder/encoder_fuzz_test.go +++ b/pkg/encoding/encoder/encoder_fuzz_test.go @@ -27,7 +27,7 @@ func FuzzOnlySystematic(f *testing.F) { //sample the correct systematic frames samples, indices := sampleFrames(frames, uint64(len(frames))) - data, err := enc.Decode(samples, indices, uint64(len(input))) + data, err := enc.DecodeBytes(samples, indices, uint64(len(input))) if err != nil { t.Errorf("Error Decoding:\n Data:\n %q \n Err: %q", input, err) } diff --git a/pkg/encoding/kzgEncoder/decode.go b/pkg/encoding/kzgEncoder/decode.go index 7b88aba90e..5a5df7367b 100644 --- a/pkg/encoding/kzgEncoder/decode.go +++ b/pkg/encoding/kzgEncoder/decode.go @@ -10,5 +10,5 @@ func (g *KzgEncoder) Decode(frames []Frame, indices []uint64, maxInputSize uint6 rsFrames[ind] = rs.Frame{Coeffs: frame.Coeffs} } - return g.Encoder.Decode(rsFrames, indices, maxInputSize) + return g.Encoder.DecodeBytes(rsFrames, indices, maxInputSize) } diff --git a/pkg/encoding/kzgEncoder/shiftedencoder_test.go b/pkg/encoding/kzgEncoder/shiftedencoder_test.go index 7d383b70a1..bea097975f 100644 --- a/pkg/encoding/kzgEncoder/shiftedencoder_test.go +++ b/pkg/encoding/kzgEncoder/shiftedencoder_test.go @@ -20,7 +20,7 @@ func TestShiftedEncoding(t *testing.T) { group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) - blobSize := 256 + blobSize := 32 blob := make([]byte, blobSize*31) _, err := rand.Read(blob) assert.NoError(t, err) @@ -29,15 +29,13 @@ func TestShiftedEncoding(t *testing.T) { params := rs.EncodingParams{ NumChunks: 4, - ChunkLen: 128, + ChunkLen: 16, } enc, err := group.NewKzgEncoder(params) if err != nil { t.Errorf("Error making rs: %q", err) } - origCommit := enc.Commit(input) - n := uint8(math.Log2(float64(enc.NumEvaluations()))) + 1 fs := kzg.NewFFTSettings(n) @@ -51,33 +49,52 @@ func TestShiftedEncoding(t *testing.T) { } //encode the data - commit, _, frames, indices, err := enc.Encode(input) + _, _, frames, indices_, err := enc.Encode(input) if err != nil { t.Errorf("Error Encoding:\n Data:\n %q \n Err: %q", input, err) } - shiftedCommit := bls.G1Point{} - bls.MulG1(&shiftedCommit, origCommit, factor) + fmt.Println("indices_", indices_) - assert.True(t, bls.EqualG1(commit, &shiftedCommit), "commitment mismatch") + // for _, frame := range frames { + // assert.NotEqual(t, len(frame.Coeffs), 0) + // } - for _, frame := range frames { - assert.NotEqual(t, len(frame.Coeffs), 0) - } + // for i := 0; i < len(frames); i++ { + // f := frames[i] + // j := indices[i] + + // q, err := rs.GetLeadingCosetIndex(uint64(i), params.NumChunks) + // assert.Nil(t, err) + + // assert.Equal(t, j, q, "leading coset inconsistency") - for i := 0; i < len(frames); i++ { - f := frames[i] - j := indices[i] + // fmt.Printf("frame %v leading coset %v\n", i, j) + // lc := enc.Fs.ExpandedRootsOfUnity[uint64(q)] - q, err := rs.GetLeadingCosetIndex(uint64(i), params.NumChunks) - assert.Nil(t, err) + // assert.True(t, f.Verify(enc.Ks, &shiftedCommit, &lc), "Proof %v failed\n", i) + // } - assert.Equal(t, j, q, "leading coset inconsistency") + samples_, indices := sampleFrames(frames, uint64(len(frames))) - fmt.Printf("frame %v leading coset %v\n", i, j) - lc := enc.Fs.ExpandedRootsOfUnity[uint64(q)] + samples := make([]rs.Frame, len(frames)) + for i, frame := range samples_ { + samples[i] = rs.Frame{ + Coeffs: frame.Coeffs, + } + } + + fmt.Println("len(samples)", len(samples), "len(frames)", len(frames), "len(indices)", len(indices)) + + recoveredCoeffs, err := enc.Encoder.Decode(samples, indices) + assert.NoError(t, err) - assert.True(t, f.Verify(enc.Ks, &shiftedCommit, &lc), "Proof %v failed\n", i) + notEqual := make([]int, 0) + for i := 0; i < len(input); i++ { + if !bls.EqualFr(&input[i], &recoveredCoeffs[i]) { + notEqual = append(notEqual, i) + } } + assert.Equal(t, []int{}, notEqual) } diff --git a/pkg/encoding/mixedencoder/allocation.go b/pkg/encoding/mixedencoder/allocation.go index 75b4c8c523..dabfa4c8b2 100644 --- a/pkg/encoding/mixedencoder/allocation.go +++ b/pkg/encoding/mixedencoder/allocation.go @@ -56,7 +56,7 @@ func AddOffsets(allocations []*Allocation) error { // Sort allocations by number of evaluations sorted := make([]*Allocation, len(allocations)) copy(sorted, allocations) - sort.Slice(sorted, func(i, j int) bool { + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].NumEvaluations > sorted[j].NumEvaluations }) diff --git a/pkg/encoding/mixedencoder/encode.go b/pkg/encoding/mixedencoder/encode.go index 169b0a8cea..96175dde13 100644 --- a/pkg/encoding/mixedencoder/encode.go +++ b/pkg/encoding/mixedencoder/encode.go @@ -66,6 +66,7 @@ func (e *MixedEncoder) Encode(input []byte, params []rs.EncodingParams) (*bls.G1 // Condition the input shiftedPolyCoeffs := ShiftPoly(coeffs, allocations[ind].Offset) + fmt.Println("Offset", allocations[ind].Offset, "RootIndex", allocations[ind].RootIndex) // Encode shiftedCommit, _, frames, indices, err := encoder.Encode(shiftedPolyCoeffs) diff --git a/pkg/encoding/mixedencoder/encode_test.go b/pkg/encoding/mixedencoder/encode_test.go index 41d5488c86..03537f883c 100644 --- a/pkg/encoding/mixedencoder/encode_test.go +++ b/pkg/encoding/mixedencoder/encode_test.go @@ -8,8 +8,9 @@ import ( rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" kzgrs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" "github.com/Layr-Labs/eigenda/pkg/encoding/mixedencoder" - "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" "github.com/stretchr/testify/assert" + + "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) var ( @@ -78,8 +79,30 @@ func TestMixedEncoding(t *testing.T) { // } // Decode - inputs := make([]*mixedencoder.MixedDecoderInput, len(outputs)) - for i, output := range outputs { + inputs := sampleInputs(outputs) + + // for _, input := range inputs { + // testDecode(t, blob, input) + // } + + numEvaluations := 0 + for _, input := range inputs { + numEvaluations += input.Allocation.NumEvaluations + } + numEvaluations = int(rs.NextPowerOf2(uint64(numEvaluations))) + + decoded, err := encoder.Decode(numEvaluations, len(blob), inputs) + + assert.NoError(t, err) + assert.Equal(t, string(blob), string(decoded)) + +} + +func sampleInputs(outputs []*mixedencoder.MixedEncodingOutput) []*mixedencoder.MixedDecoderInput { + + inputs := make([]*mixedencoder.MixedDecoderInput, 0) + + for _, output := range outputs { frames := make([]rs.Frame, len(output.Frames)) for j, frame := range output.Frames { @@ -88,31 +111,48 @@ func TestMixedEncoding(t *testing.T) { } } - inputs[i] = &mixedencoder.MixedDecoderInput{ + indices := make([]uint32, len(output.Indices)) + for j := range output.Indices { + indices[j] = uint32(j) + } + + inputs = append(inputs, &mixedencoder.MixedDecoderInput{ EncodingParams: output.Param, Allocation: output.Allocation, Frames: frames, - Indices: output.Indices, - } + Indices: indices, + }) } - numEvaluations := 0 - for _, input := range inputs { - numEvaluations += input.Allocation.NumEvaluations - } - numEvaluations = int(rs.NextPowerOf2(uint64(numEvaluations))) + return inputs - tInputs := []*mixedencoder.MixedDecoderInput{ - inputs[1], inputs[2], +} + +func testDecode(t *testing.T, blob []byte, input *mixedencoder.MixedDecoderInput) { + + enc, err := rs.NewEncoder(input.EncodingParams, false) + if err != nil { + t.Fatal(err) } - decoded, err := encoder.Decode(numEvaluations, len(blob), tInputs) + indices := make([]uint64, len(input.Indices)) + for i := range input.Indices { + indices[i] = uint64(i) + } + recoveredCoeffs, err := enc.Decode(input.Frames, indices) assert.NoError(t, err) - assert.Equal(t, string(blob), string(decoded)) - fmt.Println("Offset", tInputs[0].Allocation.RootIndex) - fmt.Println(string(decoded)) + origCoeffs := rs.ToFrArray(blob) + shiftedCoeffs := mixedencoder.ShiftPoly(origCoeffs, input.Allocation.Offset) + + notEqual := make([]int, 0) + for i := 0; i < len(shiftedCoeffs); i++ { + if !bn254.EqualFr(&shiftedCoeffs[i], &recoveredCoeffs[i]) { + notEqual = append(notEqual, i) + } + } + assert.Equal(t, []int{}, notEqual) }