From d2bfac0856ab69c8830a3101a2e3ae74da9a6727 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 3 Feb 2024 05:48:11 +0000 Subject: [PATCH] allow non encoder to not load g2 list --- clients/tests/retrieval_client_test.go | 2 +- core/encoding/encoder.go | 4 +- core/encoding/encoder_test.go | 2 +- core/test/core_test.go | 18 ++++----- disperser/batcher/batcher_test.go | 12 +++--- disperser/cmd/encoder/encoder.go | 2 +- disperser/encoder/server_test.go | 12 +++--- inabox/tests/integration_suite_test.go | 2 +- node/grpc/server_test.go | 12 +++--- node/node.go | 2 +- .../kzgEncoder/batchCommitEquivalence_test.go | 2 +- pkg/encoding/kzgEncoder/degree_test.go | 2 +- pkg/encoding/kzgEncoder/encoder.go | 37 +++++++++++-------- pkg/encoding/kzgEncoder/encoder_fuzz_test.go | 2 +- pkg/encoding/kzgEncoder/frame_test.go | 4 +- pkg/encoding/kzgEncoder/multiframe_test.go | 2 +- pkg/encoding/kzgEncoder/multiprover_test.go | 2 +- pkg/encoding/kzgEncoder/zero_padding_test.go | 2 +- pkg/encoding/main.go | 4 +- pkg/kzg/srs.go | 9 +---- retriever/cmd/main.go | 2 +- retriever/server_test.go | 14 +++---- test/integration_test.go | 13 ++++--- 23 files changed, 83 insertions(+), 80 deletions(-) diff --git a/clients/tests/retrieval_client_test.go b/clients/tests/retrieval_client_test.go index ab016ecd05..d275870b4e 100644 --- a/clients/tests/retrieval_client_test.go +++ b/clients/tests/retrieval_client_test.go @@ -34,7 +34,7 @@ func makeTestEncoder() (core.Encoder, error) { NumWorker: uint64(runtime.GOMAXPROCS(0)), } - kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config) + kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config, true) if err != nil { return nil, err } diff --git a/core/encoding/encoder.go b/core/encoding/encoder.go index 4029427f27..4583775c2d 100644 --- a/core/encoding/encoder.go +++ b/core/encoding/encoder.go @@ -26,8 +26,8 @@ type Encoder struct { var _ core.Encoder = &Encoder{} -func NewEncoder(config EncoderConfig) (*Encoder, error) { - kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(&config.KzgConfig) +func NewEncoder(config EncoderConfig, isEncoder bool) (*Encoder, error) { + kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(&config.KzgConfig, isEncoder) if err != nil { return nil, err } diff --git a/core/encoding/encoder_test.go b/core/encoding/encoder_test.go index dea882b12d..ee4d7cb4bb 100644 --- a/core/encoding/encoder_test.go +++ b/core/encoding/encoder_test.go @@ -37,7 +37,7 @@ func makeTestEncoder() (core.Encoder, error) { NumWorker: uint64(runtime.GOMAXPROCS(0)), } - return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}) + return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true) } func TestEncoder(t *testing.T) { diff --git a/core/test/core_test.go b/core/test/core_test.go index 503edcaf90..380c45633d 100644 --- a/core/test/core_test.go +++ b/core/test/core_test.go @@ -41,15 +41,15 @@ func setup(m *testing.M) { // makeTestEncoder makes an encoder currently using the only supported backend. func makeTestEncoder() (core.Encoder, error) { config := kzgEncoder.KzgConfig{ - G1Path: "../../inabox/resources/kzg/g1.point", - G2Path: "../../inabox/resources/kzg/g2.point", - CacheDir: "../../inabox/resources/kzg/SRSTables", - SRSOrder: 3000, + G1Path: "../../inabox/resources/kzg/g1.point", + G2Path: "../../inabox/resources/kzg/g2.point", + CacheDir: "../../inabox/resources/kzg/SRSTables", + SRSOrder: 3000, SRSNumberToLoad: 3000, - NumWorker: uint64(runtime.GOMAXPROCS(0)), + NumWorker: uint64(runtime.GOMAXPROCS(0)), } - return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}) + return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true) } @@ -125,10 +125,10 @@ func prepareBatch(t *testing.T, cst core.IndexedChainState, blobs []core.Blob, q blobHeader := &core.BlobHeader{ BlobCommitments: core.BlobCommitments{ - Commitment: commitments.Commitment, + Commitment: commitments.Commitment, LengthCommitment: commitments.LengthCommitment, - LengthProof: commitments.LengthProof, - Length: commitments.Length, + LengthProof: commitments.LengthProof, + Length: commitments.Length, }, QuorumInfos: []*core.BlobQuorumInfo{quorumHeader}, } diff --git a/disperser/batcher/batcher_test.go b/disperser/batcher/batcher_test.go index 3b14456d67..669fc7147e 100644 --- a/disperser/batcher/batcher_test.go +++ b/disperser/batcher/batcher_test.go @@ -43,15 +43,15 @@ type batcherComponents struct { // makeTestEncoder makes an encoder currently using the only supported backend. func makeTestEncoder() (core.Encoder, error) { config := kzgEncoder.KzgConfig{ - G1Path: "../../inabox/resources/kzg/g1.point", - G2Path: "../../inabox/resources/kzg/g2.point", - CacheDir: "../../inabox/resources/kzg/SRSTables", - SRSOrder: 3000, + G1Path: "../../inabox/resources/kzg/g1.point", + G2Path: "../../inabox/resources/kzg/g2.point", + CacheDir: "../../inabox/resources/kzg/SRSTables", + SRSOrder: 3000, SRSNumberToLoad: 3000, - NumWorker: uint64(runtime.GOMAXPROCS(0)), + NumWorker: uint64(runtime.GOMAXPROCS(0)), } - return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}) + return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true) } func makeTestBlob(securityParams []*core.SecurityParam) core.Blob { diff --git a/disperser/cmd/encoder/encoder.go b/disperser/cmd/encoder/encoder.go index 4e8e7cac3d..81fb87e3e3 100644 --- a/disperser/cmd/encoder/encoder.go +++ b/disperser/cmd/encoder/encoder.go @@ -15,7 +15,7 @@ type EncoderGRPCServer struct { func NewEncoderGRPCServer(config Config, logger common.Logger) (*EncoderGRPCServer, error) { - coreEncoder, err := encoding.NewEncoder(config.EncoderConfig) + coreEncoder, err := encoding.NewEncoder(config.EncoderConfig, true) if err != nil { return nil, fmt.Errorf("failed to create encoder: %w", err) } diff --git a/disperser/encoder/server_test.go b/disperser/encoder/server_test.go index 591ec9df36..a28b828be3 100644 --- a/disperser/encoder/server_test.go +++ b/disperser/encoder/server_test.go @@ -31,17 +31,17 @@ var logger = &cmock.Logger{} func makeTestEncoder() (*encoding.Encoder, ServerConfig) { kzgConfig := kzgEncoder.KzgConfig{ - G1Path: "../../inabox/resources/kzg/g1.point", - G2Path: "../../inabox/resources/kzg/g2.point", - CacheDir: "../../inabox/resources/kzg/SRSTables", - SRSOrder: 3000, + G1Path: "../../inabox/resources/kzg/g1.point", + G2Path: "../../inabox/resources/kzg/g2.point", + CacheDir: "../../inabox/resources/kzg/SRSTables", + SRSOrder: 3000, SRSNumberToLoad: 3000, - NumWorker: uint64(runtime.GOMAXPROCS(0)), + NumWorker: uint64(runtime.GOMAXPROCS(0)), } encodingConfig := encoding.EncoderConfig{KzgConfig: kzgConfig} - encoder, _ := encoding.NewEncoder(encodingConfig) + encoder, _ := encoding.NewEncoder(encodingConfig, true) encoderServerConfig := ServerConfig{ GrpcPort: "3000", MaxConcurrentRequests: 16, diff --git a/inabox/tests/integration_suite_test.go b/inabox/tests/integration_suite_test.go index 1903c6e41a..a169b39a5b 100644 --- a/inabox/tests/integration_suite_test.go +++ b/inabox/tests/integration_suite_test.go @@ -167,7 +167,7 @@ func setupRetrievalClient(testConfig *deploy.Config) error { Verbose: true, PreloadEncoder: false, }, - }) + }, false) if err != nil { return err } diff --git a/node/grpc/server_test.go b/node/grpc/server_test.go index da003c70d3..60c9962d41 100644 --- a/node/grpc/server_test.go +++ b/node/grpc/server_test.go @@ -44,15 +44,15 @@ func TestMain(m *testing.M) { // makeTestEncoder makes an encoder currently using the only supported backend. func makeTestEncoder() (core.Encoder, error) { config := kzgEncoder.KzgConfig{ - G1Path: "../../inabox/resources/kzg/g1.point.300000", - G2Path: "../../inabox/resources/kzg/g2.point.300000", - CacheDir: "../../inabox/resources/kzg/SRSTables", - SRSOrder: 300000, + G1Path: "../../inabox/resources/kzg/g1.point.300000", + G2Path: "../../inabox/resources/kzg/g2.point.300000", + CacheDir: "../../inabox/resources/kzg/SRSTables", + SRSOrder: 300000, SRSNumberToLoad: 300000, - NumWorker: uint64(runtime.GOMAXPROCS(0)), + NumWorker: uint64(runtime.GOMAXPROCS(0)), } - return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}) + return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true) } func newTestServer(t *testing.T, mockValidator bool) *grpc.Server { diff --git a/node/node.go b/node/node.go index 3681010c89..3a1f63c5ee 100644 --- a/node/node.go +++ b/node/node.go @@ -113,7 +113,7 @@ func NewNode(config *Config, pubIPProvider pubip.Provider, logger common.Logger) nodeApi := nodeapi.NewNodeApi(AppName, SemVer, ":"+config.NodeApiPort, logger) // Make validator - enc, err := encoding.NewEncoder(config.EncoderConfig) + enc, err := encoding.NewEncoder(config.EncoderConfig, false) if err != nil { return nil, err } diff --git a/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go b/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go index 831fdfb1a9..c874a5d41d 100644 --- a/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go +++ b/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go @@ -14,7 +14,7 @@ func TestBatchEquivalence(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) require.Nil(t, err) diff --git a/pkg/encoding/kzgEncoder/degree_test.go b/pkg/encoding/kzgEncoder/degree_test.go index 5e172e4d75..3b263ae200 100644 --- a/pkg/encoding/kzgEncoder/degree_test.go +++ b/pkg/encoding/kzgEncoder/degree_test.go @@ -13,7 +13,7 @@ func TestLengthProof(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) require.Nil(t, err) diff --git a/pkg/encoding/kzgEncoder/encoder.go b/pkg/encoding/kzgEncoder/encoder.go index ca90f76634..295050705a 100644 --- a/pkg/encoding/kzgEncoder/encoder.go +++ b/pkg/encoding/kzgEncoder/encoder.go @@ -55,7 +55,7 @@ type KzgEncoder struct { FFTPointsT [][]bls.G1Point // transpose of FFTPoints } -func NewKzgEncoderGroup(config *KzgConfig) (*KzgEncoderGroup, error) { +func NewKzgEncoderGroup(config *KzgConfig, isEncoder bool) (*KzgEncoderGroup, error) { if config.SRSNumberToLoad > config.SRSOrder { return nil, errors.New("SRSOrder is less than srsNumberToLoad") @@ -67,10 +67,27 @@ func NewKzgEncoderGroup(config *KzgConfig) (*KzgEncoderGroup, error) { log.Println("failed to read G1 points", err) return nil, err } - s2, err := utils.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker) - if err != nil { - log.Println("failed to read G2 points", err) - return nil, err + + s2 := make([]bls.G2Point, 0) + g2Trailing := make([]bls.G2Point, 0) + + // PreloadEncoder is by default not used by operator node, PreloadEncoder + if isEncoder { + s2, err = utils.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker) + if err != nil { + log.Println("failed to read G2 points", err) + return nil, err + } + + g2Trailing, err = utils.ReadG2PointSection( + config.G2Path, + config.SRSOrder-config.SRSNumberToLoad, + config.SRSOrder, // last exclusive + config.NumWorker, + ) + if err != nil { + return nil, err + } } srs, err := kzg.NewSrs(s1, s2) @@ -79,16 +96,6 @@ func NewKzgEncoderGroup(config *KzgConfig) (*KzgEncoderGroup, error) { return nil, err } - g2Trailing, err := utils.ReadG2PointSection( - config.G2Path, - config.SRSOrder-config.SRSNumberToLoad, - config.SRSOrder, // last exclusive - config.NumWorker, - ) - if err != nil { - return nil, err - } - fmt.Println("numthread", runtime.GOMAXPROCS(0)) encoderGroup := &KzgEncoderGroup{ diff --git a/pkg/encoding/kzgEncoder/encoder_fuzz_test.go b/pkg/encoding/kzgEncoder/encoder_fuzz_test.go index d14ff4abbd..9551b0c526 100644 --- a/pkg/encoding/kzgEncoder/encoder_fuzz_test.go +++ b/pkg/encoding/kzgEncoder/encoder_fuzz_test.go @@ -13,7 +13,7 @@ func FuzzOnlySystematic(f *testing.F) { f.Add(GETTYSBURG_ADDRESS_BYTES) f.Fuzz(func(t *testing.T, input []byte) { - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(10, 3, uint64(len(input))) enc, err := group.NewKzgEncoder(params) diff --git a/pkg/encoding/kzgEncoder/frame_test.go b/pkg/encoding/kzgEncoder/frame_test.go index 8d2069f7fd..2ffc008d68 100644 --- a/pkg/encoding/kzgEncoder/frame_test.go +++ b/pkg/encoding/kzgEncoder/frame_test.go @@ -16,7 +16,7 @@ func TestEncodeDecodeFrame_AreInverses(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) @@ -44,7 +44,7 @@ func TestVerify(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) diff --git a/pkg/encoding/kzgEncoder/multiframe_test.go b/pkg/encoding/kzgEncoder/multiframe_test.go index 3d3cc70386..3f72bab728 100644 --- a/pkg/encoding/kzgEncoder/multiframe_test.go +++ b/pkg/encoding/kzgEncoder/multiframe_test.go @@ -13,7 +13,7 @@ func TestUniversalVerify(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) require.Nil(t, err) diff --git a/pkg/encoding/kzgEncoder/multiprover_test.go b/pkg/encoding/kzgEncoder/multiprover_test.go index 9def8d6c1e..c80c680ea1 100644 --- a/pkg/encoding/kzgEncoder/multiprover_test.go +++ b/pkg/encoding/kzgEncoder/multiprover_test.go @@ -14,7 +14,7 @@ func TestProveAllCosetThreads(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) diff --git a/pkg/encoding/kzgEncoder/zero_padding_test.go b/pkg/encoding/kzgEncoder/zero_padding_test.go index 8c6ca041f5..af450e2fe9 100644 --- a/pkg/encoding/kzgEncoder/zero_padding_test.go +++ b/pkg/encoding/kzgEncoder/zero_padding_test.go @@ -13,7 +13,7 @@ func TestProveZeroPadding(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) diff --git a/pkg/encoding/main.go b/pkg/encoding/main.go index 9366f0d779..9889d31ca2 100644 --- a/pkg/encoding/main.go +++ b/pkg/encoding/main.go @@ -34,7 +34,7 @@ func readpoints() { } // create encoding object - kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) fmt.Println("there are ", len(kzgGroup.Srs.G1), "points") for i := 0; i < len(kzgGroup.Srs.G1); i++ { fmt.Printf("%v %v\n", i, string(kzgGroup.Srs.G1[i].MarshalText())) @@ -66,7 +66,7 @@ func TestKzgRs() { } // create encoding object - kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig) + kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) params := rs.EncodingParams{NumChunks: 200, ChunkLen: 180} enc, _ := kzgGroup.NewKzgEncoder(params) diff --git a/pkg/kzg/srs.go b/pkg/kzg/srs.go index 7a3bb5882a..1b6ffc3537 100644 --- a/pkg/kzg/srs.go +++ b/pkg/kzg/srs.go @@ -24,11 +24,9 @@ package kzg -import ( - "errors" +import bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" - bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" -) +//bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" type SRS struct { @@ -40,9 +38,6 @@ type SRS struct { func NewSrs(G1 []bls.G1Point, G2 []bls.G2Point) (*SRS, error) { - if len(G1) != len(G2) { - return nil, errors.New("secret list lengths don't match") - } return &SRS{ G1: G1, G2: G2, diff --git a/retriever/cmd/main.go b/retriever/cmd/main.go index c7fbfd3fd5..5318816296 100644 --- a/retriever/cmd/main.go +++ b/retriever/cmd/main.go @@ -75,7 +75,7 @@ func RetrieverMain(ctx *cli.Context) error { } nodeClient := clients.NewNodeClient(config.Timeout) - encoder, err := encoding.NewEncoder(config.EncoderConfig) + encoder, err := encoding.NewEncoder(config.EncoderConfig, false) if err != nil { log.Fatalln("could not start tcp listener", err) } diff --git a/retriever/server_test.go b/retriever/server_test.go index 5dc44a6736..2c0c25ab36 100644 --- a/retriever/server_test.go +++ b/retriever/server_test.go @@ -32,15 +32,15 @@ var ( func makeTestEncoder() (core.Encoder, error) { config := &kzgEncoder.KzgConfig{ - G1Path: "../inabox/resources/kzg/g1.point", - G2Path: "../inabox/resources/kzg/g2.point", - CacheDir: "../inabox/resources/kzg/SRSTables", - SRSOrder: 3000, - SRSNumberToLoad: 3000, - NumWorker: uint64(runtime.GOMAXPROCS(0)), + G1Path: "../inabox/resources/kzg/g1.point", + G2Path: "../inabox/resources/kzg/g2.point", + CacheDir: "../inabox/resources/kzg/SRSTables", + SRSOrder: 3000, + SRSNumberToLoad: 3000, + NumWorker: uint64(runtime.GOMAXPROCS(0)), } - kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config) + kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config, false) if err != nil { return nil, err } diff --git a/test/integration_test.go b/test/integration_test.go index 6a2aff84b1..e37f912f9f 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -76,16 +76,17 @@ func init() { // makeTestEncoder makes an encoder currently using the only supported backend. func mustMakeTestEncoder() core.Encoder { config := kzgEncoder.KzgConfig{ - G1Path: "../inabox/resources/kzg/g1.point", - G2Path: "../inabox/resources/kzg/g2.point", - CacheDir: "../inabox/resources/kzg/SRSTables", - SRSOrder: 3000, - SRSNumberToLoad: 3000, - NumWorker: uint64(runtime.GOMAXPROCS(0)), + G1Path: "../inabox/resources/kzg/g1.point", + G2Path: "../inabox/resources/kzg/g2.point", + CacheDir: "../inabox/resources/kzg/SRSTables", + SRSOrder: 3000, + SRSNumberToLoad: 3000, + NumWorker: uint64(runtime.GOMAXPROCS(0)), } encoder, err := encoding.NewEncoder( encoding.EncoderConfig{KzgConfig: config}, + true, ) if err != nil { log.Fatalln("failed to initialize new encoder")