Load only 28 g2point for opr node (Layr-Labs#246)

Co-authored-by: Ubuntu <[email protected]>

core/encoding/cli.go

@@ -18,6 +18,7 @@ const (
 	PreloadEncoderFlagName    = "kzg.preload-encoder"
 	CacheEncodedBlobsFlagName = "cache-encoded-blobs"
 	SRSLoadingNumberFlagName  = "kzg.srs-load"
+	G2PowerOf2PathFlagName    = "kzg.g2-power-of-2-path"
 )
 
 func CLIFlags(envPrefix string) []cli.Flag {
@@ -30,8 +31,8 @@ func CLIFlags(envPrefix string) []cli.Flag {
 		},
 		cli.StringFlag{
 			Name:     G2PathFlagName,
-			Usage:    "Path to G2 SRS",
-			Required: true,
+			Usage:    "Path to G2 SRS. Either this flag or G2_POWER_OF_2_PATH needs to be specified. For operator node, if both are specified, the node uses G2_POWER_OF_2_PATH first, if failed then tries to G2_PATH",
+			Required: false,
 			EnvVar:   common.PrefixEnvVar(envPrefix, "G2_PATH"),
 		},
 		cli.StringFlag{
@@ -77,6 +78,12 @@ func CLIFlags(envPrefix string) []cli.Flag {
 			Required: false,
 			EnvVar:   common.PrefixEnvVar(envPrefix, "PRELOAD_ENCODER"),
 		},
+		cli.StringFlag{
+			Name:     G2PowerOf2PathFlagName,
+			Usage:    "Path to G2 SRS points that are on power of 2. Either this flag or G2_PATH needs to be specified. For operator node, if both are specified, the node uses G2_POWER_OF_2_PATH first, if failed then tries to G2_PATH",
+			Required: false,
+			EnvVar:   common.PrefixEnvVar(envPrefix, "G2_POWER_OF_2_PATH"),
+		},
 	}
 }
 
@@ -90,6 +97,8 @@ func ReadCLIConfig(ctx *cli.Context) EncoderConfig {
 	cfg.NumWorker = ctx.GlobalUint64(NumWorkerFlagName)
 	cfg.Verbose = ctx.GlobalBool(VerboseFlagName)
 	cfg.PreloadEncoder = ctx.GlobalBool(PreloadEncoderFlagName)
+	cfg.G2PowerOf2Path = ctx.GlobalString(G2PowerOf2PathFlagName)
+
 	return EncoderConfig{
 		KzgConfig:         cfg,
 		CacheEncodedBlobs: ctx.GlobalBoolT(CacheEncodedBlobsFlagName),

inabox/deploy/config.go

@@ -298,6 +298,7 @@ func (env *Config) generateOperatorVars(ind int, name, key, churnerUrl, logPath,
 		NODE_EXPIRATION_POLL_INTERVAL:    "10",
 		NODE_G1_PATH:                     "",
 		NODE_G2_PATH:                     "",
+		NODE_G2_POWER_OF_2_PATH:          "",
 		NODE_CACHE_PATH:                  "",
 		NODE_SRS_ORDER:                   "",
 		NODE_SRS_LOAD:                    "",
@@ -315,7 +316,7 @@ func (env *Config) generateOperatorVars(ind int, name, key, churnerUrl, logPath,
 	}
 
 	env.applyDefaults(&v, "NODE", "opr", ind)
-
+	v.NODE_G2_PATH = ""
 	return v
 
 }
@@ -335,6 +336,7 @@ func (env *Config) generateRetrieverVars(ind int, key string, graphUrl, logPath,
 
 		RETRIEVER_G1_PATH:             "",
 		RETRIEVER_G2_PATH:             "",
+		RETRIEVER_G2_POWER_OF_2_PATH:  "",
 		RETRIEVER_CACHE_PATH:          "",
 		RETRIEVER_SRS_ORDER:           "",
 		RETRIEVER_SRS_LOAD:            "",
@@ -349,6 +351,8 @@ func (env *Config) generateRetrieverVars(ind int, key string, graphUrl, logPath,
 		RETRIEVER_LOG_PATH:       logPath,
 	}
 
+	v.RETRIEVER_G2_PATH = ""
+
 	env.applyDefaults(&v, "RETRIEVER", "retriever", ind)
 
 	return v

inabox/deploy/env_vars.go

@@ -264,6 +264,8 @@ type OperatorVars struct {
 
 	NODE_G2_PATH string
 
+	NODE_G2_POWER_OF_2_PATH string
+
 	NODE_CACHE_PATH string
 
 	NODE_SRS_ORDER string
@@ -325,6 +327,8 @@ type RetrieverVars struct {
 
 	RETRIEVER_G2_PATH string
 
+	RETRIEVER_G2_POWER_OF_2_PATH string
+
 	RETRIEVER_CACHE_PATH string
 
 	RETRIEVER_SRS_ORDER string

inabox/templates/testconfig-anvil-docker.yaml

@@ -39,6 +39,7 @@ services:
       CHAIN_ID: 40525
       G1_PATH: resources/kzg/g1.point
       G2_PATH: resources/kzg/g2.point
+      G2_POWER_OF_2_PATH: resources/kzg/g2.point.powerOf2
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
       SRS_LOAD: 2900

inabox/templates/testconfig-anvil-nochurner.yaml

@@ -38,6 +38,7 @@ services:
       CHAIN_ID: 40525
       G1_PATH: resources/kzg/g1.point
       G2_PATH: resources/kzg/g2.point
+      G2_POWER_OF_2_PATH: resources/kzg/g2.point.powerOf2
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
       SRS_LOAD: 2900

inabox/templates/testconfig-anvil-nograph.yaml

@@ -38,6 +38,7 @@ services:
       CHAIN_ID: 40525
       G1_PATH: resources/kzg/g1.point
       G2_PATH: resources/kzg/g2.point
+      G2_POWER_OF_2_PATH: resources/kzg/g2.point.powerOf2
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
       SRS_LOAD: 2900

inabox/templates/testconfig-anvil.yaml

@@ -39,6 +39,7 @@ services:
       CHAIN_ID: 40525
       G1_PATH: resources/kzg/g1.point
       G2_PATH: resources/kzg/g2.point
+      G2_POWER_OF_2_PATH: resources/kzg/g2.point.powerOf2
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
       SRS_LOAD: 2900

inabox/templates/testconfig-docker-anvil.yaml

@@ -43,6 +43,7 @@ services:
       CHAIN_ID: 40525
       G1_PATH: /data/kzg/g1.point
       G2_PATH: /data/kzg/g2.point
+      G2_POWER_OF_2_PATH: resources/kzg/g2.point.powerOf2
       CACHE_PATH: /data/kzg/SRSTables
       SRS_ORDER: 3000
       SRS_LOAD: 2000

inabox/templates/testconfig-geth.yaml

@@ -30,6 +30,7 @@ services:
       CHAIN_ID: 40525
       G1_PATH: resources/kzg/g1.point
       G2_PATH: resources/kzg/g2.point
+      G2_POWER_OF_2_PATH: resources/kzg/g2.point.powerOf2
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
       SRS_LOAD: 3000

inabox/tests/integration_suite_test.go

@@ -160,6 +160,7 @@ func setupRetrievalClient(testConfig *deploy.Config) error {
 		KzgConfig: kzgEncoder.KzgConfig{
 			G1Path:          testConfig.Retriever.RETRIEVER_G1_PATH,
 			G2Path:          testConfig.Retriever.RETRIEVER_G2_PATH,
+			G2PowerOf2Path:  testConfig.Retriever.RETRIEVER_G2_POWER_OF_2_PATH,
 			CacheDir:        testConfig.Retriever.RETRIEVER_CACHE_PATH,
 			NumWorker:       1,
 			SRSOrder:        uint64(srsOrder),

pkg/encoding/kzgEncoder/encoder.go

@@ -23,6 +23,8 @@ import (
 type KzgConfig struct {
 	G1Path          string
 	G2Path          string
+	G1PowerOf2Path  string
+	G2PowerOf2Path  string
 	CacheDir        string
 	NumWorker       uint64
 	SRSOrder        uint64 // Order is the total size of SRS
@@ -74,6 +76,10 @@ func NewKzgEncoderGroup(config *KzgConfig, loadG2Points bool) (*KzgEncoderGroup,
 
 	// PreloadEncoder is by default not used by operator node, PreloadEncoder
 	if loadG2Points {
+		if len(config.G2Path) == 0 {
+			return nil, fmt.Errorf("G2Path is empty. However, object needs to load G2Points")
+		}
+
 		s2, err = utils.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker)
 		if err != nil {
 			log.Println("failed to read G2 points", err)
@@ -89,6 +95,11 @@ func NewKzgEncoderGroup(config *KzgConfig, loadG2Points bool) (*KzgEncoderGroup,
 		if err != nil {
 			return nil, err
 		}
+	} else {
+		// todo, there are better ways to handle it
+		if len(config.G2PowerOf2Path) == 0 {
+			return nil, fmt.Errorf("G2PowerOf2Path is empty. However, object needs to load G2Points")
+		}
 	}
 
 	srs, err := kzg.NewSrs(s1, s2)
@@ -153,6 +164,38 @@ func ReadG2Point(n uint64, g *KzgConfig) (bls.G2Point, error) {
 	return g2point[0], nil
 }
 
+// Read g2 points from power of 2 file
+func ReadG2PointOnPowerOf2(exponent uint64, g *KzgConfig) (bls.G2Point, error) {
+
+	// the powerOf2 file, only [tau^exp] are stored.
+	// exponent    0,    1,       2,    , ..
+	// actual pow [tau],[tau^2],[tau^4],.. (stored in the file)
+	// In our convention SRSOrder contains the total number of series of g1, g2 starting with generator
+	// i.e. [1] [tau] [tau^2]..
+	// So the actual power of tau is SRSOrder - 1
+	// The mainnet SRS, the max power is 2^28-1, so the last power in powerOf2 file is [tau^(2^27)]
+	// For test case of 3000 SRS, the max power is 2999, so last power in powerOf2 file is [tau^2048]
+	// if a actual SRS order is 15, the file will contain four symbols (1,2,4,8) with indices [0,1,2,3]
+	// if a actual SRS order is 16, the file will contain five symbols (1,2,4,8,16) with indices [0,1,2,3,4]
+
+	actualPowerOfTau := g.SRSOrder - 1
+	largestPowerofSRS := uint64(math.Log2(float64(actualPowerOfTau)))
+	if exponent > largestPowerofSRS {
+		return bls.G2Point{}, fmt.Errorf("requested power %v is larger than largest power of SRS %v",
+			uint64(math.Pow(2, float64(exponent))), largestPowerofSRS)
+	}
+
+	if len(g.G2PowerOf2Path) == 0 {
+		return bls.G2Point{}, fmt.Errorf("G2PathPowerOf2 path is empty")
+	}
+
+	g2point, err := utils.ReadG2PointSection(g.G2PowerOf2Path, exponent, exponent+1, 1)
+	if err != nil {
+		return bls.G2Point{}, err
+	}
+	return g2point[0], nil
+}
+
 func (g *KzgEncoderGroup) PreloadAllEncoders() error {
 	paramsAll, err := GetAllPrecomputedSrsMap(g.CacheDir)
 	if err != nil {

pkg/encoding/kzgEncoder/encoder_suite_test.go

@@ -26,12 +26,13 @@ func setupSuite(t *testing.T) func(t *testing.T) {
 	log.Println("Setting up suite")
 
 	kzgConfig = &kzgRs.KzgConfig{
-		G1Path:    "../../../inabox/resources/kzg/g1.point",
-		G2Path:    "../../../inabox/resources/kzg/g2.point",
-		CacheDir:  "../../../inabox/resources/kzg/SRSTables",
-		SRSOrder:  3000,
-		SRSNumberToLoad:  2900,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		G1Path:          "../../../inabox/resources/kzg/g1.point",
+		G2Path:          "../../../inabox/resources/kzg/g2.point",
+		G2PowerOf2Path:  "../../../inabox/resources/kzg/g2.point.powerOf2",
+		CacheDir:        "../../../inabox/resources/kzg/SRSTables",
+		SRSOrder:        3000,
+		SRSNumberToLoad: 2900,
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
 	numNode = uint64(4)

pkg/encoding/kzgEncoder/multiframe.go

@@ -5,6 +5,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"math"
 
 	rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder"
 	kzg "github.com/Layr-Labs/eigenda/pkg/kzg"
@@ -199,10 +200,18 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples
 	lhsG1 := bls.LinCombG1(proofs, randomsFr)
 
 	// lhs g2
-	G2atD, err := ReadG2Point(D, group.KzgConfig)
+	exponent := uint64(math.Log2(float64(D)))
+	G2atD, err := ReadG2PointOnPowerOf2(exponent, group.KzgConfig)
+
 	if err != nil {
-		return err
+		// then try to access if there is a full list of g2 srs
+		G2atD, err = ReadG2Point(D, group.KzgConfig)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Acessed the entire G2")
 	}
+
 	lhsG2 := &G2atD
 
 	// rhs g2

pkg/encoding/kzgEncoder/multiframe_test.go

@@ -49,3 +49,45 @@ func TestUniversalVerify(t *testing.T) {
 
 	assert.True(t, group.UniversalVerify(params, samples, numBlob) == nil, "universal batch verification failed\n")
 }
+
+func TestUniversalVerifyWithPowerOf2G2(t *testing.T) {
+	teardownSuite := setupSuite(t)
+	defer teardownSuite(t)
+
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
+	group.KzgConfig.G2Path = ""
+	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
+	enc, err := group.NewKzgEncoder(params)
+	require.Nil(t, err)
+
+	numBlob := 5
+	samples := make([]kzgRs.Sample, 0)
+	for z := 0; z < numBlob; z++ {
+		inputFr := rs.ToFrArray(GETTYSBURG_ADDRESS_BYTES)
+
+		commit, _, _, frames, fIndices, err := enc.Encode(inputFr)
+		require.Nil(t, err)
+
+		// create samples
+		for i := 0; i < len(frames); i++ {
+			f := frames[i]
+			j := fIndices[i]
+
+			q, err := rs.GetLeadingCosetIndex(uint64(i), numSys+numPar)
+			require.Nil(t, err)
+
+			assert.Equal(t, j, q, "leading coset inconsistency")
+
+			sample := kzgRs.Sample{
+				Commitment: *commit,
+				Proof:      f.Proof,
+				RowIndex:   z,
+				Coeffs:     f.Coeffs,
+				X:          uint(q),
+			}
+			samples = append(samples, sample)
+		}
+	}
+
+	assert.True(t, group.UniversalVerify(params, samples, numBlob) == nil, "universal batch verification failed\n")
+}

retriever/server_test.go

@@ -34,6 +34,7 @@ func makeTestEncoder() (core.Encoder, error) {
 	config := &kzgEncoder.KzgConfig{
 		G1Path:          "../inabox/resources/kzg/g1.point",
 		G2Path:          "../inabox/resources/kzg/g2.point",
+		G2PowerOf2Path:  "../inabox/resources/kzg/g2.point.powerOf2",
 		CacheDir:        "../inabox/resources/kzg/SRSTables",
 		SRSOrder:        3000,
 		SRSNumberToLoad: 3000,