add tools (Layr-Labs#352)

Co-authored-by: Bowen Xue <[email protected]>

tools/srs-utils/README.md

@@ -0,0 +1,41 @@
+# srs-verification
+
+This project is written for EigenDA SRS related works. It includes
+1. Extract G1 and G2 points used by EigenDA from ptau challenge file, created from the perpetual power of tau MPC instance run by EF
+2. Verify the extracted points are indeed correct based on approaches used by EF KZG ceremony
+3. Provide utility tools for miscellaneous tasks
+
+## To download the ptau file
+
+`wget https://pse-trusted-setup-ppot.s3.eu-central-1.amazonaws.com/challenge_0085`
+
+See more informations from 
+1. https://docs.axiom.xyz/docs/transparency-and-security/kzg-trusted-setup
+2. https://github.com/privacy-scaling-explorations/perpetualpowersoftau/tree/master 
+
+The challenge file has 103079215232 Bytes.
+
+## How to use
+
+`go run main.go help`
+
+###  How to parse G1, G2 points from the challenge file.
+
+`go run main.go parse --ptau-path <Path to challenge file>`
+
+It produces two files, g1.point and g2.point. g1.point contains 8589934592 Bytes and g2.point 17179869184 Bytes
+
+This procedure takes roughly 10 minutes.
+
+Note: The challenge files contains 2^29 G1 points and 2^28 G2 points with secret tau. We use only the first 2^28 G1 points for EigenDA.
+
+### How to verify the parsed G1, G2 points
+
+`go run main.go verify --g1-path <Path to g1.point> --g2-path <Path to g2.point>`
+
+The verification is based on method listed here (https://github.com/ethereum/kzg-ceremony-specs/blob/master/docs/sequencer/sequencer.md#pairing-checks)
+
+This procedures takes roughly 27 Hours on a 8 thread machines.
+
+The program periodically prints out the time spent and its progress of validating 2^28 G1 and G2 points. If no error message is shown and program terminates with "Done. Everything is correct". Then SRS is deemed as correct. 
+

tools/srs-utils/go.mod

@@ -0,0 +1,24 @@
+module github.com/bxue-l2/srs-verification
+
+go 1.21.6
+
+require (
+	github.com/consensys/gnark-crypto v0.12.1
+	github.com/stretchr/testify v1.8.4
+	github.com/urfave/cli v1.22.14
+)
+
+require (
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
+)

tools/srs-utils/go.sum

@@ -0,0 +1,49 @@
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=
+github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ=
+github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
+github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
+github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
+github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
+github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
+github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
+github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
+github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
+rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=

tools/srs-utils/main.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/bxue-l2/srs-verification/parser"
+	"github.com/bxue-l2/srs-verification/verifier"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := &cli.App{
+		Commands: []cli.Command{
+			{
+				Name:    "verify",
+				Aliases: []string{"v"},
+				Usage:   "verify if the parsed SRS are consistent",
+				Action: func(cCtx *cli.Context) error {
+					config := verifier.ReadCLIConfig(cCtx)
+					verifier.VerifySRS(config)
+					return nil
+				},
+				Flags: verifier.Flags,
+			},
+			{
+				Name:    "parse",
+				Aliases: []string{"p"},
+				Usage:   "parse data from ptau challenge file into EigenDA SRS format",
+				Flags:   parser.Flags,
+				Action: func(cCtx *cli.Context) error {
+					config := parser.ReadCLIConfig(cCtx)
+					fmt.Printf("config %v\n", config.PtauPath)
+					parser.ParsePtauChallenge(config)
+					return nil
+				},
+			},
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}

tools/srs-utils/parser/flags.go

@@ -0,0 +1,67 @@
+package parser
+
+import (
+	"runtime"
+
+	"github.com/urfave/cli"
+)
+
+var (
+	/* Required Flags */
+	PtauPathFlag = cli.StringFlag{
+		Name:     "ptau-path",
+		Usage:    "File path to the ptau challenge file",
+		Required: true,
+		EnvVar:   "PTAU_PATH",
+	}
+
+	/* Optional Flags */
+	ParserNumBatchFlag = cli.Uint64Flag{
+		Name:     "parser-num-batch",
+		Usage:    "Set total number batch size for parallel parser to work on",
+		Required: false,
+		EnvVar:   "PARSER_NUM_BATCH",
+		Value:    uint64(50),
+	}
+	NumPointToParseFlag = cli.Uint64Flag{
+		Name:     "parser-num-points",
+		Usage:    "Set total number of points (g1 and g2) to parse",
+		Required: false,
+		EnvVar:   "PARSER_NUM_POINT",
+		Value:    uint64(268435456),
+	}
+	NumWorkerFlag = cli.IntFlag{
+		Name:     "verifier-num-worker",
+		Usage:    "Set total number of worker thread",
+		Required: false,
+		EnvVar:   "NUM_WORKER",
+		Value:    runtime.GOMAXPROCS(0),
+	}
+)
+
+var requiredFlags = []cli.Flag{
+	PtauPathFlag,
+}
+
+var optionalFlags = []cli.Flag{
+	ParserNumBatchFlag,
+	NumPointToParseFlag,
+	NumWorkerFlag,
+}
+
+func ReadCLIConfig(ctx *cli.Context) Config {
+	cfg := Config{}
+	cfg.PtauPath = ctx.String(PtauPathFlag.Name)
+	cfg.NumBatch = ctx.Uint64(ParserNumBatchFlag.Name)
+	cfg.NumPoint = ctx.Uint64(NumPointToParseFlag.Name)
+	cfg.NumWorker = ctx.Int(NumWorkerFlag.Name)
+
+	return cfg
+}
+
+func init() {
+	Flags = append(requiredFlags, optionalFlags...)
+}
+
+// Flags contains the list of configuration options available to the binary.
+var Flags []cli.Flag

tools/srs-utils/parser/g1FileIO.go

@@ -0,0 +1,158 @@
+package parser
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/consensys/gnark-crypto/ecc/bn254"
+	"github.com/consensys/gnark-crypto/ecc/bn254/fp"
+)
+
+func ParseG1PointSection(filepath string, params Params, numWorker uint64) ([]bn254.G1Affine, error) {
+	fmt.Printf("Start to read %v points from Byte pos at %v to at %v",
+		params.NumPoint,
+		params.G1StartByte,
+		params.GetG1EndBytePos(),
+	)
+
+	g1f, err := os.Open(filepath)
+	if err != nil {
+		log.Println("ReadG1PointSection.ERR.0", err)
+		return nil, err
+	}
+
+	defer func() {
+		if err := g1f.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	n := params.NumPoint
+	startTimer := time.Now()
+	g1r := bufio.NewReaderSize(g1f, int(params.NumPoint*params.G1Size))
+
+	_, err = g1f.Seek(int64(params.G1StartByte), 0)
+	if err != nil {
+		return nil, err
+	}
+
+	if n < numWorker {
+		numWorker = n
+	}
+
+	numToRead := params.NumPoint * params.G1Size
+	buf := make([]byte, numToRead)
+	numBytes, err := g1r.Read(buf)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if uint64(numBytes) != numToRead {
+		log.Printf("Error. Insufficient G1 points. Only contains %v. Requesting %v, NumByte %v\n", len(buf)/64, params.NumPoint, numBytes)
+		log.Println("numBytes", numBytes, "numToRead", numToRead)
+		log.Println("ReadG1PointSection.ERR.1", err)
+		return nil, err
+	}
+
+	// measure reading time
+	t := time.Now()
+	elapsed := t.Sub(startTimer)
+	log.Printf("    Reading G1 points (%v bytes) takes %v\n", (n * 64), elapsed)
+	startTimer = time.Now()
+
+	s1Outs := make([]bn254.G1Affine, n)
+
+	var wg sync.WaitGroup
+	wg.Add(int(numWorker))
+
+	start := uint64(0)
+	end := uint64(0)
+	size := n / numWorker
+
+	for i := uint64(0); i < numWorker; i++ {
+		start = i * size
+
+		if i == numWorker-1 {
+			end = n
+		} else {
+			end = (i + 1) * size
+		}
+		//todo: handle error?
+		go readG1Worker(buf, s1Outs, start, end, 64, &wg)
+	}
+	wg.Wait()
+
+	t = time.Now()
+	elapsed = t.Sub(startTimer)
+	fmt.Println("Finish Parsing takes", elapsed)
+	return s1Outs, nil
+}
+
+func WriteG1PointsForEigenDA(points []bn254.G1Affine, from uint64, to uint64) error {
+	n := to - from
+	g1f, err := os.OpenFile("g1.point", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		fmt.Printf("Canot write G1 from %v to %v . Error %v\n", from, to, err)
+		return err
+	}
+
+	g1w := bufio.NewWriter(g1f)
+
+	for i := uint64(0); i < n; i++ {
+		pointInBytes := points[i].Bytes()
+		numWritten, err := g1w.Write(pointInBytes[:])
+		if numWritten != 32 || err != nil {
+			fmt.Printf("Cannot write point %v . Error %v\n", from+i, err)
+			return err
+		}
+	}
+
+	if err = g1w.Flush(); err != nil {
+		log.Println("Cannot flush points", err)
+		return err
+	}
+	g1f.Close()
+	return nil
+
+}
+
+func readG1Worker(
+	buf []byte,
+	outs []bn254.G1Affine,
+	start uint64, // in element, not in byte
+	end uint64,
+	step uint64,
+	wg *sync.WaitGroup,
+) {
+	for i := start; i < end; i++ {
+		fieldSize := step / uint64(2)
+		g1x := buf[i*step : (i)*step+fieldSize]
+		g1y := buf[i*step+fieldSize : (i+1)*step]
+
+		point := parseG1Point(g1x, g1y)
+		outs[i] = *point
+	}
+	wg.Done()
+}
+
+func parseG1Point(xBytes, yBytes []byte) *bn254.G1Affine {
+	var x fp.Element
+	var y fp.Element
+
+	x.SetBytes(xBytes[:])
+	y.SetBytes(yBytes[:])
+
+	g1Aff := bn254.G1Affine{}
+	g1Aff.X = x
+	g1Aff.Y = y
+
+	if !g1Aff.IsOnCurve() {
+		panic("g1Affine is not on curve")
+	}
+	return &g1Aff
+}