diff --git a/.gitignore b/.gitignore index 39ced4dc..3ec43ad0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,10 @@ go-sidecar /internal/tests/testdata/**/*.json /internal/tests/testdata/*.sql !/internal/tests/testdata/**/generateExpectedResults.sql +!sqlite-extensions /*.tar /sidecar-data +__pycache__ +test-db +*.dylib +*.dylib.dSYM diff --git a/.testdataVersion b/.testdataVersion new file mode 100644 index 00000000..9b70f0f2 --- /dev/null +++ b/.testdataVersion @@ -0,0 +1 @@ +f8689176dbe66b2d7ce03665190298b9c9f7c8e9 \ No newline at end of file diff --git a/Makefile b/Makefile index d05d123e..2f9147a9 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,27 @@ .PHONY: deps proto +args=CGO_ENABLED=1 +GO=$(shell which go) + deps/dev: - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0 - go install honnef.co/go/tools/cmd/staticcheck@latest - go install github.com/google/yamlfmt/cmd/yamlfmt@latest + ${GO} install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0 + ${GO} install honnef.co/go/tools/cmd/staticcheck@latest + ${GO} install github.com/google/yamlfmt/cmd/yamlfmt@latest deps/go: - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - go get \ + ${GO} install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + ${GO} install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 + ${GO} get \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \ google.golang.org/protobuf/cmd/protoc-gen-go \ google.golang.org/grpc/cmd/protoc-gen-go-grpc - go install \ + ${GO} install \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \ google.golang.org/protobuf/cmd/protoc-gen-go \ google.golang.org/grpc/cmd/protoc-gen-go-grpc - go mod tidy + ${GO} mod tidy curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.61.0 @@ -42,7 +45,7 @@ clean: .PHONY: build/cmd/sidecar build/cmd/sidecar: - CGO_ENABLED=1 go build -o bin/sidecar main.go + ${args} ${GO} build -o bin/sidecar main.go .PHONY: build build: build/cmd/sidecar @@ -88,3 +91,38 @@ staticcheck: .PHONY: ci-test ci-test: test + +test-rewards: + TEST_REWARDS=true TESTING=true ${GO} test ./pkg/rewards -v -p 1 + +# ----------------------------------------------------------------------------- +# SQLite extension build steps +# ----------------------------------------------------------------------------- +CC = gcc -g -fPIC -shared + +PYTHON_CONFIG = python3-config +PYTHON_VERSION = $(shell python3 -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))") +PYTHON_LIBDIR := $(shell $(PYTHON_CONFIG) --prefix)/lib + +# Base flags +CFLAGS = +LDFLAGS = + +INCLUDE_DIRS = +CFLAGS += $(foreach dir,$(INCLUDE_DIRS),-I$(dir)) + +# Python flags +PYTHON_CFLAGS := $(shell $(PYTHON_CONFIG) --includes) +PYTHON_LDFLAGS := $(shell $(PYTHON_CONFIG) --ldflags) + +SQLITE_DIR = /opt/homebrew/opt/sqlite +CFLAGS += -I$(SQLITE_DIR)/include +LDFLAGS += -L$(SQLITE_DIR)/lib -lsqlite3 + +CFLAGS += $(PYTHON_CFLAGS) +LDFLAGS += $(PYTHON_LDFLAGS) -L$(PYTHON_LIBDIR) -lpython$(PYTHON_VERSION) + + +.PHONY: sqlite-extensions +sqlite-extensions: + $(CC) $(CFLAGS) -o sqlite-extensions/libcalculations.dylib sqlite-extensions/calculations.c $(LDFLAGS) diff --git a/cmd/debugger/main.go b/cmd/debugger/main.go new file mode 100644 index 00000000..6166c55e --- /dev/null +++ b/cmd/debugger/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/Layr-Labs/go-sidecar/internal/clients/ethereum" + "github.com/Layr-Labs/go-sidecar/internal/clients/etherscan" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/contractCaller" + "github.com/Layr-Labs/go-sidecar/internal/contractManager" + "github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerDelegations" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stateManager" + "github.com/Layr-Labs/go-sidecar/internal/fetcher" + "github.com/Layr-Labs/go-sidecar/internal/indexer" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/metrics" + "github.com/Layr-Labs/go-sidecar/internal/pipeline" + "github.com/Layr-Labs/go-sidecar/internal/sidecar" + "github.com/Layr-Labs/go-sidecar/internal/sqlite" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite" + "go.uber.org/zap" +) + +func main() { + ctx := context.Background() + cfg := config.NewConfig() + + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl) + if err != nil { + l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err)) + } + + etherscanClient := etherscan.NewEtherscanClient(cfg, l) + client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l) + + db := sqlite.NewSqlite(cfg.GetSqlitePath(), l) + + grm, err := sqlite.NewGormSqliteFromSqlite(db) + if err != nil { + l.Error("Failed to create gorm instance", zap.Error(err)) + panic(err) + } + + migrator := migrations.NewSqliteMigrator(grm, l) + if err = migrator.MigrateAll(); err != nil { + log.Fatalf("Failed to migrate: %v", err) + } + + contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg) + if err := contractStore.InitializeCoreContracts(); err != nil { + log.Fatalf("Failed to initialize core contracts: %v", err) + } + + cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l) + + mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg) + if err != nil { + log.Fatalln(err) + } + + sm := stateManager.NewEigenStateManager(l, grm) + + if _, err := avsOperators.NewAvsOperators(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create AvsOperatorsModel", zap.Error(err)) + } + if _, err := operatorShares.NewOperatorSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create OperatorSharesModel", zap.Error(err)) + } + if _, err := stakerDelegations.NewStakerDelegationsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerDelegationsModel", zap.Error(err)) + } + if _, err := stakerShares.NewStakerSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerSharesModel", zap.Error(err)) + } + + fetchr := fetcher.NewFetcher(client, cfg, l) + + cc := contractCaller.NewContractCaller(client, l) + + idxr := indexer.NewIndexer(mds, contractStore, etherscanClient, cm, client, fetchr, cc, l, cfg) + + p := pipeline.NewPipeline(fetchr, idxr, mds, sm, l) + + // Create new sidecar instance + sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{ + GenesisBlockNumber: cfg.GetGenesisBlockNumber(), + }, cfg, mds, p, sm, l, client) + + // RPC channel to notify the RPC server to shutdown gracefully + rpcChannel := make(chan bool) + err = sidecar.WithRpcServer(ctx, mds, sm, rpcChannel) + if err != nil { + l.Sugar().Fatalw("Failed to start RPC server", zap.Error(err)) + } + + block, err := fetchr.FetchBlock(ctx, 1215893) + if err != nil { + l.Sugar().Fatalw("Failed to fetch block", zap.Error(err)) + } + + transactionHash := "0xf6775c38af1d2802bcbc2b7c8959c0d5b48c63a14bfeda0261ba29d76c68c423" + transaction := ðereum.EthereumTransaction{} + + for _, tx := range block.Block.Transactions { + if tx.Hash.Value() == transactionHash { + transaction = tx + break + } + } + + logIndex := 4 + receipt := block.TxReceipts[transaction.Hash.Value()] + var interestingLog *ethereum.EthereumEventLog + + for _, log := range receipt.Logs { + if log.LogIndex.Value() == uint64(logIndex) { + fmt.Printf("Log: %+v\n", log) + interestingLog = log + } + } + + decodedLog, err := idxr.DecodeLogWithAbi(nil, receipt, interestingLog) + if err != nil { + l.Sugar().Fatalw("Failed to decode log", zap.Error(err)) + } + l.Sugar().Infof("Decoded log: %+v", decodedLog) +} diff --git a/cmd/run.go b/cmd/run.go index 18c38521..6671bb0e 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -59,7 +59,7 @@ var runCmd = &cobra.Command{ } } - db := sqlite.NewSqlite(cfg.GetSqlitePath()) + db := sqlite.NewSqlite(cfg.GetSqlitePath(), l) grm, err := sqlite.NewGormSqliteFromSqlite(db) if err != nil { diff --git a/cmd/sidecar/main.go b/cmd/sidecar/main.go new file mode 100644 index 00000000..7b5366b7 --- /dev/null +++ b/cmd/sidecar/main.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "log" + "time" + + "github.com/Layr-Labs/go-sidecar/internal/clients/ethereum" + "github.com/Layr-Labs/go-sidecar/internal/clients/etherscan" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/contractCaller" + "github.com/Layr-Labs/go-sidecar/internal/contractManager" + "github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/rewardSubmissions" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerDelegations" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stateManager" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/submittedDistributionRoots" + "github.com/Layr-Labs/go-sidecar/internal/fetcher" + "github.com/Layr-Labs/go-sidecar/internal/indexer" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/metrics" + "github.com/Layr-Labs/go-sidecar/internal/pipeline" + "github.com/Layr-Labs/go-sidecar/internal/shutdown" + "github.com/Layr-Labs/go-sidecar/internal/sidecar" + "github.com/Layr-Labs/go-sidecar/internal/sqlite" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite" + "go.uber.org/zap" +) + +func main() { + ctx := context.Background() + cfg := config.NewConfig() + + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl) + if err != nil { + l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err)) + } + + etherscanClient := etherscan.NewEtherscanClient(cfg, l) + client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l) + + db := sqlite.NewSqlite(cfg.GetSqlitePath(), l) + + grm, err := sqlite.NewGormSqliteFromSqlite(db) + if err != nil { + l.Error("Failed to create gorm instance", zap.Error(err)) + panic(err) + } + + migrator := migrations.NewSqliteMigrator(grm, l) + if err = migrator.MigrateAll(); err != nil { + log.Fatalf("Failed to migrate: %v", err) + } + + contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg) + if err := contractStore.InitializeCoreContracts(); err != nil { + log.Fatalf("Failed to initialize core contracts: %v", err) + } + + cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l) + + mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg) + if err != nil { + log.Fatalln(err) + } + + sm := stateManager.NewEigenStateManager(l, grm) + + if _, err := avsOperators.NewAvsOperators(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create AvsOperatorsModel", zap.Error(err)) + } + if _, err := operatorShares.NewOperatorSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create OperatorSharesModel", zap.Error(err)) + } + if _, err := stakerDelegations.NewStakerDelegationsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerDelegationsModel", zap.Error(err)) + } + if _, err := stakerShares.NewStakerSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerSharesModel", zap.Error(err)) + } + if _, err := submittedDistributionRoots.NewSubmittedDistributionRootsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create SubmittedDistributionRootsModel", zap.Error(err)) + } + if _, err := rewardSubmissions.NewRewardSubmissionsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create RewardSubmissionsModel", zap.Error(err)) + } + + fetchr := fetcher.NewFetcher(client, cfg, l) + + cc := contractCaller.NewContractCaller(client, l) + + idxr := indexer.NewIndexer(mds, contractStore, etherscanClient, cm, client, fetchr, cc, l, cfg) + + p := pipeline.NewPipeline(fetchr, idxr, mds, sm, l) + + // Create new sidecar instance + sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{ + GenesisBlockNumber: cfg.GetGenesisBlockNumber(), + }, cfg, mds, p, sm, l, client) + + // RPC channel to notify the RPC server to shutdown gracefully + rpcChannel := make(chan bool) + err = sidecar.WithRpcServer(ctx, mds, sm, rpcChannel) + if err != nil { + l.Sugar().Fatalw("Failed to start RPC server", zap.Error(err)) + } + + // Start the sidecar main process in a goroutine so that we can listen for a shutdown signal + go sidecar.Start(ctx) + + l.Sugar().Info("Started Sidecar") + + gracefulShutdown := shutdown.CreateGracefulShutdownChannel() + + done := make(chan bool) + shutdown.ListenForShutdown(gracefulShutdown, done, func() { + l.Sugar().Info("Shutting down...") + rpcChannel <- true + sidecar.ShutdownChan <- true + }, time.Second*5, l) +} diff --git a/cmd/sqlitedebug/db.py b/cmd/sqlitedebug/db.py new file mode 100644 index 00000000..5bb3c886 --- /dev/null +++ b/cmd/sqlitedebug/db.py @@ -0,0 +1,7 @@ +import sqlite3 + +conn = sqlite3.connect(":memory:") +conn.enable_load_extension(True) +conn.load_extension("/Users/seanmcgary/Code/sidecar/sqlite-extensions/yolo.dylib") +conn.execute("SELECT my_custom_function('foo')") +conn.close() diff --git a/cmd/sqlitedebug/main.go b/cmd/sqlitedebug/main.go new file mode 100644 index 00000000..0826af25 --- /dev/null +++ b/cmd/sqlitedebug/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "database/sql" + "fmt" + "github.com/mattn/go-sqlite3" + _ "github.com/mattn/go-sqlite3" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +var extensionPath = "/Users/seanmcgary/Code/sidecar/sqlite-extensions/yolo.dylib" + +func main() { + sql.Register("sqlite3_with_extensions", &sqlite3.SQLiteDriver{ + Extensions: []string{extensionPath}, + }) + dialector := gorm.Dialector(sqlite.Dialector{ + DSN: "file:yourdatabase.db?cache=shared&_fk=1", + DriverName: "sqlite3_with_extensions", + }) + db, err := gorm.Open(dialector, &gorm.Config{}) + if err != nil { + panic("failed to connect database") + } + + var r string + res := db.Raw("SELECT pre_nile_tokens_per_day('500') as result").Scan(&r) + if res.Error != nil { + panic(fmt.Sprintf("failed to load extension: %v", err)) + } + fmt.Printf("Result: %s\n", r) +} diff --git a/go.mod b/go.mod index b3ab9abc..cc49f39e 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,13 @@ go 1.22.2 require ( github.com/DataDog/datadog-go/v5 v5.5.0 github.com/ethereum/go-ethereum v1.14.9 + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 github.com/jbrower95/multicall-go v0.0.0-20240923010412-060e37b98d03 github.com/mattn/go-sqlite3 v1.14.23 github.com/rs/cors v1.7.0 + github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 @@ -27,31 +29,31 @@ require ( require ( github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect - github.com/bits-and-blooms/bitset v1.14.3 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/buger/jsonparser v1.1.1 // indirect - github.com/consensys/bavard v0.1.16 // indirect - github.com/consensys/gnark-crypto v0.14.0 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect - github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect - github.com/ethereum/c-kzg-4844 v1.0.3 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/iden3/go-iden3-crypto v0.0.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/klauspost/compress v1.17.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -61,22 +63,21 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/supranational/blst v0.3.13 // indirect - github.com/tklauser/go-sysconf v0.3.14 // indirect - github.com/tklauser/numcpus v0.8.0 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/supranational/blst v0.3.11 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.17.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index fad0d458..404dfe43 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= @@ -14,8 +16,8 @@ github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xW github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA= -github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -41,32 +43,32 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/consensys/bavard v0.1.16 h1:3+nT0BqzKg84VtCY9eNN2Glkf1X7dbS5yhh5849syJ8= -github.com/consensys/bavard v0.1.16/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= -github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= -github.com/crate-crypto/go-kzg-4844 v1.1.0 h1:EN/u9k2TF6OWSHrCCDBBU6GLNMq88OspHHlMnHfoyU4= -github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlXuJEC996PL3tTAFks= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy4WDs= -github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.14.9 h1:J7iwXDrtUyE9FUjUYbd4c9tyzwMh6dTJsKzo9i6SrwA= github.com/ethereum/go-ethereum v1.14.9/go.mod h1:QeW+MtTpRdBEm2pUFoonByee8zfHv7kGp0wK0odvU1I= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= @@ -81,10 +83,12 @@ github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -109,8 +113,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= @@ -142,8 +146,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -154,8 +158,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= -github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -199,8 +203,8 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -209,8 +213,10 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -245,14 +251,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= -github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= -github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= -github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= -github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= @@ -266,8 +272,6 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -281,11 +285,11 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -302,8 +306,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -326,13 +330,15 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/config/config.go b/internal/config/config.go index 70a1ba47..bd78cad5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "github.com/spf13/viper" "strings" @@ -13,10 +14,15 @@ type Environment int type Chain string +type ForkName string + const ( Chain_Mainnet Chain = "mainnet" Chain_Holesky Chain = "holesky" Chain_Preprod Chain = "preprod" + + Fork_Nile ForkName = "nile" + Fork_Amazon ForkName = "amazon" ) func parseListEnvVar(envVar string) []string { @@ -191,6 +197,29 @@ func (c *Config) GetGenesisBlockNumber() uint64 { } } +type ForkMap map[ForkName]string + +func (c *Config) GetForkDates() (ForkMap, error) { + switch c.Chain { + case Chain_Preprod: + return ForkMap{ + Fork_Amazon: "1970-01-01", // Amazon hard fork was never on preprod as we backfilled + Fork_Nile: "2024-08-14", // Last calculation end timestamp was 8-13: https://holesky.etherscan.io/tx/0xb5a6855e88c79312b7c0e1c9f59ae9890b97f157ea27e69e4f0fadada4712b64#eventlog + }, nil + case Chain_Holesky: + return ForkMap{ + Fork_Amazon: "1970-01-01", // Amazon hard fork was never on testnet as we backfilled + Fork_Nile: "2024-08-13", // Last calculation end timestamp was 8-12: https://holesky.etherscan.io/tx/0x5fc81b5ed2a78b017ef313c181d8627737a97fef87eee85acedbe39fc8708c56#eventlog + }, nil + case Chain_Mainnet: + return ForkMap{ + Fork_Amazon: "2024-08-02", // Last calculation end timestamp was 8-01: https://etherscan.io/tx/0x2aff6f7b0132092c05c8f6f41a5e5eeeb208aa0d95ebcc9022d7823e343dd012#eventlog + Fork_Nile: "2024-08-12", // Last calculation end timestamp was 8-11: https://etherscan.io/tx/0x922d29d93c02d189fc2332041f01a80e0007cd7a625a5663ef9d30082f7ef66f#eventlog + }, nil + } + return nil, errors.New("unsupported chain") +} + func (c *Config) GetEigenLayerGenesisBlockHeight() (uint64, error) { switch c.Chain { case Chain_Preprod, Chain_Holesky: diff --git a/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go b/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go index ce86f48a..35ac8b27 100644 --- a/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go +++ b/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go @@ -22,7 +22,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/avsOperators/avsOperators.go b/internal/eigenState/avsOperators/avsOperators.go index 8d69e264..9a1f0ab5 100644 --- a/internal/eigenState/avsOperators/avsOperators.go +++ b/internal/eigenState/avsOperators/avsOperators.go @@ -2,6 +2,7 @@ package avsOperators import ( "database/sql" + "errors" "fmt" "slices" "sort" @@ -44,6 +45,14 @@ type RegisteredAvsOperatorDiff struct { Registered bool } +type AvsOperatorStateChange struct { + Avs string + Operator string + Registered bool + LogIndex uint64 + BlockNumber uint64 +} + func NewSlotID(avs string, operator string) types.SlotID { return types.SlotID(fmt.Sprintf("%s_%s", avs, operator)) } @@ -58,6 +67,9 @@ type AvsOperatorsModel struct { // Accumulates state changes for SlotIds, grouped by block number stateAccumulator map[uint64]map[types.SlotID]*AccumulatedStateChange + + // Keep track of each distinct change, rather than accumulated change, to add to the delta table + deltaAccumulator map[uint64][]*AvsOperatorStateChange } // NewAvsOperators creates a new AvsOperatorsModel. @@ -76,6 +88,8 @@ func NewAvsOperators( globalConfig: globalConfig, stateAccumulator: make(map[uint64]map[types.SlotID]*AccumulatedStateChange), + + deltaAccumulator: make(map[uint64][]*AvsOperatorStateChange), } esm.RegisterState(s, 0) return s, nil @@ -120,6 +134,15 @@ func (a *AvsOperatorsModel) GetStateTransitions() (types.StateTransitions[Accumu registered = uint64(val.(float64)) == 1 } + // Store the change in the delta accumulator + a.deltaAccumulator[log.BlockNumber] = append(a.deltaAccumulator[log.BlockNumber], &AvsOperatorStateChange{ + Avs: avs, + Operator: operator, + Registered: registered, + LogIndex: log.LogIndex, + BlockNumber: log.BlockNumber, + }) + slotID := NewSlotID(avs, operator) record, ok := a.stateAccumulator[log.BlockNumber][slotID] if !ok { @@ -173,6 +196,7 @@ func (a *AvsOperatorsModel) IsInterestingLog(log *storage.TransactionLog) bool { func (a *AvsOperatorsModel) InitBlockProcessing(blockNumber uint64) error { a.stateAccumulator[blockNumber] = make(map[types.SlotID]*AccumulatedStateChange) + a.deltaAccumulator[blockNumber] = make([]*AvsOperatorStateChange, 0) return nil } @@ -249,6 +273,24 @@ func (a *AvsOperatorsModel) prepareState(blockNumber uint64) ([]RegisteredAvsOpe return inserts, deletes, nil } +func (a *AvsOperatorsModel) writeDeltaRecordsToDeltaTable(blockNumber uint64) error { + records, ok := a.deltaAccumulator[blockNumber] + if !ok { + msg := "Delta accumulator was not initialized" + a.logger.Sugar().Errorw(msg, zap.Uint64("blockNumber", blockNumber)) + return errors.New(msg) + } + + if len(records) > 0 { + res := a.DB.Model(&AvsOperatorStateChange{}).Clauses(clause.Returning{}).Create(&records) + if res.Error != nil { + a.logger.Sugar().Errorw("Failed to insert delta records", zap.Error(res.Error)) + return res.Error + } + } + return nil +} + // CommitFinalState commits the final state for the given block number. func (a *AvsOperatorsModel) CommitFinalState(blockNumber uint64) error { err := a.clonePreviousBlocksToNewBlock(blockNumber) @@ -280,11 +322,17 @@ func (a *AvsOperatorsModel) CommitFinalState(blockNumber uint64) error { return res.Error } } + + if err = a.writeDeltaRecordsToDeltaTable(blockNumber); err != nil { + return err + } + return nil } func (a *AvsOperatorsModel) ClearAccumulatedState(blockNumber uint64) error { delete(a.stateAccumulator, blockNumber) + delete(a.deltaAccumulator, blockNumber) return nil } diff --git a/internal/eigenState/avsOperators/avsOperators_test.go b/internal/eigenState/avsOperators/avsOperators_test.go index d7cdddc1..b467d29f 100644 --- a/internal/eigenState/avsOperators/avsOperators_test.go +++ b/internal/eigenState/avsOperators/avsOperators_test.go @@ -25,7 +25,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -40,6 +40,21 @@ func setup() ( func teardown(model *AvsOperatorsModel) { model.DB.Exec("delete from avs_operator_changes") model.DB.Exec("delete from registered_avs_operators") + model.DB.Exec("delete from avs_operator_state_changes") +} + +func getInsertedDeltaRecordsForBlock(blockNumber uint64, model *AvsOperatorsModel) ([]*AvsOperatorStateChange, error) { + results := []*AvsOperatorStateChange{} + + res := model.DB.Model(&AvsOperatorStateChange{}).Where("block_number = ?", blockNumber).Find(&results) + return results, res.Error +} + +func getInsertedDeltaRecords(model *AvsOperatorsModel) ([]*AvsOperatorStateChange, error) { + results := []*AvsOperatorStateChange{} + + res := model.DB.Model(&AvsOperatorStateChange{}).Order("block_number asc").Find(&results) + return results, res.Error } func Test_AvsOperatorState(t *testing.T) { @@ -84,7 +99,22 @@ func Test_AvsOperatorState(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, res) - teardown(avsOperatorState) + err = avsOperatorState.CommitFinalState(blockNumber) + assert.Nil(t, err) + + inserted, err := getInsertedDeltaRecordsForBlock(blockNumber, avsOperatorState) + assert.Nil(t, err) + assert.Equal(t, 1, len(inserted)) + + assert.Equal(t, "0xdf25bdcdcdd9a3dd8c9069306c4dba8d90dd8e8e", inserted[0].Avs) + assert.Equal(t, "0x870679e138bcdf293b7ff14dd44b70fc97e12fc0", inserted[0].Operator) + assert.Equal(t, true, inserted[0].Registered) + assert.Equal(t, blockNumber, inserted[0].BlockNumber) + assert.Equal(t, uint64(400), inserted[0].LogIndex) + + t.Cleanup(func() { + teardown(avsOperatorState) + }) }) t.Run("Should register AvsOperatorState and generate the table for the block", func(t *testing.T) { esm := stateManager.NewEigenStateManager(l, grm) @@ -134,7 +164,9 @@ func Test_AvsOperatorState(t *testing.T) { assert.Nil(t, err) assert.True(t, len(stateRoot) > 0) - teardown(avsOperatorState) + t.Cleanup(func() { + teardown(avsOperatorState) + }) }) t.Run("Should correctly generate state across multiple blocks", func(t *testing.T) { esm := stateManager.NewEigenStateManager(l, grm) @@ -217,6 +249,12 @@ func Test_AvsOperatorState(t *testing.T) { assert.True(t, len(stateRoot) > 0) } - teardown(avsOperatorState) + inserted, err := getInsertedDeltaRecords(avsOperatorState) + assert.Nil(t, err) + assert.Equal(t, len(logs), len(inserted)) + + t.Cleanup(func() { + teardown(avsOperatorState) + }) }) } diff --git a/internal/eigenState/eigenstate_test.go b/internal/eigenState/eigenstate_test.go index 79ce5820..a42e47f6 100644 --- a/internal/eigenState/eigenstate_test.go +++ b/internal/eigenState/eigenstate_test.go @@ -23,7 +23,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/operatorShares/operatorShares.go b/internal/eigenState/operatorShares/operatorShares.go index 7316aa84..71b6c260 100644 --- a/internal/eigenState/operatorShares/operatorShares.go +++ b/internal/eigenState/operatorShares/operatorShares.go @@ -4,6 +4,7 @@ import ( "database/sql" "encoding/json" "fmt" + pkgUtils "github.com/Layr-Labs/go-sidecar/pkg/utils" "math/big" "slices" "sort" @@ -212,32 +213,9 @@ func (osm *OperatorSharesModel) HandleStateChange(log *storage.TransactionLog) ( return nil, nil //nolint:nilnil } -func (osm *OperatorSharesModel) clonePreviousBlocksToNewBlock(blockNumber uint64) error { - query := ` - insert into operator_shares (operator, strategy, shares, block_number) - select - operator, - strategy, - shares, - @currentBlock as block_number - from operator_shares - where block_number = @previousBlock - ` - res := osm.DB.Exec(query, - sql.Named("currentBlock", blockNumber), - sql.Named("previousBlock", blockNumber-1), - ) - - if res.Error != nil { - osm.logger.Sugar().Errorw("Failed to clone previous block state to new block", zap.Error(res.Error)) - return res.Error - } - return nil -} - // prepareState prepares the state for commit by adding the new state to the existing state. -func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorSharesDiff, error) { - preparedState := make([]OperatorSharesDiff, 0) +func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]*OperatorSharesDiff, error) { + preparedState := make([]*OperatorSharesDiff, 0) accumulatedState, ok := osm.stateAccumulator[blockNumber] if !ok { @@ -253,19 +231,28 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar // Find only the records from the previous block, that are modified in this block query := ` + with ranked_rows as ( + select + operator, + strategy, + shares, + block_number, + ROW_NUMBER() OVER (PARTITION BY operator, strategy ORDER BY block_number desc) as rn + from operator_shares + where + concat(operator, '_', strategy) in @slotIds + ) select - operator, - strategy, - shares - from operator_shares - where - block_number = @previousBlock - and concat(operator, '_', strategy) in @slotIds + lb.operator, + lb.strategy, + lb.shares, + lb.block_number + from ranked_rows as lb + where rn = 1 ` - existingRecords := make([]OperatorShares, 0) + existingRecords := make([]*OperatorShares, 0) res := osm.DB.Model(&OperatorShares{}). Raw(query, - sql.Named("previousBlock", blockNumber-1), sql.Named("slotIds", slotIds), ). Scan(&existingRecords) @@ -276,9 +263,8 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar } // Map the existing records to a map for easier lookup - mappedRecords := make(map[types.SlotID]OperatorShares) + mappedRecords := make(map[types.SlotID]*OperatorShares) for _, record := range existingRecords { - fmt.Printf("Existing OperatorShares %+v\n", record) slotID := NewSlotID(record.Operator, record.Strategy) mappedRecords[slotID] = record } @@ -286,7 +272,7 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar // Loop over our new state changes. // If the record exists in the previous block, add the shares to the existing shares for slotID, newState := range accumulatedState { - prepared := OperatorSharesDiff{ + prepared := &OperatorSharesDiff{ Operator: newState.Operator, Strategy: newState.Strategy, Shares: newState.Shares, @@ -317,56 +303,27 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar } func (osm *OperatorSharesModel) CommitFinalState(blockNumber uint64) error { - // Clone the previous block state to give us a reference point. - err := osm.clonePreviousBlocksToNewBlock(blockNumber) - if err != nil { - return err - } - records, err := osm.prepareState(blockNumber) if err != nil { return err } - newRecords := make([]OperatorShares, 0) - updateRecords := make([]OperatorShares, 0) - - for _, record := range records { - r := &OperatorShares{ - Operator: record.Operator, - Strategy: record.Strategy, - Shares: record.Shares.String(), - BlockNumber: record.BlockNumber, - } - if record.IsNew { - newRecords = append(newRecords, *r) - } else { - updateRecords = append(updateRecords, *r) + recordToInsert := pkgUtils.Map(records, func(r *OperatorSharesDiff, i uint64) *OperatorShares { + return &OperatorShares{ + Operator: r.Operator, + Strategy: r.Strategy, + Shares: r.Shares.String(), + BlockNumber: blockNumber, } - } + }) - // Batch insert new records - if len(newRecords) > 0 { - res := osm.DB.Model(&OperatorShares{}).Clauses(clause.Returning{}).Create(&newRecords) + if len(recordToInsert) > 0 { + res := osm.DB.Model(&OperatorShares{}).Clauses(clause.Returning{}).Create(&recordToInsert) if res.Error != nil { osm.logger.Sugar().Errorw("Failed to create new operator_shares records", zap.Error(res.Error)) return res.Error } } - // Update existing records that were cloned from the previous block - if len(updateRecords) > 0 { - for _, record := range updateRecords { - res := osm.DB.Model(&OperatorShares{}). - Where("operator = ? and strategy = ? and block_number = ?", record.Operator, record.Strategy, record.BlockNumber). - Updates(map[string]interface{}{ - "shares": record.Shares, - }) - if res.Error != nil { - osm.logger.Sugar().Errorw("Failed to update operator_shares record", zap.Error(res.Error)) - return res.Error - } - } - } return nil } @@ -391,7 +348,7 @@ func (osm *OperatorSharesModel) GenerateStateRoot(blockNumber uint64) (types.Sta return types.StateRoot(utils.ConvertBytesToString(fullTree.Root())), nil } -func (osm *OperatorSharesModel) sortValuesForMerkleTree(diffs []OperatorSharesDiff) []*base.MerkleTreeInput { +func (osm *OperatorSharesModel) sortValuesForMerkleTree(diffs []*OperatorSharesDiff) []*base.MerkleTreeInput { inputs := make([]*base.MerkleTreeInput, 0) for _, diff := range diffs { inputs = append(inputs, &base.MerkleTreeInput{ diff --git a/internal/eigenState/operatorShares/operatorShares_test.go b/internal/eigenState/operatorShares/operatorShares_test.go index f5586822..af690c70 100644 --- a/internal/eigenState/operatorShares/operatorShares_test.go +++ b/internal/eigenState/operatorShares/operatorShares_test.go @@ -27,7 +27,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -40,7 +40,6 @@ func setup() ( } func teardown(model *OperatorSharesModel) { - model.DB.Exec("delete from operator_share_changes") model.DB.Exec("delete from operator_shares") } diff --git a/internal/eigenState/rewardSubmissions/rewardSubmissions.go b/internal/eigenState/rewardSubmissions/rewardSubmissions.go index 473d02eb..10445df2 100644 --- a/internal/eigenState/rewardSubmissions/rewardSubmissions.go +++ b/internal/eigenState/rewardSubmissions/rewardSubmissions.go @@ -34,7 +34,7 @@ type RewardSubmission struct { EndTimestamp *time.Time `gorm:"type:DATETIME"` Duration uint64 BlockNumber uint64 - IsForAll bool + RewardType string // avs, all_stakers, all_earners } type RewardSubmissionDiff struct { @@ -151,6 +151,15 @@ func (rs *RewardSubmissionsModel) handleRewardSubmissionCreatedEvent(log *storag return nil, xerrors.Errorf("Failed to parse multiplier to Big257: %s", actualOuputData.Amount.String()) } + var rewardType string + if log.EventName == "RewardsSubmissionForAllCreated" || log.EventName == "RangePaymentForAllCreated" { + rewardType = "all_stakers" + } else if log.EventName == "RangePaymentCreated" || log.EventName == "AVSRewardsSubmissionCreated" { + rewardType = "avs" + } else { + return nil, xerrors.Errorf("Unknown event name: %s", log.EventName) + } + rewardSubmission := &RewardSubmission{ Avs: strings.ToLower(arguments[0].Value.(string)), RewardHash: strings.ToLower(arguments[2].Value.(string)), @@ -162,7 +171,7 @@ func (rs *RewardSubmissionsModel) handleRewardSubmissionCreatedEvent(log *storag EndTimestamp: &endTimestamp, Duration: actualOuputData.Duration, BlockNumber: log.BlockNumber, - IsForAll: log.EventName == "RewardsSubmissionForAllCreated" || log.EventName == "RangePaymentForAllCreated", + RewardType: rewardType, } rewardSubmissions = append(rewardSubmissions, rewardSubmission) } @@ -252,7 +261,7 @@ func (rs *RewardSubmissionsModel) HandleStateChange(log *storage.TransactionLog) func (rs *RewardSubmissionsModel) clonePreviousBlocksToNewBlock(blockNumber uint64) error { query := ` - insert into reward_submissions(avs, reward_hash, token, amount, strategy, strategy_index, multiplier, start_timestamp, end_timestamp, duration, is_for_all, block_number) + insert into reward_submissions(avs, reward_hash, token, amount, strategy, strategy_index, multiplier, start_timestamp, end_timestamp, duration, reward_type, block_number) select avs, reward_hash, @@ -264,7 +273,7 @@ func (rs *RewardSubmissionsModel) clonePreviousBlocksToNewBlock(blockNumber uint start_timestamp, end_timestamp, duration, - is_for_all, + reward_type, @currentBlock as block_number from reward_submissions where block_number = @previousBlock diff --git a/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go b/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go index 2310baa2..17c8aa76 100644 --- a/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go +++ b/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go @@ -27,7 +27,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/stakerDelegations/stakerDelegations.go b/internal/eigenState/stakerDelegations/stakerDelegations.go index 72a92262..d55514a2 100644 --- a/internal/eigenState/stakerDelegations/stakerDelegations.go +++ b/internal/eigenState/stakerDelegations/stakerDelegations.go @@ -2,6 +2,7 @@ package stakerDelegations import ( "database/sql" + "errors" "fmt" "slices" "sort" @@ -36,6 +37,14 @@ type AccumulatedStateChange struct { Delegated bool } +type StakerDelegationChange struct { + Staker string + Operator string + BlockNumber uint64 + Delegated bool + LogIndex uint64 +} + func NewSlotID(staker string, operator string) types.SlotID { return types.SlotID(fmt.Sprintf("%s_%s", staker, operator)) } @@ -49,6 +58,8 @@ type StakerDelegationsModel struct { // Accumulates state changes for SlotIds, grouped by block number stateAccumulator map[uint64]map[types.SlotID]*AccumulatedStateChange + + deltaAccumulator map[uint64][]*StakerDelegationChange } type DelegatedStakersDiff struct { @@ -72,6 +83,8 @@ func NewStakerDelegationsModel( logger: logger, globalConfig: globalConfig, stateAccumulator: make(map[uint64]map[types.SlotID]*AccumulatedStateChange), + + deltaAccumulator: make(map[uint64][]*StakerDelegationChange), } esm.RegisterState(model, 2) @@ -123,6 +136,15 @@ func (s *StakerDelegationsModel) GetStateTransitions() (types.StateTransitions[A record.Delegated = true } + // Store the change in the delta accumulator + s.deltaAccumulator[log.BlockNumber] = append(s.deltaAccumulator[log.BlockNumber], &StakerDelegationChange{ + Staker: staker, + Operator: operator, + BlockNumber: log.BlockNumber, + Delegated: record.Delegated, + LogIndex: log.LogIndex, + }) + return record, nil } @@ -157,6 +179,7 @@ func (s *StakerDelegationsModel) IsInterestingLog(log *storage.TransactionLog) b // InitBlockProcessing initialize state accumulator for the block. func (s *StakerDelegationsModel) InitBlockProcessing(blockNumber uint64) error { s.stateAccumulator[blockNumber] = make(map[types.SlotID]*AccumulatedStateChange) + s.deltaAccumulator[blockNumber] = make([]*StakerDelegationChange, 0) return nil } @@ -229,6 +252,24 @@ func (s *StakerDelegationsModel) prepareState(blockNumber uint64) ([]DelegatedSt return inserts, deletes, nil } +func (s *StakerDelegationsModel) writeDeltaRecordsToDeltaTable(blockNumber uint64) error { + records, ok := s.deltaAccumulator[blockNumber] + if !ok { + msg := "Delta accumulator was not initialized" + s.logger.Sugar().Errorw(msg, zap.Uint64("blockNumber", blockNumber)) + return errors.New(msg) + } + + if len(records) > 0 { + res := s.DB.Model(&StakerDelegationChange{}).Clauses(clause.Returning{}).Create(&records) + if res.Error != nil { + s.logger.Sugar().Errorw("Failed to insert delta records", zap.Error(res.Error)) + return res.Error + } + } + return nil +} + func (s *StakerDelegationsModel) CommitFinalState(blockNumber uint64) error { // Clone the previous block state to give us a reference point. // @@ -264,12 +305,17 @@ func (s *StakerDelegationsModel) CommitFinalState(blockNumber uint64) error { return res.Error } } + + if err = s.writeDeltaRecordsToDeltaTable(blockNumber); err != nil { + return err + } return nil } // ClearAccumulatedState clears the accumulated state for the given block number to free up memory. func (s *StakerDelegationsModel) ClearAccumulatedState(blockNumber uint64) error { delete(s.stateAccumulator, blockNumber) + delete(s.deltaAccumulator, blockNumber) return nil } diff --git a/internal/eigenState/stakerDelegations/stakerDelegations_test.go b/internal/eigenState/stakerDelegations/stakerDelegations_test.go index e74bf03e..e312a465 100644 --- a/internal/eigenState/stakerDelegations/stakerDelegations_test.go +++ b/internal/eigenState/stakerDelegations/stakerDelegations_test.go @@ -24,7 +24,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -39,6 +39,7 @@ func setup() ( func teardown(model *StakerDelegationsModel) { model.DB.Exec("delete from staker_delegation_changes") model.DB.Exec("delete from delegated_stakers") + model.DB.Exec("delete from staker_delegation_changes") } func Test_DelegatedStakersState(t *testing.T) { diff --git a/internal/eigenState/stakerShares/stakerShares.go b/internal/eigenState/stakerShares/stakerShares.go index 9c9a2d63..d580825c 100644 --- a/internal/eigenState/stakerShares/stakerShares.go +++ b/internal/eigenState/stakerShares/stakerShares.go @@ -18,6 +18,7 @@ import ( "github.com/Layr-Labs/go-sidecar/internal/storage" "github.com/Layr-Labs/go-sidecar/internal/types/numbers" "github.com/Layr-Labs/go-sidecar/internal/utils" + pkgUtils "github.com/Layr-Labs/go-sidecar/pkg/utils" "go.uber.org/zap" "golang.org/x/xerrors" "gorm.io/gorm" @@ -484,32 +485,9 @@ func (ss *StakerSharesModel) HandleStateChange(log *storage.TransactionLog) (int return nil, nil } -func (ss *StakerSharesModel) clonePreviousBlocksToNewBlock(blockNumber uint64) error { - query := ` - insert into staker_shares (staker, strategy, shares, block_number) - select - staker, - strategy, - shares, - @currentBlock as block_number - from staker_shares - where block_number = @previousBlock - ` - res := ss.DB.Exec(query, - sql.Named("currentBlock", blockNumber), - sql.Named("previousBlock", blockNumber-1), - ) - - if res.Error != nil { - ss.logger.Sugar().Errorw("Failed to clone previous block state to new block", zap.Error(res.Error)) - return res.Error - } - return nil -} - // prepareState prepares the state for commit by adding the new state to the existing state. -func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDiff, error) { - preparedState := make([]StakerSharesDiff, 0) +func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]*StakerSharesDiff, error) { + preparedState := make([]*StakerSharesDiff, 0) accumulatedState, ok := ss.stateAccumulator[blockNumber] if !ok { @@ -525,19 +503,28 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif // Find only the records from the previous block, that are modified in this block query := ` + with ranked_rows as ( + select + staker, + strategy, + shares, + block_number, + ROW_NUMBER() OVER (PARTITION BY staker, strategy ORDER BY block_number desc) as rn + from staker_shares + where + concat(staker, '_', strategy) in @slotIds + ) select - staker, - strategy, - shares - from staker_shares - where - block_number = @previousBlock - and concat(staker, '_', strategy) in @slotIds + rr.staker, + rr.strategy, + rr.shares, + rr.block_number + from ranked_rows as rr + where rn = 1 ` existingRecords := make([]StakerShares, 0) res := ss.DB.Model(&StakerShares{}). Raw(query, - sql.Named("previousBlock", blockNumber-1), sql.Named("slotIds", slotIds), ). Scan(&existingRecords) @@ -557,12 +544,11 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif // Loop over our new state changes. // If the record exists in the previous block, add the shares to the existing shares for slotId, newState := range accumulatedState { - prepared := StakerSharesDiff{ + prepared := &StakerSharesDiff{ Staker: newState.Staker, Strategy: newState.Strategy, Shares: newState.Shares, BlockNumber: blockNumber, - IsNew: false, } if existingRecord, ok := mappedRecords[slotId]; ok { @@ -577,9 +563,6 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif continue } prepared.Shares = existingShares.Add(existingShares, newState.Shares) - } else { - // SlotID was not found in the previous block, so this is a new record - prepared.IsNew = true } preparedState = append(preparedState, prepared) @@ -588,56 +571,27 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif } func (ss *StakerSharesModel) CommitFinalState(blockNumber uint64) error { - // Clone the previous block state to give us a reference point. - err := ss.clonePreviousBlocksToNewBlock(blockNumber) - if err != nil { - return err - } - records, err := ss.prepareState(blockNumber) if err != nil { return err } - newRecords := make([]StakerShares, 0) - updateRecords := make([]StakerShares, 0) - - for _, record := range records { - r := &StakerShares{ - Staker: record.Staker, - Strategy: record.Strategy, - Shares: record.Shares.String(), - BlockNumber: record.BlockNumber, - } - if record.IsNew { - newRecords = append(newRecords, *r) - } else { - updateRecords = append(updateRecords, *r) + recordsToInsert := pkgUtils.Map(records, func(r *StakerSharesDiff, i uint64) *StakerShares { + return &StakerShares{ + Staker: r.Staker, + Strategy: r.Strategy, + Shares: r.Shares.String(), + BlockNumber: blockNumber, } - } + }) - // Batch insert new records - if len(newRecords) > 0 { - res := ss.DB.Model(&StakerShares{}).Clauses(clause.Returning{}).Create(&newRecords) + if len(recordsToInsert) > 0 { + res := ss.DB.Model(&StakerShares{}).Clauses(clause.Returning{}).Create(&recordsToInsert) if res.Error != nil { ss.logger.Sugar().Errorw("Failed to create new operator_shares records", zap.Error(res.Error)) return res.Error } } - // Update existing records that were cloned from the previous block - if len(updateRecords) > 0 { - for _, record := range updateRecords { - res := ss.DB.Model(&StakerShares{}). - Where("staker = ? and strategy = ? and block_number = ?", record.Staker, record.Strategy, record.BlockNumber). - Updates(map[string]interface{}{ - "shares": record.Shares, - }) - if res.Error != nil { - ss.logger.Sugar().Errorw("Failed to update operator_shares record", zap.Error(res.Error)) - return res.Error - } - } - } return nil } @@ -662,7 +616,7 @@ func (ss *StakerSharesModel) GenerateStateRoot(blockNumber uint64) (types.StateR return types.StateRoot(utils.ConvertBytesToString(fullTree.Root())), nil } -func (ss *StakerSharesModel) sortValuesForMerkleTree(diffs []StakerSharesDiff) []*base.MerkleTreeInput { +func (ss *StakerSharesModel) sortValuesForMerkleTree(diffs []*StakerSharesDiff) []*base.MerkleTreeInput { inputs := make([]*base.MerkleTreeInput, 0) for _, diff := range diffs { inputs = append(inputs, &base.MerkleTreeInput{ diff --git a/internal/eigenState/stakerShares/stakerShares_test.go b/internal/eigenState/stakerShares/stakerShares_test.go index 996ae9eb..526aabb7 100644 --- a/internal/eigenState/stakerShares/stakerShares_test.go +++ b/internal/eigenState/stakerShares/stakerShares_test.go @@ -27,7 +27,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -313,9 +313,6 @@ func Test_StakerSharesState(t *testing.T) { assert.Equal(t, "0x298afb19a105d59e74658c4c334ff360bade6dd2", preparedChange[0].Strategy) assert.Equal(t, "246393621132195985", preparedChange[0].Shares.String()) - err = model.clonePreviousBlocksToNewBlock(blockNumber) - assert.Nil(t, err) - err = model.CommitFinalState(blockNumber) assert.Nil(t, err) diff --git a/internal/eigenState/stateManager/stateManager_test.go b/internal/eigenState/stateManager/stateManager_test.go index 8ed04672..e9437576 100644 --- a/internal/eigenState/stateManager/stateManager_test.go +++ b/internal/eigenState/stateManager/stateManager_test.go @@ -22,7 +22,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go b/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go index 9cae8350..34224a19 100644 --- a/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go +++ b/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go @@ -24,7 +24,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/indexer/restakedStrategies_test.go b/internal/indexer/restakedStrategies_test.go index 0ad0ae1c..021d5ccb 100644 --- a/internal/indexer/restakedStrategies_test.go +++ b/internal/indexer/restakedStrategies_test.go @@ -41,7 +41,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/pipeline/pipeline_integration_test.go b/internal/pipeline/pipeline_integration_test.go index c9b0b910..b4f64dc3 100644 --- a/internal/pipeline/pipeline_integration_test.go +++ b/internal/pipeline/pipeline_integration_test.go @@ -62,7 +62,7 @@ func setup() ( client := ethereum.NewClient(rpcUrl, l) // database - grm, err := tests.GetSqliteDatabaseConnection() + grm, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/python/lossyTokens.py b/internal/python/lossyTokens.py new file mode 100644 index 00000000..99fdf5f3 --- /dev/null +++ b/internal/python/lossyTokens.py @@ -0,0 +1,597 @@ +from decimal import Decimal, getcontext, ROUND_UP, ROUND_HALF_UP, ROUND_DOWN, ROUND_HALF_DOWN, ROUND_FLOOR + +def lossyAdd(tokens: str): + big_amount = float(tokens) + div = 0.999999999999999 + res = big_amount * div + + res_str = "{}".format(res) + return "{}".format(int(Decimal(res_str))) + +def amazonStakerTokenRewards(sp:str, tpd:str) -> str: + getcontext().prec = 15 + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + decimal_res = Decimal(stakerProportion * tokensPerDay) + + getcontext().prec = 20 + rounded = decimal_res.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + + +def nilStakerTokenRewards(sp:str, tpd:str) -> str: + print(getcontext().prec) + # getcontext().prec = 38 + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + print("{}, {}".format(stakerProportion, tokensPerDay)) + decimal_res = Decimal(stakerProportion * tokensPerDay) + print("Decimal: {}".format(decimal_res)) + truncated = decimal_res.quantize(Decimal('0.1'), rounding=ROUND_UP) + print("Truncated: {}".format(truncated)) + rounded = truncated.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + +def stakerTokenRewards(sp:str, tpd:str) -> str: + stakerProportion = float(sp) + tokensPerDay = int(tpd) + + decimal_res = stakerProportion * tokensPerDay + + parts = str(decimal_res).split("+") + # Need more precision + if len(parts) == 2 and int(parts[1]) > 16: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + getcontext().prec = 17 + getcontext().rounding = ROUND_DOWN + decimal_res = stakerProportion * tokensPerDay + + return "{}".format(int(decimal_res)) + + return "{}".format(int(decimal_res)) + +def stakerTokenRewardsAlt(sp:str, tpd:str) -> str: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + getcontext().prec = 17 + getcontext().rounding = ROUND_DOWN + print("{} {}".format(stakerProportion, tokensPerDay)) + decimal_res = stakerProportion * tokensPerDay + print("{}".format(decimal_res)) + + return "{}".format(int(decimal_res)) + + +def nileOperatorTokenRewards(totalStakerOperatorTokens:str) -> str: + if totalStakerOperatorTokens[-1] == "0": + return "{}".format(int(totalStakerOperatorTokens) // 10) + totalStakerOperatorTokens = Decimal(totalStakerOperatorTokens) + operatorTokens = Decimal(str(totalStakerOperatorTokens)) * Decimal(0.1) + rounded = operatorTokens.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + return "{}".format(rounded) + +#print(lossyAdd("1428571428571428571428571428571428571.4142857142857143")) +#print(lossyCalc("0.092173497467823", "142857142857142700000000")) + +numbers = [ + ["2262498437749998","226249843775000"], + ["1077804256999999","107780425700000"], + ["1200335945499999","120033594550000"], + ["2579821510499997","257982151050000"], + ["1077804256999999","107780425700000"], + ["97982205000000","9798220500000"], + ["2731616159749997","273161615975000"], + ["1468013852499999","146801385250000"], + ["1460746763499999","146074676350000"], + ["97982205000000","9798220500000"], + ["2160308386499998","216030838650000"], + ["2173632495999998","217363249600000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["97982205000000","9798220500000"], + ["1205181123749999","120518112375000"], + ["1692326232749998","169232623275000"], + ["19596441000000","1959644100000"], + ["1077804256999999","107780425700000"], + ["2766886966999997","276688696700000"], + ["1421109574749999","142110957475000"], + ["1327025256749999","132702525675000"], + ["1513152979749999","151315297975000"], + ["1724319284249998","172431928425000"], + ["1881072278499998","188107227850000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2767046562249997","276704656225000"], + ["1771544047249998","177154404725000"], + ["979822051749999","97982205175000"], + ["2766618839499997","276661883950000"], + ["1111499404999999","111149940500000"], + ["1256537372499999","125653737250000"], + ["1311309356749999","131130935675000"], + ["1077804256999999","107780425700000"], + ["1476639073749999","147663907375000"], + ["619368817749999","61936881775000"], + ["2351572924249998","235157292425000"], + ["1077804256999999","107780425700000"], + ["38824366794749960","3882436679474996"], + ["1329923441749999","132992344175000"], + ["1077804256999999","107780425700000"], + ["1171094469499999","117109446950000"], + ["1077804256999999","107780425700000"], + ["2164538729999998","216453873000000"], + ["2094265522499998","209426552250000"], + ["1448972310749999","144897231075000"], + ["1077804256999999","107780425700000"], + ["1403751361499999","140375136150000"], + ["979822051749999","97982205175000"], + ["979822051749999","97982205175000"], + ["1085802969749999","108580296975000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1124308426749999","112430842675000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1182901487499999","118290148750000"], + ["1372466011999999","137246601200000"], + ["979822051749999","97982205175000"], + ["1120320939499999","112032093950000"], + ["1077804256999999","107780425700000"], + ["2813743578249997","281374357825000"], + ["1997209544499998","199720954450000"], + ["342937718000000","34293771800000"], + ["1077804256999999","107780425700000"], + ["2504518724749998","250451872475000"], + ["489911025749999","48991102575000"], + ["1169357997749999","116935799775000"], + ["1077804256999999","107780425700000"], + ["1263158553749999","126315855375000"], + ["1094640640499999","109464064050000"], + ["1284065770249999","128406577025000"], + ["1521653686749998","152165368675000"], + ["1077804256999999","107780425700000"], + ["1695020089999998","169502009000000"], + ["1077804256999999","107780425700000"], + ["1089590665249999","108959066525000"], + ["4333353594249996","433335359425000"], + ["979822051749999","97982205175000"], + ["2766860861499997","276686086150000"], + ["3737419616999996","373741961700000"], + ["1682669036499998","168266903650000"], + ["979822051749999","97982205175000"], + ["718515954363749200","71851595436374920"], + ["1077804256999999","107780425700000"], + ["3350684550249997","335068455025000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766670760499997","276667076050000"], + ["1139779232999999","113977923300000"], + ["1294703182249999","129470318225000"], + ["1401943427499999","140194342750000"], + ["1077804256999999","107780425700000"], + ["8661942075499991","866194207549999"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["979822051749999","97982205175000"], + ["9798220500000","979822050000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1329616345749999","132961634575000"], + ["2285721732999998","228572173300000"], + ["440830750000","44083075000"], + ["1309869932249999","130986993225000"], + ["1077804256999999","107780425700000"], + ["2767143131249997","276714313125000"], + ["1087449895749999","108744989575000"], + ["1440582461749999","144058246175000"], + ["1437596903499999","143759690350000"], + ["3739760468499996","373976046850000"], + ["1228930342749999","122893034275000"], + ["1668502454499998","166850245450000"], + ["1077804256999999","107780425700000"], + ["20400709468749976","2040070946874998"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1501152824749998","150115282475000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1588788984249999","158878898425000"], + ["97982205000000","9798220500000"], + ["2766955102999997","276695510300000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1213777201499999","121377720150000"], + ["2766522236999998","276652223700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1518724180249999","151872418025000"], + ["1077804256999999","107780425700000"], + ["102040547000000","10204054700000"], + ["1521663646499999","152166364650000"], + ["2766883311999998","276688331200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1239913664499999","123991366450000"], + ["1083648329249999","108364832925000"], + ["2766567664499997","276656766450000"], + ["1077804256999999","107780425700000"], + ["121497934250000","12149793425000"], + ["1489329518749999","148932951875000"], + ["705471877249999","70547187725000"], + ["1365074446999999","136507444700000"], + ["148259503000000","14825950300000"], + ["120582017500000","12058201750000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["3458772690249997","345877269025000"], + ["1096846600499999","109684660050000"], + ["2642116312499998","264211631250000"], + ["1186049670749999","118604967075000"], + ["3379361777249997","337936177725000"], + ["1073027971999999","107302797200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2205003116999998","220500311700000"], + ["1400476333249999","140047633325000"], + ["1334754243499999","133475424350000"], + ["75000261135249920","7500026113524992"], + ["2766917160499997","276691716050000"], + ["1077804256999999","107780425700000"], + ["4081356354749996","408135635475000"], + ["115064965500000","11506496550000"], + ["140863986000000","14086398600000"], + ["1421544889999999","142154489000000"], + ["1249751425499999","124975142550000"], + ["1522711647499999","152271164750000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766928990999997","276692899100000"], + ["1251222885499999","125122288550000"], + ["1804670182999998","180467018300000"], + ["1286074936749999","128607493675000"], + ["22474595357499976","2247459535749998"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766772303499997","276677230350000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1636477811999998","163647781200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766759761749998","276675976175000"], + ["1077804256999999","107780425700000"], + ["2367891235249998","236789123525000"], + ["2111355676249998","211135567625000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1335070157749999","133507015775000"], + ["1088824402499999","108882440250000"], + ["1591863078499998","159186307850000"], + ["1093209052249999","109320905225000"], + ["1231725361499999","123172536150000"], + ["1077804256999999","107780425700000"], + ["1188849744499999","118884974450000"], + ["1325974469749999","132597446975000"], + ["1077804256999999","107780425700000"], + ["1170727959499999","117072795950000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766687807749997","276668780775000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1195382903249999","119538290325000"], + ["131740039000000","13174003900000"], + ["1077804256999999","107780425700000"], + ["1584454986249999","158445498625000"], + ["1496388154999999","149638815500000"], + ["2783380170999997","278338017100000"], + ["2766853466749997","276685346675000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1259437513249999","125943751325000"], + ["2559660094249998","255966009425000"], + ["97982205000000","9798220500000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["7828778194249992","782877819424999"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1138055195249999","113805519525000"], + ["979822051749999","97982205175000"], + ["1192936111999999","119293611200000"], + ["1117604526499999","111760452650000"], + ["1077804256999999","107780425700000"], + ["391928820500000","39192882050000"], + ["1671343943749998","167134394375000"], + ["1237619152499999","123761915250000"], + ["1461227295249999","146122729525000"], + ["536422250000","53642225000"], + ["1077804256999999","107780425700000"], + ["746783849749999","74678384975000"], + ["8686889065749991","868688906574999"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["3577907333749997","357790733375000"], + ["9798220500000","979822050000"], + ["2766688656499997","276668865650000"], + ["1077804256999999","107780425700000"], + ["1190360713999999","119036071400000"], + ["1372170432249999","137217043225000"], + ["1180097737749999","118009773775000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1231665739249999","123166573925000"], + ["1250928951499999","125092895150000"], + ["1077804256999999","107780425700000"], + ["1122446018749999","112244601875000"], + ["1077804256999999","107780425700000"], + ["3465020847749997","346502084775000"], + ["497502843250000","49750284325000"], + ["2766694622499997","276669462250000"], + ["1077804256999999","107780425700000"], + ["9799200250000","979920025000"], + ["1077804256999999","107780425700000"], + ["3393183871249997","339318387125000"], + ["1224777564749999","122477756475000"], + ["1077804256999999","107780425700000"], + ["2766784225749997","276678422575000"], + ["1084088666249999","108408866625000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["120671798250000","12067179825000"], + ["97982205000000","9798220500000"], + ["1207335820749999","120733582075000"], + ["979822051749999","97982205175000"], + ["1708717776499998","170871777650000"], + ["1077804256999999","107780425700000"], + ["2171666879249998","217166687925000"], + ["2766540531749997","276654053175000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1139514050249999","113951405025000"], + ["1347136987749999","134713698775000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2373897634749998","237389763475000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1296568008249999","129656800825000"], + ["1747592574499998","174759257450000"], + ["1718446376999998","171844637700000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1126212608249999","112621260825000"], + ["1382620828249999","138262082825000"], + ["1827529957249998","182752995725000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766997874749997","276699787475000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["3558586463999997","355858646400000"], + ["148139296000000","14813929600000"], + ["1228361717249999","122836171725000"], + ["1120021586249999","112002158625000"], + ["1077804256999999","107780425700000"], + ["75838000000","7583800000"], + ["2813176036999997","281317603700000"], + ["1126418676499999","112641867650000"], + ["1077804256999999","107780425700000"], + ["1168952628749999","116895262875000"], + ["1286002109999999","128600211000000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1106691471749999","110669147175000"], + ["1077804256999999","107780425700000"], + ["2767147036499997","276714703650000"], + ["2766894477499997","276689447750000"], + ["1077804256999999","107780425700000"], + ["1221210601249999","122121060125000"], + ["1077804256999999","107780425700000"], + ["1474944833499998","147494483350000"], + ["3355500944749997","335550094475000"], + ["36042871697249960","3604287169724996"], + ["1077804256999999","107780425700000"], + ["103861137250000","10386113725000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["7982080906749991","798208090674999"], + ["1077804256999999","107780425700000"], + ["6600292500000","660029250000"], + ["158317476750000","15831747675000"], + ["1077804256999999","107780425700000"], + ["1266130821999999","126613082200000"], + ["1087461430249999","108746143025000"], + ["1925894882749998","192589488275000"], + ["109489270750000","10948927075000"], + ["3011552248749997","301155224875000"], + ["5302500000","530250000"], + ["1077804256999999","107780425700000"], + ["1295497476749999","129549747675000"], + ["1514257759249998","151425775925000"], + ["1973412247749998","197341224775000"], + ["2612205589999997","261220559000000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1215310149499999","121531014950000"], + ["1077804256999999","107780425700000"], + ["1087895020499999","108789502050000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["147078692250000","14707869225000"], + ["2464874880999998","246487488100000"], + ["1166980098249999","116698009825000"], + ["734866538749999","73486653875000"], + ["2163704354999998","216370435500000"], + ["1281449592249999","128144959225000"], + ["1198409126249999","119840912625000"], + ["2767120898499997","276712089850000"], + ["1959088190749998","195908819075000"], + ["1077804256999999","107780425700000"], + ["1170951049999999","117095105000000"], + ["1154675041749999","115467504175000"], + ["1134384833499999","113438483350000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1194101009249999","119410100925000"], + ["1388429900249999","138842990025000"], + ["979822051749999","97982205175000"], + ["1122446018749999","112244601875000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1731852292499998","173185229250000"], + ["1396115605249999","139611560525000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1081755392249999","108175539225000"], + ["1115170505749999","111517050575000"], + ["1077804256999999","107780425700000"], + ["1880982576249998","188098257625000"], + ["1253732266249999","125373226625000"], + ["1077804256999999","107780425700000"], + ["117578912750000","11757891275000"], + ["2766816535749997","276681653575000"], + ["1151717784499999","115171778450000"], + ["1073440472499999","107344047250000"], + ["1842220677249998","184222067725000"], + ["1444291238499999","144429123850000"], + ["185920687750000","18592068775000"], + ["1077804256999999","107780425700000"], + ["2766739048249997","276673904825000"], + ["1077804256999999","107780425700000"], + ["1111127251249999","111112725125000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2495212757249998","249521275725000"], + ["1109533228999999","110953322900000"], + ["1204528088999999","120452808900000"], + ["2767240998749997","276724099875000"], + ["1077804256999999","107780425700000"], + ["27022701521749972","2702270152174997"], + ["1133721705999999","113372170600000"], + ["2766637369749997","276663736975000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["979822051749999","97982205175000"], + ["861929137749999","86192913775000"], + ["1077804256999999","107780425700000"], + ["3503727170499996","350372717050000"], + ["1060787189499999","106078718950000"], + ["979822051749999","97982205175000"], + ["2045249932249998","204524993225000"], + ["482354676250000","48235467625000"], + ["1077804256999999","107780425700000"], + ["1397360907249999","139736090725000"], + ["1175341173499999","117534117350000"], + ["2767044838249997","276704483825000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1449542205249999","144954220525000"], + ["1077804256999999","107780425700000"], + ["2321707502749998","232170750275000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1513531123249999","151353112325000"], + ["2740460193499997","274046019350000"], + ["1077804256999999","107780425700000"], + ["1149339404999999","114933940500000"], + ["1151809466249999","115180946625000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1249904385249999","124990438525000"], + ["1096106635999999","109610663600000"], + ["6898005302249993","689800530224999"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766392857999997","276639285800000"], + ["1077804256999999","107780425700000"], + ["2921368330249997","292136833025000"], + ["1369704807999999","136970480800000"], + ["1077804256999999","107780425700000"], + ["5006256564499995","500625656450000"], + ["1077804256999999","107780425700000"], + ["109740069750000","10974006975000"], + ["1077804256999999","107780425700000"], + ["1346236941999999","134623694200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1103601738249999","110360173825000"], + ["2704308862999998","270430886300000"], + ["864763585499999","86476358550000"], + ["979822051749999","97982205175000"], + ["1576305397999998","157630539800000"], + ["147299275500000","14729927550000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1321185449499999","132118544950000"], + ["1077804256999999","107780425700000"], + ["1244374005749999","124437400575000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["2175404280999998","217540428100000"], + ["2324262255249998","232426225525000"], + ["1087602477499999","108760247750000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1226466551749999","122646655175000"], + ["2797069106499997","279706910650000"], + ["1122461615499999","112246161550000"], + ["1156738466749999","115673846675000"], + ["5920104740249994","592010474024999"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["13604498250000","1360449825000"], + ["1367625821749999","136762582175000"], +] +issues = 0 +indexes = [] +#for i, number in enumerate(numbers): +for i, number in enumerate(numbers): + res = nileOperatorTokenRewards(number[0]) + if res != number[1]: + issues += 1 + indexes.append(i) + print("{}".format(number[0])) + print("Expected: {}".format(number[1])) + print("Got: {}".format(res)) + print("--------") +print("issues: {} {}".format(issues, indexes)) diff --git a/internal/sqlite/migrations/202409161057_avsOperatorDeltas/up.go b/internal/sqlite/migrations/202409161057_avsOperatorDeltas/up.go new file mode 100644 index 00000000..40c42647 --- /dev/null +++ b/internal/sqlite/migrations/202409161057_avsOperatorDeltas/up.go @@ -0,0 +1,36 @@ +package _202409161057_avsOperatorDeltas + +import ( + "fmt" + "gorm.io/gorm" +) + +type SqliteMigration struct { +} + +func (m *SqliteMigration) Up(grm *gorm.DB) error { + queries := []string{ + `create table if not exists avs_operator_state_changes ( + operator TEXT NOT NULL, + avs TEXT NOT NULL, + block_number INTEGER NOT NULL, + log_index INTEGER NOT NULL, + created_at DATETIME default current_timestamp, + registered integer not null, + unique(operator, avs, block_number, log_index) + ); + `, + } + + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to execute query: %s\n", query) + return res.Error + } + } + return nil +} + +func (m *SqliteMigration) GetName() string { + return "202409161057_avsOperatorDeltas" +} diff --git a/internal/sqlite/migrations/202409181340_stakerDelegationDelta/up.go b/internal/sqlite/migrations/202409181340_stakerDelegationDelta/up.go new file mode 100644 index 00000000..b93b4048 --- /dev/null +++ b/internal/sqlite/migrations/202409181340_stakerDelegationDelta/up.go @@ -0,0 +1,29 @@ +package _202409181340_stakerDelegationDelta + +import ( + "gorm.io/gorm" +) + +type SqliteMigration struct { +} + +func (m *SqliteMigration) Up(grm *gorm.DB) error { + query := ` + create table if not exists staker_delegation_changes ( + staker TEXT NOT NULL, + operator TEXT NOT NULL, + delegated INTEGER NOT NULL, + block_number INTEGER NOT NULL, + log_index INTEGER NOT NULL + ) + ` + res := grm.Exec(query) + if res.Error != nil { + return res.Error + } + return nil +} + +func (m *SqliteMigration) GetName() string { + return "202409181340_stakerDelegationDelta" +} diff --git a/internal/sqlite/migrations/202409191425_addRewardTypeColumn/up.go b/internal/sqlite/migrations/202409191425_addRewardTypeColumn/up.go new file mode 100644 index 00000000..4a3c52c3 --- /dev/null +++ b/internal/sqlite/migrations/202409191425_addRewardTypeColumn/up.go @@ -0,0 +1,60 @@ +package _202409191425_addRewardTypeColumn + +import ( + "gorm.io/gorm" +) + +type SqliteMigration struct { +} + +func (m *SqliteMigration) Up(grm *gorm.DB) error { + queries := []string{ + `create table if not exists reward_submissions_new ( + avs TEXT NOT NULL, + reward_hash TEST NOT NULL, + token TEXT NOT NULL, + amount TEXT NOT NULL, + strategy TEXT NOT NULL, + strategy_index INTEGER NOT NULL, + multiplier TEXT NOT NULL, + start_timestamp DATETIME NOT NULL, + end_timestamp DATETIME NOT NULL, + duration INTEGER NOT NULL, + block_number INTEGER NOT NULL, + reward_type string, + unique(reward_hash, strategy, block_number) + );`, + `insert into reward_submissions_new + select + avs, + reward_hash, + token, + amount, + strategy, + strategy_index, + multiplier, + start_timestamp, + end_timestamp, + duration, + block_number, + CASE is_for_all + WHEN 1 THEN 'all_stakers' + ELSE 'avs' + END as reward_type + from reward_submissions; + `, + `drop table reward_submissions;`, + `alter table reward_submissions_new rename to reward_submissions;`, + } + + for _, query := range queries { + if err := grm.Exec(query).Error; err != nil { + return err + } + } + return nil +} + +func (m *SqliteMigration) GetName() string { + return "202409191425_addRewardTypeColumn" +} diff --git a/internal/sqlite/migrations/migrator.go b/internal/sqlite/migrations/migrator.go index 332ceec6..065632ab 100644 --- a/internal/sqlite/migrations/migrator.go +++ b/internal/sqlite/migrations/migrator.go @@ -3,6 +3,9 @@ package migrations import ( "database/sql" "fmt" + _202409161057_avsOperatorDeltas "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409161057_avsOperatorDeltas" + _202409181340_stakerDelegationDelta "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409181340_stakerDelegationDelta" + _202409191425_addRewardTypeColumn "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409191425_addRewardTypeColumn" "time" _202409061249_bootstrapDb "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409061249_bootstrapDb" @@ -52,6 +55,9 @@ func (m *SqliteMigrator) MigrateAll() error { &_202409101144_submittedDistributionRoot.SqliteMigration{}, &_202409101540_rewardSubmissions.SqliteMigration{}, &_202409111509_removeOperatorRestakedStrategiesBlockConstraint.SqliteMigration{}, + &_202409161057_avsOperatorDeltas.SqliteMigration{}, + &_202409181340_stakerDelegationDelta.SqliteMigration{}, + &_202409191425_addRewardTypeColumn.SqliteMigration{}, } m.Logger.Sugar().Info("Running migrations") diff --git a/internal/sqlite/sqlite.go b/internal/sqlite/sqlite.go index 281d48f3..63d6177b 100644 --- a/internal/sqlite/sqlite.go +++ b/internal/sqlite/sqlite.go @@ -5,6 +5,9 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/Layr-Labs/go-sidecar/internal/types/numbers" + "github.com/shopspring/decimal" + "go.uber.org/zap" "os" "path/filepath" "regexp" @@ -42,12 +45,121 @@ func InitSqliteDir(path string) error { return nil } -func NewSqlite(path string) gorm.Dialector { - sql.Register("sqlite3_with_extensions", &goSqlite.SQLiteDriver{ - ConnectHook: func(conn *goSqlite.SQLiteConn) error { - return conn.RegisterFunc("bytes_to_hex", bytesToHex, true) - }, - }) +type SumBigNumbers struct { + total decimal.Decimal +} + +func NewSumBigNumbers() *SumBigNumbers { + zero, _ := decimal.NewFromString("0") + return &SumBigNumbers{total: zero} +} + +func (s *SumBigNumbers) Step(value any) { + bigValue, err := decimal.NewFromString(value.(string)) + if err != nil { + return + } + s.total = s.total.Add(bigValue) +} + +func (s *SumBigNumbers) Done() (string, error) { + return s.total.String(), nil +} + +var hasRegisteredExtensions = false + +const SqliteInMemoryPath = "file::memory:?cache=shared" + +func NewInMemorySqlite(l *zap.Logger) gorm.Dialector { + return NewSqlite(SqliteInMemoryPath, l) +} + +func NewInMemorySqliteWithName(name string, l *zap.Logger) gorm.Dialector { + path := fmt.Sprintf("file:%s?mode=memory&cache=shared", name) + return NewSqlite(path, l) +} + +func NewSqlite(path string, l *zap.Logger) gorm.Dialector { + if !hasRegisteredExtensions { + sql.Register("sqlite3_with_extensions", &goSqlite.SQLiteDriver{ + ConnectHook: func(conn *goSqlite.SQLiteConn) error { + // Generic functions + if err := conn.RegisterAggregator("sum_big", NewSumBigNumbers, true); err != nil { + l.Sugar().Errorw("Failed to register aggregator sum_big", "error", err) + return err + } + if err := conn.RegisterFunc("subtract_big", numbers.SubtractBig, true); err != nil { + l.Sugar().Errorw("Failed to register function subtract_big", "error", err) + return err + } + if err := conn.RegisterFunc("numeric_multiply", numbers.NumericMultiply, true); err != nil { + l.Sugar().Errorw("Failed to register function NumericMultiply", "error", err) + return err + } + if err := conn.RegisterFunc("big_gt", numbers.BigGreaterThan, true); err != nil { + l.Sugar().Errorw("Failed to register function BigGreaterThan", "error", err) + return err + } + + if err := conn.RegisterFunc("bytes_to_hex", bytesToHex, true); err != nil { + l.Sugar().Errorw("Failed to register function bytes_to_hex", "error", err) + return err + } + // Raw tokens per day + if err := conn.RegisterFunc("calc_raw_tokens_per_day", numbers.CalcRawTokensPerDay, true); err != nil { + l.Sugar().Errorw("Failed to register function calc_raw_tokens_per_day", "error", err) + return err + } + // Forked tokens per day + if err := conn.RegisterFunc("post_nile_tokens_per_day", numbers.PostNileTokensPerDay, true); err != nil { + l.Sugar().Errorw("Failed to register function PostNileTokensPerDay", "error", err) + return err + } + if err := conn.RegisterFunc("pre_nile_tokens_per_day", numbers.PreNileTokensPerDay, true); err != nil { + l.Sugar().Errorw("Failed to register function PreNileTokensPerDay", "error", err) + return err + } + // Staker proportion and weight + if err := conn.RegisterFunc("calc_staker_proportion", numbers.CalculateStakerProportion, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateStakerProportion", "error", err) + return err + } + if err := conn.RegisterFunc("calc_staker_weight", numbers.CalculateStakerWeight, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateStakerWeight", "error", err) + return err + } + // Forked rewards for stakers + if err := conn.RegisterFunc("amazon_token_rewards", numbers.CalculateAmazonStakerTokenRewards, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateAmazonStakerTokenRewards", "error", err) + return err + } + if err := conn.RegisterFunc("nile_token_rewards", numbers.CalculateNileStakerTokenRewards, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateNileStakerTokenRewards", "error", err) + return err + } + if err := conn.RegisterFunc("post_nile_token_rewards", numbers.CalculatePostNileStakerTokenRewards, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculatePostNileStakerTokenRewards", "error", err) + return err + } + // Operator tokens + if err := conn.RegisterFunc("amazon_operator_tokens", numbers.CalculateAmazonOperatorTokens, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateAmazonOperatorTokens", "error", err) + return err + } + if err := conn.RegisterFunc("nile_operator_tokens", numbers.CalculateNileOperatorTokens, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateNileOperatorTokens", "error", err) + return err + } + if err := conn.RegisterFunc("post_nile_operator_tokens", numbers.CalculatePostNileOperatorTokens, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculatePostNileOperatorTokens", "error", err) + return err + } + return nil + }, + }) + hasRegisteredExtensions = true + } + return &sqlite.Dialector{ DriverName: "sqlite3_with_extensions", DSN: path, diff --git a/internal/sqlite/sqlite_test.go b/internal/sqlite/sqlite_test.go index 736b02f9..b7f98500 100644 --- a/internal/sqlite/sqlite_test.go +++ b/internal/sqlite/sqlite_test.go @@ -2,12 +2,17 @@ package sqlite import ( "encoding/hex" + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/logger" "github.com/stretchr/testify/assert" + "math/big" "strings" "testing" ) func Test_Sqlite(t *testing.T) { + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: true}) + t.Run("Should use the bytesToHex function", func(t *testing.T) { query := ` with json_values as ( @@ -20,7 +25,7 @@ func Test_Sqlite(t *testing.T) { from json_values limit 1 ` - s := NewSqlite("file::memory:?cache=shared") + s := NewSqlite("file::memory:?cache=shared", l) grm, err := NewGormSqliteFromSqlite(s) assert.Nil(t, err) @@ -36,4 +41,69 @@ func Test_Sqlite(t *testing.T) { assert.Nil(t, res.Error) assert.Equal(t, strings.ToLower(hex.EncodeToString(expectedBytes)), hexValue.WithdrawalHex) }) + t.Run("Should sum two really big numbers that are stored as strings", func(t *testing.T) { + shares1 := "1670000000000000000000" + shares2 := "1670000000000000000000" + + type shares struct { + Shares string + } + + operatorShares := []*shares{ + &shares{ + Shares: shares1, + }, + &shares{ + Shares: shares2, + }, + } + + s := NewSqlite("file::memory:?cache=shared", l) + grm, err := NewGormSqliteFromSqlite(s) + assert.Nil(t, err) + + createQuery := ` + create table shares ( + shares TEXT NOT NULL + ) + ` + res := grm.Exec(createQuery) + assert.Nil(t, res.Error) + + res = grm.Model(&shares{}).Create(&operatorShares) + assert.Nil(t, res.Error) + + query := ` + select + sum_big(shares) as total + from shares + ` + var total string + res = grm.Raw(query).Scan(&total) + assert.Nil(t, res.Error) + fmt.Printf("Total: %s\n", total) + + shares1Big, _ := new(big.Int).SetString(shares1, 10) + shares2Big, _ := new(big.Int).SetString(shares2, 10) + + expectedTotal := shares1Big.Add(shares1Big, shares2Big) + + assert.Equal(t, expectedTotal.String(), total) + }) + t.Run("Custom functions", func(t *testing.T) { + t.Run("Should call calc_raw_tokens_per_day", func(t *testing.T) { + query := `select calc_raw_tokens_per_day('100', 100) as amt` + + s := NewSqlite("file::memory:?cache=shared", l) + grm, err := NewGormSqliteFromSqlite(s) + assert.Nil(t, err) + + var result string + res := grm.Raw(query).Scan(&result) + assert.Nil(t, res.Error) + + assert.Equal(t, "86400.0000000000080179", result) + + }) + }) } diff --git a/internal/storage/sqlite/sqlite_test.go b/internal/storage/sqlite/sqlite_test.go index 41e43cc3..7109afb1 100644 --- a/internal/storage/sqlite/sqlite_test.go +++ b/internal/storage/sqlite/sqlite_test.go @@ -20,7 +20,7 @@ import ( func setup() (*gorm.DB, *zap.Logger, *config.Config) { cfg := config.NewConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: true}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/tests/testdata/combinedRewards/README.md b/internal/tests/testdata/combinedRewards/README.md new file mode 100644 index 00000000..7433b824 --- /dev/null +++ b/internal/tests/testdata/combinedRewards/README.md @@ -0,0 +1,26 @@ +## Source query + +```sql +with filtered as ( + select * from dbt_testnet_holesky_rewards.rewards_combined + where block_time < '2024-09-17' +), +expanded as ( + select + f.avs, + f.reward_hash, + f.token, + f.amount::text as amount, + f.strategy, + f.strategy_index, + f.multiplier::text as multiplier, + f.start_timestamp, + f.end_timestamp, + f.reward_type, + f.duration, + f.block_number as block_number + from filtered as f +) +select * from expanded + +``` diff --git a/internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md b/internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md new file mode 100644 index 00000000..cd153efe --- /dev/null +++ b/internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md @@ -0,0 +1,31 @@ +## Source query: + +```sql +with filtered as ( + SELECT + lower(t.arguments #>> '{0,Value}') as operator, + lower(t.arguments #>> '{1,Value}') as avs, + (t.output_data -> 'status')::int as status, + t.transaction_hash, + t.log_index, + b.block_time, + to_char(b.block_time, 'YYYY-MM-DD') AS block_date, + t.block_number + FROM transaction_logs t + LEFT JOIN blocks b ON t.block_sequence_id = b.id + WHERE t.address = '0x055733000064333caddbc92763c58bf0192ffebf' + AND t.event_name = 'OperatorAVSRegistrationStatusUpdated' + AND date_trunc('day', b.block_time) < TIMESTAMP '2024-09-17' +) +select + operator, + avs, + status as registered, + log_index, + block_number +from filtered +``` + +## Expected results + +_See `generateExpectedResults.sql`_ diff --git a/internal/tests/testdata/operatorAvsRegistrationSnapshots/generateExpectedResults.sql b/internal/tests/testdata/operatorAvsRegistrationSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..3a338ea1 --- /dev/null +++ b/internal/tests/testdata/operatorAvsRegistrationSnapshots/generateExpectedResults.sql @@ -0,0 +1,67 @@ +COPY ( + with filtered as ( + select * from dbt_testnet_holesky_rewards.operator_avs_status + where block_time < '2024-09-17' +), +marked_statuses AS ( + SELECT + operator, + avs, + registered, + block_time, + block_date, + LEAD(block_time) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_time, + LEAD(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_registration_status, + LEAD(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_date, + LAG(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_registered, + LAG(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_block_date + FROM filtered +), + removed_same_day_deregistrations AS ( + SELECT * from marked_statuses + WHERE NOT ( + (registered = TRUE AND + COALESCE(next_registration_status = FALSE, false) AND + COALESCE(block_date = next_block_date, false)) OR + (registered = FALSE AND + COALESCE(prev_registered = TRUE, false) and + COALESCE(block_date = prev_block_date, false) + ) + ) + ), + registration_periods AS ( + SELECT + operator, + avs, + block_time AS start_time, + COALESCE(next_block_time, TIMESTAMP '2024-09-01') AS end_time, + registered + FROM removed_same_day_deregistrations + WHERE registered = TRUE + ), + registration_windows_extra as ( + SELECT + operator, + avs, + date_trunc('day', start_time) + interval '1' day as start_time, + date_trunc('day', end_time) as end_time + FROM registration_periods + ), + operator_avs_registration_windows as ( + SELECT * from registration_windows_extra + WHERE start_time != end_time + ), + cleaned_records AS ( + SELECT * FROM operator_avs_registration_windows + WHERE start_time < end_time + ), + final_results as ( + SELECT + operator, + avs, + day AS snapshot + FROM cleaned_records + CROSS JOIN generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day + ) + select * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER; diff --git a/internal/tests/testdata/operatorRestakedStrategies/README.md b/internal/tests/testdata/operatorRestakedStrategies/README.md new file mode 100644 index 00000000..ab0dad6d --- /dev/null +++ b/internal/tests/testdata/operatorRestakedStrategies/README.md @@ -0,0 +1,18 @@ +## Source + +```sql +select + block_number, + operator, + avs, + strategy, + block_time, + avs_directory_address +from operator_restaked_strategies +where avs_directory_address = '0x055733000064333caddbc92763c58bf0192ffebf' +and block_time < '2024-09-17' +``` + +## Expected results + +_See `generateExpectedResults.sql`_ diff --git a/internal/tests/testdata/operatorRestakedStrategies/generateExpectedResults.sql b/internal/tests/testdata/operatorRestakedStrategies/generateExpectedResults.sql new file mode 100644 index 00000000..5236cc2d --- /dev/null +++ b/internal/tests/testdata/operatorRestakedStrategies/generateExpectedResults.sql @@ -0,0 +1,112 @@ +copy (with ranked_records AS ( + SELECT + lower(operator) as operator, + lower(avs) as avs, + lower(strategy) as strategy, + block_time, + date_trunc('day', CAST(block_time as timestamp(6))) + interval '1' day as start_time, + ROW_NUMBER() OVER ( + PARTITION BY operator, avs, strategy, date_trunc('day', CAST(block_time as timestamp(6))) + interval '1' day + ORDER BY block_time DESC + ) AS rn + FROM public.operator_restaked_strategies + WHERE avs_directory_address = lower('0x055733000064333caddbc92763c58bf0192ffebf') + and block_time < '2024-09-17' +), + latest_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + block_time + FROM ranked_records + WHERE rn = 1 + ), + grouped_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + LEAD(start_time) OVER ( + PARTITION BY operator, avs, strategy + ORDER BY start_time ASC + ) AS next_start_time + FROM latest_records + ), + parsed_ranges AS ( + SELECT + operator, + avs, + strategy, + start_time, + CASE + WHEN next_start_time IS NULL OR next_start_time > start_time + INTERVAL '1' DAY THEN start_time + ELSE next_start_time + END AS end_time + FROM grouped_records + ), + active_windows as ( + SELECT * + FROM parsed_ranges + WHERE start_time != end_time + ), + gaps_and_islands AS ( + SELECT + operator, + avs, + strategy, + start_time, + end_time, + LAG(end_time) OVER(PARTITION BY operator, avs, strategy ORDER BY start_time) as prev_end_time + FROM active_windows + ), + island_detection AS ( + SELECT operator, avs, strategy, start_time, end_time, prev_end_time, + CASE + WHEN prev_end_time = start_time THEN 0 + ELSE 1 + END as new_island + FROM gaps_and_islands + ), + island_groups AS ( + SELECT + operator, + avs, + strategy, + start_time, + end_time, + SUM(new_island) OVER ( + PARTITION BY operator, avs, strategy ORDER BY start_time + ) AS island_id + FROM island_detection + ), + operator_avs_strategy_windows AS ( + SELECT + operator, + avs, + strategy, + MIN(start_time) AS start_time, + MAX(end_time) AS end_time + FROM island_groups + GROUP BY operator, avs, strategy, island_id + ORDER BY operator, avs, strategy, start_time + ), + cleaned_records AS ( + SELECT * FROM operator_avs_strategy_windows + WHERE start_time < end_time + ), + final_results as ( +SELECT + operator, + avs, + strategy, + cast(day AS DATE) AS snapshot +FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day +) +select * from final_results +) to STDOUT DELIMITER ',' CSV HEADER; diff --git a/internal/tests/testdata/operatorShareSnapshots/README.md b/internal/tests/testdata/operatorShareSnapshots/README.md new file mode 100644 index 00000000..641ef007 --- /dev/null +++ b/internal/tests/testdata/operatorShareSnapshots/README.md @@ -0,0 +1,17 @@ +## Source data + +```sql +select + operator, + strategy, + block_number, + sum(shares)::text as shares +from dbt_testnet_holesky_rewards.operator_shares +where block_time < '2024-09-17' +group by 1, 2, 3 +``` + +## Expected results + +_See `generateExpectedResults.sql`_ + diff --git a/internal/tests/testdata/operatorShareSnapshots/generateExpectedResults.sql b/internal/tests/testdata/operatorShareSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..9d49693f --- /dev/null +++ b/internal/tests/testdata/operatorShareSnapshots/generateExpectedResults.sql @@ -0,0 +1,47 @@ +COPY ( + WITH operator_shares as ( + select * + FROM dbt_testnet_holesky_rewards.operator_shares + where block_time < '2024-09-17' +), +ranked_operator_records as ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY operator, strategy, cast(block_time AS DATE) ORDER BY block_time DESC, log_index DESC) AS rn + FROM operator_shares +), + snapshotted_records as ( + SELECT + operator, + strategy, + shares, + block_time, + date_trunc('day', block_time) + INTERVAL '1' day as snapshot_time + from ranked_operator_records + where rn = 1 + ), + operator_share_windows as ( + SELECT + operator, strategy, shares, snapshot_time as start_time, + CASE + WHEN LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) is null THEN date_trunc('day', TIMESTAMP '2024-09-01') + ELSE LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records + ), + cleaned_records as ( + SELECT * FROM operator_share_windows + WHERE start_time < end_time + ), + final_results as ( + SELECT + operator, + strategy, + shares::text, + cast(day AS DATE) AS snapshot + FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day + ) + select * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER diff --git a/internal/tests/testdata/stakerDelegationSnapshots/README.md b/internal/tests/testdata/stakerDelegationSnapshots/README.md new file mode 100644 index 00000000..bd1609a8 --- /dev/null +++ b/internal/tests/testdata/stakerDelegationSnapshots/README.md @@ -0,0 +1,22 @@ +## Source + +```sql +SELECT + staker, + operator, + log_index, + block_number, + case when src = 'undelegations' THEN false ELSE true END AS delegated +FROM ( + SELECT *, 'undelegations' AS src FROM dbt_testnet_holesky_rewards.staker_undelegations + UNION ALL + SELECT *, 'delegations' AS src FROM dbt_testnet_holesky_rewards.staker_delegations + ) as delegations_combined +where block_time < '2024-09-17' + +``` + + +```bash +psql --host localhost --port 5435 --user blocklake --dbname blocklake --password < internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql > internal/tests/testdata/stakerDelegationSnapshots/expectedResults.csv +``` diff --git a/internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql b/internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..bfee6a53 --- /dev/null +++ b/internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql @@ -0,0 +1,47 @@ +COPY ( + with delegated_stakers as ( + select + * + from dbt_testnet_holesky_rewards.staker_delegation_status + where block_time < '2024-09-17' +), +ranked_delegations as ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, cast(block_time AS DATE) ORDER BY block_time DESC, log_index DESC) AS rn + FROM delegated_stakers +), + snapshotted_records as ( + SELECT + staker, + operator, + block_time, + date_trunc('day', block_time) + INTERVAL '1' day AS snapshot_time + from ranked_delegations + where rn = 1 + ), + staker_delegation_windows as ( + SELECT + staker, operator, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the cutoff date truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) is null THEN date_trunc('day', TIMESTAMP '2024-09-01') + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records + ), +cleaned_records as ( + SELECT * FROM staker_delegation_windows + WHERE start_time < end_time +), +final_results as ( + SELECT + staker, + operator, + cast(day AS DATE) AS snapshot +FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day +) +select * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER; diff --git a/internal/tests/testdata/stakerShareSnapshots/README.md b/internal/tests/testdata/stakerShareSnapshots/README.md new file mode 100644 index 00000000..b92c0b6d --- /dev/null +++ b/internal/tests/testdata/stakerShareSnapshots/README.md @@ -0,0 +1,15 @@ +## Source data + +```sql +select + staker, + strategy, + block_number, + sum(shares)::TEXT as shares +from dbt_testnet_holesky_rewards.staker_shares +group by 1, 2, 3 +``` + +## Expected results + +_See `generateExpectedResults.sql`_ diff --git a/internal/tests/testdata/stakerShareSnapshots/generateExpectedResults.sql b/internal/tests/testdata/stakerShareSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..f766612f --- /dev/null +++ b/internal/tests/testdata/stakerShareSnapshots/generateExpectedResults.sql @@ -0,0 +1,48 @@ +COPY ( + with staker_shares as ( + select + * + from dbt_testnet_holesky_rewards.staker_shares + where block_time < '2024-09-17' +), +ranked_staker_records as ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, strategy, cast(block_time AS DATE) ORDER BY block_time DESC, log_index DESC) AS rn + FROM staker_shares +), + snapshotted_records as ( + SELECT + staker, + strategy, + shares, + block_time, + date_trunc('day', block_time) + INTERVAL '1' day AS snapshot_time + from ranked_staker_records + where rn = 1 + ), + staker_share_windows as ( + SELECT + staker, strategy, shares, snapshot_time as start_time, + CASE + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) is null THEN date_trunc('day', TIMESTAMP '2024-09-01') + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records + ), + cleaned_records as ( + SELECT * FROM staker_share_windows + WHERE start_time < end_time +), +final_results as ( + SELECT + staker, + strategy, + shares::text, + cast(day AS DATE) AS snapshot + FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day + ) +SELECT * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER diff --git a/internal/tests/utils.go b/internal/tests/utils.go index b4b8af01..c217e76b 100644 --- a/internal/tests/utils.go +++ b/internal/tests/utils.go @@ -1,27 +1,55 @@ package tests import ( - "os" - + "fmt" "github.com/Layr-Labs/go-sidecar/internal/config" sqlite2 "github.com/Layr-Labs/go-sidecar/internal/sqlite" + "github.com/gocarina/gocsv" + "github.com/google/uuid" + "go.uber.org/zap" "gorm.io/gorm" + "os" + "path/filepath" + "strings" ) func GetConfig() *config.Config { return config.NewConfig() } -const sqliteInMemoryPath = "file::memory:?cache=shared" - -func GetSqliteDatabaseConnection() (*gorm.DB, error) { - db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(sqliteInMemoryPath)) +func GetInMemorySqliteDatabaseConnection(l *zap.Logger) (*gorm.DB, error) { + db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(sqlite2.SqliteInMemoryPath, l)) if err != nil { panic(err) } return db, nil } +func GetFileBasedSqliteDatabaseConnection(l *zap.Logger) (string, *gorm.DB, error) { + fileName, err := uuid.NewRandom() + if err != nil { + panic(err) + } + basePath := fmt.Sprintf("%s%s", os.TempDir(), fileName) + if err := os.MkdirAll(basePath, os.ModePerm); err != nil { + return "", nil, err + } + + filePath := fmt.Sprintf("%s/test.db", basePath) + fmt.Printf("File path: %s\n", filePath) + db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(filePath, l)) + if err != nil { + panic(err) + } + return filePath, db, nil +} + +func DeleteTestSqliteDB(filePath string) { + if err := os.Remove(filePath); err != nil { + panic(err) + } +} + func ReplaceEnv(newValues map[string]string, previousValues *map[string]string) { for k, v := range newValues { (*previousValues)[k] = os.Getenv(k) @@ -34,3 +62,132 @@ func RestoreEnv(previousValues map[string]string) { os.Setenv(k, v) } } + +func getTestdataPathFromProjectRoot(projectRoot string, fileName string) string { + p, err := filepath.Abs(fmt.Sprintf("%s/internal/tests/testdata%s", projectRoot, fileName)) + if err != nil { + panic(err) + } + return p +} + +func getSqlFile(filePath string) (string, error) { + contents, err := os.ReadFile(filePath) + + if err != nil { + return "", err + } + + return strings.Trim(string(contents), "\n"), nil +} +func getExpectedResultsCsvFile[T any](filePath string) ([]*T, error) { + results := make([]*T, 0) + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + if err := gocsv.UnmarshalFile(file, &results); err != nil { + panic(err) + } + return results, nil +} + +func GetAllBlocksSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/allBlocks.sql") + fmt.Printf("Path: %v\n", path) + return getSqlFile(path) +} + +func GetOperatorAvsRegistrationsSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorAvsRegistrationSnapshots/operatorAvsRegistrations.sql") + return getSqlFile(path) +} + +type ExpectedOperatorAvsRegistrationSnapshot struct { + Operator string `csv:"operator"` + Avs string `csv:"avs"` + Snapshot string `csv:"snapshot"` +} + +func GetExpectedOperatorAvsSnapshotResults(projectBase string) ([]*ExpectedOperatorAvsRegistrationSnapshot, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorAvsRegistrationSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[ExpectedOperatorAvsRegistrationSnapshot](path) +} + +func GetOperatorAvsRestakedStrategiesSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorRestakedStrategies/operatorRestakedStrategies.sql") + return getSqlFile(path) +} + +type ExpectedOperatorAvsSnapshot struct { + Operator string `csv:"operator"` + Avs string `csv:"avs"` + Strategy string `csv:"strategy"` + Snapshot string `csv:"snapshot"` +} + +func GetExpectedOperatorAvsSnapshots(projectBase string) ([]*ExpectedOperatorAvsSnapshot, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorRestakedStrategies/expectedResults.csv") + return getExpectedResultsCsvFile[ExpectedOperatorAvsSnapshot](path) +} + +// OperatorShares snapshots +func GetOperatorSharesSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorShareSnapshots/operatorShares.sql") + return getSqlFile(path) +} + +type OperatorShareExpectedResult struct { + Operator string `csv:"operator"` + Strategy string `csv:"strategy"` + Snapshot string `csv:"snapshot"` + Shares string `csv:"shares"` +} + +func GetOperatorSharesExpectedResults(projectBase string) ([]*OperatorShareExpectedResult, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorShareSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[OperatorShareExpectedResult](path) +} + +// StakerShareSnapshots +func GetStakerSharesSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerShareSnapshots/stakerShares.sql") + return getSqlFile(path) +} + +type StakerShareExpectedResult struct { + Staker string `csv:"staker"` + Strategy string `csv:"strategy"` + Snapshot string `csv:"snapshot"` + Shares string `csv:"shares"` +} + +func GetStakerSharesExpectedResults(projectBase string) ([]*StakerShareExpectedResult, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerShareSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[StakerShareExpectedResult](path) +} + +// StakerDelegationSnapshots +func GetStakerDelegationsSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerDelegationSnapshots/stakerDelegations.sql") + return getSqlFile(path) +} + +type StakerDelegationExpectedResult struct { + Staker string `csv:"staker"` + Operator string `csv:"operator"` + Snapshot string `csv:"snapshot"` +} + +func GetStakerDelegationExpectedResults(projectBase string) ([]*StakerDelegationExpectedResult, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerDelegationSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[StakerDelegationExpectedResult](path) +} + +// CombinedRewards +func GetCombinedRewardsSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/combinedRewards/combinedRewards.sql") + return getSqlFile(path) +} diff --git a/internal/types/numbers/numbers.go b/internal/types/numbers/numbers.go index 10e481f9..95e9e5b5 100644 --- a/internal/types/numbers/numbers.go +++ b/internal/types/numbers/numbers.go @@ -1,6 +1,11 @@ package numbers -import "math/big" +import "C" +import ( + "fmt" + "github.com/shopspring/decimal" + "math/big" +) // NewBig257 returns a new big.Int with a size of 257 bits // This allows us to fully support math on uint256 numbers @@ -8,3 +13,100 @@ import "math/big" func NewBig257() *big.Int { return big.NewInt(257) } + +// NumericMultiply take two huge numbers, stored as strings, and multiplies them +func NumericMultiply(a, b string) (string, error) { + na, err := decimal.NewFromString(a) + if err != nil { + return "", err + } + nb, err := decimal.NewFromString(b) + if err != nil { + return "", err + } + + return na.Mul(nb).String(), nil +} + +func SubtractBig(a, b string) (string, error) { + na, err := decimal.NewFromString(a) + if err != nil { + return "", err + } + nb, err := decimal.NewFromString(b) + if err != nil { + return "", err + } + + return na.Sub(nb).String(), nil +} + +func BigGreaterThan(a, b string) (bool, error) { + na, err := decimal.NewFromString(a) + if err != nil { + return false, err + } + nb, err := decimal.NewFromString(b) + if err != nil { + return false, err + } + + return na.GreaterThan(nb), nil +} + +// CalcRawTokensPerDay calculates the raw tokens per day for a given amount and duration +// Returns the raw tokens per day in decimal format as a string +func CalcRawTokensPerDay(amountStr string, duration uint64) (string, error) { + amount, err := decimal.NewFromString(amountStr) + if err != nil { + fmt.Printf("CalcRawTokensPerDay Error: %s\n", err) + return "", err + } + + rawTokensPerDay := amount.Div(decimal.NewFromFloat(float64(duration) / 86400)) + + return rawTokensPerDay.String(), nil +} + +// PostNileTokensPerDay calculates the tokens per day for post-nile rewards +// Simply truncates the decimal portion of the of the raw tokens per day +func PostNileTokensPerDay(tokensPerDay string) (string, error) { + fmt.Printf("PostNileTokensPerDay: %s\n", tokensPerDay) + tpd, err := decimal.NewFromString(tokensPerDay) + if err != nil { + fmt.Printf("PostNileTokensPerDay Error: %s\n", err) + return "", err + } + + return tpd.BigInt().String(), nil +} + +// CalculateStakerProportion calculates the staker weight for a given staker and total weight +func CalculateStakerProportion(stakerWeightStr string, totalWeightStr string) (string, error) { + stakerWeight, err := decimal.NewFromString(stakerWeightStr) + if err != nil { + return "", err + } + totalWeight, err := decimal.NewFromString(totalWeightStr) + if err != nil { + return "", err + } + + res := ((stakerWeight.Div(totalWeight)).Mul(decimal.NewFromInt(1000000000000000))). + Div(decimal.NewFromInt(1000000000000000)). + Floor() + return res.String(), nil +} + +func CalculateStakerWeight(multiplier string, shares string) (string, error) { + m, err := decimal.NewFromString(multiplier) + if err != nil { + return "", err + } + s, err := decimal.NewFromString(shares) + if err != nil { + return "", err + } + + return m.Mul(s).String(), nil +} diff --git a/internal/types/numbers/numbersData_test.go b/internal/types/numbers/numbersData_test.go new file mode 100644 index 00000000..3fcc00c2 --- /dev/null +++ b/internal/types/numbers/numbersData_test.go @@ -0,0 +1,559 @@ +package numbers + +var ( + operatorAmazonTokens = [][]string{ + []string{"2262498437750000", "226249843775000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1200335945500000", "120033594550000"}, + []string{"2579821510500000", "257982151050000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2731616159750000", "273161615975000"}, + []string{"1468013852500000", "146801385250000"}, + []string{"1460746763500000", "146074676350000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2160308386500000", "216030838650000"}, + []string{"2173632496000000", "217363249600000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1205181123750000", "120518112375000"}, + []string{"1692326232750000", "169232623275000"}, + []string{"19596441000000", "1959644100000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"2766886967000000", "276688696700000"}, + []string{"1421109574750000", "142110957475000"}, + []string{"1327025256750000", "132702525675000"}, + []string{"1513152979750000", "151315297975000"}, + []string{"1724319284250000", "172431928425000"}, + []string{"1881072278500000", "188107227850000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"2767046562250000", "276704656225000"}, + []string{"1771544047250000", "177154404725000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2766618839500000", "276661883950000"}, + []string{"1111499405000000", "111149940500000"}, + []string{"1256537372500000", "125653737250000"}, + []string{"1311309356750000", "131130935675000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1476639073750000", "147663907375000"}, + []string{"619368817749999", "61936881775000"}, + []string{"2351572924250000", "235157292425000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"38824366794750000", "3882436679475000"}, + []string{"1329923441750000", "132992344175000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1171094469500000", "117109446950000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"2164538730000000", "216453873000000"}, + []string{"2094265522500000", "209426552250000"}, + []string{"1448972310750000", "144897231075000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1403751361500000", "140375136150000"}, + []string{"979822051749999", "97982205175000"}, + } + + operatorNileTokens = [][]string{ + []string{"2262498437749998", "226249843775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1200335945499999", "120033594550000"}, + []string{"2579821510499997", "257982151050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2731616159749997", "273161615975000"}, + []string{"1468013852499999", "146801385250000"}, + []string{"1460746763499999", "146074676350000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2160308386499998", "216030838650000"}, + []string{"2173632495999998", "217363249600000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1205181123749999", "120518112375000"}, + []string{"1692326232749998", "169232623275000"}, + []string{"19596441000000", "1959644100000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766886966999997", "276688696700000"}, + []string{"1421109574749999", "142110957475000"}, + []string{"1327025256749999", "132702525675000"}, + []string{"1513152979749999", "151315297975000"}, + []string{"1724319284249998", "172431928425000"}, + []string{"1881072278499998", "188107227850000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2767046562249997", "276704656225000"}, + []string{"1771544047249998", "177154404725000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2766618839499997", "276661883950000"}, + []string{"1111499404999999", "111149940500000"}, + []string{"1256537372499999", "125653737250000"}, + []string{"1311309356749999", "131130935675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1476639073749999", "147663907375000"}, + []string{"619368817749999", "61936881775000"}, + []string{"2351572924249998", "235157292425000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"38824366794749960", "3882436679474996"}, + []string{"1329923441749999", "132992344175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1171094469499999", "117109446950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2164538729999998", "216453873000000"}, + []string{"2094265522499998", "209426552250000"}, + []string{"1448972310749999", "144897231075000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1403751361499999", "140375136150000"}, + []string{"979822051749999", "97982205175000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1085802969749999", "108580296975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1124308426749999", "112430842675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1182901487499999", "118290148750000"}, + []string{"1372466011999999", "137246601200000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1120320939499999", "112032093950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2813743578249997", "281374357825000"}, + []string{"1997209544499998", "199720954450000"}, + []string{"342937718000000", "34293771800000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2504518724749998", "250451872475000"}, + []string{"489911025749999", "48991102575000"}, + []string{"1169357997749999", "116935799775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1263158553749999", "126315855375000"}, + []string{"1094640640499999", "109464064050000"}, + []string{"1284065770249999", "128406577025000"}, + []string{"1521653686749998", "152165368675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1695020089999998", "169502009000000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1089590665249999", "108959066525000"}, + []string{"4333353594249996", "433335359425000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2766860861499997", "276686086150000"}, + []string{"3737419616999996", "373741961700000"}, + []string{"1682669036499998", "168266903650000"}, + []string{"979822051749999", "97982205175000"}, + []string{"718515954363749200", "71851595436374920"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3350684550249997", "335068455025000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766670760499997", "276667076050000"}, + []string{"1139779232999999", "113977923300000"}, + []string{"1294703182249999", "129470318225000"}, + []string{"1401943427499999", "140194342750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"8661942075499991", "866194207549999"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"979822051749999", "97982205175000"}, + []string{"9798220500000", "979822050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1329616345749999", "132961634575000"}, + []string{"2285721732999998", "228572173300000"}, + []string{"440830750000", "44083075000"}, + []string{"1309869932249999", "130986993225000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2767143131249997", "276714313125000"}, + []string{"1087449895749999", "108744989575000"}, + []string{"1440582461749999", "144058246175000"}, + []string{"1437596903499999", "143759690350000"}, + []string{"3739760468499996", "373976046850000"}, + []string{"1228930342749999", "122893034275000"}, + []string{"1668502454499998", "166850245450000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"20400709468749976", "2040070946874998"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1501152824749998", "150115282475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1588788984249999", "158878898425000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2766955102999997", "276695510300000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1213777201499999", "121377720150000"}, + []string{"2766522236999998", "276652223700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1518724180249999", "151872418025000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"102040547000000", "10204054700000"}, + []string{"1521663646499999", "152166364650000"}, + []string{"2766883311999998", "276688331200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1239913664499999", "123991366450000"}, + []string{"1083648329249999", "108364832925000"}, + []string{"2766567664499997", "276656766450000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"121497934250000", "12149793425000"}, + []string{"1489329518749999", "148932951875000"}, + []string{"705471877249999", "70547187725000"}, + []string{"1365074446999999", "136507444700000"}, + []string{"148259503000000", "14825950300000"}, + []string{"120582017500000", "12058201750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3458772690249997", "345877269025000"}, + []string{"1096846600499999", "109684660050000"}, + []string{"2642116312499998", "264211631250000"}, + []string{"1186049670749999", "118604967075000"}, + []string{"3379361777249997", "337936177725000"}, + []string{"1073027971999999", "107302797200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2205003116999998", "220500311700000"}, + []string{"1400476333249999", "140047633325000"}, + []string{"1334754243499999", "133475424350000"}, + []string{"75000261135249920", "7500026113524992"}, + []string{"2766917160499997", "276691716050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"4081356354749996", "408135635475000"}, + []string{"115064965500000", "11506496550000"}, + []string{"140863986000000", "14086398600000"}, + []string{"1421544889999999", "142154489000000"}, + []string{"1249751425499999", "124975142550000"}, + []string{"1522711647499999", "152271164750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766928990999997", "276692899100000"}, + []string{"1251222885499999", "125122288550000"}, + []string{"1804670182999998", "180467018300000"}, + []string{"1286074936749999", "128607493675000"}, + []string{"22474595357499976", "2247459535749998"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766772303499997", "276677230350000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1636477811999998", "163647781200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766759761749998", "276675976175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2367891235249998", "236789123525000"}, + []string{"2111355676249998", "211135567625000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1335070157749999", "133507015775000"}, + []string{"1088824402499999", "108882440250000"}, + []string{"1591863078499998", "159186307850000"}, + []string{"1093209052249999", "109320905225000"}, + []string{"1231725361499999", "123172536150000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1188849744499999", "118884974450000"}, + []string{"1325974469749999", "132597446975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1170727959499999", "117072795950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766687807749997", "276668780775000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1195382903249999", "119538290325000"}, + []string{"131740039000000", "13174003900000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1584454986249999", "158445498625000"}, + []string{"1496388154999999", "149638815500000"}, + []string{"2783380170999997", "278338017100000"}, + []string{"2766853466749997", "276685346675000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1259437513249999", "125943751325000"}, + []string{"2559660094249998", "255966009425000"}, + []string{"97982205000000", "9798220500000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"7828778194249992", "782877819424999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1138055195249999", "113805519525000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1192936111999999", "119293611200000"}, + []string{"1117604526499999", "111760452650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"391928820500000", "39192882050000"}, + []string{"1671343943749998", "167134394375000"}, + []string{"1237619152499999", "123761915250000"}, + []string{"1461227295249999", "146122729525000"}, + []string{"536422250000", "53642225000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"746783849749999", "74678384975000"}, + []string{"8686889065749991", "868688906574999"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3577907333749997", "357790733375000"}, + []string{"9798220500000", "979822050000"}, + []string{"2766688656499997", "276668865650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1190360713999999", "119036071400000"}, + []string{"1372170432249999", "137217043225000"}, + []string{"1180097737749999", "118009773775000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1231665739249999", "123166573925000"}, + []string{"1250928951499999", "125092895150000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1122446018749999", "112244601875000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3465020847749997", "346502084775000"}, + []string{"497502843250000", "49750284325000"}, + []string{"2766694622499997", "276669462250000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"9799200250000", "979920025000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3393183871249997", "339318387125000"}, + []string{"1224777564749999", "122477756475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766784225749997", "276678422575000"}, + []string{"1084088666249999", "108408866625000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"120671798250000", "12067179825000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1207335820749999", "120733582075000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1708717776499998", "170871777650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2171666879249998", "217166687925000"}, + []string{"2766540531749997", "276654053175000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1139514050249999", "113951405025000"}, + []string{"1347136987749999", "134713698775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2373897634749998", "237389763475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1296568008249999", "129656800825000"}, + []string{"1747592574499998", "174759257450000"}, + []string{"1718446376999998", "171844637700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1126212608249999", "112621260825000"}, + []string{"1382620828249999", "138262082825000"}, + []string{"1827529957249998", "182752995725000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766997874749997", "276699787475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3558586463999997", "355858646400000"}, + []string{"148139296000000", "14813929600000"}, + []string{"1228361717249999", "122836171725000"}, + []string{"1120021586249999", "112002158625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"75838000000", "7583800000"}, + []string{"2813176036999997", "281317603700000"}, + []string{"1126418676499999", "112641867650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1168952628749999", "116895262875000"}, + []string{"1286002109999999", "128600211000000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1106691471749999", "110669147175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2767147036499997", "276714703650000"}, + []string{"2766894477499997", "276689447750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1221210601249999", "122121060125000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1474944833499998", "147494483350000"}, + []string{"3355500944749997", "335550094475000"}, + []string{"36042871697249960", "3604287169724996"}, + []string{"1077804256999999", "107780425700000"}, + []string{"103861137250000", "10386113725000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"7982080906749991", "798208090674999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"6600292500000", "660029250000"}, + []string{"158317476750000", "15831747675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1266130821999999", "126613082200000"}, + []string{"1087461430249999", "108746143025000"}, + []string{"1925894882749998", "192589488275000"}, + []string{"109489270750000", "10948927075000"}, + []string{"3011552248749997", "301155224875000"}, + []string{"5302500000", "530250000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1295497476749999", "129549747675000"}, + []string{"1514257759249998", "151425775925000"}, + []string{"1973412247749998", "197341224775000"}, + []string{"2612205589999997", "261220559000000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1215310149499999", "121531014950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1087895020499999", "108789502050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"147078692250000", "14707869225000"}, + []string{"2464874880999998", "246487488100000"}, + []string{"1166980098249999", "116698009825000"}, + []string{"734866538749999", "73486653875000"}, + []string{"2163704354999998", "216370435500000"}, + []string{"1281449592249999", "128144959225000"}, + []string{"1198409126249999", "119840912625000"}, + []string{"2767120898499997", "276712089850000"}, + []string{"1959088190749998", "195908819075000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1170951049999999", "117095105000000"}, + []string{"1154675041749999", "115467504175000"}, + []string{"1134384833499999", "113438483350000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1194101009249999", "119410100925000"}, + []string{"1388429900249999", "138842990025000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1122446018749999", "112244601875000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1731852292499998", "173185229250000"}, + []string{"1396115605249999", "139611560525000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1081755392249999", "108175539225000"}, + []string{"1115170505749999", "111517050575000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1880982576249998", "188098257625000"}, + []string{"1253732266249999", "125373226625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"117578912750000", "11757891275000"}, + []string{"2766816535749997", "276681653575000"}, + []string{"1151717784499999", "115171778450000"}, + []string{"1073440472499999", "107344047250000"}, + []string{"1842220677249998", "184222067725000"}, + []string{"1444291238499999", "144429123850000"}, + []string{"185920687750000", "18592068775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766739048249997", "276673904825000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1111127251249999", "111112725125000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2495212757249998", "249521275725000"}, + []string{"1109533228999999", "110953322900000"}, + []string{"1204528088999999", "120452808900000"}, + []string{"2767240998749997", "276724099875000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"27022701521749972", "2702270152174997"}, + []string{"1133721705999999", "113372170600000"}, + []string{"2766637369749997", "276663736975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"979822051749999", "97982205175000"}, + []string{"861929137749999", "86192913775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3503727170499996", "350372717050000"}, + []string{"1060787189499999", "106078718950000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2045249932249998", "204524993225000"}, + []string{"482354676250000", "48235467625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1397360907249999", "139736090725000"}, + []string{"1175341173499999", "117534117350000"}, + []string{"2767044838249997", "276704483825000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1449542205249999", "144954220525000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2321707502749998", "232170750275000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1513531123249999", "151353112325000"}, + []string{"2740460193499997", "274046019350000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1149339404999999", "114933940500000"}, + []string{"1151809466249999", "115180946625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1249904385249999", "124990438525000"}, + []string{"1096106635999999", "109610663600000"}, + []string{"6898005302249993", "689800530224999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766392857999997", "276639285800000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2921368330249997", "292136833025000"}, + []string{"1369704807999999", "136970480800000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"5006256564499995", "500625656450000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"109740069750000", "10974006975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1346236941999999", "134623694200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1103601738249999", "110360173825000"}, + []string{"2704308862999998", "270430886300000"}, + []string{"864763585499999", "86476358550000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1576305397999998", "157630539800000"}, + []string{"147299275500000", "14729927550000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1321185449499999", "132118544950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1244374005749999", "124437400575000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2175404280999998", "217540428100000"}, + []string{"2324262255249998", "232426225525000"}, + []string{"1087602477499999", "108760247750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1226466551749999", "122646655175000"}, + []string{"2797069106499997", "279706910650000"}, + []string{"1122461615499999", "112246161550000"}, + []string{"1156738466749999", "115673846675000"}, + []string{"5920104740249994", "592010474024999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"13604498250000", "1360449825000"}, + []string{"1367625821749999", "136762582175000"}, + } +) diff --git a/internal/types/numbers/numbers_test.go b/internal/types/numbers/numbers_test.go index 681a0f88..7e7cb82d 100644 --- a/internal/types/numbers/numbers_test.go +++ b/internal/types/numbers/numbers_test.go @@ -1,12 +1,49 @@ package numbers import ( + "fmt" + "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" + "math" "math/big" "testing" ) -func Test_numbers(t *testing.T) { +func roundToEven(f *big.Float) *big.Float { + // Get the integer part of the float + intPart, _ := f.Int(nil) + + // Create a big.Float from the integer part + rounded := new(big.Float).SetInt(intPart) + + // Calculate the difference (fractional part) + diff := new(big.Float).Sub(f, rounded) + + // Create a big.Float representing 0.5 + half := big.NewFloat(0.5) + + // Check if the fractional part is exactly 0.5 + if diff.Cmp(half) == 0 { + // Check if the integer part is even + intPartMod := new(big.Int).Mod(intPart, big.NewInt(2)) + if intPartMod.Cmp(big.NewInt(0)) != 0 { + // If odd, round up + rounded.Add(rounded, big.NewFloat(1)) + } + } else if diff.Cmp(half) > 0 { + // If the fractional part is greater than 0.5, round up + rounded.Add(rounded, big.NewFloat(1)) + } + + return rounded +} + +func Test_Numbers(t *testing.T) { + if err := InitPython(); err != nil { + t.Error(err) + } + defer FinalizePython() + t.Run("Test that big.Int can produce negative numbers", func(t *testing.T) { startingNum := big.Int{} startingNum.SetString("10", 10) @@ -45,4 +82,778 @@ func Test_numbers(t *testing.T) { assert.Equal(t, "13389173345999999999999980", amountToSubtract.Sub(startingNum, amountToSubtract).String()) }) + t.Run("Test floating point math to see if its correct", func(t *testing.T) { + t.Skip("math/big is not accurate enough for this test") + duration := float64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedRawTokensPerDay := "1428571428571428571428571428571428571" + //expectedFloorValue := "1428571428571428571428571428571428571" + //expectedRoundedValue := "1428571428571427000000000000000000000" + // 1428571428571428500000000000000000000 + + amount, _ := NewBig257().SetString(amountStr, 10) + //amount, _, err := big.ParseFloat("99999999999999999999999999999999999999", 10, 38, big.ToPositiveInf) + //assert.Nil(t, err) + + fmt.Printf("Amount: %+v\n", amount.String()) + + amountFloat := big.NewFloat(0) + amountFloat.SetInt(amount) + amountFloat.SetMode(big.ToNearestEven) + //amountFloat.SetPrec(256) + + divisor := big.NewFloat(float64(duration) / 86400) + divisor.SetMode(big.ToNearestEven) + fmt.Printf("Divisor: %+v\n", divisor.String()) + tokensPerDay := amountFloat.Quo(amountFloat, divisor) + tokensPerDay = roundToEven(tokensPerDay) + + rawTokensPerDay := tokensPerDay.Text('f', 0) + assert.Equal(t, expectedRawTokensPerDay, rawTokensPerDay) + + // We use floor to ensure we are always underesimating total tokens per day + tokensPerDayFloored := NewBig257() + tokensPerDayFloored, _ = tokensPerDay.Int(tokensPerDayFloored) + + precision := (math.Pow(10, 15) - float64(1)) / math.Pow(10, 15) + + // Round down to 15 sigfigs for double precision, ensuring know errouneous round up or down + tokensPerDayDecimal := tokensPerDay.Mul(tokensPerDay, big.NewFloat(precision)) + + fmt.Printf("tokensPerDayFloored: %s\n", tokensPerDayFloored.String()) + fmt.Printf("tokensPerDayDecimal: %s\n", tokensPerDayDecimal.String()) + + }) + t.Run("Test floating point math to see if its correct with shopspring/decimal", func(t *testing.T) { + duration := float64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedRawTokensPerDay := "1428571428571428571428571428571428571" + expectedFloorValue := "1428571428571428571428571428571428571" + // expectedRoundedValue := "1428571428571427000000000000000000000" + // 1428571428571428500000000000000000000 + + amount, err := decimal.NewFromString(amountStr) + assert.Nil(t, err) + + fmt.Printf("Amount: %+v\n", amount.String()) + + rawTokensPerDay := amount.Div(decimal.NewFromFloat(duration / 86400)) + fmt.Printf("Raw tokens per day: %+v\n", rawTokensPerDay.String()) + // rawTokensPerDayForRounding := amount.Div(decimal.NewFromFloat(duration / 86400)) + + assert.Equal(t, expectedRawTokensPerDay, rawTokensPerDay.BigInt().String()) + + tokensPerDayFloored := rawTokensPerDay.BigInt() + + assert.Equal(t, expectedFloorValue, tokensPerDayFloored.String()) + + one := (decimal.NewFromInt(10).Pow(decimal.NewFromInt(15))).Sub(decimal.NewFromInt(1)) + two := decimal.NewFromInt(10).Pow(decimal.NewFromInt(15)) + + fmt.Printf("one/two :%v\n", one.Div(two).String()) + assert.Equal(t, "0.999999999999999", one.Div(two).String()) + + }) + + t.Run("Test CalcRawTokensPerDay", func(t *testing.T) { + amountStr := "99999999999999999999999999999999999999" + duration := uint64(6048000) + + amount, err := CalcRawTokensPerDay(amountStr, duration) + assert.Nil(t, err) + assert.Equal(t, "1428571428571428571428571428571428571.4142857142857143", amount) + }) + t.Run("Test PostNileTokensPerDay", func(t *testing.T) { + duration := uint64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedFloorValue := "1428571428571428571428571428571428571" + + amount, err := CalcRawTokensPerDay(amountStr, duration) + assert.Nil(t, err) + + amount, err = PostNileTokensPerDay(amount) + assert.Nil(t, err) + assert.Equal(t, expectedFloorValue, amount) + }) + t.Run("Test PreNileTokensPerDay", func(t *testing.T) { + duration := uint64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedRoundedValue := "1428571428571427000000000000000000000" + + amount, err := CalcRawTokensPerDay(amountStr, duration) + assert.Nil(t, err) + + amount, err = PreNileTokensPerDay(amount) + assert.Nil(t, err) + assert.Equal(t, expectedRoundedValue, amount) + }) + + t.Run("Staker tokens", func(t *testing.T) { + t.Run("Should correctly calculate the amazon token rewards for stakers", func(t *testing.T) { + values := [][]string{ + []string{"0.000009049993751000000000", "249999999999999740000", "2262498437750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000004801343782000000000", "249999999999999740000", "1200335945500000"}, + []string{"0.000010319286042000000000", "249999999999999740000", "2579821510500000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000010926464639000000000", "249999999999999740000", "2731616159750000"}, + []string{"0.000005872055410000000000", "249999999999999740000", "1468013852500000"}, + []string{"0.000005842987054000000000", "249999999999999740000", "1460746763500000"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000008641233546000000000", "249999999999999740000", "2160308386500000"}, + []string{"0.000008694529984000000000", "249999999999999740000", "2173632496000000"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000004820724495000000000", "249999999999999740000", "1205181123750000"}, + []string{"0.000006769304931000000000", "249999999999999740000", "1692326232750000"}, + []string{"0.000000078385764000000000", "249999999999999740000", "19596441000000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000011067547868000000000", "249999999999999740000", "2766886967000000"}, + []string{"0.000005684438299000000000", "249999999999999740000", "1421109574750000"}, + []string{"0.000005308101027000000000", "249999999999999740000", "1327025256750000"}, + []string{"0.000006052611919000000000", "249999999999999740000", "1513152979750000"}, + []string{"0.000006897277137000000000", "249999999999999740000", "1724319284250000"}, + []string{"0.000007524289114000000000", "249999999999999740000", "1881072278500000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000011068186249000000000", "249999999999999740000", "2767046562250000"}, + []string{"0.000007086176189000000000", "249999999999999740000", "1771544047250000"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000011066475358000000000", "249999999999999740000", "2766618839500000"}, + []string{"0.000004445997620000000000", "249999999999999740000", "1111499405000000"}, + []string{"0.000005026149490000000000", "249999999999999740000", "1256537372500000"}, + []string{"0.000005245237427000000000", "249999999999999740000", "1311309356750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000005906556295000000000", "249999999999999740000", "1476639073750000"}, + []string{"0.000002477475271000000000", "249999999999999740000", "619368817749999"}, + []string{"0.000009406291697000000000", "249999999999999740000", "2351572924250000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.00015529746717900000", "249999999999999740000", "38824366794750000"}, + []string{"0.000005319693767000000000", "249999999999999740000", "1329923441750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000004684377878000000000", "249999999999999740000", "1171094469500000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000008658154920000000000", "249999999999999740000", "2164538730000000"}, + []string{"0.000008377062090000000000", "249999999999999740000", "2094265522500000"}, + []string{"0.000005795889243000000000", "249999999999999740000", "1448972310750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000005615005446000000000", "249999999999999740000", "1403751361500000"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + } + + for i, value := range values { + stakerProportion := value[0] + tokensPerDay := value[1] + expectedValue := value[2] + result, err := CalculateAmazonStakerTokenRewards(stakerProportion, tokensPerDay) + assert.Nil(t, err) + + assert.Equal(t, expectedValue, result, fmt.Sprintf("row %d: '%s' - '%s' - '%s'", i, stakerProportion, tokensPerDay, expectedValue)) + } + + }) + t.Run("Should correctly calculate the nile token rewards for stakers", func(t *testing.T) { + values := [][]string{ + []string{"0.000009049993751000000000", "249999999999999740000", "2262498437749998"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000004801343782000000000", "249999999999999740000", "1200335945499999"}, + []string{"0.000010319286042000000000", "249999999999999740000", "2579821510499997"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000010926464639000000000", "249999999999999740000", "2731616159749997"}, + []string{"0.000005872055410000000000", "249999999999999740000", "1468013852499999"}, + []string{"0.000005842987054000000000", "249999999999999740000", "1460746763499999"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000008641233546000000000", "249999999999999740000", "2160308386499998"}, + []string{"0.000008694529984000000000", "249999999999999740000", "2173632495999998"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000004820724495000000000", "249999999999999740000", "1205181123749999"}, + []string{"0.000006769304931000000000", "249999999999999740000", "1692326232749998"}, + []string{"0.000000078385764000000000", "249999999999999740000", "19596441000000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000011067547868000000000", "249999999999999740000", "2766886966999997"}, + []string{"0.000005684438299000000000", "249999999999999740000", "1421109574749999"}, + []string{"0.000005308101027000000000", "249999999999999740000", "1327025256749999"}, + []string{"0.000006052611919000000000", "249999999999999740000", "1513152979749999"}, + []string{"0.000006897277137000000000", "249999999999999740000", "1724319284249998"}, + []string{"0.000007524289114000000000", "249999999999999740000", "1881072278499998"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000011068186249000000000", "249999999999999740000", "2767046562249997"}, + []string{"0.000007086176189000000000", "249999999999999740000", "1771544047249998"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000011066475358000000000", "249999999999999740000", "2766618839499997"}, + []string{"0.000004445997620000000000", "249999999999999740000", "1111499404999999"}, + []string{"0.000005026149490000000000", "249999999999999740000", "1256537372499999"}, + []string{"0.000005245237427000000000", "249999999999999740000", "1311309356749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000005906556295000000000", "249999999999999740000", "1476639073749999"}, + []string{"0.000002477475271000000000", "249999999999999740000", "619368817749999"}, + []string{"0.000009406291697000000000", "249999999999999740000", "2351572924249998"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.00015529746717900000", "249999999999999740000", "38824366794749960"}, + []string{"0.000005319693767000000000", "249999999999999740000", "1329923441749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000004684377878000000000", "249999999999999740000", "1171094469499999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000008658154920000000000", "249999999999999740000", "2164538729999998"}, + []string{"0.000008377062090000000000", "249999999999999740000", "2094265522499998"}, + []string{"0.000005795889243000000000", "249999999999999740000", "1448972310749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000005615005446000000000", "249999999999999740000", "1403751361499999"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + } + + for i, value := range values { + stakerProportion := value[0] + tokensPerDay := value[1] + expectedValue := value[2] + result, err := CalculateNileStakerTokenRewards(stakerProportion, tokensPerDay) + assert.Nil(t, err) + + assert.Equal(t, expectedValue, result, fmt.Sprintf("row %d: '%s' - '%s' - '%s'", i, stakerProportion, tokensPerDay, expectedValue)) + } + }) + t.Run("Should correctly calculate the post-nile token rewards for stakers", func(t *testing.T) { + values := [][]string{ + []string{"0.000009049993751", "249999999999999740000", "2262498437749997", "2262498437749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004801343782", "249999999999999740000", "1200335945499998", "1200335945499998.8"}, + []string{"0.000010319286042", "249999999999999740000", "2579821510499997", "2579821510499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000010926464639", "249999999999999740000", "2731616159749997", "2731616159749997"}, + []string{"0.00000587205541", "249999999999999740000", "1468013852499998", "1468013852499998.5"}, + []string{"0.000005842987054", "249999999999999740000", "1460746763499998", "1460746763499998.5"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000008641233546", "249999999999999740000", "2160308386499998", "2160308386499998"}, + []string{"0.000008694529984", "249999999999999740000", "2173632495999997", "2173632495999997.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004820724495", "249999999999999740000", "1205181123749998", "1205181123749998.8"}, + []string{"0.000006769304931", "249999999999999740000", "1692326232749998", "1692326232749998.2"}, + []string{"0.000000078385764", "249999999999999740000", "19596440999999", "19596440999999.98"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067547868", "249999999999999740000", "2766886966999997", "2766886966999997"}, + []string{"0.000005684438299", "249999999999999740000", "1421109574749998", "1421109574749998.5"}, + []string{"0.000005308101027", "249999999999999740000", "1327025256749998", "1327025256749998.5"}, + []string{"0.000006052611919", "249999999999999740000", "1513152979749998", "1513152979749998.5"}, + []string{"0.000006897277137", "249999999999999740000", "1724319284249998", "1724319284249998.2"}, + []string{"0.000007524289114", "249999999999999740000", "1881072278499998", "1881072278499998"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011068186249", "249999999999999740000", "2767046562249997", "2767046562249997"}, + []string{"0.000007086176189", "249999999999999740000", "1771544047249998", "1771544047249998.2"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000011066475358", "249999999999999740000", "2766618839499997", "2766618839499997"}, + []string{"0.00000444599762", "249999999999999740000", "1111499404999998", "1111499404999998.8"}, + []string{"0.00000502614949", "249999999999999740000", "1256537372499998", "1256537372499998.8"}, + []string{"0.000005245237427", "249999999999999740000", "1311309356749998", "1311309356749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005906556295", "249999999999999740000", "1476639073749998", "1476639073749998.5"}, + []string{"0.000002477475271", "249999999999999740000", "619368817749999", "619368817749999.4"}, + []string{"0.000009406291697", "249999999999999740000", "2351572924249997", "2351572924249997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000155297467179", "249999999999999740000", "38824366794749960", "38824366794749960"}, + []string{"0.000005319693767", "249999999999999740000", "1329923441749998", "1329923441749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004684377878", "249999999999999740000", "1171094469499998", "1171094469499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000865815492", "249999999999999740000", "2164538729999997", "2164538729999997.5"}, + []string{"0.00000837706209", "249999999999999740000", "2094265522499997", "2094265522499997.8"}, + []string{"0.000005795889243", "249999999999999740000", "1448972310749998", "1448972310749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005615005446", "249999999999999740000", "1403751361499998", "1403751361499998.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004343211879", "249999999999999740000", "1085802969749998", "1085802969749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004497233707", "249999999999999740000", "1124308426749998", "1124308426749998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000473160595", "249999999999999740000", "1182901487499998", "1182901487499998.8"}, + []string{"0.000005489864048", "249999999999999740000", "1372466011999998", "1372466011999998.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004481283758", "249999999999999740000", "1120320939499998", "1120320939499998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011254974313", "249999999999999740000", "2813743578249997", "2813743578249997"}, + []string{"0.000007988838178", "249999999999999740000", "1997209544499998", "1997209544499998"}, + []string{"0.000001371750872", "249999999999999740000", "342937717999999", "342937717999999.6"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000010018074899", "249999999999999740000", "2504518724749997", "2504518724749997.5"}, + []string{"0.000001959644103", "249999999999999740000", "489911025749999", "489911025749999.44"}, + []string{"0.000004677431991", "249999999999999740000", "1169357997749998", "1169357997749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005052634215", "249999999999999740000", "1263158553749998", "1263158553749998.8"}, + []string{"0.000004378562562", "249999999999999740000", "1094640640499998", "1094640640499998.9"}, + []string{"0.000005136263081", "249999999999999740000", "1284065770249998", "1284065770249998.5"}, + []string{"0.000006086614747", "249999999999999740000", "1521653686749998", "1521653686749998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000678008036", "249999999999999740000", "1695020089999998", "1695020089999998"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004358362661", "249999999999999740000", "1089590665249998", "1089590665249998.8"}, + []string{"0.000017333414377", "249999999999999740000", "4333353594249995", "4333353594249995.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000011067443446", "249999999999999740000", "2766860861499997", "2766860861499997"}, + []string{"0.000014949678468", "249999999999999740000", "3737419616999996", "3737419616999996"}, + []string{"0.000006730676146", "249999999999999740000", "1682669036499998", "1682669036499998.2"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.002874063817455", "249999999999999740000", "718515954363749250", "718515954363749250"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013402738201", "249999999999999740000", "3350684550249996", "3350684550249996.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011066683042", "249999999999999740000", "2766670760499997", "2766670760499997"}, + []string{"0.000004559116932", "249999999999999740000", "1139779232999998", "1139779232999998.8"}, + []string{"0.000005178812729", "249999999999999740000", "1294703182249998", "1294703182249998.8"}, + []string{"0.00000560777371", "249999999999999740000", "1401943427499998", "1401943427499998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000034647768302", "249999999999999740000", "8661942075499991", "8661942075499991"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000000039192882", "249999999999999740000", "9798220499999", "9798220499999.99"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005318465383", "249999999999999740000", "1329616345749998", "1329616345749998.8"}, + []string{"0.000009142886932", "249999999999999740000", "2285721732999997", "2285721732999997.5"}, + []string{"0.000000001763323", "249999999999999740000", "440830749999", "440830749999.9995"}, + []string{"0.000005239479729", "249999999999999740000", "1309869932249998", "1309869932249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011068572525", "249999999999999740000", "2767143131249997", "2767143131249997"}, + []string{"0.000004349799583", "249999999999999740000", "1087449895749998", "1087449895749998.9"}, + []string{"0.000005762329847", "249999999999999740000", "1440582461749998", "1440582461749998.5"}, + []string{"0.000005750387614", "249999999999999740000", "1437596903499998", "1437596903499998.5"}, + []string{"0.000014959041874", "249999999999999740000", "3739760468499996", "3739760468499996"}, + []string{"0.000004915721371", "249999999999999740000", "1228930342749998", "1228930342749998.8"}, + []string{"0.000006674009818", "249999999999999740000", "1668502454499998", "1668502454499998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000081602837875", "249999999999999740000", "20400709468749976", "20400709468749976"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006004611299", "249999999999999740000", "1501152824749998", "1501152824749998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006355155937", "249999999999999740000", "1588788984249998", "1588788984249998.5"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000011067820412", "249999999999999740000", "2766955102999997", "2766955102999997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004855108806", "249999999999999740000", "1213777201499998", "1213777201499998.8"}, + []string{"0.000011066088948", "249999999999999740000", "2766522236999997", "2766522236999997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006074896721", "249999999999999740000", "1518724180249998", "1518724180249998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000408162188", "249999999999999740000", "102040546999999", "102040546999999.89"}, + []string{"0.000006086654586", "249999999999999740000", "1521663646499998", "1521663646499998.5"}, + []string{"0.000011067533248", "249999999999999740000", "2766883311999997", "2766883311999997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004959654658", "249999999999999740000", "1239913664499998", "1239913664499998.8"}, + []string{"0.000004334593317", "249999999999999740000", "1083648329249998", "1083648329249998.9"}, + []string{"0.000011066270658", "249999999999999740000", "2766567664499997", "2766567664499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000485991737", "249999999999999740000", "121497934249999", "121497934249999.86"}, + []string{"0.000005957318075", "249999999999999740000", "1489329518749998", "1489329518749998.5"}, + []string{"0.000002821887509", "249999999999999740000", "705471877249999", "705471877249999.2"}, + []string{"0.000005460297788", "249999999999999740000", "1365074446999998", "1365074446999998.5"}, + []string{"0.000000593038012", "249999999999999740000", "148259502999999", "148259502999999.84"}, + []string{"0.00000048232807", "249999999999999740000", "120582017499999", "120582017499999.88"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013835090761", "249999999999999740000", "3458772690249996", "3458772690249996.5"}, + []string{"0.000004387386402", "249999999999999740000", "1096846600499998", "1096846600499998.9"}, + []string{"0.00001056846525", "249999999999999740000", "2642116312499997", "2642116312499997.5"}, + []string{"0.000004744198683", "249999999999999740000", "1186049670749998", "1186049670749998.8"}, + []string{"0.000013517447109", "249999999999999740000", "3379361777249996", "3379361777249996.5"}, + []string{"0.000004292111888", "249999999999999740000", "1073027971999998", "1073027971999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000008820012468", "249999999999999740000", "2205003116999997", "2205003116999997.5"}, + []string{"0.000005601905333", "249999999999999740000", "1400476333249998", "1400476333249998.5"}, + []string{"0.000005339016974", "249999999999999740000", "1334754243499998", "1334754243499998.5"}, + []string{"0.000300001044541", "249999999999999740000", "75000261135249920", "75000261135249920"}, + []string{"0.000011067668642", "249999999999999740000", "2766917160499997", "2766917160499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000016325425419", "249999999999999740000", "4081356354749995", "4081356354749995.5"}, + []string{"0.000000460259862", "249999999999999740000", "115064965499999", "115064965499999.88"}, + []string{"0.000000563455944", "249999999999999740000", "140863985999999", "140863985999999.88"}, + []string{"0.00000568617956", "249999999999999740000", "1421544889999998", "1421544889999998.5"}, + []string{"0.000004999005702", "249999999999999740000", "1249751425499998", "1249751425499998.5"}, + []string{"0.00000609084659", "249999999999999740000", "1522711647499998", "1522711647499998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067715964", "249999999999999740000", "2766928990999997", "2766928990999997"}, + []string{"0.000005004891542", "249999999999999740000", "1251222885499998", "1251222885499998.8"}, + []string{"0.000007218680732", "249999999999999740000", "1804670182999998", "1804670182999998"}, + []string{"0.000005144299747", "249999999999999740000", "1286074936749998", "1286074936749998.8"}, + []string{"0.00008989838143", "249999999999999740000", "22474595357499976", "22474595357499976"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067089214", "249999999999999740000", "2766772303499997", "2766772303499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006545911248", "249999999999999740000", "1636477811999998", "1636477811999998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067039047", "249999999999999740000", "2766759761749997", "2766759761749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009471564941", "249999999999999740000", "2367891235249997", "2367891235249997.5"}, + []string{"0.000008445422705", "249999999999999740000", "2111355676249997", "2111355676249997.8"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005340280631", "249999999999999740000", "1335070157749998", "1335070157749998.5"}, + []string{"0.00000435529761", "249999999999999740000", "1088824402499998", "1088824402499998.9"}, + []string{"0.000006367452314", "249999999999999740000", "1591863078499998", "1591863078499998.2"}, + []string{"0.000004372836209", "249999999999999740000", "1093209052249998", "1093209052249998.9"}, + []string{"0.000004926901446", "249999999999999740000", "1231725361499998", "1231725361499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004755398978", "249999999999999740000", "1188849744499998", "1188849744499998.8"}, + []string{"0.000005303897879", "249999999999999740000", "1325974469749998", "1325974469749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004682911838", "249999999999999740000", "1170727959499998", "1170727959499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011066751231", "249999999999999740000", "2766687807749997", "2766687807749997"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004781531613", "249999999999999740000", "1195382903249998", "1195382903249998.8"}, + []string{"0.000000526960156", "249999999999999740000", "131740038999999", "131740038999999.86"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006337819945", "249999999999999740000", "1584454986249998", "1584454986249998.5"}, + []string{"0.00000598555262", "249999999999999740000", "1496388154999998", "1496388154999998.5"}, + []string{"0.000011133520684", "249999999999999740000", "2783380170999997", "2783380170999997"}, + []string{"0.000011067413867", "249999999999999740000", "2766853466749997", "2766853466749997"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005037750053", "249999999999999740000", "1259437513249998", "1259437513249998.8"}, + []string{"0.000010238640377", "249999999999999740000", "2559660094249997", "2559660094249997.5"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000031315112777", "249999999999999740000", "7828778194249992", "7828778194249992"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004552220781", "249999999999999740000", "1138055195249998", "1138055195249998.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004771744448", "249999999999999740000", "1192936111999998", "1192936111999998.8"}, + []string{"0.000004470418106", "249999999999999740000", "1117604526499998", "1117604526499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000001567715282", "249999999999999740000", "391928820499999", "391928820499999.56"}, + []string{"0.000006685375775", "249999999999999740000", "1671343943749998", "1671343943749998.2"}, + []string{"0.00000495047661", "249999999999999740000", "1237619152499998", "1237619152499998.8"}, + []string{"0.000005844909181", "249999999999999740000", "1461227295249998", "1461227295249998.5"}, + []string{"0.000000002145689", "249999999999999740000", "536422249999", "536422249999.99945"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000002987135399", "249999999999999740000", "746783849749999", "746783849749999.2"}, + []string{"0.000034747556263", "249999999999999740000", "8686889065749991", "8686889065749991"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000014311629335", "249999999999999740000", "3577907333749996", "3577907333749996.5"}, + []string{"0.000000039192882", "249999999999999740000", "9798220499999", "9798220499999.99"}, + []string{"0.000011066754626", "249999999999999740000", "2766688656499997", "2766688656499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004761442856", "249999999999999740000", "1190360713999998", "1190360713999998.8"}, + []string{"0.000005488681729", "249999999999999740000", "1372170432249998", "1372170432249998.5"}, + []string{"0.000004720390951", "249999999999999740000", "1180097737749998", "1180097737749998.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004926662957", "249999999999999740000", "1231665739249998", "1231665739249998.8"}, + []string{"0.000005003715806", "249999999999999740000", "1250928951499998", "1250928951499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004489784075", "249999999999999740000", "1122446018749998", "1122446018749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013860083391", "249999999999999740000", "3465020847749996", "3465020847749996.5"}, + []string{"0.000001990011373", "249999999999999740000", "497502843249999", "497502843249999.5"}, + []string{"0.00001106677849", "249999999999999740000", "2766694622499997", "2766694622499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000039196801", "249999999999999740000", "9799200249999", "9799200249999.99"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013572735485", "249999999999999740000", "3393183871249996", "3393183871249996.5"}, + []string{"0.000004899110259", "249999999999999740000", "1224777564749998", "1224777564749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067136903", "249999999999999740000", "2766784225749997", "2766784225749997"}, + []string{"0.000004336354665", "249999999999999740000", "1084088666249998", "1084088666249998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000482687193", "249999999999999740000", "120671798249999", "120671798249999.86"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004829343283", "249999999999999740000", "1207335820749998", "1207335820749998.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000006834871106", "249999999999999740000", "1708717776499998", "1708717776499998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000008686667517", "249999999999999740000", "2171666879249998", "2171666879249998"}, + []string{"0.000011066162127", "249999999999999740000", "2766540531749997", "2766540531749997"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004558056201", "249999999999999740000", "1139514050249999", "1139514050249999"}, + []string{"0.000005388547951", "249999999999999740000", "1347136987749998", "1347136987749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009495590539", "249999999999999740000", "2373897634749997", "2373897634749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005186272033", "249999999999999740000", "1296568008249998", "1296568008249998.8"}, + []string{"0.000006990370298", "249999999999999740000", "1747592574499998", "1747592574499998"}, + []string{"0.000006873785508", "249999999999999740000", "1718446376999998", "1718446376999998"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004504850433", "249999999999999740000", "1126212608249998", "1126212608249998.8"}, + []string{"0.000005530483313", "249999999999999740000", "1382620828249998", "1382620828249998.5"}, + []string{"0.000007310119829", "249999999999999740000", "1827529957249998", "1827529957249998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067991499", "249999999999999740000", "2766997874749997", "2766997874749997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000014234345856", "249999999999999740000", "3558586463999996", "3558586463999996.5"}, + []string{"0.000000592557184", "249999999999999740000", "148139295999999", "148139295999999.84"}, + []string{"0.000004913446869", "249999999999999740000", "1228361717249998", "1228361717249998.8"}, + []string{"0.000004480086345", "249999999999999740000", "1120021586249998", "1120021586249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000000303352", "249999999999999740000", "75837999999", "75837999999.99992"}, + []string{"0.000011252704148", "249999999999999740000", "2813176036999997", "2813176036999997"}, + []string{"0.000004505674706", "249999999999999740000", "1126418676499999", "1126418676499999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004675810515", "249999999999999740000", "1168952628749998", "1168952628749998.8"}, + []string{"0.00000514400844", "249999999999999740000", "1286002109999998", "1286002109999998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004426765887", "249999999999999740000", "1106691471749998", "1106691471749998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011068588146", "249999999999999740000", "2767147036499997", "2767147036499997"}, + []string{"0.00001106757791", "249999999999999740000", "2766894477499997", "2766894477499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004884842405", "249999999999999740000", "1221210601249998", "1221210601249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005899779334", "249999999999999740000", "1474944833499998", "1474944833499998.2"}, + []string{"0.000013422003779", "249999999999999740000", "3355500944749996", "3355500944749996.5"}, + []string{"0.000144171486789", "249999999999999740000", "36042871697249960", "36042871697249960"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000415444549", "249999999999999740000", "103861137249999", "103861137249999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000031928323627", "249999999999999740000", "7982080906749991", "7982080906749991"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000002640117", "249999999999999740000", "6600292499999", "6600292499999.993"}, + []string{"0.000000633269907", "249999999999999740000", "158317476749999", "158317476749999.84"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005064523288", "249999999999999740000", "1266130821999998", "1266130821999998.8"}, + []string{"0.000004349845721", "249999999999999740000", "1087461430249999", "1087461430249999"}, + []string{"0.000007703579531", "249999999999999740000", "1925894882749998", "1925894882749998.2"}, + []string{"0.000000437957083", "249999999999999740000", "109489270749999", "109489270749999.89"}, + []string{"0.000012046208995", "249999999999999740000", "3011552248749997", "3011552248749997"}, + []string{"0.00000000002121", "249999999999999740000", "5302499999", "5302499999.999994"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005181989907", "249999999999999740000", "1295497476749998", "1295497476749998.5"}, + []string{"0.000006057031037", "249999999999999740000", "1514257759249998", "1514257759249998.2"}, + []string{"0.000007893648991", "249999999999999740000", "1973412247749997", "1973412247749997.8"}, + []string{"0.00001044882236", "249999999999999740000", "2612205589999997", "2612205589999997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004861240598", "249999999999999740000", "1215310149499998", "1215310149499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004351580082", "249999999999999740000", "1087895020499999", "1087895020499999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000588314769", "249999999999999740000", "147078692249999", "147078692249999.84"}, + []string{"0.000009859499524", "249999999999999740000", "2464874880999997", "2464874880999997.5"}, + []string{"0.000004667920393", "249999999999999740000", "1166980098249999", "1166980098249999"}, + []string{"0.000002939466155", "249999999999999740000", "734866538749999", "734866538749999.2"}, + []string{"0.00000865481742", "249999999999999740000", "2163704354999997", "2163704354999997.8"}, + []string{"0.000005125798369", "249999999999999740000", "1281449592249998", "1281449592249998.5"}, + []string{"0.000004793636505", "249999999999999740000", "1198409126249998", "1198409126249998.8"}, + []string{"0.000011068483594", "249999999999999740000", "2767120898499997", "2767120898499997"}, + []string{"0.000007836352763", "249999999999999740000", "1959088190749998", "1959088190749998"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.0000046838042", "249999999999999740000", "1170951049999998", "1170951049999998.8"}, + []string{"0.000004618700167", "249999999999999740000", "1154675041749998", "1154675041749998.8"}, + []string{"0.000004537539334", "249999999999999740000", "1134384833499998", "1134384833499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004776404037", "249999999999999740000", "1194101009249998", "1194101009249998.8"}, + []string{"0.000005553719601", "249999999999999740000", "1388429900249998", "1388429900249998.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004489784075", "249999999999999740000", "1122446018749998", "1122446018749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000692740917", "249999999999999740000", "1731852292499998", "1731852292499998.2"}, + []string{"0.000005584462421", "249999999999999740000", "1396115605249998", "1396115605249998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004327021569", "249999999999999740000", "1081755392249998", "1081755392249998.9"}, + []string{"0.000004460682023", "249999999999999740000", "1115170505749998", "1115170505749998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000007523930305", "249999999999999740000", "1880982576249998", "1880982576249998"}, + []string{"0.000005014929065", "249999999999999740000", "1253732266249998", "1253732266249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000470315651", "249999999999999740000", "117578912749999", "117578912749999.88"}, + []string{"0.000011067266143", "249999999999999740000", "2766816535749997", "2766816535749997"}, + []string{"0.000004606871138", "249999999999999740000", "1151717784499998", "1151717784499998.8"}, + []string{"0.00000429376189", "249999999999999740000", "1073440472499998", "1073440472499998.9"}, + []string{"0.000007368882709", "249999999999999740000", "1842220677249998", "1842220677249998"}, + []string{"0.000005777164954", "249999999999999740000", "1444291238499998", "1444291238499998.5"}, + []string{"0.000000743682751", "249999999999999740000", "185920687749999", "185920687749999.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011066956193", "249999999999999740000", "2766739048249997", "2766739048249997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004444509005", "249999999999999740000", "1111127251249998", "1111127251249998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009980851029", "249999999999999740000", "2495212757249997", "2495212757249997.5"}, + []string{"0.000004438132916", "249999999999999740000", "1109533228999998", "1109533228999998.9"}, + []string{"0.000004818112356", "249999999999999740000", "1204528088999998", "1204528088999998.8"}, + []string{"0.000011068963995", "249999999999999740000", "2767240998749997", "2767240998749997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000108090806087", "249999999999999740000", "27022701521749972", "27022701521749972"}, + []string{"0.000004534886824", "249999999999999740000", "1133721705999998", "1133721705999998.8"}, + []string{"0.000011066549479", "249999999999999740000", "2766637369749997", "2766637369749997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000003447716551", "249999999999999740000", "861929137749999", "861929137749999.1"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000014014908682", "249999999999999740000", "3503727170499996", "3503727170499996"}, + []string{"0.000004243148758", "249999999999999740000", "1060787189499998", "1060787189499998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000008180999729", "249999999999999740000", "2045249932249997", "2045249932249997.8"}, + []string{"0.000001929418705", "249999999999999740000", "482354676249999", "482354676249999.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005589443629", "249999999999999740000", "1397360907249998", "1397360907249998.5"}, + []string{"0.000004701364694", "249999999999999740000", "1175341173499998", "1175341173499998.8"}, + []string{"0.000011068179353", "249999999999999740000", "2767044838249997", "2767044838249997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005798168821", "249999999999999740000", "1449542205249998", "1449542205249998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009286830011", "249999999999999740000", "2321707502749997", "2321707502749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006054124493", "249999999999999740000", "1513531123249998", "1513531123249998.5"}, + []string{"0.000010961840774", "249999999999999740000", "2740460193499997", "2740460193499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000459735762", "249999999999999740000", "1149339404999998", "1149339404999998.8"}, + []string{"0.000004607237865", "249999999999999740000", "1151809466249998", "1151809466249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004999617541", "249999999999999740000", "1249904385249998", "1249904385249998.8"}, + []string{"0.000004384426544", "249999999999999740000", "1096106635999998", "1096106635999998.8"}, + []string{"0.000027592021209", "249999999999999740000", "6898005302249993", "6898005302249993"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011065571432", "249999999999999740000", "2766392857999997", "2766392857999997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011685473321", "249999999999999740000", "2921368330249997", "2921368330249997"}, + []string{"0.000005478819232", "249999999999999740000", "1369704807999998", "1369704807999998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000020025026258", "249999999999999740000", "5006256564499995", "5006256564499995"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000438960279", "249999999999999740000", "109740069749999", "109740069749999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005384947768", "249999999999999740000", "1346236941999998", "1346236941999998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004414406953", "249999999999999740000", "1103601738249998", "1103601738249998.8"}, + []string{"0.000010817235452", "249999999999999740000", "2704308862999997", "2704308862999997.5"}, + []string{"0.000003459054342", "249999999999999740000", "864763585499999", "864763585499999"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000006305221592", "249999999999999740000", "1576305397999998", "1576305397999998.2"}, + []string{"0.000000589197102", "249999999999999740000", "147299275499999", "147299275499999.84"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005284741798", "249999999999999740000", "1321185449499998", "1321185449499998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004977496023", "249999999999999740000", "1244374005749998", "1244374005749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000008701617124", "249999999999999740000", "2175404280999997", "2175404280999997.8"}, + []string{"0.000009297049021", "249999999999999740000", "2324262255249997", "2324262255249997.5"}, + []string{"0.00000435040991", "249999999999999740000", "1087602477499998", "1087602477499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004905866207", "249999999999999740000", "1226466551749998", "1226466551749998.8"}, + []string{"0.000011188276426", "249999999999999740000", "2797069106499997", "2797069106499997"}, + []string{"0.000004489846462", "249999999999999740000", "1122461615499998", "1122461615499998.9"}, + []string{"0.000004626953867", "249999999999999740000", "1156738466749999", "1156738466749999"}, + []string{"0.000023680418961", "249999999999999740000", "5920104740249994", "5920104740249994"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000054417993", "249999999999999740000", "13604498249999", "13604498249999.986"}, + []string{"0.000005470503287", "249999999999999740000", "1367625821749998", "1367625821749998.5"}, + } + + for i, value := range values { + stakerProportion := value[0] + tokensPerDayDecimal := value[1] + expectedValue := value[2] + result, err := CalculatePostNileStakerTokenRewards(stakerProportion, tokensPerDayDecimal) + assert.Nil(t, err) + + assert.Equal(t, expectedValue, result, fmt.Sprintf("row %d: '%s' - '%s' - '%s'", i, stakerProportion, tokensPerDayDecimal, expectedValue)) + } + }) + }) + + t.Run("Operator tokens", func(t *testing.T) { + t.Run("Should calculate operator amazon fork tokens correctly", func(t *testing.T) { + for _, value := range operatorAmazonTokens { + totalStakerOperatorPayout := value[0] + expectedValue := value[1] + + result, err := CalculateAmazonOperatorTokens(totalStakerOperatorPayout) + assert.Nil(t, err) + assert.Equal(t, expectedValue, result) + } + }) + t.Run("Should calculate nile fork tokens correctly", func(t *testing.T) { + for _, value := range operatorNileTokens { + totalStakerOperatorPayout := value[0] + expectedValue := value[1] + + result, err := CalculateNileOperatorTokens(totalStakerOperatorPayout) + assert.Nil(t, err) + assert.Equal(t, expectedValue, result) + } + }) + }) } diff --git a/internal/types/numbers/tokenCalculations.go b/internal/types/numbers/tokenCalculations.go new file mode 100644 index 00000000..1aa778be --- /dev/null +++ b/internal/types/numbers/tokenCalculations.go @@ -0,0 +1,127 @@ +package numbers + +import ( + "errors" + "fmt" + "github.com/shopspring/decimal" +) + +/* +#cgo darwin CFLAGS: -I/opt/homebrew/opt/python@3.12/Frameworks/Python.framework/Versions/3.12/include/python3.12 -I/Users/seanmcgary/Code/sidecar/sqlite-extensions +#cgo darwin LDFLAGS: -L/opt/homebrew/opt/python@3.12/Frameworks/Python.framework/Versions/3.12/lib -lpython3.12 -L/Users/seanmcgary/Code/sidecar/sqlite-extensions -lcalculations +#cgo darwin LDFLAGS: -Wl,-rpath,/Users/seanmcgary/Code/sidecar/sqlite-extensions +#include +#include "calculations.h" +*/ +import "C" +import "unsafe" + +func InitPython() error { + if C.init_python() == 0 { + return errors.New("failed to initialize python") + } + return nil +} + +func FinalizePython() { + C.finalize_python() +} + +// CalculateAmazonStakerTokenRewards calculates the Amazon token rewards for a given staker proportion and tokens per day +// cast(staker_proportion * tokens_per_day AS DECIMAL(38,0)) +func CalculateAmazonStakerTokenRewards(stakerProportion string, tokensPerDay string) (string, error) { + cSp := C.CString(stakerProportion) + cTpd := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cSp)) + defer C.free(unsafe.Pointer(cTpd)) + + cResult := C._amazon_staker_token_rewards(cSp, cTpd) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculateNileStakerTokenRewards calculates the tokens to be rewarded for a given staker proportion and tokens per day +// (staker_proportion * tokens_per_day)::text::decimal(38,0) +func CalculateNileStakerTokenRewards(stakerProportion string, tokensPerDay string) (string, error) { + cSp := C.CString(stakerProportion) + cTpd := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cSp)) + defer C.free(unsafe.Pointer(cTpd)) + + cResult := C._nile_staker_token_rewards(cSp, cTpd) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculatePostNileStakerTokenRewards calculates the tokens to be rewarded for a given staker proportion and tokens per day +// FLOOR(staker_proportion * tokens_per_day_decimal) +func CalculatePostNileStakerTokenRewards(stakerProportion string, tokensPerDay string) (string, error) { + cSp := C.CString(stakerProportion) + cTpd := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cSp)) + defer C.free(unsafe.Pointer(cTpd)) + + cResult := C._staker_token_rewards(cSp, cTpd) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculateAmazonOperatorTokens calculates the operator payout portion for rewards (10% of total) +// +// cast(total_staker_operator_payout * 0.10 AS DECIMAL(38,0)) +func CalculateAmazonOperatorTokens(totalStakerPayout string) (string, error) { + tsp := C.CString(totalStakerPayout) + defer C.free(unsafe.Pointer(tsp)) + + cResult := C._amazon_operator_token_rewards(tsp) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculateNileOperatorTokens calculates the operator payout portion for rewards (10% of total) +// +// (total_staker_operator_payout * 0.10)::text::decimal(38,0) +func CalculateNileOperatorTokens(totalStakerPayout string) (string, error) { + tsp := C.CString(totalStakerPayout) + defer C.free(unsafe.Pointer(tsp)) + + cResult := C._nile_operator_token_rewards(tsp) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculatePostNileOperatorTokens calculates the operator payout portion for rewards (10% of total) +// +// floor(total_staker_operator_payout * 0.10) +func CalculatePostNileOperatorTokens(totalStakerPayout string) (string, error) { + tpd, err := decimal.NewFromString(totalStakerPayout) + if err != nil { + return "", err + } + + return tpd.Mul(decimal.NewFromFloat(0.10)).Floor().String(), nil +} + +// PreNileTokensPerDay calculates the tokens per day for pre-nile rewards, rounded to 15 sigfigs +// +// Not gonna lie, this is pretty annoying that it has to be this way, but in order to support backwards compatibility +// with the current/old rewards system where postgres was lossy, we have to do this. +func PreNileTokensPerDay(tokensPerDay string) (string, error) { + fmt.Printf("PreNileTokensPerDay: %s\n", tokensPerDay) + cTokens := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cTokens)) + + fmt.Printf("Calling pre_nile_tokens_per_day\n") + result := C._pre_nile_tokens_per_day(cTokens) + fmt.Printf("Successfully called") + defer C.free(unsafe.Pointer(result)) + + resultStr := C.GoString(result) + fmt.Printf("PreNileTokensPerDay Result: %+v\n", resultStr) + return resultStr, nil +} diff --git a/pkg/rewards/1_goldActiveRewards.go b/pkg/rewards/1_goldActiveRewards.go new file mode 100644 index 00000000..d9f04ced --- /dev/null +++ b/pkg/rewards/1_goldActiveRewards.go @@ -0,0 +1,170 @@ +package rewards + +import ( + "database/sql" + "fmt" +) + +var _1_goldActiveRewardsQuery = ` +-- insert into gold_1_active_rewards +WITH active_rewards_modified as ( + SELECT + *, + calc_raw_tokens_per_day(amount, duration) as tokens_per_day, + DATETIME(@cutoffDate) as global_end_inclusive -- Inclusive means we DO USE this day as a snapshot + FROM combined_rewards + WHERE end_timestamp >= DATETIME(@rewardsStart) and start_timestamp <= DATETIME(@cutoffDate) +), +-- Cut each reward's start and end windows to handle the global range +active_rewards_updated_end_timestamps as ( + SELECT + avs, + -- Cut the start and end windows to handle + -- A. Retroactive rewards that came recently whose start date is less than start_timestamp + -- B. Don't make any rewards past end_timestamp for this run + start_timestamp as reward_start_exclusive, + MIN(global_end_inclusive, end_timestamp) as reward_end_inclusive, + tokens_per_day, + token, + multiplier, + strategy, + reward_hash, + reward_type, + global_end_inclusive, + block_date as reward_submission_date + FROM active_rewards_modified +), +-- For each reward hash, find the latest snapshot +active_rewards_updated_start_timestamps as ( + SELECT + ap.avs, + ap.reward_start_exclusive as reward_start_exclusive, + ap.reward_end_inclusive, + ap.token, + post_nile_tokens_per_day(ap.tokens_per_day) as tokens_per_day_decimal, + pre_nile_tokens_per_day(ap.tokens_per_day) as tokens_per_day, + ap.multiplier, + ap.strategy, + ap.reward_hash, + ap.reward_type, + ap.global_end_inclusive, + ap.reward_submission_date + FROM active_rewards_updated_end_timestamps ap + LEFT JOIN gold_table g ON g.reward_hash = ap.reward_hash + GROUP BY ap.avs, ap.reward_end_inclusive, ap.token, ap.tokens_per_day, ap.multiplier, ap.strategy, ap.reward_hash, ap.global_end_inclusive, ap.reward_start_exclusive, ap.reward_type, ap.reward_submission_date +), +-- Parse out invalid ranges +active_reward_ranges AS ( + SELECT * from active_rewards_updated_start_timestamps + -- Take out (reward_start_exclusive, reward_end_inclusive) windows where + -- 1. reward_start_exclusive >= reward_end_inclusive: The reward period is done or we will handle on a subsequent run + WHERE reward_start_exclusive < reward_end_inclusive +), + date_bounds as ( + select + min(reward_start_exclusive) as min_start, + max(reward_end_inclusive) as max_end + from active_reward_ranges + ), + day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner + ), +-- Explode out the ranges for a day per inclusive date + exploded_active_range_rewards AS ( + SELECT + arr.*, + day_series.day as day + FROM active_reward_ranges as arr + cross join day_series + where DATE(day_series.day) between DATE(reward_start_exclusive) and DATE(reward_end_inclusive) + ), + active_rewards_final AS ( + SELECT + avs, + DATE(day) as snapshot, + token, + tokens_per_day, + tokens_per_day_decimal, + multiplier, + strategy, + reward_hash, + reward_type, + reward_submission_date + FROM exploded_active_range_rewards + -- Remove snapshots on the start day + WHERE day != reward_start_exclusive + ) +select + avs, + snapshot, + token, + tokens_per_day, + tokens_per_day_decimal, + multiplier, + strategy, + reward_hash, + reward_type, + reward_submission_date +from active_rewards_final +` + +type ResultRow struct { + Avs string + Snapshot string + Token string + TokensPerDay string + TokensPerDayDecimal string +} + +// GenerateActiveRewards generates active rewards for the gold_1_active_rewards table +// +// @param snapshotDate: The upper bound of when to calculate rewards to +// @param startDate: The lower bound of when to calculate rewards from. If we're running rewards for the first time, +// this will be "1970-01-01". If this is a subsequent run, this will be the last snapshot date. +func (r *RewardsCalculator) GenerateActiveRewards(cutoffDate string, startDate string) error { + r.logger.Sugar().Infow("Generating active rewards", "cutoffDate", cutoffDate, "startDate", startDate) + rows := make([]ResultRow, 0) + res := r.grm.Raw(_1_goldActiveRewardsQuery, + sql.Named("cutoffDate", cutoffDate), + sql.Named("rewardsStart", startDate), + ).Scan(&rows) + fmt.Printf("Rows: %+v\n", rows) + r.logger.Sugar().Infof("Rows affected: %+v", res.RowsAffected) + r.logger.Sugar().Infow("Executed active rewards query") + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate active rewards", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateGold1ActiveRewardsTable() error { + query := ` + create table if not exists gold_1_active_rewards ( + avs TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + tokens_per_day_decimal TEXT NOT NULL, + multiplier TEXT NOT NULL, + strategy TEXT NOT NULL, + reward_hash TEXT NOT NULL, + reward_type TEXT NOT NULL, + reward_submission_date DATE NOT NULL + ) + ` + res := r.grm.Exec(query) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create gold_1_active_rewards table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/2_goldStakerRewardAmounts.go b/pkg/rewards/2_goldStakerRewardAmounts.go new file mode 100644 index 00000000..9f6b2ab9 --- /dev/null +++ b/pkg/rewards/2_goldStakerRewardAmounts.go @@ -0,0 +1,180 @@ +package rewards + +import ( + "database/sql" + "github.com/Layr-Labs/go-sidecar/internal/config" +) + +const _2_goldStakerRewardAmountsQuery = ` +insert into gold_2_staker_reward_amounts +WITH reward_snapshot_operators as ( + SELECT + ap.reward_hash, + ap.snapshot, + ap.token, + ap.tokens_per_day, + ap.tokens_per_day_decimal, + ap.avs, + ap.strategy, + ap.multiplier, + ap.reward_type, + ap.reward_submission_date, + oar.operator + FROM gold_1_active_rewards ap + JOIN operator_avs_registration_snapshots oar + ON ap.avs = oar.avs and ap.snapshot = oar.snapshot + WHERE ap.reward_type = 'avs' +), +_operator_restaked_strategies AS ( + SELECT + rso.* + FROM reward_snapshot_operators rso + JOIN operator_avs_strategy_snapshots oas + ON + rso.operator = oas.operator AND + rso.avs = oas.avs AND + rso.strategy = oas.strategy AND + rso.snapshot = oas.snapshot +), +-- Get the stakers that were delegated to the operator for the snapshot +staker_delegated_operators AS ( + SELECT + ors.*, + sds.staker + FROM _operator_restaked_strategies ors + JOIN staker_delegation_snapshots sds + ON + ors.operator = sds.operator AND + ors.snapshot = sds.snapshot +), +-- Get the shares for staker delegated to the operator +staker_avs_strategy_shares AS ( + SELECT + sdo.*, + sss.shares + FROM staker_delegated_operators sdo + JOIN staker_share_snapshots sss + ON + sdo.staker = sss.staker AND + sdo.snapshot = sss.snapshot AND + sdo.strategy = sss.strategy + -- Parse out negative shares and zero multiplier so there is no division by zero case + WHERE sss.shares > 0 and sdo.multiplier != 0 +), +-- Calculate the weight of a staker +staker_weights AS ( + SELECT *, + sum_big(numeric_multiply(multiplier, shares)) OVER (PARTITION BY staker, reward_hash, snapshot) AS staker_weight + FROM staker_avs_strategy_shares +), +-- Get distinct stakers since their weights are already calculated +distinct_stakers AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (staker, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn + FROM staker_weights + ) t + WHERE rn = 1 + ORDER BY reward_hash, snapshot, staker +), +-- Calculate sum of all staker weights for each reward and snapshot +staker_weight_sum AS ( + SELECT *, + sum_big(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_weight + FROM distinct_stakers +), +-- Calculate staker proportion of tokens for each reward and snapshot +staker_proportion AS ( + SELECT *, + calc_staker_proportion(staker_weight, total_weight) as staker_proportion + FROM staker_weight_sum +), +-- Calculate total tokens to the (staker, operator) pair +staker_operator_total_tokens AS ( + SELECT *, + CASE + -- For snapshots that are before the hard fork AND submitted before the hard fork, we use the old calc method + WHEN snapshot < DATE(@amazonHardforkDate) AND reward_submission_date < DATE(@amazonHardforkDate) THEN + amazon_token_rewards(staker_proportion, tokens_per_day) + -- cast(staker_proportion * tokens_per_day AS DECIMAL(38,0)) + WHEN snapshot < DATE(@nileHardforkDate) AND reward_submission_date < DATE(@nileHardforkDate) THEN + nile_token_rewards(staker_proportion, tokens_per_day) + -- (staker_proportion * tokens_per_day)::text::decimal(38,0) + ELSE + -- FLOOR(staker_proportion * tokens_per_day_decimal) + post_nile_token_rewards(staker_proportion, tokens_per_day) + END as total_staker_operator_payout + FROM staker_proportion +), +operator_tokens as ( + select * + CASE + WHEN snapshot < DATE(@amazonHardforkDate) AND reward_submission_date < DATE(@amazonHardforkDate) THEN + amazon_operator_tokens(total_staker_operator_payout) + --cast(total_staker_operator_payout * 0.10 AS DECIMAL(38,0)) + WHEN snapshot < DATE(@nileHardforkDate) AND reward_submission_date < DATE(@nileHardforkDate) THEN + nile_operator_tokens(total_staker_operator_payout) + -- (total_staker_operator_payout * 0.10)::text::decimal(38,0) + ELSE + post_nile_operator_tokens(total_staker_operator_payout) + -- floor(total_staker_operator_payout * 0.10) + END as operator_tokens + from staker_operator_total_tokens +), +-- Calculate the token breakdown for each (staker, operator) pair +token_breakdowns AS ( + SELECT *, + subtract_big(total_staker_operator_payout, operator_tokens) as staker_tokens + FROM operator_tokens +) +SELECT * from token_breakdowns +ORDER BY reward_hash, snapshot, staker, operator +` + +func (rc *RewardsCalculator) GenerateGoldStakerRewardAmountsTable(forks config.ForkMap) error { + res := rc.grm.Exec(_2_goldStakerRewardAmountsQuery, + sql.Named("amazonHardforkDate", forks[config.Fork_Amazon]), + sql.Named("nileHardforkDate", forks[config.Fork_Nile]), + ) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_staker_reward_amounts", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold2RewardAmountsTable() error { + query := ` + create table if not exists gold_2_staker_reward_amounts ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + tokens_per_day_decimal TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + reward_submission_date DATE NOT NULL, + operator TEXT NOT NULL, + staker TEXT NOT NULL, + shares TEXT NOT NULL, + staker_weight TEXT NOT NULL, + rn INTEGER NOT NULL, + total_weight TEXT NOT NULL, + staker_proportion TEXT NOT NULL, + total_staker_operator_payout TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + staker_tokens TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_2_staker_reward_amounts table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/3_goldOperatorRewardAmounts.go b/pkg/rewards/3_goldOperatorRewardAmounts.go new file mode 100644 index 00000000..8204834f --- /dev/null +++ b/pkg/rewards/3_goldOperatorRewardAmounts.go @@ -0,0 +1,65 @@ +package rewards + +const _3_goldOperatorRewardAmountsQuery = ` +insert into gold_3_operator_reward_amounts +WITH operator_token_sums AS ( + SELECT + reward_hash, + snapshot, + token, + tokens_per_day, + avs, + strategy, + multiplier, + reward_type, + operator, + sum_big(operator_tokens) OVER (PARTITION BY operator, reward_hash, snapshot) AS operator_tokens + FROM gold_2_staker_reward_amounts +), +-- Dedupe the operator tokens across strategies for each operator, reward hash, and snapshot +distinct_operators AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (operator, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, operator ORDER BY strategy ASC) as rn + FROM operator_token_sums + ) t + WHERE rn = 1 +) +SELECT * FROM distinct_operators +` + +func (rc *RewardsCalculator) GenerateGoldOperatorRewardAmountsTable() error { + res := rc.grm.Exec(_3_goldOperatorRewardAmountsQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_operator_reward_amounts", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold3OperatorRewardsTable() error { + query := ` + create table if not exists gold_3_operator_reward_amounts ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + operator TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + rn INTEGER NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_3_operator_reward_amounts table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/4_goldRewardsForAll.go b/pkg/rewards/4_goldRewardsForAll.go new file mode 100644 index 00000000..02c00acb --- /dev/null +++ b/pkg/rewards/4_goldRewardsForAll.go @@ -0,0 +1,101 @@ +package rewards + +const _4_goldRewardsForAllQuery = ` +insert into gold_4_rewards_for_all +WITH reward_snapshot_stakers AS ( + SELECT + ap.reward_hash, + ap.snapshot, + ap.token, + ap.tokens_per_day, + ap.avs, + ap.strategy, + ap.multiplier, + ap.reward_type, + sss.staker, + sss.shares + FROM gold_1_active_rewards ap + JOIN staker_share_snapshots as sss + ON ap.strategy = sss.strategy and ap.snapshot = sss.snapshot + WHERE ap.reward_type = 'all_stakers' + -- Parse out negative shares and zero multiplier so there is no division by zero case + AND big_gt(sss.shares, "0") and ap.multiplier != "0" +), +-- Calculate the weight of a staker +staker_weights AS ( + SELECT *, + sum_big(numeric_multiply(multiplier, shares)) OVER (PARTITION BY staker, reward_hash, snapshot) AS staker_weight + FROM reward_snapshot_stakers +), +-- Get distinct stakers since their weights are already calculated +distinct_stakers AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (staker, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn + FROM staker_weights + ) t + WHERE rn = 1 + ORDER BY reward_hash, snapshot, staker +), +-- Calculate sum of all staker weights +staker_weight_sum AS ( + SELECT *, + sum_big(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_staker_weight + FROM distinct_stakers +), +-- Calculate staker token proportion +staker_proportion AS ( + SELECT *, + calc_staker_proportion(staker_weight, total_staker_weight) as staker_proportion + FROM staker_weight_sum +), +-- Calculate total tokens to staker +staker_tokens AS ( + SELECT *, + -- TODO: update to using floor when we reactivate this + nile_token_rewards(staker_proportion, tokens_per_day) as staker_tokens + -- (tokens_per_day * staker_proportion)::text::decimal(38,0) as staker_tokens + FROM staker_proportion +) +SELECT * from staker_tokens +` + +func (rc *RewardsCalculator) GenerateGoldRewardsForAllTable() error { + res := rc.grm.Exec(_4_goldRewardsForAllQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_rewards_for_all", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold4RewardsForAllTable() error { + query := ` + create table if not exists gold_4_rewards_for_all ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + staker TEXT NOT NULL, + shares TEXT NOT NULL, + staker_weight TEXT NOT NULL, + rn INTEGER NOT NULL, + total_staker_weight TEXT NOT NULL, + staker_proportion TEXT NOT NULL, + staker_tokens TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_4_rewards_for_all table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/5_goldRfaeStakers.go b/pkg/rewards/5_goldRfaeStakers.go new file mode 100644 index 00000000..b81da8fe --- /dev/null +++ b/pkg/rewards/5_goldRfaeStakers.go @@ -0,0 +1,141 @@ +package rewards + +const _5_goldRfaeStakersQuery = ` +insert into gold_5_rfae_stakers +WITH avs_opted_operators AS ( + SELECT DISTINCT + snapshot, + operator + FROM operator_avs_registration_snapshots +), +-- Get the operators who will earn rewards for the reward submission at the given snapshot +reward_snapshot_operators as ( + SELECT + ap.reward_hash, + ap.snapshot, + ap.token, + ap.tokens_per_day_decimal, + ap.avs, + ap.strategy, + ap.multiplier, + ap.reward_type, + ap.reward_submission_date, + aoo.operator + FROM gold_1_active_rewards ap + JOIN avs_opted_operators aoo + ON ap.snapshot = aoo.snapshot + WHERE ap.reward_type = 'all_earners' +), +-- Get the stakers that were delegated to the operator for the snapshot +staker_delegated_operators AS ( + SELECT + rso.*, + sds.staker + FROM reward_snapshot_operators rso + JOIN staker_delegation_snapshots sds + ON + rso.operator = sds.operator AND + rso.snapshot = sds.snapshot +), +-- Get the shares of each strategy the staker has delegated to the operator +staker_strategy_shares AS ( + SELECT + sdo.*, + sss.shares + FROM staker_delegated_operators sdo + JOIN staker_share_snapshots sss + ON + sdo.staker = sss.staker AND + sdo.snapshot = sss.snapshot AND + sdo.strategy = sss.strategy + -- Parse out negative shares and zero multiplier so there is no division by zero case + WHERE big_gt(sss.shares, "0") and sdo.multiplier != "0" +), +-- Calculate the weight of a staker +staker_weights AS ( + SELECT *, + big_sum(numeric_multiply(multiplier, shares)) OVER (PARTITION BY staker, reward_hash, snapshot) AS staker_weight + FROM staker_strategy_shares +), +-- Get distinct stakers since their weights are already calculated +distinct_stakers AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (staker, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn + FROM staker_weights + ) t + WHERE rn = 1 + ORDER BY reward_hash, snapshot, staker +), +-- Calculate sum of all staker weights for each reward and snapshot +staker_weight_sum AS ( + SELECT *, + big_sum(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_weight + FROM distinct_stakers +), +-- Calculate staker proportion of tokens for each reward and snapshot +staker_proportion AS ( + SELECT *, + calc_staker_proportion(staker_weight, total_weight) as staker_proportion + FROM staker_weight_sum +), +-- Calculate total tokens to the (staker, operator) pair +staker_operator_total_tokens AS ( + SELECT *, + post_nile_token_rewards(staker_proportion, tokens_per_day_decimal) as total_staker_operator_payout + FROM staker_proportion +), +-- Calculate the token breakdown for each (staker, operator) pair +token_breakdowns AS ( + SELECT *, + floor(total_staker_operator_payout * 0.10) as operator_tokens, + subtract_big(total_staker_operator_payout, post_nile_operator_tokens(total_staker_operator_payout)) as staker_tokens + FROM staker_operator_total_tokens +) +SELECT * from token_breakdowns +ORDER BY reward_hash, snapshot, staker, operator +` + +func (rc *RewardsCalculator) GenerateGoldRfaeStakersTable() error { + res := rc.grm.Exec(_5_goldRfaeStakersQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_rfae_stakers", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold5RfaeStakersTable() error { + query := ` + create table if not exists gold_5_rfae_stakers ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + reward_submission_date DATE NOT NULL, + operator TEXT NOT NULL, + staker TEXT NOT NULL, + shares TEXT NOT NULL, + staker_weight TEXT NOT NULL, + rn INTEGER NOT NULL, + total_weight TEXT NOT NULL, + staker_proportion TEXT NOT NULL, + total_staker_operator_payout TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + staker_tokens TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_5_rfae_stakers table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/6_goldRfaeOperators.go b/pkg/rewards/6_goldRfaeOperators.go new file mode 100644 index 00000000..d7b25f54 --- /dev/null +++ b/pkg/rewards/6_goldRfaeOperators.go @@ -0,0 +1,65 @@ +package rewards + +const _6_goldRfaeOperatorsQuery = ` +insert into gold_6_rfae_operators as +WITH operator_token_sums AS ( + SELECT + reward_hash, + snapshot, + token, + tokens_per_day_decimal, + avs, + strategy, + multiplier, + reward_type, + operator, + big_sum(operator_tokens) OVER (PARTITION BY operator, reward_hash, snapshot) AS operator_tokens + FROM gold_5_rfae_stakers +), +-- Dedupe the operator tokens across strategies for each operator, reward hash, and snapshot +distinct_operators AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (operator, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, operator ORDER BY strategy ASC) as rn + FROM operator_token_sums + ) t + WHERE rn = 1 +) +SELECT * FROM distinct_operators +` + +func (rc *RewardsCalculator) GenerateGoldRfaeOperatorsTable() error { + res := rc.grm.Exec(_6_goldRfaeOperatorsQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_rfae_operators", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold6RfaeOperatorsTable() error { + query := ` + create table if not exists gold_6_rfae_operators ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day_decimal TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + operator TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + rn INTEGER NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_6_rfae_operators", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/7_goldStaging.go b/pkg/rewards/7_goldStaging.go new file mode 100644 index 00000000..ddb93301 --- /dev/null +++ b/pkg/rewards/7_goldStaging.go @@ -0,0 +1,107 @@ +package rewards + +const _7_goldStagingQuery = ` +insert into gold_7_staging as +WITH staker_rewards AS ( + -- We can select DISTINCT here because the staker's tokens are the same for each strategy in the reward hash + SELECT DISTINCT + staker as earner, + snapshot, + reward_hash, + token, + staker_tokens as amount + FROM gold_2_staker_reward_amounts +), +operator_rewards AS ( + SELECT DISTINCT + -- We can select DISTINCT here because the operator's tokens are the same for each strategy in the reward hash + operator as earner, + snapshot, + reward_hash, + token, + operator_tokens as amount + FROM gold_3_operator_reward_amounts +), +rewards_for_all AS ( + SELECT DISTINCT + staker as earner, + snapshot, + reward_hash, + token, + staker_tokens as amount + FROM gold_4_rewards_for_all +), +rewards_for_all_earners_stakers AS ( + SELECT DISTINCT + staker as earner, + snapshot, + reward_hash, + token, + staker_tokens as amounts + FROM gold_5_rfae_stakers +), +rewards_for_all_earners_operators AS ( + SELECT DISTINCT + operator as earner, + snapshot, + reward_hash, + token, + operator_tokens as amount + FROM gold_6_rfae_operators +), +combined_rewards AS ( + SELECT * FROM operator_rewards + UNION ALL + SELECT * FROM staker_rewards + UNION ALL + SELECT * FROM rewards_for_all + UNION ALL + SELECT * FROM rewards_for_all_earners_stakers + UNION ALL + SELECT * FROM rewards_for_all_earners_operators +), +-- Dedupe earners, primarily operators who are also their own staker. +deduped_earners AS ( + SELECT + earner, + snapshot, + reward_hash, + token, + big_sum(amount) as amount + FROM combined_rewards + GROUP BY + earner, + snapshot, + reward_hash, + token +) +SELECT * +FROM deduped_earners +` + +func (rc *RewardsCalculator) GenerateGoldStagingTable() error { + res := rc.grm.Exec(_7_goldStagingQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_staging", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold7StagingTable() error { + query := ` + create table if not exists gold_7_staging ( + earner TEXT NOT NULL, + snapshot DATE NOT NULL, + reward_hash TEXT NOT NULL, + token TEXT NOT NULL, + amount TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_7_staging", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/8_goldFinal.go b/pkg/rewards/8_goldFinal.go new file mode 100644 index 00000000..7746faa7 --- /dev/null +++ b/pkg/rewards/8_goldFinal.go @@ -0,0 +1,39 @@ +package rewards + +const _8_goldFinalQuery = ` +insert into gold_final +SELECT + earner, + snapshot, + reward_hash, + token, + amount +FROM gold_7_staging +` + +func (rc *RewardsCalculator) GenerateGoldFinalTable() error { + res := rc.grm.Exec(_8_goldFinalQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_final", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) Create8GoldTable() error { + query := ` + create table if not exists gold_table ( + earner TEXT NOT NULL, + snapshot DATE NOT NULL, + reward_hash TEXT NOT NULL, + token TEXT NOT NULL, + amount TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/combinedRewards.go b/pkg/rewards/combinedRewards.go new file mode 100644 index 00000000..6b779c13 --- /dev/null +++ b/pkg/rewards/combinedRewards.go @@ -0,0 +1,100 @@ +package rewards + +import ( + "time" +) + +const rewardsCombinedQuery = ` + with combined_rewards as ( + select + rs.avs, + rs.reward_hash, + rs.token, + rs.amount, + rs.strategy, + rs.strategy_index, + rs.multiplier, + rs.start_timestamp, + rs.end_timestamp, + rs.duration, + rs.block_number, + b.block_time, + DATE(b.block_time) as block_date, + rs.reward_type, + ROW_NUMBER() OVER (PARTITION BY reward_hash, strategy_index ORDER BY block_number asc) as rn + from reward_submissions as rs + left join blocks as b on (b.number = rs.block_number) + ) + select * from combined_rewards + where rn = 1 +` + +type CombinedRewards struct { + Avs string + RewardHash string + Token string + Amount string + Strategy string + StrategyIndex uint64 + Multiplier string + StartTimestamp *time.Time `gorm:"type:DATETIME"` + EndTimestamp *time.Time `gorm:"type:DATETIME"` + Duration uint64 + BlockNumber uint64 + BlockDate string + BlockTime *time.Time `gorm:"type:DATETIME"` + RewardType string // avs, all_stakers, all_earners +} + +func (r *RewardsCalculator) GenerateCombinedRewards() ([]*CombinedRewards, error) { + combinedRewards := make([]*CombinedRewards, 0) + + res := r.grm.Raw(rewardsCombinedQuery).Scan(&combinedRewards) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate combined rewards", "error", res.Error) + return nil, res.Error + } + return combinedRewards, nil +} + +func (r *RewardsCalculator) GenerateAndInsertCombinedRewards() error { + combinedRewards, err := r.GenerateCombinedRewards() + if err != nil { + r.logger.Sugar().Errorw("Failed to generate combined rewards", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting combined rewards", "count", len(combinedRewards)) + res := r.grm.Model(&CombinedRewards{}).CreateInBatches(combinedRewards, 500) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert combined rewards", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateCombinedRewardsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS combined_rewards ( + avs TEXT, + reward_hash TEXT, + token TEXT, + amount TEXT, + strategy TEXT, + strategy_index INTEGER, + multiplier TEXT, + start_timestamp DATETIME, + end_timestamp DATETIME, + duration INTEGER, + block_number INTEGER, + block_time DATETIME, + block_date DATE, + reward_type string + )`, + ) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create combined_rewards table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/combinedRewards_test.go b/pkg/rewards/combinedRewards_test.go new file mode 100644 index 00000000..f468a0f9 --- /dev/null +++ b/pkg/rewards/combinedRewards_test.go @@ -0,0 +1,114 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "testing" +) + +func setupCombinedRewards() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbName, cfg, db, l, err +} + +func teardownCombinedRewards(grm *gorm.DB) { + queries := []string{ + `delete from reward_submissions`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateRewardSubmissionsTable(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetCombinedRewardsSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_CombinedRewards(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + dbFileName, cfg, grm, l, err := setupCombinedRewards() + + if err != nil { + t.Fatal(err) + } + + t.Run("Should hydrate blocks and reward_submissions tables", func(t *testing.T) { + err := hydrateAllBlocksTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query := "select count(*) from blocks" + var count int + res := grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, TOTAL_BLOCK_COUNT, count) + + err = hydrateRewardSubmissionsTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query = "select count(*) from reward_submissions" + res = grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, 192, count) + }) + t.Run("Should generate the proper combinedRewards", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + combinedRewards, err := rewards.GenerateCombinedRewards() + assert.Nil(t, err) + assert.NotNil(t, combinedRewards) + + t.Logf("Generated %d combinedRewards", len(combinedRewards)) + + assert.Equal(t, 192, len(combinedRewards)) + }) + t.Cleanup(func() { + tests.DeleteTestSqliteDB(dbFileName) + teardownCombinedRewards(grm) + }) +} diff --git a/pkg/rewards/operatorAvsRegistrationSnaphots.go b/pkg/rewards/operatorAvsRegistrationSnaphots.go new file mode 100644 index 00000000..9367cb99 --- /dev/null +++ b/pkg/rewards/operatorAvsRegistrationSnaphots.go @@ -0,0 +1,172 @@ +package rewards + +import ( + "database/sql" +) + +// Operator AVS Registration Windows: Ranges at which an operator has registered for an AVS +// 0. Ranked: Rank the operator state changes by block_time and log_index since sqlite lacks LEAD/LAG functions +// 1. Marked_statuses: Denote which registration status comes after one another +// 2. Removed_same_day_deregistrations: Remove a pairs of (registration, deregistration) that happen on the same day +// 3. Registration_periods: Combine registration together, only select registrations with: +// a. (Registered, Unregistered) +// b. (Registered, Null). If null, the end time is the current timestamp +// 4. Registration_snapshots: Round up each start_time to 0 UTC on NEXT DAY and round down each end_time to 0 UTC on CURRENT DAY +// 5. Operator_avs_registration_windows: Ranges that start and end on same day are invalid +// Note: We cannot assume that the operator is registered for avs at end_time because it is +// Payments calculations should only be done on snapshots from the PREVIOUS DAY. For example say we have the following: +// <-----0-------1-------2------3------> +// ^ ^ +// Entry Exit +// Since exits (deregistrations) are rounded down, we must only look at the day 2 snapshot on a pipeline run on day 3. +const operatorAvsRegistrationSnapshotsQuery = ` +WITH state_changes as ( + select + aosc.*, + b.block_time as block_time, + DATE(b.block_time) as block_date + from avs_operator_state_changes as aosc + left join blocks as b on (b.number = aosc.block_number) +), +marked_statuses as ( + SELECT + operator, + avs, + registered, + block_time, + DATE(block_time) as block_date, + -- Mark the next action as next_block_time + LEAD(block_time) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_time, + -- The below lead/lag combinations are only used in the next CTE + -- Get the next row's registered status and block_date + LEAD(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_registration_status, + LEAD(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_date, + -- Get the previous row's registered status and block_date + LAG(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_registered, + LAG(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_block_date + FROM state_changes +), +-- Ignore a (registration,deregistration) pairs that happen on the exact same date +removed_same_day_deregistrations as ( + SELECT * from marked_statuses + WHERE NOT ( + -- Remove the registration part + (registered = TRUE AND + COALESCE(next_registration_status = FALSE, false) AND -- default to false if null + COALESCE(block_date = next_block_date, false)) OR + -- Remove the deregistration part + (registered = FALSE AND + COALESCE(prev_registered = TRUE, false) and + COALESCE(block_date = prev_block_date, false) + ) + ) +), +-- Combine corresponding registrations into a single record +-- start_time is the beginning of the record +registration_periods AS ( + SELECT + operator, + avs, + block_time AS start_time, + -- Mark the next_block_time as the end_time for the range + -- Use coalesce because if the next_block_time for a registration is not closed, then we use cutoff_date + COALESCE(next_block_time, DATETIME(@cutoffDate)) AS end_time, + registered + FROM removed_same_day_deregistrations + WHERE registered = TRUE +), +-- Round UP each start_time and round DOWN each end_time +registration_windows_extra as ( + SELECT + operator, + avs, + DATE(start_time, '+1 day') as start_time, + -- End time is end time non inclusive because the operator is not registered on the AVS at the end time OR it is current timestamp rounded up + DATE(end_time) as end_time + FROM registration_periods +), +-- Ignore start_time and end_time that last less than a day +operator_avs_registration_windows as ( + SELECT * from registration_windows_extra + WHERE start_time != end_time +), +cleaned_records AS ( + SELECT * FROM operator_avs_registration_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from operator_avs_registration_windows +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + select + operator, + avs, + day as snapshot + from cleaned_records + cross join day_series +where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type OperatorAvsRegistrationSnapshots struct { + Avs string + Operator string + Snapshot string +} + +// GenerateOperatorAvsRegistrationSnapshots returns a list of OperatorAvsRegistrationSnapshots objects +func (r *RewardsCalculator) GenerateOperatorAvsRegistrationSnapshots(snapshotDate string) ([]*OperatorAvsRegistrationSnapshots, error) { + results := make([]*OperatorAvsRegistrationSnapshots, 0) + + res := r.grm.Raw(operatorAvsRegistrationSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS registration windows", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertOperatorAvsRegistrationSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateOperatorAvsRegistrationSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS registration snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting operator AVS registration snapshots", "count", len(snapshots)) + + res := r.grm.Model(&OperatorAvsRegistrationSnapshots{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert operator AVS registration snapshots", "error", res.Error) + return err + } + return nil +} + +func (r *RewardsCalculator) CreateOperatorAvsRegistrationSnapshotsTable() error { + res := r.grm.Exec(`CREATE TABLE IF NOT EXISTS operator_avs_registration_snapshots ( + avs TEXT, + operator TEXT, + snapshot TEXT + )`) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create operator_avs_registration_snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/operatorAvsRegistrationSnaphots_test.go b/pkg/rewards/operatorAvsRegistrationSnaphots_test.go new file mode 100644 index 00000000..4725ca12 --- /dev/null +++ b/pkg/rewards/operatorAvsRegistrationSnaphots_test.go @@ -0,0 +1,159 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "slices" + "testing" +) + +func setupOperatorAvsRegistrationSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownOperatorAvsRegistrationSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from avs_operator_state_changes`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateOperatorAvsStateChangesTable(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetOperatorAvsRegistrationsSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_OperatorAvsRegistrationSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupOperatorAvsRegistrationSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate blocks and operatorAvsStateChanges tables", func(t *testing.T) { + err := hydrateAllBlocksTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query := "select count(*) from blocks" + var count int + res := grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, TOTAL_BLOCK_COUNT, count) + + err = hydrateOperatorAvsStateChangesTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query = "select count(*) from avs_operator_state_changes" + res = grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, 20442, count) + }) + t.Run("Should generate the proper operatorAvsRegistrationWindows", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + snapshots, err := rewards.GenerateOperatorAvsRegistrationSnapshots(snapshotDate) + assert.Nil(t, err) + assert.NotNil(t, snapshots) + + t.Logf("Generated %d snapshots", len(snapshots)) + + expectedResults, err := tests.GetExpectedOperatorAvsSnapshotResults(projectRoot) + assert.Nil(t, err) + + t.Logf("Expected %d snapshots", len(expectedResults)) + assert.Equal(t, len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*OperatorAvsRegistrationSnapshots, 0) + + mappedExpectedResults := make(map[string][]string) + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s", expectedResult.Operator, expectedResult.Avs) + if _, ok := mappedExpectedResults[slotId]; !ok { + mappedExpectedResults[slotId] = make([]string, 0) + } + mappedExpectedResults[slotId] = append(mappedExpectedResults[slotId], expectedResult.Snapshot) + } + + // If the two result sets are different lengths, we need to find out why. + if len(expectedResults) != len(snapshots) { + // Go line-by-line in the window results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, window := range snapshots { + slotId := fmt.Sprintf("%s_%s", window.Operator, window.Avs) + found, ok := mappedExpectedResults[slotId] + if !ok { + t.Logf("Operator/AVS not found in results: %+v\n", window) + lacksExpectedResult = append(lacksExpectedResult, window) + } else { + if !slices.Contains(found, window.Snapshot) { + t.Logf("Found operator/AVS, but no snapshot: %+v - %+v\n", window, found) + lacksExpectedResult = append(lacksExpectedResult, window) + } + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Window: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + tests.DeleteTestSqliteDB(dbFileName) + teardownOperatorAvsRegistrationSnapshot(grm) + }) +} diff --git a/pkg/rewards/operatorAvsStrategySnapshots.go b/pkg/rewards/operatorAvsStrategySnapshots.go new file mode 100644 index 00000000..cfc8ce47 --- /dev/null +++ b/pkg/rewards/operatorAvsStrategySnapshots.go @@ -0,0 +1,231 @@ +package rewards + +import ( + "database/sql" +) + +// Operator AVS Strategy Windows: Ranges for which an Operator, Strategy is restaked on an AVS +// 1. Ranked_records: Order all records. Round up block_time to 0 UTC +// 2. Latest_records: Get latest records for each (operator, avs, strategy, day) combination +// 3. Grouped_records: Find the next start_time for each (operator, avs, strategy) combination +// 4. Parsed_ranges: Complicated step. Here, we set end_time = start_time in three cases: +// Case 1: end_time is null because there are no more RPC calls made. For example, if today is 4/28 and +// the start_time is 4/27 (rounded from 4/26), there is nothing we can do on the (4/27, 4/28) +// range since it has not ended. +// Case 2: end_time is null because the (operator, strategy) combo is no longer registered +// Case 3: end_time is more than 1 day greater than start_time. In this case, if there is a new range, +// it will be accounted for. Say we have a range (4/21, 4/22, 4/23), (4/23, 4/25), (4/25, 4/26). +// The second range will be discarded since its not contiguous. We will keep (4/21-4/23) and (4/25-4/26) +// 5. Active_windows: Parse out all rows whose start_time == end_time (see above conditions) +// 6. Gaps_and_islands: Mark the previous end time for each row. If null, then start of a range +// 7. Island_detection: Mark islands if the previous end time is equal to the start time +// 8. Island_groups: Group islands by summing up ids +// 9. Operator_avs_strategy_windows: Combine ranges with same id +const operatorAvsStrategyWindowsQuery = ` +with ranked_records AS ( + SELECT + lower(operator) as operator, + lower(avs) as avs, + lower(strategy) as strategy, + block_time, + DATE(block_time, '+1 day') as start_time, + ROW_NUMBER() OVER ( + PARTITION BY operator, avs, strategy, DATE(block_time, '+1 day') + ORDER BY block_time DESC -- want latest records to be ranked highest + ) AS rn + FROM operator_restaked_strategies + WHERE avs_directory_address = lower(@avsDirectoryAddress) +), +-- Get the latest records for each (operator, avs, strategy, day) combination +latest_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + block_time, + rn + FROM ranked_records + WHERE rn = 1 +), +-- Find the next entry for each (operator,avs,strategy) grouping +grouped_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + LEAD(start_time) OVER ( + PARTITION BY operator, avs, strategy + ORDER BY start_time ASC + ) AS next_start_time + FROM latest_records +), +-- Parse out any holes (ie. any next_start_times that are not exactly one day after the current start_time) +parsed_ranges AS ( + SELECT + operator, + avs, + strategy, + start_time, + -- If the next_start_time is not on the consecutive day, close off the end_time + CASE + WHEN next_start_time IS NULL OR next_start_time > DATE(start_time, '+1 day') THEN start_time + ELSE next_start_time + END AS end_time + FROM grouped_records +), +-- Remove the (operator,avs,strategy) combos where start_time == end_time +active_windows as ( + SELECT * + FROM parsed_ranges + WHERE start_time != end_time +), +partitioned_gaps_and_islands as ( + select + operator, + avs, + strategy, + start_time, + end_time, + ROW_NUMBER() OVER (PARTITION BY operator, avs, strategy ORDER BY start_time) as rn + from active_windows +), +-- Mark the prev_end_time for each row. If new window, then gap is empty +gaps_and_islands AS ( + SELECT + operator, + avs, + strategy, + start_time, + end_time, + LAG(end_time) OVER(PARTITION BY operator, avs, strategy ORDER BY start_time) as prev_end_time + FROM active_windows +), +-- Detect islands +island_detection AS ( + SELECT operator, avs, strategy, start_time, end_time, prev_end_time, + CASE + -- If the previous end time is equal to the start time, then mark as part of the island, else create a new island + WHEN prev_end_time = start_time THEN 0 + ELSE 1 + END as new_island + FROM gaps_and_islands +), +-- Group each based on their ID +island_groups AS ( + SELECT + t1.operator, + t1.avs, + t1.strategy, + t1.start_time, + t1.end_time, + ( + SELECT SUM(t2.new_island) + FROM island_detection t2 + WHERE t2.operator = t1.operator + AND t2.avs = t1.avs + AND t2.strategy = t1.strategy + AND t2.start_time <= t1.start_time + ) AS island_id + FROM island_detection t1 + ORDER BY t1.operator, t1.avs, t1.strategy, t1.start_time +), +operator_avs_strategy_windows AS ( + SELECT + operator, + avs, + strategy, + MIN(start_time) AS start_time, + MAX(end_time) AS end_time + FROM island_groups + GROUP BY operator, avs, strategy, island_id + ORDER BY operator, avs, strategy, start_time +), +cleaned_records AS ( + SELECT * + FROM operator_avs_strategy_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + select + operator, + avs, + strategy, + day as snapshot + from cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results; +` + +type OperatorAvsStrategySnapshot struct { + Operator string + Avs string + Strategy string + Snapshot string +} + +func (r *RewardsCalculator) GenerateOperatorAvsStrategySnapshots(snapshotDate string) ([]*OperatorAvsStrategySnapshot, error) { + results := make([]*OperatorAvsStrategySnapshot, 0) + + contractAddresses := r.globalConfig.GetContractsMapForChain() + + res := r.grm.Raw(operatorAvsStrategyWindowsQuery, sql.Named("avsDirectoryAddress", contractAddresses.AvsDirectory)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS strategy windows", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertOperatorAvsStrategySnapshots(snapshotDate string) error { + snapshots, err := r.GenerateOperatorAvsStrategySnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS strategy snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting operator AVS strategy snapshots", "count", len(snapshots)) + res := r.grm.Model(&OperatorAvsStrategySnapshot{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert operator AVS strategy snapshots", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateOperatorAvsStrategySnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS operator_avs_strategy_snapshots ( + operator VARCHAR(42) NOT NULL, + avs VARCHAR(42) NOT NULL, + strategy VARCHAR(42) NOT NULL, + snapshot DATE NOT NULL + ); + `) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create operator_avs_strategy_snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/operatorAvsStrategySnapshots_test.go b/pkg/rewards/operatorAvsStrategySnapshots_test.go new file mode 100644 index 00000000..5c1932e1 --- /dev/null +++ b/pkg/rewards/operatorAvsStrategySnapshots_test.go @@ -0,0 +1,148 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "slices" + "strings" + "testing" +) + +func setupOperatorAvsStrategyWindows() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + cfg.Chain = config.Chain_Holesky + + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownOperatorAvsStrategyWindows(grm *gorm.DB) { + queries := []string{ + `delete from operator_avs_strategy_snapshots`, + } + for _, query := range queries { + grm.Exec(query) + } +} + +func hydrateOperatorAvsRestakedStrategies(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetOperatorAvsRestakedStrategiesSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_OperatorAvsStrategySnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupOperatorAvsStrategyWindows() + + if err != nil { + t.Fatal(err) + } + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + t.Log("Hydrating restaked strategies") + err := hydrateOperatorAvsRestakedStrategies(grm, l) + if err != nil { + t.Fatal(err) + } + + query := `select count(*) from operator_restaked_strategies` + var count int + res := grm.Raw(query).Scan(&count) + + assert.Nil(t, res.Error) + assert.Equal(t, 3144978, count) + }) + + t.Run("Should calculate correct operatorAvsStrategy windows", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating snapshots") + windows, err := rewards.GenerateOperatorAvsStrategySnapshots("2024-09-01") + assert.Nil(t, err) + + t.Log("Getting expected results") + expectedResults, err := tests.GetExpectedOperatorAvsSnapshots(projectRoot) + assert.Nil(t, err) + t.Logf("Loaded %d expected results", len(expectedResults)) + + assert.Equal(t, len(expectedResults), len(windows)) + + // Memoize to make lookups faster rather than n^2 + mappedExpectedResults := make(map[string][]string) + for _, r := range expectedResults { + slotId := strings.ToLower(fmt.Sprintf("%s_%s_%s", r.Operator, r.Avs, r.Strategy)) + val, ok := mappedExpectedResults[slotId] + if !ok { + mappedExpectedResults[slotId] = make([]string, 0) + } + mappedExpectedResults[slotId] = append(val, r.Snapshot) + } + + lacksExpectedResult := make([]*OperatorAvsStrategySnapshot, 0) + // Go line-by-line in the window results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, window := range windows { + slotId := strings.ToLower(fmt.Sprintf("%s_%s_%s", window.Operator, window.Avs, window.Strategy)) + + found, ok := mappedExpectedResults[slotId] + if !ok { + lacksExpectedResult = append(lacksExpectedResult, window) + t.Logf("Could not find expected result for %+v", window) + continue + } + if !slices.Contains(found, window.Snapshot) { + t.Logf("Found result, but snapshot doesnt match: %+v - %v", window, found) + lacksExpectedResult = append(lacksExpectedResult, window) + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + //if len(lacksExpectedResult) > 0 { + // for i, window := range lacksExpectedResult { + // fmt.Printf("%d - Snapshot: %+v\n", i, window) + // } + //} + }) + t.Cleanup(func() { + teardownOperatorAvsStrategyWindows(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/rewards/operatorShareSnapshots.go b/pkg/rewards/operatorShareSnapshots.go new file mode 100644 index 00000000..ecc18525 --- /dev/null +++ b/pkg/rewards/operatorShareSnapshots.go @@ -0,0 +1,132 @@ +package rewards + +import ( + "database/sql" + "go.uber.org/zap" +) + +const operatorShareSnapshotsQuery = ` +with operator_shares_with_block_info as ( + select + os.operator, + os.strategy, + os.shares, + os.block_number, + b.block_time, + DATE(b.block_time) as block_date + from operator_shares as os + left join blocks as b on (b.number = os.block_number) +), +ranked_operator_records as ( + select + *, + ROW_NUMBER() OVER (PARTITION BY operator, strategy, block_date ORDER BY block_time DESC) as rn + from operator_shares_with_block_info as os +), +-- Get the latest record for each day & round up to the snapshot day +snapshotted_records as ( + SELECT + operator, + strategy, + shares, + block_time, + DATE(block_date, '+1 day') as snapshot_time + from ranked_operator_records + where rn = 1 +), +-- Get the range for each operator, strategy pairing +operator_share_windows as ( + SELECT + operator, strategy, shares, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the current timestamp truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) is null THEN DATE(@cutoffDate) + ELSE LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records +), +cleaned_records as ( + SELECT * FROM operator_share_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + SELECT + operator, + strategy, + shares, + day as snapshot + FROM cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type OperatorShareSnapshots struct { + Operator string + Strategy string + Shares string + Snapshot string +} + +func (r *RewardsCalculator) GenerateOperatorShareSnapshots(snapshotDate string) ([]*OperatorShareSnapshots, error) { + results := make([]*OperatorShareSnapshots, 0) + + res := r.grm.Raw(operatorShareSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate operator share snapshots", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertOperatorShareSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateOperatorShareSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate operator share snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting operator share snapshots", "count", len(snapshots)) + res := r.grm.Model(&OperatorShareSnapshots{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert operator share snapshots", "error", res.Error) + return res.Error + } + + return nil +} + +func (r *RewardsCalculator) CreateOperatorSharesSnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS operator_share_snapshots ( + operator TEXT, + strategy TEXT, + shares TEXT, + snapshot TEXT + ) + `) + if res.Error != nil { + r.logger.Error("Failed to create operator share snapshots table", zap.Error(res.Error)) + return res.Error + } + return nil +} diff --git a/pkg/rewards/operatorShareSnapshots_test.go b/pkg/rewards/operatorShareSnapshots_test.go new file mode 100644 index 00000000..89cd01ad --- /dev/null +++ b/pkg/rewards/operatorShareSnapshots_test.go @@ -0,0 +1,142 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "testing" +) + +func setupOperatorShareSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownOperatorShareSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from operator_shares`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateOperatorShares(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetOperatorSharesSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_OperatorShareSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupOperatorShareSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + if err = hydrateAllBlocksTable(grm, l); err != nil { + t.Error(err) + } + if err = hydrateOperatorShares(grm, l); err != nil { + t.Error(err) + } + }) + t.Run("Should generate operator share snapshots", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating operator share snapshots") + snapshots, err := rewards.GenerateOperatorShareSnapshots(snapshotDate) + assert.Nil(t, err) + + t.Log("Loading expected results") + expectedResults, err := tests.GetOperatorSharesExpectedResults(projectRoot) + assert.Nil(t, err) + + assert.Equal(t, len(expectedResults), len(snapshots)) + + mappedExpectedResults := make(map[string]string) + + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s_%s", expectedResult.Operator, expectedResult.Strategy, expectedResult.Snapshot) + mappedExpectedResults[slotId] = expectedResult.Shares + } + + if len(expectedResults) != len(snapshots) { + t.Errorf("Expected %d snapshots, got %d", len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*OperatorShareSnapshots, 0) + // Go line-by-line in the snapshot results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, snapshot := range snapshots { + + slotId := fmt.Sprintf("%s_%s_%s", snapshot.Operator, snapshot.Strategy, snapshot.Snapshot) + + found, ok := mappedExpectedResults[slotId] + if !ok { + t.Logf("Record not found %+v", snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + continue + } + if found != snapshot.Shares { + t.Logf("Expected: %s, Got: %s for %+v", found, snapshot.Shares, snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Snapshot: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + teardownOperatorShareSnapshot(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/rewards/rewards.go b/pkg/rewards/rewards.go new file mode 100644 index 00000000..f50b3b06 --- /dev/null +++ b/pkg/rewards/rewards.go @@ -0,0 +1,173 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/submittedDistributionRoots" + "go.uber.org/zap" + "gorm.io/gorm" + "time" +) + +type RewardsCalculator struct { + logger *zap.Logger + grm *gorm.DB + globalConfig *config.Config +} + +func NewRewardsCalculator( + l *zap.Logger, + grm *gorm.DB, + cfg *config.Config, +) (*RewardsCalculator, error) { + rc := &RewardsCalculator{ + logger: l, + grm: grm, + globalConfig: cfg, + } + + if err := rc.initializeRewardsSchema(); err != nil { + l.Sugar().Errorw("Failed to initialize rewards schema", zap.Error(err)) + return nil, err + } + return rc, nil +} + +// CalculateRewardsForSnapshot calculates the rewards for a given snapshot date. +// +// Rewards are calculated for the period between the last snapshot published on-chain +// via the DistributionRootSubmitted event and the desired snapshot date (exclusive). +// +// If there is no previous DistributionRoot, the rewards are calculated from EigenLayer Genesis. +func (rc *RewardsCalculator) CalculateRewardsForSnapshot(desiredSnapshotDate uint64) error { + // First make sure that the snapshot date is valid as provided. + // The time should be at 00:00:00 UTC. and should be in the past. + snapshotDate := time.Unix(int64(desiredSnapshotDate), 0).UTC() + + if !rc.isValidSnapshotDate(snapshotDate) { + return fmt.Errorf("invalid snapshot date '%s'", snapshotDate.String()) + } + + // Get the last snapshot date published on-chain. + distributionRoot, err := rc.getMostRecentDistributionRoot() + if err != nil { + rc.logger.Error("Failed to get the most recent distribution root", zap.Error(err)) + return err + } + + var lowerBoundBlockNumber uint64 + if distributionRoot != nil { + lowerBoundBlockNumber = distributionRoot.BlockNumber + } else { + lowerBoundBlockNumber = rc.globalConfig.GetGenesisBlockNumber() + } + + rc.logger.Sugar().Infow("Calculating rewards for snapshot date", + zap.String("snapshot_date", snapshotDate.String()), + zap.Uint64("lowerBoundBlockNumber", lowerBoundBlockNumber), + ) + + // Calculate the rewards for the period. + // TODO(seanmcgary): lower bound should be either 0 (i.e. 1970-01-01) or the snapshot of the previously calculated rewards. + return rc.calculateRewards("", snapshotDate) +} + +func (rc *RewardsCalculator) isValidSnapshotDate(snapshotDate time.Time) bool { + // Check if the snapshot date is in the past. + // The snapshot date should be at 00:00:00 UTC. + if snapshotDate.After(time.Now().UTC()) { + rc.logger.Error("Snapshot date is in the future") + return false + } + + if snapshotDate.Hour() != 0 || snapshotDate.Minute() != 0 || snapshotDate.Second() != 0 { + rc.logger.Error("Snapshot date is not at 00:00:00 UTC") + return false + } + + return true +} + +func (rc *RewardsCalculator) getMostRecentDistributionRoot() (*submittedDistributionRoots.SubmittedDistributionRoots, error) { + var distributionRoot *submittedDistributionRoots.SubmittedDistributionRoots + res := rc.grm.Model(&submittedDistributionRoots.SubmittedDistributionRoots{}).Order("block_number desc").First(&distributionRoot) + if res != nil { + return nil, res.Error + } + return distributionRoot, nil +} + +func (rc *RewardsCalculator) initializeRewardsSchema() error { + funcs := []func() error{ + rc.CreateOperatorAvsRegistrationSnapshotsTable, + rc.CreateOperatorAvsStrategySnapshotsTable, + rc.CreateOperatorSharesSnapshotsTable, + rc.CreateStakerShareSnapshotsTable, + rc.CreateStakerDelegationSnapshotsTable, + rc.CreateCombinedRewardsTable, + + // Gold tables + rc.CreateGold1ActiveRewardsTable, + rc.CreateGold2RewardAmountsTable, + rc.CreateGold3OperatorRewardsTable, + rc.CreateGold4RewardsForAllTable, + rc.CreateGold5RfaeStakersTable, + rc.CreateGold6RfaeOperatorsTable, + rc.CreateGold7StagingTable, + rc.Create8GoldTable, + } + for _, f := range funcs { + err := f() + if err != nil { + return err + } + } + return nil +} + +func (rc *RewardsCalculator) generateSnapshotData(snapshotDate string) error { + var err error + + if err = rc.GenerateAndInsertCombinedRewards(); err != nil { + rc.logger.Sugar().Errorw("Failed to generate combined rewards", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated combined rewards") + + if err = rc.GenerateAndInsertOperatorAvsRegistrationSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate operator AVS registration snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated operator AVS registration snapshots") + + if err = rc.GenerateAndInsertOperatorAvsStrategySnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate operator AVS strategy snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated operator AVS strategy snapshots") + + if err = rc.GenerateAndInsertOperatorShareSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate operator share snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated operator share snapshots") + + if err = rc.GenerateAndInsertStakerShareSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate staker share snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated staker share snapshots") + + if err = rc.GenerateAndInsertStakerDelegationSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate staker delegation snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated staker delegation snapshots") + + return nil +} + +func (rc *RewardsCalculator) calculateRewards(previousSnapshotDate string, snapshotDate time.Time) error { + + return nil +} diff --git a/pkg/rewards/rewards_test.go b/pkg/rewards/rewards_test.go new file mode 100644 index 00000000..e9c497a3 --- /dev/null +++ b/pkg/rewards/rewards_test.go @@ -0,0 +1,191 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/Layr-Labs/go-sidecar/internal/types/numbers" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "os" + "path/filepath" + "slices" + "testing" +) + +const TOTAL_BLOCK_COUNT = 1229187 + +func rewardsTestsEnabled() bool { + return os.Getenv("TEST_REWARDS") == "true" +} + +func getProjectRootPath() string { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + p, err := filepath.Abs(fmt.Sprintf("%s/../..", wd)) + if err != nil { + panic(err) + } + return p +} + +func hydrateAllBlocksTable(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetAllBlocksSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func setupRewards() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + cfg.Debug = true + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownRewards(grm *gorm.DB) { + teardownOperatorAvsRegistrationSnapshot(grm) + teardownOperatorAvsStrategyWindows(grm) + teardownOperatorShareSnapshot(grm) + teardownStakerDelegationSnapshot(grm) + teardownStakerShareSnapshot(grm) +} + +func Test_Rewards(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + if err := numbers.InitPython(); err != nil { + t.Fatal(err) + } + + dbFileName, cfg, grm, l, err := setupRewards() + fmt.Printf("Using db file: %+v\n", dbFileName) + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should initialize the rewards calculator", func(t *testing.T) { + rc, err := NewRewardsCalculator(l, grm, cfg) + assert.Nil(t, err) + if err != nil { + t.Fatal(err) + } + assert.NotNil(t, rc) + + fmt.Printf("DB Path: %+v", dbFileName) + + query := `select name from main.sqlite_master where type = 'table' order by name asc` + type row struct{ Name string } + var tables []row + res := rc.grm.Raw(query).Scan(&tables) + assert.Nil(t, res.Error) + + expectedTables := []string{ + "combined_rewards", + "gold_1_active_rewards", + "gold_2_staker_reward_amounts", + "gold_3_operator_reward_amounts", + "gold_4_rewards_for_all", + "gold_5_rfae_stakers", + "gold_6_rfae_operators", + "gold_7_staging", + "gold_table", + "operator_avs_registration_snapshots", + "operator_avs_strategy_snapshots", + "operator_share_snapshots", + "staker_delegation_snapshots", + "staker_share_snapshots", + } + tablesList := make([]string, 0) + for i, table := range tables { + fmt.Printf("[%v]: %+v\n", i, table.Name) + tablesList = append(tablesList, table.Name) + } + + for _, table := range expectedTables { + assert.True(t, slices.Contains(tablesList, table)) + } + + // Setup all tables and source data + err = hydrateAllBlocksTable(grm, l) + assert.Nil(t, err) + + err = hydrateOperatorAvsStateChangesTable(grm, l) + assert.Nil(t, err) + + err = hydrateOperatorAvsRestakedStrategies(grm, l) + assert.Nil(t, err) + + err = hydrateOperatorShares(grm, l) + assert.Nil(t, err) + + err = hydrateStakerDelegations(grm, l) + assert.Nil(t, err) + + err = hydrateStakerShares(grm, l) + assert.Nil(t, err) + + err = hydrateRewardSubmissionsTable(grm, l) + assert.Nil(t, err) + + t.Log("Hydrated tables") + + // Generate snapshots + err = rc.generateSnapshotData(snapshotDate) + assert.Nil(t, err) + + t.Log("Generated and inserted snapshots") + + startDate := "1970-01-01" + err = rc.GenerateActiveRewards(snapshotDate, startDate) + assert.Nil(t, err) + + query = `select count(*) from gold_1_active_rewards` + var count int + res = rc.grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + fmt.Printf("Count: %v\n", count) + + fmt.Printf("Done!\n\n") + t.Cleanup(func() { + teardownRewards(grm) + // tests.DeleteTestSqliteDB(dbFileName) + }) + }) +} diff --git a/pkg/rewards/stakerDelegationSnapshots.go b/pkg/rewards/stakerDelegationSnapshots.go new file mode 100644 index 00000000..6e7e56b4 --- /dev/null +++ b/pkg/rewards/stakerDelegationSnapshots.go @@ -0,0 +1,124 @@ +package rewards + +import "database/sql" + +const stakerDelegationSnapshotsQuery = ` +with staker_delegations_with_block_info as ( + select + sdc.staker, + case when sdc.delegated = false then '0x0000000000000000000000000000000000000000' else sdc.operator end as operator, + sdc.log_index, + sdc.block_number, + b.block_time, + DATE(b.block_time) as block_date + from staker_delegation_changes as sdc + left join blocks as b on (b.number = sdc.block_number) +), +ranked_staker_records as ( +SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, block_date ORDER BY block_time DESC, log_index desc) AS rn +FROM staker_delegations_with_block_info +), +-- Get the latest record for each day & round up to the snapshot day +snapshotted_records as ( + SELECT + staker, + operator, + block_time, + DATE(block_date, '+1 day') as snapshot_time + from ranked_staker_records + where rn = 1 +), +-- Get the range for each operator, strategy pairing +staker_share_windows as ( + SELECT + staker, operator, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the current timestamp truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) is null THEN DATE(@cutoffDate) + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records +), +cleaned_records as ( + SELECT * FROM staker_share_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + SELECT + staker, + operator, + day as snapshot + FROM cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type StakerDelegationSnapshot struct { + Staker string + Operator string + Snapshot string +} + +func (r *RewardsCalculator) GenerateStakerDelegationSnapshots(snapshotDate string) ([]*StakerDelegationSnapshot, error) { + results := make([]*StakerDelegationSnapshot, 0) + + res := r.grm.Raw(stakerDelegationSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate staker delegation snapshots", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertStakerDelegationSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateStakerDelegationSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate staker delegation snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting staker delegation snapshots", "count", len(snapshots)) + res := r.grm.Model(&StakerDelegationSnapshot{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert staker delegation snapshots", "error", res.Error) + return res.Error + } + + return nil +} + +func (r *RewardsCalculator) CreateStakerDelegationSnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS staker_delegation_snapshots ( + staker TEXT, + operator TEXT, + snapshot TEXT + ) + `) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create staker delegation snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/stakerDelegationSnapshots_test.go b/pkg/rewards/stakerDelegationSnapshots_test.go new file mode 100644 index 00000000..10850c13 --- /dev/null +++ b/pkg/rewards/stakerDelegationSnapshots_test.go @@ -0,0 +1,143 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "slices" + "testing" +) + +func setupStakerDelegationSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownStakerDelegationSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from staker_delegation_changes`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateStakerDelegations(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetStakerDelegationsSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_StakerDelegationSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupStakerDelegationSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + if err := hydrateAllBlocksTable(grm, l); err != nil { + t.Error(err) + } + if err := hydrateStakerDelegations(grm, l); err != nil { + t.Error(err) + } + }) + t.Run("Should generate staker share snapshots", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating staker delegation snapshots") + snapshots, err := rewards.GenerateStakerDelegationSnapshots(snapshotDate) + assert.Nil(t, err) + + t.Log("Getting expected results") + expectedResults, err := tests.GetStakerDelegationExpectedResults(projectRoot) + assert.Nil(t, err) + + assert.Equal(t, len(expectedResults), len(snapshots)) + + mappedExpectedResults := make(map[string][]string) + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s", expectedResult.Staker, expectedResult.Operator) + if _, ok := mappedExpectedResults[slotId]; !ok { + mappedExpectedResults[slotId] = make([]string, 0) + } + mappedExpectedResults[slotId] = append(mappedExpectedResults[slotId], expectedResult.Snapshot) + } + + if len(expectedResults) != len(snapshots) { + t.Errorf("Expected %d snapshots, got %d", len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*StakerDelegationSnapshot, 0) + // Go line-by-line in the snapshot results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, snapshot := range snapshots { + slotId := fmt.Sprintf("%s_%s", snapshot.Staker, snapshot.Operator) + found, ok := mappedExpectedResults[slotId] + if !ok { + t.Logf("Staker/operator not found in results: %+v\n", snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } else { + if !slices.Contains(found, snapshot.Snapshot) { + t.Logf("Found staker operator, but no snapshot: %+v - %+v\n", snapshot, found) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Snapshot: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + teardownStakerDelegationSnapshot(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/rewards/stakerShareSnapshots.go b/pkg/rewards/stakerShareSnapshots.go new file mode 100644 index 00000000..6b92846c --- /dev/null +++ b/pkg/rewards/stakerShareSnapshots.go @@ -0,0 +1,127 @@ +package rewards + +import "database/sql" + +const stakerShareSnapshotsQuery = ` +with staker_shares_with_block_info as ( + select + ss.staker, + ss.strategy, + ss.shares, + ss.block_number, + b.block_time, + DATE(b.block_time) as block_date + from staker_shares as ss + left join blocks as b on (b.number = ss.block_number) +), +ranked_staker_records as ( +SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, strategy, block_date ORDER BY block_time DESC) AS rn +FROM staker_shares_with_block_info +), +-- Get the latest record for each day & round up to the snapshot day +snapshotted_records as ( + SELECT + staker, + strategy, + shares, + block_time, + DATE(block_date, '+1 day') as snapshot_time + from ranked_staker_records + where rn = 1 +), +-- Get the range for each operator, strategy pairing +staker_share_windows as ( + SELECT + staker, strategy, shares, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the current timestamp truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) is null THEN DATE(@cutoffDate) + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records +), +cleaned_records as ( + SELECT * FROM staker_share_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + SELECT + staker, + strategy, + shares, + day as snapshot + FROM cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type StakerShareSnapshot struct { + Staker string + Strategy string + Snapshot string + Shares string +} + +func (r *RewardsCalculator) GenerateStakerShareSnapshots(snapshotDate string) ([]*StakerShareSnapshot, error) { + results := make([]*StakerShareSnapshot, 0) + + res := r.grm.Raw(stakerShareSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate staker share snapshots", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertStakerShareSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateStakerShareSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate staker share snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting staker share snapshots", "count", len(snapshots)) + res := r.grm.Model(&StakerShareSnapshot{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert staker share snapshots", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateStakerShareSnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS staker_share_snapshots ( + staker TEXT, + strategy TEXT, + shares TEXT, + snapshot TEXT + ) + `) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create staker share snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/stakerShareSnapshots_test.go b/pkg/rewards/stakerShareSnapshots_test.go new file mode 100644 index 00000000..0395d226 --- /dev/null +++ b/pkg/rewards/stakerShareSnapshots_test.go @@ -0,0 +1,140 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "testing" +) + +func setupStakerShareSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownStakerShareSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from staker_shares`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateStakerShares(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetStakerSharesSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error), zap.String("query", contents)) + return res.Error + } + return nil +} + +func Test_StakerShareSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupStakerShareSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + if err = hydrateAllBlocksTable(grm, l); err != nil { + t.Error(err) + } + if err = hydrateStakerShares(grm, l); err != nil { + t.Error(err) + } + }) + t.Run("Should generate staker share snapshots", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating staker share snapshots") + snapshots, err := rewards.GenerateStakerShareSnapshots(snapshotDate) + assert.Nil(t, err) + + t.Log("Getting expected results") + expectedResults, err := tests.GetStakerSharesExpectedResults(projectRoot) + assert.Nil(t, err) + + assert.Equal(t, len(expectedResults), len(snapshots)) + + t.Log("Comparing results") + mappedExpectedResults := make(map[string]string) + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s_%s", expectedResult.Staker, expectedResult.Strategy, expectedResult.Snapshot) + mappedExpectedResults[slotId] = expectedResult.Shares + } + + if len(expectedResults) != len(snapshots) { + t.Errorf("Expected %d snapshots, got %d", len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*StakerShareSnapshot, 0) + // Go line-by-line in the snapshot results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, snapshot := range snapshots { + slotId := fmt.Sprintf("%s_%s_%s", snapshot.Staker, snapshot.Strategy, snapshot.Snapshot) + + found, ok := mappedExpectedResults[slotId] + if !ok { + lacksExpectedResult = append(lacksExpectedResult, snapshot) + continue + } + if found != snapshot.Shares { + t.Logf("Record found, but shares dont match. Expected %s, got %+v", found, snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Snapshot: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + teardownStakerShareSnapshot(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 5e9ffea9..abb779f8 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -19,6 +19,15 @@ func Filter[A any](coll []A, criteria func(i A) bool) []A { return out } +func Find[A any](coll []*A, criteria func(i *A) bool) *A { + for _, item := range coll { + if criteria(item) { + return item + } + } + return nil +} + func Reduce[A any, B any](coll []A, processor func(accum B, next A) B, initialState B) B { val := initialState for _, item := range coll { diff --git a/scripts/downloadTestData.sh b/scripts/downloadTestData.sh new file mode 100755 index 00000000..39f07f1f --- /dev/null +++ b/scripts/downloadTestData.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +version=$(cat .testdataVersion) +bucketName="eigenlayer-sidecar-testdata" + +dataUrl="https://${bucketName}.s3.amazonaws.com/${version}.tar" + +if [[ -z $version ]]; then + echo "No version found in .testdataVersion" + exit 1 +fi +echo "Downloading testdata version $dataUrl" +curl -L $dataUrl | tar xvz -C ./ diff --git a/scripts/updateTestData.sh b/scripts/updateTestData.sh new file mode 100755 index 00000000..7daacbbd --- /dev/null +++ b/scripts/updateTestData.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +bucketName="eigenlayer-sidecar-testdata" +testdataVersionFile="./.testdataVersion" + +if git status --porcelain | grep -q .; +then + echo "You have uncommitted changes. Please commit or stash them before running this script." + # exit 1 +fi + +newVersion=$(git rev-parse HEAD) + +currentVersion=$(cat $testdataVersionFile) +if [[ -z $currentVersion ]]; then + echo "No current version found" +else + echo "Current version: $currentVersion" +fi + +if [[ $currentVersion == $newVersion ]]; then + echo "Current version is the same as the new version. Exiting." + exit 0 +fi + +echo "New version: $newVersion" + +tar -cvf "${newVersion}.tar" internal/tests/testdata + +aws s3 cp "${newVersion}.tar" "s3://${bucketName}/" + +rm "${newVersion}.tar" + +echo -n $newVersion > $testdataVersionFile + +git add . +git commit -m "Updated testdata version to $newVersion" + diff --git a/sql/schema.sql b/sql/schema.sql deleted file mode 100644 index 8c391d37..00000000 --- a/sql/schema.sql +++ /dev/null @@ -1,737 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 14.11 (Homebrew) --- Dumped by pg_dump version 15.8 (Homebrew) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - --- --- Name: public; Type: SCHEMA; Schema: -; Owner: - --- - --- *not* creating schema, since initdb creates it - - -SET default_tablespace = ''; - -SET default_table_access_method = heap; - --- --- Name: avs_operator_changes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.avs_operator_changes ( - id integer NOT NULL, - operator character varying, - avs character varying, - registered boolean, - transaction_hash character varying, - transaction_index bigint, - log_index bigint, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: avs_operator_changes_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.avs_operator_changes_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: avs_operator_changes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.avs_operator_changes_id_seq OWNED BY public.avs_operator_changes.id; - - --- --- Name: blocks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.blocks ( - number bigint NOT NULL, - hash character varying(255) NOT NULL, - blob_path text NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - block_time timestamp with time zone NOT NULL -); - - --- --- Name: contracts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.contracts ( - contract_address character varying(255) NOT NULL, - contract_abi text, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - bytecode_hash character varying(64) DEFAULT NULL::character varying, - verified boolean DEFAULT false, - matching_contract_address character varying(255) DEFAULT NULL::character varying, - checked_for_proxy boolean DEFAULT false NOT NULL, - id integer NOT NULL, - checked_for_abi boolean -); - - --- --- Name: contracts_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.contracts_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: contracts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.contracts_id_seq OWNED BY public.contracts.id; - - --- --- Name: delegated_stakers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.delegated_stakers ( - staker character varying, - operator character varying, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: migrations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.migrations ( - name text NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone -); - - --- --- Name: operator_restaked_strategies; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.operator_restaked_strategies ( - id integer NOT NULL, - block_number bigint NOT NULL, - operator character varying NOT NULL, - avs character varying NOT NULL, - strategy character varying NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - block_time timestamp with time zone NOT NULL, - avs_directory_address character varying -); - - --- --- Name: operator_restaked_strategies_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.operator_restaked_strategies_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: operator_restaked_strategies_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.operator_restaked_strategies_id_seq OWNED BY public.operator_restaked_strategies.id; - - --- --- Name: operator_share_changes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.operator_share_changes ( - id integer NOT NULL, - operator character varying, - strategy character varying, - shares numeric, - transaction_hash character varying, - transaction_index bigint, - log_index bigint, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: operator_share_changes_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.operator_share_changes_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: operator_share_changes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.operator_share_changes_id_seq OWNED BY public.operator_share_changes.id; - - --- --- Name: operator_shares; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.operator_shares ( - operator character varying, - strategy character varying, - shares numeric, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: proxy_contracts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.proxy_contracts ( - block_number bigint NOT NULL, - contract_address character varying(255) NOT NULL, - proxy_contract_address character varying(255) NOT NULL, - created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - updated_at timestamp with time zone, - deleted_at timestamp with time zone -); - - --- --- Name: registered_avs_operators; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.registered_avs_operators ( - operator character varying, - avs character varying, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: staker_delegation_changes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.staker_delegation_changes ( - staker character varying, - operator character varying, - delegated boolean, - transaction_hash character varying, - log_index bigint, - transaction_index bigint, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: transaction_logs; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.transaction_logs ( - transaction_hash character varying(255) NOT NULL, - address character varying(255) NOT NULL, - arguments jsonb, - event_name character varying(255) NOT NULL, - log_index bigint NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - block_number bigint NOT NULL, - transaction_index integer NOT NULL, - output_data jsonb -); - - --- --- Name: transactions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.transactions ( - block_number bigint NOT NULL, - transaction_hash character varying(255) NOT NULL, - transaction_index bigint NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - from_address character varying(255) NOT NULL, - to_address character varying(255) DEFAULT NULL::character varying, - contract_address character varying(255) DEFAULT NULL::character varying, - bytecode_hash character varying(64) DEFAULT NULL::character varying, - gas_used numeric, - cumulative_gas_used numeric, - effective_gas_price numeric -); - - --- --- Name: unverified_contracts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.unverified_contracts ( - contract_address character varying(255) NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone -); - - --- --- Name: avs_operator_changes id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.avs_operator_changes ALTER COLUMN id SET DEFAULT nextval('public.avs_operator_changes_id_seq'::regclass); - - --- --- Name: contracts id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.contracts ALTER COLUMN id SET DEFAULT nextval('public.contracts_id_seq'::regclass); - - --- --- Name: operator_restaked_strategies id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_restaked_strategies ALTER COLUMN id SET DEFAULT nextval('public.operator_restaked_strategies_id_seq'::regclass); - - --- --- Name: operator_share_changes id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_share_changes ALTER COLUMN id SET DEFAULT nextval('public.operator_share_changes_id_seq'::regclass); - - --- --- Name: avs_operator_changes avs_operator_changes_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.avs_operator_changes - ADD CONSTRAINT avs_operator_changes_pkey PRIMARY KEY (id); - - --- --- Name: blocks block_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.blocks - ADD CONSTRAINT block_pkey PRIMARY KEY (number); - - --- --- Name: blocks blocks_unique_block_number_hash; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.blocks - ADD CONSTRAINT blocks_unique_block_number_hash UNIQUE (number, hash); - - --- --- Name: contracts contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.contracts - ADD CONSTRAINT contracts_pkey PRIMARY KEY (contract_address); - - --- --- Name: delegated_stakers delegated_stakers_staker_operator_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.delegated_stakers - ADD CONSTRAINT delegated_stakers_staker_operator_block_number_key UNIQUE (staker, operator, block_number); - - --- --- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.migrations - ADD CONSTRAINT migrations_pkey PRIMARY KEY (name); - - --- --- Name: operator_restaked_strategies operator_restaked_strategies_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_restaked_strategies - ADD CONSTRAINT operator_restaked_strategies_pkey PRIMARY KEY (id); - - --- --- Name: operator_share_changes operator_share_changes_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_share_changes - ADD CONSTRAINT operator_share_changes_pkey PRIMARY KEY (id); - - --- --- Name: operator_shares operator_shares_operator_strategy_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_shares - ADD CONSTRAINT operator_shares_operator_strategy_block_number_key UNIQUE (operator, strategy, block_number); - - --- --- Name: registered_avs_operators registered_avs_operators_operator_avs_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.registered_avs_operators - ADD CONSTRAINT registered_avs_operators_operator_avs_block_number_key UNIQUE (operator, avs, block_number); - - --- --- Name: staker_delegation_changes staker_delegation_changes_staker_operator_log_index_block_n_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.staker_delegation_changes - ADD CONSTRAINT staker_delegation_changes_staker_operator_log_index_block_n_key UNIQUE (staker, operator, log_index, block_number); - - --- --- Name: transactions transactions_transaction_hash_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transactions - ADD CONSTRAINT transactions_transaction_hash_block_number_key UNIQUE (transaction_hash, block_number); - - --- --- Name: blocks unique_blocks; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.blocks - ADD CONSTRAINT unique_blocks UNIQUE (number); - - --- --- Name: unverified_contracts unverified_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.unverified_contracts - ADD CONSTRAINT unverified_contracts_pkey PRIMARY KEY (contract_address); - - --- --- Name: idx_avs_operator_changes_avs_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_avs_operator_changes_avs_operator ON public.avs_operator_changes USING btree (avs, operator); - - --- --- Name: idx_avs_operator_changes_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_avs_operator_changes_block ON public.avs_operator_changes USING btree (block_number); - - --- --- Name: idx_bytecode_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_bytecode_hash ON public.contracts USING btree (bytecode_hash); - - --- --- Name: idx_delegated_stakers_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_delegated_stakers_block ON public.delegated_stakers USING btree (block_number); - - --- --- Name: idx_delegated_stakers_staker_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_delegated_stakers_staker_operator ON public.delegated_stakers USING btree (staker, operator); - - --- --- Name: idx_operator_share_changes_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_share_changes_block ON public.operator_share_changes USING btree (block_number); - - --- --- Name: idx_operator_share_changes_operator_strat; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_share_changes_operator_strat ON public.operator_share_changes USING btree (operator, strategy); - - --- --- Name: idx_operator_shares_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_shares_block ON public.operator_shares USING btree (block_number); - - --- --- Name: idx_operator_shares_operator_strategy; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_shares_operator_strategy ON public.operator_shares USING btree (operator, strategy); - - --- --- Name: idx_proxy_contract_contract_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_proxy_contract_contract_address ON public.proxy_contracts USING btree (contract_address); - - --- --- Name: idx_proxy_contract_proxy_contract_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_proxy_contract_proxy_contract_address ON public.proxy_contracts USING btree (proxy_contract_address); - - --- --- Name: idx_registered_avs_operators_avs_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_registered_avs_operators_avs_operator ON public.registered_avs_operators USING btree (avs, operator); - - --- --- Name: idx_registered_avs_operators_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_registered_avs_operators_block ON public.registered_avs_operators USING btree (block_number); - - --- --- Name: idx_staker_delegation_changes_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_staker_delegation_changes_block ON public.staker_delegation_changes USING btree (block_number); - - --- --- Name: idx_staker_delegation_changes_staker_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_staker_delegation_changes_staker_operator ON public.staker_delegation_changes USING btree (staker, operator); - - --- --- Name: idx_transaciton_logs_block_number; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transaciton_logs_block_number ON public.transaction_logs USING btree (block_number); - - --- --- Name: idx_transaction_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_transaction_hash ON public.transaction_logs USING btree (transaction_hash, log_index); - - --- --- Name: idx_transaction_logs_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transaction_logs_address ON public.transaction_logs USING btree (address); - - --- --- Name: idx_transaction_logs_transaction_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transaction_logs_transaction_hash ON public.transaction_logs USING btree (transaction_hash); - - --- --- Name: idx_transactions_block_number; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_block_number ON public.transactions USING btree (block_number); - - --- --- Name: idx_transactions_bytecode_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_bytecode_hash ON public.transactions USING btree (bytecode_hash); - - --- --- Name: idx_transactions_from_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_from_address ON public.transactions USING btree (from_address); - - --- --- Name: idx_transactions_to_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_to_address ON public.transactions USING btree (to_address); - - --- --- Name: idx_uniq_proxy_contract; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_uniq_proxy_contract ON public.proxy_contracts USING btree (block_number, contract_address); - - --- --- Name: idx_unique_operator_restaked_strategies; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_unique_operator_restaked_strategies ON public.operator_restaked_strategies USING btree (block_number, operator, avs, strategy); - - --- --- Name: transactions_contract_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX transactions_contract_address ON public.transactions USING btree (contract_address); - - --- --- Name: transaction_logs fk_transaction_hash_block_number_key; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transaction_logs - ADD CONSTRAINT fk_transaction_hash_block_number_key FOREIGN KEY (transaction_hash, block_number) REFERENCES public.transactions(transaction_hash, block_number) ON DELETE CASCADE; - - --- --- Name: transaction_logs transaction_logs_block_number_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transaction_logs - ADD CONSTRAINT transaction_logs_block_number_fkey FOREIGN KEY (block_number) REFERENCES public.blocks(number) ON DELETE CASCADE; - - --- --- Name: transactions transactions_block_number_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transactions - ADD CONSTRAINT transactions_block_number_fkey FOREIGN KEY (block_number) REFERENCES public.blocks(number) ON DELETE CASCADE; - - --- --- Name: SCHEMA public; Type: ACL; Schema: -; Owner: - --- - -REVOKE USAGE ON SCHEMA public FROM PUBLIC; -GRANT ALL ON SCHEMA public TO PUBLIC; - - --- --- PostgreSQL database dump complete --- - --- --- PostgreSQL database dump --- - --- Dumped from database version 14.11 (Homebrew) --- Dumped by pg_dump version 15.8 (Homebrew) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - --- --- Data for Name: migrations; Type: TABLE DATA; Schema: public; Owner: seanmcgary --- - -INSERT INTO public.migrations VALUES ('202405150900_bootstrapDb', '2024-05-17 09:13:15.57261-05', NULL); -INSERT INTO public.migrations VALUES ('202405150917_insertContractAbi', '2024-05-17 09:13:15.575844-05', NULL); -INSERT INTO public.migrations VALUES ('202405151523_addTransactionToFrom', '2024-05-17 09:13:15.578099-05', NULL); -INSERT INTO public.migrations VALUES ('202405170842_addBlockInfoToTransactionLog', '2024-05-17 09:13:15.580354-05', NULL); -INSERT INTO public.migrations VALUES ('202405171056_unverifiedContracts', '2024-05-17 11:00:17.149086-05', NULL); -INSERT INTO public.migrations VALUES ('202405171345_addUpdatedPaymentCoordinatorAbi', '2024-05-17 13:51:24.584807-05', NULL); -INSERT INTO public.migrations VALUES ('202405201503_fixTransactionHashConstraint', '2024-05-20 15:10:45.476856-05', NULL); -INSERT INTO public.migrations VALUES ('202405300925_addUniqueBlockConstraint', '2024-05-30 09:33:13.115195-05', NULL); -INSERT INTO public.migrations VALUES ('202405312008_indexTransactionContractAddress', '2024-05-31 21:13:37.099393-05', NULL); -INSERT INTO public.migrations VALUES ('202405312134_handleProxyContracts', '2024-05-31 22:21:46.84577-05', NULL); -INSERT INTO public.migrations VALUES ('202406030920_addCheckedForProxyFlag', '2024-06-03 10:09:02.176827-05', NULL); -INSERT INTO public.migrations VALUES ('202406031946_addSerialIdToContracts', '2024-06-04 08:54:09.723152-05', NULL); -INSERT INTO public.migrations VALUES ('202406051937_addBytecodeIndex', '2024-06-05 19:54:36.665099-05', NULL); -INSERT INTO public.migrations VALUES ('202406071318_indexTransactionLogBlockNumber', '2024-06-07 13:20:19.291429-05', NULL); -INSERT INTO public.migrations VALUES ('202406110848_transactionLogsContractIndex', '2024-06-11 11:26:43.316616-05', NULL); -INSERT INTO public.migrations VALUES ('202406141007_addCheckedForAbiFlag', '2024-06-14 10:13:35.067238-05', NULL); -INSERT INTO public.migrations VALUES ('202406251424_addTransactionLogsOutputDataColumn', '2024-06-25 14:29:47.494612-05', NULL); -INSERT INTO public.migrations VALUES ('202406251426_addTransactionIndexes', '2024-06-25 14:29:47.500543-05', NULL); -INSERT INTO public.migrations VALUES ('202407101440_addOperatorRestakedStrategiesTable', '2024-07-11 09:48:48.933519-05', NULL); -INSERT INTO public.migrations VALUES ('202407110946_addBlockTimeToRestakedStrategies', '2024-07-11 09:49:17.325774-05', NULL); -INSERT INTO public.migrations VALUES ('202407111116_addAvsDirectoryAddress', '2024-07-24 09:13:18.235218-05', NULL); -INSERT INTO public.migrations VALUES ('202407121407_updateProxyContractIndex', '2024-07-24 09:13:18.240594-05', NULL); -INSERT INTO public.migrations VALUES ('202408200934_eigenlayerStateTables', '2024-09-05 16:16:40.950631-05', NULL); -INSERT INTO public.migrations VALUES ('202409051720_operatorShareChanges', '2024-09-05 19:14:07.595987-05', NULL); -INSERT INTO public.migrations VALUES ('202409052151_stakerDelegations', '2024-09-05 22:24:34.016039-05', NULL); -INSERT INTO public.migrations VALUES ('202409061121_removeSequenceId', '2024-09-06 11:42:21.585605-05', NULL); - - --- --- PostgreSQL database dump complete --- - diff --git a/sqlite-extensions/README.md b/sqlite-extensions/README.md new file mode 100644 index 00000000..4ab1ba2e --- /dev/null +++ b/sqlite-extensions/README.md @@ -0,0 +1,5 @@ +### Running + +```bash +PYTHONPATH="/Users/seanmcgary/Code/sidecar/sqlite-extensions:$PYTHONPATH" lldb -- /opt/homebrew/opt/sqlite/bin/sqlite3 +``` diff --git a/sqlite-extensions/calculations.c b/sqlite-extensions/calculations.c new file mode 100644 index 00000000..0e0cc28b --- /dev/null +++ b/sqlite-extensions/calculations.c @@ -0,0 +1,310 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include "calculations.h" +SQLITE_EXTENSION_INIT1 + +static PyObject *pModule = NULL; +static PyGILState_STATE gilstate; + +int init_python() { + Py_Initialize(); + PyRun_SimpleString("import sys; sys.path.append('.')"); + pModule = PyImport_ImportModule("calculations"); + if (pModule == NULL) { + PyErr_Print(); + return 0; + } + return 1; +} + +void finalize_python() { + Py_XDECREF(pModule); + Py_Finalize(); +} + +char* call_python_func(const char* func_name, const char* arg1, const char* arg2) { + char *result = NULL; + gilstate = PyGILState_Ensure(); + + PyObject *pFunc = PyObject_GetAttrString(pModule, func_name); + if (pFunc && PyCallable_Check(pFunc)) { + PyObject *pArgs; + if (arg2 == NULL) { + pArgs = Py_BuildValue("(s)", arg1); + } else { + pArgs = Py_BuildValue("(ss)", arg1, arg2); + } + PyObject *pValue = PyObject_CallObject(pFunc, pArgs); + Py_DECREF(pArgs); + if (pValue != NULL) { + PyObject *str_obj = PyObject_Str(pValue); + const char *str_result = PyUnicode_AsUTF8(str_obj); + result = strdup(str_result); + Py_DECREF(str_obj); + Py_DECREF(pValue); + } else { + PyErr_Print(); + } + } else { + PyErr_Print(); + } + Py_XDECREF(pFunc); + + PyGILState_Release(gilstate); + return result; +} + +// _pre_nile_tokens_per_day is a non-sqlite3 function that is called by pre_nile_tokens_per_day +char* _pre_nile_tokens_per_day(const char* tokens) { + return call_python_func("preNileTokensPerDay", tokens, NULL); +} + +// Your custom function +void pre_nile_tokens_per_day(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 1) { + sqlite3_result_error(context, "pre_nile_tokens_per_day() requires exactly one argument", -1); + return; + } + const char* input = (const char*)sqlite3_value_text(argv[0]); + if (!input) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _pre_nile_tokens_per_day(input); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _amazon_staker_token_rewards(const char* sp, const char* tpd) { + return call_python_func("amazonStakerTokenRewards", sp, tpd); +} + +void amazon_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 2) { + sqlite3_result_error(context, "amazon_staker_token_rewards() requires two arguments", -1); + return; + } + const char* sp = (const char*)sqlite3_value_text(argv[0]); + if (!sp) { + sqlite3_result_null(context); + return; + } + + const char* tpd = (const char*)sqlite3_value_text(argv[1]); + if (!tpd) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _amazon_staker_token_rewards(sp, tpd); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _nile_staker_token_rewards(const char* sp, const char* tpd) { + return call_python_func("nileStakerTokenRewards", sp, tpd); +} + +void nile_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 2) { + sqlite3_result_error(context, "nile_staker_token_rewards() requires two arguments", -1); + return; + } + const char* sp = (const char*)sqlite3_value_text(argv[0]); + if (!sp) { + sqlite3_result_null(context); + return; + } + + const char* tpd = (const char*)sqlite3_value_text(argv[1]); + if (!tpd) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _nile_staker_token_rewards(sp, tpd); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _staker_token_rewards(const char* sp, const char* tpd) { + return call_python_func("stakerTokenRewards", sp, tpd); +} + +void staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 2) { + sqlite3_result_error(context, "staker_token_rewards() requires two arguments", -1); + return; + } + const char* sp = (const char*)sqlite3_value_text(argv[0]); + if (!sp) { + sqlite3_result_null(context); + return; + } + + const char* tpd = (const char*)sqlite3_value_text(argv[1]); + if (!tpd) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _staker_token_rewards(sp, tpd); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _amazon_operator_token_rewards(const char* totalStakerOperatorTokens) { + return call_python_func("amazonOperatorTokenRewards", totalStakerOperatorTokens, NULL); +} + +void amazon_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 1) { + sqlite3_result_error(context, "amazon_operator_token_rewards() requires exactly one argument", -1); + return; + } + const char* input = (const char*)sqlite3_value_text(argv[0]); + if (!input) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _amazon_operator_token_rewards(input); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + + +char* _nile_operator_token_rewards(const char* totalStakerOperatorTokens) { + return call_python_func("nileOperatorTokenRewards", totalStakerOperatorTokens, NULL); +} +void nile_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 1) { + sqlite3_result_error(context, "nile_operator_token_rewards() requires exactly one argument", -1); + return; + } + const char* input = (const char*)sqlite3_value_text(argv[0]); + if (!input) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _nile_operator_token_rewards(input); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +int sqlite3_calculations_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + int rc; + rc = sqlite3_create_function(db, "pre_nile_tokens_per_day", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, pre_nile_tokens_per_day, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "amazon_staker_token_rewards", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, amazon_staker_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "nile_staker_token_rewards", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, nile_staker_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "staker_token_rewards", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, staker_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "amazon_operator_token_rewards", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, amazon_operator_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "nile_operator_token_rewards", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, nile_operator_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + return SQLITE_OK; +} diff --git a/sqlite-extensions/calculations.h b/sqlite-extensions/calculations.h new file mode 100644 index 00000000..5034fc47 --- /dev/null +++ b/sqlite-extensions/calculations.h @@ -0,0 +1,28 @@ +#ifndef CALCULATIONS_H +#define CALCULATIONS_H + +#include + +int init_python(); +void finalize_python(); +char* call_python_func(const char* func_name, const char* arg1, const char* arg2); + +char* _pre_nile_tokens_per_day(const char* tokens); +void pre_nile_tokens_per_day(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _amazon_staker_token_rewards(const char* sp, const char* tpd); +void amazon_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _nile_staker_token_rewards(const char* sp, const char* tpd); +void nile_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _staker_token_rewards(const char* sp, const char* tpd); +void staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _amazon_operator_token_rewards(const char* totalStakerOperatorTokens); +void amazon_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _nile_operator_token_rewards(const char* totalStakerOperatorTokens); +void nile_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +#endif // CALCULATIONS_H diff --git a/sqlite-extensions/calculations.py b/sqlite-extensions/calculations.py new file mode 100644 index 00000000..674e3282 --- /dev/null +++ b/sqlite-extensions/calculations.py @@ -0,0 +1,74 @@ +from decimal import Decimal, getcontext, ROUND_HALF_UP, ROUND_UP, ROUND_DOWN + +def preNileTokensPerDay(tokens: str) -> str: + big_amount = float(tokens) + div = 0.999999999999999 + res = big_amount * div + + res_str = "{}".format(res) + return "{}".format(int(Decimal(res_str))) + +def amazonStakerTokenRewards(sp:str, tpd:str) -> str: + getcontext().prec = 15 + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + decimal_res = Decimal(stakerProportion * tokensPerDay) + + getcontext().prec = 20 + rounded = decimal_res.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + + +def nileStakerTokenRewards(sp:str, tpd:str) -> str: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + decimal_res = Decimal(stakerProportion * tokensPerDay) + # Truncate to 0.x decimals + truncated = decimal_res.quantize(Decimal('0.1'), rounding=ROUND_UP) + + # Bankers rounding to match postgres + rounded = truncated.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + +def stakerTokenRewards(sp:str, tpd:str) -> str: + stakerProportion = float(sp) + tokensPerDay = int(tpd) + + decimal_res = stakerProportion * tokensPerDay + + parts = str(decimal_res).split("+") + # Need more precision + if len(parts) == 2 and int(parts[1]) > 16: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + getcontext().prec = 17 + getcontext().rounding = ROUND_DOWN + decimal_res = stakerProportion * tokensPerDay + + return "{}".format(int(decimal_res)) + + return "{}".format(int(decimal_res)) + + +def amazonOperatorTokenRewards(totalStakerOperatorTokens:str) -> str: + totalStakerOperatorTokens = Decimal(totalStakerOperatorTokens) + + operatorTokens = totalStakerOperatorTokens * Decimal(0.1) + + rounded = operatorTokens.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + +def nileOperatorTokenRewards(totalStakerOperatorTokens:str) -> str: + if totalStakerOperatorTokens[-1] == "0": + return "{}".format(int(totalStakerOperatorTokens) // 10) + totalStakerOperatorTokens = Decimal(totalStakerOperatorTokens) + operatorTokens = Decimal(str(totalStakerOperatorTokens)) * Decimal(0.1) + rounded = operatorTokens.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + return "{}".format(rounded) +