# This is a combination of 25 commits.

# This is the 1st commit message:

Calculate rewards

Add state change table for AVS operators to make windowing easier

Convert avs operator windows query to sqlite

Queries for snapshots

Testing OperatorAVSRegistrationWindows

Window query passing

Some cleanup

Expand test dataset to be more than just up to 2024-09-01

Restaked strategy snapshots

Test tweaks

Generate snapshots rather than windows

Add custom sum_big function

Adding tests for operator share snapshots

Change operatorShares to be a sparse table

Change StakerShares to a sparse table

Add snapshot generation for operator shares

Calculate rewards

remove git attributes

# This is the commit message #2:

Adding query stubs

# This is the commit message #3:

Adding more custom functions for math

# This is the commit message #4:

Copy over queries for gold table generation

# This is the commit message #5:

Use custom sql functions

# This is the commit message #6:

Use file-based db for testing rewards

# This is the commit message #7:

Staker amounts for nile and amazon forks

# This is the commit message #8:

Calculation testing

# This is the commit message #9:

Floating point math is the absolute worst

# This is the commit message #10:

Fix compliation

# This is the commit message #11:

More number tests

# This is the commit message #12:

Trying to get sqlite lib working

# This is the commit message #13:

This compiles and runs

# This is the commit message #14:

Also compiles

# This is the commit message #15:

Compiles and runs with added python stuff

# This is the commit message #16:

Delete compiled stuff

# This is the commit message #17:

Python seems to be working

# This is the commit message #18:

Working pre_nile_tokens_per_day

# This is the commit message #19:

Works with go example

# This is the commit message #20:

Port amazonStakerTokenRewards to native C

# This is the commit message #21:

Port nile_staker_token_rewards to native C

# This is the commit message #22:

Port amazon_operator_token_rewards and nile_operator_token_rewards to native C

# This is the commit message #23:

Rename yolo to calculations

# This is the commit message #24:

Add pure functions for easier testing

# This is the commit message #25:

Token calcuations test works

.gitignore

@@ -14,5 +14,10 @@ go-sidecar
 /internal/tests/testdata/**/*.json
 /internal/tests/testdata/*.sql
 !/internal/tests/testdata/**/generateExpectedResults.sql
+!sqlite-extensions
 /*.tar
 /sidecar-data
+__pycache__
+test-db
+*.dylib
+*.dylib.dSYM

.testdataVersion

@@ -0,0 +1 @@
+f8689176dbe66b2d7ce03665190298b9c9f7c8e9

Makefile

@@ -1,24 +1,27 @@
 .PHONY: deps proto
 
+args=CGO_ENABLED=1
+GO=$(shell which go)
+
 deps/dev:
-	go install github.com/golangci/golangci-lint/cmd/[email protected]
-	go install honnef.co/go/tools/cmd/staticcheck@latest
-	go install github.com/google/yamlfmt/cmd/yamlfmt@latest
+	${GO} install github.com/golangci/golangci-lint/cmd/[email protected]
+	${GO} install honnef.co/go/tools/cmd/staticcheck@latest
+	${GO} install github.com/google/yamlfmt/cmd/yamlfmt@latest
 
 deps/go:
-	go install google.golang.org/protobuf/cmd/[email protected]
-	go install google.golang.org/grpc/cmd/[email protected]
-	go get \
+	${GO} install google.golang.org/protobuf/cmd/[email protected]
+	${GO} install google.golang.org/grpc/cmd/[email protected]
+	${GO} get \
 		github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \
 		github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \
 		google.golang.org/protobuf/cmd/protoc-gen-go \
 		google.golang.org/grpc/cmd/protoc-gen-go-grpc
-	go install \
+	${GO} install \
 		github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \
 		github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \
 		google.golang.org/protobuf/cmd/protoc-gen-go \
 		google.golang.org/grpc/cmd/protoc-gen-go-grpc
-	go mod tidy
+	${GO} mod tidy
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.61.0
 
 
@@ -42,7 +45,7 @@ clean:
 
 .PHONY: build/cmd/sidecar
 build/cmd/sidecar:
-	CGO_ENABLED=1 go build -o bin/sidecar main.go
+	${args} ${GO} build -o bin/sidecar main.go
 
 .PHONY: build
 build: build/cmd/sidecar
@@ -88,3 +91,38 @@ staticcheck:
 
 .PHONY: ci-test
 ci-test: test
+
+test-rewards:
+	TEST_REWARDS=true TESTING=true ${GO} test ./pkg/rewards -v -p 1
+
+# -----------------------------------------------------------------------------
+# SQLite extension build steps
+# -----------------------------------------------------------------------------
+CC = gcc -g -fPIC -shared
+
+PYTHON_CONFIG = python3-config
+PYTHON_VERSION = $(shell python3 -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))")
+PYTHON_LIBDIR := $(shell $(PYTHON_CONFIG) --prefix)/lib
+
+# Base flags
+CFLAGS =
+LDFLAGS =
+
+INCLUDE_DIRS =
+CFLAGS += $(foreach dir,$(INCLUDE_DIRS),-I$(dir))
+
+# Python flags
+PYTHON_CFLAGS := $(shell $(PYTHON_CONFIG) --includes)
+PYTHON_LDFLAGS := $(shell $(PYTHON_CONFIG) --ldflags)
+
+SQLITE_DIR = /opt/homebrew/opt/sqlite
+CFLAGS += -I$(SQLITE_DIR)/include
+LDFLAGS += -L$(SQLITE_DIR)/lib -lsqlite3
+
+CFLAGS += $(PYTHON_CFLAGS)
+LDFLAGS += $(PYTHON_LDFLAGS) -L$(PYTHON_LIBDIR) -lpython$(PYTHON_VERSION)
+
+
+.PHONY: sqlite-extensions
+sqlite-extensions:
+	$(CC) $(CFLAGS) -o sqlite-extensions/libcalculations.dylib sqlite-extensions/calculations.c $(LDFLAGS)

cmd/debugger/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/Layr-Labs/go-sidecar/internal/clients/ethereum"
+	"github.com/Layr-Labs/go-sidecar/internal/clients/etherscan"
+	"github.com/Layr-Labs/go-sidecar/internal/config"
+	"github.com/Layr-Labs/go-sidecar/internal/contractCaller"
+	"github.com/Layr-Labs/go-sidecar/internal/contractManager"
+	"github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerDelegations"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerShares"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/stateManager"
+	"github.com/Layr-Labs/go-sidecar/internal/fetcher"
+	"github.com/Layr-Labs/go-sidecar/internal/indexer"
+	"github.com/Layr-Labs/go-sidecar/internal/logger"
+	"github.com/Layr-Labs/go-sidecar/internal/metrics"
+	"github.com/Layr-Labs/go-sidecar/internal/pipeline"
+	"github.com/Layr-Labs/go-sidecar/internal/sidecar"
+	"github.com/Layr-Labs/go-sidecar/internal/sqlite"
+	"github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations"
+	sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite"
+	"go.uber.org/zap"
+)
+
+func main() {
+	ctx := context.Background()
+	cfg := config.NewConfig()
+
+	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
+
+	sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl)
+	if err != nil {
+		l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err))
+	}
+
+	etherscanClient := etherscan.NewEtherscanClient(cfg, l)
+	client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l)
+
+	db := sqlite.NewSqlite(cfg.GetSqlitePath(), l)
+
+	grm, err := sqlite.NewGormSqliteFromSqlite(db)
+	if err != nil {
+		l.Error("Failed to create gorm instance", zap.Error(err))
+		panic(err)
+	}
+
+	migrator := migrations.NewSqliteMigrator(grm, l)
+	if err = migrator.MigrateAll(); err != nil {
+		log.Fatalf("Failed to migrate: %v", err)
+	}
+
+	contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg)
+	if err := contractStore.InitializeCoreContracts(); err != nil {
+		log.Fatalf("Failed to initialize core contracts: %v", err)
+	}
+
+	cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l)
+
+	mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	sm := stateManager.NewEigenStateManager(l, grm)
+
+	if _, err := avsOperators.NewAvsOperators(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create AvsOperatorsModel", zap.Error(err))
+	}
+	if _, err := operatorShares.NewOperatorSharesModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create OperatorSharesModel", zap.Error(err))
+	}
+	if _, err := stakerDelegations.NewStakerDelegationsModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create StakerDelegationsModel", zap.Error(err))
+	}
+	if _, err := stakerShares.NewStakerSharesModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create StakerSharesModel", zap.Error(err))
+	}
+
+	fetchr := fetcher.NewFetcher(client, cfg, l)
+
+	cc := contractCaller.NewContractCaller(client, l)
+
+	idxr := indexer.NewIndexer(mds, contractStore, etherscanClient, cm, client, fetchr, cc, l, cfg)
+
+	p := pipeline.NewPipeline(fetchr, idxr, mds, sm, l)
+
+	// Create new sidecar instance
+	sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{
+		GenesisBlockNumber: cfg.GetGenesisBlockNumber(),
+	}, cfg, mds, p, sm, l, client)
+
+	// RPC channel to notify the RPC server to shutdown gracefully
+	rpcChannel := make(chan bool)
+	err = sidecar.WithRpcServer(ctx, mds, sm, rpcChannel)
+	if err != nil {
+		l.Sugar().Fatalw("Failed to start RPC server", zap.Error(err))
+	}
+
+	block, err := fetchr.FetchBlock(ctx, 1215893)
+	if err != nil {
+		l.Sugar().Fatalw("Failed to fetch block", zap.Error(err))
+	}
+
+	transactionHash := "0xf6775c38af1d2802bcbc2b7c8959c0d5b48c63a14bfeda0261ba29d76c68c423"
+	transaction := &ethereum.EthereumTransaction{}
+
+	for _, tx := range block.Block.Transactions {
+		if tx.Hash.Value() == transactionHash {
+			transaction = tx
+			break
+		}
+	}
+
+	logIndex := 4
+	receipt := block.TxReceipts[transaction.Hash.Value()]
+	var interestingLog *ethereum.EthereumEventLog
+
+	for _, log := range receipt.Logs {
+		if log.LogIndex.Value() == uint64(logIndex) {
+			fmt.Printf("Log: %+v\n", log)
+			interestingLog = log
+		}
+	}
+
+	decodedLog, err := idxr.DecodeLogWithAbi(nil, receipt, interestingLog)
+	if err != nil {
+		l.Sugar().Fatalw("Failed to decode log", zap.Error(err))
+	}
+	l.Sugar().Infof("Decoded log: %+v", decodedLog)
+}

cmd/run.go

@@ -59,7 +59,7 @@ var runCmd = &cobra.Command{
 			}
 		}
 
-		db := sqlite.NewSqlite(cfg.GetSqlitePath())
+		db := sqlite.NewSqlite(cfg.GetSqlitePath(), l)
 
 		grm, err := sqlite.NewGormSqliteFromSqlite(db)
 		if err != nil {

cmd/sidecar/main.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"github.com/Layr-Labs/go-sidecar/internal/clients/ethereum"
+	"github.com/Layr-Labs/go-sidecar/internal/clients/etherscan"
+	"github.com/Layr-Labs/go-sidecar/internal/config"
+	"github.com/Layr-Labs/go-sidecar/internal/contractCaller"
+	"github.com/Layr-Labs/go-sidecar/internal/contractManager"
+	"github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/rewardSubmissions"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerDelegations"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerShares"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/stateManager"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/submittedDistributionRoots"
+	"github.com/Layr-Labs/go-sidecar/internal/fetcher"
+	"github.com/Layr-Labs/go-sidecar/internal/indexer"
+	"github.com/Layr-Labs/go-sidecar/internal/logger"
+	"github.com/Layr-Labs/go-sidecar/internal/metrics"
+	"github.com/Layr-Labs/go-sidecar/internal/pipeline"
+	"github.com/Layr-Labs/go-sidecar/internal/shutdown"
+	"github.com/Layr-Labs/go-sidecar/internal/sidecar"
+	"github.com/Layr-Labs/go-sidecar/internal/sqlite"
+	"github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations"
+	sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite"
+	"go.uber.org/zap"
+)
+
+func main() {
+	ctx := context.Background()
+	cfg := config.NewConfig()
+
+	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
+
+	sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl)
+	if err != nil {
+		l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err))
+	}
+
+	etherscanClient := etherscan.NewEtherscanClient(cfg, l)
+	client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l)
+
+	db := sqlite.NewSqlite(cfg.GetSqlitePath(), l)
+
+	grm, err := sqlite.NewGormSqliteFromSqlite(db)
+	if err != nil {
+		l.Error("Failed to create gorm instance", zap.Error(err))
+		panic(err)
+	}
+
+	migrator := migrations.NewSqliteMigrator(grm, l)
+	if err = migrator.MigrateAll(); err != nil {
+		log.Fatalf("Failed to migrate: %v", err)
+	}
+
+	contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg)
+	if err := contractStore.InitializeCoreContracts(); err != nil {
+		log.Fatalf("Failed to initialize core contracts: %v", err)
+	}
+
+	cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l)
+
+	mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	sm := stateManager.NewEigenStateManager(l, grm)
+
+	if _, err := avsOperators.NewAvsOperators(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create AvsOperatorsModel", zap.Error(err))
+	}
+	if _, err := operatorShares.NewOperatorSharesModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create OperatorSharesModel", zap.Error(err))
+	}
+	if _, err := stakerDelegations.NewStakerDelegationsModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create StakerDelegationsModel", zap.Error(err))
+	}
+	if _, err := stakerShares.NewStakerSharesModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create StakerSharesModel", zap.Error(err))
+	}
+	if _, err := submittedDistributionRoots.NewSubmittedDistributionRootsModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create SubmittedDistributionRootsModel", zap.Error(err))
+	}
+	if _, err := rewardSubmissions.NewRewardSubmissionsModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create RewardSubmissionsModel", zap.Error(err))
+	}
+
+	fetchr := fetcher.NewFetcher(client, cfg, l)
+
+	cc := contractCaller.NewContractCaller(client, l)
+
+	idxr := indexer.NewIndexer(mds, contractStore, etherscanClient, cm, client, fetchr, cc, l, cfg)
+
+	p := pipeline.NewPipeline(fetchr, idxr, mds, sm, l)
+
+	// Create new sidecar instance
+	sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{
+		GenesisBlockNumber: cfg.GetGenesisBlockNumber(),
+	}, cfg, mds, p, sm, l, client)
+
+	// RPC channel to notify the RPC server to shutdown gracefully
+	rpcChannel := make(chan bool)
+	err = sidecar.WithRpcServer(ctx, mds, sm, rpcChannel)
+	if err != nil {
+		l.Sugar().Fatalw("Failed to start RPC server", zap.Error(err))
+	}
+
+	// Start the sidecar main process in a goroutine so that we can listen for a shutdown signal
+	go sidecar.Start(ctx)
+
+	l.Sugar().Info("Started Sidecar")
+
+	gracefulShutdown := shutdown.CreateGracefulShutdownChannel()
+
+	done := make(chan bool)
+	shutdown.ListenForShutdown(gracefulShutdown, done, func() {
+		l.Sugar().Info("Shutting down...")
+		rpcChannel <- true
+		sidecar.ShutdownChan <- true
+	}, time.Second*5, l)
+}

cmd/sqlitedebug/db.py

@@ -0,0 +1,7 @@
+import sqlite3
+
+conn = sqlite3.connect(":memory:")
+conn.enable_load_extension(True)
+conn.load_extension("/Users/seanmcgary/Code/sidecar/sqlite-extensions/yolo.dylib")
+conn.execute("SELECT my_custom_function('foo')")
+conn.close()