From 856d11e9232d694738c3cd3c040603de8c8d6ece Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 09:27:55 -0700 Subject: [PATCH 01/20] generator date fix --- packages/eventindexer/generator/generator.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 0b933732148..2a84b70c9a5 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -149,10 +149,10 @@ func (g *Generator) getLatestDateByTask(ctx context.Context, task string) (time. latestDate = g.genesisDate } else { latestDate, err = time.Parse("2006-01-02", latestDateString) - } - - if err != nil { - return time.Time{}, err + if err != nil { + return time.Time{}, err + } + latestDate = latestDate.AddDate(0, 0, 1) } slog.Info("latest date for task", "task", task, "latestDate", latestDate.Format("2006-01-02")) From 63daf452b4b39f1d0dc3d9f981b9d001be90b405 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 09:28:08 -0700 Subject: [PATCH 02/20] lint --- packages/eventindexer/generator/generator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 2a84b70c9a5..2bbed244ec5 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -152,6 +152,7 @@ func (g *Generator) getLatestDateByTask(ctx context.Context, task string) (time. if err != nil { return time.Time{}, err } + latestDate = latestDate.AddDate(0, 0, 1) } From e57ffd85e708db4cfef39da528dc3afe31cb425a Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 09:29:49 -0700 Subject: [PATCH 03/20] rename function --- packages/eventindexer/generator/generator.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 2bbed244ec5..0641b627362 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -93,7 +93,7 @@ func (g *Generator) generateTimeSeriesData(ctx context.Context) error { func (g *Generator) generateByTask(ctx context.Context, task string) error { slog.Info("generating for task", "task", task) - latestDate, err := g.getLatestDateByTask(ctx, task) + latestDate, err := g.getStartingDateByTask(ctx, task) if err != nil { return err } @@ -132,12 +132,12 @@ func (g *Generator) generateByTask(ctx context.Context, task string) error { return nil } -// getLatestDateByTask returns the last time time series data has been generated -// for the given task. -func (g *Generator) getLatestDateByTask(ctx context.Context, task string) (time.Time, error) { +// getStartingDateByTask returns first required time series data, one after the latest date entry, +// or the genesis date. +func (g *Generator) getStartingDateByTask(ctx context.Context, task string) (time.Time, error) { var latestDateString string - var latestDate time.Time + var nextRequiredDate time.Time q := `SELECT date FROM time_series_data WHERE task = ? ORDER BY date DESC LIMIT 1;` @@ -146,19 +146,19 @@ func (g *Generator) getLatestDateByTask(ctx context.Context, task string) (time. slog.Info("latestDateString", "task", task, "date", latestDateString) if err != nil || latestDateString == "" { - latestDate = g.genesisDate + nextRequiredDate = g.genesisDate } else { - latestDate, err = time.Parse("2006-01-02", latestDateString) + latestDate, err := time.Parse("2006-01-02", latestDateString) if err != nil { return time.Time{}, err } - latestDate = latestDate.AddDate(0, 0, 1) + nextRequiredDate = latestDate.AddDate(0, 0, 1) } - slog.Info("latest date for task", "task", task, "latestDate", latestDate.Format("2006-01-02")) + slog.Info("next required date for task", "task", task, "latestDate", nextRequiredDate.Format("2006-01-02")) - return latestDate, nil + return nextRequiredDate, nil } // getCurrentDate returns the current date in YYYY-MM-DD format From 4fe00630822f09800ca81e57d4037d30f2aa594d Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 09:31:24 -0700 Subject: [PATCH 04/20] rename --- packages/eventindexer/generator/generator.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 0641b627362..77e5caf4064 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -93,19 +93,24 @@ func (g *Generator) generateTimeSeriesData(ctx context.Context) error { func (g *Generator) generateByTask(ctx context.Context, task string) error { slog.Info("generating for task", "task", task) - latestDate, err := g.getStartingDateByTask(ctx, task) + startingDate, err := g.getStartingDateByTask(ctx, task) if err != nil { return err } currentDate := g.getCurrentDate() - if latestDate.AddDate(0, 0, 1).Compare(currentDate) == 0 { - slog.Info("data already generated up-to-date for task", "task", task, "date", latestDate.Format("2006-01-02")) + if startingDate.Compare(currentDate) == 0 { + slog.Info( + "data already generated up-to-date for task", + "task", task, + "date", startingDate.Format("2006-01-02"), + ) + return nil } // Loop through each date from latestDate to currentDate - for d := latestDate; d.Before(currentDate); d = d.AddDate(0, 0, 1) { + for d := startingDate; d.Before(currentDate); d = d.AddDate(0, 0, 1) { slog.Info("Processing", "task", task, "date", d.Format("2006-01-02"), "currentDate", currentDate.Format("2006-01-02")) result, err := g.queryByTask(task, d) From 75c8c1e1c4f81d2eadb1dd00b1fd953996033ab6 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 09:31:56 -0700 Subject: [PATCH 05/20] logs --- packages/eventindexer/generator/generator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 77e5caf4064..474e46d6948 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -104,6 +104,7 @@ func (g *Generator) generateByTask(ctx context.Context, task string) error { "data already generated up-to-date for task", "task", task, "date", startingDate.Format("2006-01-02"), + "currentDate", currentDate.Format("2006-01-02"), ) return nil From 69fa5e85bec0882f1d096eff490bab85358d0426 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 09:58:33 -0700 Subject: [PATCH 06/20] fix contract deployment task --- packages/eventindexer/generator/generator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 474e46d6948..3bf5a4948fb 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -162,7 +162,7 @@ func (g *Generator) getStartingDateByTask(ctx context.Context, task string) (tim nextRequiredDate = latestDate.AddDate(0, 0, 1) } - slog.Info("next required date for task", "task", task, "latestDate", nextRequiredDate.Format("2006-01-02")) + slog.Info("next required date for task", "task", task, "nextRequiredDate", nextRequiredDate.Format("2006-01-02")) return nextRequiredDate, nil } @@ -343,14 +343,14 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { result = strconv.Itoa(dailyTxCount + tsdResult) case tasks.ContractDeploymentsPerDay: query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` - err = g.db.GormDB().Raw(query, dateString, ZeroAddress).Scan(&result).Error + err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&result).Error case tasks.TotalContractDeployments: var dailyContractCount int // get current days txs, get previous entry for the time series data, add them together. query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` - err = g.db.GormDB().Raw(query, dateString, ZeroAddress).Scan(&dailyContractCount).Error + err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&dailyContractCount).Error if err != nil { return "", err } From 8ed4b8498325fa1789c8d3922faf640ca7b1b8a9 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 11:22:33 -0700 Subject: [PATCH 07/20] track proposer/proof rewards --- packages/eventindexer/.l1.env | 3 +- packages/eventindexer/event.go | 4 + packages/eventindexer/generator/generator.go | 38 +++++++ packages/eventindexer/http/get_stats.go | 4 +- .../indexer/save_block_proposed_event.go | 100 ++++++++++++++++++ .../1666650599_create_events_table.sql | 2 + .../1666650701_create_stats_table.sql | 2 + packages/eventindexer/repo/event.go | 24 +++++ packages/eventindexer/repo/stat.go | 9 ++ packages/eventindexer/repo/stat_test.go | 23 ++-- packages/eventindexer/stat.go | 17 +-- packages/eventindexer/tasks/tasks.go | 8 ++ 12 files changed, 216 insertions(+), 18 deletions(-) diff --git a/packages/eventindexer/.l1.env b/packages/eventindexer/.l1.env index e137fc178ce..1a0131b7af8 100644 --- a/packages/eventindexer/.l1.env +++ b/packages/eventindexer/.l1.env @@ -13,4 +13,5 @@ RPC_URL=wss://l1ws.internal.taiko.xyz CORS_ORIGINS=* BLOCK_BATCH_SIZE=10 CACHE_INTERVAL_IN_SECONDS=60 -LAYER=l1 \ No newline at end of file +LAYER=l1 +GENESIS_DATE=2023-09-08 \ No newline at end of file diff --git a/packages/eventindexer/event.go b/packages/eventindexer/event.go index 288801324ac..8fd589ae4b0 100644 --- a/packages/eventindexer/event.go +++ b/packages/eventindexer/event.go @@ -34,6 +34,8 @@ type Event struct { Address string `json:"address"` BlockID sql.NullInt64 `json:"blockID"` Amount decimal.NullDecimal `json:"amount"` + ProofReward decimal.NullDecimal `json:"proofReward"` + ProposerReward decimal.NullDecimal `json:"proposerReward"` AssignedProver string `json:"assignedProver"` To string `json:"to"` TokenID sql.NullInt64 `json:"tokenID"` @@ -50,6 +52,8 @@ type SaveEventOpts struct { Address string BlockID *int64 Amount *big.Int + ProposerReward *big.Int + ProofReward *big.Int AssignedProver *string To *string TokenID *int64 diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 3bf5a4948fb..e379ebb8011 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -187,6 +187,44 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { var err error switch task { + case tasks.ProposerRewardsPerDay: + query := "SELECT SUM(proposer_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, date). + Scan(&result).Error + case tasks.TotalProposerRewards: + var dailyProposerRewards int + + query := "SELECT SUM(proposer_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, date). + Scan(&result).Error + + tsdResult, err := g.previousDayTsdResultByTask(task, date) + if err != nil { + return "", err + } + + result = strconv.Itoa(dailyProposerRewards + tsdResult) + case tasks.TotalProofRewards: + var dailyProofRewards int + + query := "SELECT SUM(proof_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, date). + Scan(&result).Error + + tsdResult, err := g.previousDayTsdResultByTask(task, date) + if err != nil { + return "", err + } + + result = strconv.Itoa(dailyProofRewards + tsdResult) + case tasks.ProofRewardsPerDay: + query := "SELECT SUM(proof_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, date). + Scan(&result).Error case tasks.BridgeMessagesSentPerDay: err = g.eventCount(task, date, eventindexer.EventNameMessageSent, &result) case tasks.TotalBridgeMessagesSent: diff --git a/packages/eventindexer/http/get_stats.go b/packages/eventindexer/http/get_stats.go index cbb68858c6d..c66e26a4109 100644 --- a/packages/eventindexer/http/get_stats.go +++ b/packages/eventindexer/http/get_stats.go @@ -2,10 +2,10 @@ package http import ( "net/http" + "time" "github.com/cyberhorsey/webutils" "github.com/labstack/echo/v4" - "github.com/patrickmn/go-cache" "github.com/taikoxyz/taiko-mono/packages/eventindexer" ) @@ -26,7 +26,7 @@ func (srv *Server) GetStats(c echo.Context) error { return webutils.LogAndRenderErrors(c, http.StatusUnprocessableEntity, err) } - srv.cache.Set(CacheKeyStats, stats, cache.DefaultExpiration) + srv.cache.Set(CacheKeyStats, stats, 1*time.Minute) } return c.JSON(http.StatusOK, stats) diff --git a/packages/eventindexer/indexer/save_block_proposed_event.go b/packages/eventindexer/indexer/save_block_proposed_event.go index 6d2c2b027fd..8a2e116eed9 100644 --- a/packages/eventindexer/indexer/save_block_proposed_event.go +++ b/packages/eventindexer/indexer/save_block_proposed_event.go @@ -84,12 +84,112 @@ func (indxr *Indexer) saveBlockProposedEvent( BlockID: &blockID, AssignedProver: &assignedProver, TransactedAt: time.Unix(int64(block.Time()), 0), + Amount: event.Reward, }) if err != nil { return errors.Wrap(err, "indxr.eventRepo.Save") } + if err := indxr.updateAverageProposerReward(ctx, event); err != nil { + return errors.Wrap(err, "indxr.updateAverageProposerReward") + } + + if err := indxr.updateAverageProverReward(ctx, event); err != nil { + return errors.Wrap(err, "indxr.updateAverageProposerReward") + } + eventindexer.BlockProposedEventsProcessed.Inc() return nil } + +func (indxr *Indexer) updateAverageProposerReward( + ctx context.Context, + event *taikol1.TaikoL1BlockProposed, +) error { + stat, err := indxr.statRepo.Find(ctx) + if err != nil { + return errors.Wrap(err, "indxr.statRepo.Find") + } + + reward := event.Reward + + avg, ok := new(big.Int).SetString(stat.AverageProposerReward, 10) + if !ok { + return errors.New("unable to convert average proposer to string") + } + + newAverageProposerReward := calcNewAverage( + avg, + new(big.Int).SetUint64(stat.NumProposerRewards), + reward, + ) + + slog.Info("newAverageProposerReward update", + "id", + event.BlockId.Int64(), + "prover", + event.Prover.Hex(), + "avg", + avg.String(), + "newAvg", + newAverageProposerReward.String(), + ) + + _, err = indxr.statRepo.Save(ctx, eventindexer.SaveStatOpts{ + ProposerReward: newAverageProposerReward, + }) + if err != nil { + return errors.Wrap(err, "indxr.statRepo.Save") + } + + return nil +} + +func (indxr *Indexer) updateAverageProverReward( + ctx context.Context, + event *taikol1.TaikoL1BlockProposed, +) error { + stat, err := indxr.statRepo.Find(ctx) + if err != nil { + return errors.Wrap(err, "indxr.statRepo.Find") + } + + tx, _, err := indxr.ethClient.TransactionByHash(ctx, event.Raw.TxHash) + if err != nil { + return errors.Wrap(err, "indxr.ethClient.TransactionByHash") + } + + reward := tx.Value() + + avg, ok := new(big.Int).SetString(stat.AverageProofReward, 10) + if !ok { + return errors.New("unable to convert average proof time to string") + } + + newAverageProofReward := calcNewAverage( + avg, + new(big.Int).SetUint64(stat.NumProofs), + reward, + ) + + slog.Info("newAverageProofReward update", + "id", + event.BlockId.Int64(), + "prover", + event.Prover.Hex(), + "avg", + avg.String(), + "newAvg", + newAverageProofReward.String(), + ) + + _, err = indxr.statRepo.Save(ctx, eventindexer.SaveStatOpts{ + ProofReward: newAverageProofReward, + }) + if err != nil { + return errors.Wrap(err, "indxr.statRepo.Save") + } + + return nil +} diff --git a/packages/eventindexer/migrations/1666650599_create_events_table.sql b/packages/eventindexer/migrations/1666650599_create_events_table.sql index e5b027fa73c..40fe273e1f1 100644 --- a/packages/eventindexer/migrations/1666650599_create_events_table.sql +++ b/packages/eventindexer/migrations/1666650599_create_events_table.sql @@ -9,6 +9,8 @@ CREATE TABLE IF NOT EXISTS events ( address VARCHAR(42) NOT NULL DEFAULT "", block_id int DEFAULT NULL, amount DECIMAL(65, 0) DEFAULT NULL, + proof_reward VARCHAR(255) DEFAULT NULL, + proposer_reward VARCHAR(255) DEFAULT NULL, assigned_prover VARCHAR(42) NOT NULL DEFAULT "", transacted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , diff --git a/packages/eventindexer/migrations/1666650701_create_stats_table.sql b/packages/eventindexer/migrations/1666650701_create_stats_table.sql index 9a8ea8a6e93..60bbbeea6fc 100644 --- a/packages/eventindexer/migrations/1666650701_create_stats_table.sql +++ b/packages/eventindexer/migrations/1666650701_create_stats_table.sql @@ -4,6 +4,8 @@ CREATE TABLE IF NOT EXISTS stats ( id int NOT NULL PRIMARY KEY AUTO_INCREMENT, average_proof_time VARCHAR(255) NOT NULL DEFAULT "0", average_proof_reward VARCHAR(255) NOT NULL DEFAULT "0", + average_proposer_reward VARCHAR(255) NOT NULL DEFAULT "0", + num_proposer_rewards int NOT NULL default 0, num_proofs int NOT NULL default 0, num_verified_blocks int NOT NULL default 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , diff --git a/packages/eventindexer/repo/event.go b/packages/eventindexer/repo/event.go index 4b940615a6d..cb28860426a 100644 --- a/packages/eventindexer/repo/event.go +++ b/packages/eventindexer/repo/event.go @@ -56,6 +56,30 @@ func (r *EventRepository) Save(ctx context.Context, opts eventindexer.SaveEventO } } + if opts.ProposerReward != nil { + amt, err := decimal.NewFromString(opts.ProposerReward.String()) + if err != nil { + return nil, errors.Wrap(err, "decimal.NewFromString") + } + + e.ProposerReward = decimal.NullDecimal{ + Valid: true, + Decimal: amt, + } + } + + if opts.ProofReward != nil { + amt, err := decimal.NewFromString(opts.ProofReward.String()) + if err != nil { + return nil, errors.Wrap(err, "decimal.NewFromString") + } + + e.ProofReward = decimal.NullDecimal{ + Valid: true, + Decimal: amt, + } + } + if opts.AssignedProver != nil { e.AssignedProver = *opts.AssignedProver } diff --git a/packages/eventindexer/repo/stat.go b/packages/eventindexer/repo/stat.go index 65174abb912..82a5efd31b4 100644 --- a/packages/eventindexer/repo/stat.go +++ b/packages/eventindexer/repo/stat.go @@ -41,6 +41,11 @@ func (r *StatRepository) Save(ctx context.Context, opts eventindexer.SaveStatOpt s.AverageProofTime = opts.ProofTime.String() } + if opts.ProposerReward != nil { + s.NumProposerRewards++ + s.AverageProposerReward = opts.ProposerReward.String() + } + if err := r.db.GormDB().Save(s).Error; err != nil { return nil, errors.Wrap(err, "r.db.Save") } @@ -66,5 +71,9 @@ func (r *StatRepository) Find(ctx context.Context) (*eventindexer.Stat, error) { s.AverageProofTime = "0" } + if s.AverageProposerReward == "" { + s.AverageProposerReward = "0" + } + return s, nil } diff --git a/packages/eventindexer/repo/stat_test.go b/packages/eventindexer/repo/stat_test.go index 1c7d9128c3a..1eca8ab3c70 100644 --- a/packages/eventindexer/repo/stat_test.go +++ b/packages/eventindexer/repo/stat_test.go @@ -54,9 +54,14 @@ func TestIntegration_Stat_Find(t *testing.T) { var proofReward = big.NewInt(4) - _, err = statRepo.Save(context.Background(), eventindexer.SaveStatOpts{ - ProofReward: proofReward, - }) + var proposerReward = big.NewInt(7) + + for i := 0; i < 3; i++ { + _, err = statRepo.Save(context.Background(), eventindexer.SaveStatOpts{ + ProofReward: proofReward, + ProposerReward: proposerReward, + }) + } assert.Equal(t, nil, err) @@ -68,11 +73,13 @@ func TestIntegration_Stat_Find(t *testing.T) { { "success", &eventindexer.Stat{ - ID: 1, - AverageProofReward: proofReward.String(), - AverageProofTime: "0", - NumProofs: 0, - NumVerifiedBlocks: 1, + ID: 1, + AverageProofReward: new(big.Int).Mul(proofReward, big.NewInt(3)).String(), + AverageProofTime: "0", + AverageProposerReward: new(big.Int).Mul(proposerReward, big.NewInt(3)).String(), + NumProposerRewards: 3, + NumProofs: 0, + NumVerifiedBlocks: 3, }, nil, }, diff --git a/packages/eventindexer/stat.go b/packages/eventindexer/stat.go index 58752298b66..0734ca64def 100644 --- a/packages/eventindexer/stat.go +++ b/packages/eventindexer/stat.go @@ -9,17 +9,20 @@ import ( // into the Data field to be unmarshalled into a concrete struct // dependant on the name of the event type Stat struct { - ID int `json:"id"` - AverageProofTime string `json:"averageProofTime"` - AverageProofReward string `json:"averageProofReward"` - NumProofs uint64 `json:"numProofs"` - NumVerifiedBlocks uint64 `json:"numVerifiedBlocks"` + ID int `json:"id"` + AverageProofTime string `json:"averageProofTime"` + AverageProofReward string `json:"averageProofReward"` + AverageProposerReward string `json:"averageProposerReward"` + NumProposerRewards uint64 `json:"numProposerRewards"` + NumProofs uint64 `json:"numProofs"` + NumVerifiedBlocks uint64 `json:"numVerifiedBlocks"` } // SaveStatOpts type SaveStatOpts struct { - ProofTime *big.Int - ProofReward *big.Int + ProofTime *big.Int + ProofReward *big.Int + ProposerReward *big.Int } // StatRepository is used to interact with stats in the store diff --git a/packages/eventindexer/tasks/tasks.go b/packages/eventindexer/tasks/tasks.go index 971a8ffb717..62e4bc5cab5 100644 --- a/packages/eventindexer/tasks/tasks.go +++ b/packages/eventindexer/tasks/tasks.go @@ -19,6 +19,10 @@ var ( TotalProposeBlockTx = "total-propose-block-tx" BridgeMessagesSentPerDay = "bridge-messages-sent-per-day" TotalBridgeMessagesSent = "total-bridge-messages-sent" + TotalProofRewards = "total-proof-rewards" + ProofRewardsPerDay = "proof-rewards-per-day" + TotalProposerRewards = "total-proposer-rewards" + ProposerRewardsPerDay = "proposer-rewards-per-day" ) var Tasks = []string{ @@ -40,4 +44,8 @@ var Tasks = []string{ TotalProposeBlockTx, BridgeMessagesSentPerDay, TotalBridgeMessagesSent, + TotalProofRewards, + ProofRewardsPerDay, + TotalProposerRewards, + ProposerRewardsPerDay, } From 70d3317ca736d86df155f40617500c3993f70472 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 11:53:37 -0700 Subject: [PATCH 08/20] save rewards --- .../indexer/save_block_proposed_event.go | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/packages/eventindexer/indexer/save_block_proposed_event.go b/packages/eventindexer/indexer/save_block_proposed_event.go index 8a2e116eed9..c791f424f60 100644 --- a/packages/eventindexer/indexer/save_block_proposed_event.go +++ b/packages/eventindexer/indexer/save_block_proposed_event.go @@ -75,6 +75,16 @@ func (indxr *Indexer) saveBlockProposedEvent( return errors.Wrap(err, "indxr.ethClient.BlockByNumber") } + proposerReward, err := indxr.updateAverageProposerReward(ctx, event) + if err != nil { + return errors.Wrap(err, "indxr.updateAverageProposerReward") + } + + proverReward, err := indxr.updateAverageProverReward(ctx, event) + if err != nil { + return errors.Wrap(err, "indxr.updateAverageProposerReward") + } + _, err = indxr.eventRepo.Save(ctx, eventindexer.SaveEventOpts{ Name: eventindexer.EventNameBlockProposed, Data: string(marshaled), @@ -85,19 +95,13 @@ func (indxr *Indexer) saveBlockProposedEvent( AssignedProver: &assignedProver, TransactedAt: time.Unix(int64(block.Time()), 0), Amount: event.Reward, + ProposerReward: proposerReward, + ProofReward: proverReward, }) if err != nil { return errors.Wrap(err, "indxr.eventRepo.Save") } - if err := indxr.updateAverageProposerReward(ctx, event); err != nil { - return errors.Wrap(err, "indxr.updateAverageProposerReward") - } - - if err := indxr.updateAverageProverReward(ctx, event); err != nil { - return errors.Wrap(err, "indxr.updateAverageProposerReward") - } - eventindexer.BlockProposedEventsProcessed.Inc() return nil @@ -106,17 +110,17 @@ func (indxr *Indexer) saveBlockProposedEvent( func (indxr *Indexer) updateAverageProposerReward( ctx context.Context, event *taikol1.TaikoL1BlockProposed, -) error { +) (*big.Int, error) { stat, err := indxr.statRepo.Find(ctx) if err != nil { - return errors.Wrap(err, "indxr.statRepo.Find") + return nil, errors.Wrap(err, "indxr.statRepo.Find") } reward := event.Reward avg, ok := new(big.Int).SetString(stat.AverageProposerReward, 10) if !ok { - return errors.New("unable to convert average proposer to string") + return nil, errors.New("unable to convert average proposer to string") } newAverageProposerReward := calcNewAverage( @@ -140,31 +144,31 @@ func (indxr *Indexer) updateAverageProposerReward( ProposerReward: newAverageProposerReward, }) if err != nil { - return errors.Wrap(err, "indxr.statRepo.Save") + return nil, errors.Wrap(err, "indxr.statRepo.Save") } - return nil + return reward, err } func (indxr *Indexer) updateAverageProverReward( ctx context.Context, event *taikol1.TaikoL1BlockProposed, -) error { +) (*big.Int, error) { stat, err := indxr.statRepo.Find(ctx) if err != nil { - return errors.Wrap(err, "indxr.statRepo.Find") + return nil, errors.Wrap(err, "indxr.statRepo.Find") } tx, _, err := indxr.ethClient.TransactionByHash(ctx, event.Raw.TxHash) if err != nil { - return errors.Wrap(err, "indxr.ethClient.TransactionByHash") + return nil, errors.Wrap(err, "indxr.ethClient.TransactionByHash") } reward := tx.Value() avg, ok := new(big.Int).SetString(stat.AverageProofReward, 10) if !ok { - return errors.New("unable to convert average proof time to string") + return nil, errors.New("unable to convert average proof time to string") } newAverageProofReward := calcNewAverage( @@ -188,8 +192,8 @@ func (indxr *Indexer) updateAverageProverReward( ProofReward: newAverageProofReward, }) if err != nil { - return errors.Wrap(err, "indxr.statRepo.Save") + return nil, errors.Wrap(err, "indxr.statRepo.Save") } - return nil + return reward, nil } From 88400c31a49455d960d27f0839914493d2781e9b Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 11:57:12 -0700 Subject: [PATCH 09/20] test --- packages/eventindexer/repo/stat_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/eventindexer/repo/stat_test.go b/packages/eventindexer/repo/stat_test.go index 1eca8ab3c70..37abb5b81cf 100644 --- a/packages/eventindexer/repo/stat_test.go +++ b/packages/eventindexer/repo/stat_test.go @@ -5,7 +5,6 @@ import ( "math/big" "testing" - "github.com/davecgh/go-spew/spew" "github.com/taikoxyz/taiko-mono/packages/eventindexer" "gotest.tools/assert" ) @@ -74,9 +73,9 @@ func TestIntegration_Stat_Find(t *testing.T) { "success", &eventindexer.Stat{ ID: 1, - AverageProofReward: new(big.Int).Mul(proofReward, big.NewInt(3)).String(), + AverageProofReward: proofReward.String(), AverageProofTime: "0", - AverageProposerReward: new(big.Int).Mul(proposerReward, big.NewInt(3)).String(), + AverageProposerReward: proposerReward.String(), NumProposerRewards: 3, NumProofs: 0, NumVerifiedBlocks: 3, @@ -88,7 +87,7 @@ func TestIntegration_Stat_Find(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { resp, err := statRepo.Find(context.Background()) - spew.Dump(resp) + assert.Equal(t, tt.wantErr, err) assert.Equal(t, *tt.wantResp, *resp) }) From ebd6f2cae4615d8c2953b89ae4270503a103aaa9 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 12:57:35 -0700 Subject: [PATCH 10/20] coalesce --- packages/eventindexer/generator/generator.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index e379ebb8011..29fc8a5034f 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -188,14 +188,14 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { switch task { case tasks.ProposerRewardsPerDay: - query := "SELECT SUM(proposer_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). Raw(query, eventindexer.EventNameBlockProposed, date). Scan(&result).Error case tasks.TotalProposerRewards: var dailyProposerRewards int - query := "SELECT SUM(proposer_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). Raw(query, eventindexer.EventNameBlockProposed, date). Scan(&result).Error @@ -209,7 +209,7 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { case tasks.TotalProofRewards: var dailyProofRewards int - query := "SELECT SUM(proof_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). Raw(query, eventindexer.EventNameBlockProposed, date). Scan(&result).Error @@ -221,7 +221,7 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { result = strconv.Itoa(dailyProofRewards + tsdResult) case tasks.ProofRewardsPerDay: - query := "SELECT SUM(proof_reward) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). Raw(query, eventindexer.EventNameBlockProposed, date). Scan(&result).Error From ee578b1e1230fc7ad960c251e9f5dd6c4eb457c3 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 13:49:57 -0700 Subject: [PATCH 11/20] use decimal type --- packages/eventindexer/generator/generator.go | 136 +++++++++--------- ...06203931_create_time_series_data_table.sql | 2 +- 2 files changed, 73 insertions(+), 65 deletions(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 29fc8a5034f..d1804de6943 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -4,11 +4,11 @@ import ( "context" "errors" "log/slog" - "strconv" "syscall" "time" "github.com/ethereum/go-ethereum/common" + "github.com/shopspring/decimal" "github.com/taikoxyz/taiko-mono/packages/eventindexer" "github.com/taikoxyz/taiko-mono/packages/eventindexer/tasks" "github.com/urfave/cli/v2" @@ -120,7 +120,7 @@ func (g *Generator) generateByTask(ctx context.Context, task string) error { return err } - slog.Info("Query successful", "task", task, "date", d.Format("2006-01-02"), "result", result) + slog.Info("Query successful", "task", task, "date", d.Format("2006-01-02"), "result", result.String()) insertStmt := ` INSERT INTO time_series_data(task, value, date) @@ -176,13 +176,13 @@ func (g *Generator) getCurrentDate() time.Time { return currentDate } -// nolint: funlen +// nolint: funlen, gocognit // queryByTask runs a database query which should return result data based on the // task -func (g *Generator) queryByTask(task string, date time.Time) (string, error) { +func (g *Generator) queryByTask(task string, date time.Time) (decimal.Decimal, error) { dateString := date.Format("2006-01-02") - var result string + var result decimal.Decimal var err error @@ -190,179 +190,181 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { case tasks.ProposerRewardsPerDay: query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, date). + Raw(query, eventindexer.EventNameBlockProposed, dateString). Scan(&result).Error + case tasks.TotalProposerRewards: - var dailyProposerRewards int + var dailyProposerRewards decimal.NullDecimal query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, date). - Scan(&result).Error + Raw(query, eventindexer.EventNameBlockProposed, dateString). + Scan(&dailyProposerRewards).Error tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProposerRewards + tsdResult) + result = tsdResult.Decimal.Add(dailyProposerRewards.Decimal) + case tasks.TotalProofRewards: - var dailyProofRewards int + var dailyProofRewards decimal.NullDecimal query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, date). - Scan(&result).Error + Raw(query, eventindexer.EventNameBlockProposed, dateString). + Scan(&dailyProofRewards).Error tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProofRewards + tsdResult) + result = tsdResult.Decimal.Add(dailyProofRewards.Decimal) case tasks.ProofRewardsPerDay: query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, date). + Raw(query, eventindexer.EventNameBlockProposed, dateString). Scan(&result).Error case tasks.BridgeMessagesSentPerDay: err = g.eventCount(task, date, eventindexer.EventNameMessageSent, &result) case tasks.TotalBridgeMessagesSent: - var dailyMsgSentCount int + var dailyMsgSentCount decimal.NullDecimal err = g.eventCount(task, date, eventindexer.EventNameMessageSent, &dailyMsgSentCount) if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyMsgSentCount + tsdResult) + result = tsdResult.Decimal.Add(dailyMsgSentCount.Decimal) case tasks.ProposeBlockTxPerDay: err = g.eventCount(task, date, eventindexer.EventNameBlockProposed, &result) case tasks.TotalProposeBlockTx: - var dailyProposerCount int + var dailyProposerCount decimal.NullDecimal err = g.eventCount(task, date, eventindexer.EventNameBlockProposed, &dailyProposerCount) if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProposerCount + tsdResult) + result = tsdResult.Decimal.Add(dailyProposerCount.Decimal) case tasks.UniqueProposersPerDay: query := "SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, date). + Raw(query, eventindexer.EventNameBlockProposed, dateString). Scan(&result).Error case tasks.TotalUniqueProposers: - var dailyProposerCount int + var dailyProposerCount decimal.NullDecimal query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProposed, dateString).Scan(&dailyProposerCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProposerCount + tsdResult) + result = tsdResult.Decimal.Add(dailyProposerCount.Decimal) case tasks.UniqueProversPerDay: query := "SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProven, date). + Raw(query, eventindexer.EventNameBlockProven, dateString). Scan(&result).Error case tasks.TotalUniqueProvers: - var dailyProposerCount int + var dailyProverCount decimal.NullDecimal query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProposerCount).Error + err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProverCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProposerCount + tsdResult) + result = tsdResult.Decimal.Add(dailyProverCount.Decimal) case tasks.ProveBlockTxPerDay: query := "SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProven, date). + Raw(query, eventindexer.EventNameBlockProven, dateString). Scan(&result).Error case tasks.TotalProveBlockTx: - var dailyProposerCount int + var dailyProveBlockCount decimal.NullDecimal query := `SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProposerCount).Error + err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProveBlockCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProposerCount + tsdResult) + result = tsdResult.Decimal.Add(dailyProveBlockCount.Decimal) case tasks.AccountsPerDay: query := `SELECT COUNT(*) FROM accounts WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalAccounts: - var dailyAccountsCount int + var dailyAccountsCount decimal.NullDecimal query := `SELECT COUNT(*) FROM accounts WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&dailyAccountsCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyAccountsCount + tsdResult) + result = tsdResult.Decimal.Add(dailyAccountsCount.Decimal) case tasks.BlocksPerDay: query := `SELECT COUNT(*) FROM blocks WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalBlocks: - var dailyBlockCount int + var dailyBlockCount decimal.NullDecimal query := `SELECT COUNT(*) FROM blocks WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&dailyBlockCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyBlockCount + tsdResult) + result = tsdResult.Decimal.Add(dailyBlockCount.Decimal) case tasks.TransactionsPerDay: query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalTransactions: - var dailyTxCount int + var dailyTxCount decimal.NullDecimal // get current days txs, get previous entry for the time series data, add them together. @@ -370,41 +372,41 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { err = g.db.GormDB().Raw(query, dateString).Scan(&dailyTxCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyTxCount + tsdResult) + result = tsdResult.Decimal.Add(dailyTxCount.Decimal) case tasks.ContractDeploymentsPerDay: query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&result).Error case tasks.TotalContractDeployments: - var dailyContractCount int + var dailyContractCount decimal.NullDecimal // get current days txs, get previous entry for the time series data, add them together. query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&dailyContractCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyContractCount + tsdResult) + result = tsdResult.Decimal.Add(dailyContractCount.Decimal) default: - return "", errors.New("task not supported") + return result, errors.New("task not supported") } if err != nil { - return "", err + return result, err } return result, nil @@ -412,16 +414,22 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { // previousDayTsdResultByTask returns the previous day's time series data, based on // task and time passed in. -func (g *Generator) previousDayTsdResultByTask(task string, date time.Time) (int, error) { - var tsdResult int +func (g *Generator) previousDayTsdResultByTask(task string, date time.Time) (decimal.NullDecimal, error) { + var tsdResult decimal.NullDecimal tsdQuery := `SELECT value FROM time_series_data WHERE task = ? AND date = ?` err := g.db.GormDB().Raw(tsdQuery, task, date.AddDate(0, 0, -1).Format("2006-01-02")).Scan(&tsdResult).Error if err != nil { - return 0, err + return tsdResult, err } + if !tsdResult.Valid { + return decimal.NullDecimal{ + Valid: true, + Decimal: decimal.Zero, + }, nil + } return tsdResult, nil } @@ -431,6 +439,6 @@ func (g *Generator) eventCount(task string, date time.Time, event string, result query := "SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?" return g.db.GormDB(). - Raw(query, event, date). + Raw(query, event, date.Format("2006-01-02")). Scan(result).Error } diff --git a/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql b/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql index 64bd23e013e..b41b7cf0bcc 100644 --- a/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql +++ b/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS time_series_data ( id int NOT NULL PRIMARY KEY AUTO_INCREMENT, task VARCHAR(40) NOT NULL, - value VARCHAR(100) NOT NULL, + value DECIMAL(65, 0) NOT NULL, date VARCHAR(20) NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, From b06e616d23d461f8da6910960df6b5f4b52ecf49 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 13:51:05 -0700 Subject: [PATCH 12/20] models --- packages/eventindexer/generator/generator.go | 1 + packages/eventindexer/repo/{chart_repo.go => chart.go} | 2 +- packages/eventindexer/time_series_data.go | 8 ++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) rename packages/eventindexer/repo/{chart_repo.go => chart.go} (96%) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index d1804de6943..f5f4bcd20f1 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -430,6 +430,7 @@ func (g *Generator) previousDayTsdResultByTask(task string, date time.Time) (dec Decimal: decimal.Zero, }, nil } + return tsdResult, nil } diff --git a/packages/eventindexer/repo/chart_repo.go b/packages/eventindexer/repo/chart.go similarity index 96% rename from packages/eventindexer/repo/chart_repo.go rename to packages/eventindexer/repo/chart.go index e1326c4a35c..590ed4e92f5 100644 --- a/packages/eventindexer/repo/chart_repo.go +++ b/packages/eventindexer/repo/chart.go @@ -48,7 +48,7 @@ func (r *ChartRepository) Find( for _, d := range tsd { chart.Chart = append(chart.Chart, eventindexer.ChartItem{ Date: d.Date, - Value: d.Value, + Value: d.Value.Decimal.String(), }) } diff --git a/packages/eventindexer/time_series_data.go b/packages/eventindexer/time_series_data.go index 4d593b901b0..35c7dd8ec84 100644 --- a/packages/eventindexer/time_series_data.go +++ b/packages/eventindexer/time_series_data.go @@ -1,11 +1,15 @@ package eventindexer -import "time" +import ( + "time" + + "github.com/shopspring/decimal" +) type TimeSeriesData struct { ID int Task string - Value string + Value decimal.NullDecimal Date string CreatedAt time.Time UpdatedAt time.Time From a25ed2053cf96806ce0fb4702a0ca4c10d57e072 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 16:38:20 -0700 Subject: [PATCH 13/20] tests --- packages/eventindexer/http/get_stats_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eventindexer/http/get_stats_test.go b/packages/eventindexer/http/get_stats_test.go index a4c5c6000fb..1212aedc014 100644 --- a/packages/eventindexer/http/get_stats_test.go +++ b/packages/eventindexer/http/get_stats_test.go @@ -37,7 +37,7 @@ func Test_GetStats(t *testing.T) { "success", "0x123", http.StatusOK, - []string{`{"id":1,"averageProofTime":"5","averageProofReward":"7","numProofs":1,"numVerifiedBlocks":1}`}, + []string{`{"id":1,"averageProofTime":"5","averageProofReward":"7","averageProposerReward":"","numProposerRewards":0,"numProofs":1,"numVerifiedBlocks":1}`}, }, } From 20194868cf5ef06e21f6807287b7651ca95fe92c Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 16:38:34 -0700 Subject: [PATCH 14/20] . --- packages/eventindexer/http/get_stats_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/eventindexer/http/get_stats_test.go b/packages/eventindexer/http/get_stats_test.go index 1212aedc014..9926d1f9a25 100644 --- a/packages/eventindexer/http/get_stats_test.go +++ b/packages/eventindexer/http/get_stats_test.go @@ -37,6 +37,7 @@ func Test_GetStats(t *testing.T) { "success", "0x123", http.StatusOK, + // nolint: lll []string{`{"id":1,"averageProofTime":"5","averageProofReward":"7","averageProposerReward":"","numProposerRewards":0,"numProofs":1,"numVerifiedBlocks":1}`}, }, } From 47cd2fa2250ffd3cc48a377c760ae0f19afe8b86 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 16:39:09 -0700 Subject: [PATCH 15/20] UTC --- packages/eventindexer/generator/generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index f5f4bcd20f1..d41313e4fff 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -170,7 +170,7 @@ func (g *Generator) getStartingDateByTask(ctx context.Context, task string) (tim // getCurrentDate returns the current date in YYYY-MM-DD format func (g *Generator) getCurrentDate() time.Time { // Get current date - currentTime := time.Now() + currentTime := time.Now().UTC() currentDate := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), 0, 0, 0, 0, time.UTC) return currentDate From dfafe59610444fc6bbc23fcd868c9f3f950f7490 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 16:47:12 -0700 Subject: [PATCH 16/20] use uTC --- packages/eventindexer/db/db.go | 6 ++++-- packages/eventindexer/indexer/save_block_proposed_event.go | 2 +- packages/relayer/db/db.go | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/eventindexer/db/db.go b/packages/eventindexer/db/db.go index 9223263b3cb..e2d19c61ca0 100644 --- a/packages/eventindexer/db/db.go +++ b/packages/eventindexer/db/db.go @@ -46,18 +46,20 @@ func OpenDBConnection(opts DBConnectionOpts) (*DB, error) { dsn := "" if opts.Password == "" { dsn = fmt.Sprintf( - "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", opts.Name, opts.Host, opts.Database, + "UTC", ) } else { dsn = fmt.Sprintf( - "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", opts.Name, opts.Password, opts.Host, opts.Database, + "UTC", ) } diff --git a/packages/eventindexer/indexer/save_block_proposed_event.go b/packages/eventindexer/indexer/save_block_proposed_event.go index c791f424f60..6564efcef1f 100644 --- a/packages/eventindexer/indexer/save_block_proposed_event.go +++ b/packages/eventindexer/indexer/save_block_proposed_event.go @@ -93,7 +93,7 @@ func (indxr *Indexer) saveBlockProposedEvent( Address: sender.Hex(), BlockID: &blockID, AssignedProver: &assignedProver, - TransactedAt: time.Unix(int64(block.Time()), 0), + TransactedAt: time.Unix(int64(block.Time()), 0).UTC(), Amount: event.Reward, ProposerReward: proposerReward, ProofReward: proverReward, diff --git a/packages/relayer/db/db.go b/packages/relayer/db/db.go index 9223263b3cb..e2d19c61ca0 100644 --- a/packages/relayer/db/db.go +++ b/packages/relayer/db/db.go @@ -46,18 +46,20 @@ func OpenDBConnection(opts DBConnectionOpts) (*DB, error) { dsn := "" if opts.Password == "" { dsn = fmt.Sprintf( - "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", opts.Name, opts.Host, opts.Database, + "UTC", ) } else { dsn = fmt.Sprintf( - "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", opts.Name, opts.Password, opts.Host, opts.Database, + "UTC", ) } From ceb0b0742d0726764316f422e60fcfdf96780fac Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 17:00:59 -0700 Subject: [PATCH 17/20] dont sysskill --- packages/eventindexer/db/db.go | 6 ++---- packages/eventindexer/generator/generator.go | 5 ----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/eventindexer/db/db.go b/packages/eventindexer/db/db.go index e2d19c61ca0..3b865643fb7 100644 --- a/packages/eventindexer/db/db.go +++ b/packages/eventindexer/db/db.go @@ -46,20 +46,18 @@ func OpenDBConnection(opts DBConnectionOpts) (*DB, error) { dsn := "" if opts.Password == "" { dsn = fmt.Sprintf( - "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", + "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True", opts.Name, opts.Host, opts.Database, - "UTC", ) } else { dsn = fmt.Sprintf( - "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", + "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True", opts.Name, opts.Password, opts.Host, opts.Database, - "UTC", ) } diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index d41313e4fff..1097e7d0d88 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -4,7 +4,6 @@ import ( "context" "errors" "log/slog" - "syscall" "time" "github.com/ethereum/go-ethereum/common" @@ -58,10 +57,6 @@ func (g *Generator) Start() error { return err } - if err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); err != nil { - return err - } - return nil } From 17f20ef250064d0a06a27257d88c96e83fb22327 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 17:19:33 -0700 Subject: [PATCH 18/20] total unique proposers/provers fix --- packages/eventindexer/generator/generator.go | 32 ++++++-------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 1097e7d0d88..4cb5c46e2bc 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -260,42 +260,30 @@ func (g *Generator) queryByTask(task string, date time.Time) (decimal.Decimal, e Raw(query, eventindexer.EventNameBlockProposed, dateString). Scan(&result).Error case tasks.TotalUniqueProposers: - var dailyProposerCount decimal.NullDecimal - - query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProposed, dateString).Scan(&dailyProposerCount).Error - if err != nil { - return result, err - } + query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ?` - tsdResult, err := g.previousDayTsdResultByTask(task, date) + err = g.db.GormDB().Raw( + query, + eventindexer.EventNameBlockProposed, + ).Scan(&result).Error if err != nil { return result, err } - - result = tsdResult.Decimal.Add(dailyProposerCount.Decimal) case tasks.UniqueProversPerDay: query := "SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). Raw(query, eventindexer.EventNameBlockProven, dateString). Scan(&result).Error case tasks.TotalUniqueProvers: - var dailyProverCount decimal.NullDecimal - - query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProverCount).Error - if err != nil { - return result, err - } + query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ?` - tsdResult, err := g.previousDayTsdResultByTask(task, date) + err = g.db.GormDB().Raw( + query, + eventindexer.EventNameBlockProven, + ).Scan(&result).Error if err != nil { return result, err } - - result = tsdResult.Decimal.Add(dailyProverCount.Decimal) case tasks.ProveBlockTxPerDay: query := "SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). From 95984fa01c98c1ac22735b288bcacf76c184ce9e Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 17:25:30 -0700 Subject: [PATCH 19/20] Regen flag --- packages/eventindexer/cmd/flags/generator.go | 8 ++++++++ packages/eventindexer/generator/config.go | 2 ++ packages/eventindexer/generator/config_test.go | 2 ++ packages/eventindexer/generator/generator.go | 18 ++++++++++++++++++ 4 files changed, 30 insertions(+) diff --git a/packages/eventindexer/cmd/flags/generator.go b/packages/eventindexer/cmd/flags/generator.go index b9388df43ac..9c53b437f50 100644 --- a/packages/eventindexer/cmd/flags/generator.go +++ b/packages/eventindexer/cmd/flags/generator.go @@ -10,7 +10,15 @@ var ( Category: generatorCategory, EnvVars: []string{"GENESIS_DATE"}, } + Regenerate = &cli.StringFlag{ + Name: "regenerate", + Usage: "True to delete all existing data and regenerate from genesis, false to not", + Required: false, + Category: generatorCategory, + EnvVars: []string{"REGENERATE"}, + } ) var GeneratorFlags = MergeFlags(CommonFlags, []cli.Flag{ GenesisDate, + Regenerate, }) diff --git a/packages/eventindexer/generator/config.go b/packages/eventindexer/generator/config.go index 1cf9ec83675..378b03c24c0 100644 --- a/packages/eventindexer/generator/config.go +++ b/packages/eventindexer/generator/config.go @@ -28,6 +28,7 @@ type Config struct { DatabaseMaxConnLifetime uint64 MetricsHTTPPort uint64 GenesisDate time.Time + Regenerate bool OpenDBFunc func() (DB, error) } @@ -48,6 +49,7 @@ func NewConfigFromCliContext(c *cli.Context) (*Config, error) { DatabaseMaxConnLifetime: c.Uint64(flags.DatabaseConnMaxLifetime.Name), MetricsHTTPPort: c.Uint64(flags.MetricsHTTPPort.Name), GenesisDate: date, + Regenerate: c.Bool(flags.Regenerate.Name), OpenDBFunc: func() (DB, error) { return db.OpenDBConnection(db.DBConnectionOpts{ Name: c.String(flags.DatabaseUsername.Name), diff --git a/packages/eventindexer/generator/config_test.go b/packages/eventindexer/generator/config_test.go index 2731d49fb0f..f38f1ed1205 100644 --- a/packages/eventindexer/generator/config_test.go +++ b/packages/eventindexer/generator/config_test.go @@ -32,6 +32,7 @@ func TestNewConfigFromCliContext(t *testing.T) { assert.Equal(t, "dbpass", c.DatabasePassword) assert.Equal(t, "dbname", c.DatabaseName) assert.Equal(t, "dbhost", c.DatabaseHost) + assert.Equal(t, true, c.Regenerate) wantTime, _ := time.Parse("2006-01-02", "2023-07-07") assert.Equal(t, wantTime, c.GenesisDate) @@ -52,5 +53,6 @@ func TestNewConfigFromCliContext(t *testing.T) { "-" + flags.DatabaseHost.Name, "dbhost", "-" + flags.DatabaseName.Name, "dbname", "-" + flags.GenesisDate.Name, "2023-07-07", + "-" + flags.Regenerate.Name, "true", })) } diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 4cb5c46e2bc..5cc9d023629 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -23,6 +23,7 @@ var ( type Generator struct { db DB genesisDate time.Time + regenerate bool } func (g *Generator) InitFromCli(ctx context.Context, c *cli.Context) error { @@ -42,6 +43,7 @@ func InitFromConfig(ctx context.Context, g *Generator, cfg *Config) error { g.db = db g.genesisDate = cfg.GenesisDate + g.regenerate = cfg.Regenerate return nil } @@ -51,6 +53,13 @@ func (g *Generator) Name() string { } func (g *Generator) Start() error { + if g.regenerate { + slog.Info("regenerating, deleting existing data") + if err := g.deleteTimeSeriesData(context.Background()); err != nil { + return err + } + } + slog.Info("generating time series data") if err := g.generateTimeSeriesData(context.Background()); err != nil { @@ -71,6 +80,15 @@ func (g *Generator) Close(ctx context.Context) { } } +func (g *Generator) deleteTimeSeriesData(ctx context.Context) error { + deleteStmt := "DELETE FROM time_series_data;" + if err := g.db.GormDB().Exec(deleteStmt).Error; err != nil { + return err + } + + return nil +} + // generateTimeSeriesData iterates over each task and generates time series data. func (g *Generator) generateTimeSeriesData(ctx context.Context) error { for _, task := range tasks.Tasks { From 3a32a7228ad627b55dd5573c4a4a02fc0c1ca466 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 13 Sep 2023 17:25:52 -0700 Subject: [PATCH 20/20] lint --- packages/eventindexer/generator/generator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 5cc9d023629..b60228f9b4c 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -55,6 +55,7 @@ func (g *Generator) Name() string { func (g *Generator) Start() error { if g.regenerate { slog.Info("regenerating, deleting existing data") + if err := g.deleteTimeSeriesData(context.Background()); err != nil { return err }