diff --git a/exp/ticker/cmd/clean.go b/exp/ticker/cmd/clean.go new file mode 100644 index 0000000000..dc00b7993c --- /dev/null +++ b/exp/ticker/cmd/clean.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "time" + + "github.com/lib/pq" + "github.com/spf13/cobra" + "github.com/stellar/go/exp/ticker/internal/tickerdb" +) + +var DaysToKeep int + +func init() { + rootCmd.AddCommand(cmdClean) + cmdClean.AddCommand(cmdCleanTrades) + + cmdCleanTrades.Flags().IntVarP( + &DaysToKeep, + "keep-days", + "k", + 7, + "Trade entries older than keep-days will be deleted", + ) +} + +var cmdClean = &cobra.Command{ + Use: "clean [data type]", + Short: "Cleans up the database for a given data type", +} + +var cmdCleanTrades = &cobra.Command{ + Use: "trades", + Short: "Cleans up old trades from the database", + Run: func(cmd *cobra.Command, args []string) { + dbInfo, err := pq.ParseURL(DatabaseURL) + if err != nil { + Logger.Fatal("could not parse db-url:", err) + } + + session, err := tickerdb.CreateSession("postgres", dbInfo) + if err != nil { + Logger.Fatal("could not connect to db:", err) + } + + now := time.Now() + minDate := now.AddDate(0, 0, -DaysToKeep) + Logger.Infof("Deleting trade entries older than %d days", DaysToKeep) + err = session.DeleteOldTrades(minDate) + if err != nil { + Logger.Fatal("could not delete trade entries:", err) + } + }, +} diff --git a/exp/ticker/internal/tickerdb/queries_trade.go b/exp/ticker/internal/tickerdb/queries_trade.go index e2ff921233..d3cb5e2be8 100644 --- a/exp/ticker/internal/tickerdb/queries_trade.go +++ b/exp/ticker/internal/tickerdb/queries_trade.go @@ -3,6 +3,7 @@ package tickerdb import ( "math" "strings" + "time" ) // BulkInsertTrades inserts a slice of trades in the database. Trades @@ -30,6 +31,12 @@ func (s *TickerSession) GetLastTrade() (trade Trade, err error) { return } +// DeleteOldTrades deletes trades in the database older than minDate. +func (s *TickerSession) DeleteOldTrades(minDate time.Time) error { + _, err := s.ExecRaw("DELETE FROM trades WHERE ledger_close_time < ?", minDate) + return err +} + // chunkifyDBTrades transforms a slice into a slice of chunks (also slices) of chunkSize // e.g.: Chunkify([b, c, d, e, f], 2) = [[b c] [d e] [f]] func chunkifyDBTrades(sl []Trade, chunkSize int) [][]Trade { diff --git a/exp/ticker/internal/tickerdb/queries_trade_test.go b/exp/ticker/internal/tickerdb/queries_trade_test.go index 3102e75877..b62c1d1943 100644 --- a/exp/ticker/internal/tickerdb/queries_trade_test.go +++ b/exp/ticker/internal/tickerdb/queries_trade_test.go @@ -223,3 +223,139 @@ func TestGetLastTrade(t *testing.T) { lastTrade.LedgerCloseTime.Local().Truncate(time.Millisecond), ) } + +func TestDeleteOldTrades(t *testing.T) { + db := dbtest.Postgres(t) + defer db.Close() + + var session TickerSession + session.DB = db.Open() + defer session.DB.Close() + + // Run migrations to make sure the tests are run + // on the most updated schema version + migrations := &migrate.FileMigrationSource{ + Dir: "./migrations", + } + _, err := migrate.Exec(session.DB.DB, "postgres", migrations, migrate.Up) + require.NoError(t, err) + + // Adding a seed issuer to be used later: + tbl := session.GetTable("issuers") + _, err = tbl.Insert(Issuer{ + PublicKey: "GCF3TQXKZJNFJK7HCMNE2O2CUNKCJH2Y2ROISTBPLC7C5EIA5NNG2XZB", + Name: "FOO BAR", + }).IgnoreCols("id").Exec() + require.NoError(t, err) + var issuer Issuer + err = session.GetRaw(&issuer, ` + SELECT * + FROM issuers + ORDER BY id DESC + LIMIT 1`, + ) + require.NoError(t, err) + + // Adding a seed asset to be used later: + err = session.InsertOrUpdateAsset(&Asset{ + Code: "XLM", + IssuerID: issuer.ID, + }, []string{"code", "issuer_id"}) + require.NoError(t, err) + var asset1 Asset + err = session.GetRaw(&asset1, ` + SELECT * + FROM assets + ORDER BY id DESC + LIMIT 1`, + ) + require.NoError(t, err) + + // Adding another asset to be used later: + err = session.InsertOrUpdateAsset(&Asset{ + Code: "BTC", + IssuerID: issuer.ID, + }, []string{"code", "issuer_id"}) + require.NoError(t, err) + var asset2 Asset + err = session.GetRaw(&asset2, ` + SELECT * + FROM assets + ORDER BY id DESC + LIMIT 1`, + ) + require.NoError(t, err) + + // Verify that we actually have two assets: + assert.NotEqual(t, asset1.ID, asset2.ID) + + // Setting up some times for testing + now := time.Now() + oneDayAgo := now.AddDate(0, 0, -1) + oneMonthAgo := now.AddDate(0, -1, 0) + oneYearAgo := now.AddDate(-1, 0, 0) + + // Now let's create the trades: + trades := []Trade{ + Trade{ + HorizonID: "hrzid1", + BaseAssetID: asset1.ID, + CounterAssetID: asset2.ID, + LedgerCloseTime: now, + }, + Trade{ + HorizonID: "hrzid2", + BaseAssetID: asset2.ID, + CounterAssetID: asset1.ID, + LedgerCloseTime: oneDayAgo, + }, + Trade{ + HorizonID: "hrzid3", + BaseAssetID: asset2.ID, + CounterAssetID: asset1.ID, + LedgerCloseTime: oneMonthAgo, + }, + Trade{ + HorizonID: "hrzid4", + BaseAssetID: asset2.ID, + CounterAssetID: asset1.ID, + LedgerCloseTime: oneYearAgo, + }, + } + err = session.BulkInsertTrades(trades) + require.NoError(t, err) + + // Deleting trades older than 1 day ago: + err = session.DeleteOldTrades(oneDayAgo) + require.NoError(t, err) + + var dbTrades []Trade + var trade1, trade2 Trade + err = session.SelectRaw(&dbTrades, "SELECT * FROM trades") + require.NoError(t, err) + assert.Equal(t, 2, len(dbTrades)) + + // Make sure we're actually deleting the entries we wanted: + for i, trade := range dbTrades { + if trade.HorizonID == "hrzid1" { + trade1 = dbTrades[i] + } + + if trade.HorizonID == "hrzid2" { + trade2 = dbTrades[i] + } + } + + assert.NotEqual(t, trade1.HorizonID, "") + assert.NotEqual(t, trade2.HorizonID, "") + assert.Equal( + t, + now.Local().Truncate(time.Millisecond), + trade1.LedgerCloseTime.Local().Truncate(time.Millisecond), + ) + assert.Equal( + t, + oneDayAgo.Local().Truncate(time.Millisecond), + trade2.LedgerCloseTime.Local().Truncate(time.Millisecond), + ) +}