diff --git a/cmd/misc/main.go b/cmd/misc/main.go index ef201dbb64..cf3201198f 100644 --- a/cmd/misc/main.go +++ b/cmd/misc/main.go @@ -1942,8 +1942,6 @@ func updateRatelimits() error { err := ratelimit.DBUpdate() if err != nil { logrus.WithError(err).Errorf("error in updateRatelimits") - } else { - logrus.Infof("updated ratelimits") } time.Sleep(time.Second * 10) } diff --git a/db/migrations/20240125120000_add_ratelimits.sql b/db/migrations/20240125120000_add_ratelimits.sql index 37086e5516..fd05e1fe2d 100644 --- a/db/migrations/20240125120000_add_ratelimits.sql +++ b/db/migrations/20240125120000_add_ratelimits.sql @@ -12,6 +12,8 @@ CREATE TABLE IF NOT EXISTS PRIMARY KEY (user_id) ); +CREATE INDEX IF NOT EXISTS idx_api_ratelimits_changed_at ON api_ratelimits (changed_at); + SELECT 'up SQL query - add table api_keys'; CREATE TABLE IF NOT EXISTS api_keys ( @@ -22,6 +24,8 @@ CREATE TABLE IF NOT EXISTS PRIMARY KEY (user_id, api_key) ); +CREATE INDEX IF NOT EXISTS idx_api_keys_changed_at ON api_keys (changed_at); + SELECT 'up SQL query - add table api_weights'; CREATE TABLE IF NOT EXISTS api_weights ( @@ -51,8 +55,12 @@ CREATE TABLE IF NOT EXISTS -- +goose StatementBegin SELECT 'down SQL query - drop table api_ratelimits'; DROP TABLE IF EXISTS api_ratelimits; +SELECT 'down SQL query - drop index idx_api_ratelimits_changed_at'; +DROP INDEX IF EXISTS idx_api_ratelimits_changed_at; SELECT 'down SQL query - drop table api_keys'; DROP TABLE IF EXISTS api_keys; +SELECT 'down SQL query - drop index idx_api_keys_changed_at'; +DROP INDEX IF EXISTS idx_api_keys_changed_at; SELECT 'down SQL query - drop table api_weights'; DROP TABLE IF EXISTS api_weights; SELECT 'down SQL query - drop table api_products'; diff --git a/db/ratelimit.go b/db/ratelimit.go deleted file mode 100644 index c350867710..0000000000 --- a/db/ratelimit.go +++ /dev/null @@ -1,64 +0,0 @@ -package db - -import ( - "database/sql" - "eth2-exporter/utils" -) - -func GetUserRatelimitProduct() {} - -func UpdateApiRatelimits() (sql.Result, error) { - return FrontendWriterDB.Exec( - `with - stripe_price_ids as ( - select product, price_id from ( values - ('sapphire', $1), - ('emerald', $2), - ('diamond', $3), - ('custom1', $4), - ('custom2', $5), - ('whale', $6), - ('goldfish', $7), - ('plankton', $8) - ) as x(product, price_id) - ), - current_api_products as ( - select distinct on (product) product, second, hour, month, valid_from - from api_products - where valid_from <= now() - order by product, valid_from desc - ) - insert into api_ratelimits (user_id, second, hour, month, valid_until, changed_at) - select - u.id as user_id, - greatest(coalesce(cap1.second,0),coalesce(cap2.second,0)) as second, - greatest(coalesce(cap1.hour ,0),coalesce(cap2.hour ,0)) as hour, - greatest(coalesce(cap1.month ,0),coalesce(cap2.month ,0)) as month, - to_timestamp('3000-01-01', 'YYYY-MM-DD') as valid_until, - now() as changed_at - from users u - left join users_stripe_subscriptions uss on uss.customer_id = u.stripe_customer_id and uss.active = true - left join stripe_price_ids spi1 on spi1.price_id = uss.price_id - left join current_api_products cap1 on cap1.product = coalesce(spi1.product,'free') - left join app_subs_view asv on asv.user_id = u.id and asv.active = true - left join current_api_products cap2 on cap2.product = coalesce(asv.product_id,'free') - on conflict (user_id) do update set - second = excluded.second, - hour = excluded.hour, - month = excluded.month, - valid_until = excluded.valid_until, - changed_at = now() - where - api_ratelimits.second != excluded.second - or api_ratelimits.hour != excluded.hour - or api_ratelimits.month != excluded.month`, - utils.Config.Frontend.Stripe.Sapphire, - utils.Config.Frontend.Stripe.Emerald, - utils.Config.Frontend.Stripe.Diamond, - utils.Config.Frontend.Stripe.Custom1, - utils.Config.Frontend.Stripe.Custom2, - utils.Config.Frontend.Stripe.Whale, - utils.Config.Frontend.Stripe.Goldfish, - utils.Config.Frontend.Stripe.Plankton, - ) -} diff --git a/ratelimit/ratelimit.go b/ratelimit/ratelimit.go index 7f916afd66..ca504ce390 100644 --- a/ratelimit/ratelimit.go +++ b/ratelimit/ratelimit.go @@ -285,6 +285,7 @@ func HttpMiddleware(next http.Handler) http.Handler { func updateWeights(firstRun bool) error { start := time.Now() defer func() { + logger.Infof("updateWeights took %v", time.Since(start).Seconds()) metrics.TaskDuration.WithLabelValues("ratelimit_updateWeights").Observe(time.Since(start).Seconds()) }() @@ -341,6 +342,7 @@ func updateRedisStatus() error { func updateStats() error { start := time.Now() defer func() { + logger.Infof("updateStats took %v", time.Since(start).Seconds()) metrics.TaskDuration.WithLabelValues("ratelimit_updateStats").Observe(time.Since(start).Seconds()) }() @@ -505,6 +507,7 @@ func updateStatsEntries(entries []dbEntry) error { func updateRateLimits() error { start := time.Now() defer func() { + logger.Infof("updateRateLimits took %v", time.Since(start).Seconds()) metrics.TaskDuration.WithLabelValues("ratelimit_updateRateLimits").Observe(time.Since(start).Seconds()) }() @@ -928,6 +931,7 @@ func DBUpdate() error { } logrus.Infof("updated %v api_keys in %v", ra, time.Since(now)) + now = time.Now() _, err = DBUpdateApiRatelimits() if err != nil { return err @@ -938,15 +942,15 @@ func DBUpdate() error { } logrus.Infof("updated %v api_ratelimits in %v", ra, time.Since(now)) - _, err = DBInvalidateApiKeys() - if err != nil { - return err - } - ra, err = res.RowsAffected() - if err != nil { - return err - } - logrus.Infof("invalidated %v api_keys in %v", ra, time.Since(now)) + // _, err = DBInvalidateApiKeys() + // if err != nil { + // return err + // } + // ra, err = res.RowsAffected() + // if err != nil { + // return err + // } + // logrus.Infof("invalidated %v api_keys in %v", ra, time.Since(now)) return nil } @@ -955,8 +959,7 @@ func DBInvalidateApiKeys() (sql.Result, error) { return db.FrontendWriterDB.Exec(` update api_ratelimits set changed_at = now(), valid_until = now() - where valid_until > now() - and user_id not in (select user_id from api_keys where api_key is not null)`) + where valid_until > now() and not exists (select id from api_keys where api_keys.user_id = api_ratelimits.user_id)`) } func DBUpdateApiKeys() (sql.Result, error) { @@ -968,7 +971,7 @@ func DBUpdateApiKeys() (sql.Result, error) { to_timestamp('3000-01-01', 'YYYY-MM-DD') as valid_until, now() as changed_at from users - where api_key is not null + where api_key is not null and not exists (select user_id from api_keys where api_keys.user_id = users.id) on conflict (user_id, api_key) do update set valid_until = excluded.valid_until, changed_at = excluded.changed_at @@ -999,6 +1002,9 @@ func DBUpdateApiRatelimits() (sql.Result, error) { left join current_api_products cap1 on cap1.name = coalesce(cap.name,'free') left join app_subs_view asv on asv.user_id = u.id and asv.active = true left join current_api_products cap2 on cap2.name = coalesce(asv.product_id,'free') + left join api_ratelimits ar on ar.user_id = u.id + where + cap1.name != 'free' or cap2.name != 'free' or ar.user_id is not null on conflict (user_id) do update set second = excluded.second, hour = excluded.hour,