(BIDS-2872) wip

db/migrations/20240125120000_add_ratelimits.sql

@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS
     api_keys (
         user_id INT NOT NULL,
         api_key VARCHAR(256) NOT NULL,
-        valid_until TIMESTAMP WITHOUT TIME ZONE NOT NULL,
-        changed_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+        valid_until TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT '9999-12-31 23:59:59',
+        changed_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
         PRIMARY KEY (user_id, api_key)
     );
 

local-deployment/provision-explorer-config.sh

@@ -101,7 +101,7 @@ frontend:
     sapphire: price_sapphire
     emerald: price_emerald
     diamond: price_diamond
-  rateLimitUpdateInterval: 1s
+  ratelimitUpdateInterval: 1s
 
 indexer:
   # fullIndexOnStartup: false # Perform a one time full db index on startup

ratelimit/ratelimit.go

@@ -1015,19 +1015,6 @@ func DBUpdate(redisClient *redis.Client) {
 		}
 		logger.WithField("duration", time.Since(start)).WithField("updates", ra).Infof("updated api_ratelimits")
 
-		start = time.Now()
-		res, err = DBUpdateUnlimitedRatelimits()
-		if err != nil {
-			logger.WithError(err).Errorf("error updating unlikmited api_ratelimit")
-			return
-		}
-		ra, err = res.RowsAffected()
-		if err != nil {
-			logger.WithError(err).Errorf("error getting rows affected")
-			return
-		}
-		logger.WithField("duration", time.Since(start)).WithField("updates", ra).Infof("updated unlimited api_ratelimits")
-
 		start = time.Now()
 		res, err = DBInvalidateApiKeys()
 		if err != nil {
@@ -1044,20 +1031,22 @@ func DBUpdate(redisClient *redis.Client) {
 	wg.Wait()
 }
 
+// DBInvalidateApiKeys invalidates api_keys that are not associated with a user. This func is only needed until api-key-mgmt is fully implemented - where users.apikey column is not used anymore.
 func DBInvalidateApiKeys() (sql.Result, error) {
 	return db.FrontendWriterDB.Exec(`
         update api_keys 
         set changed_at = now(), valid_until = now() 
         where valid_until > now() and not exists (select id from users where id = api_keys.user_id)`)
 }
 
+// DBUpdateApiKeys updates the api_keys table with the api_keys from the users table. This func is only needed until api-key-mgmt is fully implemented - where users.apikey column is not used anymore.
 func DBUpdateApiKeys() (sql.Result, error) {
 	return db.FrontendWriterDB.Exec(
 		`insert into api_keys (user_id, api_key, valid_until, changed_at)
         select 
             id as user_id, 
             api_key,
-            to_timestamp('3000-01-01', 'YYYY-MM-DD') as valid_until,
+            to_timestamp('9999-12-31 23:59:59', 'YYYY-MM-DD HH:MI:SS') as valid_until,
             now() as changed_at
         from users 
         where api_key is not null and not exists (select user_id from api_keys where api_keys.user_id = users.id)
@@ -1071,70 +1060,54 @@ func DBUpdateApiKeys() (sql.Result, error) {
 func DBUpdateApiRatelimits() (sql.Result, error) {
 	return db.FrontendWriterDB.Exec(
 		`with 
-            current_api_products as (
-                select distinct on (name) name, stripe_price_id, second, hour, month, valid_from 
-                from api_products 
-                where valid_from <= now()
-                order by name, valid_from desc
-            )
-        insert into api_ratelimits (user_id, second, hour, month, valid_until, changed_at)
-        select
-            u.id as user_id,
-            max(greatest(coalesce(cap1.second,0),coalesce(cap2.second,0))) as second,
-            max(greatest(coalesce(cap1.hour  ,0),coalesce(cap2.hour  ,0))) as hour,
-            max(greatest(coalesce(cap1.month ,0),coalesce(cap2.month ,0))) as month,
-            to_timestamp('3000-01-01', 'YYYY-MM-DD') as valid_until,
-            now() as changed_at
-        from users u
-            left join users_stripe_subscriptions uss on uss.customer_id = u.stripe_customer_id and uss.active = true
-            left join current_api_products cap on cap.stripe_price_id = uss.price_id
-            left join current_api_products cap1 on cap1.name = coalesce(cap.name,'free')
-            left join app_subs_view asv on asv.user_id = u.id and asv.active = true
-            left join current_api_products cap2 on cap2.name = coalesce(asv.product_id,'free')
-            left join api_ratelimits ar on ar.user_id = u.id
-        where
-            cap1.name != 'free' or cap2.name != 'free' or ar.user_id is not null
-		group by u.id, valid_until, changed_at
-        on conflict (user_id) do update set
-            second = excluded.second,
-            hour = excluded.hour,
-            month = excluded.month,
-            valid_until = excluded.valid_until,
-            changed_at = now()
-        where
-            api_ratelimits.second != excluded.second 
-            or api_ratelimits.hour != excluded.hour 
-            or api_ratelimits.month != excluded.month`)
-}
-
-func DBUpdateUnlimitedRatelimits() (sql.Result, error) {
-	return db.FrontendWriterDB.Exec(
-		`with
-            unlimited_ratelimit as (
-                select second, hour, month
-                from api_products
-                where name = 'unlimited' and valid_from <= now()
-                order by valid_from desc
-                limit 1
-            )
-        insert into api_ratelimits (user_id, second, hour, month, valid_until, changed_at)
-        select
-            id as user_id,
-            unlimited_ratelimit.second,
-            unlimited_ratelimit.hour,
-            unlimited_ratelimit.month,
-            to_timestamp('3000-01-01', 'YYYY-MM-DD') as valid_until,
-            now() as changed_at
-        from unlimited_ratelimit, users
-        where user_group = 'ADMIN'
-        on conflict (user_id) do update set
-            second = excluded.second,
-            hour = excluded.hour,
-            month = excluded.month,
-            valid_until = excluded.valid_until,
-            changed_at = now()
-        where
-            api_ratelimits.second != excluded.second 
-            or api_ratelimits.hour != excluded.hour 
-            or api_ratelimits.month != excluded.month`)
+			current_api_products as (
+				select distinct on (name) name, stripe_price_id, second, hour, month, valid_from 
+				from api_products 
+				where valid_from <= now()
+				order by name, valid_from desc
+			)
+		insert into api_ratelimits (user_id, second, hour, month, valid_until, changed_at)
+		select 
+			user_id,
+			max(second) as second,
+			max(hour) as hour,
+			max(month) as month,
+			to_timestamp('3000-01-01', 'YYYY-MM-DD') as valid_until,
+			now() as changed_at
+		from (
+			-- set all current ratelimits to free
+			select user_id, cap.second, cap.hour, cap.month
+			from api_ratelimits
+			left join current_api_products cap on cap.name = 'free'
+		union
+			-- set ratelimits for stripe subscriptions
+			select u.id as user_id, cap.second, cap.hour, cap.month
+			from users_stripe_subscriptions uss
+			left join users u on u.stripe_customer_id = uss.customer_id
+			left join current_api_products cap on cap.stripe_price_id = uss.price_id
+			where uss.active = true and u.id is not null
+		union
+			-- set ratelimits for app subscriptions
+			select asv.user_id, cap.second, cap.hour, cap.month
+			from app_subs_view asv
+			left join current_api_products cap on cap.name = asv.product_id
+			where asv.active = true
+		union
+			-- set ratelimits for admins to unlimited
+			select u.id as user_id, cap.second, cap.hour, cap.month
+			from users u
+			left join current_api_products cap on cap.name = 'unlimited'
+			where u.user_group = 'ADMIN' and cap.second is not null
+		) a
+		group by user_id
+		on conflict (user_id) do update set
+			second = excluded.second,
+			hour = excluded.hour,
+			month = excluded.month,
+			valid_until = excluded.valid_until,
+			changed_at = now()
+		where
+			api_ratelimits.second != excluded.second 
+			or api_ratelimits.hour != excluded.hour 
+			or api_ratelimits.month != excluded.month`)
 }