Bids 2872/ratelimits

cmd/explorer/main.go

@@ -11,6 +11,7 @@ import (
 	"eth2-exporter/handlers"
 	"eth2-exporter/metrics"
 	"eth2-exporter/price"
+	"eth2-exporter/ratelimit"
 	"eth2-exporter/rpc"
 	"eth2-exporter/services"
 	"eth2-exporter/static"
@@ -606,6 +607,9 @@ func main() {
 			router.Use(metrics.HttpMiddleware)
 		}
 
+		ratelimit.Init()
+		router.Use(ratelimit.HttpMiddleware)
+
 		n := negroni.New(negroni.NewRecovery())
 		n.Use(gzip.Gzip(gzip.DefaultCompression))
 

cmd/misc/main.go

@@ -10,6 +10,7 @@ import (
 	"eth2-exporter/cmd/misc/commands"
 	"eth2-exporter/db"
 	"eth2-exporter/exporter"
+	"eth2-exporter/ratelimit"
 	"eth2-exporter/rpc"
 	"eth2-exporter/services"
 	"eth2-exporter/types"
@@ -66,7 +67,7 @@ func main() {
 	statsPartitionCommand := commands.StatsMigratorCommand{}
 
 	configPath := flag.String("config", "config/default.config.yml", "Path to the config file")
-	flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases")
+	flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases, update-ratelimits")
 	flag.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch")
 	flag.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch")
 	flag.Uint64Var(&opts.User, "user", 0, "user id")
@@ -391,6 +392,8 @@ func main() {
 		err = fixEns(erigonClient)
 	case "fix-ens-addresses":
 		err = fixEnsAddresses(erigonClient)
+	case "update-ratelimits":
+		ratelimit.DBUpdater()
 	default:
 		utils.LogFatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0)
 	}

db/migrations/20240125110000_add_apps_subs_view.sql

@@ -0,0 +1,23 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query - add view app_subs_view';
+CREATE OR REPLACE VIEW app_subs_view AS
+    SELECT users_app_subscriptions.id,
+        users_app_subscriptions.user_id,
+        users_app_subscriptions.product_id,
+        users_app_subscriptions.created_at,
+        users_app_subscriptions.updated_at,
+        users_app_subscriptions.validate_remotely,
+        users_app_subscriptions.active,
+        users_app_subscriptions.store,
+        users_app_subscriptions.expires_at,
+        users_app_subscriptions.reject_reason,
+        users_app_subscriptions.receipt_hash
+    FROM users_app_subscriptions;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query - drop view app_subs_view';
+DROP VIEW app_subs_view;
+-- +goose StatementEnd

db/migrations/20240125120000_add_ratelimits.sql

@@ -0,0 +1,82 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query - add table api_ratelimits';
+CREATE TABLE IF NOT EXISTS
+    api_ratelimits (
+        user_id INT NOT NULL,
+        second INT NOT NULL DEFAULT 0,
+        hour INT NOT NULL DEFAULT 0,
+        month INT NOT NULL DEFAULT 0,
+        valid_until TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+        changed_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, 
+        PRIMARY KEY (user_id)
+    );
+
+CREATE INDEX IF NOT EXISTS idx_api_ratelimits_changed_at_valid_until ON api_ratelimits (changed_at, valid_until);
+
+SELECT 'up SQL query - add table api_keys';
+CREATE TABLE IF NOT EXISTS
+    api_keys (
+        api_key VARCHAR(256) NOT NULL UNIQUE,
+        user_id INT NOT NULL,
+        valid_until TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT '9999-12-31 23:59:59',
+        changed_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (api_key)
+    );
+
+CREATE INDEX IF NOT EXISTS idx_api_keys_changed_at_valid_until ON api_keys (changed_at, valid_until);
+
+SELECT 'up SQL query - add table api_weights';
+CREATE TABLE IF NOT EXISTS
+    api_weights (
+        bucket VARCHAR(20) NOT NULL,
+        endpoint TEXT NOT NULL,
+        method TEXT NOT NULL,
+        params TEXT NOT NULL,
+        weight INT NOT NULL DEFAULT 1,
+        valid_from TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT TO_TIMESTAMP(0),
+        PRIMARY KEY (endpoint, valid_from)
+    );
+
+SELECT 'up SQL query - add table api_products';
+CREATE TABLE IF NOT EXISTS
+    api_products (
+        name VARCHAR(20) NOT NULL,
+        stripe_price_id VARCHAR(256) NOT NULL DEFAULT '',
+        second INT NOT NULL DEFAULT 0,
+        hour INT NOT NULL DEFAULT 0,
+        month INT NOT NULL DEFAULT 0,
+        valid_from TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT TO_TIMESTAMP(0),
+        PRIMARY KEY (name, valid_from)
+    );
+INSERT INTO api_products (name, second, hour, month) VALUES
+    ('nokey', 2, 1000, 0),
+    ('free', 10, 0, 0),
+    ('unlimited', 100, 0, 0)
+ON CONFLICT DO NOTHING;
+
+ALTER TABLE api_statistics ADD COLUMN IF NOT EXISTS endpoint TEXT NOT NULL DEFAULT '';
+ALTER TABLE api_statistics DROP CONSTRAINT IF EXISTS api_statistics_pkey;
+ALTER TABLE api_statistics ADD PRIMARY KEY (ts, apikey, endpoint);
+ALTER TABLE api_statistics ALTER COLUMN call SET DEFAULT '';
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query - drop table api_ratelimits';
+DROP TABLE IF EXISTS api_ratelimits;
+SELECT 'down SQL query - drop index idx_api_ratelimits_changed_at';
+DROP INDEX IF EXISTS idx_api_ratelimits_changed_at;
+SELECT 'down SQL query - drop table api_keys';
+DROP TABLE IF EXISTS api_keys;
+SELECT 'down SQL query - drop index idx_api_keys_changed_at';
+DROP INDEX IF EXISTS idx_api_keys_changed_at;
+SELECT 'down SQL query - drop table api_weights';
+DROP TABLE IF EXISTS api_weights;
+SELECT 'down SQL query - drop table api_products';
+DROP TABLE IF EXISTS api_products;
+SELECT 'down SQL query - drop column api_statistics.endpoint';
+ALTER TABLE api_statistics DROP COLUMN IF EXISTS endpoint;
+ALTER TABLE api_statistics DROP CONSTRAINT IF EXISTS api_statistics_pkey;
+ALTER TABLE api_statistics ADD PRIMARY KEY (ts, apikey, call);
+-- +goose StatementEnd

handlers/user.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"eth2-exporter/db"
 	"eth2-exporter/mail"
+	"eth2-exporter/ratelimit"
 	"eth2-exporter/services"
 	"eth2-exporter/templates"
 	"eth2-exporter/types"
@@ -87,19 +88,18 @@ func UserSettings(w http.ResponseWriter, r *http.Request) {
 		statsSharing = false
 	}
 
-	maxDaily := utils.Config.Frontend.Ratelimits.FreeDay
-	maxMonthly := utils.Config.Frontend.Ratelimits.FreeMonth
-	if subscription.PriceID != nil {
-		if *subscription.PriceID == utils.Config.Frontend.Stripe.Sapphire {
-			maxDaily = utils.Config.Frontend.Ratelimits.SapphierDay
-			maxMonthly = utils.Config.Frontend.Ratelimits.SapphierMonth
-		} else if *subscription.PriceID == utils.Config.Frontend.Stripe.Emerald {
-			maxDaily = utils.Config.Frontend.Ratelimits.EmeraldDay
-			maxMonthly = utils.Config.Frontend.Ratelimits.EmeraldMonth
-		} else if *subscription.PriceID == utils.Config.Frontend.Stripe.Diamond {
-			maxDaily = utils.Config.Frontend.Ratelimits.DiamondDay
-			maxMonthly = utils.Config.Frontend.Ratelimits.DiamondMonth
-		}
+	rl, err := ratelimit.DBGetUserApiRateLimit(int64(user.UserID))
+	if err != nil {
+		logger.Errorf("Error retrieving the api-ratelimit for user: %v %v", user.UserID, err)
+		utils.SetFlash(w, r, "", "Error: Something went wrong.")
+		http.Redirect(w, r, "/user/settings", http.StatusSeeOther)
+		return
+	}
+
+	maxDaily := int(rl.Second * 24 * 3600)
+	maxMonthly := int(rl.Month)
+	if maxDaily > maxMonthly {
+		maxDaily = maxMonthly
 	}
 
 	userSettingsData.ApiStatistics = &types.ApiStatistics{}

local-deployment/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     <<: *default-service
     profiles:
       - build-once
-    command: /bin/bash -c "git config --global --add safe.directory '*' && make -B all"
+    command: /bin/bash -c "git config --global --add safe.directory '*' && make -j -B all"
   indexer:
     <<: *default-service
     command: go run ./cmd/explorer -config /app/local-deployment/config.yml
@@ -35,3 +35,18 @@ services:
     command: go run ./cmd/explorer -config /app/local-deployment/config.yml
     environment:
       - FRONTEND_ENABLED=true
+  ratelimits-updater:
+    <<: *default-service
+    command: go run ./cmd/misc -config /app/local-deployment/config.yml -command=update-ratelimits
+  misc:
+    <<: *default-service
+    command: /bin/bash -c "while true; do date; sleep 1; done"
+  redis-sessions:
+    image: redis:7
+    volumes:
+      - redis-sessions:/data
+    ports:
+      - "$REDIS_SESSIONS_PORT:6379"
+
+volumes:
+  redis-sessions:

local-deployment/network-params.json

@@ -26,7 +26,7 @@
       "electra_fork_epoch": 999999999
     },
     "global_client_log_level": "info",
-    "start_tx_spammer": true,
+    "start_tx_spammer": false,
     "start_blob_spammer": false
   }
 

local-deployment/provision-explorer-config.sh

@@ -1,4 +1,5 @@
 #! /bin/bash
+
 CL_PORT=$(kurtosis enclave inspect my-testnet | grep 4000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/http\:\/\/127.0.0.1\://' | head -n 1)
 echo "CL Node port is $CL_PORT"
 
@@ -8,6 +9,9 @@ echo "EL Node port is $EL_PORT"
 REDIS_PORT=$(kurtosis enclave inspect my-testnet | grep 6379/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | head -n 1)
 echo "Redis port is $REDIS_PORT"
 
+REDIS_SESSIONS_PORT=$(comm -23 <(seq 49152 65535 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 1)
+echo "Redis sessions port is $REDIS_SESSIONS_PORT"
+
 POSTGRES_PORT=$(kurtosis enclave inspect my-testnet | grep 5432/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/postgresql\:\/\/127.0.0.1\://' | head -n 1)
 echo "Postgres port is $POSTGRES_PORT"
 
@@ -18,6 +22,7 @@ cat <<EOF > .env
 CL_PORT=$CL_PORT
 EL_PORT=$EL_PORT
 REDIS_PORT=$REDIS_PORT
+REDIS_SESSIONS_PORT=$REDIS_SESSIONS_PORT
 POSTGRES_PORT=$POSTGRES_PORT
 LBT_PORT=$LBT_PORT
 EOF
@@ -56,6 +61,7 @@ bigtable:
 eth1ErigonEndpoint: 'http://127.0.0.1:$EL_PORT'
 eth1GethEndpoint: 'http://127.0.0.1:$EL_PORT'
 redisCacheEndpoint: '127.0.0.1:$REDIS_PORT'
+redisSessionStoreEndpoint: '127.0.0.1:$REDIS_SESSIONS_PORT'
 tieredCacheProvider: 'redis'
 frontend:
   siteDomain: "localhost:8080"
@@ -91,6 +97,11 @@ frontend:
     termsOfServiceUrl: "tos.pdf"
     privacyPolicyUrl: "privacy.pdf"
     imprintTemplate: '{{ define "js" }}{{ end }}{{ define "css" }}{{ end }}{{ define "content" }}Imprint{{ end }}'
+  stripe:
+    sapphire: price_sapphire
+    emerald: price_emerald
+    diamond: price_diamond
+  ratelimitUpdateInterval: 1s
 
 indexer:
   # fullIndexOnStartup: false # Perform a one time full db index on startup

local-deployment/run.sh

@@ -22,27 +22,48 @@ fn_main() {
             start) shift; fn_start "$@"; exit;;
             stop) shift; fn_stop "$@"; exit;;
             sql) shift; fn_sql "$@"; exit;;
+            redis) shift; fn_redis "$@"; exit;;
+            misc) shift; fn_misc "$@"; exit;;
             *) echo "$var_help"
         esac
         shift
     done
 }
 
+fn_misc() {
+    docker compose exec misc go run ./cmd/misc -config /app/local-deployment/config.yml $@
+}
+
 fn_sql() {
-    PGPASSWORD=pass psql -h localhost -p$POSTGRES_PORT -U postgres -d db
+    if [ -z "${1}" ]; then
+        PGPASSWORD=pass psql -h localhost -p$POSTGRES_PORT -U postgres -d db
+    else
+        PGPASSWORD=pass psql -h localhost -p$POSTGRES_PORT -U postgres -d db -c "$@" --csv --pset=pager=off
+    fi
+}
+
+fn_redis() {
+    if [ -z "${1}" ]; then
+        docker compose exec redis-sessions redis-cli
+    else
+        docker compose exec redis-sessions redis-cli "$@"
+    fi
+    #redis-cli -h localhost -p $REDIS_PORT
 }
 
 fn_start() {
     fn_stop
-    kurtosis run --enclave my-testnet . "$(cat network-params.json)"
+    # build once before starting all services to prevent multiple parallel builds
+    docker compose --profile=build-once run build-once &
+    kurtosis run --enclave my-testnet . "$(cat network-params.json)" &
+    wait
     bash provision-explorer-config.sh
-    docker compose --profile=build-once run build-once # build once before starting all services to prevent multiple parallel builds
     docker compose up -d
     echo "Waiting for explorer to start, then browse http://localhost:8080"
 }
 
 fn_stop() {
-    docker compose down
+    docker compose down -v
     kurtosis clean -a
 }