Merge #88766

88766: server: fix bottlenecks when purging system.web_sessions r=dhartunian a=aadityasondhi

The previous purging limit was found to be too low which resulted in the
`system.web_sessions` table growing rapidly. This happened when a
customer-side automation script was generating a large number of
sessions. Additionally, we were using `ORDER BY random()` when deleting
expired sessions from the table which caused a full table scan of a large
table. This resulted in high cpu usage on every purge attempt with only very
few rows actually being deleted. Over time, this turns into a nonideal
feedback loop.

This change fixes the two issues by:
1. Increasing the defauilt purging limit to 1000 from 10.
2. Remove the `ORDER BY random()` when purging the table to avoid full
   table scans on every purge.

support ticket: https://github.com/cockroachlabs/support/issues/1812

resolves #88622

Release note: None

Co-authored-by: Aaditya Sondhi <[email protected]>

docs/generated/settings/settings-for-tenants.txt

@@ -87,7 +87,7 @@ server.user_login.password_hashes.default_cost.scram_sha_256	integer	119680	the
 server.user_login.timeout	duration	10s	timeout after which client authentication times out if some system range is unavailable (0 = no timeout)
 server.user_login.upgrade_bcrypt_stored_passwords_to_scram.enabled	boolean	true	whether to automatically re-encode stored passwords using crdb-bcrypt to scram-sha-256
 server.web_session.auto_logout.timeout	duration	168h0m0s	the duration that web sessions will survive before being periodically purged, since they were last used
-server.web_session.purge.max_deletions_per_cycle	integer	10	the maximum number of old sessions to delete for each purge
+server.web_session.purge.max_deletions_per_cycle	integer	1000	the maximum number of old sessions to delete for each purge
 server.web_session.purge.period	duration	1h0m0s	the time until old sessions are deleted
 server.web_session.purge.ttl	duration	1h0m0s	if nonzero, entries in system.web_sessions older than this duration are periodically purged
 server.web_session_timeout	duration	168h0m0s	the duration that a newly created web session will be valid

docs/generated/settings/settings.html

@@ -106,7 +106,7 @@
 <tr><td><code>server.user_login.timeout</code></td><td>duration</td><td><code>10s</code></td><td>timeout after which client authentication times out if some system range is unavailable (0 = no timeout)</td></tr>
 <tr><td><code>server.user_login.upgrade_bcrypt_stored_passwords_to_scram.enabled</code></td><td>boolean</td><td><code>true</code></td><td>whether to automatically re-encode stored passwords using crdb-bcrypt to scram-sha-256</td></tr>
 <tr><td><code>server.web_session.auto_logout.timeout</code></td><td>duration</td><td><code>168h0m0s</code></td><td>the duration that web sessions will survive before being periodically purged, since they were last used</td></tr>
-<tr><td><code>server.web_session.purge.max_deletions_per_cycle</code></td><td>integer</td><td><code>10</code></td><td>the maximum number of old sessions to delete for each purge</td></tr>
+<tr><td><code>server.web_session.purge.max_deletions_per_cycle</code></td><td>integer</td><td><code>1000</code></td><td>the maximum number of old sessions to delete for each purge</td></tr>
 <tr><td><code>server.web_session.purge.period</code></td><td>duration</td><td><code>1h0m0s</code></td><td>the time until old sessions are deleted</td></tr>
 <tr><td><code>server.web_session.purge.ttl</code></td><td>duration</td><td><code>1h0m0s</code></td><td>if nonzero, entries in system.web_sessions older than this duration are periodically purged</td></tr>
 <tr><td><code>server.web_session_timeout</code></td><td>duration</td><td><code>168h0m0s</code></td><td>the duration that a newly created web session will be valid</td></tr>

pkg/server/purge_auth_session.go

@@ -50,7 +50,7 @@ var (
 		settings.TenantWritable,
 		"server.web_session.purge.max_deletions_per_cycle",
 		"the maximum number of old sessions to delete for each purge",
-		10,
+		1000,
 	).WithPublic()
 )
 
@@ -90,21 +90,21 @@ func (s *authenticationServer) purgeOldSessions(ctx context.Context) {
 		deleteOldExpiredSessionsStmt = `
 DELETE FROM system.web_sessions
 WHERE "expiresAt" < $1
-ORDER BY random()
+ORDER BY "expiresAt" ASC
 LIMIT $2
 RETURNING 1
 `
 		deleteOldRevokedSessionsStmt = `
 DELETE FROM system.web_sessions
 WHERE "revokedAt" < $1
-ORDER BY random()
+ORDER BY "expiresAt" ASC
 LIMIT $2
 RETURNING 1
 `
 		deleteSessionsAutoLogoutStmt = `
 DELETE FROM system.web_sessions
 WHERE "lastUsedAt" < $1
-ORDER BY random()
+ORDER BY "expiresAt" ASC
 LIMIT $2
 RETURNING 1
 `