Merge #127145

127145: jwtauthccl: Move HTTP client timeout into a cluster setting r=souravcrl a=pritesh-lahoti

Currently, the external calls made during JWT authentication use a default HTTP client timeout. We would like to make this configurable similar to OIDC by exposing this through a cluster setting. This change introduces server.jwt_authentication.client.timeout to capture this intent.
Also, updated the timeout to 15s for OIDC setting for consistency.

Fixes: #126546
Epic: CRDB-39971

Release note (enterprise change, ops change): A new public cluster setting server.jwt_authentication.client.timeout has been introduced to capture the HTTP client timeout for external calls made during JWT authentication.

Co-authored-by: Pritesh Lahoti <[email protected]>

docs/generated/settings/settings-for-tenants.txt

@@ -117,6 +117,7 @@ server.hot_ranges_request.node.timeout	duration	5m0s	the duration allowed for a
 server.hsts.enabled	boolean	false	if true, HSTS headers will be sent along with all HTTP requests. The headers will contain a max-age setting of one year. Browsers honoring the header will always use HTTPS to access the DB Console. Ensure that TLS is correctly configured prior to enabling.	application
 server.http.base_path	string	/	path to redirect the user to upon succcessful login	application
 server.identity_map.configuration	string		system-identity to database-username mappings	application
+server.jwt_authentication.client.timeout	duration	15s	sets the client timeout for external calls made during JWT authentication (e.g. fetching JWKS, etc.)	application
 server.ldap_authentication.client.tls_certificate	string		sets the client certificate PEM for establishing mTLS connection with LDAP server	application
 server.ldap_authentication.client.tls_key	string		sets the client key PEM for establishing mTLS connection with LDAP server	application
 server.ldap_authentication.domain.custom_ca	string		sets the PEM encoded custom root CA for verifying domain certificates when establishing connection with LDAP server	application
@@ -127,7 +128,7 @@ server.max_open_transactions_per_gateway	integer	-1	the maximum number of open S
 server.oidc_authentication.autologin.enabled (alias: server.oidc_authentication.autologin)	boolean	false	if true, logged-out visitors to the DB Console will be automatically redirected to the OIDC login endpoint	application
 server.oidc_authentication.button_text	string	Log in with your OIDC provider	text to show on button on DB Console login page to login with your OIDC provider (only shown if OIDC is enabled)	application
 server.oidc_authentication.claim_json_key	string		sets JSON key of principal to extract from payload after OIDC authentication completes (usually email or sid)	application
-server.oidc_authentication.client.timeout	duration	30s	sets the client timeout for external calls made during OIDC authentication (e.g. authorization code flow, etc.)	application
+server.oidc_authentication.client.timeout	duration	15s	sets the client timeout for external calls made during OIDC authentication (e.g. authorization code flow, etc.)	application
 server.oidc_authentication.client_id	string		sets OIDC client id	application
 server.oidc_authentication.client_secret	string		sets OIDC client secret	application
 server.oidc_authentication.enabled	boolean	false	enables or disabled OIDC login for the DB Console	application

docs/generated/settings/settings.html

@@ -145,6 +145,7 @@
 <tr><td><div id="setting-server-hsts-enabled" class="anchored"><code>server.hsts.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>if true, HSTS headers will be sent along with all HTTP requests. The headers will contain a max-age setting of one year. Browsers honoring the header will always use HTTPS to access the DB Console. Ensure that TLS is correctly configured prior to enabling.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-http-base-path" class="anchored"><code>server.http.base_path</code></div></td><td>string</td><td><code>/</code></td><td>path to redirect the user to upon succcessful login</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-identity-map-configuration" class="anchored"><code>server.identity_map.configuration</code></div></td><td>string</td><td><code></code></td><td>system-identity to database-username mappings</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-server-jwt-authentication-client-timeout" class="anchored"><code>server.jwt_authentication.client.timeout</code></div></td><td>duration</td><td><code>15s</code></td><td>sets the client timeout for external calls made during JWT authentication (e.g. fetching JWKS, etc.)</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-ldap-authentication-client-tls-certificate" class="anchored"><code>server.ldap_authentication.client.tls_certificate</code></div></td><td>string</td><td><code></code></td><td>sets the client certificate PEM for establishing mTLS connection with LDAP server</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-ldap-authentication-client-tls-key" class="anchored"><code>server.ldap_authentication.client.tls_key</code></div></td><td>string</td><td><code></code></td><td>sets the client key PEM for establishing mTLS connection with LDAP server</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-ldap-authentication-domain-custom-ca" class="anchored"><code>server.ldap_authentication.domain.custom_ca</code></div></td><td>string</td><td><code></code></td><td>sets the PEM encoded custom root CA for verifying domain certificates when establishing connection with LDAP server</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
@@ -155,7 +156,7 @@
 <tr><td><div id="setting-server-oidc-authentication-autologin" class="anchored"><code>server.oidc_authentication.autologin.enabled<br />(alias: server.oidc_authentication.autologin)</code></div></td><td>boolean</td><td><code>false</code></td><td>if true, logged-out visitors to the DB Console will be automatically redirected to the OIDC login endpoint</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-button-text" class="anchored"><code>server.oidc_authentication.button_text</code></div></td><td>string</td><td><code>Log in with your OIDC provider</code></td><td>text to show on button on DB Console login page to login with your OIDC provider (only shown if OIDC is enabled)</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-claim-json-key" class="anchored"><code>server.oidc_authentication.claim_json_key</code></div></td><td>string</td><td><code></code></td><td>sets JSON key of principal to extract from payload after OIDC authentication completes (usually email or sid)</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
-<tr><td><div id="setting-server-oidc-authentication-client-timeout" class="anchored"><code>server.oidc_authentication.client.timeout</code></div></td><td>duration</td><td><code>30s</code></td><td>sets the client timeout for external calls made during OIDC authentication (e.g. authorization code flow, etc.)</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-server-oidc-authentication-client-timeout" class="anchored"><code>server.oidc_authentication.client.timeout</code></div></td><td>duration</td><td><code>15s</code></td><td>sets the client timeout for external calls made during OIDC authentication (e.g. authorization code flow, etc.)</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-client-id" class="anchored"><code>server.oidc_authentication.client_id</code></div></td><td>string</td><td><code></code></td><td>sets OIDC client id</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-client-secret" class="anchored"><code>server.oidc_authentication.client_secret</code></div></td><td>string</td><td><code></code></td><td>sets OIDC client secret</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-enabled" class="anchored"><code>server.oidc_authentication.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>enables or disabled OIDC login for the DB Console</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/ccl/jwtauthccl/authentication_jwt.go

@@ -87,6 +87,7 @@ func (authenticator *jwtAuthenticator) reloadConfig(ctx context.Context, st *clu
 func (authenticator *jwtAuthenticator) reloadConfigLocked(
 	ctx context.Context, st *cluster.Settings,
 ) {
+	clientTimeout := JWTAuthClientTimeout.Get(&st.SV)
 	conf := jwtAuthenticatorConf{
 		audience:             mustParseValueOrArray(JWTAuthAudience.Get(&st.SV)),
 		enabled:              JWTAuthEnabled.Get(&st.SV),
@@ -96,8 +97,8 @@ func (authenticator *jwtAuthenticator) reloadConfigLocked(
 		claim:                JWTAuthClaim.Get(&st.SV),
 		jwksAutoFetchEnabled: JWKSAutoFetchEnabled.Get(&st.SV),
 		httpClient: httputil.NewClient(
-			httputil.WithClientTimeout(httputil.StandardHTTPTimeout),
-			httputil.WithDialerTimeout(httputil.StandardHTTPTimeout),
+			httputil.WithClientTimeout(clientTimeout),
+			httputil.WithDialerTimeout(clientTimeout),
 			httputil.WithCustomCAPEM(JWTAuthIssuerCustomCA.Get(&st.SV)),
 		),
 	}

pkg/ccl/jwtauthccl/authentication_jwt_test.go

@@ -17,11 +17,13 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -949,3 +951,94 @@ func TestJWTAuthWithCustomCACert(t *testing.T) {
 		})
 	}
 }
+
+func TestJWTAuthClientTimeout(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	ctx := context.Background()
+
+	// Initiate a test JWKS server locally.
+	testServer := httptest.NewUnstartedServer(nil)
+	waitChan := make(chan struct{}, 1)
+
+	mux := http.NewServeMux()
+	mux.HandleFunc(
+		"GET /jwks",
+		func(w http.ResponseWriter, r *http.Request) {
+			// Hang the request handler to enforce HTTP client timeout.
+			<-waitChan
+		},
+	)
+
+	testServer.Config = &http.Server{
+		Handler: mux,
+	}
+	testServer.Start()
+	defer func() {
+		waitChan <- struct{}{}
+		close(waitChan)
+		testServer.Close()
+	}()
+
+	mockGetHttpResponse := func(ctx context.Context, url string, authenticator *jwtAuthenticator) ([]byte, error) {
+		if strings.Contains(url, "/.well-known/openid-configuration") {
+			return mockGetHttpResponseWithLocalFileContent(ctx, url, authenticator)
+		} else if strings.Contains(url, "/oauth2/v3/certs") {
+			// For fetching JWKS, point to the local test server.
+			resp, err := authenticator.mu.conf.httpClient.Get(
+				context.Background(),
+				testServer.URL+"/jwks",
+			)
+			if err != nil {
+				return nil, err
+			}
+			defer resp.Body.Close()
+
+			body, err := io.ReadAll(resp.Body)
+			if err != nil {
+				return nil, err
+			}
+			return body, nil
+		}
+		return nil, errors.Newf("unsupported route: %s", url)
+	}
+	getHttpResponseTestHook := testutils.TestingHook(&getHttpResponse, mockGetHttpResponse)
+	defer getHttpResponseTestHook()
+
+	s := serverutils.StartServerOnly(t, base.TestServerArgs{})
+	defer s.Stopper().Stop(ctx)
+
+	identMapString := ""
+	identMap, err := identmap.From(strings.NewReader(identMapString))
+	require.NoError(t, err)
+
+	// Create a key to sign the token using testdata.
+	// The same will be fetched through the JWKS URL to verify the token.
+	keySet := createJWKSFromFile(t, "testdata/www.idp1apis.com_oauth2_v3_certs_private")
+	key, _ := keySet.Get(0)
+	validIssuer := "https://accounts.idp1.com"
+	token := createJWT(
+		t, username1, audience1, validIssuer, timeutil.Now().Add(time.Hour), key, jwa.RS256, "", "")
+
+	JWTAuthEnabled.Override(ctx, &s.ClusterSettings().SV, true)
+	JWTAuthIssuers.Override(ctx, &s.ClusterSettings().SV, validIssuer)
+	JWKSAutoFetchEnabled.Override(ctx, &s.ClusterSettings().SV, true)
+	JWTAuthAudience.Override(ctx, &s.ClusterSettings().SV, audience1)
+	JWTAuthClientTimeout.Override(ctx, &s.ClusterSettings().SV, time.Millisecond)
+
+	verifier := ConfigureJWTAuth(ctx, s.AmbientCtx(), s.ClusterSettings(), s.StorageClusterID())
+	errMsg, err := verifier.ValidateJWTLogin(
+		ctx,
+		s.ClusterSettings(),
+		username.MakeSQLUsernameFromPreNormalizedString(username1),
+		token,
+		identMap,
+	)
+	require.Regexp(
+		t,
+		regexp.MustCompile(`unable to fetch jwks:.*\(Client.Timeout exceeded while awaiting headers\)`),
+		errMsg,
+	)
+	require.ErrorContains(t, err, "JWT authentication: unable to validate token")
+}

pkg/ccl/jwtauthccl/settings.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"crypto/x509"
 	"encoding/json"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/errors"
@@ -28,6 +29,7 @@ const (
 	JWTAuthClaimSettingName          = baseJWTAuthSettingName + "claim"
 	JWKSAutoFetchEnabledSettingName  = baseJWTAuthSettingName + "jwks_auto_fetch.enabled"
 	jwtAuthIssuerCustomCASettingName = baseJWTAuthSettingName + "issuers.custom_ca"
+	jwtAuthClientTimeoutSettingName  = baseJWTAuthSettingName + "client.timeout"
 )
 
 // JWTAuthClaim sets the JWT claim that is parsed to get the username.
@@ -98,6 +100,18 @@ var JWKSAutoFetchEnabled = settings.RegisterBoolSetting(
 	settings.WithReportable(true),
 )
 
+// JWTAuthClientTimeout is the client timeout for all the external calls made
+// during JWT authentication (e.g. fetching JWKS, etc.).
+var JWTAuthClientTimeout = settings.RegisterDurationSetting(
+	settings.ApplicationLevel,
+	jwtAuthClientTimeoutSettingName,
+	"sets the client timeout for external calls made during JWT authentication "+
+		"(e.g. fetching JWKS, etc.)",
+	15*time.Second,
+	settings.NonNegativeDuration,
+	settings.WithPublic,
+)
+
 func validateJWTAuthIssuers(values *settings.Values, s string) error {
 	var issuers []string
 

pkg/ccl/oidcccl/settings.go

@@ -81,7 +81,7 @@ var OIDCAuthClientTimeout = settings.RegisterDurationSetting(
 	oidcAuthClientTimeoutSettingName,
 	"sets the client timeout for external calls made during OIDC authentication "+
 		"(e.g. authorization code flow, etc.)",
-	30*time.Second,
+	15*time.Second,
 	settings.NonNegativeDuration,
 	settings.WithPublic,
 )