ccl,sql,util: Fix jwt auth and add sensitive error logs

We are running into issues with jwt authentication and currently unable to
provide support as we are not logging the error from the http client used in the
authenticator. The PR looks to propagate this obtained error from
`ValidateJWTLogin` http client. We are also introducing `JWTClientCustomCA`
cluster setting such that this could be configured directly for the http client
used in authenticator. The http client now also respects the system http proxy
if set.

Validated the error details when presenting an expired token
```
ERROR: JWT authentication: invalid token
SQLSTATE: 28000
DETAIL: unable to parse token: exp not satisfied Failed running "sql"
```

Validated error on setting wrong proxy params
```
ERROR: JWT authentication: unable to validate token
SQLSTATE: 28000
Failed running "sql"
```
and logged error:
```
I240510 08:31:28.604141 1473 4@util/log/event_log.go:32 ⋮
[T1,Vsystem,n1,client=127.0.0.1:56289,hostssl,user=‹sourav.sarangi›] 3
={"Timestamp":1715329888604122000,"EventType":"client_authentication_failed","InstanceID":1,"Network":"tcp","RemoteAddress":"‹127.0.0.1:56289›","SessionID":"17ce136f2a8ecd480000000000000001","Transport":"hostssl","User":"‹sourav.sarangi›","SystemIdentity":"‹sourav.sarangi›","Reason":"CREDENTIALS_INVALID","Detail":"JWT
authentication: unable to validate token\nunable to fetch jwks: Get
\"https://accounts.google.com/.well-known/openid-configuration\": proxyconnect
tcp: dial tcp [::1]:3129: connect: connection refused","Method":"jwt_token"}
```

Verified access logs after setting up squid proxy and passing env HTTP_PROXY and
HTTPS_PROXY params
```
1715103871.761    144 ::1 TCP_TUNNEL/200 5708 CONNECT accounts.google.com:443 -
HIER_DIRECT/74.125.200.84 -
1715103871.836     73 ::1 TCP_TUNNEL/200 5964 CONNECT www.googleapis.com:443 -
HIER_DIRECT/142.250.182.10 -
```

fixes #123575,
Epic CRDB-38386, CRDB-38408

Release note(security update): We are adding a cluster settings
`server.jwt_authentication.client.custom_ca` which can tune the jwt auth
behaviour when we need custom ca http client calls.

docs/generated/settings/settings-for-tenants.txt

@@ -88,6 +88,7 @@ server.hot_ranges_request.node.timeout	duration	5m0s	the duration allowed for a
 server.hsts.enabled	boolean	false	if true, HSTS headers will be sent along with all HTTP requests. The headers will contain a max-age setting of one year. Browsers honoring the header will always use HTTPS to access the DB Console. Ensure that TLS is correctly configured prior to enabling.	application
 server.http.base_path	string	/	path to redirect the user to upon succcessful login	application
 server.identity_map.configuration	string		system-identity to database-username mappings	application
+server.jwt_authentication.client.custom_ca	string		sets the custom root CA for verifying certificates when interacting with jwt internal HTTPS client	application
 server.log_gc.max_deletions_per_cycle	integer	1000	the maximum number of entries to delete on each purge of log-like system tables	application
 server.log_gc.period	duration	1h0m0s	the period at which log-like system tables are checked for old entries	application
 server.max_connections_per_gateway	integer	-1	the maximum number of SQL connections per gateway allowed at a given time (note: this will only limit future connection attempts and will not affect already established connections). Negative values result in unlimited number of connections. Superusers are not affected by this limit.	application

docs/generated/settings/settings.html

@@ -116,6 +116,7 @@
 <tr><td><div id="setting-server-hsts-enabled" class="anchored"><code>server.hsts.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>if true, HSTS headers will be sent along with all HTTP requests. The headers will contain a max-age setting of one year. Browsers honoring the header will always use HTTPS to access the DB Console. Ensure that TLS is correctly configured prior to enabling.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-http-base-path" class="anchored"><code>server.http.base_path</code></div></td><td>string</td><td><code>/</code></td><td>path to redirect the user to upon succcessful login</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-identity-map-configuration" class="anchored"><code>server.identity_map.configuration</code></div></td><td>string</td><td><code></code></td><td>system-identity to database-username mappings</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-server-jwt-authentication-client-custom-ca" class="anchored"><code>server.jwt_authentication.client.custom_ca</code></div></td><td>string</td><td><code></code></td><td>sets the custom root CA for verifying certificates when interacting with jwt internal HTTPS client</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-log-gc-max-deletions-per-cycle" class="anchored"><code>server.log_gc.max_deletions_per_cycle</code></div></td><td>integer</td><td><code>1000</code></td><td>the maximum number of entries to delete on each purge of log-like system tables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-log-gc-period" class="anchored"><code>server.log_gc.period</code></div></td><td>duration</td><td><code>1h0m0s</code></td><td>the period at which log-like system tables are checked for old entries</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-max-connections-per-gateway" class="anchored"><code>server.max_connections_per_gateway</code></div></td><td>integer</td><td><code>-1</code></td><td>the maximum number of SQL connections per gateway allowed at a given time (note: this will only limit future connection attempts and will not affect already established connections). Negative values result in unlimited number of connections. Superusers are not affected by this limit.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/ccl/jwtauthccl/authentication_jwt.go

@@ -71,6 +71,7 @@ type jwtAuthenticatorConf struct {
 	jwks                 jwk.Set
 	claim                string
 	jwksAutoFetchEnabled bool
+	customCA             string
 }
 
 // reloadConfig locks mutex and then refreshes the values in conf from the cluster settings.
@@ -91,6 +92,7 @@ func (authenticator *jwtAuthenticator) reloadConfigLocked(
 		jwks:                 mustParseJWKS(JWTAuthJWKS.Get(&st.SV)),
 		claim:                JWTAuthClaim.Get(&st.SV),
 		jwksAutoFetchEnabled: JWKSAutoFetchEnabled.Get(&st.SV),
+		customCA:             JWTClientCustomCA.Get(&st.SV),
 	}
 
 	if !authenticator.mu.conf.enabled && conf.enabled {
@@ -126,18 +128,23 @@ func (authenticator *jwtAuthenticator) mapUsername(
 // * the audience field matches the audience cluster setting.
 // * the issuer field is one of the values in the issuer cluster setting.
 // * the cluster has an enterprise license.
+// It returns authError (which is the error sql clients will see in case of
+// failures) and detailedError (which is the internal error from http clients
+// that might contain sensitive information we do not want to send to sql
+// clients but still want to log it). We do not want to send any information
+// back to client which was not provided by the client.
 func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	ctx context.Context,
 	st *cluster.Settings,
 	user username.SQLUsername,
 	tokenBytes []byte,
 	identMap *identmap.Conf,
-) error {
+) (detailedErrors string, authError error) {
 	authenticator.mu.Lock()
 	defer authenticator.mu.Unlock()
 
 	if !authenticator.mu.enabled {
-		return errors.Newf("JWT authentication: not enabled")
+		return "", errors.Newf("JWT authentication: not enabled")
 	}
 
 	telemetry.Inc(beginAuthUseCounter)
@@ -146,7 +153,9 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	// The token will be parsed again later to actually verify the signature.
 	unverifiedToken, err := jwt.Parse(tokenBytes)
 	if err != nil {
-		return errors.Newf("JWT authentication: invalid token")
+		return "", errors.WithDetailf(
+			errors.Newf("JWT authentication: invalid token"),
+			"token parsing failed: %v", err)
 	}
 
 	// Check for issuer match against configured issuers.
@@ -160,17 +169,18 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 		}
 	}
 	if !issuerMatch {
-		return errors.WithDetailf(
+		return "", errors.WithDetailf(
 			errors.Newf("JWT authentication: invalid issuer"),
 			"token issued by %s", unverifiedToken.Issuer())
 	}
 
 	var jwkSet jwk.Set
 	// If auto-fetch is enabled, fetch the JWKS remotely from the issuer's well known jwks url.
 	if authenticator.mu.conf.jwksAutoFetchEnabled {
-		jwkSet, err = remoteFetchJWKS(ctx, issuerUrl)
+		jwkSet, err = authenticator.remoteFetchJWKS(ctx, issuerUrl)
 		if err != nil {
-			return errors.Newf("JWT authentication: unable to validate token")
+			return fmt.Sprintf("unable to fetch jwks: %v", err),
+				errors.Newf("JWT authentication: unable to validate token")
 		}
 	} else {
 		jwkSet = authenticator.mu.conf.jwks
@@ -179,7 +189,9 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	// Now that both the issuer and key-id are matched, parse the token again to validate the signature.
 	parsedToken, err := jwt.Parse(tokenBytes, jwt.WithKeySet(jwkSet), jwt.WithValidate(true), jwt.InferAlgorithmFromKey(true))
 	if err != nil {
-		return errors.Newf("JWT authentication: invalid token")
+		return "", errors.WithDetailf(
+			errors.Newf("JWT authentication: invalid token"),
+			"unable to parse token: %v", err)
 	}
 
 	// Extract all requested principals from the token. By default, we take it from the subject unless they specify
@@ -190,7 +202,7 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	} else {
 		claimValue, ok := parsedToken.Get(authenticator.mu.conf.claim)
 		if !ok {
-			return errors.WithDetailf(
+			return "", errors.WithDetailf(
 				errors.Newf("JWT authentication: missing claim"),
 				"token does not contain a claim for %s", authenticator.mu.conf.claim)
 		}
@@ -217,14 +229,14 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	for _, tokenPrincipal := range tokenPrincipals {
 		mappedUsernames, err := authenticator.mapUsername(tokenPrincipal, parsedToken.Issuer(), identMap)
 		if err != nil {
-			return errors.WithDetailf(
+			return "", errors.WithDetailf(
 				errors.Newf("JWT authentication: invalid claim value"),
 				"the value %s for the issuer %s is invalid", tokenPrincipal, parsedToken.Issuer())
 		}
 		acceptedUsernames = append(acceptedUsernames, mappedUsernames...)
 	}
 	if len(acceptedUsernames) == 0 {
-		return errors.WithDetailf(
+		return "", errors.WithDetailf(
 			errors.Newf("JWT authentication: invalid principal"),
 			"the value %s for the issuer %s is invalid", tokenPrincipals, parsedToken.Issuer())
 	}
@@ -236,12 +248,12 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 		}
 	}
 	if !principalMatch {
-		return errors.WithDetailf(
+		return "", errors.WithDetailf(
 			errors.Newf("JWT authentication: invalid principal"),
 			"token issued for %s and login was for %s", tokenPrincipals, user.Normalized())
 	}
 	if user.IsRootUser() || user.IsReserved() {
-		return errors.WithDetailf(
+		return "", errors.WithDetailf(
 			errors.Newf("JWT authentication: invalid identity"),
 			"cannot use JWT auth to login to a reserved user %s", user.Normalized())
 	}
@@ -255,26 +267,29 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 		}
 	}
 	if !audienceMatch {
-		return errors.WithDetailf(
+		return "", errors.WithDetailf(
 			errors.Newf("JWT authentication: invalid audience"),
 			"token issued with an audience of %s", parsedToken.Audience())
 	}
 
 	if err = utilccl.CheckEnterpriseEnabled(st, "JWT authentication"); err != nil {
-		return err
+		return "", err
 	}
 
 	telemetry.Inc(loginSuccessUseCounter)
-	return nil
+	return "", nil
 }
 
 // remoteFetchJWKS fetches the JWKS from the provided URI.
-func remoteFetchJWKS(ctx context.Context, issuerUrl string) (jwk.Set, error) {
-	jwksUrl, err := getJWKSUrl(ctx, issuerUrl)
+func (authenticator *jwtAuthenticator) remoteFetchJWKS(
+	ctx context.Context, issuerUrl string,
+) (jwk.Set, error) {
+	jwksUrl, err := authenticator.getJWKSUrl(ctx, issuerUrl)
 	if err != nil {
 		return nil, err
 	}
-	body, err := getHttpResponse(ctx, jwksUrl)
+
+	body, err := getHttpResponse(ctx, jwksUrl, authenticator)
 	if err != nil {
 		return nil, err
 	}
@@ -286,12 +301,14 @@ func remoteFetchJWKS(ctx context.Context, issuerUrl string) (jwk.Set, error) {
 }
 
 // getJWKSUrl returns the JWKS URI from the OpenID configuration endpoint.
-func getJWKSUrl(ctx context.Context, issuerUrl string) (string, error) {
+func (authenticator *jwtAuthenticator) getJWKSUrl(
+	ctx context.Context, issuerUrl string,
+) (string, error) {
 	type OIDCConfigResponse struct {
 		JWKSUri string `json:"jwks_uri"`
 	}
 	openIdConfigEndpoint := getOpenIdConfigEndpoint(issuerUrl)
-	body, err := getHttpResponse(ctx, openIdConfigEndpoint)
+	body, err := getHttpResponse(ctx, openIdConfigEndpoint, authenticator)
 	if err != nil {
 		return "", err
 	}
@@ -311,8 +328,14 @@ func getOpenIdConfigEndpoint(issuerUrl string) string {
 	return openIdConfigEndpoint
 }
 
-var getHttpResponse = func(ctx context.Context, url string) ([]byte, error) {
-	resp, err := httputil.Get(ctx, url)
+var getHttpResponse = func(ctx context.Context, url string, authenticator *jwtAuthenticator) ([]byte, error) {
+	defaultTimeout := httputil.StandardHTTPTimeout
+	// TODO(souravcrl): cache the http client in a callback attached to customCA
+	// and other http client cluster settings as re parsing the custom CA every
+	// time is expensive
+	httpClient := httputil.NewClientWithTimeoutsCustomCA(
+		defaultTimeout, defaultTimeout, authenticator.mu.conf.customCA)
+	resp, err := httpClient.Get(context.Background(), url)
 	if err != nil {
 		return nil, err
 	}
@@ -354,6 +377,9 @@ var ConfigureJWTAuth = func(
 	JWKSAutoFetchEnabled.SetOnChange(&st.SV, func(ctx context.Context) {
 		authenticator.reloadConfig(ambientCtx.AnnotateCtx(ctx), st)
 	})
+	JWTClientCustomCA.SetOnChange(&st.SV, func(ctx context.Context) {
+		authenticator.reloadConfig(ambientCtx.AnnotateCtx(ctx), st)
+	})
 	return &authenticator
 }