ccl,sql,util: Fix jwt auth and add sensitive error logs

We are running into issues with jwt authentication and currently unable to
provide support as we are not logging the error from the http client used in the
authenticator. Thus, the PR looks to propagate the obtained error from
`ValidateJWTLogin`. We are also introducing `JWTClientTimeout` and
`JWTClientCustomCA` cluster settings such that these could be configured
directly for the http client used in authenticator. The http client now also
respects the system http proxy if set.

fixes cockroachdb#123575,
Epic CRDB-38386,

Release note(security update): We are adding 2 cluster settings
`server.jwt_authentication.client.timeout` and
`server.jwt_authentication.client.custom_ca` which can control the jwt auth
behaviour for http client calls.

docs/generated/settings/settings-for-tenants.txt

@@ -88,6 +88,8 @@ server.hot_ranges_request.node.timeout	duration	5m0s	the duration allowed for a
 server.hsts.enabled	boolean	false	if true, HSTS headers will be sent along with all HTTP requests. The headers will contain a max-age setting of one year. Browsers honoring the header will always use HTTPS to access the DB Console. Ensure that TLS is correctly configured prior to enabling.	application
 server.http.base_path	string	/	path to redirect the user to upon succcessful login	application
 server.identity_map.configuration	string		system-identity to database-username mappings	application
+server.jwt_authentication.client.custom_ca	string		sets the custom root CA (appended to system's default CAs) for verifying certificates when interacting with jwt internal HTTPS client	application
+server.jwt_authentication.client.timeout	duration	3s	sets the timeout for jwt http client operations	application
 server.log_gc.max_deletions_per_cycle	integer	1000	the maximum number of entries to delete on each purge of log-like system tables	application
 server.log_gc.period	duration	1h0m0s	the period at which log-like system tables are checked for old entries	application
 server.max_connections_per_gateway	integer	-1	the maximum number of SQL connections per gateway allowed at a given time (note: this will only limit future connection attempts and will not affect already established connections). Negative values result in unlimited number of connections. Superusers are not affected by this limit.	application

docs/generated/settings/settings.html

@@ -116,6 +116,8 @@
 <tr><td><div id="setting-server-hsts-enabled" class="anchored"><code>server.hsts.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>if true, HSTS headers will be sent along with all HTTP requests. The headers will contain a max-age setting of one year. Browsers honoring the header will always use HTTPS to access the DB Console. Ensure that TLS is correctly configured prior to enabling.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-http-base-path" class="anchored"><code>server.http.base_path</code></div></td><td>string</td><td><code>/</code></td><td>path to redirect the user to upon succcessful login</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-identity-map-configuration" class="anchored"><code>server.identity_map.configuration</code></div></td><td>string</td><td><code></code></td><td>system-identity to database-username mappings</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-server-jwt-authentication-client-custom-ca" class="anchored"><code>server.jwt_authentication.client.custom_ca</code></div></td><td>string</td><td><code></code></td><td>sets the custom root CA (appended to system&#39;s default CAs) for verifying certificates when interacting with jwt internal HTTPS client</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-server-jwt-authentication-client-timeout" class="anchored"><code>server.jwt_authentication.client.timeout</code></div></td><td>duration</td><td><code>3s</code></td><td>sets the timeout for jwt http client operations</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-log-gc-max-deletions-per-cycle" class="anchored"><code>server.log_gc.max_deletions_per_cycle</code></div></td><td>integer</td><td><code>1000</code></td><td>the maximum number of entries to delete on each purge of log-like system tables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-log-gc-period" class="anchored"><code>server.log_gc.period</code></div></td><td>duration</td><td><code>1h0m0s</code></td><td>the period at which log-like system tables are checked for old entries</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-max-connections-per-gateway" class="anchored"><code>server.max_connections_per_gateway</code></div></td><td>integer</td><td><code>-1</code></td><td>the maximum number of SQL connections per gateway allowed at a given time (note: this will only limit future connection attempts and will not affect already established connections). Negative values result in unlimited number of connections. Superusers are not affected by this limit.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/ccl/jwtauthccl/authentication_jwt.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/ccl/utilccl"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
@@ -71,6 +72,8 @@ type jwtAuthenticatorConf struct {
 	jwks                 jwk.Set
 	claim                string
 	jwksAutoFetchEnabled bool
+	clientTimeout        time.Duration
+	customCA             string
 }
 
 // reloadConfig locks mutex and then refreshes the values in conf from the cluster settings.
@@ -91,6 +94,8 @@ func (authenticator *jwtAuthenticator) reloadConfigLocked(
 		jwks:                 mustParseJWKS(JWTAuthJWKS.Get(&st.SV)),
 		claim:                JWTAuthClaim.Get(&st.SV),
 		jwksAutoFetchEnabled: JWKSAutoFetchEnabled.Get(&st.SV),
+		clientTimeout:        JWTClientTimeout.Get(&st.SV),
+		customCA:             JWTClientCustomCA.Get(&st.SV),
 	}
 
 	if !authenticator.mu.conf.enabled && conf.enabled {
@@ -146,7 +151,8 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	// The token will be parsed again later to actually verify the signature.
 	unverifiedToken, err := jwt.Parse(tokenBytes)
 	if err != nil {
-		return errors.Newf("JWT authentication: invalid token")
+		return errors.WithDetailf(
+			errors.Newf("JWT authentication: invalid token"), "token parsing failed: %v", err)
 	}
 
 	// Check for issuer match against configured issuers.
@@ -168,9 +174,11 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	var jwkSet jwk.Set
 	// If auto-fetch is enabled, fetch the JWKS remotely from the issuer's well known jwks url.
 	if authenticator.mu.conf.jwksAutoFetchEnabled {
-		jwkSet, err = remoteFetchJWKS(ctx, issuerUrl)
+		jwkSet, err = authenticator.remoteFetchJWKS(ctx, issuerUrl)
 		if err != nil {
-			return errors.Newf("JWT authentication: unable to validate token")
+			return errors.WithDetailf(
+				errors.Newf("JWT authentication: unable to validate token"),
+				"unable to fetch jwks: %v", err)
 		}
 	} else {
 		jwkSet = authenticator.mu.conf.jwks
@@ -179,7 +187,7 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 	// Now that both the issuer and key-id are matched, parse the token again to validate the signature.
 	parsedToken, err := jwt.Parse(tokenBytes, jwt.WithKeySet(jwkSet), jwt.WithValidate(true), jwt.InferAlgorithmFromKey(true))
 	if err != nil {
-		return errors.Newf("JWT authentication: invalid token")
+		return errors.WithDetailf(errors.Newf("JWT authentication: invalid token"), "unable to parse token: %v", err)
 	}
 
 	// Extract all requested principals from the token. By default, we take it from the subject unless they specify
@@ -269,12 +277,14 @@ func (authenticator *jwtAuthenticator) ValidateJWTLogin(
 }
 
 // remoteFetchJWKS fetches the JWKS from the provided URI.
-func remoteFetchJWKS(ctx context.Context, issuerUrl string) (jwk.Set, error) {
-	jwksUrl, err := getJWKSUrl(ctx, issuerUrl)
+func (authenticator *jwtAuthenticator) remoteFetchJWKS(
+	ctx context.Context, issuerUrl string,
+) (jwk.Set, error) {
+	jwksUrl, err := authenticator.getJWKSUrl(ctx, issuerUrl)
 	if err != nil {
 		return nil, err
 	}
-	body, err := getHttpResponse(ctx, jwksUrl)
+	body, err := getHttpResponse(ctx, jwksUrl, authenticator)
 	if err != nil {
 		return nil, err
 	}
@@ -286,12 +296,14 @@ func remoteFetchJWKS(ctx context.Context, issuerUrl string) (jwk.Set, error) {
 }
 
 // getJWKSUrl returns the JWKS URI from the OpenID configuration endpoint.
-func getJWKSUrl(ctx context.Context, issuerUrl string) (string, error) {
+func (authenticator *jwtAuthenticator) getJWKSUrl(
+	ctx context.Context, issuerUrl string,
+) (string, error) {
 	type OIDCConfigResponse struct {
 		JWKSUri string `json:"jwks_uri"`
 	}
 	openIdConfigEndpoint := getOpenIdConfigEndpoint(issuerUrl)
-	body, err := getHttpResponse(ctx, openIdConfigEndpoint)
+	body, err := getHttpResponse(ctx, openIdConfigEndpoint, authenticator)
 	if err != nil {
 		return "", err
 	}
@@ -311,8 +323,11 @@ func getOpenIdConfigEndpoint(issuerUrl string) string {
 	return openIdConfigEndpoint
 }
 
-var getHttpResponse = func(ctx context.Context, url string) ([]byte, error) {
-	resp, err := httputil.Get(ctx, url)
+var getHttpResponse = func(ctx context.Context, url string, authenticator *jwtAuthenticator) ([]byte, error) {
+	responseTimeout := authenticator.mu.conf.clientTimeout
+	httpClient := httputil.NewClientWithTimeoutsCustomCA(
+		responseTimeout, responseTimeout, authenticator.mu.conf.customCA)
+	resp, err := httpClient.Get(context.Background(), url)
 	if err != nil {
 		return nil, err
 	}
@@ -354,6 +369,12 @@ var ConfigureJWTAuth = func(
 	JWKSAutoFetchEnabled.SetOnChange(&st.SV, func(ctx context.Context) {
 		authenticator.reloadConfig(ambientCtx.AnnotateCtx(ctx), st)
 	})
+	JWTClientTimeout.SetOnChange(&st.SV, func(ctx context.Context) {
+		authenticator.reloadConfig(ambientCtx.AnnotateCtx(ctx), st)
+	})
+	JWTClientCustomCA.SetOnChange(&st.SV, func(ctx context.Context) {
+		authenticator.reloadConfig(ambientCtx.AnnotateCtx(ctx), st)
+	})
 	return &authenticator
 }
 

pkg/ccl/jwtauthccl/authentication_jwt_test.go

@@ -618,7 +618,7 @@ func TestAudienceCheck(t *testing.T) {
 
 // mockGetHttpResponseWithLocalFileContent is a mock function for getHttpResponse. This is used to intercept the call to
 // getHttpResponse and return the content of a local file instead of making a http call.
-var mockGetHttpResponseWithLocalFileContent = func(ctx context.Context, url string) ([]byte, error) {
+var mockGetHttpResponseWithLocalFileContent = func(ctx context.Context, url string, authenticator *jwtAuthenticator) ([]byte, error) {
 	// remove https:// and replace / with _ in the url to get the testdata file name
 	fileName := "testdata/" + strings.ReplaceAll(strings.ReplaceAll(url, "https://", ""), "/", "_")
 	// read content of the file as a byte array

pkg/ccl/jwtauthccl/settings.go

@@ -13,19 +13,22 @@ import (
 	"encoding/json"
 
 	"github.com/cockroachdb/cockroach/pkg/settings"
+	"github.com/cockroachdb/cockroach/pkg/util/httputil"
 	"github.com/cockroachdb/errors"
 	"github.com/lestrrat-go/jwx/jwk"
 )
 
 // All cluster settings necessary for the JWT authentication feature.
 const (
-	baseJWTAuthSettingName          = "server.jwt_authentication."
-	JWTAuthAudienceSettingName      = baseJWTAuthSettingName + "audience"
-	JWTAuthEnabledSettingName       = baseJWTAuthSettingName + "enabled"
-	JWTAuthIssuersSettingName       = baseJWTAuthSettingName + "issuers"
-	JWTAuthJWKSSettingName          = baseJWTAuthSettingName + "jwks"
-	JWTAuthClaimSettingName         = baseJWTAuthSettingName + "claim"
-	JWKSAutoFetchEnabledSettingName = baseJWTAuthSettingName + "jwks_auto_fetch.enabled"
+	baseJWTAuthSettingName           = "server.jwt_authentication."
+	JWTAuthAudienceSettingName       = baseJWTAuthSettingName + "audience"
+	JWTAuthEnabledSettingName        = baseJWTAuthSettingName + "enabled"
+	JWTAuthIssuersSettingName        = baseJWTAuthSettingName + "issuers"
+	JWTAuthJWKSSettingName           = baseJWTAuthSettingName + "jwks"
+	JWTAuthClaimSettingName          = baseJWTAuthSettingName + "claim"
+	JWKSAutoFetchEnabledSettingName  = baseJWTAuthSettingName + "jwks_auto_fetch.enabled"
+	JWTAuthClientTimeoutSettingName  = baseJWTAuthSettingName + "client.timeout"
+	JWTAuthClientCustomCASettingName = baseJWTAuthSettingName + "client.custom_ca"
 )
 
 // JWTAuthClaim sets the JWT claim that is parsed to get the username.
@@ -84,6 +87,22 @@ var JWKSAutoFetchEnabled = settings.RegisterBoolSetting(
 	settings.WithReportable(true),
 )
 
+// JWTClientTimeout is a cluster setting used for setting jwt http client interactions.
+var JWTClientTimeout = settings.RegisterDurationSetting(
+	settings.ApplicationLevel,
+	JWTAuthClientTimeoutSettingName,
+	"sets the timeout for jwt http client operations",
+	httputil.StandardHTTPTimeout,
+	settings.WithPublic)
+
+var JWTClientCustomCA = settings.RegisterStringSetting(
+	settings.ApplicationLevel,
+	JWTAuthClientCustomCASettingName,
+	"sets the custom root CA (appended to system's default CAs) for verifying "+
+		"certificates when interacting with jwt internal HTTPS client",
+	"",
+	settings.WithPublic)
+
 func validateJWTAuthIssuers(values *settings.Values, s string) error {
 	var issuers []string
 

pkg/sql/pgwire/auth.go

@@ -553,7 +553,8 @@ func (p *authPipe) LogAuthFailed(
 		p.loggedFailure = true
 		var errStr redact.RedactableString
 		if detailedErr != nil {
-			errStr = redact.Sprint(detailedErr)
+			// should we apply redact here?
+			errStr = redact.Sprint(detailedErr).Redact()
 		}
 		ev := &eventpb.ClientAuthenticationFailed{
 			CommonConnectionDetails: p.connDetails,

pkg/sql/pgwire/auth_methods.go

@@ -781,7 +781,10 @@ func authJwtToken(
 			return security.NewErrPasswordUserAuthFailed(user)
 		}
 		if err = jwtVerifier.ValidateJWTLogin(ctx, execCfg.Settings, user, []byte(token), identMap); err != nil {
+			// We are logging unsafe part of error also. Also, should we look into
+			// obtaining more specific auth failure reason here?
 			c.LogAuthFailed(ctx, eventpb.AuthFailReason_CREDENTIALS_INVALID, err)
+			// we are returning unsafe error to client
 			return err
 		}
 		c.LogAuthOK(ctx)

pkg/util/httputil/client.go

@@ -12,6 +12,8 @@ package httputil
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"io"
 	"net"
 	"net/http"
@@ -32,13 +34,41 @@ func NewClientWithTimeout(timeout time.Duration) *Client {
 
 // NewClientWithTimeouts defines a http.Client with the given dialer and client timeouts.
 func NewClientWithTimeouts(dialerTimeout, clientTimeout time.Duration) *Client {
+	return NewClientWithTimeoutsCustomCA(dialerTimeout, clientTimeout, "")
+}
+
+// NewClientWithTimeoutsCustomCA defines a http.Client with the given dialer and client timeouts and custom CA pem.
+func NewClientWithTimeoutsCustomCA(
+	dialerTimeout, clientTimeout time.Duration, customCAPem string,
+) *Client {
+	var tlsConf *tls.Config
+	if customCAPem != "" {
+		roots, err := x509.SystemCertPool()
+		if err != nil {
+			return nil
+		}
+		if !roots.AppendCertsFromPEM([]byte(customCAPem)) {
+			return nil
+		}
+		tlsConf = &tls.Config{RootCAs: roots}
+	}
+	t := http.DefaultTransport.(*http.Transport)
 	return &Client{&http.Client{
 		Timeout: clientTimeout,
 		Transport: &http.Transport{
 			// Don't leak a goroutine on OSX (the TCP level timeout is probably
 			// much higher than on linux).
 			DialContext:       (&net.Dialer{Timeout: dialerTimeout}).DialContext,
 			DisableKeepAlives: true,
+
+			Proxy:                 t.Proxy,
+			MaxIdleConns:          t.MaxIdleConns,
+			IdleConnTimeout:       t.IdleConnTimeout,
+			TLSHandshakeTimeout:   t.TLSHandshakeTimeout,
+			ExpectContinueTimeout: t.ExpectContinueTimeout,
+
+			// Add our custom CA.
+			TLSClientConfig: tlsConf,
 		},
 	}}
 }