ccl, sql: add authZ handler for ldap groups retrieval

informs cockroachdb#125087
fixes CRDB-39227
Epic CRDB-33829

This PR adds support for fetching LDAP groups info for sql user trying to gain
authorization for cockroach cluster via LDAP. This change will enable us to
assign appropriate role privileges for user on CRDB.

Release note(enterprise, security): We add support for authorization to CRDB
cluster via LDAP, retrieving AD groups membership information for LDAP user. The
new HBA conf cluster setting option `ldapgrouplistfilter` performs filtered
search query on LDAP for matching groups.

Example hba conf entry to support LDAP authZ configuration:
```
 # TYPE    DATABASE      USER           ADDRESS             METHOD             OPTIONS
 # Allow all users to connect to using LDAP authentication with search and bind
   host    all           all            all                 ldap               ldapserver=ldap.example.com ldapport=636 "ldapbasedn=ou=users,dc=example,dc=com" "ldapbinddn=cn=readonly,dc=example,dc=com" ldapbindpasswd=readonly_password ldapsearchattribute=uid "ldapsearchfilter=(memberof=cn=cockroachdb_users,ou=groups,dc=example,dc=com)" "ldapgrouplistfilter=(objectClass=groupOfNames)"
 # Fallback to password authentication for the root user
   host    all           root           0.0.0.0/0          password
```
Example to use for azure AD server:
```
SET cluster setting server.host_based_authentication.configuration = 'host    all           all            all                 ldap ldapserver=azure.dev ldapport=636 "ldapbasedn=OU=AADDC Users,DC=azure,DC=dev" "ldapbinddn=CN=Some User,OU=AADDC Users,DC=azure,DC=dev" ldapbindpasswd=my_pwd ldapsearchattribute=sAMAccountName "ldapsearchfilter=(memberOf=CN=azure-dev-domain-sync-users,OU=AADDC Users,DC=crlcloud,DC=dev)" "ldapgrouplistfilter=(objectCategory=CN=Group,CN=Schema,CN=Configuration,DC=crlcloud,DC=dev)"
host    all           root           0.0.0.0/0          password';
```

Post configuration CRDB cluster should be able to authorize users via LDAP
server if:
1. Users LDAP authentication attempt is successful, and it has the user's DN for
LDAP server.
2. `ldapgrouplistfilter` is properly configured, and it successfully syncs
groups of the user.

pkg/ccl/ldapccl/BUILD.bazel

@@ -4,6 +4,9 @@ go_library(
     name = "ldapccl",
     srcs = [
         "authentication_ldap.go",
+        "authorization_ldap.go",
+        "ldap_manager.go",
+        "ldap_test_util.go",
         "ldap_util.go",
         "settings.go",
     ],
@@ -13,6 +16,7 @@ go_library(
         "//pkg/ccl/utilccl",
         "//pkg/clusterversion",
         "//pkg/security",
+        "//pkg/security/distinguishedname",
         "//pkg/security/username",
         "//pkg/server/telemetry",
         "//pkg/settings",
@@ -36,6 +40,7 @@ go_test(
     size = "small",
     srcs = [
         "authentication_ldap_test.go",
+        "authorization_ldap_test.go",
         "main_test.go",
         "settings_test.go",
     ],
@@ -49,7 +54,6 @@ go_test(
         "//pkg/security/securitytest",
         "//pkg/security/username",
         "//pkg/server",
-        "//pkg/sql/pgwire/hba",
         "//pkg/testutils",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/testcluster",
@@ -58,7 +62,6 @@ go_test(
         "//pkg/util/randutil",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_redact//:redact",
-        "@com_github_go_ldap_ldap_v3//:ldap",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/ccl/ldapccl/authentication_ldap.go

@@ -13,157 +13,36 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/ccl/utilccl"
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
+	"github.com/cockroachdb/cockroach/pkg/security/distinguishedname"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/server/telemetry"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
-	"github.com/cockroachdb/cockroach/pkg/sql/pgwire"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/hba"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/identmap"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
-	"github.com/cockroachdb/cockroach/pkg/util/log"
-	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
-	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/redact"
+	"github.com/go-ldap/ldap/v3"
 )
 
 const (
-	counterPrefix           = "auth.ldap."
-	beginAuthCounterName    = counterPrefix + "begin_auth"
+	beginAuthNCounterName   = counterPrefix + "begin_authentication"
 	loginSuccessCounterName = counterPrefix + "login_success"
-	enableCounterName       = counterPrefix + "enable"
 )
 
 var (
-	beginAuthUseCounter    = telemetry.GetCounterOnce(beginAuthCounterName)
+	beginAuthNUseCounter   = telemetry.GetCounterOnce(beginAuthNCounterName)
 	loginSuccessUseCounter = telemetry.GetCounterOnce(loginSuccessCounterName)
-	enableUseCounter       = telemetry.GetCounterOnce(enableCounterName)
 )
 
-// ldapAuthenticator is an object that is used to enable ldap connection
-// validation that are used as part of the CRDB client auth flow.
-//
-// The implementation uses the `go-ldap/ldap/` client package and is supported
-// through a number of cluster settings defined in `ldapccl/settings.go`. These
-// settings specify how the ldap auth attempt should be executed and if this
-// feature is enabled.
-type ldapAuthenticator struct {
-	mu struct {
-		syncutil.RWMutex
-		// conf contains all the values that come from cluster settings.
-		conf ldapAuthenticatorConf
-		// util contains connection object required for interfacing with ldap server.
-		util ILDAPUtil
-		// enabled represents the present state of if this feature is enabled. It
-		// is set to true once ldap util is initialized.
-		enabled bool
-	}
-	// clusterUUID is used to check the validity of the enterprise license. It is
-	// set once at initialization.
-	clusterUUID uuid.UUID
-}
-
-// ldapAuthenticatorConf contains all the values to configure LDAP
-// authentication. These values are copied from the matching cluster settings or
-// from hba conf options for LDAP entry.
-type ldapAuthenticatorConf struct {
-	domainCACert        string
-	clientTLSCert       string
-	clientTLSKey        string
-	ldapServer          string
-	ldapPort            string
-	ldapBaseDN          string
-	ldapBindDN          string
-	ldapBindPassword    string
-	ldapSearchFilter    string
-	ldapSearchAttribute string
-}
-
-// reloadConfig locks mutex and then refreshes the values in conf from the cluster settings.
-func (authenticator *ldapAuthenticator) reloadConfig(ctx context.Context, st *cluster.Settings) {
-	authenticator.mu.Lock()
-	defer authenticator.mu.Unlock()
-	authenticator.reloadConfigLocked(ctx, st)
-}
-
-// reloadConfig refreshes the values in conf from the cluster settings without locking the mutex.
-func (authenticator *ldapAuthenticator) reloadConfigLocked(
-	ctx context.Context, st *cluster.Settings,
-) {
-	conf := ldapAuthenticatorConf{
-		domainCACert:  LDAPDomainCACertificate.Get(&st.SV),
-		clientTLSCert: LDAPClientTLSCertSetting.Get(&st.SV),
-		clientTLSKey:  LDAPClientTLSKeySetting.Get(&st.SV),
-	}
-	authenticator.mu.conf = conf
-
-	var err error
-	authenticator.mu.util, err = NewLDAPUtil(ctx, authenticator.mu.conf)
-	if err != nil {
-		log.Warningf(ctx, "LDAP authentication: unable to initialize LDAP connection: %v", err)
-		return
-	}
-
-	if !authenticator.mu.enabled {
-		telemetry.Inc(enableUseCounter)
-	}
-	authenticator.mu.enabled = true
-	log.Infof(ctx, "initialized LDAP authenticator")
-}
-
-// setLDAPConfigOptions extracts hba conf parameters required for connecting and
-// querying LDAP server from hba conf entry and sets them for LDAP authenticator.
-func (authenticator *ldapAuthenticator) setLDAPConfigOptions(entry *hba.Entry) error {
-	conf := ldapAuthenticatorConf{
-		domainCACert: authenticator.mu.conf.domainCACert,
-	}
-	for _, opt := range entry.Options {
-		switch opt[0] {
-		case "ldapserver":
-			conf.ldapServer = opt[1]
-		case "ldapport":
-			conf.ldapPort = opt[1]
-		case "ldapbasedn":
-			conf.ldapBaseDN = opt[1]
-		case "ldapbinddn":
-			conf.ldapBindDN = opt[1]
-		case "ldapbindpasswd":
-			conf.ldapBindPassword = opt[1]
-		case "ldapsearchfilter":
-			conf.ldapSearchFilter = opt[1]
-		case "ldapsearchattribute":
-			conf.ldapSearchAttribute = opt[1]
-		default:
-			return errors.Newf("invalid LDAP option provided in hba conf: %s", opt[0])
-		}
-	}
-	authenticator.mu.conf = conf
-	return nil
-}
-
-// validateLDAPOptions checks the ldap authenticator config values for validity.
-func (authenticator *ldapAuthenticator) validateLDAPOptions() error {
-	const ldapOptionsErrorMsg = "ldap params in HBA conf missing"
-	if authenticator.mu.conf.ldapServer == "" {
-		return errors.New(ldapOptionsErrorMsg + " ldap server")
-	}
-	if authenticator.mu.conf.ldapPort == "" {
-		return errors.New(ldapOptionsErrorMsg + " ldap port")
-	}
-	if authenticator.mu.conf.ldapBaseDN == "" {
-		return errors.New(ldapOptionsErrorMsg + " base DN")
-	}
-	if authenticator.mu.conf.ldapBindDN == "" {
-		return errors.New(ldapOptionsErrorMsg + " bind DN")
-	}
-	if authenticator.mu.conf.ldapBindPassword == "" {
-		return errors.New(ldapOptionsErrorMsg + " bind password")
-	}
-	if authenticator.mu.conf.ldapSearchFilter == "" {
+// validateLDAPAuthNOptions checks the ldap authentication config values.
+func (authManager *ldapAuthManager) validateLDAPAuthNOptions() error {
+	const ldapOptionsErrorMsg = "ldap authentication params in HBA conf missing"
+	if authManager.mu.conf.ldapSearchFilter == "" {
 		return errors.New(ldapOptionsErrorMsg + " search filter")
 	}
-	if authenticator.mu.conf.ldapSearchAttribute == "" {
+	if authManager.mu.conf.ldapSearchAttribute == "" {
 		return errors.New(ldapOptionsErrorMsg + " search attribute")
 	}
 	return nil
@@ -173,78 +52,92 @@ func (authenticator *ldapAuthenticator) validateLDAPOptions() error {
 // In particular, it checks that:
 // * The cluster has an enterprise license.
 // * The active cluster version is 24.2 for this feature.
-// * LDAP authentication is enabled after settings were reloaded.
+// * LDAP authManager is enabled after settings were reloaded.
 // * The auth attempt is not for a reserved user.
 // * The hba conf entry options could be parsed to obtain ldap server params.
 // * All ldap server params are valid.
 // * LDAPs connection can be established with configured server.
 // * Configured bind DN and password can be used to search for the sql user DN on ldap server.
 // * The obtained user DN could be used to bind with the password from sql connection string.
-// It returns authError (which is the error sql clients will see in case of
+// It returns the retrievedUserDN which is the DN associated with the user in
+// LDAP server, authError (which is the error sql clients will see in case of
 // failures) and detailedError (which is the internal error from ldap clients
 // that might contain sensitive information we do not want to send to sql
 // clients but still want to log it). We do not want to send any information
 // back to client which was not provided by the client.
-func (authenticator *ldapAuthenticator) ValidateLDAPLogin(
+func (authManager *ldapAuthManager) ValidateLDAPLogin(
 	ctx context.Context,
 	st *cluster.Settings,
 	user username.SQLUsername,
 	ldapPwd string,
 	entry *hba.Entry,
 	_ *identmap.Conf,
-) (detailedErrorMsg redact.RedactableString, authError error) {
+) (retrievedUserDN *ldap.DN, detailedErrorMsg redact.RedactableString, authError error) {
 	if err := utilccl.CheckEnterpriseEnabled(st, "LDAP authentication"); err != nil {
-		return "", err
+		return nil, "", err
 	}
 	if !st.Version.IsActive(ctx, clusterversion.V24_2) {
-		return "", pgerror.Newf(pgcode.FeatureNotSupported, "LDAP authentication is only supported after v24.2 upgrade is finalized")
+		return nil, "", pgerror.Newf(pgcode.FeatureNotSupported, "LDAP authentication is only supported after v24.2 upgrade is finalized")
 	}
 
-	authenticator.mu.Lock()
-	defer authenticator.mu.Unlock()
+	authManager.mu.Lock()
+	defer authManager.mu.Unlock()
 
-	if !authenticator.mu.enabled {
-		return "", errors.Newf("LDAP authentication: not enabled")
+	if !authManager.mu.enabled {
+		return nil, "", errors.Newf("LDAP authentication: not enabled")
 	}
-	telemetry.Inc(beginAuthUseCounter)
+	telemetry.Inc(beginAuthNUseCounter)
 
 	if user.IsRootUser() || user.IsReserved() {
-		return "", errors.WithDetailf(
+		return nil, "", errors.WithDetailf(
 			errors.Newf("LDAP authentication: invalid identity"),
 			"cannot use LDAP auth to login to a reserved user %s", user.Normalized())
 	}
 
-	if err := authenticator.setLDAPConfigOptions(entry); err != nil {
-		return redact.Sprintf("error parsing hba conf options for LDAP: %v", err),
+	if err := authManager.setLDAPConfigOptions(entry); err != nil {
+		return nil, redact.Sprintf("error parsing hba conf options for LDAP: %v", err),
 			errors.Newf("LDAP authentication: unable to parse hba conf options")
 	}
 
-	if err := authenticator.validateLDAPOptions(); err != nil {
-		return redact.Sprintf("error validating hba conf options for LDAP: %v", err),
-			errors.Newf("LDAP authentication: unable to validate authenticator options")
+	if err := authManager.validateLDAPBaseOptions(); err != nil {
+		return nil, redact.Sprintf("error validating base hba conf options for LDAP: %v", err),
+			errors.Newf("LDAP authentication: unable to validate authManager base options")
+	}
+
+	if err := authManager.validateLDAPAuthNOptions(); err != nil {
+		return nil, redact.Sprintf("error validating authentication hba conf options for LDAP: %v", err),
+			errors.Newf("LDAP authentication: unable to validate authManager authentication options")
 	}
 
 	// Establish a LDAPs connection with the set LDAP server and port
-	err := authenticator.mu.util.InitLDAPsConn(ctx, authenticator.mu.conf)
+	err := authManager.mu.util.MaybeInitLDAPsConn(ctx, authManager.mu.conf)
 	if err != nil {
-		return redact.Sprintf("error when trying to create LDAP connection: %v", err),
+		return nil, redact.Sprintf("error when trying to create LDAP connection: %v", err),
 			errors.Newf("LDAP authentication: unable to establish LDAP connection")
 	}
 
 	// Fetch the ldap server Distinguished Name using sql username as search value
 	// for  ldap search attribute
-	userDN, err := authenticator.mu.util.Search(ctx, authenticator.mu.conf, user.Normalized())
+	userDN, err := authManager.mu.util.Search(ctx, authManager.mu.conf, user.Normalized())
 	if err != nil {
-		return redact.Sprintf("error when searching for user in LDAP server: %v", err),
+		return nil, redact.Sprintf("error when searching for user in LDAP server: %v", err),
 			errors.WithDetailf(
 				errors.Newf("LDAP authentication: unable to find LDAP user distinguished name"),
 				"cannot find provided user %s on LDAP server", user.Normalized())
 	}
 
+	retrievedUserDN, err = distinguishedname.ParseDN(userDN)
+	if err != nil {
+		return nil, redact.Sprintf("error parsing user DN %s obtained from LDAP server: %v", userDN, err),
+			errors.WithDetailf(
+				errors.Newf("LDAP authentication: unable to parse LDAP user distinguished name"),
+				"cannot find provided user %s on LDAP server", user.Normalized())
+	}
+
 	// Bind as the user to verify their password
-	err = authenticator.mu.util.Bind(ctx, userDN, ldapPwd)
+	err = authManager.mu.util.Bind(ctx, userDN, ldapPwd)
 	if err != nil {
-		return redact.Sprintf("error when binding as user %s with DN(%s) in LDAP server: %v",
+		return retrievedUserDN, redact.Sprintf("error when binding as user %s with DN(%s) in LDAP server: %v",
 				user.Normalized(), userDN, err,
 			),
 			errors.WithDetailf(
@@ -253,26 +146,5 @@ func (authenticator *ldapAuthenticator) ValidateLDAPLogin(
 	}
 
 	telemetry.Inc(loginSuccessUseCounter)
-	return "", nil
-}
-
-// ConfigureLDAPAuth initializes and returns a ldapAuthenticator. It also sets up listeners so
-// that the ldapAuthenticator's config is updated when the cluster settings values change.
-var ConfigureLDAPAuth = func(
-	serverCtx context.Context,
-	ambientCtx log.AmbientContext,
-	st *cluster.Settings,
-	clusterUUID uuid.UUID,
-) pgwire.LDAPVerifier {
-	authenticator := ldapAuthenticator{}
-	authenticator.clusterUUID = clusterUUID
-	authenticator.reloadConfig(serverCtx, st)
-	LDAPDomainCACertificate.SetOnChange(&st.SV, func(ctx context.Context) {
-		authenticator.reloadConfig(ambientCtx.AnnotateCtx(ctx), st)
-	})
-	return &authenticator
-}
-
-func init() {
-	pgwire.ConfigureLDAPAuth = ConfigureLDAPAuth
+	return retrievedUserDN, "", nil
 }