Hash token values for storage (#41792) (#42220)

This commit changes how access tokens and refresh tokens are stored
in the tokens index.

Access token values are now hashed before being stored in the id
field of the `user_token` and before becoming part of the token
document id. Refresh token values are hashed before being stored
in the token field of the `refresh_token`. The tokens are hashed
without a salt value since these are v4 UUID values that have
enough entropy themselves. Both rainbow table attacks and offline
brute force attacks are impractical.

As a side effect of this change and in order to support multiple
concurrent refreshes as introduced in #39631, upon refreshing an
<access token, refresh token> pair, the superseding access token
and refresh tokens values are stored in the superseded token doc,
encrypted with a key that is derived from the superseded refresh
token. As such, subsequent requests to refresh the same token in
the predefined time window will return the same superseding access
token and refresh token values, without hitting the tokens index
(as this only stores hashes of the token values). AES in GCM
mode is used for encrypting the token values and the key
derivation from the superseded refresh token uses a small number
of iterations as it needs to be quick.

For backwards compatibility reasons, the new behavior is only
enabled when all nodes in a cluster are in the required version
so that old nodes can cope with the token values in a mixed
cluster during a rolling upgrade.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java

@@ -351,6 +351,24 @@ public boolean verify(SecureString text, char[] hash) {
             return CharArrays.constantTimeEquals(computedHash, new String(saltAndHash, 12, saltAndHash.length - 12));
         }
     },
+    /*
+     * Unsalted SHA-256 , not suited for password storage.
+     */
+    SHA256() {
+        @Override
+        public char[] hash(SecureString text) {
+            MessageDigest md = MessageDigests.sha256();
+            md.update(CharArrays.toUtf8Bytes(text.getChars()));
+            return Base64.getEncoder().encodeToString(md.digest()).toCharArray();
+        }
+
+        @Override
+        public boolean verify(SecureString text, char[] hash) {
+            MessageDigest md = MessageDigests.sha256();
+            md.update(CharArrays.toUtf8Bytes(text.getChars()));
+            return CharArrays.constantTimeEquals(Base64.getEncoder().encodeToString(md.digest()).toCharArray(), hash);
+        }
+    },
 
     NOOP() {
         @Override

x-pack/plugin/core/src/main/resources/security-index-template-7.json

@@ -213,8 +213,19 @@
               "type": "date",
               "format": "epoch_millis"
             },
-            "superseded_by": {
-              "type": "keyword"
+            "superseding": {
+              "type": "object",
+              "properties": {
+                "encrypted_tokens": {
+                  "type": "binary"
+                },
+                "encryption_iv": {
+                  "type": "binary"
+                },
+                "encryption_salt": {
+                  "type": "binary"
+                }
+              }
             },
             "invalidated" : {
               "type" : "boolean"

x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json

@@ -35,8 +35,19 @@
               "type": "date",
               "format": "epoch_millis"
             },
-            "superseded_by": {
-              "type": "keyword"
+            "superseding": {
+              "type": "object",
+              "properties": {
+                "encrypted_tokens": {
+                  "type": "binary"
+                },
+                "encryption_iv": {
+                  "type": "binary"
+                },
+                "encryption_salt": {
+                  "type": "binary"
+                }
+              }
             },
             "invalidated" : {
               "type" : "boolean"

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectAuthenticateAction.java

@@ -7,6 +7,8 @@
 
 import com.nimbusds.oauth2.sdk.id.State;
 import com.nimbusds.openid.connect.sdk.Nonce;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
@@ -36,6 +38,7 @@ public class TransportOpenIdConnectAuthenticateAction
     private final ThreadPool threadPool;
     private final AuthenticationService authenticationService;
     private final TokenService tokenService;
+    private static final Logger logger = LogManager.getLogger(TransportOpenIdConnectAuthenticateAction.class);
 
     @Inject
     public TransportOpenIdConnectAuthenticateAction(ThreadPool threadPool, TransportService transportService,
@@ -67,9 +70,8 @@ protected void doExecute(Task task, OpenIdConnectAuthenticateRequest request,
                         .get(OpenIdConnectRealm.CONTEXT_TOKEN_DATA);
                     tokenService.createOAuth2Tokens(authentication, originatingAuthentication, tokenMetadata, true,
                         ActionListener.wrap(tuple -> {
-                            final String tokenString = tokenService.getAccessTokenAsString(tuple.v1());
                             final TimeValue expiresIn = tokenService.getExpirationDelay();
-                            listener.onResponse(new OpenIdConnectAuthenticateResponse(authentication.getUser().principal(), tokenString,
+                            listener.onResponse(new OpenIdConnectAuthenticateResponse(authentication.getUser().principal(), tuple.v1(),
                                 tuple.v2(), expiresIn));
                         }, listener::onFailure));
                 }, e -> {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlAuthenticateAction.java

@@ -63,10 +63,9 @@ protected void doExecute(Task task, SamlAuthenticateRequest request, ActionListe
                 final Map<String, Object> tokenMeta = (Map<String, Object>) result.getMetadata().get(SamlRealm.CONTEXT_TOKEN_DATA);
                 tokenService.createOAuth2Tokens(authentication, originatingAuthentication,
                         tokenMeta, true, ActionListener.wrap(tuple -> {
-                            final String tokenString = tokenService.getAccessTokenAsString(tuple.v1());
                             final TimeValue expiresIn = tokenService.getExpirationDelay();
                             listener.onResponse(
-                                    new SamlAuthenticateResponse(authentication.getUser().principal(), tokenString, tuple.v2(), expiresIn));
+                                    new SamlAuthenticateResponse(authentication.getUser().principal(), tuple.v1(), tuple.v2(), expiresIn));
                         }, listener::onFailure));
             }, e -> {
                 logger.debug(() -> new ParameterizedMessage("SamlToken [{}] could not be authenticated", saml), e);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java

@@ -88,9 +88,8 @@ private void createToken(CreateTokenRequest request, Authentication authenticati
             boolean includeRefreshToken, ActionListener<CreateTokenResponse> listener) {
         tokenService.createOAuth2Tokens(authentication, originatingAuth, Collections.emptyMap(), includeRefreshToken,
                 ActionListener.wrap(tuple -> {
-                    final String tokenStr = tokenService.getAccessTokenAsString(tuple.v1());
                     final String scope = getResponseScopeValue(request.getScope());
-                    final CreateTokenResponse response = new CreateTokenResponse(tokenStr, tokenService.getExpirationDelay(), scope,
+                    final CreateTokenResponse response = new CreateTokenResponse(tuple.v1(), tokenService.getExpirationDelay(), scope,
                             tuple.v2());
                     listener.onResponse(response);
                 }, listener::onFailure));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java

@@ -31,11 +31,9 @@ public TransportRefreshTokenAction(TransportService transportService, ActionFilt
     @Override
     protected void doExecute(Task task, CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
         tokenService.refreshToken(request.getRefreshToken(), ActionListener.wrap(tuple -> {
-            final String tokenStr = tokenService.getAccessTokenAsString(tuple.v1());
             final String scope = getResponseScopeValue(request.getScope());
-
             final CreateTokenResponse response =
-                    new CreateTokenResponse(tokenStr, tokenService.getExpirationDelay(), scope, tuple.v2());
+                    new CreateTokenResponse(tuple.v1(), tokenService.getExpirationDelay(), scope, tuple.v2());
             listener.onResponse(response);
         }, listener::onFailure));
     }