Persist HCP management token from server config

We want to move away from injecting an initial management token into
Consul clusters linked to HCP. The reasoning is that by using a separate
class of token we can have more flexibility in terms of allowing HCP's
token to co-exist with the user's management token.

Down the line we can also more easily adjust the permissions attached to
HCP's token to limit it's scope.

With these changes, the cloud management token is like the initial
management token in that iit has the same global management policy and
if it is created it effectively bootstraps the ACL system.

agent/agent.go

@@ -1517,6 +1517,8 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co
 	cfg.RequestLimitsWriteRate = runtimeCfg.RequestLimitsWriteRate
 	cfg.Locality = runtimeCfg.StructLocality()
 
+	cfg.Cloud.ManagementToken = runtimeCfg.Cloud.ManagementToken
+
 	cfg.Reporting.License.Enabled = runtimeCfg.Reporting.License.Enabled
 
 	enterpriseConsulConfig(cfg, runtimeCfg)

agent/consul/acl_server.go

@@ -153,7 +153,7 @@ func (s *Server) ResolveIdentityFromToken(token string) (bool, structs.ACLIdenti
 	}
 	if aclToken == nil && token == acl.AnonymousTokenSecret {
 		// synthesize the anonymous token for early use, bootstrapping has not completed
-		s.InsertAnonymousToken()
+		s.insertAnonymousToken()
 		fallbackId := structs.ACLToken{
 			AccessorID:  acl.AnonymousTokenID,
 			SecretID:    acl.AnonymousTokenSecret,

agent/consul/config.go

@@ -441,6 +441,8 @@ type Config struct {
 
 	Locality *structs.Locality
 
+	Cloud CloudConfig
+
 	Reporting Reporting
 
 	// Embedded Consul Enterprise specific configuration

agent/consul/config_cloud.go

@@ -0,0 +1,5 @@
+package consul
+
+type CloudConfig struct {
+	ManagementToken string
+}

agent/consul/leader.go

@@ -450,72 +450,22 @@ func (s *Server) initializeACLs(ctx context.Context) error {
 
 		// Check for configured initial management token.
 		if initialManagement := s.config.ACLInitialManagementToken; len(initialManagement) > 0 {
-			state := s.fsm.State()
-			if _, err := uuid.ParseUUID(initialManagement); err != nil {
-				s.logger.Warn("Configuring a non-UUID initial management token is deprecated")
-			}
-
-			_, token, err := state.ACLTokenGetBySecret(nil, initialManagement, nil)
+			err := s.initializeManagementToken("Initial Management Token", initialManagement)
 			if err != nil {
-				return fmt.Errorf("failed to get initial management token: %v", err)
+				return fmt.Errorf("failed to initialize initial management token: %w", err)
 			}
-			// Ignoring expiration times to avoid an insertion collision.
-			if token == nil {
-				accessor, err := lib.GenerateUUID(s.checkTokenUUID)
-				if err != nil {
-					return fmt.Errorf("failed to generate the accessor ID for the initial management token: %v", err)
-				}
-
-				token := structs.ACLToken{
-					AccessorID:  accessor,
-					SecretID:    initialManagement,
-					Description: "Initial Management Token",
-					Policies: []structs.ACLTokenPolicyLink{
-						{
-							ID: structs.ACLPolicyGlobalManagementID,
-						},
-					},
-					CreateTime:     time.Now(),
-					Local:          false,
-					EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(),
-				}
-
-				token.SetHash(true)
-
-				done := false
-				if canBootstrap, _, err := state.CanBootstrapACLToken(); err == nil && canBootstrap {
-					req := structs.ACLTokenBootstrapRequest{
-						Token:      token,
-						ResetIndex: 0,
-					}
-					if _, err := s.raftApply(structs.ACLBootstrapRequestType, &req); err == nil {
-						s.logger.Info("Bootstrapped ACL initial management token from configuration")
-						done = true
-					} else {
-						if err.Error() != structs.ACLBootstrapNotAllowedErr.Error() &&
-							err.Error() != structs.ACLBootstrapInvalidResetIndexErr.Error() {
-							return fmt.Errorf("failed to bootstrap initial management token: %v", err)
-						}
-					}
-				}
-
-				if !done {
-					// either we didn't attempt to or setting the token with a bootstrap request failed.
-					req := structs.ACLTokenBatchSetRequest{
-						Tokens: structs.ACLTokens{&token},
-						CAS:    false,
-					}
-					if _, err := s.raftApply(structs.ACLTokenSetRequestType, &req); err != nil {
-						return fmt.Errorf("failed to create initial management token: %v", err)
-					}
+		}
 
-					s.logger.Info("Created ACL initial management token from configuration")
-				}
+		// Check for configured management token from HCP. It MUST NOT override the user-provided initial management token.
+		if hcpManagement := s.config.Cloud.ManagementToken; len(hcpManagement) > 0 {
+			err := s.initializeManagementToken("HCP Management Token", hcpManagement)
+			if err != nil {
+				return fmt.Errorf("failed to initialize HCP management token: %w", err)
 			}
 		}
 
 		// Insert the anonymous token if it does not exist.
-		if err := s.InsertAnonymousToken(); err != nil {
+		if err := s.insertAnonymousToken(); err != nil {
 			return err
 		}
 	} else {
@@ -540,7 +490,74 @@ func (s *Server) initializeACLs(ctx context.Context) error {
 	return nil
 }
 
-func (s *Server) InsertAnonymousToken() error {
+func (s *Server) initializeManagementToken(name, secretID string) error {
+	state := s.fsm.State()
+	if _, err := uuid.ParseUUID(secretID); err != nil {
+		s.logger.Warn("Configuring a non-UUID management token is deprecated")
+	}
+
+	_, token, err := state.ACLTokenGetBySecret(nil, secretID, nil)
+	if err != nil {
+		return fmt.Errorf("failed to get %s: %v", name, err)
+	}
+	// Ignoring expiration times to avoid an insertion collision.
+	if token == nil {
+		accessor, err := lib.GenerateUUID(s.checkTokenUUID)
+		if err != nil {
+			return fmt.Errorf("failed to generate the accessor ID for %s: %v", name, err)
+		}
+
+		token := structs.ACLToken{
+			AccessorID:  accessor,
+			SecretID:    secretID,
+			Description: name,
+			Policies: []structs.ACLTokenPolicyLink{
+				{
+					ID: structs.ACLPolicyGlobalManagementID,
+				},
+			},
+			CreateTime:     time.Now(),
+			Local:          false,
+			EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(),
+		}
+
+		token.SetHash(true)
+
+		done := false
+		if canBootstrap, _, err := state.CanBootstrapACLToken(); err == nil && canBootstrap {
+			req := structs.ACLTokenBootstrapRequest{
+				Token:      token,
+				ResetIndex: 0,
+			}
+			if _, err := s.raftApply(structs.ACLBootstrapRequestType, &req); err == nil {
+				s.logger.Info("Bootstrapped ACL token from configuration", "description", name)
+				done = true
+			} else {
+				if err.Error() != structs.ACLBootstrapNotAllowedErr.Error() &&
+					err.Error() != structs.ACLBootstrapInvalidResetIndexErr.Error() {
+					return fmt.Errorf("failed to bootstrap with %s: %v", name, err)
+				}
+			}
+		}
+
+		if !done {
+			// either we didn't attempt to or setting the token with a bootstrap request failed.
+			req := structs.ACLTokenBatchSetRequest{
+				Tokens: structs.ACLTokens{&token},
+				CAS:    false,
+			}
+			if _, err := s.raftApply(structs.ACLTokenSetRequestType, &req); err != nil {
+				return fmt.Errorf("failed to create %s: %v", name, err)
+			}
+
+			s.logger.Info("Created ACL token from configuration", "description", name)
+		}
+	}
+
+	return nil
+}
+
+func (s *Server) insertAnonymousToken() error {
 	state := s.fsm.State()
 	_, token, err := state.ACLTokenGetBySecret(nil, anonymousToken, nil)
 	if err != nil {

agent/consul/leader_test.go

@@ -1261,47 +1261,77 @@ func TestLeader_ACL_Initialization(t *testing.T) {
 
 	tests := []struct {
 		name              string
-		build             string
 		initialManagement string
-		bootstrap         bool
+		hcpManagement     string
+
+		// canBootstrap tracks whether the ACL system can be bootstrapped
+		// after the leader initializes ACLs. Bootstrapping is the act
+		// of persisting a token with the Global Management policy.
+		canBootstrap bool
 	}{
-		{"old version, no initial management", "0.8.0", "", true},
-		{"old version, initial management", "0.8.0", "root", false},
-		{"new version, no initial management", "0.9.1", "", true},
-		{"new version, initial management", "0.9.1", "root", false},
+		{
+			name:              "bootstrap from initial management",
+			initialManagement: "c9ad785a-420d-470d-9b4d-6d9f084bfa87",
+			hcpManagement:     "",
+			canBootstrap:      false,
+		},
+		{
+			name:              "bootstrap from hcp management",
+			initialManagement: "",
+			hcpManagement:     "924bc0e1-a41b-4f3a-b5e8-0899502fc50e",
+			canBootstrap:      false,
+		},
+		{
+			name:              "bootstrap with both",
+			initialManagement: "c9ad785a-420d-470d-9b4d-6d9f084bfa87",
+			hcpManagement:     "924bc0e1-a41b-4f3a-b5e8-0899502fc50e",
+			canBootstrap:      false,
+		},
+		{
+			name:              "did not bootstrap",
+			initialManagement: "",
+			hcpManagement:     "",
+			canBootstrap:      true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			conf := func(c *Config) {
-				c.Build = tt.build
 				c.Bootstrap = true
 				c.Datacenter = "dc1"
 				c.PrimaryDatacenter = "dc1"
 				c.ACLsEnabled = true
 				c.ACLInitialManagementToken = tt.initialManagement
+				c.Cloud.ManagementToken = tt.hcpManagement
 			}
-			dir1, s1 := testServerWithConfig(t, conf)
-			defer os.RemoveAll(dir1)
-			defer s1.Shutdown()
+			_, s1 := testServerWithConfig(t, conf)
 			testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
 
+			_, policy, err := s1.fsm.State().ACLPolicyGetByID(nil, structs.ACLPolicyGlobalManagementID, nil)
+			require.NoError(t, err)
+			require.NotNil(t, policy)
+
 			if tt.initialManagement != "" {
 				_, initialManagement, err := s1.fsm.State().ACLTokenGetBySecret(nil, tt.initialManagement, nil)
 				require.NoError(t, err)
 				require.NotNil(t, initialManagement)
+				require.Equal(t, tt.initialManagement, initialManagement.SecretID)
 			}
 
-			_, anon, err := s1.fsm.State().ACLTokenGetBySecret(nil, anonymousToken, nil)
-			require.NoError(t, err)
-			require.NotNil(t, anon)
+			if tt.hcpManagement != "" {
+				_, hcpManagement, err := s1.fsm.State().ACLTokenGetBySecret(nil, tt.hcpManagement, nil)
+				require.NoError(t, err)
+				require.NotNil(t, hcpManagement)
+				require.Equal(t, tt.hcpManagement, hcpManagement.SecretID)
+			}
 
 			canBootstrap, _, err := s1.fsm.State().CanBootstrapACLToken()
 			require.NoError(t, err)
-			require.Equal(t, tt.bootstrap, canBootstrap)
+			require.Equal(t, tt.canBootstrap, canBootstrap)
 
-			_, policy, err := s1.fsm.State().ACLPolicyGetByID(nil, structs.ACLPolicyGlobalManagementID, nil)
+			_, anon, err := s1.fsm.State().ACLTokenGetBySecret(nil, anonymousToken, nil)
 			require.NoError(t, err)
-			require.NotNil(t, policy)
+			require.NotNil(t, anon)
 
 			serverToken, err := s1.GetSystemMetadata(structs.ServerManagementTokenAccessorID)
 			require.NoError(t, err)

agent/hcp/config/config.go

@@ -17,6 +17,9 @@ type CloudConfig struct {
 	Hostname     string
 	AuthURL      string
 	ScadaAddress string
+
+	// internal
+	ManagementToken string
 }
 
 func (c *CloudConfig) HCPConfig(opts ...hcpcfg.HCPConfigOption) (hcpcfg.HCPConfig, error) {