diff --git a/CHANGELOG.md b/CHANGELOG.md index a09abfda2e56..990590f8691a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ DEPRECATIONS/CHANGES: + * `sys/health` DR Secondary Reporting: The `replication_dr_secondary` bool + returned by `sys/health` could be misleading since it would be `false` both + when a cluster was not a DR secondary but also when the node is a standby in + the cluster and has not yet fully received state from the active node. This + could cause health checks on LBs to decide that the node was acceptable for + traffic even though DR secondaries cannot handle normal Vault traffic. (In + other words, the bool could only convey "yes" or "no" but not "not sure + yet".) This has been replaced by `replication_dr_state` and + `replication_performance_state` which are string values that convey the + current state of the node; a value of `disabled` indicates that replication + is disabled or the state is still being discovered. As a result, an LB check + can positively verify that the node is both not `disabled` and is not a DR + secondary, and avoid sending traffic to it if either is true. * PKI Secret Backend Roles parameter types: For `ou` and `organization` in role definitions in the PKI secret backend, input can now be a comma-separated string or an array of strings. Reading a role will diff --git a/http/sys_health.go b/http/sys_health.go index 091f829ab2ed..c2e07b3c8a26 100644 --- a/http/sys_health.go +++ b/http/sys_health.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/vault" "github.com/hashicorp/vault/version" ) @@ -111,7 +112,7 @@ func getSysHealth(core *vault.Core, r *http.Request) (int, *HealthResponse, erro // Check system status sealed, _ := core.Sealed() standby, _ := core.Standby() - drSecondary := core.IsDRSecondary() + replicationState := core.ReplicationState() init, err := core.Initialized() if err != nil { return http.StatusInternalServerError, nil, err @@ -124,7 +125,7 @@ func getSysHealth(core *vault.Core, r *http.Request) (int, *HealthResponse, erro code = uninitCode case sealed: code = sealedCode - case drSecondary: + case replicationState.HasState(consts.ReplicationDRSecondary): code = drSecondaryCode case !standbyOK && standby: code = standbyCode @@ -146,25 +147,27 @@ func getSysHealth(core *vault.Core, r *http.Request) (int, *HealthResponse, erro // Format the body body := &HealthResponse{ - Initialized: init, - Sealed: sealed, - Standby: standby, - ReplicationDRSecondary: drSecondary, - ServerTimeUTC: time.Now().UTC().Unix(), - Version: version.GetVersion().VersionNumber(), - ClusterName: clusterName, - ClusterID: clusterID, + Initialized: init, + Sealed: sealed, + Standby: standby, + ReplicationPerformanceState: replicationState.GetPerformanceString(), + ReplicationDRState: replicationState.GetDRString(), + ServerTimeUTC: time.Now().UTC().Unix(), + Version: version.GetVersion().VersionNumber(), + ClusterName: clusterName, + ClusterID: clusterID, } return code, body, nil } type HealthResponse struct { - Initialized bool `json:"initialized"` - Sealed bool `json:"sealed"` - Standby bool `json:"standby"` - ReplicationDRSecondary bool `json:"replication_dr_secondary"` - ServerTimeUTC int64 `json:"server_time_utc"` - Version string `json:"version"` - ClusterName string `json:"cluster_name,omitempty"` - ClusterID string `json:"cluster_id,omitempty"` + Initialized bool `json:"initialized"` + Sealed bool `json:"sealed"` + Standby bool `json:"standby"` + ReplicationPerformanceState string `json:"replication_performance_state"` + ReplicationDRState string `json:"replication_dr_state"` + ServerTimeUTC int64 `json:"server_time_utc"` + Version string `json:"version"` + ClusterName string `json:"cluster_name,omitempty"` + ClusterID string `json:"cluster_id,omitempty"` } diff --git a/vault/request_forwarding.go b/vault/request_forwarding.go index 95afcc7a5483..577a5f00334a 100644 --- a/vault/request_forwarding.go +++ b/vault/request_forwarding.go @@ -421,7 +421,8 @@ func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) ( s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, nil, 0) } return &EchoReply{ - Message: "pong", + Message: "pong", + ReplicationState: uint32(s.core.ReplicationState()), }, nil } @@ -461,6 +462,9 @@ func (c *forwardingClient) startHeartbeat() { c.core.logger.Debug("forwarding: unexpected echo response from active node", "message", resp.Message) return } + // Store the active node's replication state to display in + // sys/health calls + atomic.StoreUint32(c.core.replicationState, resp.ReplicationState) c.core.logger.Trace("forwarding: successful heartbeat") } diff --git a/vault/request_forwarding_service.pb.go b/vault/request_forwarding_service.pb.go index 3826f7b61a7b..80f634d98883 100644 --- a/vault/request_forwarding_service.pb.go +++ b/vault/request_forwarding_service.pb.go @@ -71,8 +71,9 @@ func (m *EchoRequest) GetClusterAddrs() []string { } type EchoReply struct { - Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` - ClusterAddrs []string `protobuf:"bytes,2,rep,name=cluster_addrs,json=clusterAddrs" json:"cluster_addrs,omitempty"` + Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` + ClusterAddrs []string `protobuf:"bytes,2,rep,name=cluster_addrs,json=clusterAddrs" json:"cluster_addrs,omitempty"` + ReplicationState uint32 `protobuf:"varint,3,opt,name=replication_state,json=replicationState" json:"replication_state,omitempty"` } func (m *EchoReply) Reset() { *m = EchoReply{} } @@ -94,6 +95,13 @@ func (m *EchoReply) GetClusterAddrs() []string { return nil } +func (m *EchoReply) GetReplicationState() uint32 { + if m != nil { + return m.ReplicationState + } + return 0 +} + func init() { proto.RegisterType((*EchoRequest)(nil), "vault.EchoRequest") proto.RegisterType((*EchoReply)(nil), "vault.EchoReply") @@ -207,22 +215,23 @@ var _RequestForwarding_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("request_forwarding_service.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 261 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0x3f, 0x4f, 0xc3, 0x30, - 0x10, 0xc5, 0x9b, 0x96, 0x3f, 0x8a, 0x5b, 0x10, 0x18, 0x86, 0x28, 0x53, 0x08, 0x4b, 0x27, 0x47, - 0x82, 0x85, 0x85, 0x81, 0x01, 0x06, 0xc6, 0x7c, 0x81, 0x28, 0xb5, 0x8f, 0x38, 0x92, 0x5b, 0x9b, - 0x3b, 0xa7, 0x28, 0x2b, 0x9f, 0x1c, 0x91, 0xa4, 0x34, 0x55, 0x25, 0xc6, 0x7b, 0x77, 0xfa, 0xbd, - 0x7b, 0x8f, 0x25, 0x08, 0x9f, 0x0d, 0x90, 0x2f, 0x3e, 0x2c, 0x7e, 0x95, 0xa8, 0xea, 0x4d, 0x55, - 0x10, 0xe0, 0xb6, 0x96, 0x20, 0x1c, 0x5a, 0x6f, 0xf9, 0xe9, 0xb6, 0x6c, 0x8c, 0x8f, 0x9f, 0xaa, - 0xda, 0xeb, 0x66, 0x25, 0xa4, 0x5d, 0x67, 0xba, 0x24, 0x5d, 0x4b, 0x8b, 0x2e, 0xeb, 0x76, 0x99, - 0x06, 0xe3, 0x00, 0xb3, 0x3d, 0x22, 0xf3, 0xad, 0x03, 0xea, 0x01, 0xa9, 0x65, 0xf3, 0x57, 0xa9, - 0x6d, 0xde, 0x1b, 0xf1, 0x88, 0x9d, 0xaf, 0x81, 0xa8, 0xac, 0x20, 0x0a, 0x92, 0x60, 0x19, 0xe6, - 0xbb, 0x91, 0xdf, 0xb1, 0x85, 0x34, 0x0d, 0x79, 0xc0, 0xa2, 0x54, 0x0a, 0xa3, 0x69, 0xb7, 0x9e, - 0x0f, 0xda, 0x8b, 0x52, 0xc8, 0xef, 0xd9, 0xc5, 0xf8, 0x84, 0xa2, 0x59, 0x32, 0x5b, 0x86, 0xf9, - 0x62, 0x74, 0x43, 0xe9, 0x3b, 0x0b, 0x7b, 0x43, 0x67, 0xda, 0x7f, 0xec, 0x8e, 0x58, 0xd3, 0x63, - 0xd6, 0xc3, 0x77, 0xc0, 0xae, 0x87, 0xcf, 0xdf, 0xfe, 0xe2, 0xf1, 0x67, 0x76, 0x39, 0x4c, 0xbb, - 0x54, 0x37, 0x62, 0x9f, 0x5e, 0x0c, 0x62, 0x7c, 0x7b, 0x28, 0x92, 0xb3, 0x1b, 0x82, 0x74, 0xc2, - 0x05, 0x3b, 0xf9, 0x7d, 0x90, 0x73, 0xd1, 0xf5, 0x27, 0x46, 0xf5, 0xc4, 0x57, 0x07, 0x9a, 0x33, - 0x6d, 0x3a, 0x59, 0x9d, 0x75, 0x45, 0x3e, 0xfe, 0x04, 0x00, 0x00, 0xff, 0xff, 0xc0, 0xa1, 0xca, - 0xfe, 0xad, 0x01, 0x00, 0x00, + // 287 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0xbf, 0x4e, 0xc3, 0x30, + 0x10, 0xc6, 0x9b, 0x96, 0x3f, 0xaa, 0xdb, 0xa2, 0xd6, 0x30, 0x44, 0x99, 0x42, 0x58, 0x22, 0x21, + 0x39, 0x12, 0x2c, 0x2c, 0x0c, 0x0c, 0xf0, 0x00, 0xe1, 0x01, 0x22, 0xd7, 0x39, 0x12, 0x4b, 0x6e, + 0x6c, 0x7c, 0x4e, 0xab, 0xac, 0x3c, 0x39, 0x6a, 0x92, 0xd2, 0x54, 0x95, 0x18, 0xef, 0x77, 0xa7, + 0xef, 0xd3, 0xfd, 0x48, 0x68, 0xe1, 0xbb, 0x06, 0x74, 0xd9, 0x97, 0xb6, 0x3b, 0x6e, 0x73, 0x59, + 0x15, 0x19, 0x82, 0xdd, 0x4a, 0x01, 0xcc, 0x58, 0xed, 0x34, 0xbd, 0xdc, 0xf2, 0x5a, 0xb9, 0xe0, + 0xa5, 0x90, 0xae, 0xac, 0xd7, 0x4c, 0xe8, 0x4d, 0x52, 0x72, 0x2c, 0xa5, 0xd0, 0xd6, 0x24, 0xed, + 0x2e, 0x29, 0x41, 0x19, 0xb0, 0xc9, 0x31, 0x22, 0x71, 0x8d, 0x01, 0xec, 0x02, 0x22, 0x4d, 0x66, + 0xef, 0xa2, 0xd4, 0x69, 0x57, 0x44, 0x7d, 0x72, 0xbd, 0x01, 0x44, 0x5e, 0x80, 0xef, 0x85, 0x5e, + 0x3c, 0x4d, 0x0f, 0x23, 0xbd, 0x27, 0x73, 0xa1, 0x6a, 0x74, 0x60, 0x33, 0x9e, 0xe7, 0xd6, 0x1f, + 0xb7, 0xeb, 0x59, 0xcf, 0xde, 0xf2, 0xdc, 0xd2, 0x07, 0xb2, 0x18, 0x9e, 0xa0, 0x3f, 0x09, 0x27, + 0xf1, 0x34, 0x9d, 0x0f, 0x6e, 0x30, 0xda, 0x91, 0x69, 0x57, 0x68, 0x54, 0xf3, 0x4f, 0xdd, 0x59, + 0xd6, 0xf8, 0x3c, 0x8b, 0x3e, 0x92, 0x95, 0x05, 0xa3, 0xa4, 0xe0, 0x4e, 0xea, 0x2a, 0x43, 0xc7, + 0x1d, 0xf8, 0x93, 0xd0, 0x8b, 0x17, 0xe9, 0x72, 0xb0, 0xf8, 0xdc, 0xf3, 0xa7, 0x1f, 0x8f, 0xac, + 0xfa, 0x37, 0x3f, 0xfe, 0x5c, 0xd0, 0x57, 0x72, 0xd3, 0x4f, 0x07, 0x05, 0xb7, 0xec, 0xa8, 0x8a, + 0xf5, 0x30, 0xb8, 0x3b, 0x85, 0x68, 0x74, 0x85, 0x10, 0x8d, 0x28, 0x23, 0x17, 0xfb, 0x6f, 0x28, + 0x65, 0xad, 0x6c, 0x36, 0x70, 0x19, 0x2c, 0x4f, 0x98, 0x51, 0x4d, 0x34, 0x5a, 0x5f, 0xb5, 0xd6, + 0x9f, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x94, 0x1d, 0xe9, 0x21, 0xda, 0x01, 0x00, 0x00, } diff --git a/vault/request_forwarding_service.proto b/vault/request_forwarding_service.proto index b62ef0c9fa6f..2a012c56c193 100644 --- a/vault/request_forwarding_service.proto +++ b/vault/request_forwarding_service.proto @@ -17,6 +17,7 @@ message EchoRequest { message EchoReply { string message = 1; repeated string cluster_addrs = 2; + uint32 replication_state = 3; } service RequestForwarding {