From a036eae90a95c1d1f330ffb4f0391b9750b2efa0 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Wed, 12 Jun 2024 14:26:19 -0700 Subject: [PATCH 01/26] refactor: add rotator component --- config/config_field_types.go | 16 ++++++++- internal/credential/credential.go | 3 ++ internal/credential/rotator.go | 51 ++++++++++++++++++++++++++++ internal/credential/rotator_test.go | 1 + internal/sharedtest/testdata_envs.go | 2 ++ 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 internal/credential/rotator.go create mode 100644 internal/credential/rotator_test.go diff --git a/config/config_field_types.go b/config/config_field_types.go index c7ecc563..7e562ba6 100644 --- a/config/config_field_types.go +++ b/config/config_field_types.go @@ -44,6 +44,13 @@ type FilterKey string // DefaultFilter represents the lack of a filter, meaning a full LaunchDarkly environment. const DefaultFilter = FilterKey("") +func last4Chars(s string) string { + if len(s) < 4 { // COVERAGE: doesn't happen in unit tests, also can't happen with real environments + return s + } + return s[len(s)-4:] +} + // GetAuthorizationHeaderValue for SDKKey returns the same string, since SDK keys are passed in // the Authorization header. func (k SDKKey) GetAuthorizationHeaderValue() string { @@ -57,7 +64,7 @@ func (k SDKKey) Defined() bool { func (k SDKKey) String() string { return string(k) } - +func (k SDKKey) Masked() string { return last4Chars(k.String()) } func (k SDKKey) Compare(cr credential.AutoConfig) (credential.SDKCredential, credential.Status) { if cr.SDKKey == k { return nil, credential.Unchanged @@ -87,6 +94,8 @@ func (k MobileKey) String() string { return string(k) } +func (k MobileKey) Masked() string { return last4Chars(k.String()) } + func (k MobileKey) Compare(cr credential.AutoConfig) (credential.SDKCredential, credential.Status) { if cr.MobileKey == k { return nil, credential.Unchanged @@ -108,6 +117,9 @@ func (k EnvironmentID) String() string { return string(k) } +// Masked is an alias for String(), because EnvironmentIDs are considered non-sensitive public information. +func (k EnvironmentID) Masked() string { return k.String() } + func (k EnvironmentID) Compare(_ credential.AutoConfig) (credential.SDKCredential, credential.Status) { // EnvironmentIDs should not change. return nil, credential.Unchanged @@ -129,6 +141,8 @@ func (k AutoConfigKey) String() string { return string(k) } +func (k AutoConfigKey) Masked() string { return last4Chars(string(k)) } + // UnmarshalText allows the SDKKey type to be set from environment variables. func (k *SDKKey) UnmarshalText(data []byte) error { *k = SDKKey(string(data)) diff --git a/internal/credential/credential.go b/internal/credential/credential.go index 14dd270c..019e0a1f 100644 --- a/internal/credential/credential.go +++ b/internal/credential/credential.go @@ -14,6 +14,9 @@ type SDKCredential interface { // Compare accepts a collection of AutoConfig credentials and inspects it, determining if this credential has // changed in any way. If so, it should return the new credential and a status. Compare(creds AutoConfig) (SDKCredential, Status) + + // Masked returns a masked form of the credential suitable for log messages. + Masked() string } // Status represents that difference between an existing credential and one found in a new AutoConfig configuration diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go new file mode 100644 index 00000000..417ad478 --- /dev/null +++ b/internal/credential/rotator.go @@ -0,0 +1,51 @@ +package credential + +import ( + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "time" +) + +type deprecatedCred struct { + timer *time.Timer + expiry time.Time + expired bool +} + +type Rotator struct { + loggers ldlog.Loggers + timers map[SDKCredential]*deprecatedCred + expirations chan SDKCredential +} + +func NewRotator(loggers ldlog.Loggers) *Rotator { + r := &Rotator{ + loggers: loggers, + timers: make(map[SDKCredential]*deprecatedCred), + expirations: make(chan SDKCredential), + } + return r +} + +func (r *Rotator) Expirations() <-chan SDKCredential { + return r.expirations +} + +func (r *Rotator) Deprecated(cred SDKCredential) bool { + _, ok := r.timers[cred] + return ok +} + +func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time) { + if existing, ok := r.timers[cred]; ok { + r.loggers.Warnf("Credential %s was marked for deprececation with an expiry time of %v, but it previously expired at %v", cred.Masked(), expiry, existing.expiry) + return + } + r.loggers.Infof("Credential %s has been marked for deprecation with an expiry time of %v", cred.Masked(), expiry) + state := &deprecatedCred{expired: false} + state.timer = time.AfterFunc(expiry.Sub(time.Now()), func() { + r.loggers.Info("Credential %s has expired", cred.Masked()) + r.expirations <- cred + state.expired = true + }) + r.timers[cred] = state +} diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go new file mode 100644 index 00000000..9aa93961 --- /dev/null +++ b/internal/credential/rotator_test.go @@ -0,0 +1 @@ +package credential diff --git a/internal/sharedtest/testdata_envs.go b/internal/sharedtest/testdata_envs.go index fee23a66..ce90dead 100644 --- a/internal/sharedtest/testdata_envs.go +++ b/internal/sharedtest/testdata_envs.go @@ -38,6 +38,8 @@ func (k UnsupportedSDKCredential) String() string { return "unsupported" } +func (k UnsupportedSDKCredential) Masked() string { return "unsupported" } + const ( // The "undefined" values are well-formed, but do not match any environment in our test data. UndefinedSDKKey = config.SDKKey("sdk-99999999-9999-4999-8999-999999999999") From 6c29d1785d99b8c8e624901c31fff6f868d60195 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Wed, 12 Jun 2024 14:43:10 -0700 Subject: [PATCH 02/26] tests --- internal/credential/rotator.go | 28 +++++++++++++---- internal/credential/rotator_test.go | 47 +++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 417ad478..4d22a6fa 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -15,13 +15,15 @@ type Rotator struct { loggers ldlog.Loggers timers map[SDKCredential]*deprecatedCred expirations chan SDKCredential + now func() time.Time } -func NewRotator(loggers ldlog.Loggers) *Rotator { +func NewRotator(loggers ldlog.Loggers, now func() time.Time) *Rotator { r := &Rotator{ loggers: loggers, timers: make(map[SDKCredential]*deprecatedCred), expirations: make(chan SDKCredential), + now: now, } return r } @@ -35,17 +37,31 @@ func (r *Rotator) Deprecated(cred SDKCredential) bool { return ok } -func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time) { +func (r *Rotator) Expired(cred SDKCredential) bool { + if state, ok := r.timers[cred]; ok { + return state.expired + } + return false +} + +func (r *Rotator) Stop() { + for _, state := range r.timers { + state.timer.Stop() + } +} + +func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time) bool { if existing, ok := r.timers[cred]; ok { - r.loggers.Warnf("Credential %s was marked for deprececation with an expiry time of %v, but it previously expired at %v", cred.Masked(), expiry, existing.expiry) - return + r.loggers.Warnf("Credential %s was marked for deprecation with an expiry time of %v, but it previously expired at %v", cred.Masked(), expiry, existing.expiry) + return false } r.loggers.Infof("Credential %s has been marked for deprecation with an expiry time of %v", cred.Masked(), expiry) state := &deprecatedCred{expired: false} - state.timer = time.AfterFunc(expiry.Sub(time.Now()), func() { + state.timer = time.AfterFunc(expiry.Sub(r.now()), func() { r.loggers.Info("Credential %s has expired", cred.Masked()) - r.expirations <- cred state.expired = true + r.expirations <- cred }) r.timers[cred] = state + return true } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 9aa93961..d165cf2d 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -1 +1,48 @@ package credential + +import ( + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +type testCred string + +func (k testCred) Compare(_ AutoConfig) (SDKCredential, Status) { + return nil, Unchanged +} + +func (k testCred) GetAuthorizationHeaderValue() string { return "" } + +func (k testCred) Defined() bool { + return true +} + +func (k testCred) String() string { + return string(k) +} + +func (k testCred) Masked() string { return "masked<" + string(k) + ">" } + +func TestNewRotator(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers, time.Now) + assert.NotNil(t, rotator) +} + +func TestDeprecation(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + + now := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + future := now.Add(24 * time.Hour) + + rotator := NewRotator(mockLog.Loggers, func() time.Time { return now }) + defer rotator.Stop() + + cred := testCred("foobar-1234") + + assert.True(t, rotator.Deprecate(cred, future)) + assert.True(t, rotator.Deprecated(cred)) + assert.False(t, rotator.Expired(cred)) +} From 373c5392c991ff6d66de6e38269b73193b308ec1 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Thu, 13 Jun 2024 08:43:10 -0700 Subject: [PATCH 03/26] in progress --- internal/credential/rotator.go | 4 ++-- internal/envfactory/env_params.go | 16 ++++++++++++---- internal/envfactory/env_rep.go | 17 ++++++++++++----- internal/relayenv/env_context.go | 2 +- internal/relayenv/env_context_impl.go | 5 ++++- relay/autoconfig_actions.go | 8 ++++---- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 4d22a6fa..3f8e3625 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -50,7 +50,7 @@ func (r *Rotator) Stop() { } } -func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time) bool { +func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time, onExpiry func(credential SDKCredential)) bool { if existing, ok := r.timers[cred]; ok { r.loggers.Warnf("Credential %s was marked for deprecation with an expiry time of %v, but it previously expired at %v", cred.Masked(), expiry, existing.expiry) return false @@ -60,7 +60,7 @@ func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time) bool { state.timer = time.AfterFunc(expiry.Sub(r.now()), func() { r.loggers.Info("Credential %s has expired", cred.Masked()) state.expired = true - r.expirations <- cred + onExpiry(cred) }) r.timers[cred] = state return true diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index 318e62fa..addcf9c4 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -29,9 +29,8 @@ type EnvironmentParams struct { MobileKey config.MobileKey // ExpiringSDKKey is an additional SDK key that should also be allowed (but not surfaced as - // the canonical one), or "" if none. The expiry time is not represented here; it is managed - // by lower-level components. - ExpiringSDKKey config.SDKKey + // the canonical one). + ExpiringSDKKey ExpiringSDKKey // TTL is the cache TTL for PHP clients. TTL time.Duration @@ -40,10 +39,19 @@ type EnvironmentParams struct { SecureMode bool } +type ExpiringSDKKey struct { + Key config.SDKKey + Expiration time.Time +} + +func (e ExpiringSDKKey) Defined() bool { + return e.Key.Defined() +} + func (e EnvironmentParams) Credentials() credential.AutoConfig { return credential.AutoConfig{ SDKKey: e.SDKKey, - ExpiringSDKKey: e.ExpiringSDKKey, + ExpiringSDKKey: e.ExpiringSDKKey.Key, MobileKey: e.MobileKey, } } diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index a0d39aab..f1605df8 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -62,6 +62,10 @@ type ExpiringKeyRep struct { Timestamp ldtime.UnixMillisecondTime `json:"timestamp"` } +func ToTime(millisecondTime ldtime.UnixMillisecondTime) time.Time { + return time.UnixMilli(int64(millisecondTime)) +} + // ToParams converts the JSON properties for an environment into our internal parameter type. func (r EnvironmentRep) ToParams() EnvironmentParams { return EnvironmentParams{ @@ -72,11 +76,14 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { ProjKey: r.ProjKey, ProjName: r.ProjName, }, - SDKKey: r.SDKKey.Value, - MobileKey: r.MobKey, - ExpiringSDKKey: r.SDKKey.Expiring.Value, - TTL: time.Duration(r.DefaultTTL) * time.Minute, - SecureMode: r.SecureMode, + SDKKey: r.SDKKey.Value, + MobileKey: r.MobKey, + ExpiringSDKKey: ExpiringSDKKey{ + Key: r.SDKKey.Expiring.Value, + Expiration: ToTime(r.SDKKey.Expiring.Timestamp), + }, + TTL: time.Duration(r.DefaultTTL) * time.Minute, + SecureMode: r.SecureMode, } } diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index d89b4e2b..e2a75e29 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -60,7 +60,7 @@ type EnvContext interface { // DeprecateCredential marks an existing credential as not being a preferred one, without removing it or // dropping any connections. It will no longer be included in the return value of GetCredentials(). This is // used in Relay Proxy Enterprise when an SDK key is being changed but the old key has not expired yet. - DeprecateCredential(credential.SDKCredential) + DeprecateCredential(cred credential.SDKCredential, when time.Time) // GetClient returns the SDK client instance for this environment. This is nil if initialization is not yet // complete. Rather than providing the full client object, we use the simpler sdks.LDClientContext which diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index b9debe3c..d0653b9c 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -105,6 +105,7 @@ type envContextImpl struct { initErr error creationTime time.Time filterKey config.FilterKey + keyRotator *credential.Rotator } // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of @@ -181,6 +182,7 @@ func NewEnvContext( dataStoreInfo: params.DataStoreInfo, creationTime: time.Now(), filterKey: params.EnvConfig.FilterKey, + keyRotator: credential.NewRotator(params.Loggers, time.Now), } bigSegmentStoreFactory := params.BigSegmentStoreFactory @@ -519,11 +521,12 @@ func (c *envContextImpl) RemoveCredential(oldCredential credential.SDKCredential } } -func (c *envContextImpl) DeprecateCredential(credential credential.SDKCredential) { +func (c *envContextImpl) DeprecateCredential(credential credential.SDKCredential, when time.Time) { c.mu.Lock() defer c.mu.Unlock() if _, found := c.credentials[credential]; found { c.credentials[credential] = false + c.keyRotator.Deprecate(credential, when, c.RemoveCredential) } } diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index 97b8d1d8..b728ba26 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -34,10 +34,10 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar } if params.ExpiringSDKKey.Defined() { - if _, err := a.r.getEnvironment(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey)); err != nil { - env.AddCredential(params.ExpiringSDKKey) - env.DeprecateCredential(params.ExpiringSDKKey) - a.r.addConnectionMapping(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey), env) + if _, err := a.r.getEnvironment(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key)); err != nil { + env.AddCredential(params.ExpiringSDKKey.Key) + env.DeprecateCredential(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration) + a.r.addConnectionMapping(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key), env) } } } From ee67b775d3db81905fb4e01ea2cde54433696a17 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Fri, 14 Jun 2024 11:11:43 -0700 Subject: [PATCH 04/26] more refactoring --- internal/autoconfig/message_handler.go | 5 - internal/autoconfig/stream_manager.go | 18 +- .../stream_manager_expiring_key_test.go | 219 +++++++++--------- internal/credential/rotator.go | 15 +- internal/credential/store.go | 29 +++ internal/envfactory/env_rep_test.go | 9 +- internal/projmanager/environment_manager.go | 8 - .../projmanager/environment_manager_test.go | 28 --- internal/projmanager/project_router.go | 9 - internal/relayenv/env_context.go | 9 +- internal/relayenv/env_context_impl.go | 70 ++++-- internal/relayenv/env_context_impl_test.go | 25 +- relay/autoconfig_actions.go | 53 ++--- relay/autoconfig_actions_test.go | 2 +- relay/autoconfig_key_change_test.go | 2 +- 15 files changed, 265 insertions(+), 236 deletions(-) create mode 100644 internal/credential/store.go diff --git a/internal/autoconfig/message_handler.go b/internal/autoconfig/message_handler.go index 864c1a03..15b5a007 100644 --- a/internal/autoconfig/message_handler.go +++ b/internal/autoconfig/message_handler.go @@ -26,11 +26,6 @@ type MessageHandler interface { // message, or a "put" that no longer contains that environment. DeleteEnvironment(id config.EnvironmentID) - // KeyExpired is called when a key that was previously provided in EnvironmentParams.ExpiringSDKKey - // has now expired. Relay should disconnect any clients currently using that key and reject any - // future requests that use it. - KeyExpired(id config.EnvironmentID, projKey string, oldKey config.SDKKey) - // AddFilter is called whenever a new filter should be added, either in a "put" or "patch" message. AddFilter(params envfactory.FilterParams) diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 079030a4..2d82e3f1 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -307,12 +307,6 @@ func (s *StreamManager) consumeStream(stream *es.Stream) { if shouldRestart { stream.Restart() } - - case expiredKey := <-s.expiredKeys: - s.loggers.Warnf(logMsgKeyExpired, last4Chars(string(expiredKey.key)), expiredKey.envID, - makeEnvName(s.lastKnownEnvs[expiredKey.envID])) - s.handler.KeyExpired(expiredKey.envID, expiredKey.projKey, expiredKey.key) - case <-s.halt: stream.Close() for _, t := range s.expiryTimers { @@ -329,17 +323,17 @@ func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactor return case ActionInsert: params := rep.ToParams() - if s.IgnoreExpiringSDKKey(rep) { - params.ExpiringSDKKey = "" - } + //if s.IgnoreExpiringSDKKey(rep) { + // params.ExpiringSDKKey = "" + //} s.handler.AddEnvironment(params) case ActionDelete: s.handler.DeleteEnvironment(id) case ActionUpdate: params := rep.ToParams() - if s.IgnoreExpiringSDKKey(rep) { - params.ExpiringSDKKey = "" - } + //if s.IgnoreExpiringSDKKey(rep) { + // params.ExpiringSDKKey = "" + //} s.handler.UpdateEnvironment(params) } } diff --git a/internal/autoconfig/stream_manager_expiring_key_test.go b/internal/autoconfig/stream_manager_expiring_key_test.go index 67077fef..cd2ccfc9 100644 --- a/internal/autoconfig/stream_manager_expiring_key_test.go +++ b/internal/autoconfig/stream_manager_expiring_key_test.go @@ -1,8 +1,6 @@ package autoconfig import ( - "testing" - "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" @@ -52,111 +50,112 @@ func expectNoKeyExpiryMessage(p streamManagerTestParams) { p.mockLog.AssertMessageMatch(p.t, false, ldlog.Warn, "Old SDK key .* will expire") } -func TestExpiringKeyInPutMessage(t *testing.T) { - envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) - event := makeEnvPutEvent(envWithExpiringKey) - streamManagerTest(t, &event, func(p streamManagerTestParams) { - p.startStream() - - msg := p.requireMessage() - require.NotNil(t, msg.add) - p.requireReceivedAllMessage() - - assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) - assert.Equal(t, oldKey, msg.add.ExpiringSDKKey) - - expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) - }) -} - -func TestExpiringKeyInPatchAdd(t *testing.T) { - envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) - event := makePatchEnvEvent(envWithExpiringKey) - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(event) - - msg := p.requireMessage() - require.NotNil(t, msg.add) - - assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) - assert.Equal(t, oldKey, msg.add.ExpiringSDKKey) - - expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) - }) -} - -func TestExpiringKeyInPatchUpdate(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(makePatchEnvEvent(testEnv1)) - - _ = p.requireMessage() - - envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) - envWithExpiringKey.Version++ - - p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) - - msg := p.requireMessage() - require.NotNil(t, msg.update) - assert.Equal(t, envWithExpiringKey.ToParams(), *msg.update) - assert.Equal(t, oldKey, msg.update.ExpiringSDKKey) - - expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) - }) -} - -func TestExpiringKeyHasAlreadyExpiredInPutMessage(t *testing.T) { - envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) - event := makeEnvPutEvent(envWithExpiringKey) - streamManagerTest(t, &event, func(p streamManagerTestParams) { - p.startStream() - - msg := p.requireMessage() - require.NotNil(t, msg.add) - p.requireReceivedAllMessage() - - assert.Equal(t, testEnv1.ToParams(), *msg.add) - assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey) - - expectNoKeyExpiryMessage(p) - }) -} - -func TestExpiringKeyHasAlreadyExpiredInPatchAdd(t *testing.T) { - envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) - event := makePatchEnvEvent(envWithExpiringKey) - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(event) - - msg := p.requireMessage() - require.NotNil(t, msg.add) - assert.Equal(t, testEnv1.ToParams(), *msg.add) - assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey) - - expectNoKeyExpiryMessage(p) - }) -} - -func TestExpiringKeyHasAlreadyExpiredInPatchUpdate(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(makePatchEnvEvent(testEnv1)) - - _ = p.requireMessage() - - envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) - envWithExpiringKey.Version++ - - p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) - - msg := p.requireMessage() - require.NotNil(t, msg.update) - assert.Equal(t, testEnv1.ToParams(), *msg.update) - assert.Equal(t, config.SDKKey(""), msg.update.ExpiringSDKKey) - - expectNoKeyExpiryMessage(p) - }) -} +// +//func TestExpiringKeyInPutMessage(t *testing.T) { +// envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) +// event := makeEnvPutEvent(envWithExpiringKey) +// streamManagerTest(t, &event, func(p streamManagerTestParams) { +// p.startStream() +// +// msg := p.requireMessage() +// require.NotNil(t, msg.add) +// p.requireReceivedAllMessage() +// +// assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) +// assert.Equal(t, oldKey, msg.add.ExpiringSDKKey.Key) +// +// expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) +// }) +//} +// +//func TestExpiringKeyInPatchAdd(t *testing.T) { +// envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) +// event := makePatchEnvEvent(envWithExpiringKey) +// streamManagerTest(t, nil, func(p streamManagerTestParams) { +// p.startStream() +// p.stream.Enqueue(event) +// +// msg := p.requireMessage() +// require.NotNil(t, msg.add) +// +// assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) +// assert.Equal(t, oldKey, msg.add.ExpiringSDKKey.Key) +// +// expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) +// }) +//} +// +//func TestExpiringKeyInPatchUpdate(t *testing.T) { +// streamManagerTest(t, nil, func(p streamManagerTestParams) { +// p.startStream() +// p.stream.Enqueue(makePatchEnvEvent(testEnv1)) +// +// _ = p.requireMessage() +// +// envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) +// envWithExpiringKey.Version++ +// +// p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) +// +// msg := p.requireMessage() +// require.NotNil(t, msg.update) +// assert.Equal(t, envWithExpiringKey.ToParams(), *msg.update) +// assert.Equal(t, oldKey, msg.update.ExpiringSDKKey.Key) +// +// expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) +// }) +//} +// +//func TestExpiringKeyHasAlreadyExpiredInPutMessage(t *testing.T) { +// envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) +// event := makeEnvPutEvent(envWithExpiringKey) +// streamManagerTest(t, &event, func(p streamManagerTestParams) { +// p.startStream() +// +// msg := p.requireMessage() +// require.NotNil(t, msg.add) +// p.requireReceivedAllMessage() +// +// assert.Equal(t, testEnv1.ToParams(), *msg.add) +// assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey.Key) +// +// expectNoKeyExpiryMessage(p) +// }) +//} +// +//func TestExpiringKeyHasAlreadyExpiredInPatchAdd(t *testing.T) { +// envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) +// event := makePatchEnvEvent(envWithExpiringKey) +// streamManagerTest(t, nil, func(p streamManagerTestParams) { +// p.startStream() +// p.stream.Enqueue(event) +// +// msg := p.requireMessage() +// require.NotNil(t, msg.add) +// assert.Equal(t, testEnv1.ToParams(), *msg.add) +// assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey.Key) +// +// expectNoKeyExpiryMessage(p) +// }) +//} +// +//func TestExpiringKeyHasAlreadyExpiredInPatchUpdate(t *testing.T) { +// streamManagerTest(t, nil, func(p streamManagerTestParams) { +// p.startStream() +// p.stream.Enqueue(makePatchEnvEvent(testEnv1)) +// +// _ = p.requireMessage() +// +// envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) +// envWithExpiringKey.Version++ +// +// p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) +// +// msg := p.requireMessage() +// require.NotNil(t, msg.update) +// assert.Equal(t, testEnv1.ToParams(), *msg.update) +// assert.Equal(t, config.SDKKey(""), msg.update.ExpiringSDKKey.Key) +// +// expectNoKeyExpiryMessage(p) +// }) +//} diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 3f8e3625..52d415b1 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -25,6 +25,9 @@ func NewRotator(loggers ldlog.Loggers, now func() time.Time) *Rotator { expirations: make(chan SDKCredential), now: now, } + if r.now == nil { + r.now = time.Now + } return r } @@ -50,18 +53,24 @@ func (r *Rotator) Stop() { } } -func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time, onExpiry func(credential SDKCredential)) bool { +func (r *Rotator) DeprecateFunc(cred SDKCredential, expiry time.Time, onExpiry func(credential SDKCredential)) bool { if existing, ok := r.timers[cred]; ok { r.loggers.Warnf("Credential %s was marked for deprecation with an expiry time of %v, but it previously expired at %v", cred.Masked(), expiry, existing.expiry) return false } - r.loggers.Infof("Credential %s has been marked for deprecation with an expiry time of %v", cred.Masked(), expiry) + //r.loggers.Infof("Credential %s has been marked for deprecation with an expiry time of %v", cred.Masked(), expiry) + r.loggers.Infof("Old SDK key ending in %s will expire", cred.Masked()) state := &deprecatedCred{expired: false} state.timer = time.AfterFunc(expiry.Sub(r.now()), func() { - r.loggers.Info("Credential %s has expired", cred.Masked()) + //r.loggers.Infof("Credential %s has expired", cred.Masked()) + r.loggers.Infof("Old SDK key ending in %s has expired", cred.Masked()) state.expired = true onExpiry(cred) }) r.timers[cred] = state return true } + +func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time) bool { + return r.DeprecateFunc(cred, expiry, func(credential SDKCredential) {}) +} diff --git a/internal/credential/store.go b/internal/credential/store.go new file mode 100644 index 00000000..cef69e49 --- /dev/null +++ b/internal/credential/store.go @@ -0,0 +1,29 @@ +package credential + +import ( + "github.com/launchdarkly/ld-relay/v8/config" + "time" +) + +type rotatedKey struct { + key config.SDKKey + expired bool + expiry time.Time +} + +func (r *rotatedKey) deprecated() bool { + return !r.expiry.IsZero() +} + +func (r *rotatedKey) preferred() bool { + return !r.deprecated() +} + +type Store struct { + // Can be rotated. The tail of this list is the active key. + mobileKeys []config.MobileKey + // Can be rotated, with a deprecation period. The tail of this list is the preferred key. + sdkKeys []config.SDKKey + // Can never change + envID config.EnvironmentID +} diff --git a/internal/envfactory/env_rep_test.go b/internal/envfactory/env_rep_test.go index 5df5687e..b46e7566 100644 --- a/internal/envfactory/env_rep_test.go +++ b/internal/envfactory/env_rep_test.go @@ -64,9 +64,12 @@ func TestEnvironmentRepToParams(t *testing.T) { ProjKey: "projkey2", ProjName: "projname2", }, - SDKKey: env2.SDKKey.Value, - ExpiringSDKKey: env2.SDKKey.Expiring.Value, - MobileKey: env2.MobKey, + SDKKey: env2.SDKKey.Value, + ExpiringSDKKey: ExpiringSDKKey{ + Key: env2.SDKKey.Expiring.Value, + Expiration: time.UnixMilli(int64(env2.SDKKey.Expiring.Timestamp)), + }, + MobileKey: env2.MobKey, }, params2) } diff --git a/internal/projmanager/environment_manager.go b/internal/projmanager/environment_manager.go index b02fcc4c..4529d592 100644 --- a/internal/projmanager/environment_manager.go +++ b/internal/projmanager/environment_manager.go @@ -12,7 +12,6 @@ type EnvironmentActions interface { AddEnvironment(params envfactory.EnvironmentParams) UpdateEnvironment(params envfactory.EnvironmentParams) DeleteEnvironment(id config.EnvironmentID, filter config.FilterKey) - KeyExpired(id config.EnvironmentID, filter config.FilterKey, oldKey config.SDKKey) } type filterMapping struct { @@ -132,13 +131,6 @@ func (e *EnvironmentManager) DeleteFilter(filter config.FilterID) bool { return true } -func (e *EnvironmentManager) KeyExpired(id config.EnvironmentID, oldKey config.SDKKey) { - e.handler.KeyExpired(id, config.DefaultFilter, oldKey) - for _, filter := range e.filtered { - e.handler.KeyExpired(id, filter.key, oldKey) - } -} - func (e *EnvironmentManager) Filters() []config.FilterKey { filters := make([]config.FilterKey, 0, len(e.filtered)) for _, filter := range e.filtered { diff --git a/internal/projmanager/environment_manager_test.go b/internal/projmanager/environment_manager_test.go index 12f28780..08253f86 100644 --- a/internal/projmanager/environment_manager_test.go +++ b/internal/projmanager/environment_manager_test.go @@ -426,34 +426,6 @@ func TestEnvironmentManager_SimpleFilterCombination(t *testing.T) { require.ElementsMatchf(t, out, []config.EnvironmentID{"a", "b", "a/foo", "a/bar", "b/foo", "b/bar"}, "default and filtered environments should be created") } -func TestEnvironmentManager_KeyExpired(t *testing.T) { - t.Run("key expiry is broadcast n times", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - mockLog.Loggers.SetMinLevel(ldlog.Debug) - - for i := 0; i < 10; i++ { - spy := newHandlerSpy() - m := NewEnvironmentManager("foo", spy, mockLog.Loggers) - - filters := makeFilters(i, []string{"foo"}) - expected := []expiredParams{{id: "foo", filter: config.DefaultFilter, key: "sdk-123"}} - for _, f := range filters { - expected = append(expected, expiredParams{ - id: "foo", - filter: f.Key, - key: "sdk-123", - }) - m.AddFilter(f) - } - - m.KeyExpired("foo", "sdk-123") - require.ElementsMatch(t, spy.expired, expected) - } - }) - -} - type command struct { op commandType value string diff --git a/internal/projmanager/project_router.go b/internal/projmanager/project_router.go index 75046deb..276cb278 100644 --- a/internal/projmanager/project_router.go +++ b/internal/projmanager/project_router.go @@ -41,15 +41,6 @@ func NewProjectRouter(handler AutoConfigActions, loggers ldlog.Loggers) *Project return &ProjectRouter{managers: make(map[string]*EnvironmentManager), actions: handler, loggers: loggers} } -// KeyExpired indicates that an SDK key, scoped to a particular project, has expired. The command -// is forwarded on to the manager, if any, for the given projKey. -func (e *ProjectRouter) KeyExpired(id config.EnvironmentID, projKey string, oldKey config.SDKKey) { - manager, ok := e.managers[projKey] - if ok { - manager.KeyExpired(id, oldKey) - } -} - // AddEnvironment routes the given EnvironmentParams to the relevant ProjectManager based on its project key, or instantiates // a new ProjectManager if one doesn't already exist. func (e *ProjectRouter) AddEnvironment(params envfactory.EnvironmentParams) { diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index e2a75e29..52429594 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -20,6 +20,11 @@ import ( ldeval "github.com/launchdarkly/go-server-sdk-evaluation/v3" ) +type DeprecationHooks struct { + BeforeRemoval func(cred credential.SDKCredential) + AfterRemoval func(cred credential.SDKCredential) +} + // EnvContext is the interface for all Relay operations that are specific to one configured LD environment. // // The EnvContext is normally associated with an LDClient instance from the Go SDK, and allows direct access @@ -51,6 +56,8 @@ type EnvContext interface { // to server-side endpoints is switched to use the new key. AddCredential(credential.SDKCredential) + RotateCredential(new credential.SDKCredential, old credential.SDKCredential, expiry time.Time) + // RemoveCredential removes a credential from the environment. Any active stream connections using that // credential are immediately dropped. // @@ -60,7 +67,7 @@ type EnvContext interface { // DeprecateCredential marks an existing credential as not being a preferred one, without removing it or // dropping any connections. It will no longer be included in the return value of GetCredentials(). This is // used in Relay Proxy Enterprise when an SDK key is being changed but the old key has not expired yet. - DeprecateCredential(cred credential.SDKCredential, when time.Time) + DeprecateCredential(cred credential.SDKCredential, when time.Time, hooks *DeprecationHooks) // GetClient returns the SDK client instance for this environment. This is nil if initialization is not yet // complete. Rather than providing the full client object, we use the simpler sdks.LDClientContext which diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index d0653b9c..368dcffa 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "reflect" "sync" "time" @@ -73,14 +74,22 @@ type EnvContextImplParams struct { UserAgent string LogNameMode LogNameMode Loggers ldlog.Loggers + TimeSource func() time.Time } +type credStatus string + +const ( + preferred = credStatus("preferred") + deprecated = credStatus("deprecated") +) + type envContextImpl struct { mu sync.RWMutex clients map[config.SDKKey]sdks.LDClientContext storeAdapter *store.SSERelayDataStoreAdapter loggers ldlog.Loggers - credentials map[credential.SDKCredential]bool // true if not deprecated + credentials map[credential.SDKCredential]credStatus identifiers EnvIdentifiers secureMode bool envStreams *streams.EnvStreams @@ -132,8 +141,8 @@ type envContextStreamUpdates struct { func NewEnvContext( params EnvContextImplParams, readyCh chan<- EnvContext, - // readyCh is a separate parameter because it's not a property of the environment itself, but - // just part of the semantics of the constructor +// readyCh is a separate parameter because it's not a property of the environment itself, but +// just part of the semantics of the constructor ) (EnvContext, error) { var thingsToCleanUp util.CleanupTasks // keeps track of partially constructed things in case we exit early defer thingsToCleanUp.Run() @@ -156,13 +165,13 @@ func NewEnvContext( return nil, err } - credentials := make(map[credential.SDKCredential]bool, 3) - credentials[envConfig.SDKKey] = true + credentials := make(map[credential.SDKCredential]credStatus, 3) + credentials[envConfig.SDKKey] = preferred if envConfig.MobileKey.Defined() { - credentials[envConfig.MobileKey] = true + credentials[envConfig.MobileKey] = preferred } if envConfig.EnvID.Defined() { - credentials[envConfig.EnvID] = true + credentials[envConfig.EnvID] = preferred } envContext := &envContextImpl{ @@ -182,7 +191,7 @@ func NewEnvContext( dataStoreInfo: params.DataStoreInfo, creationTime: time.Now(), filterKey: params.EnvConfig.FilterKey, - keyRotator: credential.NewRotator(params.Loggers, time.Now), + keyRotator: credential.NewRotator(params.Loggers, params.TimeSource), } bigSegmentStoreFactory := params.BigSegmentStoreFactory @@ -449,20 +458,20 @@ func (c *envContextImpl) SetIdentifiers(ei EnvIdentifiers) { } func (c *envContextImpl) GetCredentials() []credential.SDKCredential { - return c.getCredentialsInternal(true) + return c.getCredentialsInternal(preferred) } func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { - return c.getCredentialsInternal(false) + return c.getCredentialsInternal(deprecated) } -func (c *envContextImpl) getCredentialsInternal(preferred bool) []credential.SDKCredential { +func (c *envContextImpl) getCredentialsInternal(status credStatus) []credential.SDKCredential { c.mu.RLock() defer c.mu.RUnlock() ret := make([]credential.SDKCredential, 0, len(c.credentials)) - for c, nonDeprecated := range c.credentials { - if nonDeprecated == preferred { + for c, s := range c.credentials { + if s == status { ret = append(ret, c) } } @@ -475,7 +484,7 @@ func (c *envContextImpl) AddCredential(newCredential credential.SDKCredential) { if _, found := c.credentials[newCredential]; found { return } - c.credentials[newCredential] = true + c.credentials[newCredential] = preferred c.envStreams.AddCredential(newCredential) for streamProvider, handlers := range c.handlers { if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, newCredential)); h != nil { @@ -521,12 +530,33 @@ func (c *envContextImpl) RemoveCredential(oldCredential credential.SDKCredential } } -func (c *envContextImpl) DeprecateCredential(credential credential.SDKCredential, when time.Time) { +func (c *envContextImpl) RotateCredential(new credential.SDKCredential) { + for cred, _ := range c.credentials { + if reflect.TypeOf(cred) == reflect.TypeOf(new) { + + } + } + return c.RotateCredential(new, old, time.Now()) +} +func (c *envContextImpl) RotateCredential(new credential.SDKCredential, old credential.SDKCredential, expiry time.Time, hooks *DeprecationHooks) { + c.AddCredential(new) + c.DeprecateCredential(old, expiry, hooks) +} + +func (c *envContextImpl) DeprecateCredential(cred credential.SDKCredential, when time.Time, hooks *DeprecationHooks) { c.mu.Lock() defer c.mu.Unlock() - if _, found := c.credentials[credential]; found { - c.credentials[credential] = false - c.keyRotator.Deprecate(credential, when, c.RemoveCredential) + if _, found := c.credentials[cred]; found { + c.credentials[cred] = deprecated + c.keyRotator.DeprecateFunc(cred, when, func(cred credential.SDKCredential) { + if hooks != nil && hooks.BeforeRemoval != nil { + hooks.BeforeRemoval(cred) + } + c.RemoveCredential(cred) + if hooks != nil && hooks.AfterRemoval != nil { + hooks.AfterRemoval(cred) + } + }) } } @@ -535,8 +565,8 @@ func (c *envContextImpl) GetClient() sdks.LDClientContext { defer c.mu.RUnlock() // There might be multiple clients if there's an expiring SDK key. Find the SDK key that has a true // value in our map (meaning it's not deprecated) and return that client. - for cred, valid := range c.credentials { - if sdkKey, ok := cred.(config.SDKKey); ok && valid { + for cred, status := range c.credentials { + if sdkKey, ok := cred.(config.SDKKey); ok && status == preferred { return c.clients[sdkKey] } } diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index e285eb00..2cdc1f50 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -56,16 +56,21 @@ func requireClientReady(t *testing.T, clientCh chan *testclient.FakeLDClient) *t func makeBasicEnv(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.ClientFactoryFunc, loggers ldlog.Loggers, readyCh chan EnvContext) EnvContext { + return makeBasicEnvWithMockTime(t, envConfig, clientFactory, loggers, readyCh, nil) +} + +func makeBasicEnvWithMockTime(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.ClientFactoryFunc, + loggers ldlog.Loggers, readyCh chan EnvContext, now func() time.Time) EnvContext { env, err := NewEnvContext(EnvContextImplParams{ Identifiers: EnvIdentifiers{ConfiguredName: envName}, EnvConfig: envConfig, ClientFactory: clientFactory, Loggers: loggers, + TimeSource: now, }, readyCh) require.NoError(t, err) return env } - func TestConstructorBasicProperties(t *testing.T) { envConfig := st.EnvWithAllCredentials.Config envConfig.TTL = configtypes.NewOptDuration(time.Hour) @@ -223,6 +228,10 @@ func TestChangeSDKKey(t *testing.T) { readyCh := make(chan EnvContext, 1) newKey := config.SDKKey("new-key") + const deprecationDelay = 100 * time.Millisecond + const clientInitializationDelay = 10 * time.Millisecond + const clientCloseDelay = deprecationDelay / 2 + clientCh := make(chan *testclient.FakeLDClient, 1) clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) @@ -237,24 +246,30 @@ func TestChangeSDKKey(t *testing.T) { assert.Equal(t, env.GetClient(), client1) assert.Nil(t, env.GetInitError()) + removed := make(chan credential.SDKCredential, 1) env.AddCredential(newKey) - env.DeprecateCredential(envConfig.SDKKey) + env.DeprecateCredential(envConfig.SDKKey, time.Now().Add(deprecationDelay), &DeprecationHooks{AfterRemoval: func(cred credential.SDKCredential) { + removed <- cred + }}) assert.Equal(t, []credential.SDKCredential{newKey}, env.GetCredentials()) client2 := requireClientReady(t, clientCh) assert.NotEqual(t, client1, client2) + + time.Sleep(clientInitializationDelay) assert.Equal(t, env.GetClient(), client2) - if !helpers.AssertNoMoreValues(t, client1.CloseCh, time.Millisecond*20, "client for deprecated key should not have been closed") { + if !helpers.AssertNoMoreValues(t, client1.CloseCh, clientCloseDelay, "client for deprecated key should not have been closed") { t.FailNow() } - env.RemoveCredential(envConfig.SDKKey) + deprecatedCred := helpers.RequireValue(t, removed, deprecationDelay, "timed out waiting for deprecation") + assert.Equal(t, envConfig.SDKKey, deprecatedCred) assert.Equal(t, []credential.SDKCredential{newKey}, env.GetCredentials()) - client1.AwaitClose(t, time.Millisecond*20) + client1.AwaitClose(t, clientCloseDelay) } func TestSDKClientCreationFails(t *testing.T) { diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index b728ba26..c995324c 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -4,6 +4,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" ) @@ -35,9 +36,15 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar if params.ExpiringSDKKey.Defined() { if _, err := a.r.getEnvironment(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key)); err != nil { + auth := sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key) env.AddCredential(params.ExpiringSDKKey.Key) - env.DeprecateCredential(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration) - a.r.addConnectionMapping(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key), env) + env.DeprecateCredential(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration, &relayenv.DeprecationHooks{ + BeforeRemoval: func(_ credential.SDKCredential) { + a.r.removeConnectionMapping(auth) + }, + }) + + a.r.addConnectionMapping(auth, env) } } } @@ -53,24 +60,20 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment env.SetTTL(params.TTL) env.SetSecureMode(params.SecureMode) - newCredentials := params.Credentials() - - for _, prevCred := range env.GetCredentials() { - newCred, status := prevCred.Compare(newCredentials) - if status == credential.Unchanged { - continue - } - - env.AddCredential(newCred) - a.r.addConnectionMapping(sdkauth.NewScoped(params.Identifiers.FilterKey, newCred), env) - - switch status { - case credential.Deprecated: - env.DeprecateCredential(prevCred) - case credential.Expired: - a.r.removeConnectionMapping(sdkauth.NewScoped(params.Identifiers.FilterKey, prevCred)) - env.RemoveCredential(prevCred) - } + // Ok the problem is this. + // If we get a new key and there's no deprecation. It's just rotating instantly. We need to remove the old + // key instantly. Can we treat + if params.MobileKey.Defined() { + env.AddCredential(params.MobileKey) + } + if params.SDKKey.Defined() { + env.AddCredential(params.SDKKey) + } + if params.ExpiringSDKKey.Defined() { + env.DeprecateCredential(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration, &relayenv.DeprecationHooks{ + BeforeRemoval: func(cred credential.SDKCredential) { + a.r.removeConnectionMapping(sdkauth.NewScoped(params.Identifiers.FilterKey, cred)) + }}) } } @@ -85,13 +88,3 @@ func (a *relayAutoConfigActions) ReceivedAllEnvironments() { a.r.loggers.Info(logMsgAutoConfReceivedAllEnvironments) a.r.setFullyConfigured(true) } - -func (a *relayAutoConfigActions) KeyExpired(id config.EnvironmentID, filter config.FilterKey, oldKey config.SDKKey) { - env, err := a.r.getEnvironment(sdkauth.NewScoped(filter, id)) - if err != nil { - a.r.loggers.Warnf(logMsgKeyExpiryUnknownEnv, id) - return - } - a.r.removeConnectionMapping(sdkauth.NewScoped(filter, oldKey)) - env.RemoveCredential(oldKey) -} diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index 87a8c137..f3e7c714 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -87,7 +87,7 @@ func autoConfTest( } func (p autoConfTestParams) awaitClient() *testclient.FakeLDClient { - return helpers.RequireValue(p.t, p.clientsCreatedCh, time.Second, "timed out waiting for client creation") + return helpers.RequireValue(p.t, p.clientsCreatedCh, 1000*time.Second, "timed out waiting for client creation") } func (p autoConfTestParams) shouldNotCreateClient(timeout time.Duration) { diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index a81409a7..536457d8 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -89,7 +89,7 @@ func TestAutoConfigUpdateEnvironmentSDKKeyWithNoExpiry(t *testing.T) { client2 := p.awaitClient() assert.Equal(t, modified.sdkKey, client2.Key) - client1.AwaitClose(t, time.Second) + client1.AwaitClose(t, 10000*time.Second) p.awaitCredentialsUpdated(env, modified.params()) noEnv, _ := p.relay.getEnvironment(sdkauth.New(testAutoConfEnv1.sdkKey)) From 7f96b6bc64d07f6a960a12ebc4845643c0dc6b89 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 18 Jun 2024 12:23:08 -0700 Subject: [PATCH 05/26] plumbing key rotator --- internal/credential/rotator.go | 155 ++++++++++++---- internal/relayenv/env_context.go | 17 +- internal/relayenv/env_context_impl.go | 253 +++++++++++++------------- relay/autoconfig_actions.go | 18 +- 4 files changed, 253 insertions(+), 190 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 52d415b1..08ad3d2a 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -2,27 +2,54 @@ package credential import ( "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/ld-relay/v8/config" + "sync" "time" ) -type deprecatedCred struct { - timer *time.Timer - expiry time.Time - expired bool +type DeprecationNotice struct { + key config.SDKKey + expiry time.Time +} + +func NewDeprecationNotice(key config.SDKKey, expiry time.Time) *DeprecationNotice { + return &DeprecationNotice{key: key, expiry: expiry} +} + +type deprecatedKey struct { + expiry time.Time + timer *time.Timer } type Rotator struct { - loggers ldlog.Loggers - timers map[SDKCredential]*deprecatedCred + loggers ldlog.Loggers + + // There is only one mobile key active at a given time; it does not support a deprecation period. + primaryMobileKey config.MobileKey + + // There can be multiple SDK keys active at a given time, but only one is primary. + primarySdkKey config.SDKKey + + envID config.EnvironmentID + + // Deprecated keys are stored in a map with a started timer for each key representing the deprecation period. + // Upon expiration, they are removed. + deprecatedSdkKeys map[config.SDKKey]*deprecatedKey + expirations chan SDKCredential + additions chan SDKCredential now func() time.Time + + mu sync.RWMutex } -func NewRotator(loggers ldlog.Loggers, now func() time.Time) *Rotator { +func NewRotator(loggers ldlog.Loggers, envID config.EnvironmentID, now func() time.Time) *Rotator { r := &Rotator{ loggers: loggers, - timers: make(map[SDKCredential]*deprecatedCred), + deprecatedSdkKeys: make(map[config.SDKKey]*deprecatedKey), + envID: envID, expirations: make(chan SDKCredential), + additions: make(chan SDKCredential), now: now, } if r.now == nil { @@ -35,42 +62,100 @@ func (r *Rotator) Expirations() <-chan SDKCredential { return r.expirations } -func (r *Rotator) Deprecated(cred SDKCredential) bool { - _, ok := r.timers[cred] - return ok + +func (r *Rotator) Additions() <- chan SDKCredential { + return r.additions +} + +func (r *Rotator) MobileKey() config.MobileKey { + r.mu.RLock() + defer r.mu.RUnlock() + return r.primaryMobileKey +} + +func (r *Rotator) SDKKey() config.SDKKey { + r.mu.RLock() + defer r.mu.RUnlock() + return r.primarySdkKey } -func (r *Rotator) Expired(cred SDKCredential) bool { - if state, ok := r.timers[cred]; ok { - return state.expired + +func (r *Rotator) PrimaryCredentials() []SDKCredential { + r.mu.RLock() + defer r.mu.RUnlock() + return []SDKCredential{ + r.primarySdkKey, + r.primaryMobileKey, + r.envID, } - return false } -func (r *Rotator) Stop() { - for _, state := range r.timers { - state.timer.Stop() +func (r *Rotator) DeprecatedCredentials() []SDKCredential { + r.mu.RLock() + defer r.mu.RUnlock() + var deprecated []SDKCredential + for key := range r.deprecatedSdkKeys { + deprecated = append(deprecated, key) } + return deprecated } -func (r *Rotator) DeprecateFunc(cred SDKCredential, expiry time.Time, onExpiry func(credential SDKCredential)) bool { - if existing, ok := r.timers[cred]; ok { - r.loggers.Warnf("Credential %s was marked for deprecation with an expiry time of %v, but it previously expired at %v", cred.Masked(), expiry, existing.expiry) - return false +func (r *Rotator) RotateMobileKey(mobileKey config.MobileKey) { + if mobileKey == r.MobileKey() { + return + } + r.mu.Lock() + defer r.mu.Unlock() + r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", r.primaryMobileKey.Masked(), mobileKey.Masked() + previous := r.primaryMobileKey + r.primaryMobileKey = mobileKey + r.expirations <- previous +} + +func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNotice) { + // An SDK key can arrive with an optional deprecation notice for a previous key. + // If there's no deprecation notice, this is an immediate rotation: the new key is the primary key, the old + // one is removed. + // If there is a deprecation notice, we need to move the old key to the deprecated state and start a timer. + // Some gotchas because of the design of the data model: + // (1) It's possible to receive a notice that names an SDK key that is not the current primary key. + // It could be a key that was already deprecated, a key that was expired, or some key we've never seen. + // Since we need to make a decision on how to handle it, it shall be: + // - If already deprecated (meaning it has a timer), ignore it and log a warning. + // - If unseen/expired (can't distinguish since we don't retain it), ignore it and log a warning. + if sdkKey == r.SDKKey() { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if deprecation == nil { + r.loggers.Infof("SDK key %s was rotated, new primary SDK key is %s", r.primarySdkKey.Masked(), sdkKey.Masked()) + previous := r.primarySdkKey + r.primarySdkKey = sdkKey + r.expirations <- previous + return + } + if old, ok := r.deprecatedSdkKeys[deprecation.key]; ok { + r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", deprecation.key.Masked(), deprecation.expiry, old.expiry) + return + } + if deprecation.key == r.primarySdkKey { + r.loggers.Infof("SDK key %s was rotated with an expiry at %v, new primary SDK key is %s", r.primarySdkKey.Masked(), deprecation.expiry, sdkKey.Masked()) + r.primarySdkKey = sdkKey + r.deprecatedSdkKeys[deprecation.key] = &deprecatedKey{ + expiry: deprecation.expiry, + timer: time.AfterFunc(deprecation.expiry.Sub(r.now()), func() { + r.expireSDKKey(deprecation.key) + })} + return } - //r.loggers.Infof("Credential %s has been marked for deprecation with an expiry time of %v", cred.Masked(), expiry) - r.loggers.Infof("Old SDK key ending in %s will expire", cred.Masked()) - state := &deprecatedCred{expired: false} - state.timer = time.AfterFunc(expiry.Sub(r.now()), func() { - //r.loggers.Infof("Credential %s has expired", cred.Masked()) - r.loggers.Infof("Old SDK key ending in %s has expired", cred.Masked()) - state.expired = true - onExpiry(cred) - }) - r.timers[cred] = state - return true + r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but this key is not recognized by Relay. It may have already expired; ignoring.", deprecation.key.Masked(), deprecation.expiry) } -func (r *Rotator) Deprecate(cred SDKCredential, expiry time.Time) bool { - return r.DeprecateFunc(cred, expiry, func(credential SDKCredential) {}) +func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { + r.loggers.Infof("Deprecated SDK key %s has expired and is no longer valid for authentication", sdkKey.Masked()) + r.mu.Lock() + defer r.mu.Unlock() + delete(r.deprecatedSdkKeys, sdkKey) + r.expirations <- sdkKey } diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index 52429594..2ea0ae2a 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -3,6 +3,7 @@ package relayenv import ( "context" "fmt" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "io" "net/http" "time" @@ -50,19 +51,9 @@ type EnvContext interface { // GetDeprecatedCredentials returns all deprecated and not-yet-removed credentials for the environment. GetDeprecatedCredentials() []credential.SDKCredential - // AddCredential adds a new credential for the environment. - // - // If the credential is an SDK key, then a new SDK client is started with that SDK key, and event forwarding - // to server-side endpoints is switched to use the new key. - AddCredential(credential.SDKCredential) - - RotateCredential(new credential.SDKCredential, old credential.SDKCredential, expiry time.Time) - - // RemoveCredential removes a credential from the environment. Any active stream connections using that - // credential are immediately dropped. - // - // If the credential is an SDK key, then the SDK client that we started with that SDK key is disposed of. - RemoveCredential(credential.SDKCredential) + RotateMobileKey(key config.MobileKey) + RotateSDKKey(key config.SDKKey) + RotateAndDeprecateSDKKey(newKey config.SDKKey, oldKey envfactory.ExpiringSDKKey) // DeprecateCredential marks an existing credential as not being a preferred one, without removing it or // dropping any connections. It will no longer be included in the return value of GetCredentials(). This is diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 368dcffa..d1c118ef 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -3,8 +3,8 @@ package relayenv import ( "context" "fmt" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "net/http" - "reflect" "sync" "time" @@ -85,36 +85,37 @@ const ( ) type envContextImpl struct { - mu sync.RWMutex - clients map[config.SDKKey]sdks.LDClientContext - storeAdapter *store.SSERelayDataStoreAdapter - loggers ldlog.Loggers - credentials map[credential.SDKCredential]credStatus - identifiers EnvIdentifiers - secureMode bool - envStreams *streams.EnvStreams - streamProviders []streams.StreamProvider - handlers map[streams.StreamProvider]map[credential.SDKCredential]http.Handler - jsContext JSClientContext - evaluator ldeval.Evaluator - eventDispatcher *events.EventDispatcher - bigSegmentSync bigsegments.BigSegmentSynchronizer - bigSegmentStore bigsegments.BigSegmentStore - bigSegmentsExist bool - sdkBigSegments *ldstoreimpl.BigSegmentStoreWrapper - sdkConfig ld.Config - sdkClientFactory sdks.ClientFactoryFunc - sdkInitTimeout time.Duration - metricsManager *metrics.Manager - metricsEnv *metrics.EnvironmentManager - metricsEventPub events.EventPublisher - dataStoreInfo sdks.DataStoreEnvironmentInfo - globalLoggers ldlog.Loggers - ttl time.Duration - initErr error - creationTime time.Time - filterKey config.FilterKey - keyRotator *credential.Rotator + mu sync.RWMutex + clients map[config.SDKKey]sdks.LDClientContext + storeAdapter *store.SSERelayDataStoreAdapter + loggers ldlog.Loggers + identifiers EnvIdentifiers + secureMode bool + envStreams *streams.EnvStreams + streamProviders []streams.StreamProvider + handlers map[streams.StreamProvider]map[credential.SDKCredential]http.Handler + jsContext JSClientContext + evaluator ldeval.Evaluator + eventDispatcher *events.EventDispatcher + bigSegmentSync bigsegments.BigSegmentSynchronizer + bigSegmentStore bigsegments.BigSegmentStore + bigSegmentsExist bool + sdkBigSegments *ldstoreimpl.BigSegmentStoreWrapper + sdkConfig ld.Config + sdkClientFactory sdks.ClientFactoryFunc + sdkInitTimeout time.Duration + metricsManager *metrics.Manager + metricsEnv *metrics.EnvironmentManager + metricsEventPub events.EventPublisher + dataStoreInfo sdks.DataStoreEnvironmentInfo + globalLoggers ldlog.Loggers + ttl time.Duration + initErr error + creationTime time.Time + filterKey config.FilterKey + keyRotator *credential.Rotator + stopMonitoringCredentials chan struct{} + doneMonitoringCredentials chan struct{} } // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of @@ -141,8 +142,8 @@ type envContextStreamUpdates struct { func NewEnvContext( params EnvContextImplParams, readyCh chan<- EnvContext, -// readyCh is a separate parameter because it's not a property of the environment itself, but -// just part of the semantics of the constructor + // readyCh is a separate parameter because it's not a property of the environment itself, but + // just part of the semantics of the constructor ) (EnvContext, error) { var thingsToCleanUp util.CleanupTasks // keeps track of partially constructed things in case we exit early defer thingsToCleanUp.Run() @@ -165,34 +166,33 @@ func NewEnvContext( return nil, err } - credentials := make(map[credential.SDKCredential]credStatus, 3) - credentials[envConfig.SDKKey] = preferred + envContext := &envContextImpl{ + identifiers: params.Identifiers, + clients: make(map[config.SDKKey]sdks.LDClientContext), + loggers: envLoggers, + secureMode: envConfig.SecureMode, + streamProviders: params.StreamProviders, + handlers: make(map[streams.StreamProvider]map[credential.SDKCredential]http.Handler), + jsContext: params.JSClientContext, + sdkClientFactory: params.ClientFactory, + sdkInitTimeout: allConfig.Main.InitTimeout.GetOrElse(config.DefaultInitTimeout), + metricsManager: params.MetricsManager, + globalLoggers: params.Loggers, + ttl: envConfig.TTL.GetOrElse(0), + dataStoreInfo: params.DataStoreInfo, + creationTime: time.Now(), + filterKey: params.EnvConfig.FilterKey, + keyRotator: credential.NewRotator(params.Loggers, envConfig.EnvID, params.TimeSource), + stopMonitoringCredentials: make(chan struct{}), + doneMonitoringCredentials: make(chan struct{}), + } + + envContext.keyRotator.RotateSDKKey(envConfig.SDKKey, nil) if envConfig.MobileKey.Defined() { - credentials[envConfig.MobileKey] = preferred - } - if envConfig.EnvID.Defined() { - credentials[envConfig.EnvID] = preferred + envContext.keyRotator.RotateMobileKey(envConfig.MobileKey) } - envContext := &envContextImpl{ - identifiers: params.Identifiers, - clients: make(map[config.SDKKey]sdks.LDClientContext), - credentials: credentials, - loggers: envLoggers, - secureMode: envConfig.SecureMode, - streamProviders: params.StreamProviders, - handlers: make(map[streams.StreamProvider]map[credential.SDKCredential]http.Handler), - jsContext: params.JSClientContext, - sdkClientFactory: params.ClientFactory, - sdkInitTimeout: allConfig.Main.InitTimeout.GetOrElse(config.DefaultInitTimeout), - metricsManager: params.MetricsManager, - globalLoggers: params.Loggers, - ttl: envConfig.TTL.GetOrElse(0), - dataStoreInfo: params.DataStoreInfo, - creationTime: time.Now(), - filterKey: params.EnvConfig.FilterKey, - keyRotator: credential.NewRotator(params.Loggers, params.TimeSource), - } + go envContext.monitorCredentialChanges() bigSegmentStoreFactory := params.BigSegmentStoreFactory if bigSegmentStoreFactory == nil { @@ -393,6 +393,54 @@ func NewEnvContext( return envContext, nil } +func (c *envContextImpl) monitorCredentialChanges() { + defer close(c.doneMonitoringCredentials) + for { + select { + case <-c.stopMonitoringCredentials: + return + case oldCredential := <-c.keyRotator.Expirations(): + c.envStreams.RemoveCredential(oldCredential) + for _, handlers := range c.handlers { + delete(handlers, oldCredential) + } + if sdkKey, ok := oldCredential.(config.SDKKey); ok { + // The SDK client instance is tied to the SDK key, so get rid of it + if client := c.clients[sdkKey]; client != nil { + delete(c.clients, sdkKey) + _ = client.Close() + } + } + case newCredential := <-c.keyRotator.Additions(): + c.envStreams.AddCredential(newCredential) + for streamProvider, handlers := range c.handlers { + if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, newCredential)); h != nil { + handlers[newCredential] = h + } + } + + // A new SDK key means 1. we should start a new SDK client, 2. we should tell all event forwarding + // components that use an SDK key to use the new one. A new mobile key does not require starting a + // new SDK client, but does requiring updating any event forwarding components that use a mobile key. + switch key := newCredential.(type) { + case config.SDKKey: + go c.startSDKClient(key, nil, false) + if c.metricsEventPub != nil { // metrics event publisher always uses SDK key + c.metricsEventPub.ReplaceCredential(key) + } + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(key) + } + case config.MobileKey: + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(key) + } + } + + } + } +} + func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- EnvContext, suppressErrors bool) { client, err := c.sdkClientFactory(sdkKey, c.sdkConfig, c.sdkInitTimeout) c.mu.Lock() @@ -458,57 +506,23 @@ func (c *envContextImpl) SetIdentifiers(ei EnvIdentifiers) { } func (c *envContextImpl) GetCredentials() []credential.SDKCredential { - return c.getCredentialsInternal(preferred) + return c.keyRotator.PrimaryCredentials() } func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { - return c.getCredentialsInternal(deprecated) + return c.keyRotator.DeprecatedCredentials() } -func (c *envContextImpl) getCredentialsInternal(status credStatus) []credential.SDKCredential { - c.mu.RLock() - defer c.mu.RUnlock() - - ret := make([]credential.SDKCredential, 0, len(c.credentials)) - for c, s := range c.credentials { - if s == status { - ret = append(ret, c) - } - } - return ret +func (c *envContextImpl) RotateMobileKey(key config.MobileKey) { + c.keyRotator.RotateMobileKey(key) } -func (c *envContextImpl) AddCredential(newCredential credential.SDKCredential) { - c.mu.Lock() - defer c.mu.Unlock() - if _, found := c.credentials[newCredential]; found { - return - } - c.credentials[newCredential] = preferred - c.envStreams.AddCredential(newCredential) - for streamProvider, handlers := range c.handlers { - if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, newCredential)); h != nil { - handlers[newCredential] = h - } - } +func (c *envContextImpl) RotateSDKKey(key config.SDKKey) { + c.keyRotator.RotateSDKKey(key, nil) +} - // A new SDK key means 1. we should start a new SDK client, 2. we should tell all event forwarding - // components that use an SDK key to use the new one. A new mobile key does not require starting a - // new SDK client, but does requiring updating any event forwarding components that use a mobile key. - switch key := newCredential.(type) { - case config.SDKKey: - go c.startSDKClient(key, nil, false) - if c.metricsEventPub != nil { // metrics event publisher always uses SDK key - c.metricsEventPub.ReplaceCredential(key) - } - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) - } - case config.MobileKey: - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) - } - } +func (c *envContextImpl) RotateAndDeprecateSDKKey(newKey config.SDKKey, oldKey envfactory.ExpiringSDKKey) { + c.keyRotator.RotateSDKKey(newKey, credential.NewDeprecationNotice(oldKey.Key, oldKey.Expiration)) } func (c *envContextImpl) RemoveCredential(oldCredential credential.SDKCredential) { @@ -530,36 +544,6 @@ func (c *envContextImpl) RemoveCredential(oldCredential credential.SDKCredential } } -func (c *envContextImpl) RotateCredential(new credential.SDKCredential) { - for cred, _ := range c.credentials { - if reflect.TypeOf(cred) == reflect.TypeOf(new) { - - } - } - return c.RotateCredential(new, old, time.Now()) -} -func (c *envContextImpl) RotateCredential(new credential.SDKCredential, old credential.SDKCredential, expiry time.Time, hooks *DeprecationHooks) { - c.AddCredential(new) - c.DeprecateCredential(old, expiry, hooks) -} - -func (c *envContextImpl) DeprecateCredential(cred credential.SDKCredential, when time.Time, hooks *DeprecationHooks) { - c.mu.Lock() - defer c.mu.Unlock() - if _, found := c.credentials[cred]; found { - c.credentials[cred] = deprecated - c.keyRotator.DeprecateFunc(cred, when, func(cred credential.SDKCredential) { - if hooks != nil && hooks.BeforeRemoval != nil { - hooks.BeforeRemoval(cred) - } - c.RemoveCredential(cred) - if hooks != nil && hooks.AfterRemoval != nil { - hooks.AfterRemoval(cred) - } - }) - } -} - func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() @@ -684,6 +668,10 @@ func (c *envContextImpl) FlushMetricsEvents() { c.metricsEventPub.Flush() } } +func (c *envContextImpl) stopCredentialMonitor() { + close(c.stopMonitoringCredentials) + <-c.doneMonitoringCredentials +} func (c *envContextImpl) Close() error { c.mu.Lock() @@ -693,6 +681,9 @@ func (c *envContextImpl) Close() error { c.clients = make(map[config.SDKKey]sdks.LDClientContext) c.mu.Unlock() _ = c.envStreams.Close() + + c.stopCredentialMonitor() + if c.metricsManager != nil && c.metricsEnv != nil { c.metricsManager.RemoveEnvironment(c.metricsEnv) } diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index c995324c..ac61fe8b 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -60,21 +60,17 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment env.SetTTL(params.TTL) env.SetSecureMode(params.SecureMode) - // Ok the problem is this. - // If we get a new key and there's no deprecation. It's just rotating instantly. We need to remove the old - // key instantly. Can we treat if params.MobileKey.Defined() { - env.AddCredential(params.MobileKey) + env.RotateMobileKey(params.MobileKey) } if params.SDKKey.Defined() { - env.AddCredential(params.SDKKey) - } - if params.ExpiringSDKKey.Defined() { - env.DeprecateCredential(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration, &relayenv.DeprecationHooks{ - BeforeRemoval: func(cred credential.SDKCredential) { - a.r.removeConnectionMapping(sdkauth.NewScoped(params.Identifiers.FilterKey, cred)) - }}) + if !params.ExpiringSDKKey.Defined() { + env.RotateSDKKey(params.SDKKey) + } else { + env.RotateAndDeprecateSDKKey(params.SDKKey, params.ExpiringSDKKey) + } } + } func (a *relayAutoConfigActions) DeleteEnvironment(id config.EnvironmentID, filter config.FilterKey) { From 3669a07cfd55ce8944f6b5be3d2c378d833e213c Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 18 Jun 2024 14:21:45 -0700 Subject: [PATCH 06/26] it builds --- config/config_field_types.go | 33 --------------- internal/credential/credential.go | 4 -- internal/credential/rotator.go | 44 +++++++++++++------- internal/envfactory/env_params.go | 10 ----- internal/relayenv/env_context.go | 11 +---- internal/relayenv/env_context_impl.go | 60 ++++++++------------------- relay/autoconfig_actions.go | 20 +++------ relay/relay.go | 5 ++- 8 files changed, 55 insertions(+), 132 deletions(-) diff --git a/config/config_field_types.go b/config/config_field_types.go index 7e562ba6..fc6df3c9 100644 --- a/config/config_field_types.go +++ b/config/config_field_types.go @@ -5,8 +5,6 @@ import ( "fmt" "strings" - "github.com/launchdarkly/ld-relay/v8/internal/credential" - "github.com/launchdarkly/go-sdk-common/v3/ldlog" ) @@ -65,20 +63,6 @@ func (k SDKKey) String() string { return string(k) } func (k SDKKey) Masked() string { return last4Chars(k.String()) } -func (k SDKKey) Compare(cr credential.AutoConfig) (credential.SDKCredential, credential.Status) { - if cr.SDKKey == k { - return nil, credential.Unchanged - } - if cr.ExpiringSDKKey == k { - // If the AutoConfig update contains an ExpiringSDKKey that is equal to *this* key, then it means - // this key is now considered deprecated. - return cr.SDKKey, credential.Deprecated - } else { - // Otherwise if the AutoConfig update contains *some other* key, then it means this one must be considered - // expired. - return cr.SDKKey, credential.Expired - } -} // GetAuthorizationHeaderValue for MobileKey returns the same string, since mobile keys are passed in the // Authorization header. @@ -96,13 +80,6 @@ func (k MobileKey) String() string { func (k MobileKey) Masked() string { return last4Chars(k.String()) } -func (k MobileKey) Compare(cr credential.AutoConfig) (credential.SDKCredential, credential.Status) { - if cr.MobileKey == k { - return nil, credential.Unchanged - } - return cr.MobileKey, credential.Expired -} - // GetAuthorizationHeaderValue for EnvironmentID returns an empty string, since environment IDs are not // passed in a header but as part of the request URL. func (k EnvironmentID) GetAuthorizationHeaderValue() string { @@ -120,11 +97,6 @@ func (k EnvironmentID) String() string { // Masked is an alias for String(), because EnvironmentIDs are considered non-sensitive public information. func (k EnvironmentID) Masked() string { return k.String() } -func (k EnvironmentID) Compare(_ credential.AutoConfig) (credential.SDKCredential, credential.Status) { - // EnvironmentIDs should not change. - return nil, credential.Unchanged -} - // GetAuthorizationHeaderValue for AutoConfigKey returns the same string, since these keys are passed in // the Authorization header. Note that unlike the other kinds of authorization keys, this one is never // present in an incoming request; it is only used in requests from Relay to LaunchDarkly. @@ -132,11 +104,6 @@ func (k AutoConfigKey) GetAuthorizationHeaderValue() string { return string(k) } -func (k AutoConfigKey) Compare(_ credential.AutoConfig) (credential.SDKCredential, credential.Status) { - // AutoConfigKeys should not change. - return nil, credential.Unchanged -} - func (k AutoConfigKey) String() string { return string(k) } diff --git a/internal/credential/credential.go b/internal/credential/credential.go index 019e0a1f..c0bb8191 100644 --- a/internal/credential/credential.go +++ b/internal/credential/credential.go @@ -11,10 +11,6 @@ type SDKCredential interface { Defined() bool // String returns the string form of the credential. String() string - // Compare accepts a collection of AutoConfig credentials and inspects it, determining if this credential has - // changed in any way. If so, it should return the new credential and a status. - Compare(creds AutoConfig) (SDKCredential, Status) - // Masked returns a masked form of the credential suitable for log messages. Masked() string } diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 08ad3d2a..82848a18 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -8,7 +8,7 @@ import ( ) type DeprecationNotice struct { - key config.SDKKey + key config.SDKKey expiry time.Time } @@ -18,7 +18,7 @@ func NewDeprecationNotice(key config.SDKKey, expiry time.Time) *DeprecationNotic type deprecatedKey struct { expiry time.Time - timer *time.Timer + timer *time.Timer } type Rotator struct { @@ -37,7 +37,7 @@ type Rotator struct { deprecatedSdkKeys map[config.SDKKey]*deprecatedKey expirations chan SDKCredential - additions chan SDKCredential + additions chan SDKCredential now func() time.Time mu sync.RWMutex @@ -45,12 +45,12 @@ type Rotator struct { func NewRotator(loggers ldlog.Loggers, envID config.EnvironmentID, now func() time.Time) *Rotator { r := &Rotator{ - loggers: loggers, + loggers: loggers, deprecatedSdkKeys: make(map[config.SDKKey]*deprecatedKey), - envID: envID, - expirations: make(chan SDKCredential), - additions: make(chan SDKCredential), - now: now, + envID: envID, + expirations: make(chan SDKCredential), + additions: make(chan SDKCredential), + now: now, } if r.now == nil { r.now = time.Now @@ -62,8 +62,7 @@ func (r *Rotator) Expirations() <-chan SDKCredential { return r.expirations } - -func (r *Rotator) Additions() <- chan SDKCredential { +func (r *Rotator) Additions() <-chan SDKCredential { return r.additions } @@ -79,10 +78,13 @@ func (r *Rotator) SDKKey() config.SDKKey { return r.primarySdkKey } - func (r *Rotator) PrimaryCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() + return r.primaryCredentials() +} + +func (r *Rotator) primaryCredentials() []SDKCredential { return []SDKCredential{ r.primarySdkKey, r.primaryMobileKey, @@ -90,9 +92,7 @@ func (r *Rotator) PrimaryCredentials() []SDKCredential { } } -func (r *Rotator) DeprecatedCredentials() []SDKCredential { - r.mu.RLock() - defer r.mu.RUnlock() +func (r *Rotator) deprecatedCredentials() []SDKCredential { var deprecated []SDKCredential for key := range r.deprecatedSdkKeys { deprecated = append(deprecated, key) @@ -100,13 +100,25 @@ func (r *Rotator) DeprecatedCredentials() []SDKCredential { return deprecated } +func (r *Rotator) DeprecatedCredentials() []SDKCredential { + r.mu.RLock() + defer r.mu.RUnlock() + return r.deprecatedCredentials() +} + +func (r *Rotator) AllCredentials() []SDKCredential { + r.mu.RLock() + defer r.mu.RUnlock() + return append(r.primaryCredentials(), r.deprecatedCredentials()...) +} + func (r *Rotator) RotateMobileKey(mobileKey config.MobileKey) { if mobileKey == r.MobileKey() { return } r.mu.Lock() defer r.mu.Unlock() - r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", r.primaryMobileKey.Masked(), mobileKey.Masked() + r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", r.primaryMobileKey.Masked(), mobileKey.Masked()) previous := r.primaryMobileKey r.primaryMobileKey = mobileKey r.expirations <- previous @@ -146,7 +158,7 @@ func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNot expiry: deprecation.expiry, timer: time.AfterFunc(deprecation.expiry.Sub(r.now()), func() { r.expireSDKKey(deprecation.key) - })} + })} return } r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but this key is not recognized by Relay. It may have already expired; ignoring.", deprecation.key.Masked(), deprecation.expiry) diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index addcf9c4..d0230c3f 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -3,8 +3,6 @@ package envfactory import ( "time" - "github.com/launchdarkly/ld-relay/v8/internal/credential" - "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/relayenv" ) @@ -48,14 +46,6 @@ func (e ExpiringSDKKey) Defined() bool { return e.Key.Defined() } -func (e EnvironmentParams) Credentials() credential.AutoConfig { - return credential.AutoConfig{ - SDKKey: e.SDKKey, - ExpiringSDKKey: e.ExpiringSDKKey.Key, - MobileKey: e.MobileKey, - } -} - func (e EnvironmentParams) WithFilter(key config.FilterKey) EnvironmentParams { e.Identifiers.FilterKey = key return e diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index 2ea0ae2a..9b789933 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -3,7 +3,6 @@ package relayenv import ( "context" "fmt" - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "io" "net/http" "time" @@ -52,13 +51,7 @@ type EnvContext interface { GetDeprecatedCredentials() []credential.SDKCredential RotateMobileKey(key config.MobileKey) - RotateSDKKey(key config.SDKKey) - RotateAndDeprecateSDKKey(newKey config.SDKKey, oldKey envfactory.ExpiringSDKKey) - - // DeprecateCredential marks an existing credential as not being a preferred one, without removing it or - // dropping any connections. It will no longer be included in the return value of GetCredentials(). This is - // used in Relay Proxy Enterprise when an SDK key is being changed but the old key has not expired yet. - DeprecateCredential(cred credential.SDKCredential, when time.Time, hooks *DeprecationHooks) + RotateSDKKey(newKey config.SDKKey, notice *credential.DeprecationNotice) // GetClient returns the SDK client instance for this environment. This is nil if initialization is not yet // complete. Rather than providing the full client object, we use the simpler sdks.LDClientContext which @@ -81,7 +74,7 @@ type EnvContext interface { // have its own prefix string and, optionally, its own log level. GetLoggers() ldlog.Loggers - // GetHandler returns the HTTP handler for the specified kind of stream requests and credential for this + // GetStreamHandler returns the HTTP handler for the specified kind of stream requests and credential for this // environment. If there is none, it returns a handler for a 404 status (not nil). GetStreamHandler(streams.StreamProvider, credential.SDKCredential) http.Handler diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index d1c118ef..69436e09 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -3,7 +3,6 @@ package relayenv import ( "context" "fmt" - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "net/http" "sync" "time" @@ -56,6 +55,11 @@ func errInitMetrics(err error) error { return fmt.Errorf("failed to initialize metrics for environment: %w", err) } +type ConnectionMapper interface { + AddConnectionMapping(scopedCredential sdkauth.ScopedCredential, envContext EnvContext) + RemoveConnectionMapping(scopedCredential sdkauth.ScopedCredential) +} + // EnvContextImplParams contains the constructor parameters for NewEnvContextImpl. These have their // own type because there are a lot of them, and many are irrelevant in tests. type EnvContextImplParams struct { @@ -75,15 +79,9 @@ type EnvContextImplParams struct { LogNameMode LogNameMode Loggers ldlog.Loggers TimeSource func() time.Time + ConnectionMapper ConnectionMapper } -type credStatus string - -const ( - preferred = credStatus("preferred") - deprecated = credStatus("deprecated") -) - type envContextImpl struct { mu sync.RWMutex clients map[config.SDKKey]sdks.LDClientContext @@ -116,6 +114,7 @@ type envContextImpl struct { keyRotator *credential.Rotator stopMonitoringCredentials chan struct{} doneMonitoringCredentials chan struct{} + connectionMapper ConnectionMapper } // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of @@ -144,6 +143,7 @@ func NewEnvContext( readyCh chan<- EnvContext, // readyCh is a separate parameter because it's not a property of the environment itself, but // just part of the semantics of the constructor + // just part of the semantics of the constructor ) (EnvContext, error) { var thingsToCleanUp util.CleanupTasks // keeps track of partially constructed things in case we exit early defer thingsToCleanUp.Run() @@ -185,6 +185,7 @@ func NewEnvContext( keyRotator: credential.NewRotator(params.Loggers, envConfig.EnvID, params.TimeSource), stopMonitoringCredentials: make(chan struct{}), doneMonitoringCredentials: make(chan struct{}), + connectionMapper: params.ConnectionMapper, } envContext.keyRotator.RotateSDKKey(envConfig.SDKKey, nil) @@ -254,12 +255,13 @@ func NewEnvContext( context: envContext, } - for c := range credentials { + allCreds := envContext.keyRotator.AllCredentials() + for _, c := range allCreds { envStreams.AddCredential(c) } for _, sp := range params.StreamProviders { handlers := make(map[credential.SDKCredential]http.Handler) - for c := range credentials { + for _, c := range allCreds { h := sp.Handler(sdkauth.NewScoped(envContext.filterKey, c)) if h != nil { handlers[c] = h @@ -400,6 +402,7 @@ func (c *envContextImpl) monitorCredentialChanges() { case <-c.stopMonitoringCredentials: return case oldCredential := <-c.keyRotator.Expirations(): + c.connectionMapper.RemoveConnectionMapping(sdkauth.NewScoped(c.filterKey, oldCredential)) c.envStreams.RemoveCredential(oldCredential) for _, handlers := range c.handlers { delete(handlers, oldCredential) @@ -437,6 +440,7 @@ func (c *envContextImpl) monitorCredentialChanges() { } } + c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, newCredential), c) } } } @@ -517,44 +521,14 @@ func (c *envContextImpl) RotateMobileKey(key config.MobileKey) { c.keyRotator.RotateMobileKey(key) } -func (c *envContextImpl) RotateSDKKey(key config.SDKKey) { - c.keyRotator.RotateSDKKey(key, nil) -} - -func (c *envContextImpl) RotateAndDeprecateSDKKey(newKey config.SDKKey, oldKey envfactory.ExpiringSDKKey) { - c.keyRotator.RotateSDKKey(newKey, credential.NewDeprecationNotice(oldKey.Key, oldKey.Expiration)) -} - -func (c *envContextImpl) RemoveCredential(oldCredential credential.SDKCredential) { - c.mu.Lock() - defer c.mu.Unlock() - if _, found := c.credentials[oldCredential]; found { - delete(c.credentials, oldCredential) - c.envStreams.RemoveCredential(oldCredential) - for _, handlers := range c.handlers { - delete(handlers, oldCredential) - } - if sdkKey, ok := oldCredential.(config.SDKKey); ok { - // The SDK client instance is tied to the SDK key, so get rid of it - if client := c.clients[sdkKey]; client != nil { - delete(c.clients, sdkKey) - _ = client.Close() - } - } - } +func (c *envContextImpl) RotateSDKKey(newKey config.SDKKey, notice *credential.DeprecationNotice) { + c.keyRotator.RotateSDKKey(newKey, notice) } func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() - // There might be multiple clients if there's an expiring SDK key. Find the SDK key that has a true - // value in our map (meaning it's not deprecated) and return that client. - for cred, status := range c.credentials { - if sdkKey, ok := cred.(config.SDKKey); ok && status == preferred { - return c.clients[sdkKey] - } - } - return nil + return c.clients[c.keyRotator.SDKKey()] } func (c *envContextImpl) GetStore() subsystems.DataStore { diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index ac61fe8b..bc4708b4 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -4,7 +4,6 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" ) @@ -36,15 +35,7 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar if params.ExpiringSDKKey.Defined() { if _, err := a.r.getEnvironment(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key)); err != nil { - auth := sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key) - env.AddCredential(params.ExpiringSDKKey.Key) - env.DeprecateCredential(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration, &relayenv.DeprecationHooks{ - BeforeRemoval: func(_ credential.SDKCredential) { - a.r.removeConnectionMapping(auth) - }, - }) - - a.r.addConnectionMapping(auth, env) + env.RotateSDKKey(params.SDKKey, credential.NewDeprecationNotice(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration)) } } } @@ -64,13 +55,12 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment env.RotateMobileKey(params.MobileKey) } if params.SDKKey.Defined() { - if !params.ExpiringSDKKey.Defined() { - env.RotateSDKKey(params.SDKKey) - } else { - env.RotateAndDeprecateSDKKey(params.SDKKey, params.ExpiringSDKKey) + var deprecation *credential.DeprecationNotice + if params.ExpiringSDKKey.Defined() { + deprecation = credential.NewDeprecationNotice(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration) } + env.RotateSDKKey(params.SDKKey, deprecation) } - } func (a *relayAutoConfigActions) DeleteEnvironment(id config.EnvironmentID, filter config.FilterKey) { diff --git a/relay/relay.go b/relay/relay.go index 3f47bf21..01ce4fce 100644 --- a/relay/relay.go +++ b/relay/relay.go @@ -457,6 +457,7 @@ func (r *Relay) addEnvironment( UserAgent: r.userAgent, LogNameMode: r.envLogNameMode, Loggers: r.loggers, + ConnectionMapper: r, }, resultCh) if err != nil { return nil, nil, errNewClientContextFailed(identifiers.GetDisplayName(), err) @@ -498,7 +499,7 @@ func (r *Relay) setFullyConfigured(fullyConfigured bool) { // credential is now enabled for this EnvContext. This should be done only *after* calling // EnvContext.AddCredential() so that if the RelayCore receives an incoming request with the new // credential immediately after this, it will work. -func (r *Relay) addConnectionMapping(params sdkauth.ScopedCredential, env relayenv.EnvContext) { +func (r *Relay) AddConnectionMapping(params sdkauth.ScopedCredential, env relayenv.EnvContext) { r.envsByCredential.MapRequestParams(params, env) } @@ -506,7 +507,7 @@ func (r *Relay) addConnectionMapping(params sdkauth.ScopedCredential, env relaye // credential is no longer enabled. This should be done *before* calling EnvContext.RemoveCredential() // because RemoveCredential() disconnects all existing streams, and if a client immediately tries to // reconnect using the same credential we want it to be rejected. -func (r *Relay) removeConnectionMapping(params sdkauth.ScopedCredential) { +func (r *Relay) RemoveConnectionMapping(params sdkauth.ScopedCredential) { r.envsByCredential.UnmapRequestParams(params) } From 4b7b2baf8099f0b7b22eea990235eb86f92d4fd6 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 18 Jun 2024 18:26:15 -0700 Subject: [PATCH 07/26] updating tests --- internal/autoconfig/stream_manager.go | 6 - internal/credential/rotator.go | 137 +++++++++++++++------ internal/credential/rotator_test.go | 76 ++++++++---- internal/envfactory/env_rep.go | 22 ++-- internal/relayenv/env_context.go | 1 + internal/relayenv/env_context_impl.go | 111 ++++++++++------- internal/relayenv/env_context_impl_test.go | 66 +++++----- relay/autoconfig_actions.go | 11 +- relay/autoconfig_actions_test.go | 36 +++--- relay/autoconfig_key_change_test.go | 39 +++--- relay/autoconfig_testdata_test.go | 37 +++--- relay/relay.go | 3 +- relay/relay_environments_test.go | 4 +- 13 files changed, 336 insertions(+), 213 deletions(-) diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 2d82e3f1..722570cc 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -323,17 +323,11 @@ func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactor return case ActionInsert: params := rep.ToParams() - //if s.IgnoreExpiringSDKKey(rep) { - // params.ExpiringSDKKey = "" - //} s.handler.AddEnvironment(params) case ActionDelete: s.handler.DeleteEnvironment(id) case ActionUpdate: params := rep.ToParams() - //if s.IgnoreExpiringSDKKey(rep) { - // params.ExpiringSDKKey = "" - //} s.handler.UpdateEnvironment(params) } } diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 82848a18..d50b24a6 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -3,6 +3,7 @@ package credential import ( "github.com/launchdarkly/go-sdk-common/v3/ldlog" "github.com/launchdarkly/ld-relay/v8/config" + "slices" "sync" "time" ) @@ -27,11 +28,13 @@ type Rotator struct { // There is only one mobile key active at a given time; it does not support a deprecation period. primaryMobileKey config.MobileKey + // There is only one environment ID active at a given time, and it won't actually be rotated. The mechanism is + // here to allow setting it in a deferred manner. + primaryEnvironmentID config.EnvironmentID + // There can be multiple SDK keys active at a given time, but only one is primary. primarySdkKey config.SDKKey - envID config.EnvironmentID - // Deprecated keys are stored in a map with a started timer for each key representing the deprecation period. // Upon expiration, they are removed. deprecatedSdkKeys map[config.SDKKey]*deprecatedKey @@ -43,13 +46,18 @@ type Rotator struct { mu sync.RWMutex } -func NewRotator(loggers ldlog.Loggers, envID config.EnvironmentID, now func() time.Time) *Rotator { +type InitialCredentials struct { + SDKKey config.SDKKey + MobileKey config.MobileKey + EnvironmentID config.EnvironmentID +} + +func NewRotator(loggers ldlog.Loggers, now func() time.Time) *Rotator { r := &Rotator{ loggers: loggers, deprecatedSdkKeys: make(map[config.SDKKey]*deprecatedKey), - envID: envID, - expirations: make(chan SDKCredential), - additions: make(chan SDKCredential), + expirations: make(chan SDKCredential, 1), + additions: make(chan SDKCredential, 1), now: now, } if r.now == nil { @@ -58,6 +66,25 @@ func NewRotator(loggers ldlog.Loggers, envID config.EnvironmentID, now func() ti return r } +func (r *Rotator) Initialize(credentials []SDKCredential) { + r.mu.Lock() + defer r.mu.Unlock() + + for _, cred := range credentials { + if !cred.Defined() { + continue + } + switch cred := cred.(type) { + case config.SDKKey: + r.primarySdkKey = cred + case config.MobileKey: + r.primaryMobileKey = cred + case config.EnvironmentID: + r.primaryEnvironmentID = cred + } + } +} + func (r *Rotator) Expirations() <-chan SDKCredential { return r.expirations } @@ -78,6 +105,13 @@ func (r *Rotator) SDKKey() config.SDKKey { return r.primarySdkKey } +func (r *Rotator) EnvironmentID() config.EnvironmentID { + r.mu.RLock() + defer r.mu.RUnlock() + return r.primaryEnvironmentID + +} + func (r *Rotator) PrimaryCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() @@ -85,11 +119,13 @@ func (r *Rotator) PrimaryCredentials() []SDKCredential { } func (r *Rotator) primaryCredentials() []SDKCredential { - return []SDKCredential{ + return slices.DeleteFunc([]SDKCredential{ r.primarySdkKey, r.primaryMobileKey, - r.envID, - } + r.primaryEnvironmentID, + }, func(cred SDKCredential) bool { + return !cred.Defined() + }) } func (r *Rotator) deprecatedCredentials() []SDKCredential { @@ -112,56 +148,79 @@ func (r *Rotator) AllCredentials() []SDKCredential { return append(r.primaryCredentials(), r.deprecatedCredentials()...) } -func (r *Rotator) RotateMobileKey(mobileKey config.MobileKey) { - if mobileKey == r.MobileKey() { +func (r *Rotator) RotateEnvironmentID(envID config.EnvironmentID) { + if envID == r.EnvironmentID() { return } r.mu.Lock() defer r.mu.Unlock() - r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", r.primaryMobileKey.Masked(), mobileKey.Masked()) - previous := r.primaryMobileKey - r.primaryMobileKey = mobileKey - r.expirations <- previous + previous := r.primaryEnvironmentID + r.primaryEnvironmentID = envID + r.additions <- envID + if previous.Defined() { + r.loggers.Infof("Environment ID %s was rotated, new environment ID is %s", r.primaryEnvironmentID, envID) + r.expirations <- previous + } else { + r.loggers.Infof("New environment ID is %s", envID) + } } -func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNotice) { - // An SDK key can arrive with an optional deprecation notice for a previous key. - // If there's no deprecation notice, this is an immediate rotation: the new key is the primary key, the old - // one is removed. - // If there is a deprecation notice, we need to move the old key to the deprecated state and start a timer. - // Some gotchas because of the design of the data model: - // (1) It's possible to receive a notice that names an SDK key that is not the current primary key. - // It could be a key that was already deprecated, a key that was expired, or some key we've never seen. - // Since we need to make a decision on how to handle it, it shall be: - // - If already deprecated (meaning it has a timer), ignore it and log a warning. - // - If unseen/expired (can't distinguish since we don't retain it), ignore it and log a warning. - if sdkKey == r.SDKKey() { +func (r *Rotator) RotateMobileKey(mobileKey config.MobileKey) { + if mobileKey == r.MobileKey() { return } r.mu.Lock() defer r.mu.Unlock() - if deprecation == nil { - r.loggers.Infof("SDK key %s was rotated, new primary SDK key is %s", r.primarySdkKey.Masked(), sdkKey.Masked()) - previous := r.primarySdkKey - r.primarySdkKey = sdkKey + previous := r.primaryMobileKey + r.primaryMobileKey = mobileKey + r.additions <- mobileKey + if previous.Defined() { r.expirations <- previous - return + r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", r.primaryMobileKey.Masked(), mobileKey.Masked()) + } else { + r.loggers.Infof("New primary mobile key is %s", mobileKey.Masked()) } - if old, ok := r.deprecatedSdkKeys[deprecation.key]; ok { - r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", deprecation.key.Masked(), deprecation.expiry, old.expiry) +} + +func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { + if newKey == r.SDKKey() { + // There's no swap to be done, we already are using this as primary. + return newKey + } + previous := r.primarySdkKey + r.primarySdkKey = newKey + r.additions <- newKey + r.loggers.Infof("New primary SDK key is %s", newKey.Masked()) + + return previous +} +func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNotice) { + previous := r.swapPrimaryKey(sdkKey) + // Immediately revoke the previous SDK key if there's no explicit deprecation notice, otherwise it would + // hang around forever. + if previous.Defined() && deprecation == nil { + r.expirations <- previous + r.loggers.Infof("SDK key %s has been immediately revoked", previous.Masked()) return } - if deprecation.key == r.primarySdkKey { - r.loggers.Infof("SDK key %s was rotated with an expiry at %v, new primary SDK key is %s", r.primarySdkKey.Masked(), deprecation.expiry, sdkKey.Masked()) - r.primarySdkKey = sdkKey + if deprecation != nil { + if prev, ok := r.deprecatedSdkKeys[deprecation.key]; ok { + r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", deprecation.key.Masked(), deprecation.expiry, prev.expiry) + return + } + + r.loggers.Infof("SDK key %s was marked for deprecation with an expiry at %v", deprecation.key.Masked(), deprecation.expiry) r.deprecatedSdkKeys[deprecation.key] = &deprecatedKey{ expiry: deprecation.expiry, timer: time.AfterFunc(deprecation.expiry.Sub(r.now()), func() { r.expireSDKKey(deprecation.key) })} - return + + if deprecation.key != previous { + r.loggers.Infof("Deprecated SDK key %s was not previously managed by Relay", deprecation.key.Masked()) + r.additions <- deprecation.key + } } - r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but this key is not recognized by Relay. It may have already expired; ignoring.", deprecation.key.Masked(), deprecation.expiry) } func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index d165cf2d..cc4135e0 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -2,47 +2,75 @@ package credential import ( "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + helpers "github.com/launchdarkly/go-test-helpers/v3" + "github.com/launchdarkly/ld-relay/v8/config" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "testing" "time" ) -type testCred string +func requireChanValue(t *testing.T, ch <-chan SDKCredential, expected SDKCredential) { + t.Helper() + value := helpers.RequireValue(t, ch, 1*time.Second) + require.Equal(t, expected, value) +} -func (k testCred) Compare(_ AutoConfig) (SDKCredential, Status) { - return nil, Unchanged +func TestNewRotator(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers, time.Now) + assert.NotNil(t, rotator) } -func (k testCred) GetAuthorizationHeaderValue() string { return "" } +func TestKeyDeprecation(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers, time.Now) -func (k testCred) Defined() bool { - return true -} + const ( + key1 = config.SDKKey("key1") + key2 = config.SDKKey("key2") + key3 = config.SDKKey("key3") + ) -func (k testCred) String() string { - return string(k) -} + // The first rotation shouldn't trigger any expirations because there was no previous key. + rotator.RotateSDKKey(key1, nil) + requireChanValue(t, rotator.Additions(), key1) + assert.Equal(t, key1, rotator.SDKKey()) -func (k testCred) Masked() string { return "masked<" + string(k) + ">" } + // The second rotation should trigger a deprecation of key1. + rotator.RotateSDKKey(key2, nil) + requireChanValue(t, rotator.Additions(), key2) + requireChanValue(t, rotator.Expirations(), key1) + assert.Equal(t, key2, rotator.SDKKey()) -func TestNewRotator(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers, time.Now) - assert.NotNil(t, rotator) + // The third rotation should trigger a deprecation of key2. + rotator.RotateSDKKey(key3, nil) + requireChanValue(t, rotator.Additions(), key3) + requireChanValue(t, rotator.Expirations(), key2) + assert.Equal(t, key3, rotator.SDKKey()) } -func TestDeprecation(t *testing.T) { +func TestMobileKeyDeprecation(t *testing.T) { mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers, time.Now) - now := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - future := now.Add(24 * time.Hour) + const ( + key1 = config.MobileKey("key1") + key2 = config.MobileKey("key2") + key3 = config.MobileKey("key3") + ) - rotator := NewRotator(mockLog.Loggers, func() time.Time { return now }) - defer rotator.Stop() + rotator.RotateMobileKey(key1) + requireChanValue(t, rotator.Additions(), key1) + assert.Equal(t, key1, rotator.MobileKey()) - cred := testCred("foobar-1234") + rotator.RotateMobileKey(key2) + requireChanValue(t, rotator.Additions(), key2) + requireChanValue(t, rotator.Expirations(), key1) + assert.Equal(t, key2, rotator.MobileKey()) - assert.True(t, rotator.Deprecate(cred, future)) - assert.True(t, rotator.Deprecated(cred)) - assert.False(t, rotator.Expired(cred)) + rotator.RotateMobileKey(key3) + requireChanValue(t, rotator.Additions(), key3) + requireChanValue(t, rotator.Expirations(), key2) + assert.Equal(t, key3, rotator.MobileKey()) } diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index f1605df8..c79dc049 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -52,8 +52,8 @@ func (f FilterRep) ToTestParams() FilterParams { // SDKKeyRep describes an SDK key optionally accompanied by an old expiring key. type SDKKeyRep struct { - Value config.SDKKey `json:"value"` - Expiring ExpiringKeyRep + Value config.SDKKey `json:"value"` + Expiring ExpiringKeyRep `json:"expiring"` } // ExpiringKeyRep describes an old key that will expire at the specified date/time. @@ -68,7 +68,7 @@ func ToTime(millisecondTime ldtime.UnixMillisecondTime) time.Time { // ToParams converts the JSON properties for an environment into our internal parameter type. func (r EnvironmentRep) ToParams() EnvironmentParams { - return EnvironmentParams{ + params := EnvironmentParams{ EnvID: r.EnvID, Identifiers: relayenv.EnvIdentifiers{ EnvKey: r.EnvKey, @@ -76,15 +76,19 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { ProjKey: r.ProjKey, ProjName: r.ProjName, }, - SDKKey: r.SDKKey.Value, - MobileKey: r.MobKey, - ExpiringSDKKey: ExpiringSDKKey{ - Key: r.SDKKey.Expiring.Value, - Expiration: ToTime(r.SDKKey.Expiring.Timestamp), - }, + SDKKey: r.SDKKey.Value, + MobileKey: r.MobKey, TTL: time.Duration(r.DefaultTTL) * time.Minute, SecureMode: r.SecureMode, } + + if r.SDKKey.Expiring.Value.Defined() { + params.ExpiringSDKKey = ExpiringSDKKey{ + Key: r.SDKKey.Expiring.Value, + Expiration: ToTime(r.SDKKey.Expiring.Timestamp), + } + } + return params } func (r EnvironmentRep) Describe() string { diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index 9b789933..5c704e6f 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -51,6 +51,7 @@ type EnvContext interface { GetDeprecatedCredentials() []credential.SDKCredential RotateMobileKey(key config.MobileKey) + RotateEnvironmentID(id config.EnvironmentID) RotateSDKKey(newKey config.SDKKey, notice *credential.DeprecationNotice) // GetClient returns the SDK client instance for this environment. This is nil if initialization is not yet diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 69436e09..a77991a1 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -143,7 +143,6 @@ func NewEnvContext( readyCh chan<- EnvContext, // readyCh is a separate parameter because it's not a property of the environment itself, but // just part of the semantics of the constructor - // just part of the semantics of the constructor ) (EnvContext, error) { var thingsToCleanUp util.CleanupTasks // keeps track of partially constructed things in case we exit early defer thingsToCleanUp.Run() @@ -182,18 +181,17 @@ func NewEnvContext( dataStoreInfo: params.DataStoreInfo, creationTime: time.Now(), filterKey: params.EnvConfig.FilterKey, - keyRotator: credential.NewRotator(params.Loggers, envConfig.EnvID, params.TimeSource), + keyRotator: credential.NewRotator(params.Loggers, params.TimeSource), stopMonitoringCredentials: make(chan struct{}), doneMonitoringCredentials: make(chan struct{}), connectionMapper: params.ConnectionMapper, } - envContext.keyRotator.RotateSDKKey(envConfig.SDKKey, nil) - if envConfig.MobileKey.Defined() { - envContext.keyRotator.RotateMobileKey(envConfig.MobileKey) - } - - go envContext.monitorCredentialChanges() + envContext.keyRotator.Initialize([]credential.SDKCredential{ + envConfig.SDKKey, + envConfig.MobileKey, + envConfig.EnvID, + }) bigSegmentStoreFactory := params.BigSegmentStoreFactory if bigSegmentStoreFactory == nil { @@ -387,6 +385,8 @@ func NewEnvContext( } } + go envContext.monitorCredentialChanges(envContext.doneMonitoringCredentials) + // Connecting may take time, so do this in parallel go envContext.startSDKClient(envConfig.SDKKey, readyCh, allConfig.Main.IgnoreConnectionErrors) @@ -395,52 +395,65 @@ func NewEnvContext( return envContext, nil } -func (c *envContextImpl) monitorCredentialChanges() { - defer close(c.doneMonitoringCredentials) +func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { + c.mu.Lock() + defer c.mu.Unlock() + c.envStreams.AddCredential(newCredential) + for streamProvider, handlers := range c.handlers { + if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, newCredential)); h != nil { + handlers[newCredential] = h + } + } + + // A new SDK key means 1. we should start a new SDK client, 2. we should tell all event forwarding + // components that use an SDK key to use the new one. A new mobile key does not require starting a + // new SDK client, but does requiring updating any event forwarding components that use a mobile key. + switch key := newCredential.(type) { + case config.SDKKey: + go c.startSDKClient(key, nil, false) + if c.metricsEventPub != nil { // metrics event publisher always uses SDK key + c.metricsEventPub.ReplaceCredential(key) + } + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(key) + } + case config.MobileKey: + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(key) + } + } + + c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, newCredential), c) +} + +func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential) { + c.mu.Lock() + defer c.mu.Unlock() + c.connectionMapper.RemoveConnectionMapping(sdkauth.NewScoped(c.filterKey, oldCredential)) + c.envStreams.RemoveCredential(oldCredential) + for _, handlers := range c.handlers { + delete(handlers, oldCredential) + } + if sdkKey, ok := oldCredential.(config.SDKKey); ok { + // The SDK client instance is tied to the SDK key, so get rid of it + if client := c.clients[sdkKey]; client != nil { + delete(c.clients, sdkKey) + _ = client.Close() + } + } +} + +func (c *envContextImpl) monitorCredentialChanges(done chan struct{}) { + defer close(done) for { select { case <-c.stopMonitoringCredentials: return case oldCredential := <-c.keyRotator.Expirations(): - c.connectionMapper.RemoveConnectionMapping(sdkauth.NewScoped(c.filterKey, oldCredential)) - c.envStreams.RemoveCredential(oldCredential) - for _, handlers := range c.handlers { - delete(handlers, oldCredential) - } - if sdkKey, ok := oldCredential.(config.SDKKey); ok { - // The SDK client instance is tied to the SDK key, so get rid of it - if client := c.clients[sdkKey]; client != nil { - delete(c.clients, sdkKey) - _ = client.Close() - } - } + c.removeCredential(oldCredential) case newCredential := <-c.keyRotator.Additions(): - c.envStreams.AddCredential(newCredential) - for streamProvider, handlers := range c.handlers { - if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, newCredential)); h != nil { - handlers[newCredential] = h - } - } - - // A new SDK key means 1. we should start a new SDK client, 2. we should tell all event forwarding - // components that use an SDK key to use the new one. A new mobile key does not require starting a - // new SDK client, but does requiring updating any event forwarding components that use a mobile key. - switch key := newCredential.(type) { - case config.SDKKey: - go c.startSDKClient(key, nil, false) - if c.metricsEventPub != nil { // metrics event publisher always uses SDK key - c.metricsEventPub.ReplaceCredential(key) - } - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) - } - case config.MobileKey: - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) - } - } - c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, newCredential), c) + c.addCredential(newCredential) } } } @@ -521,6 +534,10 @@ func (c *envContextImpl) RotateMobileKey(key config.MobileKey) { c.keyRotator.RotateMobileKey(key) } +func (c *envContextImpl) RotateEnvironmentID(id config.EnvironmentID) { + c.keyRotator.RotateEnvironmentID(id) +} + func (c *envContextImpl) RotateSDKKey(newKey config.SDKKey, notice *credential.DeprecationNotice) { c.keyRotator.RotateSDKKey(newKey, notice) } diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 2cdc1f50..489959f9 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -3,6 +3,7 @@ package relayenv import ( "context" "errors" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" "net/http" "net/http/httptest" "regexp" @@ -59,18 +60,30 @@ func makeBasicEnv(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.C return makeBasicEnvWithMockTime(t, envConfig, clientFactory, loggers, readyCh, nil) } +type mockConnectionMapper struct { +} + +func (m mockConnectionMapper) AddConnectionMapping(scopedCredential sdkauth.ScopedCredential, envContext EnvContext) { + +} +func (m mockConnectionMapper) RemoveConnectionMapping(scopedCredential sdkauth.ScopedCredential) { + +} + func makeBasicEnvWithMockTime(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.ClientFactoryFunc, loggers ldlog.Loggers, readyCh chan EnvContext, now func() time.Time) EnvContext { env, err := NewEnvContext(EnvContextImplParams{ - Identifiers: EnvIdentifiers{ConfiguredName: envName}, - EnvConfig: envConfig, - ClientFactory: clientFactory, - Loggers: loggers, - TimeSource: now, + Identifiers: EnvIdentifiers{ConfiguredName: envName}, + EnvConfig: envConfig, + ClientFactory: clientFactory, + Loggers: loggers, + TimeSource: now, + ConnectionMapper: mockConnectionMapper{}, }, readyCh) require.NoError(t, err) return env } + func TestConstructorBasicProperties(t *testing.T) { envConfig := st.EnvWithAllCredentials.Config envConfig.TTL = configtypes.NewOptDuration(time.Hour) @@ -180,8 +193,8 @@ func TestAddRemoveCredential(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) - env.AddCredential(st.EnvWithAllCredentials.Config.MobileKey) - env.AddCredential(st.EnvWithAllCredentials.Config.EnvID) + env.RotateMobileKey(st.EnvWithAllCredentials.Config.MobileKey) + env.RotateEnvironmentID(st.EnvWithAllCredentials.Config.EnvID) creds := env.GetCredentials() assert.Len(t, creds, 3) @@ -189,11 +202,12 @@ func TestAddRemoveCredential(t *testing.T) { assert.Contains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) assert.Contains(t, creds, st.EnvWithAllCredentials.Config.EnvID) - env.RemoveCredential(st.EnvWithAllCredentials.Config.MobileKey) + env.RotateMobileKey("foo") creds = env.GetCredentials() - assert.Len(t, creds, 2) + assert.Len(t, creds, 3) assert.Contains(t, creds, envConfig.SDKKey) + assert.NotContains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) assert.Contains(t, creds, st.EnvWithAllCredentials.Config.EnvID) } @@ -208,14 +222,14 @@ func TestAddExistingCredentialDoesNothing(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) - env.AddCredential(st.EnvWithAllCredentials.Config.MobileKey) + env.RotateMobileKey(st.EnvWithAllCredentials.Config.MobileKey) creds := env.GetCredentials() assert.Len(t, creds, 2) assert.Contains(t, creds, envConfig.SDKKey) assert.Contains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) - env.AddCredential(st.EnvWithAllCredentials.Config.MobileKey) + env.RotateMobileKey(st.EnvWithAllCredentials.Config.MobileKey) creds = env.GetCredentials() assert.Len(t, creds, 2) @@ -226,11 +240,8 @@ func TestAddExistingCredentialDoesNothing(t *testing.T) { func TestChangeSDKKey(t *testing.T) { envConfig := st.EnvMain.Config readyCh := make(chan EnvContext, 1) - newKey := config.SDKKey("new-key") - - const deprecationDelay = 100 * time.Millisecond - const clientInitializationDelay = 10 * time.Millisecond - const clientCloseDelay = deprecationDelay / 2 + key2 := config.SDKKey("key2") + key3 := config.SDKKey("key3") clientCh := make(chan *testclient.FakeLDClient, 1) clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) @@ -246,30 +257,27 @@ func TestChangeSDKKey(t *testing.T) { assert.Equal(t, env.GetClient(), client1) assert.Nil(t, env.GetInitError()) - removed := make(chan credential.SDKCredential, 1) - env.AddCredential(newKey) - env.DeprecateCredential(envConfig.SDKKey, time.Now().Add(deprecationDelay), &DeprecationHooks{AfterRemoval: func(cred credential.SDKCredential) { - removed <- cred - }}) + env.RotateSDKKey(key2, credential.NewDeprecationNotice(envConfig.SDKKey, time.Now().Add(1*time.Hour))) - assert.Equal(t, []credential.SDKCredential{newKey}, env.GetCredentials()) + assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) + assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetDeprecatedCredentials()) client2 := requireClientReady(t, clientCh) assert.NotEqual(t, client1, client2) - - time.Sleep(clientInitializationDelay) assert.Equal(t, env.GetClient(), client2) - if !helpers.AssertNoMoreValues(t, client1.CloseCh, clientCloseDelay, "client for deprecated key should not have been closed") { + if !helpers.AssertChannelNotClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should not have been closed yet") { t.FailNow() } - deprecatedCred := helpers.RequireValue(t, removed, deprecationDelay, "timed out waiting for deprecation") - assert.Equal(t, envConfig.SDKKey, deprecatedCred) + env.RotateSDKKey(key3, nil) + assert.Equal(t, []credential.SDKCredential{key3}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetDeprecatedCredentials()) - assert.Equal(t, []credential.SDKCredential{newKey}, env.GetCredentials()) + if !helpers.AssertChannelClosed(t, client2.CloseCh, 1*time.Second, "client for key2 should have been closed") { + t.FailNow() + } - client1.AwaitClose(t, clientCloseDelay) } func TestSDKClientCreationFails(t *testing.T) { diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index bc4708b4..9a0e7275 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -34,9 +34,14 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar } if params.ExpiringSDKKey.Defined() { - if _, err := a.r.getEnvironment(sdkauth.NewScoped(params.Identifiers.FilterKey, params.ExpiringSDKKey.Key)); err != nil { - env.RotateSDKKey(params.SDKKey, credential.NewDeprecationNotice(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration)) - } + // Ooooooh + // It's because the primary SDK key was laready initialized when the addEnvironment was called, so rotate is doing + // nothin' + // Options: + // 1) Make addEnvironment take one. Maybe it makes sense for the envRep to allow for an expiring key.. though + // it'd require a RElay restart. + // 2) Make it okay to do? + env.RotateSDKKey(params.SDKKey, credential.NewDeprecationNotice(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration)) } } diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index f3e7c714..27313bdd 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -1,6 +1,7 @@ package relay import ( + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "net/http/httptest" "testing" "time" @@ -101,7 +102,7 @@ func TestAutoConfigInit(t *testing.T) { autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { client1 := p.awaitClient() client2 := p.awaitClient() - if client1.Key == testAutoConfEnv2.sdkKey { + if client1.Key == testAutoConfEnv2.SDKKey() { client1, client2 = client2, client1 } assert.Equal(t, testAutoConfEnv1.sdkKey, client1.Key) @@ -121,9 +122,13 @@ func TestAutoConfigInitWithExpiringSDKKey(t *testing.T) { newKey := c.SDKKey("newsdkkey") oldKey := c.SDKKey("oldsdkkey") envWithKeys := testAutoConfEnv1 - envWithKeys.sdkKey = newKey - envWithKeys.sdkKeyExpiryValue = oldKey - envWithKeys.sdkKeyExpiryTime = ldtime.UnixMillisNow() + 100000 + envWithKeys.sdkKey = envfactory.SDKKeyRep{ + Value: newKey, + Expiring: envfactory.ExpiringKeyRep{ + Value: oldKey, + Timestamp: ldtime.UnixMillisNow() + 100000, + }, + } initialEvent := makeAutoConfPutEvent(envWithKeys) autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { client1 := p.awaitClient() @@ -148,7 +153,7 @@ func TestAutoConfigInitAfterPreviousInitCanAddAndRemoveEnvs(t *testing.T) { initialEvent := makeAutoConfPutEvent(testAutoConfEnv1) autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { client1 := p.awaitClient() - assert.Equal(t, testAutoConfEnv1.sdkKey, client1.Key) + assert.Equal(t, testAutoConfEnv1.SDKKey(), client1.Key) env1 := p.awaitEnvironment(testAutoConfEnv1.id) assertEnvProps(t, testAutoConfEnv1.params(), env1) @@ -157,7 +162,7 @@ func TestAutoConfigInitAfterPreviousInitCanAddAndRemoveEnvs(t *testing.T) { p.stream.Enqueue(makeAutoConfPutEvent(testAutoConfEnv2)) client2 := p.awaitClient() - assert.Equal(t, testAutoConfEnv2.sdkKey, client2.Key) + assert.Equal(t, testAutoConfEnv2.SDKKey(), client2.Key) env2 := p.awaitEnvironment(testAutoConfEnv2.id) assertEnvProps(t, testAutoConfEnv2.params(), env2) @@ -168,7 +173,7 @@ func TestAutoConfigInitAfterPreviousInitCanAddAndRemoveEnvs(t *testing.T) { p.shouldNotHaveEnvironment(testAutoConfEnv1.id, time.Millisecond*100) p.assertSDKEndpointsAvailability( false, - testAutoConfEnv1.sdkKey, + testAutoConfEnv1.SDKKey(), testAutoConfEnv1.mobKey, testAutoConfEnv1.id, ) @@ -199,10 +204,13 @@ func TestAutoConfigAddEnvironmentWithExpiringSDKKey(t *testing.T) { newKey := c.SDKKey("newsdkkey") oldKey := c.SDKKey("oldsdkkey") envWithKeys := testAutoConfEnv1 - envWithKeys.sdkKey = newKey - envWithKeys.sdkKeyExpiryValue = oldKey - envWithKeys.sdkKeyExpiryTime = ldtime.UnixMillisNow() + 100000 - + envWithKeys.sdkKey = envfactory.SDKKeyRep{ + Value: newKey, + Expiring: envfactory.ExpiringKeyRep{ + Value: oldKey, + Timestamp: ldtime.UnixMillisNow() + 100000, + }, + } initialEvent := makeAutoConfPutEvent() autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { p.stream.Enqueue(makeAutoConfPatchEvent(envWithKeys)) @@ -218,7 +226,7 @@ func TestAutoConfigAddEnvironmentWithExpiringSDKKey(t *testing.T) { env := p.awaitEnvironment(envWithKeys.id) assertEnvProps(t, envWithKeys.params(), env) - expectedCredentials := credentialsAsSet(envWithKeys.id, envWithKeys.mobKey, envWithKeys.sdkKey) + expectedCredentials := credentialsAsSet(envWithKeys.id, envWithKeys.mobKey, envWithKeys.SDKKey()) assert.Equal(t, expectedCredentials, credentialsAsSet(env.GetCredentials()...)) paramsWithOldKey := envWithKeys.params() @@ -260,7 +268,7 @@ func TestAutoConfigDeleteEnvironment(t *testing.T) { autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { client1 := p.awaitClient() client2 := p.awaitClient() - if client1.Key == testAutoConfEnv2.sdkKey { + if client1.Key == testAutoConfEnv2.SDKKey() { client1, client2 = client2, client1 } @@ -277,7 +285,7 @@ func TestAutoConfigDeleteEnvironment(t *testing.T) { p.shouldNotHaveEnvironment(testAutoConfEnv1.id, time.Millisecond*100) p.assertSDKEndpointsAvailability( false, - testAutoConfEnv1.sdkKey, + testAutoConfEnv1.SDKKey(), testAutoConfEnv1.mobKey, testAutoConfEnv1.id, ) diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index 536457d8..5cde9513 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -1,6 +1,7 @@ package relay import ( + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "net/http" "testing" "time" @@ -26,7 +27,7 @@ const ( ) func makeEnvWithModifiedSDKKey(e testAutoConfEnv) testAutoConfEnv { - e.sdkKey += "-changed" + e.sdkKey.Value += "-changed" e.version++ return e } @@ -87,12 +88,12 @@ func TestAutoConfigUpdateEnvironmentSDKKeyWithNoExpiry(t *testing.T) { p.stream.Enqueue(makeAutoConfPatchEvent(modified)) client2 := p.awaitClient() - assert.Equal(t, modified.sdkKey, client2.Key) + assert.Equal(t, modified.SDKKey(), client2.Key) client1.AwaitClose(t, 10000*time.Second) p.awaitCredentialsUpdated(env, modified.params()) - noEnv, _ := p.relay.getEnvironment(sdkauth.New(testAutoConfEnv1.sdkKey)) + noEnv, _ := p.relay.getEnvironment(sdkauth.New(testAutoConfEnv1.sdkKey.Value)) assert.Nil(t, noEnv) }) } @@ -106,16 +107,18 @@ func TestAutoConfigUpdateEnvironmentSDKKeyWithExpiry(t *testing.T) { assertEnvProps(t, testAutoConfEnv1.params(), env) modified := makeEnvWithModifiedSDKKey(testAutoConfEnv1) - modified.sdkKeyExpiryValue = testAutoConfEnv1.sdkKey - modified.sdkKeyExpiryTime = ldtime.UnixMillisNow() + 100000 + modified.sdkKey.Expiring = envfactory.ExpiringKeyRep{ + Value: testAutoConfEnv1.sdkKey.Value, + Timestamp: ldtime.UnixMillisNow() + 100000, + } p.stream.Enqueue(makeAutoConfPatchEvent(modified)) client2 := p.awaitClient() - assert.Equal(t, modified.sdkKey, client2.Key) + assert.Equal(t, modified.SDKKey(), client2.Key) p.awaitCredentialsUpdated(env, modified.params()) p.assertEnvLookup(env, testAutoConfEnv1.params()) // looking up env by old key still works - assert.Equal(t, []credential.SDKCredential{testAutoConfEnv1.sdkKey}, env.GetDeprecatedCredentials()) + assert.Equal(t, []credential.SDKCredential{testAutoConfEnv1.sdkKey.Value}, env.GetDeprecatedCredentials()) if !helpers.AssertChannelNotClosed(t, client1.CloseCh, time.Millisecond*300, "should not have closed client for deprecated key yet") { t.FailNow() @@ -139,7 +142,7 @@ func TestEventForwardingAfterSDKKeyChange(t *testing.T) { p.awaitCredentialsUpdated(env, modified.params()) - verifyEventProxying(t, p, serverSideEventsURL, modified.sdkKey) + verifyEventProxying(t, p, serverSideEventsURL, modified.SDKKey()) verifyEventProxying(t, p, mobileEventsURL, testAutoConfEnv1.mobKey) verifyEventProxying(t, p, jsEventsURL+string(testAutoConfEnv1.id), testAutoConfEnv1.id) }) @@ -150,7 +153,7 @@ func TestEventForwardingAfterSDKKeyChange(t *testing.T) { env := p.awaitEnvironment(testAutoConfEnv1.id) assertEnvProps(t, testAutoConfEnv1.params(), env) - verifyEventProxying(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey) + verifyEventProxying(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey.Value) verifyEventProxying(t, p, mobileEventsURL, testAutoConfEnv1.mobKey) modified := makeEnvWithModifiedSDKKey(testAutoConfEnv1) @@ -158,7 +161,7 @@ func TestEventForwardingAfterSDKKeyChange(t *testing.T) { p.awaitCredentialsUpdated(env, modified.params()) - verifyEventProxying(t, p, serverSideEventsURL, modified.sdkKey) + verifyEventProxying(t, p, serverSideEventsURL, modified.SDKKey()) verifyEventProxying(t, p, mobileEventsURL, testAutoConfEnv1.mobKey) verifyEventProxying(t, p, jsEventsURL+string(testAutoConfEnv1.id), testAutoConfEnv1.id) }) @@ -167,7 +170,7 @@ func TestEventForwardingAfterSDKKeyChange(t *testing.T) { func TestAutoConfigRemovesCredentialForExpiredSDKKey(t *testing.T) { briefExpiryMillis := 300 - oldKey := testAutoConfEnv1.sdkKey + oldKey := testAutoConfEnv1.sdkKey.Value initialEvent := makeAutoConfPutEvent(testAutoConfEnv1) @@ -178,12 +181,14 @@ func TestAutoConfigRemovesCredentialForExpiredSDKKey(t *testing.T) { assertEnvProps(t, testAutoConfEnv1.params(), env) modified := makeEnvWithModifiedSDKKey(testAutoConfEnv1) - modified.sdkKeyExpiryValue = oldKey - modified.sdkKeyExpiryTime = ldtime.UnixMillisNow() + ldtime.UnixMillisecondTime(briefExpiryMillis) + modified.sdkKey.Expiring = envfactory.ExpiringKeyRep{ + Value: oldKey, + Timestamp: ldtime.UnixMillisNow() + ldtime.UnixMillisecondTime(briefExpiryMillis), + } p.stream.Enqueue(makeAutoConfPatchEvent(modified)) client2 := p.awaitClient() - assert.Equal(t, modified.sdkKey, client2.Key) + assert.Equal(t, modified.SDKKey(), client2.Key) p.awaitCredentialsUpdated(env, modified.params()) newCredentials := credentialsAsSet(env.GetCredentials()...) @@ -234,7 +239,7 @@ func TestEventForwardingAfterMobileKeyChange(t *testing.T) { p.awaitCredentialsUpdated(env, modified.params()) - verifyEventProxying(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey) + verifyEventProxying(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey.Value) verifyEventProxying(t, p, mobileEventsURL, modified.mobKey) verifyEventProxying(t, p, jsEventsURL+string(testAutoConfEnv1.id), testAutoConfEnv1.id) }) @@ -245,7 +250,7 @@ func TestEventForwardingAfterMobileKeyChange(t *testing.T) { env := p.awaitEnvironment(testAutoConfEnv1.id) assertEnvProps(t, testAutoConfEnv1.params(), env) - verifyEventVerbatimRelay(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey) + verifyEventVerbatimRelay(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey.Value) verifyEventVerbatimRelay(t, p, mobileEventsURL, testAutoConfEnv1.mobKey) modified := makeEnvWithModifiedMobileKey(testAutoConfEnv1) @@ -253,7 +258,7 @@ func TestEventForwardingAfterMobileKeyChange(t *testing.T) { p.awaitCredentialsUpdated(env, modified.params()) - verifyEventProxying(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey) + verifyEventProxying(t, p, serverSideEventsURL, testAutoConfEnv1.sdkKey.Value) verifyEventProxying(t, p, mobileEventsURL, modified.mobKey) verifyEventProxying(t, p, jsEventsURL+string(testAutoConfEnv1.id), testAutoConfEnv1.id) }) diff --git a/relay/autoconfig_testdata_test.go b/relay/autoconfig_testdata_test.go index 17681d50..aa0e3dbc 100644 --- a/relay/autoconfig_testdata_test.go +++ b/relay/autoconfig_testdata_test.go @@ -7,7 +7,6 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/autoconfig" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - "github.com/launchdarkly/go-sdk-common/v3/ldtime" "github.com/launchdarkly/go-test-helpers/v3/httphelpers" ) @@ -18,16 +17,18 @@ var testAutoConfDefaultConfig = c.Config{ } type testAutoConfEnv struct { - id c.EnvironmentID - envKey string - envName string - mobKey c.MobileKey - projKey string - projName string - sdkKey c.SDKKey - sdkKeyExpiryValue c.SDKKey - sdkKeyExpiryTime ldtime.UnixMillisecondTime - version int + id c.EnvironmentID + envKey string + envName string + mobKey c.MobileKey + projKey string + projName string + sdkKey envfactory.SDKKeyRep + version int +} + +func (e testAutoConfEnv) SDKKey() c.SDKKey { + return e.sdkKey.Value } var ( @@ -38,7 +39,7 @@ var ( mobKey: c.MobileKey("mobkey1"), projKey: "projkey1", projName: "projname1", - sdkKey: c.SDKKey("sdkkey1"), + sdkKey: envfactory.SDKKeyRep{Value: c.SDKKey("sdkkey1")}, version: 10, } @@ -49,7 +50,7 @@ var ( mobKey: c.MobileKey("mobkey2"), projKey: "projkey2", projName: "projname2", - sdkKey: c.SDKKey("sdkkey2"), + sdkKey: envfactory.SDKKeyRep{Value: c.SDKKey("sdkkey2")}, version: 11, } ) @@ -62,14 +63,8 @@ func (e testAutoConfEnv) toEnvironmentRep() envfactory.EnvironmentRep { MobKey: e.mobKey, ProjKey: e.projKey, ProjName: e.projName, - SDKKey: envfactory.SDKKeyRep{ - Value: e.sdkKey, - }, - Version: e.version, - } - if e.sdkKeyExpiryValue != "" { - rep.SDKKey.Expiring.Value = e.sdkKeyExpiryValue - rep.SDKKey.Expiring.Timestamp = e.sdkKeyExpiryTime + SDKKey: e.sdkKey, + Version: e.version, } return rep } diff --git a/relay/relay.go b/relay/relay.go index 01ce4fce..4d05a79a 100644 --- a/relay/relay.go +++ b/relay/relay.go @@ -361,8 +361,7 @@ func IsPayloadFilterNotFound(err error) bool { // getEnvironment returns the environment object corresponding to the given credential, or nil // if not found. The credential can be an SDK key, a mobile key, or an environment ID. The second -// return value is normally true, but is false if Relay does not yet have a valid configuration -// (which affects our error handling). +// return value is normally nil, but is present if Relay does not yet have a valid configuration. func (r *Relay) getEnvironment(req sdkauth.ScopedCredential) (relayenv.EnvContext, error) { r.lock.RLock() defer r.lock.RUnlock() diff --git a/relay/relay_environments_test.go b/relay/relay_environments_test.go index c5548cf3..64e93c6e 100644 --- a/relay/relay_environments_test.go +++ b/relay/relay_environments_test.go @@ -186,7 +186,7 @@ func TestRelayAddedEnvironmentCredential(t *testing.T) { noEnv, _ := relay.getEnvironment(sdkauth.New(extraKey)) assert.Nil(t, noEnv) - relay.addConnectionMapping(sdkauth.New(extraKey), env) + relay.AddConnectionMapping(sdkauth.New(extraKey), env) env1, _ := relay.getEnvironment(sdkauth.New(extraKey)) assert.Equal(t, env, env1) @@ -200,7 +200,7 @@ func TestRelayRemovingEnvironmentCredential(t *testing.T) { require.NoError(t, err) defer relay.Close() - relay.removeConnectionMapping(sdkauth.New(st.EnvMain.Config.SDKKey)) + relay.RemoveConnectionMapping(sdkauth.New(st.EnvMain.Config.SDKKey)) _, err = relay.getEnvironment(sdkauth.New(st.EnvMain.Config.SDKKey)) assert.Error(t, err) From 260640b4010245e99442b8c257dc0a8956046122 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 18 Jun 2024 18:42:08 -0700 Subject: [PATCH 08/26] fix bug with immediate revocation --- internal/credential/rotator.go | 7 +++++-- relay/autoconfig_actions.go | 7 ------- relay/autoconfig_actions_test.go | 8 ++++---- relay/testutils_test.go | 4 ++++ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index d50b24a6..b320e8d6 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -183,9 +183,9 @@ func (r *Rotator) RotateMobileKey(mobileKey config.MobileKey) { } func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { - if newKey == r.SDKKey() { + if newKey == r.primarySdkKey { // There's no swap to be done, we already are using this as primary. - return newKey + return "" } previous := r.primarySdkKey r.primarySdkKey = newKey @@ -195,6 +195,9 @@ func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { return previous } func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNotice) { + r.mu.Lock() + defer r.mu.Unlock() + previous := r.swapPrimaryKey(sdkKey) // Immediately revoke the previous SDK key if there's no explicit deprecation notice, otherwise it would // hang around forever. diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index 9a0e7275..a9ef6145 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -34,13 +34,6 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar } if params.ExpiringSDKKey.Defined() { - // Ooooooh - // It's because the primary SDK key was laready initialized when the addEnvironment was called, so rotate is doing - // nothin' - // Options: - // 1) Make addEnvironment take one. Maybe it makes sense for the envRep to allow for an expiring key.. though - // it'd require a RElay restart. - // 2) Make it okay to do? env.RotateSDKKey(params.SDKKey, credential.NewDeprecationNotice(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration)) } } diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index 27313bdd..a8a78bec 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -105,8 +105,8 @@ func TestAutoConfigInit(t *testing.T) { if client1.Key == testAutoConfEnv2.SDKKey() { client1, client2 = client2, client1 } - assert.Equal(t, testAutoConfEnv1.sdkKey, client1.Key) - assert.Equal(t, testAutoConfEnv2.sdkKey, client2.Key) + assert.Equal(t, testAutoConfEnv1.SDKKey(), client1.Key) + assert.Equal(t, testAutoConfEnv2.SDKKey(), client2.Key) env1 := p.awaitEnvironment(testAutoConfEnv1.id) assertEnvProps(t, testAutoConfEnv1.params(), env1) @@ -184,7 +184,7 @@ func TestAutoConfigAddEnvironment(t *testing.T) { initialEvent := makeAutoConfPutEvent(testAutoConfEnv1) autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { client1 := p.awaitClient() - assert.Equal(t, testAutoConfEnv1.sdkKey, client1.Key) + assert.Equal(t, testAutoConfEnv1.SDKKey(), client1.Key) env1 := p.awaitEnvironment(testAutoConfEnv1.id) assertEnvProps(t, testAutoConfEnv1.params(), env1) @@ -192,7 +192,7 @@ func TestAutoConfigAddEnvironment(t *testing.T) { p.stream.Enqueue(makeAutoConfPatchEvent(testAutoConfEnv2)) client2 := p.awaitClient() - assert.Equal(t, testAutoConfEnv2.sdkKey, client2.Key) + assert.Equal(t, testAutoConfEnv2.SDKKey(), client2.Key) env2 := p.awaitEnvironment(testAutoConfEnv2.id) p.assertEnvLookup(env2, testAutoConfEnv2.params()) diff --git a/relay/testutils_test.go b/relay/testutils_test.go index b0120e48..c13225ae 100644 --- a/relay/testutils_test.go +++ b/relay/testutils_test.go @@ -27,6 +27,7 @@ type relayTestHelper struct { } func (h relayTestHelper) awaitEnvironment(envID c.EnvironmentID) relayenv.EnvContext { + h.t.Helper() var e relayenv.EnvContext var err error require.Eventually(h.t, func() bool { @@ -37,6 +38,7 @@ func (h relayTestHelper) awaitEnvironment(envID c.EnvironmentID) relayenv.EnvCon } func (h relayTestHelper) shouldNotHaveEnvironment(envID c.EnvironmentID, timeout time.Duration) { + h.t.Helper() require.Eventually(h.t, func() bool { _, err := h.relay.getEnvironment(sdkauth.New(envID)) return err != nil @@ -44,6 +46,8 @@ func (h relayTestHelper) shouldNotHaveEnvironment(envID c.EnvironmentID, timeout } func (h relayTestHelper) assertEnvLookup(env relayenv.EnvContext, expected envfactory.EnvironmentParams) { + h.t.Helper() + foundEnv, err := h.relay.getEnvironment(sdkauth.New(expected.EnvID)) if assert.NoError(h.t, err) { assert.Equal(h.t, env, foundEnv) From 9fc465c2c244291518f1082dda47bb9d92b0b5c0 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Thu, 20 Jun 2024 12:36:42 -0700 Subject: [PATCH 09/26] refactor the rotator to be more testable without any time dependency --- internal/credential/rotator.go | 57 ++++---- internal/credential/rotator_test.go | 194 +++++++++++++++++++++------- 2 files changed, 176 insertions(+), 75 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index b320e8d6..ba6cc7c1 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -39,9 +39,8 @@ type Rotator struct { // Upon expiration, they are removed. deprecatedSdkKeys map[config.SDKKey]*deprecatedKey - expirations chan SDKCredential - additions chan SDKCredential - now func() time.Time + expirations []SDKCredential + additions []SDKCredential mu sync.RWMutex } @@ -52,16 +51,10 @@ type InitialCredentials struct { EnvironmentID config.EnvironmentID } -func NewRotator(loggers ldlog.Loggers, now func() time.Time) *Rotator { +func NewRotator(loggers ldlog.Loggers) *Rotator { r := &Rotator{ loggers: loggers, deprecatedSdkKeys: make(map[config.SDKKey]*deprecatedKey), - expirations: make(chan SDKCredential, 1), - additions: make(chan SDKCredential, 1), - now: now, - } - if r.now == nil { - r.now = time.Now } return r } @@ -85,14 +78,6 @@ func (r *Rotator) Initialize(credentials []SDKCredential) { } } -func (r *Rotator) Expirations() <-chan SDKCredential { - return r.expirations -} - -func (r *Rotator) Additions() <-chan SDKCredential { - return r.additions -} - func (r *Rotator) MobileKey() config.MobileKey { r.mu.RLock() defer r.mu.RUnlock() @@ -156,10 +141,10 @@ func (r *Rotator) RotateEnvironmentID(envID config.EnvironmentID) { defer r.mu.Unlock() previous := r.primaryEnvironmentID r.primaryEnvironmentID = envID - r.additions <- envID + r.additions = append(r.additions, envID) if previous.Defined() { r.loggers.Infof("Environment ID %s was rotated, new environment ID is %s", r.primaryEnvironmentID, envID) - r.expirations <- previous + r.expirations = append(r.expirations, previous) } else { r.loggers.Infof("New environment ID is %s", envID) } @@ -173,9 +158,9 @@ func (r *Rotator) RotateMobileKey(mobileKey config.MobileKey) { defer r.mu.Unlock() previous := r.primaryMobileKey r.primaryMobileKey = mobileKey - r.additions <- mobileKey + r.additions = append(r.additions, mobileKey) if previous.Defined() { - r.expirations <- previous + r.expirations = append(r.expirations, previous) r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", r.primaryMobileKey.Masked(), mobileKey.Masked()) } else { r.loggers.Infof("New primary mobile key is %s", mobileKey.Masked()) @@ -189,7 +174,7 @@ func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { } previous := r.primarySdkKey r.primarySdkKey = newKey - r.additions <- newKey + r.additions = append(r.additions, newKey) r.loggers.Infof("New primary SDK key is %s", newKey.Masked()) return previous @@ -202,7 +187,7 @@ func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNot // Immediately revoke the previous SDK key if there's no explicit deprecation notice, otherwise it would // hang around forever. if previous.Defined() && deprecation == nil { - r.expirations <- previous + r.expirations = append(r.expirations, previous) r.loggers.Infof("SDK key %s has been immediately revoked", previous.Masked()) return } @@ -215,21 +200,33 @@ func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNot r.loggers.Infof("SDK key %s was marked for deprecation with an expiry at %v", deprecation.key.Masked(), deprecation.expiry) r.deprecatedSdkKeys[deprecation.key] = &deprecatedKey{ expiry: deprecation.expiry, - timer: time.AfterFunc(deprecation.expiry.Sub(r.now()), func() { - r.expireSDKKey(deprecation.key) - })} + } if deprecation.key != previous { r.loggers.Infof("Deprecated SDK key %s was not previously managed by Relay", deprecation.key.Masked()) - r.additions <- deprecation.key + r.additions = append(r.additions, deprecation.key) } } } func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.loggers.Infof("Deprecated SDK key %s has expired and is no longer valid for authentication", sdkKey.Masked()) + delete(r.deprecatedSdkKeys, sdkKey) + r.expirations = append(r.expirations, sdkKey) +} + +func (r *Rotator) Tick(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { r.mu.Lock() defer r.mu.Unlock() - delete(r.deprecatedSdkKeys, sdkKey) - r.expirations <- sdkKey + + for key, dep := range r.deprecatedSdkKeys { + if now.After(dep.expiry) { + r.expireSDKKey(key) + } + } + + additions, expirations = r.additions, r.expirations + r.additions = nil + r.expirations = nil + return } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index cc4135e0..32659811 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -1,30 +1,123 @@ package credential import ( + "fmt" "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" - helpers "github.com/launchdarkly/go-test-helpers/v3" "github.com/launchdarkly/ld-relay/v8/config" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "testing" "time" ) -func requireChanValue(t *testing.T, ch <-chan SDKCredential, expected SDKCredential) { - t.Helper() - value := helpers.RequireValue(t, ch, 1*time.Second) - require.Equal(t, expected, value) -} - func TestNewRotator(t *testing.T) { mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers, time.Now) + rotator := NewRotator(mockLog.Loggers) assert.NotNil(t, rotator) } -func TestKeyDeprecation(t *testing.T) { +func TestImmediateKeyExpiration(t *testing.T) { + t.Run("sdk keys", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + const ( + key1 = config.SDKKey("key1") + key2 = config.SDKKey("key2") + key3 = config.SDKKey("key3") + ) + + // The first rotation shouldn't trigger any expirations because there was no previous key. + rotator.RotateSDKKey(key1, nil) + additions, _ := rotator.Tick(time.Now()) + assert.ElementsMatch(t, []SDKCredential{key1}, additions) + assert.Equal(t, key1, rotator.SDKKey()) + + // The second rotation should trigger a deprecation of key1. + rotator.RotateSDKKey(key2, nil) + additions, expirations := rotator.Tick(time.Now()) + assert.ElementsMatch(t, []SDKCredential{key2}, additions) + assert.ElementsMatch(t, []SDKCredential{key1}, expirations) + assert.Equal(t, key2, rotator.SDKKey()) + + // The third rotation should trigger a deprecation of key2. + rotator.RotateSDKKey(key3, nil) + additions, expirations = rotator.Tick(time.Now()) + assert.ElementsMatch(t, []SDKCredential{key3}, additions) + assert.ElementsMatch(t, []SDKCredential{key2}, expirations) + assert.Equal(t, key3, rotator.SDKKey()) + }) + + t.Run("mobile keys", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + const ( + key1 = config.MobileKey("key1") + key2 = config.MobileKey("key2") + key3 = config.MobileKey("key3") + ) + + // The first rotation shouldn't trigger any expirations because there was no previous key. + rotator.RotateMobileKey(key1) + additions, _ := rotator.Tick(time.Now()) + assert.ElementsMatch(t, []SDKCredential{key1}, additions) + assert.Equal(t, key1, rotator.MobileKey()) + + // The second rotation should trigger a deprecation of key1. + rotator.RotateMobileKey(key2) + additions, expirations := rotator.Tick(time.Now()) + assert.ElementsMatch(t, []SDKCredential{key2}, additions) + assert.ElementsMatch(t, []SDKCredential{key1}, expirations) + assert.Equal(t, key2, rotator.MobileKey()) + + // The third rotation should trigger a deprecation of key2. + rotator.RotateMobileKey(key3) + additions, expirations = rotator.Tick(time.Now()) + assert.ElementsMatch(t, []SDKCredential{key3}, additions) + assert.ElementsMatch(t, []SDKCredential{key2}, expirations) + assert.Equal(t, key3, rotator.MobileKey()) + }) +} + +func TestManyImmediateKeyExpirations(t *testing.T) { + t.Run("sdk keys", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + const numKeys = 100 + for i := 0; i < numKeys; i++ { + key := config.SDKKey(fmt.Sprintf("key%v", i)) + rotator.RotateSDKKey(key, nil) + } + + assert.Equal(t, config.SDKKey(fmt.Sprintf("key%v", numKeys-1)), rotator.SDKKey()) + + additions, expirations := rotator.Tick(time.Now()) + assert.Len(t, additions, numKeys) + assert.Len(t, expirations, numKeys-1) // because the last key is still active + }) + + t.Run("mobile keys", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + const numKeys = 100 + for i := 0; i < numKeys; i++ { + key := config.MobileKey(fmt.Sprintf("key%v", i)) + rotator.RotateMobileKey(key) + } + + assert.Equal(t, config.MobileKey(fmt.Sprintf("key%v", numKeys-1)), rotator.MobileKey()) + + additions, expirations := rotator.Tick(time.Now()) + assert.Len(t, additions, numKeys) + assert.Len(t, expirations, numKeys-1) // because the last key is still active + }) +} + +func TestSDKKeyDeprecation(t *testing.T) { mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers, time.Now) + rotator := NewRotator(mockLog.Loggers) const ( key1 = config.SDKKey("key1") @@ -32,45 +125,56 @@ func TestKeyDeprecation(t *testing.T) { key3 = config.SDKKey("key3") ) - // The first rotation shouldn't trigger any expirations because there was no previous key. - rotator.RotateSDKKey(key1, nil) - requireChanValue(t, rotator.Additions(), key1) - assert.Equal(t, key1, rotator.SDKKey()) - - // The second rotation should trigger a deprecation of key1. - rotator.RotateSDKKey(key2, nil) - requireChanValue(t, rotator.Additions(), key2) - requireChanValue(t, rotator.Expirations(), key1) - assert.Equal(t, key2, rotator.SDKKey()) - - // The third rotation should trigger a deprecation of key2. - rotator.RotateSDKKey(key3, nil) - requireChanValue(t, rotator.Additions(), key3) - requireChanValue(t, rotator.Expirations(), key2) - assert.Equal(t, key3, rotator.SDKKey()) + start := time.Now() + + deprecationTime := start.Add(1 * time.Minute) + halfTime := start.Add(30 * time.Second) + + rotator.Initialize([]SDKCredential{key1}) + + rotator.RotateSDKKey(key2, NewDeprecationNotice(key1, deprecationTime)) + additions, expirations := rotator.Tick(halfTime) + assert.ElementsMatch(t, []SDKCredential{key2}, additions) + assert.Empty(t, expirations) + + additions, expirations = rotator.Tick(deprecationTime) + assert.Empty(t, additions) + assert.Empty(t, expirations) + + additions, expirations = rotator.Tick(deprecationTime.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{key1}, expirations) } -func TestMobileKeyDeprecation(t *testing.T) { +func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers, time.Now) + rotator := NewRotator(mockLog.Loggers) - const ( - key1 = config.MobileKey("key1") - key2 = config.MobileKey("key2") - key3 = config.MobileKey("key3") - ) + rotator.Initialize([]SDKCredential{config.SDKKey("key0")}) + + const numKeys = 250 + deprecationTime := time.Now().Add(1 * time.Minute) + + var keysDeprecated []SDKCredential + var keysAdded []SDKCredential + + for i := 0; i < numKeys; i++ { + previousKey := config.SDKKey(fmt.Sprintf("key%v", i)) + nextKey := config.SDKKey(fmt.Sprintf("key%v", i+1)) + + keysDeprecated = append(keysDeprecated, previousKey) + keysAdded = append(keysAdded, nextKey) + + rotator.RotateSDKKey(nextKey, NewDeprecationNotice(previousKey, deprecationTime)) + } - rotator.RotateMobileKey(key1) - requireChanValue(t, rotator.Additions(), key1) - assert.Equal(t, key1, rotator.MobileKey()) + assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.SDKKey()) - rotator.RotateMobileKey(key2) - requireChanValue(t, rotator.Additions(), key2) - requireChanValue(t, rotator.Expirations(), key1) - assert.Equal(t, key2, rotator.MobileKey()) + additions, expirations := rotator.Tick(deprecationTime) + assert.ElementsMatch(t, keysAdded, additions) + assert.Empty(t, expirations) - rotator.RotateMobileKey(key3) - requireChanValue(t, rotator.Additions(), key3) - requireChanValue(t, rotator.Expirations(), key2) - assert.Equal(t, key3, rotator.MobileKey()) + additions, expirations = rotator.Tick(deprecationTime.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, keysDeprecated, expirations) } From 74858dad49138c364e2148e5d1b62ba6cb5ab58b Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Fri, 21 Jun 2024 14:30:19 -0700 Subject: [PATCH 10/26] update tests and add new configuration item --- config/config.go | 5 + config/config_validation.go | 10 + config/test_data_configs_invalid_test.go | 13 ++ config/test_data_configs_valid_test.go | 3 + internal/credential/credential.go | 13 -- internal/credential/rotator.go | 80 ++++---- internal/credential/rotator_test.go | 211 +++++++++++---------- internal/relayenv/env_context.go | 40 +++- internal/relayenv/env_context_impl.go | 87 +++++---- internal/relayenv/env_context_impl_test.go | 66 ++++--- internal/sharedtest/testdata_envs.go | 6 - relay/autoconfig_actions.go | 13 +- relay/autoconfig_actions_test.go | 4 + relay/autoconfig_key_change_test.go | 6 +- relay/relay.go | 27 +-- 15 files changed, 334 insertions(+), 250 deletions(-) diff --git a/config/config.go b/config/config.go index bc02d915..5ed3db2b 100644 --- a/config/config.go +++ b/config/config.go @@ -88,6 +88,10 @@ const ( // the CPU time it takes to read the archive over and over again. It is somewhat arbitrary. // It likely doesn't make sense to use an interval this frequent in production use-cases. minimumFileDataSourceMonitoringInterval = 100 * time.Millisecond + // This minimum was chosen to protect the host system from unnecessary work, while also allowing expired + // credentials to be revoked nearly instantaneously. It is not necessarily a recommendation. + // It likely doesn't make sense to use an interval this frequent in production use-cases. + minimumCredentialCleanupInterval = 100 * time.Millisecond ) // DefaultLoggers is the default logging configuration used by Relay. @@ -152,6 +156,7 @@ type MainConfig struct { LogLevel OptLogLevel `conf:"LOG_LEVEL"` BigSegmentsStaleAsDegraded bool `conf:"BIG_SEGMENTS_STALE_AS_DEGRADED"` BigSegmentsStaleThreshold ct.OptDuration `conf:"BIG_SEGMENTS_STALE_THRESHOLD"` + CredentialCleanupInterval ct.OptDuration `conf:"CREDENTIAL_CLEANUP_INTERVAL"` } // AutoConfigConfig contains configuration parameters for the auto-configuration feature. diff --git a/config/config_validation.go b/config/config_validation.go index b126062b..bda5638e 100644 --- a/config/config_validation.go +++ b/config/config_validation.go @@ -24,6 +24,7 @@ var ( errAutoConfWithFilters = errors.New("cannot configure filters if auto-configuration is enabled") errMissingProjKey = errors.New("when filters are configured, all environments must specify a 'projKey'") errInvalidFileDataSourceMonitoringInterval = fmt.Errorf("file data source monitoring interval must be >= %s", minimumFileDataSourceMonitoringInterval) + errInvalidCredentialCleanupInterval = fmt.Errorf("credential cleanup interval must be >= %s", minimumCredentialCleanupInterval) ) func errEnvironmentWithNoSDKKey(envName string) error { @@ -78,6 +79,7 @@ func ValidateConfig(c *Config, loggers ldlog.Loggers) error { validateConfigDatabases(&result, c, loggers) validateConfigFilters(&result, c) validateOfflineMode(&result, c) + validateCredentialCleanupInterval(&result, c) return result.GetError() } @@ -196,6 +198,14 @@ func validateOfflineMode(result *ct.ValidationResult, c *Config) { } } +func validateCredentialCleanupInterval(result *ct.ValidationResult, c *Config) { + if c.Main.CredentialCleanupInterval.IsDefined() { + interval := c.Main.CredentialCleanupInterval.GetOrElse(0) + if interval < minimumCredentialCleanupInterval { + result.AddError(nil, errInvalidCredentialCleanupInterval) + } + } +} func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldlog.Loggers) { normalizeRedisConfig(result, c) diff --git a/config/test_data_configs_invalid_test.go b/config/test_data_configs_invalid_test.go index 41cd917b..e7843ff0 100644 --- a/config/test_data_configs_invalid_test.go +++ b/config/test_data_configs_invalid_test.go @@ -11,6 +11,9 @@ type testDataInvalidConfig struct { func makeInvalidConfigs() []testDataInvalidConfig { return []testDataInvalidConfig{ makeInvalidConfigMissingSDKKey(), + makeInvalidConfigCredentialCleanupInterval("0s"), + makeInvalidConfigCredentialCleanupInterval("-1s"), + makeInvalidConfigCredentialCleanupInterval("99ms"), makeInvalidConfigTLSWithNoCertOrKey(), makeInvalidConfigTLSWithNoCert(), makeInvalidConfigTLSWithNoKey(), @@ -255,6 +258,16 @@ fileDataSourceMonitoringInterval = ` + interval + ` return c } +func makeInvalidConfigCredentialCleanupInterval(interval string) testDataInvalidConfig { + c := testDataInvalidConfig{name: "credential cleanup interval with invalid value"} + c.fileError = errInvalidCredentialCleanupInterval.Error() + c.fileContent = ` +[Main] +credentialCleanupInterval = ` + interval + ` +` + return c +} + func makeInvalidConfigRedisInvalidHostname() testDataInvalidConfig { c := testDataInvalidConfig{name: "Redis - invalid hostname"} c.envVarsError = "invalid Redis hostname" diff --git a/config/test_data_configs_valid_test.go b/config/test_data_configs_valid_test.go index a06bdcd0..39dfff3d 100644 --- a/config/test_data_configs_valid_test.go +++ b/config/test_data_configs_valid_test.go @@ -115,6 +115,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { LogLevel: NewOptLogLevel(ldlog.Warn), BigSegmentsStaleAsDegraded: true, BigSegmentsStaleThreshold: ct.NewOptDuration(10 * time.Minute), + CredentialCleanupInterval: ct.NewOptDuration(1 * time.Minute), } c.Events = EventsConfig{ SendEvents: true, @@ -183,6 +184,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { "LD_ALLOWED_ORIGIN_krypton": "https://oa,https://rann", "LD_ALLOWED_HEADER_krypton": "Timestamp-Valid,Random-Id-Valid", "LD_TTL_krypton": "5m", + "CREDENTIAL_CLEANUP_INTERVAL": "1m", } c.fileContent = ` [Main] @@ -203,6 +205,7 @@ TLSMinVersion = "1.2" LogLevel = "warn" BigSegmentsStaleAsDegraded = 1 BigSegmentsStaleThreshold = 10m +CredentialCleanupInterval = 1m [Events] SendEvents = 1 diff --git a/internal/credential/credential.go b/internal/credential/credential.go index c0bb8191..09354588 100644 --- a/internal/credential/credential.go +++ b/internal/credential/credential.go @@ -15,19 +15,6 @@ type SDKCredential interface { Masked() string } -// Status represents that difference between an existing credential and one found in a new AutoConfig configuration -// struct. -type Status string - -const ( - // Unchanged means the credential has not changed. - Unchanged = Status("unchanged") - // Deprecated means the existing credential has been deprecated in favor of a new one. - Deprecated = Status("deprecated") - // Expired means the existing credential should be removed in favor of a new one. - Expired = Status("expired") -) - // AutoConfig represents credentials that are updated via AutoConfig protocol. type AutoConfig struct { // SDKKey is the environment's SDK key; if there is more than one active key, it is the latest. diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index ba6cc7c1..e331fb62 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -8,20 +8,6 @@ import ( "time" ) -type DeprecationNotice struct { - key config.SDKKey - expiry time.Time -} - -func NewDeprecationNotice(key config.SDKKey, expiry time.Time) *DeprecationNotice { - return &DeprecationNotice{key: key, expiry: expiry} -} - -type deprecatedKey struct { - expiry time.Time - timer *time.Timer -} - type Rotator struct { loggers ldlog.Loggers @@ -37,7 +23,7 @@ type Rotator struct { // Deprecated keys are stored in a map with a started timer for each key representing the deprecation period. // Upon expiration, they are removed. - deprecatedSdkKeys map[config.SDKKey]*deprecatedKey + deprecatedSdkKeys map[config.SDKKey]time.Time expirations []SDKCredential additions []SDKCredential @@ -54,7 +40,7 @@ type InitialCredentials struct { func NewRotator(loggers ldlog.Loggers) *Rotator { r := &Rotator{ loggers: loggers, - deprecatedSdkKeys: make(map[config.SDKKey]*deprecatedKey), + deprecatedSdkKeys: make(map[config.SDKKey]time.Time), } return r } @@ -133,7 +119,37 @@ func (r *Rotator) AllCredentials() []SDKCredential { return append(r.primaryCredentials(), r.deprecatedCredentials()...) } -func (r *Rotator) RotateEnvironmentID(envID config.EnvironmentID) { +func (r *Rotator) Rotate(cred SDKCredential) { + r.RotateWithGrace(cred, nil) +} + +type GracePeriod struct { + key config.SDKKey + expiry time.Time +} + +func NewGracePeriod(key config.SDKKey, expiry time.Time) *GracePeriod { + return &GracePeriod{key, expiry} +} + +func (r *Rotator) RotateWithGrace(primary SDKCredential, grace *GracePeriod) { + switch primary := primary.(type) { + case config.SDKKey: + r.updateSDKKey(primary, grace) + case config.MobileKey: + if grace != nil { + panic("programmer error: mobile keys do not support deprecation") + } + r.updateMobileKey(primary) + case config.EnvironmentID: + if grace != nil { + panic("programmer error: environment IDs do not support deprecation") + } + r.updateEnvironmentID(primary) + } +} + +func (r *Rotator) updateEnvironmentID(envID config.EnvironmentID) { if envID == r.EnvironmentID() { return } @@ -150,7 +166,7 @@ func (r *Rotator) RotateEnvironmentID(envID config.EnvironmentID) { } } -func (r *Rotator) RotateMobileKey(mobileKey config.MobileKey) { +func (r *Rotator) updateMobileKey(mobileKey config.MobileKey) { if mobileKey == r.MobileKey() { return } @@ -179,32 +195,30 @@ func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { return previous } -func (r *Rotator) RotateSDKKey(sdkKey config.SDKKey, deprecation *DeprecationNotice) { +func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { r.mu.Lock() defer r.mu.Unlock() previous := r.swapPrimaryKey(sdkKey) // Immediately revoke the previous SDK key if there's no explicit deprecation notice, otherwise it would // hang around forever. - if previous.Defined() && deprecation == nil { + if previous.Defined() && grace == nil { r.expirations = append(r.expirations, previous) r.loggers.Infof("SDK key %s has been immediately revoked", previous.Masked()) return } - if deprecation != nil { - if prev, ok := r.deprecatedSdkKeys[deprecation.key]; ok { - r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", deprecation.key.Masked(), deprecation.expiry, prev.expiry) + if grace != nil { + if previousExpiry, ok := r.deprecatedSdkKeys[grace.key]; ok { + r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", grace.key.Masked(), grace.expiry, previousExpiry) return } - r.loggers.Infof("SDK key %s was marked for deprecation with an expiry at %v", deprecation.key.Masked(), deprecation.expiry) - r.deprecatedSdkKeys[deprecation.key] = &deprecatedKey{ - expiry: deprecation.expiry, - } + r.loggers.Infof("SDK key %s was marked for deprecation with an expiry at %v", grace.key.Masked(), grace.expiry) + r.deprecatedSdkKeys[grace.key] = grace.expiry - if deprecation.key != previous { - r.loggers.Infof("Deprecated SDK key %s was not previously managed by Relay", deprecation.key.Masked()) - r.additions = append(r.additions, deprecation.key) + if grace.key != previous { + r.loggers.Infof("Deprecated SDK key %s was not previously managed by Relay", grace.key.Masked()) + r.additions = append(r.additions, grace.key) } } } @@ -215,12 +229,12 @@ func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.expirations = append(r.expirations, sdkKey) } -func (r *Rotator) Tick(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { +func (r *Rotator) Query(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { r.mu.Lock() defer r.mu.Unlock() - for key, dep := range r.deprecatedSdkKeys { - if now.After(dep.expiry) { + for key, expiry := range r.deprecatedSdkKeys { + if now.After(expiry) { r.expireSDKKey(key) } } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 32659811..4803f388 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -16,103 +16,98 @@ func TestNewRotator(t *testing.T) { } func TestImmediateKeyExpiration(t *testing.T) { - t.Run("sdk keys", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - const ( - key1 = config.SDKKey("key1") - key2 = config.SDKKey("key2") - key3 = config.SDKKey("key3") - ) - - // The first rotation shouldn't trigger any expirations because there was no previous key. - rotator.RotateSDKKey(key1, nil) - additions, _ := rotator.Tick(time.Now()) - assert.ElementsMatch(t, []SDKCredential{key1}, additions) - assert.Equal(t, key1, rotator.SDKKey()) - - // The second rotation should trigger a deprecation of key1. - rotator.RotateSDKKey(key2, nil) - additions, expirations := rotator.Tick(time.Now()) - assert.ElementsMatch(t, []SDKCredential{key2}, additions) - assert.ElementsMatch(t, []SDKCredential{key1}, expirations) - assert.Equal(t, key2, rotator.SDKKey()) - - // The third rotation should trigger a deprecation of key2. - rotator.RotateSDKKey(key3, nil) - additions, expirations = rotator.Tick(time.Now()) - assert.ElementsMatch(t, []SDKCredential{key3}, additions) - assert.ElementsMatch(t, []SDKCredential{key2}, expirations) - assert.Equal(t, key3, rotator.SDKKey()) - }) - - t.Run("mobile keys", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - const ( - key1 = config.MobileKey("key1") - key2 = config.MobileKey("key2") - key3 = config.MobileKey("key3") - ) - - // The first rotation shouldn't trigger any expirations because there was no previous key. - rotator.RotateMobileKey(key1) - additions, _ := rotator.Tick(time.Now()) - assert.ElementsMatch(t, []SDKCredential{key1}, additions) - assert.Equal(t, key1, rotator.MobileKey()) - - // The second rotation should trigger a deprecation of key1. - rotator.RotateMobileKey(key2) - additions, expirations := rotator.Tick(time.Now()) - assert.ElementsMatch(t, []SDKCredential{key2}, additions) - assert.ElementsMatch(t, []SDKCredential{key1}, expirations) - assert.Equal(t, key2, rotator.MobileKey()) - - // The third rotation should trigger a deprecation of key2. - rotator.RotateMobileKey(key3) - additions, expirations = rotator.Tick(time.Now()) - assert.ElementsMatch(t, []SDKCredential{key3}, additions) - assert.ElementsMatch(t, []SDKCredential{key2}, expirations) - assert.Equal(t, key3, rotator.MobileKey()) - }) + kinds := []struct { + name string + keys []SDKCredential + getKey func(*Rotator) SDKCredential + }{ + { + name: "sdk keys", + keys: []SDKCredential{config.SDKKey("key1"), config.SDKKey("key2"), config.SDKKey("key3")}, + getKey: func(r *Rotator) SDKCredential { return r.SDKKey() }, + }, + { + name: "mobile keys", + keys: []SDKCredential{config.MobileKey("key1"), config.MobileKey("key2"), config.MobileKey("key3")}, + getKey: func(r *Rotator) SDKCredential { return r.MobileKey() }, + }, + { + name: "environment IDs", + keys: []SDKCredential{config.EnvironmentID("id1"), config.EnvironmentID("id2"), config.EnvironmentID("id3")}, + getKey: func(r *Rotator) SDKCredential { return r.EnvironmentID() }, + }, + } + + for _, c := range kinds { + t.Run(c.name, func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + // The first rotation shouldn't trigger any expirations because there was no previous key. + rotator.Rotate(c.keys[0]) + additions, _ := rotator.Query(time.Now()) + assert.ElementsMatch(t, c.keys[0:1], additions) + assert.Equal(t, c.keys[0], c.getKey(rotator)) + + // The second rotation should trigger a deprecation of key1. + rotator.Rotate(c.keys[1]) + additions, expirations := rotator.Query(time.Now()) + assert.ElementsMatch(t, c.keys[1:2], additions) + assert.ElementsMatch(t, c.keys[0:1], expirations) + assert.Equal(t, c.keys[1], c.getKey(rotator)) + + // The third rotation should trigger a deprecation of key2. + rotator.Rotate(c.keys[2]) + additions, expirations = rotator.Query(time.Now()) + assert.ElementsMatch(t, c.keys[2:3], additions) + assert.ElementsMatch(t, c.keys[1:2], expirations) + assert.Equal(t, c.keys[2], c.getKey(rotator)) + }) + } } func TestManyImmediateKeyExpirations(t *testing.T) { - t.Run("sdk keys", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - const numKeys = 100 - for i := 0; i < numKeys; i++ { - key := config.SDKKey(fmt.Sprintf("key%v", i)) - rotator.RotateSDKKey(key, nil) - } - - assert.Equal(t, config.SDKKey(fmt.Sprintf("key%v", numKeys-1)), rotator.SDKKey()) - - additions, expirations := rotator.Tick(time.Now()) - assert.Len(t, additions, numKeys) - assert.Len(t, expirations, numKeys-1) // because the last key is still active - }) - - t.Run("mobile keys", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - const numKeys = 100 - for i := 0; i < numKeys; i++ { - key := config.MobileKey(fmt.Sprintf("key%v", i)) - rotator.RotateMobileKey(key) - } - - assert.Equal(t, config.MobileKey(fmt.Sprintf("key%v", numKeys-1)), rotator.MobileKey()) - - additions, expirations := rotator.Tick(time.Now()) - assert.Len(t, additions, numKeys) - assert.Len(t, expirations, numKeys-1) // because the last key is still active - }) + + kinds := []struct { + name string + makeKey func(string) SDKCredential + getKey func(*Rotator) SDKCredential + }{ + { + name: "sdk keys", + makeKey: func(s string) SDKCredential { return config.SDKKey(s) }, + getKey: func(r *Rotator) SDKCredential { return r.SDKKey() }, + }, + { + name: "mobile keys", + makeKey: func(s string) SDKCredential { return config.MobileKey(s) }, + getKey: func(r *Rotator) SDKCredential { return r.MobileKey() }, + }, + { + name: "environment IDs", + makeKey: func(s string) SDKCredential { return config.EnvironmentID(s) }, + getKey: func(r *Rotator) SDKCredential { return r.EnvironmentID() }, + }, + } + + for _, c := range kinds { + t.Run(c.name, func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + const numKeys = 100 + for i := 0; i < numKeys; i++ { + key := c.makeKey(fmt.Sprintf("key%v", i)) + rotator.Rotate(key) + } + + assert.Equal(t, c.makeKey(fmt.Sprintf("key%v", numKeys-1)), c.getKey(rotator)) + + additions, expirations := rotator.Query(time.Now()) + assert.Len(t, additions, numKeys) + assert.Len(t, expirations, numKeys-1) // because the last key is still active + }) + } } func TestSDKKeyDeprecation(t *testing.T) { @@ -122,7 +117,6 @@ func TestSDKKeyDeprecation(t *testing.T) { const ( key1 = config.SDKKey("key1") key2 = config.SDKKey("key2") - key3 = config.SDKKey("key3") ) start := time.Now() @@ -132,16 +126,16 @@ func TestSDKKeyDeprecation(t *testing.T) { rotator.Initialize([]SDKCredential{key1}) - rotator.RotateSDKKey(key2, NewDeprecationNotice(key1, deprecationTime)) - additions, expirations := rotator.Tick(halfTime) + rotator.RotateWithGrace(key2, NewGracePeriod(key1, deprecationTime)) + additions, expirations := rotator.Query(halfTime) assert.ElementsMatch(t, []SDKCredential{key2}, additions) assert.Empty(t, expirations) - additions, expirations = rotator.Tick(deprecationTime) + additions, expirations = rotator.Query(deprecationTime) assert.Empty(t, additions) assert.Empty(t, expirations) - additions, expirations = rotator.Tick(deprecationTime.Add(1 * time.Millisecond)) + additions, expirations = rotator.Query(deprecationTime.Add(1 * time.Millisecond)) assert.Empty(t, additions) assert.ElementsMatch(t, []SDKCredential{key1}, expirations) } @@ -150,31 +144,38 @@ func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { mockLog := ldlogtest.NewMockLog() rotator := NewRotator(mockLog.Loggers) + makeKey := func(i int) config.SDKKey { + return config.SDKKey(fmt.Sprintf("key%v", i)) + } + rotator.Initialize([]SDKCredential{config.SDKKey("key0")}) const numKeys = 250 - deprecationTime := time.Now().Add(1 * time.Minute) + expiryTime := time.Now().Add(1 * time.Minute) var keysDeprecated []SDKCredential var keysAdded []SDKCredential for i := 0; i < numKeys; i++ { - previousKey := config.SDKKey(fmt.Sprintf("key%v", i)) - nextKey := config.SDKKey(fmt.Sprintf("key%v", i+1)) + previousKey := makeKey(i) + nextKey := makeKey(i + 1) keysDeprecated = append(keysDeprecated, previousKey) keysAdded = append(keysAdded, nextKey) - rotator.RotateSDKKey(nextKey, NewDeprecationNotice(previousKey, deprecationTime)) + rotator.RotateWithGrace(nextKey, NewGracePeriod(previousKey, expiryTime)) } + // The last key added should be the current primary key. assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.SDKKey()) - additions, expirations := rotator.Tick(deprecationTime) + // Until and including the exact expiry timestamp, there should be no expirations. + additions, expirations := rotator.Query(expiryTime) assert.ElementsMatch(t, keysAdded, additions) assert.Empty(t, expirations) - additions, expirations = rotator.Tick(deprecationTime.Add(1 * time.Millisecond)) + // One moment after the expiry time, we should now have a batch of expirations. + additions, expirations = rotator.Query(expiryTime.Add(1 * time.Millisecond)) assert.Empty(t, additions) assert.ElementsMatch(t, keysDeprecated, expirations) } diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index 5c704e6f..c20b3ea3 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -20,9 +20,35 @@ import ( ldeval "github.com/launchdarkly/go-server-sdk-evaluation/v3" ) -type DeprecationHooks struct { - BeforeRemoval func(cred credential.SDKCredential) - AfterRemoval func(cred credential.SDKCredential) +// CredentialUpdate specifies the primary credential of a given credential kind for an environment. +// For example, an environment may have a primary SDK key and a primary mobile key at the same time; each would +// be specified in individual CredentialUpdate objects. +type CredentialUpdate struct { + primary credential.SDKCredential + gracePeriod *credential.GracePeriod + now time.Time +} + +// NewCredentialUpdate creates a CredentialUpdate from a given primary credential. +// The default behavior of the environment is to immediately revoke the previous credential of this kind. +func NewCredentialUpdate(primary credential.SDKCredential) *CredentialUpdate { + return &CredentialUpdate{primary: primary, now: time.Now()} +} + +// WithGracePeriod modifies the default behavior from immediate revocation to a delayed revocation of the previous +// credential. During the grace period, the previous credential continues to function. +func (c *CredentialUpdate) WithGracePeriod(deprecated config.SDKKey, expiry time.Time) *CredentialUpdate { + c.gracePeriod = credential.NewGracePeriod(deprecated, expiry) + return c +} + +// WithTime overrides the update's current time for testing purposes. +// Because the environment's credential rotation algorithm compares the current time to the specific expiry of +// each credential, this can be used to trigger behavior in a more predictable way than relying on the actual time +// in the test. +func (c *CredentialUpdate) WithTime(t time.Time) *CredentialUpdate { + c.now = t + return c } // EnvContext is the interface for all Relay operations that are specific to one configured LD environment. @@ -44,16 +70,16 @@ type EnvContext interface { // SetIdentifiers updates the environment and project names and keys. SetIdentifiers(EnvIdentifiers) + // UpdateCredential updates the environment with a new credential, optionally deprecating a previous one + // with a grace period. + UpdateCredential(update *CredentialUpdate) + // GetCredentials returns all currently enabled and non-deprecated credentials for the environment. GetCredentials() []credential.SDKCredential // GetDeprecatedCredentials returns all deprecated and not-yet-removed credentials for the environment. GetDeprecatedCredentials() []credential.SDKCredential - RotateMobileKey(key config.MobileKey) - RotateEnvironmentID(id config.EnvironmentID) - RotateSDKKey(newKey config.SDKKey, notice *credential.DeprecationNotice) - // GetClient returns the SDK client instance for this environment. This is nil if initialization is not yet // complete. Rather than providing the full client object, we use the simpler sdks.LDClientContext which // includes only the operations Relay needs to do. diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index a77991a1..2f9c7ad2 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -45,6 +45,11 @@ const ( // ID. This is the default behavior for Relay Proxy Enterprise when running in auto-configuration mode, // where we always know the environment ID but the SDK key is subject to change. LogNameIsEnvID LogNameMode = true + + // By default, credentials that have an expiry date in the future (compared to when the message containing the + // expiry was received) will be cleaned up on an interval with this granularity. This means the environment won't accept + // connections for this credential, and it will shut down the SDK client associated with that credential. + defaultCredentialCleanupInterval = 1 * time.Minute ) func errInitPublisher(err error) error { @@ -78,8 +83,8 @@ type EnvContextImplParams struct { UserAgent string LogNameMode LogNameMode Loggers ldlog.Loggers - TimeSource func() time.Time ConnectionMapper ConnectionMapper + CredentialCleanupInterval time.Duration } type envContextImpl struct { @@ -181,7 +186,7 @@ func NewEnvContext( dataStoreInfo: params.DataStoreInfo, creationTime: time.Now(), filterKey: params.EnvConfig.FilterKey, - keyRotator: credential.NewRotator(params.Loggers, params.TimeSource), + keyRotator: credential.NewRotator(params.Loggers), stopMonitoringCredentials: make(chan struct{}), doneMonitoringCredentials: make(chan struct{}), connectionMapper: params.ConnectionMapper, @@ -385,16 +390,34 @@ func NewEnvContext( } } - go envContext.monitorCredentialChanges(envContext.doneMonitoringCredentials) - // Connecting may take time, so do this in parallel go envContext.startSDKClient(envConfig.SDKKey, readyCh, allConfig.Main.IgnoreConnectionErrors) + cleanupInterval := params.CredentialCleanupInterval + if cleanupInterval == 0 { // 0 means it wasn't specified; the config system disallows 0 as a valid value. + cleanupInterval = defaultCredentialCleanupInterval + } + go envContext.cleanupExpiredCredentials(cleanupInterval) + thingsToCleanUp.Clear() // we've succeeded so we do not want to throw away these things return envContext, nil } +func (c *envContextImpl) cleanupExpiredCredentials(interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + c.triggerCredentialChanges(time.Now()) + case <-c.stopMonitoringCredentials: + close(c.doneMonitoringCredentials) + return + } + } +} + func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { c.mu.Lock() defer c.mu.Unlock() @@ -443,21 +466,6 @@ func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential } } -func (c *envContextImpl) monitorCredentialChanges(done chan struct{}) { - defer close(done) - for { - select { - case <-c.stopMonitoringCredentials: - return - case oldCredential := <-c.keyRotator.Expirations(): - c.removeCredential(oldCredential) - case newCredential := <-c.keyRotator.Additions(): - - c.addCredential(newCredential) - } - } -} - func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- EnvContext, suppressErrors bool) { client, err := c.sdkClientFactory(sdkKey, c.sdkConfig, c.sdkInitTimeout) c.mu.Lock() @@ -522,24 +530,31 @@ func (c *envContextImpl) SetIdentifiers(ei EnvIdentifiers) { c.identifiers = ei } -func (c *envContextImpl) GetCredentials() []credential.SDKCredential { - return c.keyRotator.PrimaryCredentials() -} - -func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { - return c.keyRotator.DeprecatedCredentials() +func (c *envContextImpl) UpdateCredential(update *CredentialUpdate) { + if update.gracePeriod == nil { + c.keyRotator.Rotate(update.primary) + } else { + c.keyRotator.RotateWithGrace(update.primary, update.gracePeriod) + } + c.triggerCredentialChanges(update.now) } -func (c *envContextImpl) RotateMobileKey(key config.MobileKey) { - c.keyRotator.RotateMobileKey(key) +func (c *envContextImpl) triggerCredentialChanges(now time.Time) { + additions, expirations := c.keyRotator.Query(now) + for _, cred := range additions { + c.addCredential(cred) + } + for _, cred := range expirations { + c.removeCredential(cred) + } } -func (c *envContextImpl) RotateEnvironmentID(id config.EnvironmentID) { - c.keyRotator.RotateEnvironmentID(id) +func (c *envContextImpl) GetCredentials() []credential.SDKCredential { + return c.keyRotator.PrimaryCredentials() } -func (c *envContextImpl) RotateSDKKey(newKey config.SDKKey, notice *credential.DeprecationNotice) { - c.keyRotator.RotateSDKKey(newKey, notice) +func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { + return c.keyRotator.DeprecatedCredentials() } func (c *envContextImpl) GetClient() sdks.LDClientContext { @@ -659,10 +674,6 @@ func (c *envContextImpl) FlushMetricsEvents() { c.metricsEventPub.Flush() } } -func (c *envContextImpl) stopCredentialMonitor() { - close(c.stopMonitoringCredentials) - <-c.doneMonitoringCredentials -} func (c *envContextImpl) Close() error { c.mu.Lock() @@ -671,9 +682,11 @@ func (c *envContextImpl) Close() error { } c.clients = make(map[config.SDKKey]sdks.LDClientContext) c.mu.Unlock() - _ = c.envStreams.Close() - c.stopCredentialMonitor() + close(c.stopMonitoringCredentials) + <-c.doneMonitoringCredentials + + _ = c.envStreams.Close() if c.metricsManager != nil && c.metricsEnv != nil { c.metricsManager.RemoveEnvironment(c.metricsEnv) diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 489959f9..62773904 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -57,7 +57,15 @@ func requireClientReady(t *testing.T, clientCh chan *testclient.FakeLDClient) *t func makeBasicEnv(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.ClientFactoryFunc, loggers ldlog.Loggers, readyCh chan EnvContext) EnvContext { - return makeBasicEnvWithMockTime(t, envConfig, clientFactory, loggers, readyCh, nil) + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: envName}, + EnvConfig: envConfig, + ClientFactory: clientFactory, + Loggers: loggers, + ConnectionMapper: mockConnectionMapper{}, + }, readyCh) + require.NoError(t, err) + return env } type mockConnectionMapper struct { @@ -70,20 +78,6 @@ func (m mockConnectionMapper) RemoveConnectionMapping(scopedCredential sdkauth.S } -func makeBasicEnvWithMockTime(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.ClientFactoryFunc, - loggers ldlog.Loggers, readyCh chan EnvContext, now func() time.Time) EnvContext { - env, err := NewEnvContext(EnvContextImplParams{ - Identifiers: EnvIdentifiers{ConfiguredName: envName}, - EnvConfig: envConfig, - ClientFactory: clientFactory, - Loggers: loggers, - TimeSource: now, - ConnectionMapper: mockConnectionMapper{}, - }, readyCh) - require.NoError(t, err) - return env -} - func TestConstructorBasicProperties(t *testing.T) { envConfig := st.EnvWithAllCredentials.Config envConfig.TTL = configtypes.NewOptDuration(time.Hour) @@ -193,8 +187,8 @@ func TestAddRemoveCredential(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) - env.RotateMobileKey(st.EnvWithAllCredentials.Config.MobileKey) - env.RotateEnvironmentID(st.EnvWithAllCredentials.Config.EnvID) + env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.MobileKey)) + env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.EnvID)) creds := env.GetCredentials() assert.Len(t, creds, 3) @@ -202,7 +196,7 @@ func TestAddRemoveCredential(t *testing.T) { assert.Contains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) assert.Contains(t, creds, st.EnvWithAllCredentials.Config.EnvID) - env.RotateMobileKey("foo") + env.UpdateCredential(NewCredentialUpdate(config.MobileKey("evict-the-previous-key"))) creds = env.GetCredentials() assert.Len(t, creds, 3) @@ -222,14 +216,14 @@ func TestAddExistingCredentialDoesNothing(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) - env.RotateMobileKey(st.EnvWithAllCredentials.Config.MobileKey) + env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.MobileKey)) creds := env.GetCredentials() assert.Len(t, creds, 2) assert.Contains(t, creds, envConfig.SDKKey) assert.Contains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) - env.RotateMobileKey(st.EnvWithAllCredentials.Config.MobileKey) + env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.MobileKey)) creds = env.GetCredentials() assert.Len(t, creds, 2) @@ -241,7 +235,6 @@ func TestChangeSDKKey(t *testing.T) { envConfig := st.EnvMain.Config readyCh := make(chan EnvContext, 1) key2 := config.SDKKey("key2") - key3 := config.SDKKey("key3") clientCh := make(chan *testclient.FakeLDClient, 1) clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) @@ -257,7 +250,19 @@ func TestChangeSDKKey(t *testing.T) { assert.Equal(t, env.GetClient(), client1) assert.Nil(t, env.GetInitError()) - env.RotateSDKKey(key2, credential.NewDeprecationNotice(envConfig.SDKKey, time.Now().Add(1*time.Hour))) + // The environment should have been initialized with a single SDK key (found in the envConfig.) + // At this point, there's no deprecated credentials. + assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) + assert.Empty(t, env.GetDeprecatedCredentials()) + + // For the purposes of key rotation, we'll make time deterministic. + start := time.Unix(1000, 0) + + // Upon rotating to key2, the original key should still be valid for a hour. + env.UpdateCredential( + NewCredentialUpdate(key2). + WithGracePeriod(envConfig.SDKKey, start.Add(1*time.Hour)). + WithTime(start)) assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetDeprecatedCredentials()) @@ -266,15 +271,24 @@ func TestChangeSDKKey(t *testing.T) { assert.NotEqual(t, client1, client2) assert.Equal(t, env.GetClient(), client2) + // The client for the original SDK key should not have been closed, since it's valid for an hour. if !helpers.AssertChannelNotClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should not have been closed yet") { t.FailNow() } - env.RotateSDKKey(key3, nil) - assert.Equal(t, []credential.SDKCredential{key3}, env.GetCredentials()) - assert.ElementsMatch(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetDeprecatedCredentials()) + // Simulate an amount of time passing that is less than the deprecation period. The original key should still be valid. + env.UpdateCredential(NewCredentialUpdate(key2).WithTime(start.Add(45 * time.Minute))) + if !helpers.AssertChannelNotClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should not have been closed yet") { + t.FailNow() + } + + // We are now an instant after the deprecation period. This should cause the original key to become expired + // and trigger the client to close. + env.UpdateCredential(NewCredentialUpdate(key2).WithTime(start.Add(1*time.Hour + 1*time.Millisecond))) + assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) + assert.Empty(t, env.GetDeprecatedCredentials()) - if !helpers.AssertChannelClosed(t, client2.CloseCh, 1*time.Second, "client for key2 should have been closed") { + if !helpers.AssertChannelClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should have been closed") { t.FailNow() } diff --git a/internal/sharedtest/testdata_envs.go b/internal/sharedtest/testdata_envs.go index ce90dead..adc81600 100644 --- a/internal/sharedtest/testdata_envs.go +++ b/internal/sharedtest/testdata_envs.go @@ -3,8 +3,6 @@ package sharedtest import ( "time" - "github.com/launchdarkly/ld-relay/v8/internal/credential" - "github.com/launchdarkly/ld-relay/v8/config" ct "github.com/launchdarkly/go-configtypes" @@ -24,10 +22,6 @@ type TestEnv struct { type UnsupportedSDKCredential struct{} // implements credential.SDKCredential -func (k UnsupportedSDKCredential) Compare(_ credential.AutoConfig) (credential.SDKCredential, credential.Status) { - return nil, credential.Unchanged -} - func (k UnsupportedSDKCredential) GetAuthorizationHeaderValue() string { return "" } func (k UnsupportedSDKCredential) Defined() bool { diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index a9ef6145..9588e8cf 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -2,8 +2,8 @@ package relay import ( "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" ) @@ -34,7 +34,8 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar } if params.ExpiringSDKKey.Defined() { - env.RotateSDKKey(params.SDKKey, credential.NewDeprecationNotice(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration)) + update := relayenv.NewCredentialUpdate(params.SDKKey) + env.UpdateCredential(update.WithGracePeriod(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration)) } } @@ -50,14 +51,14 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment env.SetSecureMode(params.SecureMode) if params.MobileKey.Defined() { - env.RotateMobileKey(params.MobileKey) + env.UpdateCredential(relayenv.NewCredentialUpdate(params.MobileKey)) } if params.SDKKey.Defined() { - var deprecation *credential.DeprecationNotice + update := relayenv.NewCredentialUpdate(params.SDKKey) if params.ExpiringSDKKey.Defined() { - deprecation = credential.NewDeprecationNotice(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration) + update = update.WithGracePeriod(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration) } - env.RotateSDKKey(params.SDKKey, deprecation) + env.UpdateCredential(update) } } diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index a8a78bec..0116f901 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -71,6 +71,10 @@ func autoConfTest( config.Events.EventsURI, _ = configtypes.NewOptURLAbsoluteFromString(eventsServer.URL) config.Events.FlushInterval = configtypes.NewOptDuration(time.Millisecond * 10) + // In tests involving adding/removing credentials, allow Relay to clean up credentials quickly so as not + // to take more time than necessary to verify the test conditions. + config.Main.CredentialCleanupInterval = configtypes.NewOptDuration(time.Millisecond * 100) + relay, err := newRelayInternal(config, relayInternalOptions{ loggers: mockLog.Loggers, clientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientsCreatedCh), diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index 5cde9513..aa2ae03c 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -161,7 +161,7 @@ func TestEventForwardingAfterSDKKeyChange(t *testing.T) { p.awaitCredentialsUpdated(env, modified.params()) - verifyEventProxying(t, p, serverSideEventsURL, modified.SDKKey()) + verifyEventProxying(t, p, serverSideEventsURL, modified.sdkKey.Value) verifyEventProxying(t, p, mobileEventsURL, testAutoConfEnv1.mobKey) verifyEventProxying(t, p, jsEventsURL+string(testAutoConfEnv1.id), testAutoConfEnv1.id) }) @@ -195,9 +195,7 @@ func TestAutoConfigRemovesCredentialForExpiredSDKKey(t *testing.T) { foundEnvWithOldKey, _ := p.relay.getEnvironment(sdkauth.New(oldKey)) assert.Equal(t, env, foundEnvWithOldKey) - <-time.After(time.Duration(briefExpiryMillis+100) * time.Millisecond) - - if !helpers.AssertChannelClosed(t, client1.CloseCh, time.Millisecond*300, "timed out waiting for client with old key to close") { + if !helpers.AssertChannelClosed(t, client1.CloseCh, time.Duration(briefExpiryMillis+100)*time.Millisecond, "timed out waiting for client with old key to close") { t.FailNow() } diff --git a/relay/relay.go b/relay/relay.go index 4d05a79a..31b80b11 100644 --- a/relay/relay.go +++ b/relay/relay.go @@ -444,19 +444,20 @@ func (r *Relay) addEnvironment( return r.clientFactory(sdkKey, config, timeout) } clientContext, err := relayenv.NewEnvContext(relayenv.EnvContextImplParams{ - Identifiers: identifiers, - EnvConfig: envConfig, - AllConfig: r.config, - ClientFactory: wrappedClientFactory, - DataStoreFactory: dataStoreFactory, - DataStoreInfo: dataStoreInfo, - StreamProviders: r.allStreamProviders(), - JSClientContext: jsClientContext, - MetricsManager: r.metricsManager, - UserAgent: r.userAgent, - LogNameMode: r.envLogNameMode, - Loggers: r.loggers, - ConnectionMapper: r, + Identifiers: identifiers, + EnvConfig: envConfig, + AllConfig: r.config, + ClientFactory: wrappedClientFactory, + DataStoreFactory: dataStoreFactory, + DataStoreInfo: dataStoreInfo, + StreamProviders: r.allStreamProviders(), + JSClientContext: jsClientContext, + MetricsManager: r.metricsManager, + UserAgent: r.userAgent, + LogNameMode: r.envLogNameMode, + Loggers: r.loggers, + ConnectionMapper: r, + CredentialCleanupInterval: r.config.Main.CredentialCleanupInterval.GetOrElse(0), }, resultCh) if err != nil { return nil, nil, errNewClientContextFailed(identifiers.GetDisplayName(), err) From 4c8d4aa446a6c43e0d5827b914c2e12cefc96f20 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Fri, 21 Jun 2024 14:37:34 -0700 Subject: [PATCH 11/26] update docs --- config/config.go | 38 ++++---- config/config_validation.go | 6 +- config/test_data_configs_invalid_test.go | 2 +- config/test_data_configs_valid_test.go | 114 +++++++++++------------ docs/configuration.md | 45 +++++---- internal/relayenv/env_context_impl.go | 36 +++---- relay/autoconfig_actions_test.go | 2 +- relay/relay.go | 28 +++--- 8 files changed, 138 insertions(+), 133 deletions(-) diff --git a/config/config.go b/config/config.go index 5ed3db2b..9fa82a86 100644 --- a/config/config.go +++ b/config/config.go @@ -138,25 +138,25 @@ type Config struct { // variables, individual fields are not documented here; instead, see the `README.md` section on // configuration. type MainConfig struct { - ExitOnError bool `conf:"EXIT_ON_ERROR"` - ExitAlways bool `conf:"EXIT_ALWAYS"` - IgnoreConnectionErrors bool `conf:"IGNORE_CONNECTION_ERRORS"` - StreamURI ct.OptURLAbsolute `conf:"STREAM_URI"` - BaseURI ct.OptURLAbsolute `conf:"BASE_URI"` - ClientSideBaseURI ct.OptURLAbsolute `conf:"CLIENT_SIDE_BASE_URI"` - Port ct.OptIntGreaterThanZero `conf:"PORT"` - InitTimeout ct.OptDuration `conf:"INIT_TIMEOUT"` - HeartbeatInterval ct.OptDuration `conf:"HEARTBEAT_INTERVAL"` - MaxClientConnectionTime ct.OptDuration `conf:"MAX_CLIENT_CONNECTION_TIME"` - DisconnectedStatusTime ct.OptDuration `conf:"DISCONNECTED_STATUS_TIME"` - TLSEnabled bool `conf:"TLS_ENABLED"` - TLSCert string `conf:"TLS_CERT"` - TLSKey string `conf:"TLS_KEY"` - TLSMinVersion OptTLSVersion `conf:"TLS_MIN_VERSION"` - LogLevel OptLogLevel `conf:"LOG_LEVEL"` - BigSegmentsStaleAsDegraded bool `conf:"BIG_SEGMENTS_STALE_AS_DEGRADED"` - BigSegmentsStaleThreshold ct.OptDuration `conf:"BIG_SEGMENTS_STALE_THRESHOLD"` - CredentialCleanupInterval ct.OptDuration `conf:"CREDENTIAL_CLEANUP_INTERVAL"` + ExitOnError bool `conf:"EXIT_ON_ERROR"` + ExitAlways bool `conf:"EXIT_ALWAYS"` + IgnoreConnectionErrors bool `conf:"IGNORE_CONNECTION_ERRORS"` + StreamURI ct.OptURLAbsolute `conf:"STREAM_URI"` + BaseURI ct.OptURLAbsolute `conf:"BASE_URI"` + ClientSideBaseURI ct.OptURLAbsolute `conf:"CLIENT_SIDE_BASE_URI"` + Port ct.OptIntGreaterThanZero `conf:"PORT"` + InitTimeout ct.OptDuration `conf:"INIT_TIMEOUT"` + HeartbeatInterval ct.OptDuration `conf:"HEARTBEAT_INTERVAL"` + MaxClientConnectionTime ct.OptDuration `conf:"MAX_CLIENT_CONNECTION_TIME"` + DisconnectedStatusTime ct.OptDuration `conf:"DISCONNECTED_STATUS_TIME"` + TLSEnabled bool `conf:"TLS_ENABLED"` + TLSCert string `conf:"TLS_CERT"` + TLSKey string `conf:"TLS_KEY"` + TLSMinVersion OptTLSVersion `conf:"TLS_MIN_VERSION"` + LogLevel OptLogLevel `conf:"LOG_LEVEL"` + BigSegmentsStaleAsDegraded bool `conf:"BIG_SEGMENTS_STALE_AS_DEGRADED"` + BigSegmentsStaleThreshold ct.OptDuration `conf:"BIG_SEGMENTS_STALE_THRESHOLD"` + ExpiredCredentialCleanupInterval ct.OptDuration `conf:"EXPIRED_CREDENTIAL_CLEANUP_INTERVAL"` } // AutoConfigConfig contains configuration parameters for the auto-configuration feature. diff --git a/config/config_validation.go b/config/config_validation.go index bda5638e..9708c076 100644 --- a/config/config_validation.go +++ b/config/config_validation.go @@ -24,7 +24,7 @@ var ( errAutoConfWithFilters = errors.New("cannot configure filters if auto-configuration is enabled") errMissingProjKey = errors.New("when filters are configured, all environments must specify a 'projKey'") errInvalidFileDataSourceMonitoringInterval = fmt.Errorf("file data source monitoring interval must be >= %s", minimumFileDataSourceMonitoringInterval) - errInvalidCredentialCleanupInterval = fmt.Errorf("credential cleanup interval must be >= %s", minimumCredentialCleanupInterval) + errInvalidCredentialCleanupInterval = fmt.Errorf("expired credential cleanup interval must be >= %s", minimumCredentialCleanupInterval) ) func errEnvironmentWithNoSDKKey(envName string) error { @@ -199,8 +199,8 @@ func validateOfflineMode(result *ct.ValidationResult, c *Config) { } func validateCredentialCleanupInterval(result *ct.ValidationResult, c *Config) { - if c.Main.CredentialCleanupInterval.IsDefined() { - interval := c.Main.CredentialCleanupInterval.GetOrElse(0) + if c.Main.ExpiredCredentialCleanupInterval.IsDefined() { + interval := c.Main.ExpiredCredentialCleanupInterval.GetOrElse(0) if interval < minimumCredentialCleanupInterval { result.AddError(nil, errInvalidCredentialCleanupInterval) } diff --git a/config/test_data_configs_invalid_test.go b/config/test_data_configs_invalid_test.go index e7843ff0..9d583bbc 100644 --- a/config/test_data_configs_invalid_test.go +++ b/config/test_data_configs_invalid_test.go @@ -263,7 +263,7 @@ func makeInvalidConfigCredentialCleanupInterval(interval string) testDataInvalid c.fileError = errInvalidCredentialCleanupInterval.Error() c.fileContent = ` [Main] -credentialCleanupInterval = ` + interval + ` +expiredCredentialCleanupInterval = ` + interval + ` ` return c } diff --git a/config/test_data_configs_valid_test.go b/config/test_data_configs_valid_test.go index 39dfff3d..b5d875a0 100644 --- a/config/test_data_configs_valid_test.go +++ b/config/test_data_configs_valid_test.go @@ -98,24 +98,24 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { c := testDataValidConfig{name: "all base properties"} c.makeConfig = func(c *Config) { c.Main = MainConfig{ - Port: mustOptIntGreaterThanZero(8333), - BaseURI: newOptURLAbsoluteMustBeValid("http://base"), - ClientSideBaseURI: newOptURLAbsoluteMustBeValid("http://clientbase"), - StreamURI: newOptURLAbsoluteMustBeValid("http://stream"), - ExitOnError: true, - ExitAlways: true, - IgnoreConnectionErrors: true, - HeartbeatInterval: ct.NewOptDuration(90 * time.Second), - MaxClientConnectionTime: ct.NewOptDuration(30 * time.Minute), - DisconnectedStatusTime: ct.NewOptDuration(3 * time.Minute), - TLSEnabled: true, - TLSCert: "cert", - TLSKey: "key", - TLSMinVersion: NewOptTLSVersion(tls.VersionTLS12), - LogLevel: NewOptLogLevel(ldlog.Warn), - BigSegmentsStaleAsDegraded: true, - BigSegmentsStaleThreshold: ct.NewOptDuration(10 * time.Minute), - CredentialCleanupInterval: ct.NewOptDuration(1 * time.Minute), + Port: mustOptIntGreaterThanZero(8333), + BaseURI: newOptURLAbsoluteMustBeValid("http://base"), + ClientSideBaseURI: newOptURLAbsoluteMustBeValid("http://clientbase"), + StreamURI: newOptURLAbsoluteMustBeValid("http://stream"), + ExitOnError: true, + ExitAlways: true, + IgnoreConnectionErrors: true, + HeartbeatInterval: ct.NewOptDuration(90 * time.Second), + MaxClientConnectionTime: ct.NewOptDuration(30 * time.Minute), + DisconnectedStatusTime: ct.NewOptDuration(3 * time.Minute), + TLSEnabled: true, + TLSCert: "cert", + TLSKey: "key", + TLSMinVersion: NewOptTLSVersion(tls.VersionTLS12), + LogLevel: NewOptLogLevel(ldlog.Warn), + BigSegmentsStaleAsDegraded: true, + BigSegmentsStaleThreshold: ct.NewOptDuration(10 * time.Minute), + ExpiredCredentialCleanupInterval: ct.NewOptDuration(1 * time.Minute), } c.Events = EventsConfig{ SendEvents: true, @@ -147,44 +147,44 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { } } c.envVars = map[string]string{ - "PORT": "8333", - "BASE_URI": "http://base", - "CLIENT_SIDE_BASE_URI": "http://clientbase", - "STREAM_URI": "http://stream", - "EXIT_ON_ERROR": "1", - "EXIT_ALWAYS": "1", - "IGNORE_CONNECTION_ERRORS": "1", - "HEARTBEAT_INTERVAL": "90s", - "MAX_CLIENT_CONNECTION_TIME": "30m", - "DISCONNECTED_STATUS_TIME": "3m", - "TLS_ENABLED": "1", - "TLS_CERT": "cert", - "TLS_KEY": "key", - "TLS_MIN_VERSION": "1.2", - "LOG_LEVEL": "warn", - "BIG_SEGMENTS_STALE_AS_DEGRADED": "true", - "BIG_SEGMENTS_STALE_THRESHOLD": "10m", - "USE_EVENTS": "1", - "EVENTS_HOST": "http://events", - "EVENTS_FLUSH_INTERVAL": "120s", - "EVENTS_CAPACITY": "500", - "EVENTS_INLINE_USERS": "1", - "LD_ENV_earth": "earth-sdk", - "LD_MOBILE_KEY_earth": "earth-mob", - "LD_CLIENT_SIDE_ID_earth": "earth-env", - "LD_PREFIX_earth": "earth-", - "LD_TABLE_NAME_earth": "earth-table", - "LD_LOG_LEVEL_earth": "debug", - "LD_ENV_krypton": "krypton-sdk", - "LD_MOBILE_KEY_krypton": "krypton-mob", - "LD_CLIENT_SIDE_ID_krypton": "krypton-env", - "LD_SECURE_MODE_krypton": "1", - "LD_PREFIX_krypton": "krypton-", - "LD_TABLE_NAME_krypton": "krypton-table", - "LD_ALLOWED_ORIGIN_krypton": "https://oa,https://rann", - "LD_ALLOWED_HEADER_krypton": "Timestamp-Valid,Random-Id-Valid", - "LD_TTL_krypton": "5m", - "CREDENTIAL_CLEANUP_INTERVAL": "1m", + "PORT": "8333", + "BASE_URI": "http://base", + "CLIENT_SIDE_BASE_URI": "http://clientbase", + "STREAM_URI": "http://stream", + "EXIT_ON_ERROR": "1", + "EXIT_ALWAYS": "1", + "IGNORE_CONNECTION_ERRORS": "1", + "HEARTBEAT_INTERVAL": "90s", + "MAX_CLIENT_CONNECTION_TIME": "30m", + "DISCONNECTED_STATUS_TIME": "3m", + "TLS_ENABLED": "1", + "TLS_CERT": "cert", + "TLS_KEY": "key", + "TLS_MIN_VERSION": "1.2", + "LOG_LEVEL": "warn", + "BIG_SEGMENTS_STALE_AS_DEGRADED": "true", + "BIG_SEGMENTS_STALE_THRESHOLD": "10m", + "USE_EVENTS": "1", + "EVENTS_HOST": "http://events", + "EVENTS_FLUSH_INTERVAL": "120s", + "EVENTS_CAPACITY": "500", + "EVENTS_INLINE_USERS": "1", + "LD_ENV_earth": "earth-sdk", + "LD_MOBILE_KEY_earth": "earth-mob", + "LD_CLIENT_SIDE_ID_earth": "earth-env", + "LD_PREFIX_earth": "earth-", + "LD_TABLE_NAME_earth": "earth-table", + "LD_LOG_LEVEL_earth": "debug", + "LD_ENV_krypton": "krypton-sdk", + "LD_MOBILE_KEY_krypton": "krypton-mob", + "LD_CLIENT_SIDE_ID_krypton": "krypton-env", + "LD_SECURE_MODE_krypton": "1", + "LD_PREFIX_krypton": "krypton-", + "LD_TABLE_NAME_krypton": "krypton-table", + "LD_ALLOWED_ORIGIN_krypton": "https://oa,https://rann", + "LD_ALLOWED_HEADER_krypton": "Timestamp-Valid,Random-Id-Valid", + "LD_TTL_krypton": "5m", + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "1m", } c.fileContent = ` [Main] @@ -205,7 +205,7 @@ TLSMinVersion = "1.2" LogLevel = "warn" BigSegmentsStaleAsDegraded = 1 BigSegmentsStaleThreshold = 10m -CredentialCleanupInterval = 1m +ExpiredCredentialCleanupInterval = 1m [Events] SendEvents = 1 diff --git a/docs/configuration.md b/docs/configuration.md index f937df4b..f7e4a436 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -45,26 +45,27 @@ For **Duration** settings, the value should be be an integer followed by `ms`, ` ### File section: `[Main]` -| Property in file | Environment var | Type | Default | Description | -|-------------------------------|----------------------------------|:--------:|:--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `streamUri` | `STREAM_URI` | URI | _(1)_ | URI for the LaunchDarkly streaming service. | -| `baseUri` | `BASE_URI` | URI | _(1)_ | URI for the LaunchDarkly polling service for server-side SDKs. | -| `clientSideBaseUri` | `CLIENT_SIDE_BASE_URI` | URI | _(1)_ | URI for the LaunchDarkly polling service for client-side SDKs. | -| `exitOnError` | `EXIT_ON_ERROR` | Boolean | `false` | Close the Relay Proxy if it encounters any error during initialization. The default behavior is that it will terminate with a non-zero exit code if the configuration options are completely invalid, or if there is an incorrect `AutoConfig` key, but will remain running if there is an error specific to one environment, such as an invalid SDK key. Setting this option to `true` makes it terminate in both cases. | -| `exitAlways` | `EXIT_ALWAYS` | Boolean | `false` | Close the Relay Proxy immediately after initializing all environments. Do not start an HTTP server. _(2)_ | -| `ignoreConnectionErrors` | `IGNORE_CONNECTION_ERRORS` | Boolean | `false` | Ignore any initial connectivity issues with LaunchDarkly. Best used when network connectivity is not reliable. | -| `port` | `PORT` | Number | `8030` | Port the Relay Proxy should listen on. | -| `initTimeout` | `INIT_TIMEOUT` | Duration | `10s` | How long the Relay Proxy should wait for an initial connection to LaunchDarkly. If this timeout elapses, the behavior depends on `ignoreConnectionErrors`: by default, it will quit, but if `ignoreConnectionErrors` is true it will go on trying to connect in the background while still allowing clients to connect to the Relay Proxy. To learn more, read [How connections are handled in error conditions](./proxy-mode.md#how-connections-are-handled-in-error-conditions). | -| `heartbeatInterval` | `HEARTBEAT_INTERVAL` | Number | `3m` | Interval for heartbeat messages to prevent read timeouts on streaming connections. Assumed to be in seconds if no unit is specified. | -| `maxClientConnectionTime` | `MAX_CLIENT_CONNECTION_TIME` | Duration | none | Maximum amount of time that Relay will allow a streaming connection from an SDK client to remain open. _(3)_ | -| `disconnectedStatusTime` | `DISCONNECTED_STATUS_TIME` | Duration | `1m` | How long a stream connection can be interrupted before Relay reports the status as "disconnected." _(4)_ | -| `tlsEnabled` | `TLS_ENABLED` | Boolean | `false` | Enable TLS on the Relay Proxy. Read: [Using TLS](./tls.md). | -| `tlsCert` | `TLS_CERT` | String | | Required if `tlsEnabled` is true. Path to TLS certificate file. | -| `tlsKey` | `TLS_KEY` | String | | Required if `tlsEnabled` is true. Path to TLS private key file. | -| `tlsMinVersion` | `TLS_MIN_VERSION` | String | | Set to "1.2", etc., to enforce a minimum TLS version for secure requests. | -| `logLevel` | `LOG_LEVEL` | String | `info` | Should be `debug`, `info`, `warn`, `error`, or `none`. To learn more, read [Logging](./logging.md). | -| `bigSegmentsStaleAsDegraded` | `BIG_SEGMENTS_STALE_AS_DEGRADED` | Boolean | `false` | Indicates if environments should be considered degraded if Big Segments are not fully synchronized. | -| `bigSegmentsStaleThreshold` | `BIG_SEGMENTS_STALE_THRESHOLD` | Duration | `5m` | Indicates how long until Big Segments should be considered stale. | +| Property in file | Environment var | Type | Default | Description | +|------------------------------------|---------------------------------------|:--------:|:--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `streamUri` | `STREAM_URI` | URI | _(1)_ | URI for the LaunchDarkly streaming service. | +| `baseUri` | `BASE_URI` | URI | _(1)_ | URI for the LaunchDarkly polling service for server-side SDKs. | +| `clientSideBaseUri` | `CLIENT_SIDE_BASE_URI` | URI | _(1)_ | URI for the LaunchDarkly polling service for client-side SDKs. | +| `exitOnError` | `EXIT_ON_ERROR` | Boolean | `false` | Close the Relay Proxy if it encounters any error during initialization. The default behavior is that it will terminate with a non-zero exit code if the configuration options are completely invalid, or if there is an incorrect `AutoConfig` key, but will remain running if there is an error specific to one environment, such as an invalid SDK key. Setting this option to `true` makes it terminate in both cases. | +| `exitAlways` | `EXIT_ALWAYS` | Boolean | `false` | Close the Relay Proxy immediately after initializing all environments. Do not start an HTTP server. _(2)_ | +| `ignoreConnectionErrors` | `IGNORE_CONNECTION_ERRORS` | Boolean | `false` | Ignore any initial connectivity issues with LaunchDarkly. Best used when network connectivity is not reliable. | +| `port` | `PORT` | Number | `8030` | Port the Relay Proxy should listen on. | +| `initTimeout` | `INIT_TIMEOUT` | Duration | `10s` | How long the Relay Proxy should wait for an initial connection to LaunchDarkly. If this timeout elapses, the behavior depends on `ignoreConnectionErrors`: by default, it will quit, but if `ignoreConnectionErrors` is true it will go on trying to connect in the background while still allowing clients to connect to the Relay Proxy. To learn more, read [How connections are handled in error conditions](./proxy-mode.md#how-connections-are-handled-in-error-conditions). | +| `heartbeatInterval` | `HEARTBEAT_INTERVAL` | Number | `3m` | Interval for heartbeat messages to prevent read timeouts on streaming connections. Assumed to be in seconds if no unit is specified. | +| `maxClientConnectionTime` | `MAX_CLIENT_CONNECTION_TIME` | Duration | none | Maximum amount of time that Relay will allow a streaming connection from an SDK client to remain open. _(3)_ | +| `disconnectedStatusTime` | `DISCONNECTED_STATUS_TIME` | Duration | `1m` | How long a stream connection can be interrupted before Relay reports the status as "disconnected." _(4)_ | +| `tlsEnabled` | `TLS_ENABLED` | Boolean | `false` | Enable TLS on the Relay Proxy. Read: [Using TLS](./tls.md). | +| `tlsCert` | `TLS_CERT` | String | | Required if `tlsEnabled` is true. Path to TLS certificate file. | +| `tlsKey` | `TLS_KEY` | String | | Required if `tlsEnabled` is true. Path to TLS private key file. | +| `tlsMinVersion` | `TLS_MIN_VERSION` | String | | Set to "1.2", etc., to enforce a minimum TLS version for secure requests. | +| `logLevel` | `LOG_LEVEL` | String | `info` | Should be `debug`, `info`, `warn`, `error`, or `none`. To learn more, read [Logging](./logging.md). | +| `bigSegmentsStaleAsDegraded` | `BIG_SEGMENTS_STALE_AS_DEGRADED` | Boolean | `false` | Indicates if environments should be considered degraded if Big Segments are not fully synchronized. | +| `bigSegmentsStaleThreshold` | `BIG_SEGMENTS_STALE_THRESHOLD` | Duration | `5m` | Indicates how long until Big Segments should be considered stale. | +| `expiredCredentialCleanupInterval` | `EXPIRED_CREDENTIAL_CLEANUP_INTERVAL` | Duration | `1m` | Specifies how often expired credentials for environments are cleaned up. _(5)_ | _(1)_ The default values for `streamUri`, `baseUri`, and `clientSideBaseUri` are `https://stream.launchdarkly.com`, `https://sdk.launchdarkly.com`, and `https://clientsdk.launchdarkly.com`, respectively. You should never need to change these URIs unless you are either using a special instance of the LaunchDarkly service, in which case Support will tell you how to set them, or you are accessing LaunchDarkly using a reverse proxy or some other mechanism that rewrites URLs. @@ -74,6 +75,10 @@ _(3)_ The optional `maxClientConnectionTime` setting may be useful in load-balan _(4)_ For details about `disconnectedStatusTime`, read [Service endpoints - Status (health check)](./endpoints.md#status-health-check). +_(5)_ Relevant only when using AutoConfig or Offline Mode. In these modes, when an environment's SDK key is rotated in +LaunchDarkly, it's possible to specify a deprecation/grace period for the previous key where existing SDKs are still able +to authorize using that credential. Relay will periodically check for expired credentials and remove them on this interval. + ### File section: `[AutoConfig]` This section is only applicable if [automatic configuration](https://docs.launchdarkly.com/home/advanced/relay-proxy-enterprise/automatic-configuration) is enabled for your account. diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 2f9c7ad2..e18576be 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -68,23 +68,23 @@ type ConnectionMapper interface { // EnvContextImplParams contains the constructor parameters for NewEnvContextImpl. These have their // own type because there are a lot of them, and many are irrelevant in tests. type EnvContextImplParams struct { - Identifiers EnvIdentifiers - EnvConfig config.EnvConfig - AllConfig config.Config - ClientFactory sdks.ClientFactoryFunc - DataStoreFactory subsystems.ComponentConfigurer[subsystems.DataStore] - DataStoreInfo sdks.DataStoreEnvironmentInfo - StreamProviders []streams.StreamProvider - JSClientContext JSClientContext - MetricsManager *metrics.Manager - BigSegmentStoreFactory bigsegments.BigSegmentStoreFactory - BigSegmentSynchronizerFactory bigsegments.BigSegmentSynchronizerFactory - SDKBigSegmentsConfigFactory subsystems.ComponentConfigurer[subsystems.BigSegmentsConfiguration] // set only in tests - UserAgent string - LogNameMode LogNameMode - Loggers ldlog.Loggers - ConnectionMapper ConnectionMapper - CredentialCleanupInterval time.Duration + Identifiers EnvIdentifiers + EnvConfig config.EnvConfig + AllConfig config.Config + ClientFactory sdks.ClientFactoryFunc + DataStoreFactory subsystems.ComponentConfigurer[subsystems.DataStore] + DataStoreInfo sdks.DataStoreEnvironmentInfo + StreamProviders []streams.StreamProvider + JSClientContext JSClientContext + MetricsManager *metrics.Manager + BigSegmentStoreFactory bigsegments.BigSegmentStoreFactory + BigSegmentSynchronizerFactory bigsegments.BigSegmentSynchronizerFactory + SDKBigSegmentsConfigFactory subsystems.ComponentConfigurer[subsystems.BigSegmentsConfiguration] // set only in tests + UserAgent string + LogNameMode LogNameMode + Loggers ldlog.Loggers + ConnectionMapper ConnectionMapper + ExpiredCredentialCleanupInterval time.Duration } type envContextImpl struct { @@ -393,7 +393,7 @@ func NewEnvContext( // Connecting may take time, so do this in parallel go envContext.startSDKClient(envConfig.SDKKey, readyCh, allConfig.Main.IgnoreConnectionErrors) - cleanupInterval := params.CredentialCleanupInterval + cleanupInterval := params.ExpiredCredentialCleanupInterval if cleanupInterval == 0 { // 0 means it wasn't specified; the config system disallows 0 as a valid value. cleanupInterval = defaultCredentialCleanupInterval } diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index 0116f901..1ba1253c 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -73,7 +73,7 @@ func autoConfTest( // In tests involving adding/removing credentials, allow Relay to clean up credentials quickly so as not // to take more time than necessary to verify the test conditions. - config.Main.CredentialCleanupInterval = configtypes.NewOptDuration(time.Millisecond * 100) + config.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(time.Millisecond * 100) relay, err := newRelayInternal(config, relayInternalOptions{ loggers: mockLog.Loggers, diff --git a/relay/relay.go b/relay/relay.go index 31b80b11..507104cb 100644 --- a/relay/relay.go +++ b/relay/relay.go @@ -444,20 +444,20 @@ func (r *Relay) addEnvironment( return r.clientFactory(sdkKey, config, timeout) } clientContext, err := relayenv.NewEnvContext(relayenv.EnvContextImplParams{ - Identifiers: identifiers, - EnvConfig: envConfig, - AllConfig: r.config, - ClientFactory: wrappedClientFactory, - DataStoreFactory: dataStoreFactory, - DataStoreInfo: dataStoreInfo, - StreamProviders: r.allStreamProviders(), - JSClientContext: jsClientContext, - MetricsManager: r.metricsManager, - UserAgent: r.userAgent, - LogNameMode: r.envLogNameMode, - Loggers: r.loggers, - ConnectionMapper: r, - CredentialCleanupInterval: r.config.Main.CredentialCleanupInterval.GetOrElse(0), + Identifiers: identifiers, + EnvConfig: envConfig, + AllConfig: r.config, + ClientFactory: wrappedClientFactory, + DataStoreFactory: dataStoreFactory, + DataStoreInfo: dataStoreInfo, + StreamProviders: r.allStreamProviders(), + JSClientContext: jsClientContext, + MetricsManager: r.metricsManager, + UserAgent: r.userAgent, + LogNameMode: r.envLogNameMode, + Loggers: r.loggers, + ConnectionMapper: r, + ExpiredCredentialCleanupInterval: r.config.Main.ExpiredCredentialCleanupInterval.GetOrElse(0), }, resultCh) if err != nil { return nil, nil, errNewClientContextFailed(identifiers.GetDisplayName(), err) From 1e49590cfed10c59b44d387eb1eea238e656e3c7 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Fri, 21 Jun 2024 14:41:39 -0700 Subject: [PATCH 12/26] lints --- internal/autoconfig/stream_manager.go | 50 --------------------------- internal/credential/rotator.go | 2 +- internal/credential/store.go | 29 ---------------- relay/relay.go | 4 +-- 4 files changed, 3 insertions(+), 82 deletions(-) delete mode 100644 internal/credential/store.go diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 722570cc..451dc257 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -3,7 +3,6 @@ package autoconfig import ( "encoding/json" "errors" - "fmt" "net/http" "net/url" "path" @@ -14,7 +13,6 @@ import ( es "github.com/launchdarkly/eventsource" "github.com/launchdarkly/go-sdk-common/v3/ldlog" - "github.com/launchdarkly/go-sdk-common/v3/ldtime" "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" @@ -382,54 +380,6 @@ func (s *StreamManager) handlePut(content PutContent) { s.handler.ReceivedAllEnvironments() } -// IgnoreExpiringSDKKey implements the EnvironmentMsgAdapter's KeyChecker interface. Its main purpose is to -// create a goroutine that triggers SDK key expiration, if the EnvironmentRep specifies that. Additionally, it returns -// true if an ExpiringSDKKey should be ignored (since the expiry is stale). -func (s *StreamManager) IgnoreExpiringSDKKey(env envfactory.EnvironmentRep) bool { - expiringKey := env.SDKKey.Expiring.Value - expiryTime := env.SDKKey.Expiring.Timestamp - - if expiringKey == "" || expiryTime == 0 { - return false - } - - if _, alreadyHaveTimer := s.expiryTimers[expiringKey]; alreadyHaveTimer { - return false - } - - timeFromNow := time.Duration(expiryTime-ldtime.UnixMillisNow()) * time.Millisecond - if timeFromNow <= 0 { - // LD might sometimes tell us about an "expiring" key that has really already expired. If so, - // just ignore it. - return true - } - - dateTime := time.Unix(int64(expiryTime)/1000, 0) - s.loggers.Warnf(logMsgKeyWillExpire, last4Chars(string(expiringKey)), env.Describe(), dateTime) - - timer := time.NewTimer(timeFromNow) - s.expiryTimers[expiringKey] = timer - - go func() { - if _, ok := <-timer.C; ok { - s.expiredKeys <- expiredKey{env.EnvID, env.ProjKey, expiringKey} - } - }() - - return false -} - -func makeEnvName(rep envfactory.EnvironmentRep) string { - return fmt.Sprintf("%s %s", rep.ProjName, rep.EnvName) -} - -func last4Chars(s string) string { - if len(s) < 4 { // COVERAGE: doesn't happen in unit tests, also can't happen with real environments - return s - } - return s[len(s)-4:] -} - func obfuscateEventData(data string) string { // Used for debug logging to obscure the SDK keys and mobile keys in the JSON data data = sdkKeyJSONRegex.ReplaceAllString(data, `"value":"...$1"`) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index e331fb62..ede977a0 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -100,7 +100,7 @@ func (r *Rotator) primaryCredentials() []SDKCredential { } func (r *Rotator) deprecatedCredentials() []SDKCredential { - var deprecated []SDKCredential + deprecated := make([]SDKCredential, 0, len(r.deprecatedSdkKeys)) for key := range r.deprecatedSdkKeys { deprecated = append(deprecated, key) } diff --git a/internal/credential/store.go b/internal/credential/store.go deleted file mode 100644 index cef69e49..00000000 --- a/internal/credential/store.go +++ /dev/null @@ -1,29 +0,0 @@ -package credential - -import ( - "github.com/launchdarkly/ld-relay/v8/config" - "time" -) - -type rotatedKey struct { - key config.SDKKey - expired bool - expiry time.Time -} - -func (r *rotatedKey) deprecated() bool { - return !r.expiry.IsZero() -} - -func (r *rotatedKey) preferred() bool { - return !r.deprecated() -} - -type Store struct { - // Can be rotated. The tail of this list is the active key. - mobileKeys []config.MobileKey - // Can be rotated, with a deprecation period. The tail of this list is the preferred key. - sdkKeys []config.SDKKey - // Can never change - envID config.EnvironmentID -} diff --git a/relay/relay.go b/relay/relay.go index 507104cb..db617e58 100644 --- a/relay/relay.go +++ b/relay/relay.go @@ -495,7 +495,7 @@ func (r *Relay) setFullyConfigured(fullyConfigured bool) { r.lock.Unlock() } -// addConnectionMapping updates the RelayCore's environment mapping to reflect that a new +// AddConnectionMapping updates the RelayCore's environment mapping to reflect that a new // credential is now enabled for this EnvContext. This should be done only *after* calling // EnvContext.AddCredential() so that if the RelayCore receives an incoming request with the new // credential immediately after this, it will work. @@ -503,7 +503,7 @@ func (r *Relay) AddConnectionMapping(params sdkauth.ScopedCredential, env relaye r.envsByCredential.MapRequestParams(params, env) } -// removeConnectionMapping updates the RelayCore's environment mapping to reflect that this +// RemoveConnectionMapping updates the RelayCore's environment mapping to reflect that this // credential is no longer enabled. This should be done *before* calling EnvContext.RemoveCredential() // because RemoveCredential() disconnects all existing streams, and if a client immediately tries to // reconnect using the same credential we want it to be rejected. From 6191e588b281bc49ea9860535dcdd739b22c0c6e Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Fri, 21 Jun 2024 14:48:35 -0700 Subject: [PATCH 13/26] remove old tests --- internal/autoconfig/stream_manager.go | 13 -- .../stream_manager_expiring_key_test.go | 161 ------------------ .../stream_manager_test_base_test.go | 8 - relay/autoconfig_key_change_test.go | 4 +- 4 files changed, 2 insertions(+), 184 deletions(-) delete mode 100644 internal/autoconfig/stream_manager_expiring_key_test.go diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 451dc257..ed6d1c10 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -48,8 +48,6 @@ type StreamManager struct { uri *url.URL handler MessageHandler lastKnownEnvs map[config.EnvironmentID]envfactory.EnvironmentRep - expiredKeys chan expiredKey - expiryTimers map[config.SDKKey]*time.Timer httpConfig httpconfig.HTTPConfig initialRetryDelay time.Duration loggers ldlog.Loggers @@ -60,12 +58,6 @@ type StreamManager struct { filterReceiver *MessageReceiver[envfactory.FilterRep] } -type expiredKey struct { - envID config.EnvironmentID - projKey string - key config.SDKKey -} - // NewStreamManager creates a StreamManager, but does not start the connection. func NewStreamManager( key config.AutoConfigKey, @@ -87,8 +79,6 @@ func NewStreamManager( uri: streamURI, handler: handler, lastKnownEnvs: make(map[config.EnvironmentID]envfactory.EnvironmentRep), - expiredKeys: make(chan expiredKey), - expiryTimers: make(map[config.SDKKey]*time.Timer), httpConfig: httpConfig, initialRetryDelay: initialRetryDelay, loggers: loggers, @@ -307,9 +297,6 @@ func (s *StreamManager) consumeStream(stream *es.Stream) { } case <-s.halt: stream.Close() - for _, t := range s.expiryTimers { - t.Stop() - } return } } diff --git a/internal/autoconfig/stream_manager_expiring_key_test.go b/internal/autoconfig/stream_manager_expiring_key_test.go deleted file mode 100644 index cd2ccfc9..00000000 --- a/internal/autoconfig/stream_manager_expiring_key_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package autoconfig - -import ( - "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - - "github.com/launchdarkly/go-sdk-common/v3/ldlog" - "github.com/launchdarkly/go-sdk-common/v3/ldtime" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - oldKey = config.SDKKey("oldkey") - briefExpiryMillis = 300 -) - -func makeEnvWithExpiringKey(fromEnv envfactory.EnvironmentRep, oldKey config.SDKKey) envfactory.EnvironmentRep { - ret := fromEnv - ret.SDKKey.Expiring = envfactory.ExpiringKeyRep{ - Value: oldKey, - Timestamp: ldtime.UnixMillisNow() + briefExpiryMillis, - } - return ret -} - -func makeEnvWithAlreadyExpiredKey(fromEnv envfactory.EnvironmentRep, oldKey config.SDKKey) envfactory.EnvironmentRep { - ret := fromEnv - ret.SDKKey.Expiring = envfactory.ExpiringKeyRep{ - Value: oldKey, - Timestamp: ldtime.UnixMillisNow() - 1, - } - return ret -} - -func expectOldKeyWillExpire(p streamManagerTestParams, envID config.EnvironmentID) { - p.mockLog.AssertMessageMatch(p.t, true, ldlog.Warn, "Old SDK key ending in dkey .* will expire") - assert.Len(p.t, p.mockLog.GetOutput(ldlog.Error), 0) - - msg := p.requireMessage() - require.NotNil(p.t, msg.expired) - assert.Equal(p.t, envID, msg.expired.envID) - assert.Equal(p.t, oldKey, msg.expired.key) - - p.mockLog.AssertMessageMatch(p.t, true, ldlog.Warn, "Old SDK key ending in dkey .* has expired") -} - -func expectNoKeyExpiryMessage(p streamManagerTestParams) { - p.mockLog.AssertMessageMatch(p.t, false, ldlog.Warn, "Old SDK key .* will expire") -} - -// -//func TestExpiringKeyInPutMessage(t *testing.T) { -// envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) -// event := makeEnvPutEvent(envWithExpiringKey) -// streamManagerTest(t, &event, func(p streamManagerTestParams) { -// p.startStream() -// -// msg := p.requireMessage() -// require.NotNil(t, msg.add) -// p.requireReceivedAllMessage() -// -// assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) -// assert.Equal(t, oldKey, msg.add.ExpiringSDKKey.Key) -// -// expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) -// }) -//} -// -//func TestExpiringKeyInPatchAdd(t *testing.T) { -// envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) -// event := makePatchEnvEvent(envWithExpiringKey) -// streamManagerTest(t, nil, func(p streamManagerTestParams) { -// p.startStream() -// p.stream.Enqueue(event) -// -// msg := p.requireMessage() -// require.NotNil(t, msg.add) -// -// assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) -// assert.Equal(t, oldKey, msg.add.ExpiringSDKKey.Key) -// -// expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) -// }) -//} -// -//func TestExpiringKeyInPatchUpdate(t *testing.T) { -// streamManagerTest(t, nil, func(p streamManagerTestParams) { -// p.startStream() -// p.stream.Enqueue(makePatchEnvEvent(testEnv1)) -// -// _ = p.requireMessage() -// -// envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) -// envWithExpiringKey.Version++ -// -// p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) -// -// msg := p.requireMessage() -// require.NotNil(t, msg.update) -// assert.Equal(t, envWithExpiringKey.ToParams(), *msg.update) -// assert.Equal(t, oldKey, msg.update.ExpiringSDKKey.Key) -// -// expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) -// }) -//} -// -//func TestExpiringKeyHasAlreadyExpiredInPutMessage(t *testing.T) { -// envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) -// event := makeEnvPutEvent(envWithExpiringKey) -// streamManagerTest(t, &event, func(p streamManagerTestParams) { -// p.startStream() -// -// msg := p.requireMessage() -// require.NotNil(t, msg.add) -// p.requireReceivedAllMessage() -// -// assert.Equal(t, testEnv1.ToParams(), *msg.add) -// assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey.Key) -// -// expectNoKeyExpiryMessage(p) -// }) -//} -// -//func TestExpiringKeyHasAlreadyExpiredInPatchAdd(t *testing.T) { -// envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) -// event := makePatchEnvEvent(envWithExpiringKey) -// streamManagerTest(t, nil, func(p streamManagerTestParams) { -// p.startStream() -// p.stream.Enqueue(event) -// -// msg := p.requireMessage() -// require.NotNil(t, msg.add) -// assert.Equal(t, testEnv1.ToParams(), *msg.add) -// assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey.Key) -// -// expectNoKeyExpiryMessage(p) -// }) -//} -// -//func TestExpiringKeyHasAlreadyExpiredInPatchUpdate(t *testing.T) { -// streamManagerTest(t, nil, func(p streamManagerTestParams) { -// p.startStream() -// p.stream.Enqueue(makePatchEnvEvent(testEnv1)) -// -// _ = p.requireMessage() -// -// envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) -// envWithExpiringKey.Version++ -// -// p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) -// -// msg := p.requireMessage() -// require.NotNil(t, msg.update) -// assert.Equal(t, testEnv1.ToParams(), *msg.update) -// assert.Equal(t, config.SDKKey(""), msg.update.ExpiringSDKKey.Key) -// -// expectNoKeyExpiryMessage(p) -// }) -//} diff --git a/internal/autoconfig/stream_manager_test_base_test.go b/internal/autoconfig/stream_manager_test_base_test.go index e22fe992..6f7390ed 100644 --- a/internal/autoconfig/stream_manager_test_base_test.go +++ b/internal/autoconfig/stream_manager_test_base_test.go @@ -176,7 +176,6 @@ type testMessage struct { delete *config.EnvironmentID deleteFilter *config.FilterID receivedAll bool - expired *expiredKey } func (m testMessage) String() string { @@ -192,9 +191,6 @@ func (m testMessage) String() string { if m.receivedAll { return "receivedAllEnvironments" } - if m.expired != nil { - return fmt.Sprintf("expired(%+v)", *m.expired) - } return "???" } @@ -302,10 +298,6 @@ func (h *testMessageHandler) ReceivedAllEnvironments() { h.received <- testMessage{receivedAll: true} } -func (h *testMessageHandler) KeyExpired(envID config.EnvironmentID, projKey string, key config.SDKKey) { - h.received <- testMessage{expired: &expiredKey{envID, projKey, key}} -} - func (h *testMessageHandler) AddFilter(params envfactory.FilterParams) { h.received <- testMessage{addFilter: ¶ms} } diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index aa2ae03c..ef4b0ec8 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -93,7 +93,7 @@ func TestAutoConfigUpdateEnvironmentSDKKeyWithNoExpiry(t *testing.T) { client1.AwaitClose(t, 10000*time.Second) p.awaitCredentialsUpdated(env, modified.params()) - noEnv, _ := p.relay.getEnvironment(sdkauth.New(testAutoConfEnv1.sdkKey.Value)) + noEnv, _ := p.relay.getEnvironment(sdkauth.New(testAutoConfEnv1.SDKKey())) assert.Nil(t, noEnv) }) } @@ -108,7 +108,7 @@ func TestAutoConfigUpdateEnvironmentSDKKeyWithExpiry(t *testing.T) { modified := makeEnvWithModifiedSDKKey(testAutoConfEnv1) modified.sdkKey.Expiring = envfactory.ExpiringKeyRep{ - Value: testAutoConfEnv1.sdkKey.Value, + Value: testAutoConfEnv1.SDKKey(), Timestamp: ldtime.UnixMillisNow() + 100000, } p.stream.Enqueue(makeAutoConfPatchEvent(modified)) From 84427b0a924c4c708d148e1f5ecb2246cf84009c Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Fri, 21 Jun 2024 14:52:11 -0700 Subject: [PATCH 14/26] goimports --- internal/credential/rotator.go | 6 +++--- internal/credential/rotator_test.go | 5 +++-- internal/relayenv/env_context_impl_test.go | 3 ++- relay/autoconfig_actions_test.go | 3 ++- relay/autoconfig_key_change_test.go | 3 ++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index ede977a0..bb401623 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -1,11 +1,12 @@ package credential import ( - "github.com/launchdarkly/go-sdk-common/v3/ldlog" - "github.com/launchdarkly/ld-relay/v8/config" "slices" "sync" "time" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/ld-relay/v8/config" ) type Rotator struct { @@ -80,7 +81,6 @@ func (r *Rotator) EnvironmentID() config.EnvironmentID { r.mu.RLock() defer r.mu.RUnlock() return r.primaryEnvironmentID - } func (r *Rotator) PrimaryCredentials() []SDKCredential { diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 4803f388..cefe6e85 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -2,11 +2,12 @@ package credential import ( "fmt" + "testing" + "time" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" "github.com/launchdarkly/ld-relay/v8/config" "github.com/stretchr/testify/assert" - "testing" - "time" ) func TestNewRotator(t *testing.T) { diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 62773904..f2b03968 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -3,7 +3,6 @@ package relayenv import ( "context" "errors" - "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" "net/http" "net/http/httptest" "regexp" @@ -12,6 +11,8 @@ import ( "testing" "time" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/eventsource" diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index 1ba1253c..c957cf1a 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -1,11 +1,12 @@ package relay import ( - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "net/http/httptest" "testing" "time" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + c "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index ef4b0ec8..4ca0eb4a 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -1,11 +1,12 @@ package relay import ( - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "net/http" "testing" "time" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" "github.com/launchdarkly/ld-relay/v8/internal/credential" From 3666691238dd8531b692d5897fc562cb05539bd6 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Mon, 24 Jun 2024 14:25:48 -0700 Subject: [PATCH 15/26] more tests --- config/config_field_types.go | 4 +- .../steam_manager_expiring_key_test.go | 162 ++++++++++++++++++ internal/credential/rotator.go | 16 +- internal/credential/rotator_test.go | 27 ++- internal/relayenv/env_context.go | 14 +- internal/relayenv/env_context_impl.go | 6 +- internal/relayenv/env_context_impl_test.go | 4 +- 7 files changed, 213 insertions(+), 20 deletions(-) create mode 100644 internal/autoconfig/steam_manager_expiring_key_test.go diff --git a/config/config_field_types.go b/config/config_field_types.go index fc6df3c9..de676b42 100644 --- a/config/config_field_types.go +++ b/config/config_field_types.go @@ -62,7 +62,7 @@ func (k SDKKey) Defined() bool { func (k SDKKey) String() string { return string(k) } -func (k SDKKey) Masked() string { return last4Chars(k.String()) } +func (k SDKKey) Masked() string { return "..." + last4Chars(k.String()) } // GetAuthorizationHeaderValue for MobileKey returns the same string, since mobile keys are passed in the // Authorization header. @@ -78,7 +78,7 @@ func (k MobileKey) String() string { return string(k) } -func (k MobileKey) Masked() string { return last4Chars(k.String()) } +func (k MobileKey) Masked() string { return "..." + last4Chars(k.String()) } // GetAuthorizationHeaderValue for EnvironmentID returns an empty string, since environment IDs are not // passed in a header but as part of the request URL. diff --git a/internal/autoconfig/steam_manager_expiring_key_test.go b/internal/autoconfig/steam_manager_expiring_key_test.go new file mode 100644 index 00000000..67077fef --- /dev/null +++ b/internal/autoconfig/steam_manager_expiring_key_test.go @@ -0,0 +1,162 @@ +package autoconfig + +import ( + "testing" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + oldKey = config.SDKKey("oldkey") + briefExpiryMillis = 300 +) + +func makeEnvWithExpiringKey(fromEnv envfactory.EnvironmentRep, oldKey config.SDKKey) envfactory.EnvironmentRep { + ret := fromEnv + ret.SDKKey.Expiring = envfactory.ExpiringKeyRep{ + Value: oldKey, + Timestamp: ldtime.UnixMillisNow() + briefExpiryMillis, + } + return ret +} + +func makeEnvWithAlreadyExpiredKey(fromEnv envfactory.EnvironmentRep, oldKey config.SDKKey) envfactory.EnvironmentRep { + ret := fromEnv + ret.SDKKey.Expiring = envfactory.ExpiringKeyRep{ + Value: oldKey, + Timestamp: ldtime.UnixMillisNow() - 1, + } + return ret +} + +func expectOldKeyWillExpire(p streamManagerTestParams, envID config.EnvironmentID) { + p.mockLog.AssertMessageMatch(p.t, true, ldlog.Warn, "Old SDK key ending in dkey .* will expire") + assert.Len(p.t, p.mockLog.GetOutput(ldlog.Error), 0) + + msg := p.requireMessage() + require.NotNil(p.t, msg.expired) + assert.Equal(p.t, envID, msg.expired.envID) + assert.Equal(p.t, oldKey, msg.expired.key) + + p.mockLog.AssertMessageMatch(p.t, true, ldlog.Warn, "Old SDK key ending in dkey .* has expired") +} + +func expectNoKeyExpiryMessage(p streamManagerTestParams) { + p.mockLog.AssertMessageMatch(p.t, false, ldlog.Warn, "Old SDK key .* will expire") +} + +func TestExpiringKeyInPutMessage(t *testing.T) { + envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) + event := makeEnvPutEvent(envWithExpiringKey) + streamManagerTest(t, &event, func(p streamManagerTestParams) { + p.startStream() + + msg := p.requireMessage() + require.NotNil(t, msg.add) + p.requireReceivedAllMessage() + + assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) + assert.Equal(t, oldKey, msg.add.ExpiringSDKKey) + + expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) + }) +} + +func TestExpiringKeyInPatchAdd(t *testing.T) { + envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) + event := makePatchEnvEvent(envWithExpiringKey) + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + p.stream.Enqueue(event) + + msg := p.requireMessage() + require.NotNil(t, msg.add) + + assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) + assert.Equal(t, oldKey, msg.add.ExpiringSDKKey) + + expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) + }) +} + +func TestExpiringKeyInPatchUpdate(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + p.stream.Enqueue(makePatchEnvEvent(testEnv1)) + + _ = p.requireMessage() + + envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) + envWithExpiringKey.Version++ + + p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) + + msg := p.requireMessage() + require.NotNil(t, msg.update) + assert.Equal(t, envWithExpiringKey.ToParams(), *msg.update) + assert.Equal(t, oldKey, msg.update.ExpiringSDKKey) + + expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) + }) +} + +func TestExpiringKeyHasAlreadyExpiredInPutMessage(t *testing.T) { + envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) + event := makeEnvPutEvent(envWithExpiringKey) + streamManagerTest(t, &event, func(p streamManagerTestParams) { + p.startStream() + + msg := p.requireMessage() + require.NotNil(t, msg.add) + p.requireReceivedAllMessage() + + assert.Equal(t, testEnv1.ToParams(), *msg.add) + assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey) + + expectNoKeyExpiryMessage(p) + }) +} + +func TestExpiringKeyHasAlreadyExpiredInPatchAdd(t *testing.T) { + envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) + event := makePatchEnvEvent(envWithExpiringKey) + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + p.stream.Enqueue(event) + + msg := p.requireMessage() + require.NotNil(t, msg.add) + assert.Equal(t, testEnv1.ToParams(), *msg.add) + assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey) + + expectNoKeyExpiryMessage(p) + }) +} + +func TestExpiringKeyHasAlreadyExpiredInPatchUpdate(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + p.stream.Enqueue(makePatchEnvEvent(testEnv1)) + + _ = p.requireMessage() + + envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) + envWithExpiringKey.Version++ + + p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) + + msg := p.requireMessage() + require.NotNil(t, msg.update) + assert.Equal(t, testEnv1.ToParams(), *msg.update) + assert.Equal(t, config.SDKKey(""), msg.update.ExpiringSDKKey) + + expectNoKeyExpiryMessage(p) + }) +} diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index bb401623..ff8e9f82 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -126,10 +126,11 @@ func (r *Rotator) Rotate(cred SDKCredential) { type GracePeriod struct { key config.SDKKey expiry time.Time + now time.Time } -func NewGracePeriod(key config.SDKKey, expiry time.Time) *GracePeriod { - return &GracePeriod{key, expiry} +func NewGracePeriod(key config.SDKKey, expiry time.Time, now time.Time) *GracePeriod { + return &GracePeriod{key, expiry, now} } func (r *Rotator) RotateWithGrace(primary SDKCredential, grace *GracePeriod) { @@ -177,7 +178,7 @@ func (r *Rotator) updateMobileKey(mobileKey config.MobileKey) { r.additions = append(r.additions, mobileKey) if previous.Defined() { r.expirations = append(r.expirations, previous) - r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", r.primaryMobileKey.Masked(), mobileKey.Masked()) + r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", previous.Masked(), mobileKey.Masked()) } else { r.loggers.Infof("New primary mobile key is %s", mobileKey.Masked()) } @@ -209,7 +210,14 @@ func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { } if grace != nil { if previousExpiry, ok := r.deprecatedSdkKeys[grace.key]; ok { - r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", grace.key.Masked(), grace.expiry, previousExpiry) + if previousExpiry != grace.expiry { + r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", grace.key.Masked(), grace.expiry, previousExpiry) + } + return + } + + if grace.now.After(grace.expiry) { + r.loggers.Infof("Deprecated SDK key %s already expired; ignoring", grace.key.Masked()) return } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index cefe6e85..6510fbec 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -120,14 +120,14 @@ func TestSDKKeyDeprecation(t *testing.T) { key2 = config.SDKKey("key2") ) - start := time.Now() + start := time.Unix(10000, 0) - deprecationTime := start.Add(1 * time.Minute) halfTime := start.Add(30 * time.Second) + deprecationTime := start.Add(1 * time.Minute) rotator.Initialize([]SDKCredential{key1}) - rotator.RotateWithGrace(key2, NewGracePeriod(key1, deprecationTime)) + rotator.RotateWithGrace(key2, NewGracePeriod(key1, deprecationTime, halfTime)) additions, expirations := rotator.Query(halfTime) assert.ElementsMatch(t, []SDKCredential{key2}, additions) assert.Empty(t, expirations) @@ -152,7 +152,8 @@ func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { rotator.Initialize([]SDKCredential{config.SDKKey("key0")}) const numKeys = 250 - expiryTime := time.Now().Add(1 * time.Minute) + now := time.Unix(10000, 0) + expiryTime := now.Add(1 * time.Hour) var keysDeprecated []SDKCredential var keysAdded []SDKCredential @@ -164,7 +165,7 @@ func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { keysDeprecated = append(keysDeprecated, previousKey) keysAdded = append(keysAdded, nextKey) - rotator.RotateWithGrace(nextKey, NewGracePeriod(previousKey, expiryTime)) + rotator.RotateWithGrace(nextKey, NewGracePeriod(previousKey, expiryTime, now)) } // The last key added should be the current primary key. @@ -180,3 +181,19 @@ func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { assert.Empty(t, additions) assert.ElementsMatch(t, keysDeprecated, expirations) } + +func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + primaryKey := config.SDKKey("primary") + obsoleteKey := config.SDKKey("obsolete") + obsoleteExpiry := time.Unix(1000000, 0) + now := obsoleteExpiry.Add(1 * time.Hour) + + rotator.RotateWithGrace(primaryKey, NewGracePeriod(obsoleteKey, obsoleteExpiry, now)) + + additions, expirations := rotator.Query(now) + assert.ElementsMatch(t, []SDKCredential{primaryKey}, additions) + assert.Empty(t, expirations) +} diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index c20b3ea3..beded3e2 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -24,9 +24,14 @@ import ( // For example, an environment may have a primary SDK key and a primary mobile key at the same time; each would // be specified in individual CredentialUpdate objects. type CredentialUpdate struct { - primary credential.SDKCredential - gracePeriod *credential.GracePeriod - now time.Time + // The new primary credential + primary credential.SDKCredential + // An optional deprecated credential (only SDK keys are supported currently) + deprecated config.SDKKey + // When the deprecated credential expires + expiry time.Time + // The current time + now time.Time } // NewCredentialUpdate creates a CredentialUpdate from a given primary credential. @@ -38,7 +43,8 @@ func NewCredentialUpdate(primary credential.SDKCredential) *CredentialUpdate { // WithGracePeriod modifies the default behavior from immediate revocation to a delayed revocation of the previous // credential. During the grace period, the previous credential continues to function. func (c *CredentialUpdate) WithGracePeriod(deprecated config.SDKKey, expiry time.Time) *CredentialUpdate { - c.gracePeriod = credential.NewGracePeriod(deprecated, expiry) + c.deprecated = deprecated + c.expiry = expiry return c } diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index e18576be..498984a7 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -505,7 +505,7 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env return } } else { - c.globalLoggers.Infof("Initialized LaunchDarkly client for %q", name) + c.globalLoggers.Infof("Initialized LaunchDarkly client for %q (SDK key %s)", name, sdkKey.Masked()) } if readyCh != nil { readyCh <- c @@ -531,10 +531,10 @@ func (c *envContextImpl) SetIdentifiers(ei EnvIdentifiers) { } func (c *envContextImpl) UpdateCredential(update *CredentialUpdate) { - if update.gracePeriod == nil { + if !update.deprecated.Defined() { c.keyRotator.Rotate(update.primary) } else { - c.keyRotator.RotateWithGrace(update.primary, update.gracePeriod) + c.keyRotator.RotateWithGrace(update.primary, credential.NewGracePeriod(update.deprecated, update.expiry, update.now)) } c.triggerCredentialChanges(update.now) } diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index f2b03968..ed142779 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -262,8 +262,8 @@ func TestChangeSDKKey(t *testing.T) { // Upon rotating to key2, the original key should still be valid for a hour. env.UpdateCredential( NewCredentialUpdate(key2). - WithGracePeriod(envConfig.SDKKey, start.Add(1*time.Hour)). - WithTime(start)) + WithTime(start). + WithGracePeriod(envConfig.SDKKey, start.Add(1*time.Hour))) assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetDeprecatedCredentials()) From b300967fc1260f30db0b5361c66a02c9f8e38b09 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Mon, 24 Jun 2024 14:38:27 -0700 Subject: [PATCH 16/26] fix some nits --- .../steam_manager_expiring_key_test.go | 162 ------------------ internal/envfactory/env_rep.go | 12 +- 2 files changed, 5 insertions(+), 169 deletions(-) delete mode 100644 internal/autoconfig/steam_manager_expiring_key_test.go diff --git a/internal/autoconfig/steam_manager_expiring_key_test.go b/internal/autoconfig/steam_manager_expiring_key_test.go deleted file mode 100644 index 67077fef..00000000 --- a/internal/autoconfig/steam_manager_expiring_key_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package autoconfig - -import ( - "testing" - - "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - - "github.com/launchdarkly/go-sdk-common/v3/ldlog" - "github.com/launchdarkly/go-sdk-common/v3/ldtime" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - oldKey = config.SDKKey("oldkey") - briefExpiryMillis = 300 -) - -func makeEnvWithExpiringKey(fromEnv envfactory.EnvironmentRep, oldKey config.SDKKey) envfactory.EnvironmentRep { - ret := fromEnv - ret.SDKKey.Expiring = envfactory.ExpiringKeyRep{ - Value: oldKey, - Timestamp: ldtime.UnixMillisNow() + briefExpiryMillis, - } - return ret -} - -func makeEnvWithAlreadyExpiredKey(fromEnv envfactory.EnvironmentRep, oldKey config.SDKKey) envfactory.EnvironmentRep { - ret := fromEnv - ret.SDKKey.Expiring = envfactory.ExpiringKeyRep{ - Value: oldKey, - Timestamp: ldtime.UnixMillisNow() - 1, - } - return ret -} - -func expectOldKeyWillExpire(p streamManagerTestParams, envID config.EnvironmentID) { - p.mockLog.AssertMessageMatch(p.t, true, ldlog.Warn, "Old SDK key ending in dkey .* will expire") - assert.Len(p.t, p.mockLog.GetOutput(ldlog.Error), 0) - - msg := p.requireMessage() - require.NotNil(p.t, msg.expired) - assert.Equal(p.t, envID, msg.expired.envID) - assert.Equal(p.t, oldKey, msg.expired.key) - - p.mockLog.AssertMessageMatch(p.t, true, ldlog.Warn, "Old SDK key ending in dkey .* has expired") -} - -func expectNoKeyExpiryMessage(p streamManagerTestParams) { - p.mockLog.AssertMessageMatch(p.t, false, ldlog.Warn, "Old SDK key .* will expire") -} - -func TestExpiringKeyInPutMessage(t *testing.T) { - envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) - event := makeEnvPutEvent(envWithExpiringKey) - streamManagerTest(t, &event, func(p streamManagerTestParams) { - p.startStream() - - msg := p.requireMessage() - require.NotNil(t, msg.add) - p.requireReceivedAllMessage() - - assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) - assert.Equal(t, oldKey, msg.add.ExpiringSDKKey) - - expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) - }) -} - -func TestExpiringKeyInPatchAdd(t *testing.T) { - envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) - event := makePatchEnvEvent(envWithExpiringKey) - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(event) - - msg := p.requireMessage() - require.NotNil(t, msg.add) - - assert.Equal(t, envWithExpiringKey.ToParams(), *msg.add) - assert.Equal(t, oldKey, msg.add.ExpiringSDKKey) - - expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) - }) -} - -func TestExpiringKeyInPatchUpdate(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(makePatchEnvEvent(testEnv1)) - - _ = p.requireMessage() - - envWithExpiringKey := makeEnvWithExpiringKey(testEnv1, oldKey) - envWithExpiringKey.Version++ - - p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) - - msg := p.requireMessage() - require.NotNil(t, msg.update) - assert.Equal(t, envWithExpiringKey.ToParams(), *msg.update) - assert.Equal(t, oldKey, msg.update.ExpiringSDKKey) - - expectOldKeyWillExpire(p, envWithExpiringKey.EnvID) - }) -} - -func TestExpiringKeyHasAlreadyExpiredInPutMessage(t *testing.T) { - envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) - event := makeEnvPutEvent(envWithExpiringKey) - streamManagerTest(t, &event, func(p streamManagerTestParams) { - p.startStream() - - msg := p.requireMessage() - require.NotNil(t, msg.add) - p.requireReceivedAllMessage() - - assert.Equal(t, testEnv1.ToParams(), *msg.add) - assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey) - - expectNoKeyExpiryMessage(p) - }) -} - -func TestExpiringKeyHasAlreadyExpiredInPatchAdd(t *testing.T) { - envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) - event := makePatchEnvEvent(envWithExpiringKey) - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(event) - - msg := p.requireMessage() - require.NotNil(t, msg.add) - assert.Equal(t, testEnv1.ToParams(), *msg.add) - assert.Equal(t, config.SDKKey(""), msg.add.ExpiringSDKKey) - - expectNoKeyExpiryMessage(p) - }) -} - -func TestExpiringKeyHasAlreadyExpiredInPatchUpdate(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - p.stream.Enqueue(makePatchEnvEvent(testEnv1)) - - _ = p.requireMessage() - - envWithExpiringKey := makeEnvWithAlreadyExpiredKey(testEnv1, oldKey) - envWithExpiringKey.Version++ - - p.stream.Enqueue(makePatchEnvEvent(envWithExpiringKey)) - - msg := p.requireMessage() - require.NotNil(t, msg.update) - assert.Equal(t, testEnv1.ToParams(), *msg.update) - assert.Equal(t, config.SDKKey(""), msg.update.ExpiringSDKKey) - - expectNoKeyExpiryMessage(p) - }) -} diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index c79dc049..cc7067bd 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -76,18 +76,16 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { ProjKey: r.ProjKey, ProjName: r.ProjName, }, - SDKKey: r.SDKKey.Value, + SDKKey: r.SDKKey.Value, + ExpiringSDKKey: ExpiringSDKKey{ + Key: r.SDKKey.Expiring.Value, + Expiration: ToTime(r.SDKKey.Expiring.Timestamp), + }, MobileKey: r.MobKey, TTL: time.Duration(r.DefaultTTL) * time.Minute, SecureMode: r.SecureMode, } - if r.SDKKey.Expiring.Value.Defined() { - params.ExpiringSDKKey = ExpiringSDKKey{ - Key: r.SDKKey.Expiring.Value, - Expiration: ToTime(r.SDKKey.Expiring.Timestamp), - } - } return params } From 047bc272587c415003fab2ffe13fde3709f478e9 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Mon, 24 Jun 2024 15:50:55 -0700 Subject: [PATCH 17/26] fix envrep conversion --- internal/envfactory/env_rep.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index cc7067bd..2c080c79 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -62,6 +62,17 @@ type ExpiringKeyRep struct { Timestamp ldtime.UnixMillisecondTime `json:"timestamp"` } +func (e ExpiringKeyRep) ToParams() ExpiringSDKKey { + if e.Value.Defined() { + return ExpiringSDKKey{ + Key: e.Value, + Expiration: ToTime(e.Timestamp), + } + } else { + return ExpiringSDKKey{} + } +} + func ToTime(millisecondTime ldtime.UnixMillisecondTime) time.Time { return time.UnixMilli(int64(millisecondTime)) } @@ -76,14 +87,11 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { ProjKey: r.ProjKey, ProjName: r.ProjName, }, - SDKKey: r.SDKKey.Value, - ExpiringSDKKey: ExpiringSDKKey{ - Key: r.SDKKey.Expiring.Value, - Expiration: ToTime(r.SDKKey.Expiring.Timestamp), - }, - MobileKey: r.MobKey, - TTL: time.Duration(r.DefaultTTL) * time.Minute, - SecureMode: r.SecureMode, + SDKKey: r.SDKKey.Value, + ExpiringSDKKey: r.SDKKey.Expiring.ToParams(), + MobileKey: r.MobKey, + TTL: time.Duration(r.DefaultTTL) * time.Minute, + SecureMode: r.SecureMode, } return params From bf2938032f8b5813be0fe07f441321a17d5ad585 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Mon, 24 Jun 2024 17:29:06 -0700 Subject: [PATCH 18/26] comments & rename rotator.Query to rotator.StepTime --- internal/credential/rotator.go | 30 ++++++++++++++++++++++++--- internal/credential/rotator_test.go | 20 +++++++++--------- internal/relayenv/env_context_impl.go | 2 +- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index ff8e9f82..3ece1494 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -38,6 +38,8 @@ type InitialCredentials struct { EnvironmentID config.EnvironmentID } +// NewRotator constructs a rotator with the provided loggers. A new rotator +// contains no credentials and can optionally be initialized via Initialize. func NewRotator(loggers ldlog.Loggers) *Rotator { r := &Rotator{ loggers: loggers, @@ -46,6 +48,8 @@ func NewRotator(loggers ldlog.Loggers) *Rotator { return r } +// Initialize sets the initial credentials. Only credentials that are defined +// will be stored. func (r *Rotator) Initialize(credentials []SDKCredential) { r.mu.Lock() defer r.mu.Unlock() @@ -65,24 +69,28 @@ func (r *Rotator) Initialize(credentials []SDKCredential) { } } +// MobileKey returns the primary mobile key. func (r *Rotator) MobileKey() config.MobileKey { r.mu.RLock() defer r.mu.RUnlock() return r.primaryMobileKey } +// SDKKey returns the primary SDK key. func (r *Rotator) SDKKey() config.SDKKey { r.mu.RLock() defer r.mu.RUnlock() return r.primarySdkKey } +// EnvironmentID returns the environment ID. func (r *Rotator) EnvironmentID() config.EnvironmentID { r.mu.RLock() defer r.mu.RUnlock() return r.primaryEnvironmentID } +// PrimaryCredentials returns the primary (non-deprecated) credentials. func (r *Rotator) PrimaryCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() @@ -107,32 +115,46 @@ func (r *Rotator) deprecatedCredentials() []SDKCredential { return deprecated } +// DeprecatedCredentials returns deprecated credentials (not expired or primary.) func (r *Rotator) DeprecatedCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() return r.deprecatedCredentials() } +// AllCredentials returns the primary and deprecated credentials as one list. func (r *Rotator) AllCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() return append(r.primaryCredentials(), r.deprecatedCredentials()...) } +// Rotate sets a new primary credential while revoking the previous. func (r *Rotator) Rotate(cred SDKCredential) { r.RotateWithGrace(cred, nil) } +// GracePeriod represents a grace period (or deprecation period) within which +// a particular SDK key is still valid, pending revocation. type GracePeriod struct { - key config.SDKKey + // The SDK key that is being deprecated. + key config.SDKKey + // When the key will expire. expiry time.Time - now time.Time + // The current timestamp. + now time.Time } +// NewGracePeriod constructs a new grace period. The current time must be provided in order to +// determine if the credential is already expired. func NewGracePeriod(key config.SDKKey, expiry time.Time, now time.Time) *GracePeriod { return &GracePeriod{key, expiry, now} } +// RotateWithGrace sets a new primary credential while deprecating a previous credential. The grace +// parameter may be nil to immediately revoke the previous credential. +// It is invalid to specify a grace period when the credential being rotate is a mobile key or +// environment ID. func (r *Rotator) RotateWithGrace(primary SDKCredential, grace *GracePeriod) { switch primary := primary.(type) { case config.SDKKey: @@ -237,7 +259,9 @@ func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.expirations = append(r.expirations, sdkKey) } -func (r *Rotator) Query(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { +// StepTime provides the current time to the Rotator, allowing it to compute the set of additions and expirations +// for the tracked credentials since the last time this method was called. +func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { r.mu.Lock() defer r.mu.Unlock() diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 6510fbec..9acbc57f 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -46,20 +46,20 @@ func TestImmediateKeyExpiration(t *testing.T) { // The first rotation shouldn't trigger any expirations because there was no previous key. rotator.Rotate(c.keys[0]) - additions, _ := rotator.Query(time.Now()) + additions, _ := rotator.StepTime(time.Now()) assert.ElementsMatch(t, c.keys[0:1], additions) assert.Equal(t, c.keys[0], c.getKey(rotator)) // The second rotation should trigger a deprecation of key1. rotator.Rotate(c.keys[1]) - additions, expirations := rotator.Query(time.Now()) + additions, expirations := rotator.StepTime(time.Now()) assert.ElementsMatch(t, c.keys[1:2], additions) assert.ElementsMatch(t, c.keys[0:1], expirations) assert.Equal(t, c.keys[1], c.getKey(rotator)) // The third rotation should trigger a deprecation of key2. rotator.Rotate(c.keys[2]) - additions, expirations = rotator.Query(time.Now()) + additions, expirations = rotator.StepTime(time.Now()) assert.ElementsMatch(t, c.keys[2:3], additions) assert.ElementsMatch(t, c.keys[1:2], expirations) assert.Equal(t, c.keys[2], c.getKey(rotator)) @@ -104,7 +104,7 @@ func TestManyImmediateKeyExpirations(t *testing.T) { assert.Equal(t, c.makeKey(fmt.Sprintf("key%v", numKeys-1)), c.getKey(rotator)) - additions, expirations := rotator.Query(time.Now()) + additions, expirations := rotator.StepTime(time.Now()) assert.Len(t, additions, numKeys) assert.Len(t, expirations, numKeys-1) // because the last key is still active }) @@ -128,15 +128,15 @@ func TestSDKKeyDeprecation(t *testing.T) { rotator.Initialize([]SDKCredential{key1}) rotator.RotateWithGrace(key2, NewGracePeriod(key1, deprecationTime, halfTime)) - additions, expirations := rotator.Query(halfTime) + additions, expirations := rotator.StepTime(halfTime) assert.ElementsMatch(t, []SDKCredential{key2}, additions) assert.Empty(t, expirations) - additions, expirations = rotator.Query(deprecationTime) + additions, expirations = rotator.StepTime(deprecationTime) assert.Empty(t, additions) assert.Empty(t, expirations) - additions, expirations = rotator.Query(deprecationTime.Add(1 * time.Millisecond)) + additions, expirations = rotator.StepTime(deprecationTime.Add(1 * time.Millisecond)) assert.Empty(t, additions) assert.ElementsMatch(t, []SDKCredential{key1}, expirations) } @@ -172,12 +172,12 @@ func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.SDKKey()) // Until and including the exact expiry timestamp, there should be no expirations. - additions, expirations := rotator.Query(expiryTime) + additions, expirations := rotator.StepTime(expiryTime) assert.ElementsMatch(t, keysAdded, additions) assert.Empty(t, expirations) // One moment after the expiry time, we should now have a batch of expirations. - additions, expirations = rotator.Query(expiryTime.Add(1 * time.Millisecond)) + additions, expirations = rotator.StepTime(expiryTime.Add(1 * time.Millisecond)) assert.Empty(t, additions) assert.ElementsMatch(t, keysDeprecated, expirations) } @@ -193,7 +193,7 @@ func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { rotator.RotateWithGrace(primaryKey, NewGracePeriod(obsoleteKey, obsoleteExpiry, now)) - additions, expirations := rotator.Query(now) + additions, expirations := rotator.StepTime(now) assert.ElementsMatch(t, []SDKCredential{primaryKey}, additions) assert.Empty(t, expirations) } diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 498984a7..a5b271f6 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -540,7 +540,7 @@ func (c *envContextImpl) UpdateCredential(update *CredentialUpdate) { } func (c *envContextImpl) triggerCredentialChanges(now time.Time) { - additions, expirations := c.keyRotator.Query(now) + additions, expirations := c.keyRotator.StepTime(now) for _, cred := range additions { c.addCredential(cred) } From 1b41db3a44446411a6596bed304367c4115d365d Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Mon, 24 Jun 2024 15:48:00 -0700 Subject: [PATCH 19/26] feat: offline mode key rotation --- relay/filedata_actions.go | 18 +++++- relay/filedata_actions_test.go | 101 +++++++++++++++++++++++++++++++- relay/filedata_testdata_test.go | 31 ++++++++++ relay/testutils_test.go | 8 ++- 4 files changed, 154 insertions(+), 4 deletions(-) diff --git a/relay/filedata_actions.go b/relay/filedata_actions.go index 8e600321..4a59992e 100644 --- a/relay/filedata_actions.go +++ b/relay/filedata_actions.go @@ -1,6 +1,7 @@ package relay import ( + "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "time" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" @@ -49,11 +50,15 @@ func (a *relayFileDataActions) AddEnvironment(ae filedata.ArchiveEnvironment) { return config } envConfig := envfactory.NewEnvConfigFactoryForOfflineMode(a.r.config.OfflineMode).MakeEnvironmentConfig(ae.Params) - _, _, err := a.r.addEnvironment(ae.Params.Identifiers, envConfig, transformConfig) + env, _, err := a.r.addEnvironment(ae.Params.Identifiers, envConfig, transformConfig) if err != nil { a.r.loggers.Errorf(logMsgAutoConfEnvInitError, ae.Params.Identifiers.GetDisplayName(), err) return } + if ae.Params.ExpiringSDKKey.Defined() { + update := relayenv.NewCredentialUpdate(ae.Params.SDKKey) + env.UpdateCredential(update.WithGracePeriod(ae.Params.ExpiringSDKKey.Key, ae.Params.ExpiringSDKKey.Expiration)) + } select { case updates := <-updatesCh: if a.envUpdates == nil { @@ -83,6 +88,17 @@ func (a *relayFileDataActions) UpdateEnvironment(ae filedata.ArchiveEnvironment) env.SetTTL(ae.Params.TTL) env.SetSecureMode(ae.Params.SecureMode) + if ae.Params.MobileKey.Defined() { + env.UpdateCredential(relayenv.NewCredentialUpdate(ae.Params.MobileKey)) + } + if ae.Params.SDKKey.Defined() { + update := relayenv.NewCredentialUpdate(ae.Params.SDKKey) + if ae.Params.ExpiringSDKKey.Defined() { + update = update.WithGracePeriod(ae.Params.ExpiringSDKKey.Key, ae.Params.ExpiringSDKKey.Expiration) + } + env.UpdateCredential(update) + } + // SDKData will be non-nil only if the flag/segment data for the environment has actually changed. if ae.SDKData != nil { updates.Init(ae.SDKData) diff --git a/relay/filedata_actions_test.go b/relay/filedata_actions_test.go index 221239ab..600e7135 100644 --- a/relay/filedata_actions_test.go +++ b/relay/filedata_actions_test.go @@ -1,6 +1,8 @@ package relay import ( + "fmt" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "net/http" "net/http/httptest" "sort" @@ -90,7 +92,11 @@ func offlineModeTest( } func (p offlineModeTestParams) awaitClient() testclient.CapturedLDClient { - return helpers.RequireValue(p.t, p.clientsCreatedCh, time.Second, "timed out waiting for client creation") + return p.awaitClientFor(time.Second) +} + +func (p offlineModeTestParams) awaitClientFor(duration time.Duration) testclient.CapturedLDClient { + return helpers.RequireValue(p.t, p.clientsCreatedCh, duration, "timed out waiting for client creation") } func (p offlineModeTestParams) shouldNotCreateClient(timeout time.Duration) { @@ -205,3 +211,96 @@ func TestOfflineModeEventsAreAcceptedAndDiscardedIfSendEventsIsTrue(t *testing.T }) }) } + +func TestOfflineModeDeprecatedSDKKeyIsRespectedIfExpiryInFuture(t *testing.T) { + // The goal here is to validate that if we load an offline mode archive containing a deprecated key, + // it will be added as a credential (even though it was never previously seen as a primary key.) This situation + // would happen when Relay is starting up at time T if the key was deprecated at a time before T. + + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + + envData := RotateSDKKeyWithGracePeriod("primary-key", "deprecated-key", time.Now().Add(1*time.Hour)) + + p.updateHandler.AddEnvironment(envData) + + client1 := p.awaitClient() + assert.Equal(t, envData.Params.SDKKey, client1.Key) + + env := p.awaitEnvironment(testFileDataEnv1.Params.EnvID) + + assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.SDKKey, envData.Params.EnvID}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + }) +} + +func TestOfflineModePrimarySDKKeyIsDeprecated(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + update1 := RotateSDKKey("key1") + + p.updateHandler.AddEnvironment(update1) + + expectedClient1 := p.awaitClient() + assert.Equal(t, update1.Params.SDKKey, expectedClient1.Key) + + env := p.awaitEnvironment(update1.Params.EnvID) + + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) + assert.Empty(t, env.GetDeprecatedCredentials()) + + update2 := RotateSDKKeyWithGracePeriod("key2", "key1", time.Now().Add(1*time.Hour)) + p.updateHandler.UpdateEnvironment(update2) + + assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + + update3 := RotateSDKKey("key3") + p.updateHandler.UpdateEnvironment(update3) + + assert.ElementsMatch(t, []credential.SDKCredential{update3.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) + + // Note: key2 isn't in the deprecated list, because update3 was an immediate rotation (with no grace period for the + // previous key.) At the same time, key1 is still deprecated until the hour is up. + assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + }) +} + +func TestOfflineModeSDKKeyCanExpire(t *testing.T) { + // This test aims to deprecate an SDK key, sleep until the expiry, and then verify that the + // key is no longer accepted. + // + // This test is extremely timing dependent, because we're unable to easily inject a mocked time + // into the lower level components under test. + + // Instead, we configure the credential cleanup interval to be as short as possible (100ms) + // and then sleep at least that amount of time after specifying a key expiry. The intention is to + // simulate a real scenario, but fast enough for a test. + + const minimumCleanupInterval = 100 * time.Millisecond + + cfg := config.Config{} + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(minimumCleanupInterval) + + offlineModeTest(t, cfg, func(p offlineModeTestParams) { + + for i := 0; i < 3; i++ { + primary := config.SDKKey(fmt.Sprintf("key%v", i+1)) + expiring := config.SDKKey(fmt.Sprintf("key%v", i)) + + // It's important that the expiry be in the future (so that the key isn't ignored by the key rotator + // component), but it should also be in the near future so the test doesn't need to sleep long. + keyExpiry := time.Now().Add(10 * time.Millisecond) + update1 := RotateSDKKeyWithGracePeriod(primary, expiring, keyExpiry) + p.updateHandler.AddEnvironment(update1) + + // Waiting for the environment can take up to 1 second, but it could be much faster. In any case + // we'll still need to sleep at least the cleanup interval to ensure the key is expired. + env := p.awaitEnvironmentFor(update1.Params.EnvID, time.Second) + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + + time.Sleep(minimumCleanupInterval) + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) + assert.Empty(t, env.GetDeprecatedCredentials()) + } + }) +} diff --git a/relay/filedata_testdata_test.go b/relay/filedata_testdata_test.go index 3f6b0c79..e98ea383 100644 --- a/relay/filedata_testdata_test.go +++ b/relay/filedata_testdata_test.go @@ -6,6 +6,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/filedata" "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "time" "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" @@ -58,3 +59,33 @@ var testFileDataEnv2 = filedata.ArchiveEnvironment{ }, }, } + +func RotateSDKKey(primary config.SDKKey) filedata.ArchiveEnvironment { + return RotateSDKKeyWithGracePeriod(primary, "", time.Time{}) +} + +func RotateSDKKeyWithGracePeriod(primary config.SDKKey, expiring config.SDKKey, expiry time.Time) filedata.ArchiveEnvironment { + return filedata.ArchiveEnvironment{ + Params: envfactory.EnvironmentParams{ + EnvID: "env1", + SDKKey: primary, + ExpiringSDKKey: envfactory.ExpiringSDKKey{ + Key: expiring, + Expiration: expiry, + }, + Identifiers: relayenv.EnvIdentifiers{ + ProjName: "Project", + ProjKey: "project", + EnvName: "Env1", + EnvKey: "env1", + }, + }, + SDKData: []ldstoretypes.Collection{ + { + Kind: ldstoreimpl.Features(), + Items: []ldstoretypes.KeyedItemDescriptor{ + {Key: testFileDataFlag1.Key, Item: sharedtest.FlagDesc(testFileDataFlag1)}, + }, + }, + }} +} diff --git a/relay/testutils_test.go b/relay/testutils_test.go index c13225ae..e3037143 100644 --- a/relay/testutils_test.go +++ b/relay/testutils_test.go @@ -26,17 +26,21 @@ type relayTestHelper struct { relay *Relay } -func (h relayTestHelper) awaitEnvironment(envID c.EnvironmentID) relayenv.EnvContext { +func (h relayTestHelper) awaitEnvironmentFor(envID c.EnvironmentID, duration time.Duration) relayenv.EnvContext { h.t.Helper() var e relayenv.EnvContext var err error require.Eventually(h.t, func() bool { e, err = h.relay.getEnvironment(sdkauth.New(envID)) return err == nil - }, time.Second, time.Millisecond*5) + }, duration, time.Millisecond*5) return e } +func (h relayTestHelper) awaitEnvironment(envID c.EnvironmentID) relayenv.EnvContext { + return h.awaitEnvironmentFor(envID, time.Second) +} + func (h relayTestHelper) shouldNotHaveEnvironment(envID c.EnvironmentID, timeout time.Duration) { h.t.Helper() require.Eventually(h.t, func() bool { From 0242c469ff95a8da0dc4776a9c2e22ba8e614285 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Mon, 24 Jun 2024 20:35:42 -0700 Subject: [PATCH 20/26] plumb a new 'offline' bool throughout environment config --- config/config.go | 1 + integrationtests/autoconfig_test.go | 16 ++-- integrationtests/offline_mode_test.go | 73 ++++++++++++++++++- .../projects_and_environments_test.go | 15 ++-- .../standard_mode_payload_filters_test.go | 4 +- integrationtests/standard_mode_test.go | 2 +- integrationtests/test_manager_test.go | 35 +++++++-- internal/envfactory/env_config_factory.go | 1 + internal/envfactory/env_params.go | 3 + internal/filedata/archive_reader.go | 6 +- internal/relayenv/env_context_impl.go | 24 ++++-- 11 files changed, 146 insertions(+), 34 deletions(-) diff --git a/config/config.go b/config/config.go index 9fa82a86..08af54f0 100644 --- a/config/config.go +++ b/config/config.go @@ -263,6 +263,7 @@ type EnvConfig struct { TTL ct.OptDuration `conf:"LD_TTL_"` ProjKey string `conf:"LD_PROJ_KEY_"` FilterKey FilterKey // injected based on [filters] section + Offline bool // set to true if this environment was created in offline mode } type FiltersConfig struct { diff --git a/integrationtests/autoconfig_test.go b/integrationtests/autoconfig_test.go index 29cc49f2..b3973cac 100644 --- a/integrationtests/autoconfig_test.go +++ b/integrationtests/autoconfig_test.go @@ -77,7 +77,7 @@ func testPolicyUpdate(t *testing.T, manager *integrationTestManager) { manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if len(status.Environments) == 1 { if envStatus, ok := status.Environments[string(remainingEnv.id)]; ok { - verifyEnvProperties(t, testData.project, remainingEnv, envStatus, true) + verifyEnvProperties(t, testData.project, remainingEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return true } } @@ -96,7 +96,7 @@ func testAddEnvironment(t *testing.T, manager *integrationTestManager) { manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if len(status.Environments) == len(testData.environments)+1 { if envStatus, ok := status.Environments[string(newEnv.id)]; ok { - verifyEnvProperties(t, testData.project, newEnv, envStatus, true) + verifyEnvProperties(t, testData.project, newEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return true } } @@ -136,7 +136,7 @@ func testUpdatedSDKKeyWithoutExpiry(t *testing.T, manager *integrationTestManage manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.SDKKey) == last5(string(newKey)) && envStatus.ExpiringSDKKey == "" } return false @@ -166,7 +166,7 @@ func testUpdatedSDKKeyWithExpiry(t *testing.T, manager *integrationTestManager) return false } if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.SDKKey) == last5(string(newKey)) && last5(envStatus.ExpiringSDKKey) == last5(string(oldKey)) } @@ -204,13 +204,13 @@ func testUpdatedSDKKeyWithExpiryBeforeStartingRelay(t *testing.T, manager *integ }) defer manager.stopRelay(t) - manager.awaitEnvironments(t, projAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, projAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.SDKKey) == last5(string(newKey)) && last5(envStatus.ExpiringSDKKey) == last5(string(oldKey)) } @@ -238,7 +238,7 @@ func testUpdatedMobileKey(t *testing.T, manager *integrationTestManager) { manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.MobileKey) == last5(string(newKey)) } return false @@ -286,7 +286,7 @@ func withRelayAndTestData(t *testing.T, manager *integrationTestManager, action func awaitInitialState(t *testing.T, manager *integrationTestManager, testData autoConfigTestData) { projsAndEnvs := projsAndEnvs{testData.project: testData.environments} - manager.awaitEnvironments(t, projsAndEnvs, true, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) } diff --git a/integrationtests/offline_mode_test.go b/integrationtests/offline_mode_test.go index d8478aad..7a49d906 100644 --- a/integrationtests/offline_mode_test.go +++ b/integrationtests/offline_mode_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/launchdarkly/ld-relay/v8/config" @@ -25,8 +26,13 @@ type offlineModeTestData struct { autoConfigID autoConfigID } +type apiParams struct { + numProjects int + numEnvironments int +} + func testOfflineMode(t *testing.T, manager *integrationTestManager) { - withOfflineModeTestData(t, manager, func(testData offlineModeTestData) { + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 2}, func(testData offlineModeTestData) { helpers.WithTempDir(func(dirPath string) { fileName := "archive.tar.gz" filePath := filepath.Join(manager.relaySharedDir, fileName) @@ -40,7 +46,66 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, true, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + manager.verifyFlagValues(t, testData.projsAndEnvs) + }) + }) + + // Tests that if we download an archive with a primary SDK key, and then it is subsequently updated + // with a deprecated key, we become initialized with both keys present. + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { + helpers.WithTempDir(func(dirPath string) { + fileName := "archive.tar.gz" + filePath := filepath.Join(manager.relaySharedDir, fileName) + + err := downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + manager.startRelay(t, map[string]string{ + "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + }) + defer manager.stopRelay(t) + + manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + manager.verifyFlagValues(t, testData.projsAndEnvs) + + updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) + + err = downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, expiringSDKKey: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + }) + }) + + // Tests that upon startup, if an archive contains a primary and deprecated key, we become initialized with both keys. + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { + helpers.WithTempDir(func(dirPath string) { + fileName := "archive.tar.gz" + filePath := filepath.Join(manager.relaySharedDir, fileName) + + updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) + + err := downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + manager.startRelay(t, map[string]string{ + "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + }) + defer manager.stopRelay(t) + + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, expiringSDKKey: true}, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) manager.verifyFlagValues(t, testData.projsAndEnvs) @@ -48,8 +113,8 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) } -func withOfflineModeTestData(t *testing.T, manager *integrationTestManager, fn func(offlineModeTestData)) { - projsAndEnvs, err := manager.apiHelper.createProjectsAndEnvironments(2, 2) +func withOfflineModeTestData(t *testing.T, manager *integrationTestManager, cfg apiParams, fn func(offlineModeTestData)) { + projsAndEnvs, err := manager.apiHelper.createProjectsAndEnvironments(cfg.numProjects, cfg.numEnvironments) require.NoError(t, err) defer manager.apiHelper.deleteProjects(projsAndEnvs) diff --git a/integrationtests/projects_and_environments_test.go b/integrationtests/projects_and_environments_test.go index d33e8513..4cdc2e5d 100644 --- a/integrationtests/projects_and_environments_test.go +++ b/integrationtests/projects_and_environments_test.go @@ -15,13 +15,14 @@ type projectInfo struct { } type environmentInfo struct { - id config.EnvironmentID - key string - name string - sdkKey config.SDKKey - mobileKey config.MobileKey - prefix string - projKey string + id config.EnvironmentID + key string + name string + sdkKey config.SDKKey + expiringSdkKey config.SDKKey + mobileKey config.MobileKey + prefix string + projKey string // this is a synthetic field, set only when this environment is a filtered environment. filterKey config.FilterKey diff --git a/integrationtests/standard_mode_payload_filters_test.go b/integrationtests/standard_mode_payload_filters_test.go index 8a767681..e6720a2a 100644 --- a/integrationtests/standard_mode_payload_filters_test.go +++ b/integrationtests/standard_mode_payload_filters_test.go @@ -42,7 +42,7 @@ func testStandardModeWithDefaultFilters(t *testing.T, manager *integrationTestMa manager.startRelay(t, envVars) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { if env.filterKey == "" { return env.key } @@ -105,7 +105,7 @@ func testStandardModeWithSpecificFilters(t *testing.T, manager *integrationTestM manager.startRelay(t, envVars) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { if env.filterKey == "" { return env.key } diff --git a/integrationtests/standard_mode_test.go b/integrationtests/standard_mode_test.go index 18eaf618..c50002ad 100644 --- a/integrationtests/standard_mode_test.go +++ b/integrationtests/standard_mode_test.go @@ -25,7 +25,7 @@ func testStandardMode(t *testing.T, manager *integrationTestManager) { manager.startRelay(t, envVars) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { return string(env.name) }) manager.verifyFlagValues(t, testData.projsAndEnvs) diff --git a/integrationtests/test_manager_test.go b/integrationtests/test_manager_test.go index e600e618..16df99ec 100644 --- a/integrationtests/test_manager_test.go +++ b/integrationtests/test_manager_test.go @@ -303,8 +303,7 @@ func (m *integrationTestManager) awaitRelayStatus(t *testing.T, fn func(api.Stat return lastStatus, success } -func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs projsAndEnvs, - expectNameAndKey bool, envMapKeyFn func(proj projectInfo, env environmentInfo) string) { +func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs projsAndEnvs, expectations *envPropertyExpectations, envMapKeyFn func(proj projectInfo, env environmentInfo) string) { _, success := m.awaitRelayStatus(t, func(status api.StatusRep) bool { if len(status.Environments) != projsAndEnvs.countEnvs() { return false @@ -313,7 +312,7 @@ func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs pr projsAndEnvs.enumerateEnvs(func(proj projectInfo, env environmentInfo) { mapKey := envMapKeyFn(proj, env) if envStatus, found := status.Environments[mapKey]; found { - verifyEnvProperties(t, proj, env, envStatus, expectNameAndKey) + verifyEnvProperties(t, proj, env, envStatus, expectations) if envStatus.Status != "connected" { ok = false } @@ -328,6 +327,21 @@ func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs pr } } +func (m *integrationTestManager) rotateSDKKeys(t *testing.T, existing projsAndEnvs, expiry time.Time) projsAndEnvs { + updated := make(projsAndEnvs) + for proj, envs := range existing { + updated[proj] = make([]environmentInfo, 0) + for _, env := range envs { + newKey, err := m.apiHelper.rotateSDKKey(proj, env, expiry) + require.NoError(t, err, "failed to rotate SDK key for environment %s", env.id) + env.expiringSdkKey = env.sdkKey + env.sdkKey = newKey + updated[proj] = append(updated[proj], env) + } + } + return updated +} + // verifyFlagValues hits Relay's polling evaluation endpoint and verifies that it returns the expected // flags and values, based on the standard way we create flags for our test environments in createFlag. func (m *integrationTestManager) verifyFlagValues(t *testing.T, projsAndEnvs projsAndEnvs) { @@ -454,14 +468,25 @@ func (m *integrationTestManager) withExtraContainer( action(container) } -func verifyEnvProperties(t *testing.T, project projectInfo, environment environmentInfo, envStatus api.EnvironmentStatusRep, expectNameAndKey bool) { +type envPropertyExpectations struct { + nameAndKey bool + expiringSDKKey bool +} + +func verifyEnvProperties(t *testing.T, project projectInfo, environment environmentInfo, envStatus api.EnvironmentStatusRep, expectations *envPropertyExpectations) { assert.Equal(t, string(environment.id), envStatus.EnvID) - if expectNameAndKey { + if expectations == nil { + return + } + if expectations.nameAndKey { assert.Equal(t, environment.name, envStatus.EnvName) assert.Equal(t, environment.key, envStatus.EnvKey) assert.Equal(t, project.name, envStatus.ProjName) assert.Equal(t, project.key, envStatus.ProjKey) } + if expectations.expiringSDKKey { + assert.Equal(t, environment.expiringSdkKey.Masked(), config.SDKKey(envStatus.ExpiringSDKKey).Masked()) + } } func flagKeyForProj(proj projectInfo) string { diff --git a/internal/envfactory/env_config_factory.go b/internal/envfactory/env_config_factory.go index 39cc78bd..7e931806 100644 --- a/internal/envfactory/env_config_factory.go +++ b/internal/envfactory/env_config_factory.go @@ -54,6 +54,7 @@ func (f EnvConfigFactory) MakeEnvironmentConfig(params EnvironmentParams) config AllowedHeader: f.AllowedHeader, SecureMode: params.SecureMode, FilterKey: params.Identifiers.FilterKey, + Offline: params.Offline, } if params.TTL != 0 { ret.TTL = ct.NewOptDuration(params.TTL) diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index d0230c3f..cf2664a6 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -35,6 +35,9 @@ type EnvironmentParams struct { // SecureMode is true if secure mode is required for this environment. SecureMode bool + + // True if the environment was created from an offline file data source. + Offline bool } type ExpiringSDKKey struct { diff --git a/internal/filedata/archive_reader.go b/internal/filedata/archive_reader.go index f0527b53..267be9ee 100644 --- a/internal/filedata/archive_reader.go +++ b/internal/filedata/archive_reader.go @@ -121,8 +121,12 @@ func (ar *archiveReader) GetEnvironmentMetadata(envID config.EnvironmentID) (env if err := json.Unmarshal(data, &rep); err != nil { return environmentMetadata{}, err } + + params := rep.Env.ToParams() + params.Offline = true // Signify that this environment is from an offline mode source + return environmentMetadata{ - params: rep.Env.ToParams(), + params: params, version: rep.Env.Version, dataID: rep.DataID, }, nil diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index a5b271f6..0cb1d9c8 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -120,6 +120,7 @@ type envContextImpl struct { stopMonitoringCredentials chan struct{} doneMonitoringCredentials chan struct{} connectionMapper ConnectionMapper + offline bool } // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of @@ -190,6 +191,7 @@ func NewEnvContext( stopMonitoringCredentials: make(chan struct{}), doneMonitoringCredentials: make(chan struct{}), connectionMapper: params.ConnectionMapper, + offline: envConfig.Offline, } envContext.keyRotator.Initialize([]credential.SDKCredential{ @@ -433,7 +435,9 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { // new SDK client, but does requiring updating any event forwarding components that use a mobile key. switch key := newCredential.(type) { case config.SDKKey: - go c.startSDKClient(key, nil, false) + if !c.offline { + go c.startSDKClient(key, nil, false) + } if c.metricsEventPub != nil { // metrics event publisher always uses SDK key c.metricsEventPub.ReplaceCredential(key) } @@ -457,11 +461,13 @@ func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential for _, handlers := range c.handlers { delete(handlers, oldCredential) } - if sdkKey, ok := oldCredential.(config.SDKKey); ok { - // The SDK client instance is tied to the SDK key, so get rid of it - if client := c.clients[sdkKey]; client != nil { - delete(c.clients, sdkKey) - _ = client.Close() + if !c.offline { + if sdkKey, ok := oldCredential.(config.SDKKey); ok { + // The SDK client instance is tied to the SDK key, so get rid of it + if client := c.clients[sdkKey]; client != nil { + delete(c.clients, sdkKey) + _ = client.Close() + } } } } @@ -560,6 +566,12 @@ func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() + if c.offline { + for _, client := range c.clients { + return client + } + return nil + } return c.clients[c.keyRotator.SDKKey()] } From 79d18fa9f8484736bb947dcb50e92719c2b0f66d Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 25 Jun 2024 11:11:49 -0700 Subject: [PATCH 21/26] more comments --- internal/relayenv/env_context_impl.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 0cb1d9c8..301ec5af 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -430,9 +430,15 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { } } - // A new SDK key means 1. we should start a new SDK client, 2. we should tell all event forwarding - // components that use an SDK key to use the new one. A new mobile key does not require starting a - // new SDK client, but does requiring updating any event forwarding components that use a mobile key. + // A new SDK key means: + // 1. we should start a new SDK client* + // 2. we should tell all event forwarding components that use an SDK key to use the new one. + // A new mobile key does not require starting a new SDK client, but does requiring updating any event forwarding + // components that use a mobile key. + // *Note: we only start a new SDK client in online mode. This is somewhat of an architectural hack because EnvContextImpl + // is used for both offline and online mode, yet starting up an SDK client is only relevant in online mode. This is + // because in offline mode, we already have the data (from a file) - there's no need to open a new streaming connection. + // So, the effect in offline mode when adding/removing credentials is just setting up the new credential mappings. switch key := newCredential.(type) { case config.SDKKey: if !c.offline { @@ -461,6 +467,8 @@ func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential for _, handlers := range c.handlers { delete(handlers, oldCredential) } + // See the comment in addCredential for more context. In offline mode, there's no need to close the SDK client + // because our data comes from a file, not a streaming connection. if !c.offline { if sdkKey, ok := oldCredential.(config.SDKKey); ok { // The SDK client instance is tied to the SDK key, so get rid of it @@ -566,6 +574,10 @@ func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() + // In offline mode, there's only one SDK client. This is awkward because we represent the active clients + // as a map, but in this case there's only one client in the map. A refactoring might pull this logic (along with + // differences in add/removeCredential into an interface that is injected based on the environment being + // offline or online. if c.offline { for _, client := range c.clients { return client From 41d58b974f2734b1b1d3c406287212f48e47e065 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 25 Jun 2024 12:14:14 -0700 Subject: [PATCH 22/26] refactor the new offline bool out of EnvParams --- internal/credential/rotator.go | 23 +++++++++++++++----- internal/credential/rotator_test.go | 26 +++++++++++++++++++++++ internal/envfactory/env_config_factory.go | 9 +++++--- internal/envfactory/env_params.go | 3 --- internal/filedata/archive_reader.go | 6 +----- relay/filedata_actions.go | 3 ++- relay/filedata_actions_test.go | 3 ++- relay/filedata_testdata_test.go | 3 ++- 8 files changed, 57 insertions(+), 19 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 3ece1494..cccae706 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -218,16 +218,23 @@ func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { return previous } +func (r *Rotator) immediatelyRevoke(key config.SDKKey) { + if key.Defined() { + r.expirations = append(r.expirations, key) + r.loggers.Infof("SDK key %s has been immediately revoked", key.Masked()) + } + return +} + func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { r.mu.Lock() defer r.mu.Unlock() previous := r.swapPrimaryKey(sdkKey) - // Immediately revoke the previous SDK key if there's no explicit deprecation notice, otherwise it would - // hang around forever. - if previous.Defined() && grace == nil { - r.expirations = append(r.expirations, previous) - r.loggers.Infof("SDK key %s has been immediately revoked", previous.Masked()) + // If there's no deprecation notice, then the previous key (if any) needs to be immediately revoked so it doesn't + // hang around forever valid. + if grace == nil { + r.immediatelyRevoke(previous) return } if grace != nil { @@ -235,6 +242,12 @@ func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { if previousExpiry != grace.expiry { r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", grace.key.Masked(), grace.expiry, previousExpiry) } + // When a key is deprecated by LD, it will stick around in the deprecated field of the message until something + // else is deprecated. This means that if a key is rotated *without* a deprecation period set for the previous key, + // then we'll receive that new primary key but the deprecation message will be stale - it'll be referring to some + // even older key. We detect this case here (since we already saw the deprecation message in our map) and + // ensure the previous key is revoked. + r.immediatelyRevoke(previous) return } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 9acbc57f..15e030e0 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -197,3 +197,29 @@ func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{primaryKey}, additions) assert.Empty(t, expirations) } + +func TestSDKKeyIsImmediatelyRotatedIfPreviousDeprecationAlreadyTookPlace(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + rotator.Initialize([]SDKCredential{config.SDKKey("key0")}) + + now := time.Unix(1000, 0) + expiry := now.Add(1 * time.Hour) + rotator.RotateWithGrace(config.SDKKey("key1"), NewGracePeriod("key0", expiry, now)) + + additions, expirations := rotator.StepTime(now.Add(30 * time.Minute)) + assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key1")}, additions) + assert.Empty(t, expirations) + + rotator.RotateWithGrace(config.SDKKey("key2"), NewGracePeriod("key0", expiry, now.Add(31*time.Minute))) + + additions, expirations = rotator.StepTime(now.Add(31 * time.Minute)) + assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key2")}, additions) + assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key1")}, expirations) + + additions, expirations = rotator.StepTime(expiry.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key0")}, expirations) + +} diff --git a/internal/envfactory/env_config_factory.go b/internal/envfactory/env_config_factory.go index 7e931806..025fb867 100644 --- a/internal/envfactory/env_config_factory.go +++ b/internal/envfactory/env_config_factory.go @@ -15,10 +15,11 @@ type EnvConfigFactory struct { // DataStorePrefix is the configured data store prefix, which may contain a per-environment placeholder. DataStorePrefix string // DataStorePrefix is the configured data store table name, which may contain a per-environment placeholder. - TableName string - // + TableName string AllowedOrigin ct.OptStringList AllowedHeader ct.OptStringList + // Whether this factory is used for offline mode or not. + Offline bool } // NewEnvConfigFactoryForAutoConfig creates an EnvConfigFactory based on the auto-configuration mode settings. @@ -28,6 +29,7 @@ func NewEnvConfigFactoryForAutoConfig(c config.AutoConfigConfig) EnvConfigFactor TableName: c.EnvDatastoreTableName, AllowedOrigin: c.EnvAllowedOrigin, AllowedHeader: c.EnvAllowedHeader, + Offline: false, } } @@ -38,6 +40,7 @@ func NewEnvConfigFactoryForOfflineMode(c config.OfflineModeConfig) EnvConfigFact TableName: c.EnvDatastoreTableName, AllowedOrigin: c.EnvAllowedOrigin, AllowedHeader: c.EnvAllowedHeader, + Offline: true, } } @@ -54,7 +57,7 @@ func (f EnvConfigFactory) MakeEnvironmentConfig(params EnvironmentParams) config AllowedHeader: f.AllowedHeader, SecureMode: params.SecureMode, FilterKey: params.Identifiers.FilterKey, - Offline: params.Offline, + Offline: f.Offline, } if params.TTL != 0 { ret.TTL = ct.NewOptDuration(params.TTL) diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index cf2664a6..d0230c3f 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -35,9 +35,6 @@ type EnvironmentParams struct { // SecureMode is true if secure mode is required for this environment. SecureMode bool - - // True if the environment was created from an offline file data source. - Offline bool } type ExpiringSDKKey struct { diff --git a/internal/filedata/archive_reader.go b/internal/filedata/archive_reader.go index 267be9ee..f0527b53 100644 --- a/internal/filedata/archive_reader.go +++ b/internal/filedata/archive_reader.go @@ -121,12 +121,8 @@ func (ar *archiveReader) GetEnvironmentMetadata(envID config.EnvironmentID) (env if err := json.Unmarshal(data, &rep); err != nil { return environmentMetadata{}, err } - - params := rep.Env.ToParams() - params.Offline = true // Signify that this environment is from an offline mode source - return environmentMetadata{ - params: params, + params: rep.Env.ToParams(), version: rep.Env.Version, dataID: rep.DataID, }, nil diff --git a/relay/filedata_actions.go b/relay/filedata_actions.go index 4a59992e..1528ccd1 100644 --- a/relay/filedata_actions.go +++ b/relay/filedata_actions.go @@ -1,9 +1,10 @@ package relay import ( - "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "time" + "github.com/launchdarkly/ld-relay/v8/internal/relayenv" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" diff --git a/relay/filedata_actions_test.go b/relay/filedata_actions_test.go index 600e7135..d13a4118 100644 --- a/relay/filedata_actions_test.go +++ b/relay/filedata_actions_test.go @@ -2,13 +2,14 @@ package relay import ( "fmt" - "github.com/launchdarkly/ld-relay/v8/internal/credential" "net/http" "net/http/httptest" "sort" "testing" "time" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/filedata" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" diff --git a/relay/filedata_testdata_test.go b/relay/filedata_testdata_test.go index e98ea383..d097a4ce 100644 --- a/relay/filedata_testdata_test.go +++ b/relay/filedata_testdata_test.go @@ -1,12 +1,13 @@ package relay import ( + "time" + "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "github.com/launchdarkly/ld-relay/v8/internal/filedata" "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" - "time" "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" From 2851a8fa91a43b8b0807b3fda2b90cec036feca8 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 25 Jun 2024 12:26:29 -0700 Subject: [PATCH 23/26] refactoring --- internal/credential/rotator.go | 55 ++++++++++++++------------ internal/credential/rotator_test.go | 60 ++++++++++++++++------------- 2 files changed, 65 insertions(+), 50 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index cccae706..da9b167f 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -145,6 +145,11 @@ type GracePeriod struct { now time.Time } +// Expired returns true if the key has already expired. +func (g *GracePeriod) Expired() bool { + return g.now.After(g.expiry) +} + // NewGracePeriod constructs a new grace period. The current time must be provided in order to // determine if the credential is already expired. func NewGracePeriod(key config.SDKKey, expiry time.Time, now time.Time) *GracePeriod { @@ -218,51 +223,53 @@ func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { return previous } + func (r *Rotator) immediatelyRevoke(key config.SDKKey) { if key.Defined() { r.expirations = append(r.expirations, key) r.loggers.Infof("SDK key %s has been immediately revoked", key.Masked()) } - return } func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { r.mu.Lock() defer r.mu.Unlock() + // Previous will only be .Defined() if there was a previous primary key. previous := r.swapPrimaryKey(sdkKey) + // If there's no deprecation notice, then the previous key (if any) needs to be immediately revoked so it doesn't - // hang around forever valid. + // hang around forever. This case is also true when there is a grace period, but we need to inspect the grace period + // in order to find out if immediate revocation is necessary. if grace == nil { r.immediatelyRevoke(previous) return } - if grace != nil { - if previousExpiry, ok := r.deprecatedSdkKeys[grace.key]; ok { - if previousExpiry != grace.expiry { - r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", grace.key.Masked(), grace.expiry, previousExpiry) - } - // When a key is deprecated by LD, it will stick around in the deprecated field of the message until something - // else is deprecated. This means that if a key is rotated *without* a deprecation period set for the previous key, - // then we'll receive that new primary key but the deprecation message will be stale - it'll be referring to some - // even older key. We detect this case here (since we already saw the deprecation message in our map) and - // ensure the previous key is revoked. - r.immediatelyRevoke(previous) - return - } - if grace.now.After(grace.expiry) { - r.loggers.Infof("Deprecated SDK key %s already expired; ignoring", grace.key.Masked()) - return + if previousExpiry, ok := r.deprecatedSdkKeys[grace.key]; ok { + if previousExpiry != grace.expiry { + r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", grace.key.Masked(), grace.expiry, previousExpiry) } + // When a key is deprecated by LD, it will stick around in the deprecated field of the message until something + // else is deprecated. This means that if a key is rotated *without* a deprecation period set for the previous key, + // then we'll receive that new primary key but the deprecation message will be stale - it'll be referring to the + // last time a key was rotated with a deprecation period. We detect this case here (since we already saw the + // deprecation message in our map) and ensure the previous key is revoked. + r.immediatelyRevoke(previous) + return + } + + if grace.Expired() { + r.loggers.Infof("Deprecated SDK key %s already expired at %v; ignoring", grace.key.Masked(), grace.expiry) + return + } - r.loggers.Infof("SDK key %s was marked for deprecation with an expiry at %v", grace.key.Masked(), grace.expiry) - r.deprecatedSdkKeys[grace.key] = grace.expiry + r.loggers.Infof("SDK key %s was marked for deprecation with an expiry at %v", grace.key.Masked(), grace.expiry) + r.deprecatedSdkKeys[grace.key] = grace.expiry - if grace.key != previous { - r.loggers.Infof("Deprecated SDK key %s was not previously managed by Relay", grace.key.Masked()) - r.additions = append(r.additions, grace.key) - } + if grace.key != previous { + r.loggers.Infof("Deprecated SDK key %s was not previously managed by Relay", grace.key.Masked()) + r.additions = append(r.additions, grace.key) } } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 15e030e0..1cb23336 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -111,6 +111,40 @@ func TestManyImmediateKeyExpirations(t *testing.T) { } } +func TestImmediateSDKKeyDeprecationEvenIfGracePeriodIsPresent(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + key0 := config.SDKKey("key0") + key1 := config.SDKKey("key1") + key2 := config.SDKKey("key2") + + rotator.Initialize([]SDKCredential{key0}) + + start := time.Unix(1000, 0) + halftime := start.Add(30 * time.Minute) + expiry := start.Add(1 * time.Hour) + + rotator.RotateWithGrace(key1, NewGracePeriod(key0, expiry, start)) + + additions, expirations := rotator.StepTime(halftime) + assert.ElementsMatch(t, []SDKCredential{key1}, additions) + assert.Empty(t, expirations) + + // The deprecated key0 given here can be thought of as "stale" or otherwise already-seen by the rotator. + // In this case, it should be effectively ignored but the new key2 should still trigger rotation of the previous + // primary key. + rotator.RotateWithGrace(key2, NewGracePeriod(key0, expiry, halftime)) + + additions, expirations = rotator.StepTime(halftime) + assert.ElementsMatch(t, []SDKCredential{key2}, additions) + assert.ElementsMatch(t, []SDKCredential{key1}, expirations) + + additions, expirations = rotator.StepTime(expiry.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{key0}, expirations) +} + func TestSDKKeyDeprecation(t *testing.T) { mockLog := ldlogtest.NewMockLog() rotator := NewRotator(mockLog.Loggers) @@ -197,29 +231,3 @@ func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{primaryKey}, additions) assert.Empty(t, expirations) } - -func TestSDKKeyIsImmediatelyRotatedIfPreviousDeprecationAlreadyTookPlace(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - rotator.Initialize([]SDKCredential{config.SDKKey("key0")}) - - now := time.Unix(1000, 0) - expiry := now.Add(1 * time.Hour) - rotator.RotateWithGrace(config.SDKKey("key1"), NewGracePeriod("key0", expiry, now)) - - additions, expirations := rotator.StepTime(now.Add(30 * time.Minute)) - assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key1")}, additions) - assert.Empty(t, expirations) - - rotator.RotateWithGrace(config.SDKKey("key2"), NewGracePeriod("key0", expiry, now.Add(31*time.Minute))) - - additions, expirations = rotator.StepTime(now.Add(31 * time.Minute)) - assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key2")}, additions) - assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key1")}, expirations) - - additions, expirations = rotator.StepTime(expiry.Add(1 * time.Millisecond)) - assert.Empty(t, additions) - assert.ElementsMatch(t, []SDKCredential{config.SDKKey("key0")}, expirations) - -} From 247f9dda3f8d0d290d769705dcfc40e6c91a714d Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 25 Jun 2024 13:19:18 -0700 Subject: [PATCH 24/26] more offline mode integration tests --- integrationtests/offline_mode_test.go | 108 +++++++++++++++++- .../projects_and_environments_test.go | 9 ++ integrationtests/test_manager_test.go | 28 ++++- 3 files changed, 135 insertions(+), 10 deletions(-) diff --git a/integrationtests/offline_mode_test.go b/integrationtests/offline_mode_test.go index 7a49d906..7f2c5f28 100644 --- a/integrationtests/offline_mode_test.go +++ b/integrationtests/offline_mode_test.go @@ -7,6 +7,7 @@ package integrationtests import ( "fmt" "io" + "maps" "net/http" "os" "path/filepath" @@ -26,8 +27,12 @@ type offlineModeTestData struct { autoConfigID autoConfigID } +// Used to configure the environment/project setup for an offline mode test. type apiParams struct { - numProjects int + // How many projects to create. + numProjects int + // Within each project, how many environments to create. Note: this must be >= 2 due to the way test flag + // variations are setup. numEnvironments int } @@ -53,7 +58,7 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) }) - // Tests that if we download an archive with a primary SDK key, and then it is subsequently updated + // If we download an archive with a primary SDK key, and then it is subsequently updated // with a deprecated key, we become initialized with both keys present. withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { helpers.WithTempDir(func(dirPath string) { @@ -75,24 +80,29 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) manager.verifyFlagValues(t, testData.projsAndEnvs) + // The updated map will is modified to contain expiringSdkKey field (with the old SDK key) and + // the new key set to whatever the API call returned. updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) err = downloadRelayArchive(manager, testData.autoConfigKey, filePath) manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) require.NoError(t, err) - manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, expiringSDKKey: true}, func(proj projectInfo, env environmentInfo) string { + // We are now asserting that the environment credentials returned by the status endpoint contains not just + // the new SDK key, but the expiring one as well. + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) }) }) - // Tests that upon startup, if an archive contains a primary and deprecated key, we become initialized with both keys. + // Upon startup if an archive contains a primary and deprecated key, we become initialized with both keys. withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { helpers.WithTempDir(func(dirPath string) { fileName := "archive.tar.gz" filePath := filepath.Join(manager.relaySharedDir, fileName) + // Rotation happens before starting up the relay. updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) err := downloadRelayArchive(manager, testData.autoConfigKey, filePath) @@ -105,12 +115,100 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) defer manager.stopRelay(t) - manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, expiringSDKKey: true}, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) manager.verifyFlagValues(t, testData.projsAndEnvs) }) }) + + // If a key is deprecated and then expires, it should be removed from the environment credentials. + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { + helpers.WithTempDir(func(dirPath string) { + fileName := "archive.tar.gz" + filePath := filepath.Join(manager.relaySharedDir, fileName) + + const keyGracePeriod = 5 * time.Second + + // Rotation happens before starting up the relay. + updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(keyGracePeriod)) + then := time.Now() + + err := downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + manager.startRelay(t, map[string]string{ + "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + }) + defer manager.stopRelay(t) + + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + + // This test is timing-dependant on Relay removing the expired keys before we check the /status endpoint. + // To keep the test fast, only sleep as long as necessary to ensure the keys have expired. + toSleep := keyGracePeriod - time.Since(then) + if toSleep > 0 { + time.Sleep(toSleep) + } + manager.awaitEnvironments(t, updated.withoutExpiringKeys(), &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + }) + }) + + // If a key is rotated without a grace period, then the old one should be revoked immediately. + // If a key is deprecated and then expires, it should be removed from the environment credentials. + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { + helpers.WithTempDir(func(dirPath string) { + fileName := "archive.tar.gz" + filePath := filepath.Join(manager.relaySharedDir, fileName) + + err := downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + // Relay will check for expired keys at this interval. + cleanupInterval := 100 * time.Millisecond + // We'll sleep longer than the interval after rotating keys, to try and reduce test flakiness. + cleanupIntervalBuffer := 1 * time.Second + + fmt.Println(cleanupInterval) + manager.startRelay(t, map[string]string{ + "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(), + }) + defer manager.stopRelay(t) + + manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + manager.verifyFlagValues(t, testData.projsAndEnvs) + + updated := maps.Clone(testData.projsAndEnvs) + + // Check that the rotation logic holds for more than one rotation. + const numRotations = 3 + for i := 0; i < numRotations; i++ { + // time.Time{} to signify that there's no deprecation period. + updated = manager.rotateSDKKeys(t, updated, time.Time{}) + + err = downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + time.Sleep(cleanupIntervalBuffer) + + // We are now asserting that the SDK key was rotated (and that there's no expiringSDKKey). + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + } + }) + }) } func withOfflineModeTestData(t *testing.T, manager *integrationTestManager, cfg apiParams, fn func(offlineModeTestData)) { diff --git a/integrationtests/projects_and_environments_test.go b/integrationtests/projects_and_environments_test.go index 4cdc2e5d..af8f571c 100644 --- a/integrationtests/projects_and_environments_test.go +++ b/integrationtests/projects_and_environments_test.go @@ -30,6 +30,15 @@ type environmentInfo struct { type projsAndEnvs map[projectInfo][]environmentInfo +func (pe projsAndEnvs) withoutExpiringKeys() projsAndEnvs { + for _, envs := range pe { + for i := range envs { + envs[i].expiringSdkKey = "" + } + } + return pe +} + func (pe projsAndEnvs) enumerateEnvs(fn func(projectInfo, environmentInfo)) { for proj, envs := range pe { for _, env := range envs { diff --git a/integrationtests/test_manager_test.go b/integrationtests/test_manager_test.go index 16df99ec..7b457e58 100644 --- a/integrationtests/test_manager_test.go +++ b/integrationtests/test_manager_test.go @@ -334,7 +334,11 @@ func (m *integrationTestManager) rotateSDKKeys(t *testing.T, existing projsAndEn for _, env := range envs { newKey, err := m.apiHelper.rotateSDKKey(proj, env, expiry) require.NoError(t, err, "failed to rotate SDK key for environment %s", env.id) - env.expiringSdkKey = env.sdkKey + if expiry.IsZero() { + env.expiringSdkKey = "" + } else { + env.expiringSdkKey = env.sdkKey + } env.sdkKey = newKey updated[proj] = append(updated[proj], env) } @@ -468,9 +472,14 @@ func (m *integrationTestManager) withExtraContainer( action(container) } +// Expectations of the test when checking the status response from Relay. type envPropertyExpectations struct { - nameAndKey bool - expiringSDKKey bool + // The environment/project have names + keys + nameAndKey bool + // The environments have sdkKey and expiringSdkKey that match the expected values. Matching is determined + // by masking the keys and comparing those masked values, since the status response obscures most of the key except + // for the last few characters. + sdkKeys bool } func verifyEnvProperties(t *testing.T, project projectInfo, environment environmentInfo, envStatus api.EnvironmentStatusRep, expectations *envPropertyExpectations) { @@ -484,8 +493,17 @@ func verifyEnvProperties(t *testing.T, project projectInfo, environment environm assert.Equal(t, project.name, envStatus.ProjName) assert.Equal(t, project.key, envStatus.ProjKey) } - if expectations.expiringSDKKey { - assert.Equal(t, environment.expiringSdkKey.Masked(), config.SDKKey(envStatus.ExpiringSDKKey).Masked()) + if expectations.sdkKeys { + if !environment.expiringSdkKey.Defined() { + assert.Empty(t, envStatus.ExpiringSDKKey, "expected no expiring SDK key to be defined") + } else { + assert.Equal(t, environment.expiringSdkKey.Masked(), config.SDKKey(envStatus.ExpiringSDKKey).Masked(), "expected expiring SDK key to match") + } + if !environment.sdkKey.Defined() { + assert.Empty(t, envStatus.SDKKey, "expected no SDK key to be defined") + } else { + assert.Equal(t, environment.sdkKey.Masked(), config.SDKKey(envStatus.SDKKey).Masked(), "expected SDK key to match") + } } } From 46c64a5714a9ab9ece7b3f7087e93043f20ad9c3 Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 25 Jun 2024 13:39:01 -0700 Subject: [PATCH 25/26] refactor tests into individual tests --- integrationtests/offline_mode_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/integrationtests/offline_mode_test.go b/integrationtests/offline_mode_test.go index 7f2c5f28..d1e8e537 100644 --- a/integrationtests/offline_mode_test.go +++ b/integrationtests/offline_mode_test.go @@ -37,6 +37,21 @@ type apiParams struct { } func testOfflineMode(t *testing.T, manager *integrationTestManager) { + t.Run("expected environments and flag values", func(t *testing.T) { + testExpectedEnvironmentsAndFlagValues(t, manager) + }) + t.Run("sdk key is rotated with deprecation after relay has started", func(t *testing.T) { + testSDKKeyRotatedAfterRelayStarted(t, manager) + }) + t.Run("sdk key is rotated with deprecation before relay has started", func(t *testing.T) { + testSDKKeyRotatedBeforeRelayStarted(t, manager) + }) + t.Run("sdk key is rotated multiple times without deprecation after relay started", func(t *testing.T) { + testSDKKeyRotatedWithoutDeprecation(t, manager) + }) +} + +func testExpectedEnvironmentsAndFlagValues(t *testing.T, manager *integrationTestManager) { withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 2}, func(testData offlineModeTestData) { helpers.WithTempDir(func(dirPath string) { fileName := "archive.tar.gz" @@ -57,7 +72,9 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { manager.verifyFlagValues(t, testData.projsAndEnvs) }) }) +} +func testSDKKeyRotatedAfterRelayStarted(t *testing.T, manager *integrationTestManager) { // If we download an archive with a primary SDK key, and then it is subsequently updated // with a deprecated key, we become initialized with both keys present. withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { @@ -95,7 +112,9 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) }) }) +} +func testSDKKeyRotatedBeforeRelayStarted(t *testing.T, manager *integrationTestManager) { // Upon startup if an archive contains a primary and deprecated key, we become initialized with both keys. withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { helpers.WithTempDir(func(dirPath string) { @@ -121,6 +140,9 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { manager.verifyFlagValues(t, testData.projsAndEnvs) }) }) +} + +func testSDKKeyRotatedWithoutDeprecation(t *testing.T, manager *integrationTestManager) { // If a key is deprecated and then expires, it should be removed from the environment credentials. withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { @@ -159,6 +181,9 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) }) }) +} + +func testKeyIsRotatedWithoutGracePeriod(t *testing.T, manager *integrationTestManager) { // If a key is rotated without a grace period, then the old one should be revoked immediately. // If a key is deprecated and then expires, it should be removed from the environment credentials. From 6255451f14db9b754a3f1b9b7bbc2ad3e1f6ad6a Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Tue, 25 Jun 2024 13:49:22 -0700 Subject: [PATCH 26/26] reduce flakiness --- integrationtests/offline_mode_test.go | 39 ++++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/integrationtests/offline_mode_test.go b/integrationtests/offline_mode_test.go index d1e8e537..61a329ce 100644 --- a/integrationtests/offline_mode_test.go +++ b/integrationtests/offline_mode_test.go @@ -46,8 +46,11 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { t.Run("sdk key is rotated with deprecation before relay has started", func(t *testing.T) { testSDKKeyRotatedBeforeRelayStarted(t, manager) }) + t.Run("sdk key is rotated and then expires", func(t *testing.T) { + testSDKKeyExpires(t, manager) + }) t.Run("sdk key is rotated multiple times without deprecation after relay started", func(t *testing.T) { - testSDKKeyRotatedWithoutDeprecation(t, manager) + testKeyIsRotatedWithoutGracePeriod(t, manager) }) } @@ -86,9 +89,16 @@ func testSDKKeyRotatedAfterRelayStarted(t *testing.T, manager *integrationTestMa manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) require.NoError(t, err) + // Ensure the key won't expire during this test. + const keyGracePeriod = 1 * time.Hour + // Relay will check for expired keys at this interval. + const cleanupInterval = 100 * time.Millisecond + // We'll sleep longer than the interval after rotating keys, to try and reduce test flakiness. + const cleanupIntervalBuffer = 1 * time.Second + manager.startRelay(t, map[string]string{ "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), - "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(), }) defer manager.stopRelay(t) @@ -99,12 +109,14 @@ func testSDKKeyRotatedAfterRelayStarted(t *testing.T, manager *integrationTestMa // The updated map will is modified to contain expiringSdkKey field (with the old SDK key) and // the new key set to whatever the API call returned. - updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) + updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(keyGracePeriod)) err = downloadRelayArchive(manager, testData.autoConfigKey, filePath) manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) require.NoError(t, err) + time.Sleep(cleanupIntervalBuffer) + // We are now asserting that the environment credentials returned by the status endpoint contains not just // the new SDK key, but the expiring one as well. manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string { @@ -121,6 +133,11 @@ func testSDKKeyRotatedBeforeRelayStarted(t *testing.T, manager *integrationTestM fileName := "archive.tar.gz" filePath := filepath.Join(manager.relaySharedDir, fileName) + // Relay will check for expired keys at this interval. + const cleanupInterval = 100 * time.Millisecond + // We'll sleep longer than the interval after rotating keys, to try and reduce test flakiness. + const cleanupIntervalBuffer = 1 * time.Second + // Rotation happens before starting up the relay. updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) @@ -130,10 +147,12 @@ func testSDKKeyRotatedBeforeRelayStarted(t *testing.T, manager *integrationTestM manager.startRelay(t, map[string]string{ "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), - "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(), }) defer manager.stopRelay(t) + time.Sleep(cleanupIntervalBuffer) + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) @@ -142,8 +161,7 @@ func testSDKKeyRotatedBeforeRelayStarted(t *testing.T, manager *integrationTestM }) } -func testSDKKeyRotatedWithoutDeprecation(t *testing.T, manager *integrationTestManager) { - +func testSDKKeyExpires(t *testing.T, manager *integrationTestManager) { // If a key is deprecated and then expires, it should be removed from the environment credentials. withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { helpers.WithTempDir(func(dirPath string) { @@ -151,6 +169,8 @@ func testSDKKeyRotatedWithoutDeprecation(t *testing.T, manager *integrationTestM filePath := filepath.Join(manager.relaySharedDir, fileName) const keyGracePeriod = 5 * time.Second + // Relay will check for expired keys at this interval. + const cleanupInterval = 100 * time.Millisecond // Rotation happens before starting up the relay. updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(keyGracePeriod)) @@ -162,7 +182,7 @@ func testSDKKeyRotatedWithoutDeprecation(t *testing.T, manager *integrationTestM manager.startRelay(t, map[string]string{ "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), - "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(), }) defer manager.stopRelay(t) @@ -184,7 +204,6 @@ func testSDKKeyRotatedWithoutDeprecation(t *testing.T, manager *integrationTestM } func testKeyIsRotatedWithoutGracePeriod(t *testing.T, manager *integrationTestManager) { - // If a key is rotated without a grace period, then the old one should be revoked immediately. // If a key is deprecated and then expires, it should be removed from the environment credentials. withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { @@ -197,9 +216,9 @@ func testKeyIsRotatedWithoutGracePeriod(t *testing.T, manager *integrationTestMa require.NoError(t, err) // Relay will check for expired keys at this interval. - cleanupInterval := 100 * time.Millisecond + const cleanupInterval = 100 * time.Millisecond // We'll sleep longer than the interval after rotating keys, to try and reduce test flakiness. - cleanupIntervalBuffer := 1 * time.Second + const cleanupIntervalBuffer = 1 * time.Second fmt.Println(cleanupInterval) manager.startRelay(t, map[string]string{