From 7bfc26e1f56c08849a5ef0441f2acc445b6f241a Mon Sep 17 00:00:00 2001 From: Casey Waldren Date: Mon, 24 Jun 2024 20:35:42 -0700 Subject: [PATCH] plumb a new 'offline' bool throughout environment config --- config/config.go | 1 + integrationtests/api_helpers_test.go | 3 +- integrationtests/autoconfig_test.go | 16 ++-- integrationtests/offline_mode_test.go | 73 ++++++++++++++++++- .../projects_and_environments_test.go | 15 ++-- .../standard_mode_payload_filters_test.go | 4 +- integrationtests/standard_mode_test.go | 2 +- integrationtests/test_manager_test.go | 35 +++++++-- internal/envfactory/env_config_factory.go | 1 + internal/envfactory/env_params.go | 3 + internal/filedata/archive_reader.go | 6 +- internal/relayenv/env_context_impl.go | 24 ++++-- 12 files changed, 148 insertions(+), 35 deletions(-) diff --git a/config/config.go b/config/config.go index 9fa82a86..08af54f0 100644 --- a/config/config.go +++ b/config/config.go @@ -263,6 +263,7 @@ type EnvConfig struct { TTL ct.OptDuration `conf:"LD_TTL_"` ProjKey string `conf:"LD_PROJ_KEY_"` FilterKey FilterKey // injected based on [filters] section + Offline bool // set to true if this environment was created in offline mode } type FiltersConfig struct { diff --git a/integrationtests/api_helpers_test.go b/integrationtests/api_helpers_test.go index 3498e5ea..990d74e2 100644 --- a/integrationtests/api_helpers_test.go +++ b/integrationtests/api_helpers_test.go @@ -337,13 +337,14 @@ func (a *apiHelper) createFlag( flagPost.Variations = append(flagPost.Variations, ldapi.Variation{Value: &valueAsInterface}) } - _, _, err := a.apiClient.FeatureFlagsApi. + _, rsp, err := a.apiClient.FeatureFlagsApi. PostFeatureFlag(a.apiContext, proj.key). FeatureFlagBody(flagPost). Execute() err = a.logResult("Create flag "+flagKey+" in "+proj.key, err) if err != nil { + fmt.Println(rsp.Body) return err } diff --git a/integrationtests/autoconfig_test.go b/integrationtests/autoconfig_test.go index 29cc49f2..b3973cac 100644 --- a/integrationtests/autoconfig_test.go +++ b/integrationtests/autoconfig_test.go @@ -77,7 +77,7 @@ func testPolicyUpdate(t *testing.T, manager *integrationTestManager) { manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if len(status.Environments) == 1 { if envStatus, ok := status.Environments[string(remainingEnv.id)]; ok { - verifyEnvProperties(t, testData.project, remainingEnv, envStatus, true) + verifyEnvProperties(t, testData.project, remainingEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return true } } @@ -96,7 +96,7 @@ func testAddEnvironment(t *testing.T, manager *integrationTestManager) { manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if len(status.Environments) == len(testData.environments)+1 { if envStatus, ok := status.Environments[string(newEnv.id)]; ok { - verifyEnvProperties(t, testData.project, newEnv, envStatus, true) + verifyEnvProperties(t, testData.project, newEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return true } } @@ -136,7 +136,7 @@ func testUpdatedSDKKeyWithoutExpiry(t *testing.T, manager *integrationTestManage manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.SDKKey) == last5(string(newKey)) && envStatus.ExpiringSDKKey == "" } return false @@ -166,7 +166,7 @@ func testUpdatedSDKKeyWithExpiry(t *testing.T, manager *integrationTestManager) return false } if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.SDKKey) == last5(string(newKey)) && last5(envStatus.ExpiringSDKKey) == last5(string(oldKey)) } @@ -204,13 +204,13 @@ func testUpdatedSDKKeyWithExpiryBeforeStartingRelay(t *testing.T, manager *integ }) defer manager.stopRelay(t) - manager.awaitEnvironments(t, projAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, projAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.SDKKey) == last5(string(newKey)) && last5(envStatus.ExpiringSDKKey) == last5(string(oldKey)) } @@ -238,7 +238,7 @@ func testUpdatedMobileKey(t *testing.T, manager *integrationTestManager) { manager.awaitRelayStatus(t, func(status api.StatusRep) bool { if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok { - verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true) + verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true}) return last5(envStatus.MobileKey) == last5(string(newKey)) } return false @@ -286,7 +286,7 @@ func withRelayAndTestData(t *testing.T, manager *integrationTestManager, action func awaitInitialState(t *testing.T, manager *integrationTestManager, testData autoConfigTestData) { projsAndEnvs := projsAndEnvs{testData.project: testData.environments} - manager.awaitEnvironments(t, projsAndEnvs, true, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) } diff --git a/integrationtests/offline_mode_test.go b/integrationtests/offline_mode_test.go index d8478aad..7a49d906 100644 --- a/integrationtests/offline_mode_test.go +++ b/integrationtests/offline_mode_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/launchdarkly/ld-relay/v8/config" @@ -25,8 +26,13 @@ type offlineModeTestData struct { autoConfigID autoConfigID } +type apiParams struct { + numProjects int + numEnvironments int +} + func testOfflineMode(t *testing.T, manager *integrationTestManager) { - withOfflineModeTestData(t, manager, func(testData offlineModeTestData) { + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 2}, func(testData offlineModeTestData) { helpers.WithTempDir(func(dirPath string) { fileName := "archive.tar.gz" filePath := filepath.Join(manager.relaySharedDir, fileName) @@ -40,7 +46,66 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, true, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + manager.verifyFlagValues(t, testData.projsAndEnvs) + }) + }) + + // Tests that if we download an archive with a primary SDK key, and then it is subsequently updated + // with a deprecated key, we become initialized with both keys present. + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { + helpers.WithTempDir(func(dirPath string) { + fileName := "archive.tar.gz" + filePath := filepath.Join(manager.relaySharedDir, fileName) + + err := downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + manager.startRelay(t, map[string]string{ + "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + }) + defer manager.stopRelay(t) + + manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + manager.verifyFlagValues(t, testData.projsAndEnvs) + + updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) + + err = downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, expiringSDKKey: true}, func(proj projectInfo, env environmentInfo) string { + return string(env.id) + }) + }) + }) + + // Tests that upon startup, if an archive contains a primary and deprecated key, we become initialized with both keys. + withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) { + helpers.WithTempDir(func(dirPath string) { + fileName := "archive.tar.gz" + filePath := filepath.Join(manager.relaySharedDir, fileName) + + updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour)) + + err := downloadRelayArchive(manager, testData.autoConfigKey, filePath) + manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err) + require.NoError(t, err) + + manager.startRelay(t, map[string]string{ + "FILE_DATA_SOURCE": filepath.Join(relayContainerSharedDir, fileName), + "EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": "100ms", + }) + defer manager.stopRelay(t) + + manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, expiringSDKKey: true}, func(proj projectInfo, env environmentInfo) string { return string(env.id) }) manager.verifyFlagValues(t, testData.projsAndEnvs) @@ -48,8 +113,8 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) { }) } -func withOfflineModeTestData(t *testing.T, manager *integrationTestManager, fn func(offlineModeTestData)) { - projsAndEnvs, err := manager.apiHelper.createProjectsAndEnvironments(2, 2) +func withOfflineModeTestData(t *testing.T, manager *integrationTestManager, cfg apiParams, fn func(offlineModeTestData)) { + projsAndEnvs, err := manager.apiHelper.createProjectsAndEnvironments(cfg.numProjects, cfg.numEnvironments) require.NoError(t, err) defer manager.apiHelper.deleteProjects(projsAndEnvs) diff --git a/integrationtests/projects_and_environments_test.go b/integrationtests/projects_and_environments_test.go index d33e8513..4cdc2e5d 100644 --- a/integrationtests/projects_and_environments_test.go +++ b/integrationtests/projects_and_environments_test.go @@ -15,13 +15,14 @@ type projectInfo struct { } type environmentInfo struct { - id config.EnvironmentID - key string - name string - sdkKey config.SDKKey - mobileKey config.MobileKey - prefix string - projKey string + id config.EnvironmentID + key string + name string + sdkKey config.SDKKey + expiringSdkKey config.SDKKey + mobileKey config.MobileKey + prefix string + projKey string // this is a synthetic field, set only when this environment is a filtered environment. filterKey config.FilterKey diff --git a/integrationtests/standard_mode_payload_filters_test.go b/integrationtests/standard_mode_payload_filters_test.go index 8a767681..e6720a2a 100644 --- a/integrationtests/standard_mode_payload_filters_test.go +++ b/integrationtests/standard_mode_payload_filters_test.go @@ -42,7 +42,7 @@ func testStandardModeWithDefaultFilters(t *testing.T, manager *integrationTestMa manager.startRelay(t, envVars) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { if env.filterKey == "" { return env.key } @@ -105,7 +105,7 @@ func testStandardModeWithSpecificFilters(t *testing.T, manager *integrationTestM manager.startRelay(t, envVars) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { if env.filterKey == "" { return env.key } diff --git a/integrationtests/standard_mode_test.go b/integrationtests/standard_mode_test.go index 18eaf618..c50002ad 100644 --- a/integrationtests/standard_mode_test.go +++ b/integrationtests/standard_mode_test.go @@ -25,7 +25,7 @@ func testStandardMode(t *testing.T, manager *integrationTestManager) { manager.startRelay(t, envVars) defer manager.stopRelay(t) - manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string { + manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string { return string(env.name) }) manager.verifyFlagValues(t, testData.projsAndEnvs) diff --git a/integrationtests/test_manager_test.go b/integrationtests/test_manager_test.go index e600e618..16df99ec 100644 --- a/integrationtests/test_manager_test.go +++ b/integrationtests/test_manager_test.go @@ -303,8 +303,7 @@ func (m *integrationTestManager) awaitRelayStatus(t *testing.T, fn func(api.Stat return lastStatus, success } -func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs projsAndEnvs, - expectNameAndKey bool, envMapKeyFn func(proj projectInfo, env environmentInfo) string) { +func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs projsAndEnvs, expectations *envPropertyExpectations, envMapKeyFn func(proj projectInfo, env environmentInfo) string) { _, success := m.awaitRelayStatus(t, func(status api.StatusRep) bool { if len(status.Environments) != projsAndEnvs.countEnvs() { return false @@ -313,7 +312,7 @@ func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs pr projsAndEnvs.enumerateEnvs(func(proj projectInfo, env environmentInfo) { mapKey := envMapKeyFn(proj, env) if envStatus, found := status.Environments[mapKey]; found { - verifyEnvProperties(t, proj, env, envStatus, expectNameAndKey) + verifyEnvProperties(t, proj, env, envStatus, expectations) if envStatus.Status != "connected" { ok = false } @@ -328,6 +327,21 @@ func (m *integrationTestManager) awaitEnvironments(t *testing.T, projsAndEnvs pr } } +func (m *integrationTestManager) rotateSDKKeys(t *testing.T, existing projsAndEnvs, expiry time.Time) projsAndEnvs { + updated := make(projsAndEnvs) + for proj, envs := range existing { + updated[proj] = make([]environmentInfo, 0) + for _, env := range envs { + newKey, err := m.apiHelper.rotateSDKKey(proj, env, expiry) + require.NoError(t, err, "failed to rotate SDK key for environment %s", env.id) + env.expiringSdkKey = env.sdkKey + env.sdkKey = newKey + updated[proj] = append(updated[proj], env) + } + } + return updated +} + // verifyFlagValues hits Relay's polling evaluation endpoint and verifies that it returns the expected // flags and values, based on the standard way we create flags for our test environments in createFlag. func (m *integrationTestManager) verifyFlagValues(t *testing.T, projsAndEnvs projsAndEnvs) { @@ -454,14 +468,25 @@ func (m *integrationTestManager) withExtraContainer( action(container) } -func verifyEnvProperties(t *testing.T, project projectInfo, environment environmentInfo, envStatus api.EnvironmentStatusRep, expectNameAndKey bool) { +type envPropertyExpectations struct { + nameAndKey bool + expiringSDKKey bool +} + +func verifyEnvProperties(t *testing.T, project projectInfo, environment environmentInfo, envStatus api.EnvironmentStatusRep, expectations *envPropertyExpectations) { assert.Equal(t, string(environment.id), envStatus.EnvID) - if expectNameAndKey { + if expectations == nil { + return + } + if expectations.nameAndKey { assert.Equal(t, environment.name, envStatus.EnvName) assert.Equal(t, environment.key, envStatus.EnvKey) assert.Equal(t, project.name, envStatus.ProjName) assert.Equal(t, project.key, envStatus.ProjKey) } + if expectations.expiringSDKKey { + assert.Equal(t, environment.expiringSdkKey.Masked(), config.SDKKey(envStatus.ExpiringSDKKey).Masked()) + } } func flagKeyForProj(proj projectInfo) string { diff --git a/internal/envfactory/env_config_factory.go b/internal/envfactory/env_config_factory.go index 39cc78bd..7e931806 100644 --- a/internal/envfactory/env_config_factory.go +++ b/internal/envfactory/env_config_factory.go @@ -54,6 +54,7 @@ func (f EnvConfigFactory) MakeEnvironmentConfig(params EnvironmentParams) config AllowedHeader: f.AllowedHeader, SecureMode: params.SecureMode, FilterKey: params.Identifiers.FilterKey, + Offline: params.Offline, } if params.TTL != 0 { ret.TTL = ct.NewOptDuration(params.TTL) diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index d0230c3f..cf2664a6 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -35,6 +35,9 @@ type EnvironmentParams struct { // SecureMode is true if secure mode is required for this environment. SecureMode bool + + // True if the environment was created from an offline file data source. + Offline bool } type ExpiringSDKKey struct { diff --git a/internal/filedata/archive_reader.go b/internal/filedata/archive_reader.go index f0527b53..267be9ee 100644 --- a/internal/filedata/archive_reader.go +++ b/internal/filedata/archive_reader.go @@ -121,8 +121,12 @@ func (ar *archiveReader) GetEnvironmentMetadata(envID config.EnvironmentID) (env if err := json.Unmarshal(data, &rep); err != nil { return environmentMetadata{}, err } + + params := rep.Env.ToParams() + params.Offline = true // Signify that this environment is from an offline mode source + return environmentMetadata{ - params: rep.Env.ToParams(), + params: params, version: rep.Env.Version, dataID: rep.DataID, }, nil diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 498984a7..bc1a5f72 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -120,6 +120,7 @@ type envContextImpl struct { stopMonitoringCredentials chan struct{} doneMonitoringCredentials chan struct{} connectionMapper ConnectionMapper + offline bool } // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of @@ -190,6 +191,7 @@ func NewEnvContext( stopMonitoringCredentials: make(chan struct{}), doneMonitoringCredentials: make(chan struct{}), connectionMapper: params.ConnectionMapper, + offline: envConfig.Offline, } envContext.keyRotator.Initialize([]credential.SDKCredential{ @@ -433,7 +435,9 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { // new SDK client, but does requiring updating any event forwarding components that use a mobile key. switch key := newCredential.(type) { case config.SDKKey: - go c.startSDKClient(key, nil, false) + if !c.offline { + go c.startSDKClient(key, nil, false) + } if c.metricsEventPub != nil { // metrics event publisher always uses SDK key c.metricsEventPub.ReplaceCredential(key) } @@ -457,11 +461,13 @@ func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential for _, handlers := range c.handlers { delete(handlers, oldCredential) } - if sdkKey, ok := oldCredential.(config.SDKKey); ok { - // The SDK client instance is tied to the SDK key, so get rid of it - if client := c.clients[sdkKey]; client != nil { - delete(c.clients, sdkKey) - _ = client.Close() + if !c.offline { + if sdkKey, ok := oldCredential.(config.SDKKey); ok { + // The SDK client instance is tied to the SDK key, so get rid of it + if client := c.clients[sdkKey]; client != nil { + delete(c.clients, sdkKey) + _ = client.Close() + } } } } @@ -560,6 +566,12 @@ func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() + if c.offline { + for _, client := range c.clients { + return client + } + return nil + } return c.clients[c.keyRotator.SDKKey()] }