chore: Introduce opt-in flag to enable persistence support

docs/running.md

@@ -16,6 +16,7 @@ Options besides `-url`:
 * `-junit <FILEPATH>` - writes test results in JUnit XML format to the specified file
 * `-debug` - enables verbose logging of test actions for failed tests
 * `-debug-all` - enables verbose logging of test actions for all tests
+* `-enable-persistence-tests` - enables tests that require external persistence (e.g. a database like redis)
 * `-record-failures` - record failed test IDs to the given file. Recorded tests can be skipped by the next run of 
 the harness via `-skip-from`.
 * `-skip-from` - skips any test IDs recorded in the specified file. May be used in conjunction with `-record-failures`

docs/service_spec.md

@@ -102,11 +102,23 @@ v4 of the event schema originally required a `contextKeys` property on all featu
 
 This means that the SDK supports technology migrations, a feature which allows customers to migrate between data sources using well-defined migration stages.
 
-#### Capability `"persistent-data-store"`
+#### Capability `"persistent-data-store-{integration}"`
 
-This means the SDK is capable of interacting with external persistent data stores. The test harness must further identify which store types are supported through additional capabilities listed below.
+This means the SDK is capable of interacting with external persistent data stores. Each capability defines the specific persistence technology the SDK is capable of supporting. The full list of supported integrations is as follows:
 
-- `persistent-data-store-redis`: This means the SDK is capable of interacting with a Redis data store.
+- `consul`
+- `dynamodb`
+- `redis`
+
+The persistence store tests are unique in that they rely on external services to be available prior to running the test harness. Because of this, the test harness will not run these tests by default, even if the capabilities exist, without also provided the `-enable-persistence-tests` flag.
+
+These services can be easily provided through the use of docker. Example commands are shown below for your convenience.
+
+```shell
+docker run -p 8000:8000 amazon/dynamodb-local
+docker run -p 8500:8500 hashicorp/consul
+docker run -p 6379:6379 redis
+```
 
 #### Capability `"polling-gzip"`
 

framework/harness/harness.go

@@ -108,6 +108,7 @@ func NewTestHarness(
 	testServiceBaseURL string,
 	testHarnessExternalHostname string,
 	testHarnessPort int,
+	testHarnessEnablePersistenceTests bool,
 	statusQueryTimeout time.Duration,
 	debugLogger framework.Logger,
 	startupOutput io.Writer,
@@ -129,6 +130,21 @@ func NewTestHarness(
 	if err != nil {
 		return nil, err
 	}
+
+	// If we aren't running persistence tests, remove the capabilities that would enable it.
+	if !testHarnessEnablePersistenceTests {
+		filteredCapabilities := make([]string, 0, len(testServiceInfo.Capabilities))
+		for _, c := range testServiceInfo.Capabilities {
+			if c == servicedef.CapabilityPersistentDataStoreRedis ||
+				c == servicedef.CapabilityPersistentDataStoreDynamoDB ||
+				c == servicedef.CapabilityPersistentDataStoreConsul {
+				debugLogger.Printf("Disabling capability %q because persistence tests are disabled", c)
+				continue
+			}
+			filteredCapabilities = append(filteredCapabilities, c)
+		}
+		testServiceInfo.Capabilities = filteredCapabilities
+	}
 	h.testServiceInfo = testServiceInfo
 
 	if err := startServer(testHarnessPort, http.HandlerFunc(h.serveHTTP)); err != nil {

main.go

@@ -55,6 +55,7 @@ func run(params commandParams) (*ldtest.Results, error) {
 		params.serviceURL,
 		params.host,
 		params.port,
+		params.enablePersistenceTests,
 		time.Duration(params.queryTimeoutSeconds)*time.Second,
 		mainDebugLogger,
 		os.Stdout,

params.go

@@ -9,17 +9,18 @@ import (
 )
 
 type commandParams struct {
-	serviceURL          string
-	port                int
-	host                string
-	filters             ldtest.RegexFilters
-	stopServiceAtEnd    bool
-	debug               bool
-	debugAll            bool
-	jUnitFile           string
-	recordFailures      string
-	skipFile            string
-	queryTimeoutSeconds int
+	serviceURL             string
+	port                   int
+	host                   string
+	filters                ldtest.RegexFilters
+	stopServiceAtEnd       bool
+	debug                  bool
+	debugAll               bool
+	enablePersistenceTests bool
+	jUnitFile              string
+	recordFailures         string
+	skipFile               string
+	queryTimeoutSeconds    int
 }
 
 func (c *commandParams) Read(args []string) bool {
@@ -33,6 +34,8 @@ func (c *commandParams) Read(args []string) bool {
 	fs.BoolVar(&c.stopServiceAtEnd, "stop-service-at-end", false, "tell test service to exit after the test run")
 	fs.BoolVar(&c.debug, "debug", false, "enable debug logging for failed tests")
 	fs.BoolVar(&c.debugAll, "debug-all", false, "enable debug logging for all tests")
+	fs.BoolVar(&c.enablePersistenceTests, "enable-persistence-tests", false,
+		"enable tests that require external persistence support")
 	fs.StringVar(&c.jUnitFile, "junit", "", "write JUnit XML output to the specified path")
 	fs.StringVar(&c.recordFailures, "record-failures", "", "record failed test IDs to the given file.\n"+
 		"recorded tests can be skipped by the next run of the harness via -skip-from")

sdktests/server_side_persistence_base.go

@@ -41,7 +41,10 @@ const (
 )
 
 func doServerSidePersistentTests(t *ldtest.T) {
+	ranAtLeastOnce := false
+
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreRedis) {
+		ranAtLeastOnce = true
 		rdb := redis.NewClient(&redis.Options{
 			Addr:     "localhost:6379",
 			Password: "", // no password set
@@ -52,6 +55,7 @@ func doServerSidePersistentTests(t *ldtest.T) {
 	}
 
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreConsul) {
+		ranAtLeastOnce = true
 		config := consul.DefaultConfig()
 		config.Address = "localhost:8500"
 
@@ -62,6 +66,7 @@ func doServerSidePersistentTests(t *ldtest.T) {
 	}
 
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreDynamoDB) {
+		ranAtLeastOnce = true
 		mySession := session.Must(session.NewSession(
 			aws.NewConfig().
 				WithRegion("us-east-1").
@@ -81,6 +86,10 @@ func doServerSidePersistentTests(t *ldtest.T) {
 
 		t.Run("dynamodb", newServerSidePersistentTests(t, &store, "").Run)
 	}
+
+	if !ranAtLeastOnce {
+		t.Skip()
+	}
 }
 
 type PersistentStore interface {

sdktests/testsuite_entry_point.go

@@ -92,10 +92,7 @@ func doAllServerSideTests(t *ldtest.T) {
 	t.Run("migrations", doServerSideMigrationTests)
 	t.Run("hooks", doCommonHooksTests)
 	t.Run("wrapper", doServerSideWrapperTests)
-
-	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStore) {
-		t.Run("persistent data store", doServerSidePersistentTests)
-	}
+	t.Run("persistent data store", doServerSidePersistentTests)
 }
 
 func doAllClientSideTests(t *ldtest.T) {

servicedef/service_params.go

@@ -39,7 +39,6 @@ const (
 	CapabilityPollingGzip                 = "polling-gzip"
 	CapabilityEvaluationHooks             = "evaluation-hooks"
 	CapabilityClientPrereqEvents          = "client-prereq-events"
-	CapabilityPersistentDataStore         = "persistent-data-store"
 	CapabilityPersistentDataStoreRedis    = "persistent-data-store-redis"
 	CapabilityPersistentDataStoreConsul   = "persistent-data-store-consul"
 	CapabilityPersistentDataStoreDynamoDB = "persistent-data-store-dynamodb"