fix: couchbase containers intermittently hang on startup

docker.go

@@ -1653,6 +1653,7 @@ var permanentClientErrors = []func(error) bool{
 	errdefs.IsUnauthorized,
 	errdefs.IsForbidden,
 	errdefs.IsNotImplemented,
+	errdefs.IsSystem,
 }
 
 func isPermanentClientError(err error) bool {

docker_auth.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"net/url"
 	"os"
+	"sync"
 
 	"github.com/cpuguy83/dockercfg"
 	"github.com/docker/docker/api/types/registry"
@@ -79,52 +80,76 @@ func defaultRegistry(ctx context.Context) string {
 	return info.IndexServerAddress
 }
 
+// authConfig represents the details of the auth config for a registry.
+type authConfig struct {
+	key string
+	cfg registry.AuthConfig
+}
+
 // getDockerAuthConfigs returns a map with the auth configs from the docker config file
 // using the registry as the key
-func getDockerAuthConfigs() (map[string]registry.AuthConfig, error) {
+var getDockerAuthConfigs = sync.OnceValues(func() (map[string]registry.AuthConfig, error) {
 	cfg, err := getDockerConfig()
 	if err != nil {
 		return nil, err
 	}
 
 	cfgs := map[string]registry.AuthConfig{}
+	results := make(chan authConfig, len(cfg.AuthConfigs)+len(cfg.CredentialHelpers))
+	var wg sync.WaitGroup
+	wg.Add(len(cfg.AuthConfigs) + len(cfg.CredentialHelpers))
 	for k, v := range cfg.AuthConfigs {
-		ac := registry.AuthConfig{
-			Auth:          v.Auth,
-			Email:         v.Email,
-			IdentityToken: v.IdentityToken,
-			Password:      v.Password,
-			RegistryToken: v.RegistryToken,
-			ServerAddress: v.ServerAddress,
-			Username:      v.Username,
-		}
+		go func(k string, v dockercfg.AuthConfig) {
+			defer wg.Done()
+
+			ac := registry.AuthConfig{
+				Auth:          v.Auth,
+				Email:         v.Email,
+				IdentityToken: v.IdentityToken,
+				Password:      v.Password,
+				RegistryToken: v.RegistryToken,
+				ServerAddress: v.ServerAddress,
+				Username:      v.Username,
+			}
+
+			if v.Username == "" && v.Password == "" {
+				u, p, _ := dockercfg.GetRegistryCredentials(k)
+				ac.Username = u
+				ac.Password = p
+			}
+
+			if v.Auth == "" {
+				ac.Auth = base64.StdEncoding.EncodeToString([]byte(ac.Username + ":" + ac.Password))
+			}
+			results <- authConfig{key: k, cfg: ac}
+		}(k, v)
+	}
+
+	// in the case where the auth field in the .docker/conf.json is empty, and the user has credential helpers registered
+	// the auth comes from there
+	for k := range cfg.CredentialHelpers {
+		go func(k string) {
+			defer wg.Done()
 
-		if v.Username == "" && v.Password == "" {
+			ac := registry.AuthConfig{}
 			u, p, _ := dockercfg.GetRegistryCredentials(k)
 			ac.Username = u
 			ac.Password = p
-		}
-
-		if v.Auth == "" {
-			ac.Auth = base64.StdEncoding.EncodeToString([]byte(ac.Username + ":" + ac.Password))
-		}
-
-		cfgs[k] = ac
+			results <- authConfig{key: k, cfg: ac}
+		}(k)
 	}
 
-	// in the case where the auth field in the .docker/conf.json is empty, and the user has credential helpers registered
-	// the auth comes from there
-	for k := range cfg.CredentialHelpers {
-		ac := registry.AuthConfig{}
-		u, p, _ := dockercfg.GetRegistryCredentials(k)
-		ac.Username = u
-		ac.Password = p
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
 
-		cfgs[k] = ac
+	for ac := range results {
+		cfgs[ac.key] = ac.cfg
 	}
 
 	return cfgs, nil
-}
+})
 
 // getDockerConfig returns the docker config file. It will internally check, in this particular order:
 // 1. the DOCKER_AUTH_CONFIG environment variable, unmarshalling it into a dockercfg.Config

modules/couchbase/couchbase.go

@@ -491,9 +491,14 @@ func (c *CouchbaseContainer) createPrimaryIndex(ctx context.Context, bucket buck
 	body := map[string]string{
 		"statement": "CREATE PRIMARY INDEX on `" + bucket.name + "`",
 	}
-
-	_, err := c.doHttpRequest(ctx, QUERY_PORT, "/query/service", http.MethodPost, body, true)
-
+	err := backoff.Retry(func() error {
+		response, err := c.doHttpRequest(ctx, QUERY_PORT, "/query/service", http.MethodPost, body, true)
+		firstError := gjson.Get(string(response), "errors.0.code").Int()
+		if firstError != 0 {
+			return errors.New("index creation failed")
+		}
+		return err
+	}, backoff.WithContext(backoff.NewExponentialBackOff(), ctx))
 	return err
 }
 

modules/couchbase/couchbase_test.go

@@ -12,7 +12,7 @@ import (
 
 const (
 	// dockerImages {
-	enterpriseEdition = "couchbase:enterprise-7.1.3"
+	enterpriseEdition = "couchbase:enterprise-7.6.1"
 	communityEdition  = "couchbase:community-7.1.1"
 	// }
 )
@@ -54,7 +54,15 @@ func TestCouchbaseWithEnterpriseContainer(t *testing.T) {
 	ctx := context.Background()
 
 	bucketName := "testBucket"
-	container, err := tccouchbase.Run(ctx, enterpriseEdition, tccouchbase.WithBuckets(tccouchbase.NewBucket(bucketName)))
+	bucket := tccouchbase.NewBucket(bucketName).
+		WithQuota(100).
+		WithReplicas(0).
+		WithFlushEnabled(true).
+		WithPrimaryIndex(true)
+	container, err := tccouchbase.Run(ctx,
+		enterpriseEdition,
+		tccouchbase.WithBuckets(bucket),
+	)
 	if err != nil {
 		t.Fatal(err)
 	}