internal/civisibility: automatic test retries (DataDog#2892)

internal/civisibility/constants/env.go

@@ -27,4 +27,14 @@ const (
 
 	// CIVisibilityTestSessionNameEnvironmentVariable indicate the test session name to be used on CI Visibility payloads
 	CIVisibilityTestSessionNameEnvironmentVariable = "DD_TEST_SESSION_NAME"
+
+	// CIVisibilityFlakyRetryEnabledEnvironmentVariable kill-switch that allows to explicitly disable retries even if the remote setting is enabled.
+	// This environment variable should be set to "0" or "false" to disable the flaky retry feature.
+	CIVisibilityFlakyRetryEnabledEnvironmentVariable = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"
+
+	// CIVisibilityFlakyRetryCountEnvironmentVariable indicates the maximum number of retry attempts for a single test case.
+	CIVisibilityFlakyRetryCountEnvironmentVariable = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT"
+
+	// CIVisibilityTotalFlakyRetryCountEnvironmentVariable indicates the maximum number of retry attempts for the entire session.
+	CIVisibilityTotalFlakyRetryCountEnvironmentVariable = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT"
 )

internal/civisibility/constants/test_tags.go

@@ -65,6 +65,10 @@ const (
 	// TestSessionName indicates the test session name
 	// This constant is used to tag traces with the test session name
 	TestSessionName = "test_session.name"
+
+	// TestIsRetry indicates a retry execution
+	// This constant is used to tag test events that are part of a retry execution
+	TestIsRetry = "test.is_retry"
 )
 
 // Define valid test status types.

internal/civisibility/integrations/civisibility.go

@@ -70,18 +70,23 @@ func internalCiVisibilityInitialization(tracerInitializer func([]tracer.StartOpt
 
 		// Check if DD_SERVICE has been set; otherwise default to the repo name (from the spec).
 		var opts []tracer.StartOption
-		if v := os.Getenv("DD_SERVICE"); v == "" {
+		serviceName := os.Getenv("DD_SERVICE")
+		if serviceName == "" {
 			if repoURL, ok := ciTags[constants.GitRepositoryURL]; ok {
 				// regex to sanitize the repository url to be used as a service name
 				repoRegex := regexp.MustCompile(`(?m)/([a-zA-Z0-9\-_.]*)$`)
 				matches := repoRegex.FindStringSubmatch(repoURL)
 				if len(matches) > 1 {
 					repoURL = strings.TrimSuffix(matches[1], ".git")
 				}
-				opts = append(opts, tracer.WithService(repoURL))
+				serviceName = repoURL
+				opts = append(opts, tracer.WithService(serviceName))
 			}
 		}
 
+		// Initializing additional features asynchronously
+		go func() { ensureAdditionalFeaturesInitialization(serviceName) }()
+
 		// Initialize the tracer
 		tracerInitializer(opts)
 

internal/civisibility/integrations/civisibility_features.go

@@ -0,0 +1,113 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2024 Datadog, Inc.
+
+package integrations
+
+import (
+	"sync"
+
+	"gopkg.in/DataDog/dd-trace-go.v1/internal"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/constants"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/utils/net"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/log"
+)
+
+const (
+	DefaultFlakyRetryCount      = 5
+	DefaultFlakyTotalRetryCount = 1_000
+)
+
+type (
+	// FlakyRetriesSetting struct to hold all the settings related to flaky tests retries
+	FlakyRetriesSetting struct {
+		RetryCount               int64
+		TotalRetryCount          int64
+		RemainingTotalRetryCount int64
+	}
+)
+
+var (
+	// additionalFeaturesInitializationOnce ensures we do the additional features initialization just once
+	additionalFeaturesInitializationOnce sync.Once
+
+	// ciVisibilityRapidClient contains the http rapid client to do CI Visibility queries and upload to the rapid backend
+	ciVisibilityClient net.Client
+
+	// ciVisibilitySettings contains the CI Visibility settings for this session
+	ciVisibilitySettings net.SettingsResponseData
+
+	// ciVisibilityEarlyFlakyDetectionSettings contains the CI Visibility Early Flake Detection data for this session
+	ciVisibilityEarlyFlakyDetectionSettings net.EfdResponseData
+
+	// ciVisibilityFlakyRetriesSettings contains the CI Visibility Flaky Retries settings for this session
+	ciVisibilityFlakyRetriesSettings FlakyRetriesSetting
+)
+
+// ensureAdditionalFeaturesInitialization initialize all the additional features
+func ensureAdditionalFeaturesInitialization(serviceName string) {
+	additionalFeaturesInitializationOnce.Do(func() {
+		// Create the CI Visibility client
+		ciVisibilityClient = net.NewClientWithServiceName(serviceName)
+		if ciVisibilityClient == nil {
+			log.Error("CI Visibility: error getting the ci visibility http client")
+			return
+		}
+
+		// Get the CI Visibility settings payload for this test session
+		ciSettings, err := ciVisibilityClient.GetSettings()
+		if err != nil {
+			log.Error("CI Visibility: error getting CI visibility settings: %v", err)
+		} else if ciSettings != nil {
+			ciVisibilitySettings = *ciSettings
+		}
+
+		// if early flake detection is enabled then we run the early flake detection request
+		if ciVisibilitySettings.EarlyFlakeDetection.Enabled {
+			ciEfdData, err := ciVisibilityClient.GetEarlyFlakeDetectionData()
+			if err != nil {
+				log.Error("CI Visibility: error getting CI visibility early flake detection data: %v", err)
+			} else if ciEfdData != nil {
+				ciVisibilityEarlyFlakyDetectionSettings = *ciEfdData
+			}
+		}
+
+		// if flaky test retries is enabled then let's load the flaky retries settings
+		if ciVisibilitySettings.FlakyTestRetriesEnabled {
+			flakyRetryEnabledByEnv := internal.BoolEnv(constants.CIVisibilityFlakyRetryEnabledEnvironmentVariable, true)
+			if flakyRetryEnabledByEnv {
+				totalRetriesCount := (int64)(internal.IntEnv(constants.CIVisibilityTotalFlakyRetryCountEnvironmentVariable, DefaultFlakyTotalRetryCount))
+				ciVisibilityFlakyRetriesSettings = FlakyRetriesSetting{
+					RetryCount:               (int64)(internal.IntEnv(constants.CIVisibilityFlakyRetryCountEnvironmentVariable, DefaultFlakyRetryCount)),
+					TotalRetryCount:          totalRetriesCount,
+					RemainingTotalRetryCount: totalRetriesCount,
+				}
+			} else {
+				log.Warn("CI Visibility: flaky test retries was disabled by the environment variable")
+				ciVisibilitySettings.FlakyTestRetriesEnabled = false
+			}
+		}
+	})
+}
+
+// GetSettings gets the settings from the backend settings endpoint
+func GetSettings() *net.SettingsResponseData {
+	// call to ensure the additional features initialization is completed (service name can be null here)
+	ensureAdditionalFeaturesInitialization("")
+	return &ciVisibilitySettings
+}
+
+// GetEarlyFlakeDetectionSettings gets the early flake detection known tests data
+func GetEarlyFlakeDetectionSettings() *net.EfdResponseData {
+	// call to ensure the additional features initialization is completed (service name can be null here)
+	ensureAdditionalFeaturesInitialization("")
+	return &ciVisibilityEarlyFlakyDetectionSettings
+}
+
+// GetFlakyRetriesSettings gets the flaky retries settings
+func GetFlakyRetriesSettings() *FlakyRetriesSetting {
+	// call to ensure the additional features initialization is completed (service name can be null here)
+	ensureAdditionalFeaturesInitialization("")
+	return &ciVisibilityFlakyRetriesSettings
+}