kola: support denylist Warn feature

coreos#3344

mantle/kola/harness.go

@@ -128,6 +128,7 @@ var (
 	// SkipConsoleWarnings is set via SkipConsoleWarningsTag in kola-denylist.yaml
 	SkipConsoleWarnings bool
 	DenylistedTests     []string // tests which are on the denylist
+	WarnOnErrorTests    []string // denylisted tests we are going to run and warn in case of error
 	Tags                []string // tags to be ran
 
 	// Sharding is a string of the form: hash:m/n where m and n are integers to run only tests which hash to m.
@@ -356,6 +357,7 @@ type DenyListObj struct {
 	Platforms  []string `yaml:"platforms"`
 	SnoozeDate string   `yaml:"snooze"`
 	OsVersion  []string `yaml:"osversion"`
+	Warn       bool     `yaml:"warn,omitempty"`
 }
 
 type ManifestData struct {
@@ -457,7 +459,12 @@ func ParseDenyListYaml(pltfrm string) error {
 
 			fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
 		} else {
-			fmt.Printf("⚠️  Skipping kola test pattern \"%s\":\n", obj.Pattern)
+			if obj.Warn {
+				fmt.Printf("⚠️  Warning kola test pattern: \"%s\":\n", obj.Pattern)
+				WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern)
+			} else {
+				fmt.Printf("⚠️  Skipping kola test pattern \"%s\":\n", obj.Pattern)
+			}
 		}
 
 		if obj.Tracker != "" {
@@ -467,7 +474,7 @@ func ParseDenyListYaml(pltfrm string) error {
 		/// Process "special" patterns which aren't test names, but influence overall behavior
 		if obj.Pattern == SkipConsoleWarningsTag {
 			SkipConsoleWarnings = true
-		} else {
+		} else if !obj.Warn {
 			DenylistedTests = append(DenylistedTests, obj.Pattern)
 		}
 	}
@@ -854,10 +861,29 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 			return reRunErr
 		}
 	}
+
+	// Ignore the error when only denied tests with Warn:true feaute failed
+	if runErr != nil && allTestsWarnOnError(testResults.getResults()) {
+		return nil
+	}
+
 	// If the intial run failed and the rerun passed, we still return an error
 	return runErr
 }
 
+func allTestsWarnOnError(tests []*harness.H) bool {
+	for _, test := range tests {
+		if !test.Failed() {
+			continue
+		}
+		if !IsWanringOnFailure(test.Name()) {
+			return false
+		}
+		fmt.Printf("--- ⚠️ IGNORE: \"%s\"\n", test.Name())
+	}
+	return true
+}
+
 func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun, rerunSuccessTags []string) bool {
 	// Always consider the special AllowRerunSuccessTag that is added
 	// by the test harness in some failure scenarios.
@@ -875,6 +901,9 @@ func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun
 	// Iterate over the tests that were re-ran. If any of them don't
 	// allow rerun success then just exit early.
 	for _, test := range testsToRerun {
+		if IsWanringOnFailure(test) {
+			continue
+		}
 		testAllowsRerunSuccess := false
 		for _, tag := range testsBank[test].Tags {
 			if rerunSuccessTagMap[tag] {
@@ -913,6 +942,16 @@ func GetRerunnableTestName(testName string) (string, bool) {
 	}
 }
 
+func IsWanringOnFailure(testName string) bool {
+	for _, pattern := range WarnOnErrorTests {
+		found, _ := filepath.Match(pattern, testName)
+		if found {
+			return true
+		}
+	}
+	return false
+}
+
 func getRerunnable(tests []*harness.H) []string {
 	var testsToRerun []string
 	for _, h := range tests {