kola: support denylist Warn feature

coreos#3344

We bubble denylisted tests with 'Warn: true' option as warnings rather than hard failures:
```
kola -p qemu run --parallel 8 ext.config.ntp.* --output-dir tmp/kola

⚠️  Snoozing expired on: Jul 07 2023. Warning kola test pattern: "ext.config.ntp.chrony.dhcp-propagation":
  👉 coreos/fedora-coreos-tracker#1508
⚠️  Warning kola test pattern: "ext.config.ntp.timesyncd.dhcp-propagation":
  👉 coreos/fedora-coreos-tracker#1508
=== RUN   ext.config.ntp.timesyncd.dhcp-propagation
=== RUN   ext.config.ntp.chrony.coreos-platform-chrony-config
=== RUN   ext.config.ntp.chrony.dhcp-propagation
--- PASS: ext.config.ntp.chrony.coreos-platform-chrony-config (27.02s)
--- FAIL: ext.config.ntp.chrony.dhcp-propagation (107.62s)
--- FAIL: ext.config.ntp.timesyncd.dhcp-propagation (118.02s)
FAIL, output in tmp/kola
--- ⚠️ IGNORE: "ext.config.ntp.chrony.dhcp-propagation"
--- ⚠️ IGNORE: "ext.config.ntp.timesyncd.dhcp-propagation"
+ rc=0
+ set +x
```

mantle/kola/harness.go

@@ -128,6 +128,7 @@ var (
 	// SkipConsoleWarnings is set via SkipConsoleWarningsTag in kola-denylist.yaml
 	SkipConsoleWarnings bool
 	DenylistedTests     []string // tests which are on the denylist
+	WarnOnErrorTests    []string // denylisted tests we are going to run and warn in case of error
 	Tags                []string // tags to be ran
 
 	// Sharding is a string of the form: hash:m/n where m and n are integers to run only tests which hash to m.
@@ -356,6 +357,7 @@ type DenyListObj struct {
 	Platforms  []string `yaml:"platforms"`
 	SnoozeDate string   `yaml:"snooze"`
 	OsVersion  []string `yaml:"osversion"`
+	Warn       *bool    `yaml:"warn,omitempty"`
 }
 
 type ManifestData struct {
@@ -447,17 +449,29 @@ func ParseDenyListYaml(pltfrm string) error {
 			continue
 		}
 
+		// If Warn is not set in yaml, assume 'true' by default
+		warn := obj.Warn == nil || *obj.Warn
 		if obj.SnoozeDate != "" {
 			snoozeDate, err := time.Parse(snoozeFormat, obj.SnoozeDate)
 			if err != nil {
 				return err
-			} else if today.After(snoozeDate) {
-				continue
 			}
-
-			fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+			if today.After(snoozeDate) {
+				if !warn {
+					continue
+				}
+				fmt.Printf("⚠️  Snoozing expired on: %s. Warning kola test pattern: \"%s\":\n", snoozeDate.Format("Jan 02 2006"), obj.Pattern)
+				WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern)
+			} else {
+				fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+			}
 		} else {
-			fmt.Printf("⚠️  Skipping kola test pattern \"%s\":\n", obj.Pattern)
+			if warn {
+				fmt.Printf("⚠️  Warning kola test pattern: \"%s\":\n", obj.Pattern)
+				WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern)
+			} else {
+				fmt.Printf("⚠️  Skipping kola test pattern \"%s\":\n", obj.Pattern)
+			}
 		}
 
 		if obj.Tracker != "" {
@@ -467,7 +481,7 @@ func ParseDenyListYaml(pltfrm string) error {
 		/// Process "special" patterns which aren't test names, but influence overall behavior
 		if obj.Pattern == SkipConsoleWarningsTag {
 			SkipConsoleWarnings = true
-		} else {
+		} else if !warn {
 			DenylistedTests = append(DenylistedTests, obj.Pattern)
 		}
 	}
@@ -854,10 +868,29 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 			return reRunErr
 		}
 	}
+
+	// Ignore the error when only denied tests with Warn:true feaute failed
+	if runErr != nil && allTestsWarnOnError(testResults.getResults()) {
+		return nil
+	}
+
 	// If the intial run failed and the rerun passed, we still return an error
 	return runErr
 }
 
+func allTestsWarnOnError(tests []*harness.H) bool {
+	for _, test := range tests {
+		if !test.Failed() {
+			continue
+		}
+		if !IsWarningOnFailure(test.Name()) {
+			return false
+		}
+		fmt.Printf("--- ⚠️ IGNORE: \"%s\"\n", test.Name())
+	}
+	return true
+}
+
 func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun, rerunSuccessTags []string) bool {
 	// Always consider the special AllowRerunSuccessTag that is added
 	// by the test harness in some failure scenarios.
@@ -875,6 +908,9 @@ func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun
 	// Iterate over the tests that were re-ran. If any of them don't
 	// allow rerun success then just exit early.
 	for _, test := range testsToRerun {
+		if IsWarningOnFailure(test) {
+			continue
+		}
 		testAllowsRerunSuccess := false
 		for _, tag := range testsBank[test].Tags {
 			if rerunSuccessTagMap[tag] {
@@ -913,6 +949,16 @@ func GetRerunnableTestName(testName string) (string, bool) {
 	}
 }
 
+func IsWarningOnFailure(testName string) bool {
+	for _, pattern := range WarnOnErrorTests {
+		found, _ := filepath.Match(pattern, testName)
+		if found {
+			return true
+		}
+	}
+	return false
+}
+
 func getRerunnable(tests []*harness.H) []string {
 	var testsToRerun []string
 	for _, h := range tests {