From 1071f29a57cdd76d5f187424c72d003b2284e615 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 17 Jul 2023 08:14:07 +0200 Subject: [PATCH] kola: support denylist Warn feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/coreos/coreos-assembler/issues/3344 We bubble denylisted tests with 'Warn: true' option as warnings rather than hard failures: ``` kola -p qemu run --parallel 8 ext.config.ntp.* --output-dir tmp/kola ⚠️ Snoozing expired on: Jul 07 2023. Warning kola test pattern: "ext.config.ntp.chrony.dhcp-propagation": 👉 https://github.com/coreos/fedora-coreos-tracker/issues/1508 ⚠️ Warning kola test pattern: "ext.config.ntp.timesyncd.dhcp-propagation": 👉 https://github.com/coreos/fedora-coreos-tracker/issues/1508 === RUN ext.config.ntp.timesyncd.dhcp-propagation === RUN ext.config.ntp.chrony.coreos-platform-chrony-config === RUN ext.config.ntp.chrony.dhcp-propagation --- PASS: ext.config.ntp.chrony.coreos-platform-chrony-config (27.02s) --- FAIL: ext.config.ntp.chrony.dhcp-propagation (107.62s) --- FAIL: ext.config.ntp.timesyncd.dhcp-propagation (118.02s) FAIL, output in tmp/kola --- ⚠️ IGNORE: "ext.config.ntp.chrony.dhcp-propagation" --- ⚠️ IGNORE: "ext.config.ntp.timesyncd.dhcp-propagation" + rc=0 + set +x ``` --- mantle/kola/harness.go | 58 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index f4283e72cf..8e9c03f381 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -128,6 +128,7 @@ var ( // SkipConsoleWarnings is set via SkipConsoleWarningsTag in kola-denylist.yaml SkipConsoleWarnings bool DenylistedTests []string // tests which are on the denylist + WarnOnErrorTests []string // denylisted tests we are going to run and warn in case of error Tags []string // tags to be ran // Sharding is a string of the form: hash:m/n where m and n are integers to run only tests which hash to m. @@ -356,6 +357,7 @@ type DenyListObj struct { Platforms []string `yaml:"platforms"` SnoozeDate string `yaml:"snooze"` OsVersion []string `yaml:"osversion"` + Warn *bool `yaml:"warn,omitempty"` } type ManifestData struct { @@ -447,17 +449,29 @@ func ParseDenyListYaml(pltfrm string) error { continue } + // If Warn is not set in yaml, assume 'true' by default + warn := obj.Warn == nil || *obj.Warn if obj.SnoozeDate != "" { snoozeDate, err := time.Parse(snoozeFormat, obj.SnoozeDate) if err != nil { return err - } else if today.After(snoozeDate) { - continue } - - fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006")) + if today.After(snoozeDate) { + if !warn { + continue + } + fmt.Printf("⚠️ Snoozing expired on: %s. Warning kola test pattern: \"%s\":\n", snoozeDate.Format("Jan 02 2006"), obj.Pattern) + WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern) + } else { + fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006")) + } } else { - fmt.Printf("⚠️ Skipping kola test pattern \"%s\":\n", obj.Pattern) + if warn { + fmt.Printf("⚠️ Warning kola test pattern: \"%s\":\n", obj.Pattern) + WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern) + } else { + fmt.Printf("⚠️ Skipping kola test pattern \"%s\":\n", obj.Pattern) + } } if obj.Tracker != "" { @@ -467,7 +481,7 @@ func ParseDenyListYaml(pltfrm string) error { /// Process "special" patterns which aren't test names, but influence overall behavior if obj.Pattern == SkipConsoleWarningsTag { SkipConsoleWarnings = true - } else { + } else if !warn { DenylistedTests = append(DenylistedTests, obj.Pattern) } } @@ -854,10 +868,29 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu return reRunErr } } + + // Ignore the error when only denied tests with Warn:true feaute failed + if runErr != nil && allTestsWarnOnError(testResults.getResults()) { + return nil + } + // If the intial run failed and the rerun passed, we still return an error return runErr } +func allTestsWarnOnError(tests []*harness.H) bool { + for _, test := range tests { + if !test.Failed() { + continue + } + if !IsWarningOnFailure(test.Name()) { + return false + } + fmt.Printf("--- ⚠️ IGNORE: \"%s\"\n", test.Name()) + } + return true +} + func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun, rerunSuccessTags []string) bool { // Always consider the special AllowRerunSuccessTag that is added // by the test harness in some failure scenarios. @@ -875,6 +908,9 @@ func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun // Iterate over the tests that were re-ran. If any of them don't // allow rerun success then just exit early. for _, test := range testsToRerun { + if IsWarningOnFailure(test) { + continue + } testAllowsRerunSuccess := false for _, tag := range testsBank[test].Tags { if rerunSuccessTagMap[tag] { @@ -913,6 +949,16 @@ func GetRerunnableTestName(testName string) (string, bool) { } } +func IsWarningOnFailure(testName string) bool { + for _, pattern := range WarnOnErrorTests { + found, _ := filepath.Match(pattern, testName) + if found { + return true + } + } + return false +} + func getRerunnable(tests []*harness.H) []string { var testsToRerun []string for _, h := range tests {