Add certified conformance mode and non-disruptive-conformance mode (#885

) With K8s v1.16, disruptive tests may not be part of the conformance suite. As a result we need to ensure that user workloads are protected by default while still allowing for CNCF certification test runs. In this PR we: - rename the default `conformance` mode to be `non-disruptive-conformance` and modified the skip list appropriately. - added a `certified-conformance` mode which does not provide a skip list and will run even the disruptive tests - tweaked the naming to use lower case values by default to be more consistent with our other flags/naming Fixes #877 Fixes #875 Signed-off-by: John Schnake <[email protected]>
vmware-tanzu · Sep 17, 2019 · 90a586f · 90a586f
1 parent cb43587
commit 90a586f
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 25 deletions.
diff --git a/cmd/sonobuoy/app/args.go b/cmd/sonobuoy/app/args.go
@@ -55,7 +55,7 @@ func AddNamespaceFlag(str *string, flags *pflag.FlagSet) {
 // Mode can be partially or fully overridden by specifying config, e2e-focus, and e2e-skip.
 // The variables specified by those flags will overlay the defaults provided by the given mode.
 func AddModeFlag(mode *ops.Mode, flags *pflag.FlagSet) {
-	*mode = ops.Conformance // default
+	*mode = ops.NonDisruptiveConformance // default
 	flags.VarP(
 		mode, "mode", "m",
 		fmt.Sprintf("What mode to run sonobuoy in. Valid modes are %s.", strings.Join(ops.GetModes(), ", ")),
@@ -132,7 +132,7 @@ const (
 // users. Using e2e-parallel incorrectly has the potential to destroy clusters!
 func AddE2EConfigFlags(flags *pflag.FlagSet) *pflag.FlagSet {
 	e2eFlags := pflag.NewFlagSet("e2e", pflag.ExitOnError)
-	modeName := ops.Conformance
+	modeName := ops.NonDisruptiveConformance
 	defaultMode := modeName.Get()
 	e2eFlags.String(
 		e2eFocusFlag, defaultMode.E2EConfig.Focus,

diff --git a/cmd/sonobuoy/app/args_test.go b/cmd/sonobuoy/app/args_test.go
@@ -34,16 +34,15 @@ func TestGetE2EConfig(t *testing.T) {
 	}{
 		{
 			desc:     "Default",
-			mode:     "Conformance",
+			mode:     "certified-conformance",
 			flagArgs: []string{},
 			expect: &ops.E2EConfig{
 				Focus:    `\[Conformance\]`,
-				Skip:     `Alpha|\[(Disruptive|Feature:[^\]]+|Flaky)\]`,
 				Parallel: "1",
 			},
 		}, {
 			desc:     "Flags settable",
-			mode:     "Conformance",
+			mode:     "certified-conformance",
 			flagArgs: []string{"--e2e-focus=foo", "--e2e-skip=bar", "--e2e-parallel=2"},
 			expect: &ops.E2EConfig{
 				Focus:    `foo`,
@@ -52,12 +51,12 @@ func TestGetE2EConfig(t *testing.T) {
 			},
 		}, {
 			desc:      "Focus regexp validated settable",
-			mode:      "Conformance",
+			mode:      "certified-conformance",
 			flagArgs:  []string{"--e2e-focus=*"},
 			expectErr: true,
 		}, {
 			desc:      "Skip regexp validated settable",
-			mode:      "Conformance",
+			mode:      "certified-conformance",
 			flagArgs:  []string{"--e2e-skip=*"},
 			expectErr: true,
 		},

diff --git a/cmd/sonobuoy/app/gen_test.go b/cmd/sonobuoy/app/gen_test.go
@@ -93,9 +93,9 @@ func TestResolveConfig(t *testing.T) {
 		cliInput string
 	}{
 		{
-			name: "Conformance mode when supplied config is nil (nothing interesting happens)",
+			name: "NonDisruptiveConformance mode when supplied config is nil (nothing interesting happens)",
 			input: &genFlags{
-				mode:           client.Conformance,
+				mode:           client.NonDisruptiveConformance,
 				sonobuoyConfig: SonobuoyConfig{},
 			},
 			expected: &config.Config{
@@ -139,9 +139,9 @@ func TestResolveConfig(t *testing.T) {
 				Resources:        config.DefaultResources,
 			},
 		}, {
-			name: "Conformance mode with plugin selection specified",
+			name: "NonDisruptiveConformance mode with plugin selection specified",
 			input: &genFlags{
-				mode: client.Conformance,
+				mode: client.NonDisruptiveConformance,
 				sonobuoyConfig: SonobuoyConfig{
 					Config: config.Config{
 						PluginSelections: []plugin.Selection{

diff --git a/pkg/client/defaults.go b/pkg/client/defaults.go
@@ -22,9 +22,9 @@ import (
 	"github.com/heptio/sonobuoy/pkg/config"
 )
 
-// NewGenConfig is a GenConfig using the default config and Conformance mode
+// NewGenConfig is a GenConfig using the default config and NonDisruptiveConformance mode
 func NewGenConfig() *GenConfig {
-	modeName := Conformance
+	modeName := NonDisruptiveConformance
 	defaultE2E := modeName.Get().E2EConfig
 
 	return &GenConfig{

diff --git a/pkg/client/mode.go b/pkg/client/mode.go
@@ -31,17 +31,26 @@ type Mode string
 
 const (
 	// Quick runs a single E2E test and the systemd log tests.
-	Quick Mode = "Quick"
+	Quick Mode = "quick"
 
-	// Conformance runs all of the E2E tests and the systemd log tests.
-	Conformance Mode = "Conformance"
+	// NonDisruptiveConformance runs all of the `Conformance` E2E tests which are not marked as disuprtive and the systemd log tests.
+	NonDisruptiveConformance Mode = "non-disruptive-conformance"
+
+	// CertifiedConformance runs all of the `Conformance` E2E tests and the systemd log tests.
+	CertifiedConformance Mode = "certified-conformance"
 )
 
-const defaultSkipList = `Alpha|\[(Disruptive|Feature:[^\]]+|Flaky)\]`
+// nonDisruptiveSkipList should generally just need to skip disruptive tests since upstream
+// will disallow the other types of tests from being tagged as Conformance. However, in v1.16
+// two disruptive tests were  not marked as such, meaning we needed to specify them here to ensure
+// user workload safety. See https://github.com/kubernetes/kubernetes/issues/82663
+// and https://github.com/kubernetes/kubernetes/issues/82787
+const nonDisruptiveSkipList = `\[Disruptive\]|NoExecuteTaintManager`
 
 var modeMap = map[string]Mode{
-	string(Conformance): Conformance,
-	string(Quick):       Quick,
+	string(NonDisruptiveConformance): NonDisruptiveConformance,
+	string(Quick):                    Quick,
+	string(CertifiedConformance):     CertifiedConformance,
 }
 
 // ModeConfig represents the sonobuoy configuration for a given mode.
@@ -60,9 +69,9 @@ func (m *Mode) Type() string { return "Mode" }
 
 // Set the name with a given string. Returns error on unknown mode.
 func (m *Mode) Set(str string) error {
-	// Allow lowercase "conformance", "quick" etc in command line
-	upcase := strings.Title(str)
-	mode, ok := modeMap[upcase]
+	// Allow other casing from user in command line (e.g. Quick & quick)
+	lcase := strings.ToLower(str)
+	mode, ok := modeMap[lcase]
 	if !ok {
 		return fmt.Errorf("unknown mode %s", str)
 	}
@@ -74,11 +83,22 @@ func (m *Mode) Set(str string) error {
 // if there's no associated mode
 func (m *Mode) Get() *ModeConfig {
 	switch *m {
-	case Conformance:
+	case CertifiedConformance:
+		return &ModeConfig{
+			E2EConfig: E2EConfig{
+				Focus:    `\[Conformance\]`,
+				Parallel: "1",
+			},
+			Selectors: []plugin.Selection{
+				{Name: "e2e"},
+				{Name: "systemd-logs"},
+			},
+		}
+	case NonDisruptiveConformance:
 		return &ModeConfig{
 			E2EConfig: E2EConfig{
 				Focus:    `\[Conformance\]`,
-				Skip:     defaultSkipList,
+				Skip:     nonDisruptiveSkipList,
 				Parallel: "1",
 			},
 			Selectors: []plugin.Selection{
@@ -90,7 +110,6 @@ func (m *Mode) Get() *ModeConfig {
 		return &ModeConfig{
 			E2EConfig: E2EConfig{
 				Focus:    "Pods should be submitted and removed",
-				Skip:     defaultSkipList,
 				Parallel: "1",
 			},
 			Selectors: []plugin.Selection{