Add ability to skip some tests if provider is excluded

README.md

@@ -72,31 +72,24 @@ The following categories exist.
 You can run the tests in the following ways.
 
 ### Preliminaries
-
-1. Build the project
-    - Before running the tests, ensure you have built the project using the previously specified instrunctions regarding
-      how
-      to [build the project](#build-the-project).
+1. Build the project 
+   - Before running the tests, ensure you have built the project using the previously specified instrunctions regarding how
+to [build the project](#build-the-project).
 2. Implement a webhook to ensure tests are exclusively scheduled on Windows nodes.
-    - For you to reliably target all the Kubernetes e2e tests to be scheduled on a Windows node, you will need to
-      set up
-      a [mutating webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)
-      which dynamically adds a `kubernetes.io/os:windows` pod selector to every scheduled pod.
-      [`kubernetes.io/os`](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os) is a
-      well-known
-      label which is used to indicate the operating system for the node, so that it can be taken into account by the
-      _kubelet_
-      and by the _kube-scheduler_.
-    - The reason it is necessary to add the webhook is to ensure appropriate and successful scheduling of pods on
-      Windows
-      nodes while running the `ops-readiness` tests. Incorrectly scheduling the pods on a different operating system may
-      result in false positives or false negatives in the test results. Some Kubernetes e2e tests do not specify a pod
-      selector and therefore there can be no guarantee that the pods will be scheduled on Windows node when running
-      the `ops-readiness` tests. To reliably run the operational readiness tests on Windows nodes, it is required to
-      configure such a webhook.
-    - To set up your webhook, you can write your own webhook, or repurpose the one provided in this project.
-      See the [webhook README](webhook/README.md) for more instrunctions on how to use the webhook provided in this
-      project.
+   - For you to reliably target all the Kubernetes e2e tests to be scheduled on a Windows node, you will need to 
+set up a [mutating webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)
+which dynamically adds a `kubernetes.io/os:windows` pod selector to every scheduled pod.
+[`kubernetes.io/os`](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os) is a well-known
+label which is used to indicate the operating system for the node, so that it can be taken into account by the _kubelet_
+and by the _kube-scheduler_.
+   - The reason it is necessary to add the webhook is to ensure appropriate and successful scheduling of pods on Windows
+     nodes while running the `ops-readiness` tests. Incorrectly scheduling the pods on a different operating system may
+     result in false positives or false negatives in the test results. Some Kubernetes e2e tests do not specify a pod
+     selector and therefore there can be no guarantee that the pods will be scheduled on Windows node when running
+     the `ops-readiness` tests. To reliably run the operational readiness tests on Windows nodes, it is required to
+     configure such a webhook.
+   - To set up your webhook, you can write your own webhook, or repurpose the one provided in this project.
+     See the [webhook README](webhook/README.md) for more instrunctions on how to use the webhook provided in this project.
 
 ### Run the tests using the ops-readiness binary
 

cmd/root.go

@@ -33,7 +33,7 @@ import (
 func init() {
 	rootCmd.PersistentFlags().StringVar(&testDirectory, "test-directory", "", "Path to YAML root directory containing the tests.")
 	rootCmd.PersistentFlags().StringVar(&E2EBinary, "e2e-binary", "./e2e.test", "The E2E Ginkgo default binary used to run the tests.")
-	rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, etc.)")
+	rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, azure etc.)")
 	rootCmd.PersistentFlags().StringVar(&kubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
 	rootCmd.PersistentFlags().StringVar(&reportDir, "report-dir", getEnvOrDefault("ARTIFACTS", ""), "Report dump directory, uses artifact for CI integration when set.")
 	rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Do not run actual tests, used for sanity check.")
@@ -100,7 +100,19 @@ var (
 					prefix := fmt.Sprintf("%d%d", specIdx+1, testIdx+1)
 					logPrefix := fmt.Sprintf("[%s] %v / %v Tests - ", s.Category, testIdx+1, len(s.TestCases))
 					zap.L().Info(fmt.Sprintf("%sRunning Operational Readiness Test: %v", logPrefix, t.Description))
-					if err = t.RunTest(testCtx, prefix); err != nil {
+
+					var skipProvider = false
+					if len(t.SkipProviders) > 0 {
+						for _, p := range t.SkipProviders {
+							if p == testCtx.Provider {
+								skipProvider = true
+							}
+						}
+					}
+
+					if skipProvider {
+						zap.L().Info(fmt.Sprintf("%sSkipping Operational Readiness Test for Provider %v: %v", logPrefix, provider, t.Description))
+					} else if err = t.RunTest(testCtx, prefix); err != nil {
 						zap.L().Error(fmt.Sprintf("%sFailed Operational Readiness Test: %v, error is %v", logPrefix, t.Description, zap.Error(err)))
 					} else {
 						zap.L().Info(fmt.Sprintf("%sPassed Operational Readiness Test: %v", logPrefix, t.Description))

pkg/testcases/cases.go

@@ -35,6 +35,7 @@ type TestCase struct {
 	Focus              []string `yaml:"focus,omitempty"`
 	Skip               []string `yaml:"skip,omitempty"`
 	KubernetesVersions []string `yaml:"kubernetesVersions,omitempty"` // TODO: If versions are specified, only run tests against specified versions
+	SkipProviders      []string `yaml:"skipProviders,omitempty"`      // Test will be skipped for those providers. Use if there is known issues with a particular provider for a given test
 }
 
 type Category string