diff --git a/README.md b/README.md index 790526f..0798b21 100644 --- a/README.md +++ b/README.md @@ -72,31 +72,24 @@ The following categories exist. You can run the tests in the following ways. ### Preliminaries - -1. Build the project - - Before running the tests, ensure you have built the project using the previously specified instrunctions regarding - how - to [build the project](#build-the-project). +1. Build the project + - Before running the tests, ensure you have built the project using the previously specified instrunctions regarding how +to [build the project](#build-the-project). 2. Implement a webhook to ensure tests are exclusively scheduled on Windows nodes. - - For you to reliably target all the Kubernetes e2e tests to be scheduled on a Windows node, you will need to - set up - a [mutating webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) - which dynamically adds a `kubernetes.io/os:windows` pod selector to every scheduled pod. - [`kubernetes.io/os`](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os) is a - well-known - label which is used to indicate the operating system for the node, so that it can be taken into account by the - _kubelet_ - and by the _kube-scheduler_. - - The reason it is necessary to add the webhook is to ensure appropriate and successful scheduling of pods on - Windows - nodes while running the `ops-readiness` tests. Incorrectly scheduling the pods on a different operating system may - result in false positives or false negatives in the test results. Some Kubernetes e2e tests do not specify a pod - selector and therefore there can be no guarantee that the pods will be scheduled on Windows node when running - the `ops-readiness` tests. To reliably run the operational readiness tests on Windows nodes, it is required to - configure such a webhook. - - To set up your webhook, you can write your own webhook, or repurpose the one provided in this project. - See the [webhook README](webhook/README.md) for more instrunctions on how to use the webhook provided in this - project. + - For you to reliably target all the Kubernetes e2e tests to be scheduled on a Windows node, you will need to +set up a [mutating webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) +which dynamically adds a `kubernetes.io/os:windows` pod selector to every scheduled pod. +[`kubernetes.io/os`](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os) is a well-known +label which is used to indicate the operating system for the node, so that it can be taken into account by the _kubelet_ +and by the _kube-scheduler_. + - The reason it is necessary to add the webhook is to ensure appropriate and successful scheduling of pods on Windows + nodes while running the `ops-readiness` tests. Incorrectly scheduling the pods on a different operating system may + result in false positives or false negatives in the test results. Some Kubernetes e2e tests do not specify a pod + selector and therefore there can be no guarantee that the pods will be scheduled on Windows node when running + the `ops-readiness` tests. To reliably run the operational readiness tests on Windows nodes, it is required to + configure such a webhook. + - To set up your webhook, you can write your own webhook, or repurpose the one provided in this project. + See the [webhook README](webhook/README.md) for more instrunctions on how to use the webhook provided in this project. ### Run the tests using the ops-readiness binary diff --git a/cmd/root.go b/cmd/root.go index d40f185..2bf9bc7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -33,7 +33,7 @@ import ( func init() { rootCmd.PersistentFlags().StringVar(&testDirectory, "test-directory", "", "Path to YAML root directory containing the tests.") rootCmd.PersistentFlags().StringVar(&E2EBinary, "e2e-binary", "./e2e.test", "The E2E Ginkgo default binary used to run the tests.") - rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, etc.)") + rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, azure etc.)") rootCmd.PersistentFlags().StringVar(&kubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") rootCmd.PersistentFlags().StringVar(&reportDir, "report-dir", getEnvOrDefault("ARTIFACTS", ""), "Report dump directory, uses artifact for CI integration when set.") rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Do not run actual tests, used for sanity check.") @@ -100,7 +100,19 @@ var ( prefix := fmt.Sprintf("%d%d", specIdx+1, testIdx+1) logPrefix := fmt.Sprintf("[%s] %v / %v Tests - ", s.Category, testIdx+1, len(s.TestCases)) zap.L().Info(fmt.Sprintf("%sRunning Operational Readiness Test: %v", logPrefix, t.Description)) - if err = t.RunTest(testCtx, prefix); err != nil { + + var skipProvider = false + if len(t.SkipProviders) > 0 { + for _, p := range t.SkipProviders { + if p == testCtx.Provider { + skipProvider = true + } + } + } + + if skipProvider { + zap.L().Info(fmt.Sprintf("%sSkipping Operational Readiness Test for Provider %v: %v", logPrefix, provider, t.Description)) + } else if err = t.RunTest(testCtx, prefix); err != nil { zap.L().Error(fmt.Sprintf("%sFailed Operational Readiness Test: %v, error is %v", logPrefix, t.Description, zap.Error(err))) } else { zap.L().Info(fmt.Sprintf("%sPassed Operational Readiness Test: %v", logPrefix, t.Description)) diff --git a/pkg/testcases/cases.go b/pkg/testcases/cases.go index 3935f4c..c7e090e 100644 --- a/pkg/testcases/cases.go +++ b/pkg/testcases/cases.go @@ -35,6 +35,7 @@ type TestCase struct { Focus []string `yaml:"focus,omitempty"` Skip []string `yaml:"skip,omitempty"` KubernetesVersions []string `yaml:"kubernetesVersions,omitempty"` // TODO: If versions are specified, only run tests against specified versions + SkipProviders []string `yaml:"skipProviders,omitempty"` // Test will be skipped for those providers. Use if there is known issues with a particular provider for a given test } type Category string