initial content

This uses a fork of Kubernetes while the E2E
refactoring (kubernetes/kubernetes#66649) is
being worked on. cli-runtime must be pulled from a private repo
because it is not yet on
github.com/kubernetes (kubernetes/org#58).

Gopkg.toml

@@ -0,0 +1,97 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[prune]
+  go-tests = true
+  unused-packages = true
+
+[[constraint]]
+  name = "k8s.io/kubernetes"
+  branch = "e2e-refactor"
+  source = "github.com/pohly/kubernetes"
+
+[[override]]
+  name = "k8s.io/apimachinery"
+  branch = "master"
+[[override]]
+  name = "k8s.io/api"
+  branch = "master"
+[[override]]
+  name = "k8s.io/apiserver"
+  branch = "master"
+[[override]]
+  name = "k8s.io/kube-aggregator"
+  branch = "master"
+[[override]]
+  name = "k8s.io/client-go"
+  branch = "master"
+[[override]]
+  name = "k8s.io/cli-runtime"
+  branch = "master"
+  # https://github.com/kubernetes/org/issues/58
+  source = "github.com/pohly/k8s-cli-runtime"
+
+[[override]]
+  name = "github.com/docker/docker"
+  revision = "4f3616fb1c112e206b88cb7a9922bf49067a7756"
+[[override]]
+  name = "github.com/codedellemc/goscaleio"
+  revision = "20e2ce2cf8852dc78bd42b76698dcd8dcd77b7b1"
+[[override]]
+  name = "github.com/vmware/photon-controller-go-sdk"
+  revision = "4a435daef6ccd3d0edaac1161e76f51a70c2589a"
+[[override]]
+  name = "github.com/libopenstorage/openstorage"
+  revision = "093a0c3888753c2056e7373183693d670c6bba01"
+[[override]]
+  name = "github.com/opencontainers/runc"
+  revision = "871ba2e58e24314d1fab4517a80410191ba5ad01"
+[[override]]
+  name = "github.com/json-iterator/go"
+  revision = "f2b4162afba35581b6d4a50d3b8f34e33c144682"
+
+[[override]]
+  # Both Kubernetes and docker/docker need the new reference.ParseNormalizedNamed
+  # API call, which isn't yet in any stable release.
+  name = "github.com/docker/distribution"
+  branch = "master"
+
+[[override]]
+  # The only release is too old for Kubernetes.
+  name = "github.com/rancher/go-rancher"
+  branch = "master"
+
+[[override]]
+  # Kubernetes needs older version.
+  name = "github.com/russross/blackfriday"
+  version = "^1.5.1"
+
+# This prevents dep from pulling an empty repo for gopkg.in/fsnotify.v1,
+# something that it does not recover from.
+[[override]]
+  name = "gopkg.in/fsnotify.v1"
+  source = "github.com/fsnotify/fsnotify"

Makefile

@@ -0,0 +1,34 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+IMAGE_VERSION = canary
+REGISTRY_NAME=quay.io/k8scsi
+
+ifdef V
+TESTARGS = -v -args -alsologtostderr -v 5
+else
+TESTARGS =
+endif
+
+test:
+	files=$$(find ./ -name '*.go' | grep -v '^./vendor' ); \
+        if [ $$(gofmt -d $$files | wc -l) -ne 0 ]; then \
+                echo "formatting errors:"; \
+                gofmt -d $$files; \
+                false; \
+        fi
+	go vet $$(go list ./... | grep -v vendor)
+	go test -v $$(go list ./... | grep -v vendor) -args -provider=local -ginkgo.focus=CSI.Volumes
+
+.PHONY: test

README.md

@@ -0,0 +1,32 @@
+Introduction
+============
+
+This repository contains the Kubernetes E2E test framework set up in
+such a way that it runs the Kubernetes storage tests.
+
+Usage
+=====
+
+No cloud-specific code gets imported, so tests can only be run against
+a cluster that has already been set up. Set the the usual
+`KUBECONFIG=<config file>` and then run `go test -v ./test/e2e` or
+`ginkgo ./test/e2e`.
+
+Note that all `test/e2e/storage` tests are imported. To run only the
+CSI tests, use `go test -v ./test/e2e -args -ginkgo.focus=CSI.Volumes`.
+
+Adding `-provider=local` suppresses a message about the flag not being
+set and treating the run as "conformance test", but has no other
+effect in practice.
+
+The
+[upstream documentation](https://github.com/kubernetes/community/blob/master/contributors/devel/e2e-tests.md#local-clusters)
+uses `hack/e2e.go` as wrapper around the test execution. This is not
+necessary for the test suite defined in this repository.
+
+Adding Tests
+============
+
+New tests can be written in their own packages under `test/e2e` and
+then need to be added to the import list in `test/e2e_test.go`.
+

test/e2e/e2e.go

@@ -0,0 +1,202 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/version"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
+	"k8s.io/kubernetes/test/e2e/manifest"
+	testutils "k8s.io/kubernetes/test/utils"
+)
+
+// There are certain operations we only want to run once per overall test invocation
+// (such as deleting old namespaces, or verifying that all system pods are running.
+// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
+// to ensure that these operations only run on the first parallel Ginkgo node.
+//
+// This function takes two parameters: one function which runs on only the first Ginkgo node,
+// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes,
+// accepting the byte array.
+var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
+	// Run only on Ginkgo node 1
+	var data []byte
+
+	framework.Logf("checking config")
+
+	if err := framework.SetupProviderConfig(); err != nil {
+		framework.Failf("Failed to setup provider config: %v", err)
+	}
+
+	c, err := framework.LoadClientset()
+	if err != nil {
+		framework.Failf("Error loading client: ", err)
+	}
+
+	// Delete any namespaces except those created by the system. This ensures no
+	// lingering resources are left over from a previous test run.
+	if framework.TestContext.CleanStart {
+		deleted, err := framework.DeleteNamespaces(c, nil, /* deleteFilter */
+			[]string{
+				metav1.NamespaceSystem,
+				metav1.NamespaceDefault,
+				metav1.NamespacePublic,
+			})
+		if err != nil {
+			framework.Failf("Error deleting orphaned namespaces: %v", err)
+		}
+		framework.Logf("Waiting for deletion of the following namespaces: %v", deleted)
+		if err := framework.WaitForNamespacesDeleted(c, deleted, framework.NamespaceCleanupTimeout); err != nil {
+			framework.Failf("Failed to delete orphaned namespaces %v: %v", deleted, err)
+		}
+	}
+
+	// In large clusters we may get to this point but still have a bunch
+	// of nodes without Routes created. Since this would make a node
+	// unschedulable, we need to wait until all of them are schedulable.
+	framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+
+	// Ensure all pods are running and ready before starting tests (otherwise,
+	// cluster infrastructure pods that are being pulled or started can block
+	// test pods from running, and tests that ensure all pods are running and
+	// ready will fail).
+	podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout
+	// TODO: In large clusters, we often observe a non-starting pods due to
+	// #41007. To avoid those pods preventing the whole test runs (and just
+	// wasting the whole run), we allow for some not-ready pods (with the
+	// number equal to the number of allowed not-ready nodes).
+	if err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, framework.ImagePullerLabels); err != nil {
+		framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem)
+		framework.LogFailedContainers(c, metav1.NamespaceSystem, framework.Logf)
+		runKubernetesServiceTestContainer(c, metav1.NamespaceDefault)
+		framework.Failf("Error waiting for all pods to be running and ready: %v", err)
+	}
+
+	if err := framework.WaitForPodsSuccess(c, metav1.NamespaceSystem, framework.ImagePullerLabels, framework.ImagePrePullingTimeout); err != nil {
+		// There is no guarantee that the image pulling will succeed in 3 minutes
+		// and we don't even run the image puller on all platforms (including GKE).
+		// We wait for it so we get an indication of failures in the logs, and to
+		// maximize benefit of image pre-pulling.
+		framework.Logf("WARNING: Image pulling pods failed to enter success in %v: %v", framework.ImagePrePullingTimeout, err)
+	}
+
+	// Dump the output of the nethealth containers only once per run
+	if framework.TestContext.DumpLogsOnFailure {
+		logFunc := framework.Logf
+		if framework.TestContext.ReportDir != "" {
+			filePath := path.Join(framework.TestContext.ReportDir, "nethealth.txt")
+			file, err := os.Create(filePath)
+			if err != nil {
+				framework.Logf("Failed to create a file with network health data %v: %v\nPrinting to stdout", filePath, err)
+			} else {
+				defer file.Close()
+				if err = file.Chmod(0644); err != nil {
+					framework.Logf("Failed to chmod to 644 of %v: %v", filePath, err)
+				}
+				logFunc = framework.GetLogToFileFunc(file)
+				framework.Logf("Dumping network health container logs from all nodes to file %v", filePath)
+			}
+		} else {
+			framework.Logf("Dumping network health container logs from all nodes...")
+		}
+		framework.LogContainersInPodsWithLabels(c, metav1.NamespaceSystem, framework.ImagePullerLabels, "nethealth", logFunc)
+	}
+
+	// Log the version of the server and this client.
+	framework.Logf("e2e test version: %s", version.Get().GitVersion)
+
+	dc := c.DiscoveryClient
+
+	serverVersion, serverErr := dc.ServerVersion()
+	if serverErr != nil {
+		framework.Logf("Unexpected server error retrieving version: %v", serverErr)
+	}
+	if serverVersion != nil {
+		framework.Logf("kube-apiserver version: %s", serverVersion.GitVersion)
+	}
+
+	// Reference common test to make the import valid.
+	// commontest.CurrentSuite = commontest.E2E
+
+	return data
+
+}, func(data []byte) {
+	// Run on all Ginkgo nodes
+})
+
+// Similar to SynchornizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
+// Here, the order of functions is reversed; first, the function which runs everywhere,
+// and then the function that only runs on the first Ginkgo node.
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	// Run on all Ginkgo nodes
+	framework.Logf("Running AfterSuite actions on all node")
+	framework.RunCleanupActions()
+}, func() {
+	// Run only Ginkgo on node 1
+	framework.Logf("Running AfterSuite actions on node 1")
+})
+
+// RunE2ETests checks configuration parameters (specified through flags) and then runs
+// E2E tests using the Ginkgo runner.
+// This function is called on each Ginkgo node in parallel mode.
+func RunE2ETests(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgowrapper.Fail)
+	ginkgo.RunSpecs(t, "Kubernetes CSI E2E suite")
+}
+
+// Run a test container to try and contact the Kubernetes api-server from a pod, wait for it
+// to flip to Ready, log its output and delete it.
+func runKubernetesServiceTestContainer(c clientset.Interface, ns string) {
+	path := "test/images/clusterapi-tester/pod.yaml"
+	framework.Logf("Parsing pod from %v", path)
+	p, err := manifest.PodFromManifest(path)
+	if err != nil {
+		framework.Logf("Failed to parse clusterapi-tester from manifest %v: %v", path, err)
+		return
+	}
+	p.Namespace = ns
+	if _, err := c.CoreV1().Pods(ns).Create(p); err != nil {
+		framework.Logf("Failed to create %v: %v", p.Name, err)
+		return
+	}
+	defer func() {
+		if err := c.CoreV1().Pods(ns).Delete(p.Name, nil); err != nil {
+			framework.Logf("Failed to delete pod %v: %v", p.Name, err)
+		}
+	}()
+	timeout := 5 * time.Minute
+	if err := framework.WaitForPodCondition(c, ns, p.Name, "clusterapi-tester", timeout, testutils.PodRunningReady); err != nil {
+		framework.Logf("Pod %v took longer than %v to enter running/ready: %v", p.Name, timeout, err)
+		return
+	}
+	logs, err := framework.GetPodLogs(c, ns, p.Name, p.Spec.Containers[0].Name)
+	if err != nil {
+		framework.Logf("Failed to retrieve logs from %v: %v", p.Name, err)
+	} else {
+		framework.Logf("Output of clusterapi-tester:\n%v", logs)
+	}
+}