build test images for e2e (#974)

* build test images before e2e * build test images before e2e * build test images for e2e * build test images for e2e * build test images for e2e * build test images for e2e * use built test image in smoke test * use built test image in smoke test * change tests to use built test image * change tests to use built test image * change tests to use built test image * change tests to use built test image * add common base image for all purpose * build images for e2e * rebase with master * add non root base image * add non root base image
knative · Aug 20, 2020 · 4ea31c4 · 4ea31c4
1 parent c39025f
commit 4ea31c4
Show file tree

Hide file tree

Showing 134 changed files with 12,808 additions and 34 deletions.
diff --git a/.ko.yaml b/.ko.yaml
@@ -0,0 +1,3 @@
+defaultBaseImage: gcr.io/distroless/static:nonroot
+baseImageOverrides:
+  knative.dev/client/cmd/kn: docker.io/library/alpine:latest
diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
@@ -42,6 +42,10 @@
 | Add mock test client for dynamic client
 | https://github.com/knative/client/pull/972[#972]
 
+| 🐣
+| Build test images for e2e tests, add .ko.yaml specifying base image
+| https://github.com/knative/client/pull/974[#974]
+
 | 🐛
 | Fix client side volume name generation
 | https://github.com/knative/client/pull/975[#975]

diff --git a/lib/test/integration.go b/lib/test/integration.go
@@ -26,10 +26,8 @@ import (
 )
 
 const (
-	KnDefaultTestImage string        = "gcr.io/knative-samples/helloworld-go"
 	MaxRetries         int           = 10
 	RetrySleepDuration time.Duration = 5 * time.Second
-	KnTestImageEnv     string        = "KN_TEST_IMAGE" // Allow test image to be customized
 )
 
 var nsMutex sync.Mutex
@@ -43,15 +41,6 @@ type KnTest struct {
 	kn        Kn
 }
 
-// GetKnTestImage returns either customized or default kn test image
-func GetKnTestImage() string {
-	value := os.Getenv(KnTestImageEnv)
-	if value == "" {
-		return KnDefaultTestImage
-	}
-	return value
-}
-
 // NewKnTest creates a new KnTest object
 func NewKnTest() (*KnTest, error) {
 	ns := ""

diff --git a/lib/test/service.go b/lib/test/service.go
@@ -21,14 +21,15 @@ import (
 	"gotest.tools/assert"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	pkgtest "knative.dev/pkg/test"
 	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 
 	"knative.dev/client/pkg/util"
 )
 
 // ServiceCreate verifies given service creation in sync mode and also verifies output
 func ServiceCreate(r *KnRunResultCollector, serviceName string) {
-	out := r.KnTest().Kn().Run("service", "create", serviceName, "--image", GetKnTestImage())
+	out := r.KnTest().Kn().Run("service", "create", serviceName, "--image", pkgtest.ImagePath("helloworld"))
 	r.AssertNoError(out)
 	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "service", serviceName, "creating", "namespace", r.KnTest().Kn().Namespace(), "ready"))
 }
@@ -51,7 +52,7 @@ func ServiceList(r *KnRunResultCollector, serviceName string) {
 func ServiceDescribe(r *KnRunResultCollector, serviceName string) {
 	out := r.KnTest().Kn().Run("service", "describe", serviceName)
 	r.AssertNoError(out)
-	assert.Assert(r.T(), util.ContainsAll(out.Stdout, serviceName, r.KnTest().Kn().Namespace(), GetKnTestImage()))
+	assert.Assert(r.T(), util.ContainsAll(out.Stdout, serviceName, r.KnTest().Kn().Namespace(), pkgtest.ImagePath("helloworld")))
 	assert.Assert(r.T(), util.ContainsAll(out.Stdout, "Conditions", "ConfigurationsReady", "Ready", "RoutesReady"))
 	assert.Assert(r.T(), util.ContainsAll(out.Stdout, "Name", "Namespace", "URL", "Age", "Revisions"))
 }

diff --git a/test/README.md b/test/README.md
@@ -66,3 +66,30 @@ mode, use
 ```bash
 test/local-e2e-tests.sh -short
 ```
+
+## Test images
+
+### Building the test images
+
+The [`upload-test-images.sh`](./upload-test-images.sh) script can be used to
+build and push the test images used by the e2e tests. The script
+expects your environment to be setup as described in
+[DEVELOPMENT.md](https://github.com/knative/serving/blob/master/DEVELOPMENT.md#install-requirements).
+
+To run the script for all end to end test images:
+
+```bash
+./test/upload-test-images.sh
+```
+
+A docker tag may be passed as an optional parameter. This can be useful on
+Minikube in tandem with the `--tag` [flag](#using-a-docker-tag):
+
+```bash
+eval $(minikube docker-env)
+./test/upload-test-images.sh any-old-tag
+```
+
+### Adding new test images
+
+New test images should be placed in `./test/test_images`.
diff --git a/test/common.sh b/test/common.sh
@@ -51,3 +51,9 @@ function knative_setup() {
     wait_until_pods_running knative-eventing || return 1
   fi
 }
+
+# Create test resources and images
+function test_setup() {
+  echo ">> Uploading test images..."
+  ${REPO_ROOT_DIR}/test/upload-test-images.sh || return 1
+}
diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh
@@ -62,10 +62,7 @@ smoke_test() {
   ns="kne2esmoketests"
 
   # Test image
-  img=${KN_TEST_IMAGE}
-  if [[ -z "${KN_TEST_IMAGE}" ]]; then
-    img="gcr.io/knative-samples/helloworld-go"
-  fi
+  img=${KO_DOCKER_REPO}/helloworld
 
   set -x
 

diff --git a/test/e2e/service_export_test.go b/test/e2e/service_export_test.go
@@ -32,6 +32,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientv1alpha1 "knative.dev/client/pkg/apis/client/v1alpha1"
+	pkgtest "knative.dev/pkg/test"
 	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 )
 
@@ -136,7 +137,7 @@ func TestServiceExport(t *testing.T) {
 			withRevisionName("hello-rev1"),
 			withRevisionAnnotations(
 				map[string]string{
-					"client.knative.dev/user-image": test.GetKnTestImage(),
+					"client.knative.dev/user-image": pkgtest.ImagePath("helloworld"),
 				}),
 			withRevisionLabels(
 				map[string]string{
@@ -198,7 +199,7 @@ func TestServiceExport(t *testing.T) {
 			withRevisionName("hello-rev1"),
 			withRevisionAnnotations(
 				map[string]string{
-					"client.knative.dev/user-image": test.GetKnTestImage(),
+					"client.knative.dev/user-image": pkgtest.ImagePath("helloworld"),
 				}),
 			withRevisionLabels(
 				map[string]string{
@@ -399,7 +400,7 @@ func withConfigurationLabels(labels map[string]string) expectedServiceOption {
 func withConfigurationAnnotations() expectedServiceOption {
 	return func(svc *servingv1.Service) {
 		svc.Spec.Template.ObjectMeta.Annotations = map[string]string{
-			"client.knative.dev/user-image": test.GetKnTestImage(),
+			"client.knative.dev/user-image": pkgtest.ImagePath("helloworld"),
 		}
 	}
 }
@@ -489,7 +490,7 @@ func withContainer() podSpecOption {
 		spec.Containers = []corev1.Container{
 			{
 				Name:      "user-container",
-				Image:     test.GetKnTestImage(),
+				Image:     pkgtest.ImagePath("helloworld"),
 				Resources: corev1.ResourceRequirements{},
 				ReadinessProbe: &corev1.Probe{
 					SuccessThreshold: int32(1),

diff --git a/test/e2e/service_file_test.go b/test/e2e/service_file_test.go
@@ -28,6 +28,7 @@ import (
 
 	"knative.dev/client/lib/test"
 	"knative.dev/client/pkg/util"
+	pkgtest "knative.dev/pkg/test"
 )
 
 const (
@@ -87,8 +88,8 @@ func TestServiceCreateFromFile(t *testing.T) {
 	defer os.RemoveAll(tempDir)
 	assert.NilError(t, err)
 
-	test.CreateFile("foo.json", fmt.Sprintf(ServiceJSON, test.GetKnTestImage()), tempDir, test.FileModeReadWrite)
-	test.CreateFile("foo.yaml", fmt.Sprintf(ServiceYAML, test.GetKnTestImage()), tempDir, test.FileModeReadWrite)
+	test.CreateFile("foo.json", fmt.Sprintf(ServiceJSON, pkgtest.ImagePath("helloworld")), tempDir, test.FileModeReadWrite)
+	test.CreateFile("foo.yaml", fmt.Sprintf(ServiceYAML, pkgtest.ImagePath("helloworld")), tempDir, test.FileModeReadWrite)
 
 	t.Log("create foo-json service from JSON file")
 	serviceCreateFromFile(r, "foo-json", filepath.Join(tempDir, "foo.json"), true)

diff --git a/test/e2e/service_options_test.go b/test/e2e/service_options_test.go
@@ -26,10 +26,11 @@ import (
 	"testing"
 
 	"gotest.tools/assert"
-	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 
 	"knative.dev/client/lib/test"
 	"knative.dev/client/pkg/util"
+	pkgtest "knative.dev/pkg/test"
+	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 )
 
 func TestServiceOptions(t *testing.T) {
@@ -111,8 +112,8 @@ func TestServiceOptions(t *testing.T) {
 	test.ServiceDelete(r, "svc4")
 
 	t.Log("create, update and validate service with cmd and arg options")
-	serviceCreateWithOptions(r, "svc5", "--cmd", "/go/bin/helloworld")
-	validateContainerField(r, "svc5", "command", "[/go/bin/helloworld]")
+	serviceCreateWithOptions(r, "svc5", "--cmd", "/ko-app/helloworld")
+	validateContainerField(r, "svc5", "command", "[/ko-app/helloworld]")
 	test.ServiceUpdate(r, "svc5", "--arg", "myArg1", "--arg", "--myArg2")
 	validateContainerField(r, "svc5", "args", "[myArg1 --myArg2]")
 	test.ServiceUpdate(r, "svc5", "--arg", "myArg1")
@@ -140,7 +141,7 @@ func TestServiceOptions(t *testing.T) {
 }
 
 func serviceCreateWithOptions(r *test.KnRunResultCollector, serviceName string, options ...string) {
-	command := []string{"service", "create", serviceName, "--image", test.GetKnTestImage()}
+	command := []string{"service", "create", serviceName, "--image", pkgtest.ImagePath("helloworld")}
 	command = append(command, options...)
 	out := r.KnTest().Kn().Run(command...)
 	assert.Check(r.T(), util.ContainsAll(out.Stdout, "service", serviceName, "Creating", "namespace", r.KnTest().Kn().Namespace(), "Ready"))

diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go
@@ -26,6 +26,7 @@ import (
 
 	"knative.dev/client/lib/test"
 	"knative.dev/client/pkg/util"
+	pkgtest "knative.dev/pkg/test"
 	"knative.dev/serving/pkg/apis/serving"
 )
 
@@ -72,7 +73,7 @@ func TestService(t *testing.T) {
 
 func serviceCreatePrivate(r *test.KnRunResultCollector, serviceName string) {
 	out := r.KnTest().Kn().Run("service", "create", serviceName,
-		"--image", test.GetKnTestImage(), "--cluster-local")
+		"--image", pkgtest.ImagePath("helloworld"), "--cluster-local")
 	r.AssertNoError(out)
 	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "service", serviceName, "creating", "namespace", r.KnTest().Kn().Namespace(), "ready"))
 
@@ -83,7 +84,7 @@ func serviceCreatePrivate(r *test.KnRunResultCollector, serviceName string) {
 
 func serviceCreatePrivateUpdatePublic(r *test.KnRunResultCollector, serviceName string) {
 	out := r.KnTest().Kn().Run("service", "create", serviceName,
-		"--image", test.GetKnTestImage(), "--cluster-local")
+		"--image", pkgtest.ImagePath("helloworld"), "--cluster-local")
 	r.AssertNoError(out)
 	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "service", serviceName, "creating", "namespace", r.KnTest().Kn().Namespace(), "ready"))
 
@@ -92,7 +93,7 @@ func serviceCreatePrivateUpdatePublic(r *test.KnRunResultCollector, serviceName
 	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, serving.VisibilityLabelKey, serving.VisibilityClusterLocal))
 
 	out = r.KnTest().Kn().Run("service", "update", serviceName,
-		"--image", test.GetKnTestImage(), "--no-cluster-local")
+		"--image", pkgtest.ImagePath("helloworld"), "--no-cluster-local")
 	r.AssertNoError(out)
 	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "service", serviceName, "updated", "namespace", r.KnTest().Kn().Namespace(), "ready"))
 
@@ -106,7 +107,7 @@ func serviceCreateDuplicate(r *test.KnRunResultCollector, serviceName string) {
 	r.AssertNoError(out)
 	assert.Check(r.T(), strings.Contains(out.Stdout, serviceName), "The service does not exist yet")
 
-	out = r.KnTest().Kn().Run("service", "create", serviceName, "--image", test.GetKnTestImage())
+	out = r.KnTest().Kn().Run("service", "create", serviceName, "--image", pkgtest.ImagePath("helloworld"))
 	r.AssertError(out)
 	assert.Check(r.T(), util.ContainsAll(out.Stderr, "the service already exists"))
 }

diff --git a/test/test_images/helloworld/go.mod b/test/test_images/helloworld/go.mod
diff --git a/test/test_images/helloworld/release.yaml b/test/test_images/helloworld/release.yaml
@@ -0,0 +1,15 @@
+# Copyright 2020 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+image: ko://knative.dev/client/test/test_images/helloworld
diff --git a/test/upload-test-images.sh b/test/upload-test-images.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+#
+# Copyright 2020 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+
+function upload_test_images() {
+  echo ">> Publishing test images"
+  # Script needs to be executed from the root directory
+  # to pickup .ko.yaml
+  cd "$( dirname "$0")/.."
+  local image_dir="test/test_images"
+  local docker_tag=$1
+  local tag_option=""
+  if [ -n "${docker_tag}" ]; then
+    tag_option="--tags $docker_tag,latest"
+  fi
+
+  # ko resolve is being used for the side-effect of publishing images,
+  # so the resulting yaml produced is ignored.
+  # We limit the number of concurrent builds (jobs) to avoid OOMs.
+  ko resolve --jobs=4 ${tag_option} -RBf "${image_dir}" > /dev/null
+}
+
+: ${KO_DOCKER_REPO:?"You must set 'KO_DOCKER_REPO'"}
+
+upload_test_images $@