From 395d0123e0fd799b7c256bdc717d63ab70d17cd4 Mon Sep 17 00:00:00 2001 From: Christie Wilson Date: Wed, 25 Jul 2018 15:16:02 -0700 Subject: [PATCH] Only build images for tests are we are running If we are running only one individual test, we don't want to build all of the images, so this commit creates a builder which tracks which images it has built and can be used by a tests to check if it should build an image before running, or it will use the images that have already been built by a previous test. The name of the context tarball has also been made unique (it includes the unix timestamp) to avoid potential test flakes if two tests using the same GCS bucket run simultaneously. --- DEVELOPMENT.md | 14 +-- integration/cleanup.go | 6 +- integration/gcs.go | 22 ++-- integration/images.go | 15 ++- integration/integration_test.go | 193 +++++--------------------------- integration/randomstring.go | 45 -------- 6 files changed, 66 insertions(+), 229 deletions(-) delete mode 100644 integration/randomstring.go diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c54193b739..eb35949408 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -75,12 +75,15 @@ export IMAGE_REPO="gcr.io/somerepo" make integration-test ``` +If you want to run `make integration-test`, you must override the project using environment variables: + +* `GCS_BUCKET` - The name of your GCS bucket +* `IMAGE_REPO` - The path to your docker image repo + You can also run tests with `go test`, for example to run tests individually: ```shell -export GCS_BUCKET="gs://" -export IMAGE_REPO="gcr.io/somerepo" -go test -v --bucket $GCS_BUCKET --repo $IMAGE_REPO -run TestLayers/test_layer_dockerfiles/Dockerfile_test_copy +go test -v --bucket $GCS_BUCKET --repo $IMAGE_REPO -run TestLayers/test_layer_Dockerfile_test_copy_bucket ``` Requirements: @@ -92,11 +95,6 @@ Requirements: the user currently logged into `gcloud` * An image repo which you have write access to via the user currently logged into `gcloud` -If you want to run these tests yourself, you must override the project using environment variables: - -* `GCS_BUCKET` - The name of your GCS bucket -* `IMAGE_REPO` - The path to your docker image repo - These tests will be kicked off by [reviewers](#reviews) for submitted PRs. ## Creating a PR diff --git a/integration/cleanup.go b/integration/cleanup.go index ddc5cf1199..135f4a014b 100644 --- a/integration/cleanup.go +++ b/integration/cleanup.go @@ -22,11 +22,13 @@ import ( "os/signal" ) -func runOnInterrupt(f func()) { +// RunOnInterrupt will execute the function f if execution is interrupted with the +// interrupt signal. +func RunOnInterrupt(f func()) { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { - for _ = range c { + for range c { log.Println("Interrupted, cleaning up.") f() os.Exit(1) diff --git a/integration/gcs.go b/integration/gcs.go index 1f4875624b..2604f1397a 100644 --- a/integration/gcs.go +++ b/integration/gcs.go @@ -23,9 +23,12 @@ import ( "os" "os/exec" "path/filepath" + "time" ) -func createIntegrationTarball() (string, error) { +// CreateIntegrationTarball will take the contents of the integration directory and write +// them to a tarball in a temmporary dir. It will return a path to the tarball. +func CreateIntegrationTarball() (string, error) { log.Println("Creating tarball of integration test files to use as build context") dir, err := os.Getwd() if err != nil { @@ -35,8 +38,7 @@ func createIntegrationTarball() (string, error) { if err != nil { return "", fmt.Errorf("Failed to create temporary directoy to hold tarball: %s", err) } - uuid := randomString(32) - contextFile := fmt.Sprintf("%s/context_%s.tar.gz", tempDir, uuid) + contextFile := fmt.Sprintf("%s/context_%d.tar.gz", tempDir, time.Now().UnixNano()) cmd := exec.Command("tar", "-C", dir, "-zcvf", contextFile, ".") _, err = RunCommandWithoutTest(cmd) if err != nil { @@ -45,19 +47,23 @@ func createIntegrationTarball() (string, error) { return contextFile, err } -func uploadBuildContext(gcsBucket string, contextFile string) (string, error) { - log.Printf("Uploading tarball at %s to GCS bucket at %s\n", contextFile, gcsBucket) +// UploadFileToBucket will upload the at filePath to gcsBucket. It will return the path +// of the file in gcsBucket. +func UploadFileToBucket(gcsBucket string, filePath string) (string, error) { + log.Printf("Uploading file at %s to GCS bucket at %s\n", filePath, gcsBucket) - cmd := exec.Command("gsutil", "cp", contextFile, gcsBucket) + cmd := exec.Command("gsutil", "cp", filePath, gcsBucket) _, err := RunCommandWithoutTest(cmd) if err != nil { return "", fmt.Errorf("Failed to copy tarball to GCS bucket %s: %s", gcsBucket, err) } - return filepath.Join(gcsBucket, contextFile), err + return filepath.Join(gcsBucket, filePath), err } -func deleteFromGCS(path string) error { +// DeleteFromBucket will remove the content at path. path should be the full path +// to a file in GCS. +func DeleteFromBucket(path string) error { cmd := exec.Command("gsutil", "rm", path) _, err := RunCommandWithoutTest(cmd) if err != nil { diff --git a/integration/images.go b/integration/images.go index 19f345c703..b215aa4239 100644 --- a/integration/images.go +++ b/integration/images.go @@ -61,15 +61,22 @@ var singleSnapshotImages = map[string]bool{ var bucketContextTests = []string{"Dockerfile_test_copy_bucket"} var reproducibleTests = []string{"Dockerfile_test_env"} -func getDockerImage(imageRepo, dockerfile string) string { +// GetDockerImage constructs the name of the docker image that would be built with +// dockerfile if it was tagged with imageRepo. +func GetDockerImage(imageRepo, dockerfile string) string { return strings.ToLower(imageRepo + dockerPrefix + dockerfile) } -func getKanikoImage(imageRepo, dockerfile string) string { +// GetKanikoImage constructs the name of the kaniko image that would be built with +// dockerfile if it was tagged with imageRepo. +func GetKanikoImage(imageRepo, dockerfile string) string { return strings.ToLower(imageRepo + kanikoPrefix + dockerfile) } -func findDockerFiles(dockerfilesPath string) ([]string, error) { +// FindDockerFiles will look for test docker files in the directory dockerfilesPath. +// These files must start with `Dockerfile_test`. If the file is one we are intentionally +// skipping, it will not be included in the returned list. +func FindDockerFiles(dockerfilesPath string) ([]string, error) { // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed testsToIgnore := map[string]bool{"Dockerfile_test_user_run": true} allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*")) @@ -156,7 +163,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do if singleSnapshotImages[dockerfile] { buildArgs = append(buildArgs, singleSnapshotFlag) } - kanikoImage := getKanikoImage(imageRepo, dockerfile) + kanikoImage := GetKanikoImage(imageRepo, dockerfile) kanikoCmd := exec.Command("docker", append([]string{"run", "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", diff --git a/integration/integration_test.go b/integration/integration_test.go index 7522b5ac73..f1d5e528e5 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -24,9 +24,6 @@ import ( "math" "os" "os/exec" - "path" - "path/filepath" - "runtime" "strings" "testing" @@ -37,6 +34,7 @@ import ( ) var config = initGCPConfig() +var imageBuilder *DockerFileBuilder type gcpConfig struct { gcsBucket string @@ -61,15 +59,9 @@ func initGCPConfig() *gcpConfig { } const ( - executorImage = "executor-image" - dockerImage = "gcr.io/cloud-builders/docker" ubuntuImage = "ubuntu" - dockerPrefix = "docker-" - kanikoPrefix = "kaniko-" daemonPrefix = "daemon://" dockerfilesPath = "dockerfiles" - buildContextPath = "/workspace" - singleSnapshotFlag = "--single-snapshot" emptyContainerDiff = `[ { "Image1": "%s", @@ -93,9 +85,6 @@ const ( ]` ) -// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed -var testsToIgnore = map[string]bool{"Dockerfile_test_user_run": true} - func meetsRequirements() bool { requiredTools := []string{"container-diff", "gsutil"} hasRequirements := true @@ -114,13 +103,13 @@ func TestMain(m *testing.M) { fmt.Println("Missing required tools") os.Exit(1) } - contextFile, err := createIntegrationTarball() + contextFile, err := CreateIntegrationTarball() if err != nil { fmt.Println("Failed to create tarball of integration files for build context", err) os.Exit(1) } - fileInBucket, err := uploadBuildContext(config.gcsBucket, contextFile) + fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile) if err != nil { fmt.Println("Failed to upload build context", err) os.Exit(1) @@ -129,13 +118,14 @@ func TestMain(m *testing.M) { err = os.Remove(contextFile) if err != nil { err = fmt.Errorf("Failed to remove tarball at %s: %s", contextFile, err) + os.Exit(1) } - runOnInterrupt(func() { deleteFromGCS(fileInBucket) }) - defer deleteFromGCS(fileInBucket) + RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) }) + defer DeleteFromBucket(fileInBucket) fmt.Println("Building kaniko image") - buildKaniko := exec.Command("docker", "build", "-t", executorImage, "-f", "../deploy/Dockerfile", "..") + buildKaniko := exec.Command("docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", "..") err = buildKaniko.Run() if err != nil { fmt.Print(err) @@ -143,143 +133,25 @@ func TestMain(m *testing.M) { os.Exit(1) } - fmt.Println("Building all test images with both docker and kaniko") - err = buildImages() - os.Exit(m.Run()) -} - -func getDockerFiles(dockerfilesPath string) ([]string, error) { - dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*")) + dockerfiles, err := FindDockerFiles(dockerfilesPath) if err != nil { - return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err) - } - return dockerfiles, nil -} - -func getDockerImage(imageRepo, dockerfile string) string { - return strings.ToLower(imageRepo + dockerPrefix + dockerfile) -} - -func getKanikoImage(imageRepo, dockerfile string) string { - return strings.ToLower(config.imageRepo + kanikoPrefix + dockerfile) -} - -// buildImages will bulid all dockerfils in ./integration/dockerfiles using both docker -// and Kaniko, so subsequent tests can compare the results. -func buildImages() error { - dockerfiles, err := getDockerFiles(dockerfilesPath) - if err != nil { - return fmt.Errorf("Couldn't build images because files couldn't be found: %s", err) - } - - // Maps Dockerfiles to the args that they should be build with - argsMap := map[string][]string{ - "Dockerfile_test_run": {"file=/file"}, - "Dockerfile_test_workdir": {"workdir=/arg/workdir"}, - "Dockerfile_test_add": {"file=context/foo"}, - "Dockerfile_test_onbuild": {"file=/tmp/onbuild"}, - "Dockerfile_test_scratch": { - "image=scratch", - "hello=hello-value", - "file=context/foo", - "file3=context/b*", - }, - "Dockerfile_test_multistage": {"file=/foo2"}, - } - - // These images will be built via Kaniko with only one layer/snapshot, to test - // the single snapshot functionality. - singleSnapshotImages := map[string]bool{ - "Dockerfile_test_add": true, - "Dockerfile_test_scratch": true, - } - - bucketContextTests := []string{"Dockerfile_test_copy_bucket"} - reproducibleTests := []string{"Dockerfile_test_env"} - - _, ex, _, _ := runtime.Caller(0) - cwd := filepath.Dir(ex) - - for _, dockerfile := range dockerfiles { - dockerfile = dockerfile[len("dockerfile/")+1:] - if testsToIgnore[dockerfile] { - continue - } - fmt.Printf("Building images for Dockerfile %s\n", dockerfile) - - var buildArgs []string - buildArgFlag := "--build-arg" - for _, arg := range argsMap[dockerfile] { - buildArgs = append(buildArgs, buildArgFlag) - buildArgs = append(buildArgs, arg) - } - // build docker image - dockerImage := strings.ToLower(config.imageRepo + dockerPrefix + dockerfile) - dockerCmd := exec.Command("docker", - append([]string{"build", - "-t", dockerImage, - "-f", path.Join(dockerfilesPath, dockerfile), - "."}, - buildArgs...)..., - ) - _, err := RunCommandWithoutTest(dockerCmd) - if err != nil { - return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s", dockerImage, dockerCmd.Args, err) - } - - contextFlag := "-c" - contextPath := buildContextPath - for _, d := range bucketContextTests { - if d == dockerfile { - contextFlag = "-b" - contextPath = config.gcsBucket - break - } - } - - reproducibleFlag := "" - for _, d := range reproducibleTests { - if d == dockerfile { - reproducibleFlag = "--reproducible" - break - } - } - - // build kaniko image - if singleSnapshotImages[dockerfile] { - buildArgs = append(buildArgs, singleSnapshotFlag) - } - kanikoImage := getKanikoImage(config.imageRepo, dockerfile) - kanikoCmd := exec.Command("docker", - append([]string{"run", - "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", - "-v", cwd + ":/workspace", - executorImage, - "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), - "-d", kanikoImage, reproducibleFlag, - contextFlag, contextPath}, - buildArgs...)..., - ) - - _, err = RunCommandWithoutTest(kanikoCmd) - if err != nil { - return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s", dockerImage, kanikoCmd.Args, err) - } + fmt.Printf("Coudn't create map of dockerfiles: %s", err) + os.Exit(1) } - return nil + imageBuilder = NewDockerFileBuilder(dockerfiles) + os.Exit(m.Run()) } - func TestRun(t *testing.T) { - dockerfiles, err := getDockerFiles(dockerfilesPath) - if err != nil { - t.Fatalf("Couldn't run tests because files couldn't be found: %s", err) - } - for _, dockerfile := range dockerfiles { + for dockerfile, built := range imageBuilder.FilesBuilt { t.Run("test_"+dockerfile, func(t *testing.T) { - if testsToIgnore[dockerfile] { - t.SkipNow() + if !built { + err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile) + if err != nil { + t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err) + } } - kanikoImage := getKanikoImage(config.imageRepo, dockerfile) + dockerImage := GetDockerImage(config.imageRepo, dockerfile) + kanikoImage := GetKanikoImage(config.imageRepo, dockerfile) // container-diff daemonDockerImage := daemonPrefix + dockerImage @@ -296,7 +168,7 @@ func TestRun(t *testing.T) { var diffInt interface{} var expectedInt interface{} - err = json.Unmarshal(diff, &diffInt) + err := json.Unmarshal(diff, &diffInt) if err != nil { t.Error(err) t.Fail() @@ -314,11 +186,6 @@ func TestRun(t *testing.T) { } func TestLayers(t *testing.T) { - dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*")) - if err != nil { - t.Error(err) - t.FailNow() - } offset := map[string]int{ "Dockerfile_test_add": 9, "Dockerfile_test_scratch": 3, @@ -326,18 +193,20 @@ func TestLayers(t *testing.T) { // which is why this offset exists "Dockerfile_test_volume": 1, } - for _, dockerfile := range dockerfiles { + for dockerfile, built := range imageBuilder.FilesBuilt { t.Run("test_layer_"+dockerfile, func(t *testing.T) { - dockerfile = dockerfile[len("dockerfile/")+1:] - if testsToIgnore[dockerfile] { - t.SkipNow() + if !built { + err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile) + if err != nil { + t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err) + } } // Pull the kaniko image - dockerImage := getDockerImage(config.imageRepo, dockerfile) - kanikoImage := getKanikoImage(config.imageRepo, dockerfile) + dockerImage := GetDockerImage(config.imageRepo, dockerfile) + kanikoImage := GetKanikoImage(config.imageRepo, dockerfile) pullCmd := exec.Command("docker", "pull", kanikoImage) RunCommand(pullCmd, t) - if err := checkLayers(t, dockerImage, kanikoImage, offset[dockerfile]); err != nil { + if err := checkLayers(dockerImage, kanikoImage, offset[dockerfile]); err != nil { t.Error(err) t.Fail() } @@ -345,7 +214,7 @@ func TestLayers(t *testing.T) { } } -func checkLayers(t *testing.T, image1, image2 string, offset int) error { +func checkLayers(image1, image2 string, offset int) error { lenImage1, err := numLayers(image1) if err != nil { return fmt.Errorf("Couldn't get number of layers for image1 (%s): %s", image1, err) diff --git a/integration/randomstring.go b/integration/randomstring.go deleted file mode 100644 index 56b7e4ad5e..0000000000 --- a/integration/randomstring.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package integration - -import ( - "math/rand" - "sync" - "time" -) - -// r is used by randomString to generate a random string. It is seeded with the time -// at import so the strings will be different between test runs. - -var r *rand.Rand - -// once is used to initialize r -var once sync.Once - -const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - -func randomString(n int) string { - once.Do(func() { - seed := time.Now().UTC().UnixNano() - r = rand.New(rand.NewSource(seed)) - }) - b := make([]byte, n) - for i := range b { - b[i] = letterBytes[r.Intn(len(letterBytes))] - } - return string(b) -}