Support multi stage builds

cmd/executor/cmd/root.go

@@ -23,9 +23,9 @@ import (
 
 	"github.com/genuinetools/amicontained/container"
 
-	"github.com/GoogleContainerTools/kaniko/pkg/executor"
-
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/executor"
+	"github.com/GoogleContainerTools/kaniko/pkg/image"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -72,7 +72,12 @@ var RootCmd = &cobra.Command{
 			}
 			logrus.Warn("kaniko is being run outside of a container. This can have dangerous effects on your system")
 		}
-		if err := executor.DoBuild(dockerfilePath, srcContext, destination, snapshotMode, dockerInsecureSkipTLSVerify); err != nil {
+		buildImage, err := executor.DoBuild(dockerfilePath, srcContext, snapshotMode)
+		if err != nil {
+			logrus.Error(err)
+			os.Exit(1)
+		}
+		if err := image.PushImage(buildImage, destination, dockerInsecureSkipTLSVerify); err != nil {
 			logrus.Error(err)
 			os.Exit(1)
 		}

integration_tests/dockerfiles/Dockerfile_test_multistage

@@ -0,0 +1,9 @@
+FROM gcr.io/distroless/base:latest
+COPY . .
+
+FROM scratch as second
+ENV foopath context/foo
+COPY --from=0 $foopath context/b* /foo/
+
+FROM gcr.io/distroless/base:latest
+COPY --from=second /foo /foo2

integration_tests/dockerfiles/config_test_multistage.json

@@ -0,0 +1,12 @@
+[
+  {
+    "Image1": "gcr.io/kaniko-test/docker-test-multistage:latest",
+    "Image2": "gcr.io/kaniko-test/kaniko-test-multistage:latest",
+    "DiffType": "File",
+    "Diff": {
+      "Adds": null,
+      "Dels": null,
+      "Mods": null
+    }
+  }
+]

integration_tests/integration_test_yaml.go

@@ -139,6 +139,14 @@ var fileTests = []struct {
 		kanikoContext:  buildcontextPath,
 		repo:           "test-scratch",
 	},
+	{
+		description:    "test multistage",
+		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_multistage",
+		configPath:     "/workspace/integration_tests/dockerfiles/config_test_multistage.json",
+		dockerContext:  buildcontextPath,
+		kanikoContext:  buildcontextPath,
+		repo:           "test-multistage",
+	},
 }
 
 var structureTests = []struct {

pkg/commands/copy.go

@@ -17,6 +17,7 @@ limitations under the License.
 package commands
 
 import (
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/containers/image/manifest"
 	"github.com/docker/docker/builder/dockerfile/instructions"
@@ -39,6 +40,11 @@ func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error {
 	logrus.Infof("cmd: copy %s", srcs)
 	logrus.Infof("dest: %s", dest)
 
+	// Resolve from
+	if c.cmd.From != "" {
+		c.buildcontext = filepath.Join(constants.BuildContextDir, c.cmd.From)
+	}
+
 	// First, resolve any environment replacement
 	resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.cmd.SourcesAndDest, config.Env, true)
 	if err != nil {
@@ -57,11 +63,19 @@ func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error {
 		if err != nil {
 			return err
 		}
-		destPath, err := util.DestinationFilepath(src, dest, config.WorkingDir)
+		cwd := config.WorkingDir
+		if cwd == "" {
+			cwd = constants.RootDir
+		}
+		destPath, err := util.DestinationFilepath(src, dest, cwd)
 		if err != nil {
 			return err
 		}
 		if fi.IsDir() {
+			if !filepath.IsAbs(dest) {
+				// we need to add '/' to the end to indicate the destination is a directory
+				dest = filepath.Join(cwd, dest) + "/"
+			}
 			if err := util.CopyDir(fullPath, dest); err != nil {
 				return err
 			}

pkg/commands/env.go

@@ -29,6 +29,12 @@ type EnvCommand struct {
 	cmd *instructions.EnvCommand
 }
 
+func NewEnvCommand(cmd *instructions.EnvCommand) EnvCommand {
+	return EnvCommand{
+		cmd: cmd,
+	}
+}
+
 func (e *EnvCommand) ExecuteCommand(config *manifest.Schema2Config) error {
 	logrus.Info("cmd: ENV")
 	newEnvs := e.cmd.Env

pkg/dockerfile/dockerfile.go

@@ -18,10 +18,15 @@ package dockerfile
 
 import (
 	"bytes"
-	"strings"
-
+	"github.com/GoogleContainerTools/kaniko/pkg/commands"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/image"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"path/filepath"
+	"strconv"
+	"strings"
 )
 
 // Parse parses the contents of a Dockerfile and returns a list of commands
@@ -37,6 +42,25 @@ func Parse(b []byte) ([]instructions.Stage, error) {
 	return stages, err
 }
 
+// ResolveStages resolves any calls to previous stages with names to indices
+// Ex. --from=second_stage should be --from=1 for easier processing later on
+func ResolveStages(stages []instructions.Stage) {
+	nameToIndex := make(map[string]string)
+	for i, stage := range stages {
+		index := strconv.Itoa(i)
+		nameToIndex[stage.Name] = index
+		nameToIndex[index] = index
+		for _, cmd := range stage.Commands {
+			switch c := cmd.(type) {
+			case *instructions.CopyCommand:
+				if c.From != "" {
+					c.From = nameToIndex[c.From]
+				}
+			}
+		}
+	}
+}
+
 // ParseCommands parses an array of commands into an array of instructions.Command; used for onbuild
 func ParseCommands(cmdArray []string) ([]instructions.Command, error) {
 	var cmds []instructions.Command
@@ -54,3 +78,47 @@ func ParseCommands(cmdArray []string) ([]instructions.Command, error) {
 	}
 	return cmds, nil
 }
+
+// Dependencies returns a list of files in this stage that will be needed in later stages
+func Dependencies(index int, stages []instructions.Stage) ([]string, error) {
+	var dependencies []string
+	for stageIndex, stage := range stages {
+		if stageIndex <= index {
+			continue
+		}
+		ms, err := image.NewSourceImage(stage.BaseName)
+		if err != nil {
+			return nil, err
+		}
+		for _, cmd := range stage.Commands {
+			switch c := cmd.(type) {
+			case *instructions.EnvCommand:
+				envCommand := commands.NewEnvCommand(c)
+				if err := envCommand.ExecuteCommand(ms.Config()); err != nil {
+					return nil, err
+				}
+			case *instructions.CopyCommand:
+				if c.From != strconv.Itoa(index) {
+					continue
+				}
+				// First, resolve any environment replacement
+				resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.SourcesAndDest, ms.Config().Env, true)
+				if err != nil {
+					return nil, err
+				}
+				// Resolve wildcards and get a list of resolved sources
+				srcs, err := util.ResolveSources(resolvedEnvs, constants.RootDir)
+				if err != nil {
+					return nil, err
+				}
+				for index, src := range srcs {
+					if !filepath.IsAbs(src) {
+						srcs[index] = filepath.Join(constants.RootDir, src)
+					}
+				}
+				dependencies = append(dependencies, srcs...)
+			}
+		}
+	}
+	return dependencies, nil
+}

pkg/dockerfile/dockerfile_test.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockerfile
+
+import (
+	"fmt"
+	"github.com/GoogleContainerTools/kaniko/testutil"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+)
+
+var dockerfile = `
+FROM scratch
+RUN echo hi > /hi
+
+FROM scratch AS second
+COPY --from=0 /hi /hi2
+
+FROM scratch
+COPY --from=second /hi2 /hi3
+`
+
+func Test_ResolveStages(t *testing.T) {
+	stages, err := Parse([]byte(dockerfile))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ResolveStages(stages)
+	for index, stage := range stages {
+		if index == 0 {
+			continue
+		}
+		copyCmd := stage.Commands[0].(*instructions.CopyCommand)
+		expectedStage := strconv.Itoa(index - 1)
+		if copyCmd.From != expectedStage {
+			t.Fatalf("unexpected copy command: %s resolved to stage %s, expected %s", copyCmd.String(), copyCmd.From, expectedStage)
+		}
+	}
+}
+
+func Test_Dependencies(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	helloPath := filepath.Join(testDir, "hello")
+	if err := os.Mkdir(helloPath, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	dockerfile := fmt.Sprintf(`
+	FROM scratch
+	COPY %s %s
+
+	FROM scratch AS second
+	ENV hienv %s
+	COPY a b
+	COPY --from=0 /$hienv %s /hi2/
+	`, helloPath, helloPath, helloPath, testDir)
+
+	stages, err := Parse([]byte(dockerfile))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedDependencies := [][]string{
+		{
+			helloPath,
+			testDir,
+		},
+		nil,
+	}
+
+	for index := range stages {
+		actualDeps, err := Dependencies(index, stages)
+		testutil.CheckErrorAndDeepEqual(t, false, err, expectedDependencies[index], actualDeps)
+	}
+}