diff --git a/Gopkg.lock b/Gopkg.lock index 31d93acf2c..d44f11eba6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -399,7 +399,7 @@ "pkg/v1/types", "pkg/v1/v1util" ] - revision = "5e2bd1f4bf61add62944828d54e239d352daaabf" + revision = "3f6471078a9661a9a439bd5e71a371aff429566a" [[projects]] name = "github.com/googleapis/gax-go" diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 7de1ead9fc..f138e7cc5a 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -41,6 +41,7 @@ var ( buildArgs multiArg tarPath string singleSnapshot bool + reproducible bool ) func init() { @@ -56,6 +57,7 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&force, "force", "", false, "Force building outside of a container") RootCmd.PersistentFlags().StringVarP(&tarPath, "tarPath", "", "", "Path to save the image in as a tarball instead of pushing") RootCmd.PersistentFlags().BoolVarP(&singleSnapshot, "single-snapshot", "", false, "Set this flag to take a single snapshot at the end of the build.") + RootCmd.PersistentFlags().BoolVarP(&reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible") } var RootCmd = &cobra.Command{ @@ -87,6 +89,7 @@ var RootCmd = &cobra.Command{ SnapshotMode: snapshotMode, Args: buildArgs, SingleSnapshot: singleSnapshot, + Reproducible: reproducible, }) if err != nil { logrus.Error(err) diff --git a/integration/dockerfiles/Dockerfile_test_copy_reproducible b/integration/dockerfiles/Dockerfile_test_copy_reproducible new file mode 100644 index 0000000000..fd184394d5 --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_copy_reproducible @@ -0,0 +1,20 @@ +FROM alpine:3.7 +COPY context/foo foo +COPY context/foo /foodir/ +COPY context/bar/b* bar/ +COPY context/fo? /foo2 +COPY context/bar/doesnotexist* context/foo hello +COPY ./context/empty /empty +COPY ./ dir/ +COPY . newdir +COPY context/bar /baz/ +COPY ["context/foo", "/tmp/foo" ] +COPY context/b* /baz/ +COPY context/foo context/bar/ba? /test/ +COPY context/arr[[]0].txt /mydir/ +COPY context/bar/bat . + +ENV contextenv ./context +COPY ${contextenv}/foo /tmp/foo2 +COPY $contextenv/foo /tmp/foo3 +COPY $contextenv/* /tmp/${contextenv}/ diff --git a/integration/integration_test.go b/integration/integration_test.go index 41bce735e1..81ee83f7ac 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -119,7 +119,10 @@ func TestRun(t *testing.T) { "Dockerfile_test_scratch": {"--single-snapshot"}, } + // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed + testsToIgnore := []string{"Dockerfile_test_user_run"} bucketContextTests := []string{"Dockerfile_test_copy_bucket"} + reproducibleTests := []string{"Dockerfile_test_env"} _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) @@ -161,6 +164,14 @@ func TestRun(t *testing.T) { } } + reproducibleFlag := "" + for _, d := range reproducibleTests { + if d == dockerfile { + reproducibleFlag = "--reproducible" + break + } + } + // build kaniko image additionalFlags := append(buildArgs, additionalFlagsMap[dockerfile]...) kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile) @@ -170,7 +181,7 @@ func TestRun(t *testing.T) { "-v", cwd + ":/workspace", executorImage, "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), - "-d", kanikoImage, + "-d", kanikoImage, reproducibleFlag, contextFlag, contextPath}, additionalFlags...)..., ) diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 8a79129e68..308e78b68b 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -108,7 +108,7 @@ func Dependencies(index int, stages []instructions.Stage, buildArgs *BuildArgs) if err != nil { return nil, err } - sourceImage, err = remote.Image(ref, auth, http.DefaultTransport) + sourceImage, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport)) if err != nil { return nil, err } diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 85dd7a9d32..dccfffa05c 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -52,6 +52,7 @@ type KanikoBuildArgs struct { SnapshotMode string Args []string SingleSnapshot bool + Reproducible bool } func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) { @@ -94,7 +95,7 @@ func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) { if err != nil { return nil, nil, err } - sourceImage, err = remote.Image(ref, auth, http.DefaultTransport) + sourceImage, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport)) if err != nil { return nil, nil, err } @@ -174,6 +175,14 @@ func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) { if err != nil { return nil, nil, err } + + if k.Reproducible { + sourceImage, err = mutate.Canonical(sourceImage) + if err != nil { + return nil, nil, err + } + } + return ref, sourceImage, nil } if err := saveStageDependencies(index, stages, buildArgs.Clone()); err != nil { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index e557645675..b24d6896b8 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -21,12 +21,17 @@ import ( "errors" "fmt" "io" + "io/ioutil" "path/filepath" "strings" + "time" "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/v1util" ) const whiteoutPrefix = ".wh." @@ -128,22 +133,26 @@ func Append(base v1.Image, adds ...Addendum) (v1.Image, error) { // Config mutates the provided v1.Image to have the provided v1.Config func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { - m, err := base.Manifest() + cf, err := base.ConfigFile() if err != nil { return nil, err } - cf, err := base.ConfigFile() + cf.Config = cfg + + return configFile(base, cf) +} + +func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { + m, err := base.Manifest() if err != nil { return nil, err } - cf.Config = cfg - image := &image{ Image: base, manifest: m.DeepCopy(), - configFile: cf.DeepCopy(), + configFile: cfg, digestMap: make(map[v1.Hash]v1.Layer), } @@ -160,13 +169,8 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { return image, nil } -// Created mutates the provided v1.Image to have the provided v1.Time +// CreatedAt mutates the provided v1.Image to have the provided v1.Time func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) { - m, err := base.Manifest() - if err != nil { - return nil, err - } - cf, err := base.ConfigFile() if err != nil { return nil, err @@ -175,18 +179,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) { cfg := cf.DeepCopy() cfg.Created = created - image := &image{ - Image: base, - manifest: m.DeepCopy(), - configFile: cfg, - digestMap: make(map[v1.Hash]v1.Layer), - } - - image.manifest.Config.Digest, err = image.ConfigName() - if err != nil { - return nil, err - } - return image, nil + return configFile(base, cfg) } type image struct { @@ -392,3 +385,129 @@ func inWhiteoutDir(fileMap map[string]bool, file string) bool { } return false } + +// Time sets all timestamps in an image to the given timestamp. +func Time(img v1.Image, t time.Time) (v1.Image, error) { + newImage := empty.Image + + layers, err := img.Layers() + if err != nil { + + return nil, fmt.Errorf("Error getting image layers: %v", err) + } + + // Strip away all timestamps from layers + var newLayers []v1.Layer + for _, layer := range layers { + newLayer, err := layerTime(layer, t) + if err != nil { + return nil, fmt.Errorf("Error setting layer times: %v", err) + } + newLayers = append(newLayers, newLayer) + } + + newImage, err = AppendLayers(newImage, newLayers...) + if err != nil { + return nil, fmt.Errorf("Error appending layers: %v", err) + } + + ocf, err := img.ConfigFile() + if err != nil { + return nil, fmt.Errorf("Error getting original config file: %v", err) + } + + cf, err := newImage.ConfigFile() + if err != nil { + return nil, fmt.Errorf("Error setting config file: %v", err) + } + + cfg := cf.DeepCopy() + + // Copy basic config over + cfg.Config = ocf.Config + cfg.ContainerConfig = ocf.ContainerConfig + + // Strip away timestamps from the config file + cfg.Created = v1.Time{Time: t} + + for _, h := range cfg.History { + h.Created = v1.Time{Time: t} + } + + return configFile(newImage, cfg) +} + +func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { + layerReader, err := layer.Uncompressed() + if err != nil { + return nil, fmt.Errorf("Error getting layer: %v", err) + } + w := new(bytes.Buffer) + tarWriter := tar.NewWriter(w) + defer tarWriter.Close() + + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("Error reading layer: %v", err) + } + + header.ModTime = t + if err := tarWriter.WriteHeader(header); err != nil { + return nil, fmt.Errorf("Error writing tar header: %v", err) + } + + if header.Typeflag == tar.TypeReg { + if _, err = io.Copy(tarWriter, tarReader); err != nil { + return nil, fmt.Errorf("Error writing layer file: %v", err) + } + } + } + + b := w.Bytes() + // gzip the contents, then create the layer + opener := func() (io.ReadCloser, error) { + g, err := v1util.GzipReadCloser(ioutil.NopCloser(bytes.NewReader(b))) + if err != nil { + return nil, fmt.Errorf("Error compressing layer: %v", err) + } + + return g, nil + } + layer, err = tarball.LayerFromOpener(opener) + if err != nil { + return nil, fmt.Errorf("Error creating layer: %v", err) + } + + return layer, nil +} + +// Canonical is a helper function to combine Time and configFile +// to remove any randomness during a docker build. +func Canonical(img v1.Image) (v1.Image, error) { + // Set all timestamps to 0 + created := time.Time{} + img, err := Time(img, created) + if err != nil { + return nil, err + } + + cf, err := img.ConfigFile() + if err != nil { + return nil, err + } + + // Get rid of host-dependent random config + cfg := cf.DeepCopy() + + cfg.Container = "" + cfg.Config.Hostname = "" + cfg.ContainerConfig.Hostname = "" + cfg.DockerVersion = "" + + return configFile(img, cfg) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index ab1764cd22..c532e46b4d 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -42,30 +42,55 @@ type remoteImage struct { config []byte } +type ImageOption func(*imageOpener) error + var _ partial.CompressedImageCore = (*remoteImage)(nil) -// Image accesses a given image reference over the provided transport, with the provided authentication. -func Image(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) (v1.Image, error) { - scopes := []string{ref.Scope(transport.PullScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) +type imageOpener struct { + auth authn.Authenticator + transport http.RoundTripper + ref name.Reference + client *http.Client +} + +func (i *imageOpener) Open() (v1.Image, error) { + tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) if err != nil { return nil, err } - img, err := partial.CompressedToImage(&remoteImage{ - ref: ref, + ri := &remoteImage{ + ref: i.ref, client: &http.Client{Transport: tr}, - }) + } + imgCore, err := partial.CompressedToImage(ri) if err != nil { - return nil, err + return imgCore, err } // Wrap the v1.Layers returned by this v1.Image in a hint for downstream // remote.Write calls to facilitate cross-repo "mounting". return &mountableImage{ - Image: img, - Repository: ref.Context(), + Image: imgCore, + Reference: i.ref, }, nil } +// Image provides access to a remote image reference, applying functional options +// to the underlying imageOpener before resolving the reference into a v1.Image. +func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { + img := &imageOpener{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + ref: ref, + } + + for _, option := range options { + if err := option(img); err != nil { + return nil, err + } + } + return img.Open() +} + func (r *remoteImage) url(resource, identifier string) url.URL { return url.URL{ Scheme: transport.Scheme(r.ref.Context().Registry), diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/mount.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/mount.go index fb65b34da0..13b79064da 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/mount.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/mount.go @@ -24,7 +24,7 @@ import ( type MountableLayer struct { v1.Layer - Repository name.Repository + Reference name.Reference } // mountableImage wraps the v1.Layer references returned by the embedded v1.Image @@ -33,7 +33,7 @@ type MountableLayer struct { type mountableImage struct { v1.Image - Repository name.Repository + Reference name.Reference } // Layers implements v1.Image @@ -45,8 +45,8 @@ func (mi *mountableImage) Layers() ([]v1.Layer, error) { mls := make([]v1.Layer, 0, len(ls)) for _, l := range ls { mls = append(mls, &MountableLayer{ - Layer: l, - Repository: mi.Repository, + Layer: l, + Reference: mi.Reference, }) } return mls, nil @@ -59,8 +59,8 @@ func (mi *mountableImage) LayerByDigest(d v1.Hash) (v1.Layer, error) { return nil, err } return &MountableLayer{ - Layer: l, - Repository: mi.Repository, + Layer: l, + Reference: mi.Reference, }, nil } @@ -71,7 +71,7 @@ func (mi *mountableImage) LayerByDiffID(d v1.Hash) (v1.Layer, error) { return nil, err } return &MountableLayer{ - Layer: l, - Repository: mi.Repository, + Layer: l, + Reference: mi.Reference, }, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go new file mode 100644 index 0000000000..7299055acb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -0,0 +1,61 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// WithTransport is a functional option for overriding the default transport +// on a remote image +func WithTransport(t http.RoundTripper) ImageOption { + return func(i *imageOpener) error { + return i.setTransport(t) + } +} + +// WithAuth is a functional option for overriding the default authenticator +// on a remote image +func WithAuth(auth authn.Authenticator) ImageOption { + return func(i *imageOpener) error { + return i.setAuth(auth) + } +} + +// WithAuthFromKeychain is a functional option for overriding the default +// authenticator on a remote image using an authn.Keychain +func WithAuthFromKeychain(keys authn.Keychain) ImageOption { + return func(i *imageOpener) error { + auth, err := keys.Resolve(i.ref.Context().Registry) + if err != nil { + return err + } + return i.setAuth(auth) + } +} + +// Set client on image using provided transport, and the default authenticator +func (i *imageOpener) setTransport(t http.RoundTripper) error { + i.transport = t + return nil +} + +// Set client on image using provided authenticator, and the default transport +func (i *imageOpener) setAuth(auth authn.Authenticator) error { + i.auth = auth + return nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 283f817c85..57e4a1fad5 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -45,7 +45,7 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro scopes := []string{ref.Scope(transport.PushScope)} for _, l := range ls { if ml, ok := l.(*MountableLayer); ok { - scopes = append(scopes, ml.Repository.Scope(transport.PullScope)) + scopes = append(scopes, ml.Reference.Context().Scope(transport.PullScope)) } } @@ -145,7 +145,7 @@ func (w *writer) initiateUpload(h v1.Hash) (location string, mounted bool, err e // if "mount" is specified, even if no "from" sources are specified. If this turns out // to not be broadly applicable then we should replace mounts without "from"s with a HEAD. if ml, ok := l.(*MountableLayer); ok { - uv["from"] = []string{ml.Repository.RepositoryStr()} + uv["from"] = []string{ml.Reference.Context().RepositoryStr()} } u.RawQuery = uv.Encode()