From 3da57f1a11a71309a116f4d4e238607c069abffb Mon Sep 17 00:00:00 2001 From: Glyn Normington Date: Tue, 9 Apr 2019 13:36:29 +0100 Subject: [PATCH] Image relocation See `duffle relocate -h` for documentation. Adds the originalImage field to images in both the image map and invocation images. Fixes https://github.com/deislabs/duffle/issues/668 --- Gopkg.lock | 108 +++++++---- Gopkg.toml | 4 + cmd/duffle/build.go | 25 ++- cmd/duffle/relocate.go | 233 +++++++++++++++++++++++ cmd/duffle/relocate_test.go | 183 ++++++++++++++++++ cmd/duffle/root.go | 1 + cmd/duffle/testdata/relocate/bundle.json | 30 +++ 7 files changed, 540 insertions(+), 44 deletions(-) create mode 100644 cmd/duffle/relocate.go create mode 100644 cmd/duffle/relocate_test.go create mode 100644 cmd/duffle/testdata/relocate/bundle.json diff --git a/Gopkg.lock b/Gopkg.lock index a8607f44..9d026377 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -21,7 +21,7 @@ version = "v1.4.2" [[projects]] - digest = "1:2be791e7b333ff7c06f8fb3dc18a7d70580e9399dbdffd352621d067ff260b6e" + digest = "1:26b14a6dc72ace253599e969997d5ecf2143c63833c015179786bc756c76eaa4" name = "github.com/Microsoft/go-winio" packages = ["."] pruneopts = "NUT" @@ -29,7 +29,7 @@ version = "v0.4.11" [[projects]] - digest = "1:22efafa7722ed855e8bab8039d2ee6b121971c2945804ec759f5ff1b1681ad08" + digest = "1:51c7734aae4502a76644c6eed2a4b68f98406e263468789f26512810095a43fa" name = "github.com/Microsoft/hcsshim" packages = [ ".", @@ -80,7 +80,7 @@ revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] - digest = "1:c41ccacec3d1871a9a76c0a105cb4e50bd7d6489d91f0262e37ab66c3d79cc6a" + digest = "1:aaf95e919b44127f05cc1d3159f1b7b29d2cfdec597fa32dcfd0de7382031146" name = "github.com/containerd/containerd" packages = [ ".", @@ -137,7 +137,7 @@ [[projects]] branch = "master" - digest = "1:91f1813e0a9661e295671cd1004ace6ed4ffa9cb68d9d0c798badbd5ce6354ef" + digest = "1:4a940afce0fedb15226cd903bdd7207cb66a5c3b120a143ce309a2833bc09568" name = "github.com/containerd/continuity" packages = [ "fs", @@ -149,7 +149,7 @@ revision = "bd77b46c8352f74eb12c85bdc01f4b90f69d66b4" [[projects]] - digest = "1:a72272dfefbb22c87cda3dbefbb256741d861d2fb727210a66e0df0df6084b30" + digest = "1:393d96d76afa9703cc64a43ca2b27c1e32a9be64fed9546a120a5e188b53a193" name = "github.com/containerd/cri" packages = ["pkg/util"] pruneopts = "NUT" @@ -182,14 +182,14 @@ [[projects]] branch = "master" - digest = "1:f5537515bc64edb9117ec396d197d88852d927185352ce08e2667562ef532a96" + digest = "1:8cd271f26db91626bd5993ed540069dbade9f6149134e7f8a0ed951cbbf4f2ec" name = "github.com/deislabs/cnab-go" packages = ["bundle"] pruneopts = "NUT" - revision = "1a73873b84025123a0f8fe84086e59493e051302" + revision = "7cd944a8ee450ee465795d21ad7302bed34ad1b0" [[projects]] - digest = "1:e71d64468873ca819b91b975e67e71f349a89f41ab38040bcbd8840f5e193654" + digest = "1:43d1a6ccaa7b9e319dbff50d89e744db06ae63af20aae1d0f4fb743fa525ce97" name = "github.com/docker/cli" packages = [ "cli", @@ -221,7 +221,7 @@ revision = "f95ca8e1ba6c22c9abcdbf65e8dcc39c53958bba" [[projects]] - digest = "1:a828cce5747a0ac32a41fd9644660a815cbc3c61caa85c03b0a651d62320aa0b" + digest = "1:54b22de865db36f1ffaf2bb4de6cc6c39148cae03c13c58758f2af5a74466d4f" name = "github.com/docker/compose-on-kubernetes" packages = [ "api", @@ -235,7 +235,7 @@ version = "v0.4.18" [[projects]] - digest = "1:c2fd3505322eed56c220992927a13029d32b7fea0e9cc1ece7a2217369d76914" + digest = "1:79f22d913ffae103c0ee94bb2ea0c6ddd52239b442501a935be3da935f44a237" name = "github.com/docker/distribution" packages = [ ".", @@ -259,7 +259,7 @@ revision = "83389a148052d74ac602f5f1d62f86ff2f3c4aa5" [[projects]] - digest = "1:6e8fe686e68797b78d54d112f816448cf5d4bbf33c24c2cd12da0508ec600dc8" + digest = "1:ac3d0158a1eb7e6508eac6ce7ff6fba0cadde0f48b5a17949dd2e221c94a2fd6" name = "github.com/docker/docker" packages = [ "api", @@ -308,7 +308,7 @@ revision = "f76d6a078d881f410c00e8d900dcdfc2e026c841" [[projects]] - digest = "1:8866486038791fe65ea1abf660041423954b1f3fb99ea6a0ad8424422e943458" + digest = "1:15e7be8bdfcc1ea804cedc8e3187605c58642cddd1d333781b888e4873c1ed5b" name = "github.com/docker/docker-credential-helpers" packages = [ "client", @@ -327,7 +327,7 @@ version = "v1.5.1-1" [[projects]] - digest = "1:2a47f7eb1a2c30428d1ee6808cb66d4deb17e68a3e55d696f03c8068552ba5e8" + digest = "1:d131afebb270bf014e49fcdff0da3e861124976b7c1402181fac5196a57f3b31" name = "github.com/docker/go-connections" packages = [ "nat", @@ -386,7 +386,7 @@ version = "v1.1.0" [[projects]] - digest = "1:fd13acfe94ed604682ce01528a1ae0399391e64054143a8a0a98399c78696679" + digest = "1:7666507274154769bb45fbe0db36309e026f88ca08377aa28c201ee4c37ae5b2" name = "github.com/gogo/protobuf" packages = [ "proto", @@ -406,7 +406,7 @@ revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] - digest = "1:63ccdfbd20f7ccd2399d0647a7d100b122f79c13bb83da9660b1598396fd9f62" + digest = "1:094772cd3a2a5b9bb621111bfbe964a2fbee790b014a645ea09eb842b1b658c1" name = "github.com/golang/protobuf" packages = [ "proto", @@ -427,6 +427,26 @@ pruneopts = "NUT" revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306" +[[projects]] + branch = "master" + digest = "1:e56e3dbbe1b106b456ef279639d809c03cace64dc015dbb0646cd27fc1cb2cb0" + name = "github.com/google/go-containerregistry" + packages = [ + "pkg/authn", + "pkg/name", + "pkg/v1", + "pkg/v1/empty", + "pkg/v1/layout", + "pkg/v1/partial", + "pkg/v1/random", + "pkg/v1/remote", + "pkg/v1/remote/transport", + "pkg/v1/types", + "pkg/v1/v1util", + ] + pruneopts = "NUT" + revision = "f1df91a4a813cbc183527dc7b9a31ea6454557b5" + [[projects]] branch = "master" digest = "1:52c5834e2bebac9030c97cc0798ac11c3aa8a39f098aeb419f142533da6cd3cc" @@ -436,7 +456,7 @@ revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" [[projects]] - digest = "1:06a7dadb7b760767341ffb6c8d377238d68a1226f2b21b5d497d2e3f6ecf6b4e" + digest = "1:add738701bd5b2b985c0c37011092c57218bdc46caf1e682a73dc210ad36b03f" name = "github.com/googleapis/gnostic" packages = [ "OpenAPIv2", @@ -465,7 +485,7 @@ [[projects]] branch = "master" - digest = "1:2d30806677673203e02f0e36069be246e5fb819337f3e2200a322c1daa66fae3" + digest = "1:42df938c96e4a54d2c19da35d6d3f3a6c432dd942a7375343a6bf06e1945ca5b" name = "github.com/gosuri/uitable" packages = [ ".", @@ -477,7 +497,7 @@ [[projects]] branch = "master" - digest = "1:7fdf3223c7372d1ced0b98bf53457c5e89d89aecbad9a77ba9fcc6e01f9e5621" + digest = "1:a1db0214936912602a7a8cedc09a2e3211f5c097dc89189fb4b3bc86346c9e89" name = "github.com/gregjones/httpcache" packages = [ ".", @@ -495,7 +515,7 @@ version = "v1.1.0" [[projects]] - digest = "1:11c6c696067d3127ecf332b10f89394d386d9083f82baf71f40f2da31841a009" + digest = "1:02e4365951e1bc55bd6505938ea88e12b5a9d5dfedcb1ae35d5a3833e502833e" name = "github.com/hashicorp/hcl" packages = [ ".", @@ -650,7 +670,7 @@ version = "v1.0.0-rc1" [[projects]] - digest = "1:70711188c19c53147099d106169d6a81941ed5c2658651432de564a7d60fd288" + digest = "1:be059bd8249ad99021a870bf6ecd3da2e72107eeb1891e2ef2a3d350b1e00b35" name = "github.com/opencontainers/image-spec" packages = [ "identity", @@ -662,7 +682,7 @@ version = "v1.0.1" [[projects]] - digest = "1:ea8780346eba74e5eb3e8b6bded2ec46c1c81cd745bc00e9964b730955bf4252" + digest = "1:c5a5a1953c1f70bab6e3e82880af4f8752aeace36a37612e4b718403e43ace1b" name = "github.com/opencontainers/runc" packages = [ "libcontainer/system", @@ -704,6 +724,18 @@ revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" version = "v2.0.1" +[[projects]] + branch = "master" + digest = "1:1c5a18736f0071a05ff992f89516ba037020643c932ed5c4ed144f63dcc02ab0" + name = "github.com/pivotal/image-relocation" + packages = [ + "pkg/image", + "pkg/pathmapping", + "pkg/registry", + ] + pruneopts = "NUT" + revision = "532dd0b42e7a50010d7868364309cd314a2bb376" + [[projects]] digest = "1:5cf3f025cbee5951a4ee961de067c8a89fc95a5adabead774f82822efabab121" name = "github.com/pkg/errors" @@ -730,7 +762,7 @@ [[projects]] branch = "master" - digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" + digest = "1:0f37e09b3e92aaeda5991581311f8dbf38944b36a3edec61cc2d1991f527554a" name = "github.com/prometheus/client_model" packages = ["go"] pruneopts = "NUT" @@ -738,7 +770,7 @@ [[projects]] branch = "master" - digest = "1:fad5a35eea6a1a33d6c8f949fbc146f24275ca809ece854248187683f52cc30b" + digest = "1:edddf4ceb3a45778dfbb9364f128fffd92ea9bd6abd70edf1474f56565832682" name = "github.com/prometheus/common" packages = [ "expfmt", @@ -750,7 +782,7 @@ [[projects]] branch = "master" - digest = "1:26a2f5e891cc4d2321f18a0caa84c8e788663c17bed6a487f3cbe2c4295292d0" + digest = "1:c0acd6d63adc573089863ec85a5d23f2a86eb70ebdfd756621c39f3e682b9c2e" name = "github.com/prometheus/procfs" packages = [ ".", @@ -770,7 +802,7 @@ version = "v1.1.0" [[projects]] - digest = "1:330e9062b308ac597e28485699c02223bd052437a6eed32a173c9227dcb9d95a" + digest = "1:a36d61943d51cd4a1d7ecaf6993190527535a57382114ebf6549956b3e4cb612" name = "github.com/spf13/afero" packages = [ ".", @@ -789,7 +821,7 @@ version = "v1.2.0" [[projects]] - digest = "1:343d44e06621142ab09ae0c76c1799104cdfddd3ffb445d78b1adf8dc3ffaf3d" + digest = "1:2c3b60fc961b7ddca4336bb7bb39146cb73ea2ad73d4afc6b4ffea05571e712a" name = "github.com/spf13/cobra" packages = ["."] pruneopts = "NUT" @@ -845,7 +877,7 @@ version = "0.2.0" [[projects]] - digest = "1:5657d02714d03999f98887942ca12e7d0492676d4485966a5a7819b84887790f" + digest = "1:91e0f8f0d700349de5a6a77910e37bb54002c77d35850930f766ec9a8ac335bb" name = "github.com/theupdateframework/notary" packages = [ ".", @@ -869,7 +901,7 @@ [[projects]] branch = "master" - digest = "1:cb77e5934866333fa0784326a57e64c4da128001c94fbd1d29819d79bd3b1087" + digest = "1:53078c9e741e588d7cbf480c07c7b5099b3099a607f6c4b5ba3ac7821af9a267" name = "golang.org/x/crypto" packages = [ "pbkdf2", @@ -880,7 +912,7 @@ [[projects]] branch = "master" - digest = "1:5ffb087f0be0bc3afdfe8d98692d63c65b424b34ff328d36e38e1fdd052c22b1" + digest = "1:7b5d2581000653d028cc75ed40863c28f2d030f16babd80776d7b48b526b4a1a" name = "golang.org/x/net" packages = [ "context", @@ -907,7 +939,7 @@ [[projects]] branch = "master" - digest = "1:4ea08b5c4c51887a54caee2954db97b620736ae510bdc8514f691d3a9375d93a" + digest = "1:5ffbacb0c244e2f37080002d0154e28024f1c9350f74d788a70ab741dd858a1f" name = "golang.org/x/sys" packages = [ "unix", @@ -917,7 +949,7 @@ revision = "e4b3c5e9061176387e7cea65e4dc5853801f3fb7" [[projects]] - digest = "1:e7071ed636b5422cc51c0e3a6cebc229d6c9fffc528814b519a980641422d619" + digest = "1:a0f29009397dc27c9dc8440f0945d49e5cbb9b72d0b0fc745474d9bfdea2d9f8" name = "golang.org/x/text" packages = [ "collate", @@ -956,7 +988,7 @@ revision = "aa24cbd621fe1e2867110ea8c673d99a1a92efdc" [[projects]] - digest = "1:4e459e7e437c4ad1ae163acec3d1359bfc88d22142764ca5893d8d380e79988d" + digest = "1:3a3d0f6abdb6ec808baccd7b5ac3b40a6d5eb94f8294ff18a4f50c7ae2c2abc9" name = "google.golang.org/grpc" packages = [ ".", @@ -997,7 +1029,7 @@ version = "v1.18.0" [[projects]] - digest = "1:3f36ff57a1033b3e4eb2c03c311b23c9da6890fd94ddd56dc05c57b4214eb782" + digest = "1:5c8fd19f63d241febadd44d4b060faecc2cff31a41e55dd8b719312eabb23b1a" name = "gopkg.in/AlecAivazis/survey.v1" packages = [ ".", @@ -1026,7 +1058,7 @@ [[projects]] branch = "release-1.11" - digest = "1:b1c6723e934087c2fa159e1c6a309c3c5c0b9a7d209c2ba6028f21240ebe7606" + digest = "1:be4113d6193256a33770227e4dd9866cef2a2dc47d9be015783fd9898fe946b3" name = "k8s.io/api" packages = [ "admissionregistration/v1alpha1", @@ -1063,7 +1095,7 @@ revision = "912cbe2bfef3d832db29f9d3125307fe907102e9" [[projects]] - digest = "1:9c366639c38f347bf6a05e18b943a39fb6030a10d58c3686b518248660551a6f" + digest = "1:ca8ff78a26b50d456fa30a4ebe5791ad7ae6b4849114d118093c50faf9b61b4e" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -1106,7 +1138,7 @@ revision = "kubernetes-1.11.5" [[projects]] - digest = "1:276fd1cd7407399ad7852ba5fa66fdbce1eb3c26bd3cdbe5cb31d52cbd94bcb5" + digest = "1:3410feb3ae6a78ca0e6a0e97e6e9687108b6c3fd74e41e371d4700d40f51c58a" name = "k8s.io/client-go" packages = [ "discovery", @@ -1204,6 +1236,9 @@ "github.com/gosuri/uitable", "github.com/oklog/ulid", "github.com/opencontainers/go-digest", + "github.com/pivotal/image-relocation/pkg/image", + "github.com/pivotal/image-relocation/pkg/pathmapping", + "github.com/pivotal/image-relocation/pkg/registry", "github.com/pkg/errors", "github.com/sirupsen/logrus", "github.com/spf13/cobra", @@ -1214,6 +1249,7 @@ "golang.org/x/net/context", "gopkg.in/AlecAivazis/survey.v1", "gopkg.in/yaml.v2", + "k8s.io/apimachinery/pkg/util/validation", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 93480b9a..1c18036a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -49,3 +49,7 @@ [[constraint]] name = "github.com/docker/go" version = "1.5.1-1" + +[[constraint]] + branch = "master" + name = "github.com/pivotal/image-relocation" diff --git a/cmd/duffle/build.go b/cmd/duffle/build.go index b7373973..79074c9d 100644 --- a/cmd/duffle/build.go +++ b/cmd/duffle/build.go @@ -149,15 +149,9 @@ func (b *buildCmd) run() (err error) { } func (b *buildCmd) writeBundle(bf *bundle.Bundle) (string, error) { - data, err := json.MarshalCanonical(bf) + data, digest, err := marshalBundle(bf) if err != nil { - return "", err - } - data = append(data, '\n') //TODO: why? - - digest, err := digest.OfBuffer(data) - if err != nil { - return "", fmt.Errorf("cannot compute digest from bundle: %v", err) + return "", fmt.Errorf("cannot marshal bundle: %v", err) } if b.outputFile != "" { @@ -169,6 +163,21 @@ func (b *buildCmd) writeBundle(bf *bundle.Bundle) (string, error) { return digest, ioutil.WriteFile(filepath.Join(b.home.Bundles(), digest), data, 0644) } +func marshalBundle(bf *bundle.Bundle) ([]byte, string, error) { + data, err := json.MarshalCanonical(bf) + if err != nil { + return nil, "", err + } + data = append(data, '\n') //TODO: why? + + digest, err := digest.OfBuffer(data) + if err != nil { + return nil, "", fmt.Errorf("cannot compute digest from bundle: %v", err) + } + + return data, digest, nil +} + func defaultDockerTLS() bool { return os.Getenv(dockerTLSEnvVar) != "" } diff --git a/cmd/duffle/relocate.go b/cmd/duffle/relocate.go new file mode 100644 index 00000000..a6260d75 --- /dev/null +++ b/cmd/duffle/relocate.go @@ -0,0 +1,233 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + + "github.com/pivotal/image-relocation/pkg/image" + "github.com/pivotal/image-relocation/pkg/pathmapping" + "github.com/pivotal/image-relocation/pkg/registry" + + "github.com/deislabs/cnab-go/bundle" + + "github.com/deislabs/duffle/pkg/duffle/home" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + relocateDesc = ` +Relocates any docker and oci images referenced by a bundle, tags and pushes the images to a registry, and creates a new +bundle with an updated image map. + +The --repository-prefix flag determines the repositories for the relocated images. +Each image is tagged with a name starting with the given prefix and pushed to the repository. + +For example, if the repository-prefix is example.com/user, the image istio/proxyv2 is relocated +to a name starting with example.com/user/ and pushed to a repository hosted by example.com. +` + invalidRepositoryChars = ":@\" " +) + +type relocateCmd struct { + // args + inputBundle string + outputBundle string + + // flags + repoPrefix string + inputBundleIsFile bool + outputBundleIsFile bool + + // context + home home.Home + out io.Writer + + // dependencies + mapping pathmapping.PathMapping + registryClient registry.Client +} + +func newRelocateCmd(w io.Writer) *cobra.Command { + relocate := &relocateCmd{out: w} + + cmd := &cobra.Command{ + Use: "relocate [INPUT-BUNDLE] [OUTPUT-BUNDLE]", + Short: "relocate images in a CNAB bundle", + Long: relocateDesc, + Example: `duffle relocate helloworld hellorelocated --repository-prefix example.com/user +duffle relocate path/to/bundle.json relocatedbundle --repository-prefix example.com/user --input-bundle-is-file +duffle relocate helloworld path/to/relocatedbundle.json --repository-prefix example.com/user --output-bundle-is-file`, + Args: cobra.ExactArgs(2), + PreRunE: func(cmd *cobra.Command, args []string) error { + // validate --repository-prefix if it is set, otherwise fall through so that cobra will report the missing flag in its usual manner + if cmd.Flags().Changed("repository-prefix") { + if err := validateRepository(relocate.repoPrefix); err != nil { + return err + } + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + relocate.inputBundle = args[0] + relocate.outputBundle = args[1] + + relocate.home = home.Home(homePath()) + + relocate.mapping = pathmapping.FlattenRepoPathPreserveTagDigest + relocate.registryClient = registry.NewRegistryClient() + + return relocate.run() + }, + } + + f := cmd.Flags() + f.BoolVarP(&relocate.inputBundleIsFile, "input-bundle-is-file", "", false, "Indicates that the input bundle source is a file path") + f.BoolVarP(&relocate.outputBundleIsFile, "output-bundle-is-file", "", false, "Indicates that the output bundle destination is a file path") + f.StringVarP(&relocate.repoPrefix, "repository-prefix", "r", "", "Prefix for relocated image names") + cmd.MarkFlagRequired("repository-prefix") + + return cmd +} + +func (r *relocateCmd) run() error { + bun, err := r.setup() + if err != nil { + return err + } + + if err := r.relocate(bun); err != nil { + return err + } + + return r.writeBundle(bun) +} + +func (r *relocateCmd) relocate(bun *bundle.Bundle) error { + // mutate the input bundle to become the output bundle + if !r.outputBundleIsFile { + bun.Name = r.outputBundle + } + + for k := range bun.Images { + im := bun.Images[k] + if isOCI(im.ImageType) || isDocker(im.ImageType) { + // map the image name + n, err := image.NewName(im.Image) + if err != nil { + return err + } + rn := r.mapping(r.repoPrefix, n) + + // tag/push the image to its new repository + dig, err := r.registryClient.Copy(n, rn) + if err != nil { + return err + } + if dig.String() != im.Digest { + // should not happen + return fmt.Errorf("digest of image %s not preserved: old digest %s; new digest %s", im.Image, im.Digest, dig.String()) + } + + // update the imagemap + im.OriginalImage = im.Image + im.Image = rn.String() + bun.Images[k] = im + } + } + + return nil +} + +func isOCI(imageType string) bool { + return imageType == "" || imageType == "oci" +} + +func isDocker(imageType string) bool { + return imageType == "docker" +} + +func (r *relocateCmd) setup() (*bundle.Bundle, error) { + bundleFile, err := resolveBundleFilePath(r.inputBundle, r.home.String(), r.inputBundleIsFile) + if err != nil { + return nil, err + } + + bun, err := loadBundle(bundleFile) + if err != nil { + return nil, err + } + + if err = bun.Validate(); err != nil { + return nil, err + } + + return bun, nil +} + +func (r *relocateCmd) writeBundle(bf *bundle.Bundle) error { + data, digest, err := marshalBundle(bf) + if err != nil { + return fmt.Errorf("cannot marshal bundle: %v", err) + } + + if r.outputBundleIsFile { + if err := ioutil.WriteFile(r.outputBundle, data, 0644); err != nil { + return fmt.Errorf("cannot write bundle to %s: %v", r.outputBundle, err) + } + return nil + } + + if err := ioutil.WriteFile(filepath.Join(r.home.Bundles(), digest), data, 0644); err != nil { + return fmt.Errorf("cannot store bundle : %v", err) + + } + + // record the new bundle in repositories.json + if err := recordBundleReference(r.home, bf.Name, bf.Version, digest); err != nil { + return fmt.Errorf("cannot record bundle: %v", err) + } + + return nil +} + +func validateRepository(repo string) error { + if strings.HasSuffix(repo, "/") || strings.Contains(repo, "//") { + return fmt.Errorf("invalid repository: trailing '/' and '//' not allowed: %s", repo) + } + + for i, part := range strings.Split(repo, "/") { + if i != 0 { + if strings.ContainsAny(part, invalidRepositoryChars) { + return fmt.Errorf("invalid repository: characters '%s' not allowed: %s", invalidRepositoryChars, repo) + } + continue + } + + authorityParts := strings.Split(part, ":") + if len(authorityParts) > 2 { + return fmt.Errorf("invalid repository hostname: %s", part) + } + if errs := validation.IsDNS1123Subdomain(authorityParts[0]); len(errs) > 0 { + return fmt.Errorf("invalid repository hostname: %s", strings.Join(errs, "; ")) + } + if len(authorityParts) == 2 { + portNumber, err := strconv.Atoi(authorityParts[1]) + if err != nil { + return fmt.Errorf("invalid repository port number: %s", authorityParts[1]) + } + + if errs := validation.IsValidPortNum(portNumber); len(errs) > 0 { + return fmt.Errorf("invalid repository port number: %s", strings.Join(errs, "; ")) + } + } + } + + return nil +} diff --git a/cmd/duffle/relocate_test.go b/cmd/duffle/relocate_test.go new file mode 100644 index 00000000..1792f186 --- /dev/null +++ b/cmd/duffle/relocate_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "testing" + + "github.com/pivotal/image-relocation/pkg/image" + "github.com/pivotal/image-relocation/pkg/registry" + + "github.com/deislabs/duffle/pkg/duffle/home" +) + +const ( + testRepositoryPrefix = "example.com/user" + + originalImageNameA = "deislabs/duffle@sha256:4d41eeb38fb14266b7c0461ef1ef0b2f8c05f41cd544987a259a9d92cdad2540" + relocatedImageNameA = "example.com/user/deislabs/duffle/relocated@sha256:4d41eeb38fb14266b7c0461ef1ef0b2f8c05f41cd544987a259a9d92cdad2540" + imageDigestA = "sha256:4d41eeb38fb14266b7c0461ef1ef0b2f8c05f41cd544987a259a9d92cdad2540" + + originalImageNameB = "deislabs/duffle:0.1.0-ralpha.5-englishrose" + relocatedImageNameB = "example.com/user/deislabs/duffle/relocated:0.1.0-ralpha.5-englishrose" + originalImageDigestB = "sha256:14d6134d892aeccb7e142557fe746ccd0a8f736a747c195ef04c9f3f0f0bbd49" + relocatedImageDigestB = "sha256:deadbeef892aeccb7e142557fe746ccd0a8f736a747c195ef04c9f3f0f0bbd49" +) + +func TestRelocateFileToFilePreservingDigests(t *testing.T) { + relocateFileToFile(t, true, nil) +} + +func TestRelocateFileToFileMutatingDigests(t *testing.T) { + relocateFileToFile(t, false, fmt.Errorf("digest of image %s not preserved: old digest %s; new digest %s", originalImageNameB, originalImageDigestB, relocatedImageDigestB)) +} + +func relocateFileToFile(t *testing.T, preserveDigest bool, expectedErr error) { + + duffleHome, err := ioutil.TempDir("", "dufflehome") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(duffleHome) + + work, err := ioutil.TempDir("", "relocatetest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(work) + + outputBundle := filepath.Join(work, "relocated.json") + + cmd := &relocateCmd{ + inputBundle: "testdata/relocate/bundle.json", + outputBundle: outputBundle, + + repoPrefix: testRepositoryPrefix, + inputBundleIsFile: true, + outputBundleIsFile: true, + + home: home.Home(duffleHome), + out: ioutil.Discard, + + mapping: func(repoPrefix string, originalImage image.Name) image.Name { + if repoPrefix != testRepositoryPrefix { + t.Fatalf("Unexpected repository prefix %s", repoPrefix) + } + return testMapping(originalImage, t) + }, + registryClient: &mockRegClient{ + copyStub: func(source image.Name, target image.Name) (image.Digest, error) { + oinA, err := image.NewName(originalImageNameA) + if err != nil { + t.Fatal(err) + } + oinB, err := image.NewName(originalImageNameB) + if err != nil { + t.Fatal(err) + } + switch source { + case oinA: + if target.String() == relocatedImageNameA { + return image.NewDigest(imageDigestA) + } + case oinB: + if target.String() == relocatedImageNameB { + if preserveDigest { + return image.NewDigest(originalImageDigestB) + } + // check behaviour if digest is modified, even though this is not normally expected + return image.NewDigest(relocatedImageDigestB) + } + default: + t.Fatalf("unexpected source %v", source) + } + t.Fatalf("unexpected mapping from %v to %v", source, target) + return image.EmptyDigest, nil // unreachable + }, + }, + } + + if err := cmd.run(); (err != nil && expectedErr != nil && err.Error() != expectedErr.Error()) || + ((err == nil || expectedErr == nil) && err != expectedErr) { + t.Fatalf("unexpected error %v (expected %v)", err, expectedErr) + } + + if expectedErr != nil { + return + } + + // check output bundle + bundleFile, err := resolveBundleFilePath(outputBundle, "", true) + if err != nil { + t.Fatal(err) + } + + bun, err := loadBundle(bundleFile) + if err != nil { + t.Fatal(err) + } + + if err = bun.Validate(); err != nil { + t.Fatal(err) + } + + assertImage := func(t *testing.T, i string, expectedOriginalImageName string, expectedImageName string) { + img := bun.Images[i] + + actualImageName := img.Image + if actualImageName != expectedImageName { + t.Fatalf("output bundle has image %s with unexpected name: %q (expected %q)", i, actualImageName, + expectedImageName) + } + + actualOriginalImageName := img.OriginalImage + if actualOriginalImageName != expectedOriginalImageName { + t.Fatalf("output bundle has image %s with unexpected original name: %q (expected %q)", i, + actualOriginalImageName, expectedOriginalImageName) + } + } + + assertImage(t, "a", originalImageNameA, relocatedImageNameA) + assertImage(t, "b", originalImageNameB, relocatedImageNameB) + assertImage(t, "c", "", "c") +} + +// naïve test mapping, preserving any tag and/or digest +func testMapping(originalImage image.Name, t *testing.T) image.Name { + rn, err := image.NewName(path.Join(testRepositoryPrefix, originalImage.Path(), "relocated")) + if err != nil { + t.Fatal(err) + } + if tag := originalImage.Tag(); tag != "" { + rn, err = rn.WithTag(tag) + if err != nil { + t.Fatal(err) + } + } + if dig := originalImage.Digest(); dig != image.EmptyDigest { + rn, err = rn.WithDigest(dig) + if err != nil { + t.Fatal(err) + } + } + return rn +} + +type mockRegClient struct { + copyStub func(source image.Name, target image.Name) (image.Digest, error) + digestStub func(n image.Name) (image.Digest, error) + newLayoutStub func(path string) (registry.Layout, error) + readLayoutStub func(path string) (registry.Layout, error) +} + +func (r *mockRegClient) Digest(n image.Name) (image.Digest, error) { return r.digestStub(n) } +func (r *mockRegClient) Copy(src image.Name, tgt image.Name) (image.Digest, error) { + return r.copyStub(src, tgt) +} +func (r *mockRegClient) NewLayout(path string) (registry.Layout, error) { return r.newLayoutStub(path) } +func (r *mockRegClient) ReadLayout(path string) (registry.Layout, error) { + return r.readLayoutStub(path) +} diff --git a/cmd/duffle/root.go b/cmd/duffle/root.go index 63ceaf04..1da4c285 100644 --- a/cmd/duffle/root.go +++ b/cmd/duffle/root.go @@ -47,6 +47,7 @@ func newRootCmd(outputRedirect io.Writer) *cobra.Command { newInitCmd(outLog), newShowCmd(outLog), newListCmd(outLog), + newRelocateCmd(outLog), newVersionCmd(outLog), newInstallCmd(outLog), newStatusCmd(outLog), diff --git a/cmd/duffle/testdata/relocate/bundle.json b/cmd/duffle/testdata/relocate/bundle.json new file mode 100644 index 00000000..1772c2c9 --- /dev/null +++ b/cmd/duffle/testdata/relocate/bundle.json @@ -0,0 +1,30 @@ +{ + "name": "testrelocate", + "version": "0.1", + "description": "a bundle with images", + "invocationImages": [ + { + "image": "deislabs/helloworld-cnab:675c2daf7449ae58927f251acdd88acb9d1e9767", + "imageType": "docker" + } + ], + "images": { + "a": { + "description": "digested oci", + "image": "deislabs/duffle@sha256:4d41eeb38fb14266b7c0461ef1ef0b2f8c05f41cd544987a259a9d92cdad2540", + "digest": "sha256:4d41eeb38fb14266b7c0461ef1ef0b2f8c05f41cd544987a259a9d92cdad2540", + "imageType": "oci" + }, + "b": { + "description": "tagged docker", + "image": "deislabs/duffle:0.1.0-ralpha.5-englishrose", + "digest": "sha256:14d6134d892aeccb7e142557fe746ccd0a8f736a747c195ef04c9f3f0f0bbd49", + "imageType": "docker" + }, + "c": { + "description": "neither oci nor docker", + "image": "c", + "imageType": "c" + } + } +}