copy: add --dest-compress-format and --dest-compress-level

cmd/skopeo/flag.go

@@ -73,3 +73,37 @@ func (ob *optionalStringValue) String() string {
 	}
 	return ob.value
 }
+
+// optionalInt is a int with a separate presence flag.
+type optionalInt struct {
+	present bool
+	value   int
+}
+
+// optionalInt is a cli.Generic == flag.Value implementation equivalent to
+// the one underlying flag.Int, except that it records whether the flag has been set.
+// This is distinct from optionalInt to (pretend to) force callers to use
+// newoptionalIntValue
+type optionalIntValue optionalInt
+
+func newOptionalIntValue(p *optionalInt) cli.Generic {
+	p.present = false
+	return (*optionalIntValue)(p)
+}
+
+func (ob *optionalIntValue) Set(s string) error {
+	v, err := strconv.ParseInt(s, 0, strconv.IntSize)
+	if err != nil {
+		return err
+	}
+	ob.value = int(v)
+	ob.present = true
+	return nil
+}
+
+func (ob *optionalIntValue) String() string {
+	if !ob.present {
+		return "" // If the value is not present, just return an empty string, any other value wouldn't make sense.
+	}
+	return strconv.Itoa(int(ob.value))
+}

cmd/skopeo/utils.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"strings"
 
+	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/transports/alltransports"
 	"github.com/containers/image/types"
 	"github.com/urfave/cli"
@@ -147,15 +148,18 @@ func (opts *imageOptions) newSystemContext() (*types.SystemContext, error) {
 	if opts.noCreds {
 		ctx.DockerAuthConfig = &types.DockerAuthConfig{}
 	}
+
 	return ctx, nil
 }
 
 // imageDestOptions is a superset of imageOptions specialized for iamge destinations.
 type imageDestOptions struct {
 	*imageOptions
-	osTreeTmpDir                string // A directory to use for OSTree temporary files
-	dirForceCompression         bool   // Compress layers when saving to the dir: transport
-	ociAcceptUncompressedLayers bool   // Whether to accept uncompressed layers in the oci: transport
+	osTreeTmpDir                string      // A directory to use for OSTree temporary files
+	dirForceCompression         bool        // Compress layers when saving to the dir: transport
+	ociAcceptUncompressedLayers bool        // Whether to accept uncompressed layers in the oci: transport
+	compressionFormat           string      // Format to use for the compression
+	compressionLevel            optionalInt // Level to use for the compression
 }
 
 // imageDestFlags prepares a collection of CLI flags writing into imageDestOptions, and the managed imageDestOptions structure.
@@ -179,6 +183,16 @@ func imageDestFlags(global *globalOptions, shared *sharedImageOptions, flagPrefi
 			Usage:       "Allow uncompressed image layers when saving to an OCI image using the 'oci' transport. (default is to compress things that aren't compressed)",
 			Destination: &opts.ociAcceptUncompressedLayers,
 		},
+		cli.StringFlag{
+			Name:        flagPrefix + "compress-format",
+			Usage:       "`FORMAT` to use for the compression",
+			Destination: &opts.compressionFormat,
+		},
+		cli.GenericFlag{
+			Name:  flagPrefix + "compress-level",
+			Usage: "`LEVEL` to use for the compression",
+			Value: newOptionalIntValue(&opts.compressionLevel),
+		},
 	}...), &opts
 }
 
@@ -193,6 +207,16 @@ func (opts *imageDestOptions) newSystemContext() (*types.SystemContext, error) {
 	ctx.OSTreeTmpDirPath = opts.osTreeTmpDir
 	ctx.DirForceCompress = opts.dirForceCompression
 	ctx.OCIAcceptUncompressedLayers = opts.ociAcceptUncompressedLayers
+	if opts.compressionFormat != "" {
+		cf, err := compression.AlgorithmByName(opts.compressionFormat)
+		if err != nil {
+			return nil, err
+		}
+		ctx.CompressionFormat = &cf
+	}
+	if opts.compressionLevel.present {
+		ctx.CompressionLevel = &opts.compressionLevel.value
+	}
 	return ctx, err
 }
 

docs/skopeo-copy.1.md

@@ -58,6 +58,10 @@ If the authorization state is not found there, $HOME/.docker/config.json is chec
 
 Existing signatures, if any, are preserved as well.
 
+**--dest-compress-format** _format_ Specifies the compression format to use.  Supported values are: `gzip` and `zstd`.
+
+**--dest-compress-level** _format_ Specifies the compression level to use.  The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive).
+
 ## EXAMPLES
 
 To copy the layers of the docker.io busybox image to a local directory:

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/VividCortex/ewma v1.1.1 // indirect
 	github.com/containerd/continuity v0.0.0-20180216233310-d8fb8589b0e8 // indirect
 	github.com/containers/buildah v1.8.4
-	github.com/containers/image v3.0.1+incompatible
+	github.com/containers/image v1.5.2-0.20190821161828-0cc0e97405db
 	github.com/containers/storage v1.13.0
 	github.com/docker/distribution v0.0.0-20170817175659-5f6282db7d65 // indirect
 	github.com/docker/docker v0.0.0-20180522102801-da99009bbb11

go.sum

@@ -17,6 +17,10 @@ github.com/containers/image v1.5.2-0.20190717062552-2178abd5f9b1 h1:RGlzwWSoGBbc
 github.com/containers/image v1.5.2-0.20190717062552-2178abd5f9b1/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/image v1.5.2-0.20190725091050-48acc3dcbb76 h1:+9unAKrV92Jvifb06UK8H4xTKf7h7XQDOsn4EC9eqH4=
 github.com/containers/image v1.5.2-0.20190725091050-48acc3dcbb76/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
+github.com/containers/image v1.5.2-0.20190821113801-e003ccfc74de h1:6M4DvYlNvVUJppB9rWaYFw30fgzt09z2dCj4VlOl3e4=
+github.com/containers/image v1.5.2-0.20190821113801-e003ccfc74de/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
+github.com/containers/image v1.5.2-0.20190821161828-0cc0e97405db h1:ZtTWr+vb7qTLWjEW0CNP56ltYuj5t6y8AUkCxQAfH+Y=
+github.com/containers/image v1.5.2-0.20190821161828-0cc0e97405db/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/image v2.0.0+incompatible h1:FTr6Br7jlIKNCKMjSOMbAxKp2keQ0//jzJaYNTVhauk=
 github.com/containers/image v2.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
 github.com/containers/image v3.0.0+incompatible h1:pdUHY//H+3jYNnoTt+rqY8NsStX4ZBLKzPTlMC+XvnU=
@@ -49,6 +53,12 @@ github.com/etcd-io/bbolt v1.3.2 h1:RLRQ0TKLX7DlBRXAJHvbmXL17Q3KNnTBtZ9B6Qo+/Y0=
 github.com/etcd-io/bbolt v1.3.2/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680 h1:ZktWZesgun21uEDrwW7iEV1zPCGQldM2atlJZ3TdvVM=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/giuseppe/image v0.0.0-20190723195852-da7a70cbb5f1 h1:j+wcaSLeY2xPBzeSU37obWjSdWvYjf8soPscPvmOGpY=
+github.com/giuseppe/image v0.0.0-20190723195852-da7a70cbb5f1/go.mod h1:t1Xf5SPi6pYsMJfQTMHfTpWgaQkRuy+0HMhAOJOj01E=
+github.com/giuseppe/image v0.0.0-20190806095912-8e15c8a868fc h1:qvH3jFNG7pyTd58rs/gNmtfTv7E9q8Xo7/RKLZn+zag=
+github.com/giuseppe/image v0.0.0-20190806095912-8e15c8a868fc/go.mod h1:t1Xf5SPi6pYsMJfQTMHfTpWgaQkRuy+0HMhAOJOj01E=
+github.com/giuseppe/image v0.0.0-20190813140229-9b055a514f21 h1:nUR9MenOi9gs8LlU5BSQ4zCmdTLc1Js5gT7rdymI3ZE=
+github.com/giuseppe/image v0.0.0-20190813140229-9b055a514f21/go.mod h1:t1Xf5SPi6pYsMJfQTMHfTpWgaQkRuy+0HMhAOJOj01E=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/gogo/protobuf v0.0.0-20170815085658-fcdc5011193f h1:r/AdTzqktq9nQpFlFePWcp+scVi+oFRajfjRJ3UnETg=

systemtest/020-copy.bats

@@ -44,6 +44,21 @@ function setup() {
     diff -urN $dir1 $dir2
 }
 
+# Compression zstd
+@test "copy: oci, round trip" {
+    local remote_image=docker://busybox:latest
+
+    local dir=$TESTDIR/dir
+
+    run_skopeo copy --dest-compress --dest-compress-format=zstd $remote_image oci:$dir:latest
+
+    # zstd magic number
+    local magic=$(printf "\x28\xb5\x2f\xfd")
+
+    # Check there is at least one file that has the zstd magic number as the first 4 bytes
+    (for i in $dir/blobs/sha256/*; do test "$(head -c 4 $i)" = $magic && exit 0; done; exit 1)
+}
+
 # Same image, extracted once with :tag and once without
 @test "copy: oci w/ and w/o tags" {
     local remote_image=docker://busybox:latest