From b93a495de44e6d9ec349b77d88b00f86b3c5b6c6 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Fri, 26 Jul 2019 14:55:36 +0200 Subject: [PATCH] copy: set media types When copying an image, record the compression in the BlobInfo and use the information when updating the manifest's layer infos to set the layers' media types correctly. Note that consumers of the containers/image library need to update opencontainers/image-spec to commit 775207bd45b6cb8153ce218cc59351799217451f. Fixes: github.com/containers/libpod/issues/2013 Fixes: github.com/containers/buildah/issues/1589 Signed-off-by: Valentin Rothberg --- copy/copy.go | 11 ++++++++++ image/docker_schema2.go | 14 ++++++++---- image/oci.go | 14 +++++++++++- image/sourced.go | 1 + manifest/docker_schema2.go | 26 +++++++++++++++++++++- manifest/manifest.go | 4 ++++ manifest/oci.go | 26 +++++++++++++++++++++- pkg/compression/compression.go | 19 ++++++++++++---- storage/storage_image.go | 6 ++--- types/types.go | 40 ++++++++++++++++++---------------- vendor.conf | 2 +- 11 files changed, 129 insertions(+), 34 deletions(-) diff --git a/copy/copy.go b/copy/copy.go index 16c7900c67..30cfd8c9df 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -185,8 +185,13 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, blobInfoCache: blobinfocache.DefaultCache(options.DestinationCtx), } // Default to using gzip compression unless specified otherwise. +<<<<<<< HEAD if options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil { algo, err := compression.AlgorithmByName("gzip") +======= + if options.DestinationCtx.CompressionFormat == nil { + algo, err := compression.AlgorithmByName(compression.Gzip) +>>>>>>> copy: set media types if err != nil { return nil, err } @@ -911,6 +916,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr return types.BlobInfo{}, errors.Wrap(err, "Error writing blob") } + uploadedInfo.CompressionOperation = compressionOperation + // If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest. + if canModifyBlob && !isConfig { + uploadedInfo.CompressionAlgorithm = &desiredCompressionFormat + } + // This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consumer // all of the input (to compute DiffIDs), even if dest.PutBlob does not need it. // So, read everything from originalLayerReader, which will cause the rest to be diff --git a/image/docker_schema2.go b/image/docker_schema2.go index 351e73ea1d..09c35ac498 100644 --- a/image/docker_schema2.go +++ b/image/docker_schema2.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "io/ioutil" "strings" @@ -207,12 +208,17 @@ func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context) (types.Imag layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors)) for idx := range layers { layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx]) - if m.m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType { + switch m.m.LayersDescriptors[idx].MediaType { + case manifest.DockerV2Schema2ForeignLayerMediaType: layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable - } else { - // we assume layers are gzip'ed because docker v2s2 only deals with - // gzip'ed layers. However, OCI has non-gzip'ed layers as well. + case manifest.DockerV2SchemaLayerMediaTypeUncompressed: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayer + case manifest.DockerV2Schema2LayerMediaType: layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip + case manifest.DockerV2Schema2LayerMediaTypeZstd: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerZstd + default: + return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType) } } diff --git a/image/oci.go b/image/oci.go index cdff26e06a..87352f31ef 100644 --- a/image/oci.go +++ b/image/oci.go @@ -3,6 +3,7 @@ package image import ( "context" "encoding/json" + "fmt" "io/ioutil" "github.com/containers/image/docker/reference" @@ -187,7 +188,18 @@ func (m *manifestOCI1) convertToManifestSchema2() (types.Image, error) { layers := make([]manifest.Schema2Descriptor, len(m.m.Layers)) for idx := range layers { layers[idx] = schema2DescriptorFromOCI1Descriptor(m.m.Layers[idx]) - layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType + switch layers[idx].MediaType { + case imgspecv1.MediaTypeImageLayerNonDistributable: + layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaType + case imgspecv1.MediaTypeImageLayer: + layers[idx].MediaType = manifest.DockerV2SchemaLayerMediaTypeUncompressed + case imgspecv1.MediaTypeImageLayerGzip: + layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType + case imgspecv1.MediaTypeImageLayerZstd: + layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaTypeZstd + default: + return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", layers[idx].MediaType) + } } // Rather than copying the ConfigBlob now, we just pass m.src to the diff --git a/image/sourced.go b/image/sourced.go index 01cc28bbd2..c8364a1454 100644 --- a/image/sourced.go +++ b/image/sourced.go @@ -5,6 +5,7 @@ package image import ( "context" + "github.com/containers/image/types" ) diff --git a/manifest/docker_schema2.go b/manifest/docker_schema2.go index 76a80e5a6f..c602b0a7e5 100644 --- a/manifest/docker_schema2.go +++ b/manifest/docker_schema2.go @@ -2,12 +2,15 @@ package manifest import ( "encoding/json" + "fmt" "time" + "github.com/containers/image/pkg/compression" "github.com/containers/image/pkg/strslice" "github.com/containers/image/types" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // Schema2Descriptor is a “descriptor” in docker/distribution schema 2. @@ -207,7 +210,28 @@ func (m *Schema2) UpdateLayerInfos(layerInfos []types.BlobInfo) error { original := m.LayersDescriptors m.LayersDescriptors = make([]Schema2Descriptor, len(layerInfos)) for i, info := range layerInfos { - m.LayersDescriptors[i].MediaType = original[i].MediaType + switch info.CompressionOperation { + case types.PreserveOriginal: + m.LayersDescriptors[i].MediaType = original[i].MediaType + case types.Decompress: + m.LayersDescriptors[i].MediaType = DockerV2SchemaLayerMediaTypeUncompressed + case types.Compress: + if info.CompressionAlgorithm == nil { + logrus.Debugf("Preparing updated manifest: blob %q was compressed but does not specify by which algorithm: falling back to use the original blob", info.Digest) + m.LayersDescriptors[i].MediaType = original[i].MediaType + break + } + switch info.CompressionAlgorithm.Name() { + case compression.Gzip: + m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaType + case compression.Zstd: + m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaTypeZstd + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q fo layer %q", info.CompressionAlgorithm.Name(), info.Digest) + } + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression operation (%d) for layer %q", info.CompressionOperation, info.Digest) + } m.LayersDescriptors[i].Digest = info.Digest m.LayersDescriptors[i].Size = info.Size m.LayersDescriptors[i].URLs = info.URLs diff --git a/manifest/manifest.go b/manifest/manifest.go index ae1921b6cc..6d560c23a3 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -24,6 +24,10 @@ const ( DockerV2Schema2ConfigMediaType = "application/vnd.docker.container.image.v1+json" // DockerV2Schema2LayerMediaType is the MIME type used for schema 2 layers. DockerV2Schema2LayerMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + // DockerV2Schema2LayerMediaTypeZstd is the MIME type used for schema 2 layers compressed with zstd. + DockerV2Schema2LayerMediaTypeZstd = "application/vnd.docker.image.rootfs.diff.tar.zstd" + // DockerV2SchemaLayerMediaTypeUncompressed is the mediaType used for uncompressed layers. + DockerV2SchemaLayerMediaTypeUncompressed = "application/vnd.docker.image.rootfs.diff.tar" // DockerV2ListMediaType MIME type represents Docker manifest schema 2 list DockerV2ListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" // DockerV2Schema2ForeignLayerMediaType is the MIME type used for schema 2 foreign layers. diff --git a/manifest/oci.go b/manifest/oci.go index dd65e0ba27..08fbce6ff1 100644 --- a/manifest/oci.go +++ b/manifest/oci.go @@ -2,12 +2,15 @@ package manifest import ( "encoding/json" + "fmt" + "github.com/containers/image/pkg/compression" "github.com/containers/image/types" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // BlobInfoFromOCI1Descriptor returns a types.BlobInfo based on the input OCI1 descriptor. @@ -81,7 +84,28 @@ func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { original := m.Layers m.Layers = make([]imgspecv1.Descriptor, len(layerInfos)) for i, info := range layerInfos { - m.Layers[i].MediaType = original[i].MediaType + switch info.CompressionOperation { + case types.PreserveOriginal: + m.Layers[i].MediaType = original[i].MediaType + case types.Decompress: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayer + case types.Compress: + if info.CompressionAlgorithm == nil { + logrus.Debugf("Preparing updated manifest: blob %q was compressed but does not specify by which algorithm: falling back to use the original blob", info.Digest) + m.Layers[i].MediaType = original[i].MediaType + break + } + switch info.CompressionAlgorithm.Name() { + case compression.Gzip: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerGzip + case compression.Zstd: + m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerZstd + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q for layer %q", info.CompressionAlgorithm.Name(), info.Digest) + } + default: + return fmt.Errorf("Error preparing updated manifest: unknown compression operation (%d) for layer %q", info.CompressionOperation, info.Digest) + } m.Layers[i].Digest = info.Digest m.Layers[i].Size = info.Size m.Layers[i].Annotations = info.Annotations diff --git a/pkg/compression/compression.go b/pkg/compression/compression.go index b42151cffc..370df3331c 100644 --- a/pkg/compression/compression.go +++ b/pkg/compression/compression.go @@ -13,6 +13,17 @@ import ( "github.com/ulikunitz/xz" ) +const ( + // Gzip compression. + Gzip = "gzip" + // Bzip2 compression. + Bzip2 = "bzip2" + // Xz compression. + Xz = "xz" + // Zstd compression. + Zstd = "zstd" +) + // DecompressorFunc returns the decompressed stream, given a compressed stream. // The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!). type DecompressorFunc func(io.Reader) (io.ReadCloser, error) @@ -73,10 +84,10 @@ func (c Algorithm) Name() string { // compressionAlgos is an internal implementation detail of DetectCompression var compressionAlgos = []Algorithm{ - {"gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor}, // gzip (RFC 1952) - {"bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor}, // bzip2 (decompress.c:BZ2_decompress) - {"xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor}, // xz (/usr/share/doc/xz/xz-file-format.txt) - {"zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor}, // zstd (http://www.zstd.net) + {Gzip, []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor}, // gzip (RFC 1952) + {Bzip2, []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor}, // bzip2 (decompress.c:BZ2_decompress) + {Xz, []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor}, // xz (/usr/share/doc/xz/xz-file-format.txt) + {Zstd, []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor}, // zstd (http://www.zstd.net) } // AlgorithmByName returns the compressor by its name diff --git a/storage/storage_image.go b/storage/storage_image.go index 946a85f7b1..7d8860a508 100644 --- a/storage/storage_image.go +++ b/storage/storage_image.go @@ -345,9 +345,9 @@ func (s *storageImageDestination) Close() error { } func (s *storageImageDestination) DesiredLayerCompression() types.LayerCompression { - // We ultimately have to decompress layers to populate trees on disk, - // so callers shouldn't bother compressing them before handing them to - // us, if they're not already compressed. + // We ultimately have to decompress layers to populate trees on disk + // and need to explicitly ask for it here, so that the layers' MIME + // types can be set accordingly. return types.PreserveOriginal } diff --git a/types/types.go b/types/types.go index b94af8dccb..852fe91b7d 100644 --- a/types/types.go +++ b/types/types.go @@ -8,7 +8,7 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/pkg/compression" "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go/v1" + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) // ImageTransport is a top-level namespace for ways to to store/load an image. @@ -91,14 +91,29 @@ type ImageReference interface { DeleteImage(ctx context.Context, sys *SystemContext) error } +// LayerCompression indicates if layers must be compressed, decompressed or preserved +type LayerCompression int + +const ( + // PreserveOriginal indicates the layer must be preserved, ie + // no compression or decompression. + PreserveOriginal LayerCompression = iota + // Decompress indicates the layer must be decompressed + Decompress + // Compress indicates the layer must be compressed + Compress +) + // BlobInfo collects known information about a blob (layer/config). // In some situations, some fields may be unknown, in others they may be mandatory; documenting an “unknown” value here does not override that. type BlobInfo struct { - Digest digest.Digest // "" if unknown. - Size int64 // -1 if unknown - URLs []string - Annotations map[string]string - MediaType string + Digest digest.Digest // "" if unknown. + Size int64 // -1 if unknown + URLs []string + Annotations map[string]string + MediaType string + CompressionOperation LayerCompression + CompressionAlgorithm *compression.Algorithm } // BICTransportScope encapsulates transport-dependent representation of a “scope” where blobs are or are not present. @@ -212,19 +227,6 @@ type ImageSource interface { LayerInfosForCopy(ctx context.Context) ([]BlobInfo, error) } -// LayerCompression indicates if layers must be compressed, decompressed or preserved -type LayerCompression int - -const ( - // PreserveOriginal indicates the layer must be preserved, ie - // no compression or decompression. - PreserveOriginal LayerCompression = iota - // Decompress indicates the layer must be decompressed - Decompress - // Compress indicates the layer must be compressed - Compress -) - // ImageDestination is a service, possibly remote (= slow), to store components of a single image. // // There is a specific required order for some of the calls: diff --git a/vendor.conf b/vendor.conf index 7efc86e9e2..7bcd43bdde 100644 --- a/vendor.conf +++ b/vendor.conf @@ -16,7 +16,7 @@ github.com/imdario/mergo 50d4dbd4eb0e84778abe37cefef140271d96fade github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 -github.com/opencontainers/image-spec v1.0.0 +github.com/opencontainers/image-spec 775207bd45b6cb8153ce218cc59351799217451f github.com/opencontainers/runc 6b1d0e76f239ffb435445e5ae316d2676c07c6e3 github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 github.com/pkg/errors 248dadf4e9068a0b3e79f02ed0a610d935de5302