Skip to content

Commit

Permalink
Fix image size calculation in importer
Browse files Browse the repository at this point in the history 
After upgrading docker/distibution V1Compatibility no longer contains Size. For
this reason, the image size calculation is wrong. A new way to calculate the
size repeats the way that dockerregistry calculates the size.
  • Loading branch information
@legionus
legionus committed Jul 5, 2016
1 parent 8f01725 commit 4e57bf3
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 11 deletions.
2 changes: 2 additions & 0 deletions pkg/image/api/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,8 @@ func ImageWithMetadata(image *Image) error {
image.DockerImageLayers[i].Name = layer.DockerBlobSum
}
if len(manifest.History) == len(image.DockerImageLayers) {
// This code does not work since hub.docker.com has been updated.
// The V1Compatibility no longer contains Size.
image.DockerImageLayers[0].LayerSize = v1Metadata.Size
var size = DockerV1CompatibilityImageSize{}
for i, obj := range manifest.History[1:] {
Expand Down
80 changes: 69 additions & 11 deletions pkg/image/importer/importer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ type ImageStreamImporter struct {
limiter flowcontrol.RateLimiter

digestToRepositoryCache map[gocontext.Context]map[manifestKey]*api.Image
digestToLayerSizeCache map[gocontext.Context]map[string]int64
}

// NewImageStreamImport creates an importer that will load images from a remote Docker registry into an
Expand All @@ -65,6 +66,7 @@ func NewImageStreamImporter(retriever RepositoryRetriever, maximumTagsPerRepo in
limiter: limiter,

digestToRepositoryCache: make(map[gocontext.Context]map[manifestKey]*api.Image),
digestToLayerSizeCache: make(map[gocontext.Context]map[string]int64),
}
}

Expand All @@ -78,17 +80,35 @@ func (i *ImageStreamImporter) contextImageCache(ctx gocontext.Context) map[manif
return cache
}

// contextLayerSizeCache returns the image layers cache entry for a context.
func (i *ImageStreamImporter) contextLayerSizeCache(ctx gocontext.Context) map[string]int64 {
cache := i.digestToLayerSizeCache[ctx]
if cache == nil {
cache = make(map[string]int64)
i.digestToLayerSizeCache[ctx] = cache
}
return cache
}

type imageStreamImporterCache struct {
ImageCache map[manifestKey]*api.Image
LayerSizeCache map[string]int64
}

// Import tries to complete the provided isi object with images loaded from remote registries.
func (i *ImageStreamImporter) Import(ctx gocontext.Context, isi *api.ImageStreamImport) error {
cache := i.contextImageCache(ctx)
cache := &imageStreamImporterCache{
ImageCache: i.contextImageCache(ctx),
LayerSizeCache: i.contextLayerSizeCache(ctx),
}
importImages(ctx, i.retriever, isi, cache, i.limiter)
importFromRepository(ctx, i.retriever, isi, i.maximumTagsPerRepo, cache, i.limiter)
return nil
}

// importImages updates the passed ImageStreamImport object and sets Status for each image based on whether the import
// succeeded or failed. Cache is updated with any loaded images. Limiter is optional and controls how fast images are updated.
func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, cache map[manifestKey]*api.Image, limiter flowcontrol.RateLimiter) {
func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, cache *imageStreamImporterCache, limiter flowcontrol.RateLimiter) {
tags := make(map[manifestKey][]int)
ids := make(map[manifestKey][]int)
repositories := make(map[repositoryKey]*importRepository)
Expand Down Expand Up @@ -128,7 +148,7 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api
if len(ids[id]) == 1 {
repo.Digests = append(repo.Digests, importDigest{
Name: defaultRef.ID,
Image: cache[id],
Image: cache.ImageCache[id],
})
}
} else {
Expand All @@ -138,20 +158,20 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api
if len(tags[tag]) == 1 {
repo.Tags = append(repo.Tags, importTag{
Name: defaultRef.Tag,
Image: cache[tag],
Image: cache.ImageCache[tag],
})
}
}
}

// for each repository we found, import all tags and digests
for key, repo := range repositories {
importRepositoryFromDocker(ctx, retriever, repo, limiter)
importRepositoryFromDocker(ctx, retriever, repo, limiter, cache)
for _, tag := range repo.Tags {
j := manifestKey{repositoryKey: key}
j.value = tag.Name
if tag.Image != nil {
cache[j] = tag.Image
cache.ImageCache[j] = tag.Image
}
for _, index := range tags[j] {
if tag.Err != nil {
Expand All @@ -172,7 +192,7 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api
j := manifestKey{repositoryKey: key}
j.value = digest.Name
if digest.Image != nil {
cache[j] = digest.Image
cache.ImageCache[j] = digest.Image
}
for _, index := range ids[j] {
if digest.Err != nil {
Expand All @@ -194,7 +214,7 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api
// importFromRepository imports the repository named on the ImageStreamImport, if any, importing up to maximumTags, and reporting
// status on each image that is attempted to be imported. If the repository cannot be found or tags cannot be retrieved, the repository
// status field is set.
func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, maximumTags int, cache map[manifestKey]*api.Image, limiter flowcontrol.RateLimiter) {
func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, maximumTags int, cache *imageStreamImporterCache, limiter flowcontrol.RateLimiter) {
if isi.Spec.Repository == nil {
return
}
Expand Down Expand Up @@ -223,7 +243,7 @@ func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever,
Insecure: spec.ImportPolicy.Insecure,
MaximumTags: maximumTags,
}
importRepositoryFromDocker(ctx, retriever, repo, limiter)
importRepositoryFromDocker(ctx, retriever, repo, limiter, cache)

if repo.Err != nil {
status.Status = imageImportStatus(repo.Err, "", "repository")
Expand All @@ -234,7 +254,7 @@ func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever,
tagKey := manifestKey{repositoryKey: key}
for _, s := range repo.AdditionalTags {
tagKey.value = s
if image, ok := cache[tagKey]; ok {
if image, ok := cache.ImageCache[tagKey]; ok {
repo.Tags = append(repo.Tags, importTag{
Name: s,
Image: image,
Expand Down Expand Up @@ -299,9 +319,35 @@ func formatRepositoryError(repository *importRepository, refName string, refID s
return
}

func calculateImageSize(ctx gocontext.Context, repo distribution.Repository, image *api.Image, cache *imageStreamImporterCache) error {
bs := repo.Blobs(ctx)

size := int64(0)
for i := range image.DockerImageLayers {
layer := &image.DockerImageLayers[i]

if layerSize, ok := cache.LayerSizeCache[layer.Name]; ok {
size += layerSize
continue
}

desc, err := bs.Stat(ctx, digest.Digest(layer.Name))
if err != nil {
return err
}

cache.LayerSizeCache[layer.Name] = desc.Size
layer.LayerSize = desc.Size
size += desc.Size
}

image.DockerImageMetadata.Size = size
return nil
}

// importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the
// optional rate limiter. Errors are set onto the individual tags and digest objects.
func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter) {
func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter, cache *imageStreamImporterCache) {
glog.V(5).Infof("importing remote Docker repository registry=%s repository=%s insecure=%t", repository.Registry, repository.Name, repository.Insecure)
// retrieve the repository
repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure)
Expand Down Expand Up @@ -430,6 +476,12 @@ func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetri
importDigest.Err = err
continue
}
if importDigest.Image.DockerImageMetadata.Size == 0 {
if err := calculateImageSize(ctx, repo, importDigest.Image, cache); err != nil {
importDigest.Err = err
continue
}
}
}

for i := range repository.Tags {
Expand Down Expand Up @@ -474,6 +526,12 @@ func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetri
importTag.Err = err
continue
}
if importTag.Image.DockerImageMetadata.Size == 0 {
if err := calculateImageSize(ctx, repo, importTag.Image, cache); err != nil {
importTag.Err = err
continue
}
}
}
}

Expand Down

0 comments on commit 4e57bf3

Please sign in to comment.