docker: support for requesting chunks without end offset

if the specified length for the range is set to -1, then request all
the data possible by using the "Range: <unit>=<range-start>-" syntax
for the HTTP range.

Signed-off-by: Giuseppe Scrivano <[email protected]>

copy/progress_bars.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"time"
 
 	"github.com/containers/image/v5/internal/private"
@@ -151,12 +152,18 @@ type blobChunkAccessorProxy struct {
 // The specified chunks must be not overlapping and sorted by their offset.
 // The readers must be fully consumed, in the order they are returned, before blocking
 // to read the next chunk.
+// If the Length for the last chunk is set to math.MaxUint64, then it
+// fully fetches the remaining data from the offset to the end of the blob.
 func (s *blobChunkAccessorProxy) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
 	start := time.Now()
 	rc, errs, err := s.wrapped.GetBlobAt(ctx, info, chunks)
 	if err == nil {
 		total := int64(0)
 		for _, c := range chunks {
+			// do not update the progress bar if there is a chunk with unknown length.
+			if c.Length == math.MaxUint64 {
+				return rc, errs, err
+			}
 			total += int64(c.Length)
 		}
 		s.bar.EwmaIncrInt64(total, time.Since(start))

docker/docker_image_src.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -260,9 +261,15 @@ func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error,
 			}
 			currentOffset += toSkip
 		}
+		var reader io.Reader
+		if c.Length == math.MaxUint64 {
+			reader = body
+		} else {
+			reader = io.LimitReader(body, int64(c.Length))
+		}
 		s := signalCloseReader{
 			closed:        make(chan struct{}),
-			stream:        io.NopCloser(io.LimitReader(body, int64(c.Length))),
+			stream:        io.NopCloser(reader),
 			consumeStream: true,
 		}
 		streams <- s
@@ -343,12 +350,24 @@ func parseMediaType(contentType string) (string, map[string]string, error) {
 // The specified chunks must be not overlapping and sorted by their offset.
 // The readers must be fully consumed, in the order they are returned, before blocking
 // to read the next chunk.
+// If the Length for the last chunk is set to math.MaxUint64, then it
+// fully fetches the remaining data from the offset to the end of the blob.
 func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
 	headers := make(map[string][]string)
 
 	rangeVals := make([]string, 0, len(chunks))
+	lastFound := false
 	for _, c := range chunks {
-		rangeVals = append(rangeVals, fmt.Sprintf("%d-%d", c.Offset, c.Offset+c.Length-1))
+		if lastFound {
+			return nil, nil, fmt.Errorf("internal error: another chunk requested after an util-EOF chunk")
+		}
+		// If the Length is set to -1, then request anything after the specified offset.
+		if c.Length == math.MaxUint64 {
+			lastFound = true
+			rangeVals = append(rangeVals, fmt.Sprintf("%d-", c.Offset))
+		} else {
+			rangeVals = append(rangeVals, fmt.Sprintf("%d-%d", c.Offset, c.Offset+c.Length-1))
+		}
 	}
 
 	headers["Range"] = []string{fmt.Sprintf("bytes=%s", strings.Join(rangeVals, ","))}

internal/imagesource/stubs/get_blob_at.go

@@ -39,6 +39,8 @@ func (stub NoGetBlobAtInitialize) SupportsGetBlobAt() bool {
 // The specified chunks must be not overlapping and sorted by their offset.
 // The readers must be fully consumed, in the order they are returned, before blocking
 // to read the next chunk.
+// If the Length for the last chunk is set to math.MaxUint64, then it
+// fully fetches the remaining data from the offset to the end of the blob.
 func (stub NoGetBlobAtInitialize) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
 	return nil, nil, fmt.Errorf("internal error: GetBlobAt is not supported by the %q transport", stub.transportName)
 }

internal/private/private.go

@@ -143,7 +143,11 @@ type ReusedBlob struct {
 // ImageSourceChunk is a portion of a blob.
 // This API is experimental and can be changed without bumping the major version number.
 type ImageSourceChunk struct {
+	// Offset specifies the starting position of the chunk within the source blob.
 	Offset uint64
+
+	// Length specifies the size of the chunk.  If it is set to math.MaxUint64,
+	// then it refers to all the data from Offset to the end of the blob.
 	Length uint64
 }
 
@@ -154,6 +158,8 @@ type BlobChunkAccessor interface {
 	// The specified chunks must be not overlapping and sorted by their offset.
 	// The readers must be fully consumed, in the order they are returned, before blocking
 	// to read the next chunk.
+	// If the Length for the last chunk is set to math.MaxUint64, then it
+	// fully fetches the remaining data from the offset to the end of the blob.
 	GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []ImageSourceChunk) (chan io.ReadCloser, chan error, error)
 }
 

oci/archive/oci_src.go

@@ -149,6 +149,8 @@ func (s *ociArchiveImageSource) SupportsGetBlobAt() bool {
 // The specified chunks must be not overlapping and sorted by their offset.
 // The readers must be fully consumed, in the order they are returned, before blocking
 // to read the next chunk.
+// If the Length for the last chunk is set to math.MaxUint64, then it
+// fully fetches the remaining data from the offset to the end of the blob.
 func (s *ociArchiveImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
 	return s.unpackedSrc.GetBlobAt(ctx, info, chunks)
 }

pkg/blobcache/src.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"sync"
 
@@ -226,9 +227,15 @@ func streamChunksFromFile(streams chan io.ReadCloser, errs chan error, file io.R
 			errs <- err
 			break
 		}
+		var stream io.Reader
+		if c.Length != math.MaxUint64 {
+			stream = io.LimitReader(file, int64(c.Length))
+		} else {
+			stream = file
+		}
 		s := signalCloseReader{
 			closed: make(chan struct{}),
-			stream: io.LimitReader(file, int64(c.Length)),
+			stream: stream,
 		}
 		streams <- s
 
@@ -256,6 +263,8 @@ func (s signalCloseReader) Close() error {
 // The specified chunks must be not overlapping and sorted by their offset.
 // The readers must be fully consumed, in the order they are returned, before blocking
 // to read the next chunk.
+// If the Length for the last chunk is set to math.MaxUint64, then it
+// fully fetches the remaining data from the offset to the end of the blob.
 func (s *blobCacheSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
 	blobPath, _, _, err := s.reference.findBlob(info)
 	if err != nil {