Add oci/layout.PutBlobFromLocalFile

Try to reflink the file and restort to copying it in case of failure.
Also add an Options struct to be future proof.

Signed-off-by: Miloslav Trmač <[email protected]>
Signed-off-by: Valentin Rothberg <[email protected]>

oci/layout/fixtures/files/a.txt

@@ -0,0 +1 @@
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa

oci/layout/fixtures/files/b.txt

@@ -0,0 +1 @@
+bbbbbbbbbbbbbbbbbbbbbbbb

oci/layout/oci_dest.go

@@ -18,10 +18,12 @@ import (
 	"github.com/containers/image/v5/internal/private"
 	"github.com/containers/image/v5/internal/putblobdigest"
 	"github.com/containers/image/v5/types"
+	reflinkCopy "github.com/containers/storage/drivers/copy"
 	"github.com/containers/storage/pkg/fileutils"
 	digest "github.com/opencontainers/go-digest"
 	imgspec "github.com/opencontainers/image-spec/specs-go"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
 )
 
 type ociImageDestination struct {
@@ -162,7 +164,7 @@ func (d *ociImageDestination) PutBlobWithOptions(ctx context.Context, stream io.
 		return private.UploadedBlob{}, err
 	}
 
-	// need to explicitly close the file, since a rename won't otherwise not work on Windows
+	// need to explicitly close the file, since a rename won't otherwise work on Windows
 	blobFile.Close()
 	explicitClosed = true
 	if err := os.Rename(blobFile.Name(), blobPath); err != nil {
@@ -302,6 +304,102 @@ func (d *ociImageDestination) CommitWithOptions(ctx context.Context, options pri
 	return os.WriteFile(d.ref.indexPath(), indexJSON, 0644)
 }
 
+// tryReflinkLocalFile attempts to reflink the specified file and digest it.
+// If relinking does not work, reset to doing a verbatim copy of the file.
+func tryReflinkLocalFile(dest *ociImageDestination, file string) (private.UploadedBlob, bool, error) {
+	fInfo, err := os.Stat(file)
+	if err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+
+	blobFile, err := os.CreateTemp(dest.ref.dir, "oci-put-blob")
+	if err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+	blobName := blobFile.Name()
+
+	copyRange := false
+	copyClone := true
+	err = reflinkCopy.CopyRegularToFile(file, blobFile, fInfo, &copyRange, &copyClone)
+	if err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+
+	_, err = blobFile.Seek(0, 0)
+	if err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+
+	blobFile, err = os.Open(blobName)
+	if err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+	blobDigest, err := digest.FromReader(blobFile)
+	if err != nil {
+		blobFile.Close()
+		return private.UploadedBlob{}, false, err
+	}
+	blobPath, err := dest.ref.blobPath(blobDigest, dest.sharedBlobDir)
+	if err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+	if err := ensureParentDirectoryExists(blobPath); err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+
+	// need to explicitly close the file, since a rename won't otherwise work on Windows
+	blobFile.Close()
+	if err := os.Rename(blobName, blobPath); err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+
+	fileInfo, err := os.Stat(blobPath)
+	if err != nil {
+		return private.UploadedBlob{}, false, err
+	}
+	return private.UploadedBlob{Digest: blobDigest, Size: fileInfo.Size()}, false, nil
+}
+
+// PutBlobFromLocalFileOptions is unused but may receive functionality in the future.
+type PutBlobFromLocalFileOptions struct{}
+
+// PutBlobFromLocalFile arranges the data from path to be used as blob with digest.
+// It computes, and returns, the digest and size of the used file.
+//
+// This function can be used instead of dest.PutBlob() where the ImageDestination requires PutBlob() to be called.
+func PutBlobFromLocalFile(ctx context.Context, dest types.ImageDestination, file string, options *PutBlobFromLocalFileOptions) (digest.Digest, int64, error) {
+	d, ok := dest.(*ociImageDestination)
+	if !ok {
+		return "", -1, errors.New("internal error: PutBlobFromLocalFile called with a non-oci: destination")
+	}
+
+	uploaded, fallback, err := tryReflinkLocalFile(d, file)
+	if err == nil {
+		return uploaded.Digest, uploaded.Size, nil
+	} else if fallback {
+		logrus.Debugf("Falling back to copying. Error trying to hardlink file: %v", err)
+	} else {
+		return "", -1, fmt.Errorf("trying to hardlink file: %w", err)
+	}
+
+	// Fallback to copying the file
+	reader, err := os.Open(file)
+	if err != nil {
+		return "", -1, fmt.Errorf("opening %q: %w", file, err)
+	}
+	defer reader.Close()
+
+	// This makes a full copy; instead, if possible, we could only digest the file and reflink (hard link?)
+	uploaded, err = d.PutBlobWithOptions(ctx, reader, types.BlobInfo{
+		Digest: "",
+		Size:   -1,
+	}, private.PutBlobOptions{})
+	if err != nil {
+		return "", -1, err
+	}
+	return uploaded.Digest, uploaded.Size, nil
+}
+
 func ensureDirectoryExists(path string) error {
 	if err := fileutils.Exists(path); err != nil && errors.Is(err, fs.ErrNotExist) {
 		if err := os.MkdirAll(path, 0755); err != nil {

oci/layout/oci_dest_test.go

@@ -178,3 +178,41 @@ func putTestManifest(t *testing.T, ociRef ociReference, tmpDir string) {
 	digest := digest.FromBytes(data).Encoded()
 	assert.Contains(t, paths, filepath.Join(tmpDir, "blobs", "sha256", digest), "The OCI directory does not contain the new manifest data")
 }
+
+func TestPutbloFromLocalFile(t *testing.T) {
+	ref, _ := refToTempOCI(t, false)
+	dest, err := ref.NewImageDestination(context.Background(), nil)
+	require.NoError(t, err)
+	defer dest.Close()
+	ociDest, ok := dest.(*ociImageDestination)
+	require.True(t, ok)
+
+	for _, test := range []struct {
+		path   string
+		size   int64
+		digest string
+	}{
+		{path: "fixtures/files/a.txt", size: 31, digest: "sha256:c8a3f498ce6aaa13c803fa3a6a0d5fd6b5d75be5781f98f56c0f960efcc53174"},
+		{path: "fixtures/files/b.txt", size: 25, digest: "sha256:8c1e9b03116b95e6dfac68c588190d56bfc82b9cc550ada726e882e138a3b93b"},
+		{path: "fixtures/files/b.txt", size: 25, digest: "sha256:8c1e9b03116b95e6dfac68c588190d56bfc82b9cc550ada726e882e138a3b93b"}, // Must not fail
+	} {
+		digest, size, err := PutBlobFromLocalFile(context.Background(), dest, test.path, nil)
+		require.NoError(t, err)
+		require.Equal(t, test.size, size)
+		require.Equal(t, test.digest, digest.String())
+
+		blobPath, err := ociDest.ref.blobPath(digest, ociDest.sharedBlobDir)
+		require.NoError(t, err)
+		require.FileExists(t, blobPath)
+
+		expectedContent, err := os.ReadFile(test.path)
+		require.NoError(t, err)
+		require.NotEmpty(t, expectedContent)
+		blobContent, err := os.ReadFile(blobPath)
+		require.NoError(t, err)
+		require.Equal(t, expectedContent, blobContent)
+	}
+
+	err = ociDest.CommitWithOptions(context.Background(), private.CommitOptions{})
+	require.NoError(t, err)
+}