From 6481647ad504ef3d25c34451e4f04bd61d19efda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Tue, 24 Sep 2024 21:10:50 +0200 Subject: [PATCH] Add oci/layout.PutBlobFromLocalFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Try to reflink the file and restort to copying it in case of failure. Also add an Options struct to be future proof. Signed-off-by: Miloslav Trmač Signed-off-by: Valentin Rothberg --- oci/layout/fixtures/files/a.txt | 1 + oci/layout/fixtures/files/b.txt | 1 + oci/layout/oci_dest.go | 100 +++++++++++++++++++++++++++++++- oci/layout/oci_dest_test.go | 38 ++++++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 oci/layout/fixtures/files/a.txt create mode 100644 oci/layout/fixtures/files/b.txt diff --git a/oci/layout/fixtures/files/a.txt b/oci/layout/fixtures/files/a.txt new file mode 100644 index 0000000000..5582bcf401 --- /dev/null +++ b/oci/layout/fixtures/files/a.txt @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/oci/layout/fixtures/files/b.txt b/oci/layout/fixtures/files/b.txt new file mode 100644 index 0000000000..59f630b80c --- /dev/null +++ b/oci/layout/fixtures/files/b.txt @@ -0,0 +1 @@ +bbbbbbbbbbbbbbbbbbbbbbbb diff --git a/oci/layout/oci_dest.go b/oci/layout/oci_dest.go index 1f0b81d423..4a3029e017 100644 --- a/oci/layout/oci_dest.go +++ b/oci/layout/oci_dest.go @@ -18,10 +18,12 @@ import ( "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/types" + reflinkCopy "github.com/containers/storage/drivers/copy" "github.com/containers/storage/pkg/fileutils" digest "github.com/opencontainers/go-digest" imgspec "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" ) type ociImageDestination struct { @@ -162,7 +164,7 @@ func (d *ociImageDestination) PutBlobWithOptions(ctx context.Context, stream io. return private.UploadedBlob{}, err } - // need to explicitly close the file, since a rename won't otherwise not work on Windows + // need to explicitly close the file, since a rename won't otherwise work on Windows blobFile.Close() explicitClosed = true if err := os.Rename(blobFile.Name(), blobPath); err != nil { @@ -302,6 +304,102 @@ func (d *ociImageDestination) CommitWithOptions(ctx context.Context, options pri return os.WriteFile(d.ref.indexPath(), indexJSON, 0644) } +// tryReflinkLocalFile attempts to reflink the specified file and digest it. +// If relinking does not work, reset to doing a verbatim copy of the file. +func tryReflinkLocalFile(dest *ociImageDestination, file string) (private.UploadedBlob, bool, error) { + fInfo, err := os.Stat(file) + if err != nil { + return private.UploadedBlob{}, false, err + } + + blobFile, err := os.CreateTemp(dest.ref.dir, "oci-put-blob") + if err != nil { + return private.UploadedBlob{}, false, err + } + blobName := blobFile.Name() + + copyRange := false + copyClone := true + err = reflinkCopy.CopyRegularToFile(file, blobFile, fInfo, ©Range, ©Clone) + if err != nil { + return private.UploadedBlob{}, false, err + } + + _, err = blobFile.Seek(0, 0) + if err != nil { + return private.UploadedBlob{}, false, err + } + + blobFile, err = os.Open(blobName) + if err != nil { + return private.UploadedBlob{}, false, err + } + blobDigest, err := digest.FromReader(blobFile) + if err != nil { + blobFile.Close() + return private.UploadedBlob{}, false, err + } + blobPath, err := dest.ref.blobPath(blobDigest, dest.sharedBlobDir) + if err != nil { + return private.UploadedBlob{}, false, err + } + if err := ensureParentDirectoryExists(blobPath); err != nil { + return private.UploadedBlob{}, false, err + } + + // need to explicitly close the file, since a rename won't otherwise work on Windows + blobFile.Close() + if err := os.Rename(blobName, blobPath); err != nil { + return private.UploadedBlob{}, false, err + } + + fileInfo, err := os.Stat(blobPath) + if err != nil { + return private.UploadedBlob{}, false, err + } + return private.UploadedBlob{Digest: blobDigest, Size: fileInfo.Size()}, false, nil +} + +// PutBlobFromLocalFileOptions is unused but may receive functionality in the future. +type PutBlobFromLocalFileOptions struct{} + +// PutBlobFromLocalFile arranges the data from path to be used as blob with digest. +// It computes, and returns, the digest and size of the used file. +// +// This function can be used instead of dest.PutBlob() where the ImageDestination requires PutBlob() to be called. +func PutBlobFromLocalFile(ctx context.Context, dest types.ImageDestination, file string, options *PutBlobFromLocalFileOptions) (digest.Digest, int64, error) { + d, ok := dest.(*ociImageDestination) + if !ok { + return "", -1, errors.New("internal error: PutBlobFromLocalFile called with a non-oci: destination") + } + + uploaded, fallback, err := tryReflinkLocalFile(d, file) + if err == nil { + return uploaded.Digest, uploaded.Size, nil + } else if fallback { + logrus.Debugf("Falling back to copying. Error trying to hardlink file: %v", err) + } else { + return "", -1, fmt.Errorf("trying to hardlink file: %w", err) + } + + // Fallback to copying the file + reader, err := os.Open(file) + if err != nil { + return "", -1, fmt.Errorf("opening %q: %w", file, err) + } + defer reader.Close() + + // This makes a full copy; instead, if possible, we could only digest the file and reflink (hard link?) + uploaded, err = d.PutBlobWithOptions(ctx, reader, types.BlobInfo{ + Digest: "", + Size: -1, + }, private.PutBlobOptions{}) + if err != nil { + return "", -1, err + } + return uploaded.Digest, uploaded.Size, nil +} + func ensureDirectoryExists(path string) error { if err := fileutils.Exists(path); err != nil && errors.Is(err, fs.ErrNotExist) { if err := os.MkdirAll(path, 0755); err != nil { diff --git a/oci/layout/oci_dest_test.go b/oci/layout/oci_dest_test.go index 57981c37a3..ae47caea28 100644 --- a/oci/layout/oci_dest_test.go +++ b/oci/layout/oci_dest_test.go @@ -178,3 +178,41 @@ func putTestManifest(t *testing.T, ociRef ociReference, tmpDir string) { digest := digest.FromBytes(data).Encoded() assert.Contains(t, paths, filepath.Join(tmpDir, "blobs", "sha256", digest), "The OCI directory does not contain the new manifest data") } + +func TestPutbloFromLocalFile(t *testing.T) { + ref, _ := refToTempOCI(t, false) + dest, err := ref.NewImageDestination(context.Background(), nil) + require.NoError(t, err) + defer dest.Close() + ociDest, ok := dest.(*ociImageDestination) + require.True(t, ok) + + for _, test := range []struct { + path string + size int64 + digest string + }{ + {path: "fixtures/files/a.txt", size: 31, digest: "sha256:c8a3f498ce6aaa13c803fa3a6a0d5fd6b5d75be5781f98f56c0f960efcc53174"}, + {path: "fixtures/files/b.txt", size: 25, digest: "sha256:8c1e9b03116b95e6dfac68c588190d56bfc82b9cc550ada726e882e138a3b93b"}, + {path: "fixtures/files/b.txt", size: 25, digest: "sha256:8c1e9b03116b95e6dfac68c588190d56bfc82b9cc550ada726e882e138a3b93b"}, // Must not fail + } { + digest, size, err := PutBlobFromLocalFile(context.Background(), dest, test.path, nil) + require.NoError(t, err) + require.Equal(t, test.size, size) + require.Equal(t, test.digest, digest.String()) + + blobPath, err := ociDest.ref.blobPath(digest, ociDest.sharedBlobDir) + require.NoError(t, err) + require.FileExists(t, blobPath) + + expectedContent, err := os.ReadFile(test.path) + require.NoError(t, err) + require.NotEmpty(t, expectedContent) + blobContent, err := os.ReadFile(blobPath) + require.NoError(t, err) + require.Equal(t, expectedContent, blobContent) + } + + err = ociDest.CommitWithOptions(context.Background(), private.CommitOptions{}) + require.NoError(t, err) +}