diff --git a/pkg/sif/testdata/TestUpdate/AddImage.golden b/pkg/sif/testdata/TestUpdate/AddImage.golden new file mode 100644 index 0000000..59afbd8 Binary files /dev/null and b/pkg/sif/testdata/TestUpdate/AddImage.golden differ diff --git a/pkg/sif/testdata/TestUpdate/AddImageIndex.golden b/pkg/sif/testdata/TestUpdate/AddImageIndex.golden new file mode 100644 index 0000000..fafc2ae Binary files /dev/null and b/pkg/sif/testdata/TestUpdate/AddImageIndex.golden differ diff --git a/pkg/sif/testdata/TestUpdate/AddLayer.golden b/pkg/sif/testdata/TestUpdate/AddLayer.golden new file mode 100644 index 0000000..f954480 Binary files /dev/null and b/pkg/sif/testdata/TestUpdate/AddLayer.golden differ diff --git a/pkg/sif/testdata/TestUpdate/ReplaceLayers.golden b/pkg/sif/testdata/TestUpdate/ReplaceLayers.golden new file mode 100644 index 0000000..be8de88 Binary files /dev/null and b/pkg/sif/testdata/TestUpdate/ReplaceLayers.golden differ diff --git a/pkg/sif/update.go b/pkg/sif/update.go new file mode 100644 index 0000000..1838097 --- /dev/null +++ b/pkg/sif/update.go @@ -0,0 +1,341 @@ +// Copyright 2024 Sylabs Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package sif + +import ( + "bytes" + "io" + "os" + "path/filepath" + "slices" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/sylabs/sif/v2/pkg/sif" +) + +// updateOpts accumulates update options. +type updateOpts struct { + tempDir string +} + +// UpdateOpt are used to specify options to apply when updating a SIF. +type UpdateOpt func(*updateOpts) error + +// OptUpdateTempDir sets the directory to use for temporary files. If not set, the +// directory returned by TempDir is used. +func OptTarTempDir(d string) UpdateOpt { + return func(c *updateOpts) error { + c.tempDir = d + return nil + } +} + +// Update modifies the SIF file associated with fi so that it holds the content +// of ImageIndex ii. Any blobs in the SIF that are not referenced in ii are +// removed from the SIF. Any blobs that are referenced in ii but not present in +// the SIF are added to the SIF. The RootIndex of the SIF is replaced with ii. +// +// Update may create one or more temporary files during the update process. By +// default, the directory returned by TempDir is used. To override this, +// consider using OptUpdateTmpDir. +func Update(fi *sif.FileImage, ii v1.ImageIndex, opts ...UpdateOpt) error { + uo := updateOpts{} + for _, opt := range opts { + if err := opt(&uo); err != nil { + return err + } + } + + // If the existing OCI.RootIndex in the SIF matches ii, then there is nothing to do. + sifRootIndex, err := ImageIndexFromFileImage(fi) + if err != nil { + return err + } + sifRootDigest, err := sifRootIndex.Digest() + if err != nil { + return err + } + newRootDigest, err := ii.Digest() + if err != nil { + return err + } + if sifRootDigest == newRootDigest { + return nil + } + + // Get a list of all existing OCI.Blob digests in the SIF + sifBlobs, err := sifBlobs(fi) + if err != nil { + return err + } + + // Cache all new blobs referenced by the new ImageIndex and its child + // indices / images, which aren't already in the SIF. cachedblobs are new + // things to add. keepBlobs already exist in the SIF and should be kept. + blobCache, err := os.MkdirTemp(uo.tempDir, "") + if err != nil { + return err + } + defer os.RemoveAll(blobCache) + cachedBlobs, keepBlobs, err := cacheIndexBlobs(ii, sifBlobs, blobCache) + if err != nil { + return err + } + + // Compute the new RootIndex. + ri, err := ii.RawManifest() + if err != nil { + return err + } + + // Delete existing blobs from the SIF except those we want to keep. + if err := deleteBlobsExcept(fi, keepBlobs); err != nil { + return err + } + // Delete old RootIndex. + if err := deleteRootIndex(fi); err != nil { + return err + } + + // Write new (cached) blobs from ii into the SIF. + f := fileImage{fi} + for _, b := range cachedBlobs { + rc, err := readCacheBlob(b, blobCache) + if err != nil { + return err + } + if err := f.writeBlobToFileImage(rc, false); err != nil { + return err + } + if err := rc.Close(); err != nil { + return err + } + } + + // Write the new RootIndex into the SIF. + return f.writeBlobToFileImage(bytes.NewReader(ri), true) +} + +// sifBlobs will return a list of digests for all OCI.Blob descriptors in fi. +func sifBlobs(fi *sif.FileImage) ([]v1.Hash, error) { + descrs, err := fi.GetDescriptors(sif.WithDataType(sif.DataOCIBlob)) + if err != nil { + return nil, err + } + sifBlobs := make([]v1.Hash, len(descrs)) + for i, d := range descrs { + dDigest, err := d.OCIBlobDigest() + if err != nil { + return nil, err + } + sifBlobs[i] = dDigest + } + return sifBlobs, nil +} + +// cacheIndexBlobs will cache all blobs referenced by ii, except those specified +// in skipDigests. The blobs will be cached as files in cacheDir, with filenames +// equal to their digest. The function returns lists of blobs that were cached +// (in ii but not skipDigests), and those that were skipped (in ii and +// skipDigests). +func cacheIndexBlobs(ii v1.ImageIndex, skipDigests []v1.Hash, cacheDir string) (cached []v1.Hash, skipped []v1.Hash, err error) { + index, err := ii.IndexManifest() + if err != nil { + return nil, nil, err + } + + for _, desc := range index.Manifests { + //nolint:exhaustive + switch desc.MediaType { + case types.DockerManifestList, types.OCIImageIndex: + childIndex, err := ii.ImageIndex(desc.Digest) + if err != nil { + return nil, nil, err + } + // Cache children of this ImageIndex + childCached, childSkipped, err := cacheIndexBlobs(childIndex, skipDigests, cacheDir) + if err != nil { + return nil, nil, err + } + cached = append(cached, childCached...) + skipped = append(skipped, childSkipped...) + // Cache ImageIndex itself. + if slices.Contains(skipDigests, desc.Digest) { + skipped = append(skipped, desc.Digest) + continue + } + rm, err := childIndex.RawManifest() + if err != nil { + return nil, nil, err + } + rc := io.NopCloser(bytes.NewReader(rm)) + if err := writeCacheBlob(rc, desc.Digest, cacheDir); err != nil { + return nil, nil, err + } + cached = append(cached, desc.Digest) + + case types.DockerManifestSchema2, types.OCIManifestSchema1: + childImage, err := ii.Image(desc.Digest) + if err != nil { + return nil, nil, err + } + // Cache children of this image (layers, config) + childCached, childSkipped, err := cacheImageBlobs(childImage, skipDigests, cacheDir) + if err != nil { + return nil, nil, err + } + cached = append(cached, childCached...) + skipped = append(skipped, childSkipped...) + // Cache image manifest itself. + if slices.Contains(skipDigests, desc.Digest) { + skipped = append(skipped, desc.Digest) + continue + } + rm, err := childImage.RawManifest() + if err != nil { + return nil, nil, err + } + rc := io.NopCloser(bytes.NewReader(rm)) + if err := writeCacheBlob(rc, desc.Digest, cacheDir); err != nil { + return nil, nil, err + } + cached = append(cached, desc.Digest) + + default: + if slices.Contains(skipDigests, desc.Digest) { + skipped = append(skipped, desc.Digest) + continue + } + rc, err := blobFromIndex(ii, desc.Digest) + if err != nil { + return nil, nil, err + } + if err := writeCacheBlob(rc, desc.Digest, cacheDir); err != nil { + return nil, nil, err + } + cached = append(cached, desc.Digest) + } + } + return cached, skipped, nil +} + +// cacheImageBlobs will cache all blobs referenced by im, except those specified +// in skipDigests. The blobs will be cached as files in cacheDir, with filenames +// equal to their digest. The function returns lists of blobs that were cached +// (in ii but not skipDigests), and those that were skipped (in ii and +// skipDigests). +func cacheImageBlobs(im v1.Image, skipDigests []v1.Hash, cacheDir string) (cached []v1.Hash, skipped []v1.Hash, err error) { + // Cache layers first. + layers, err := im.Layers() + if err != nil { + return nil, nil, err + } + for _, l := range layers { + ld, err := l.Digest() + if err != nil { + return nil, nil, err + } + + if slices.Contains(skipDigests, ld) { + skipped = append(skipped, ld) + continue + } + + rc, err := l.Compressed() + if err != nil { + return nil, nil, err + } + if err := writeCacheBlob(rc, ld, cacheDir); err != nil { + return nil, nil, err + } + cached = append(cached, ld) + } + + // Cache image config. + // Note - Manifest is cached from parent ImageIndex. + mf, err := im.Manifest() + if err != nil { + return nil, nil, err + } + if slices.Contains(skipDigests, mf.Config.Digest) { + skipped = append(skipped, mf.Config.Digest) + return cached, skipped, nil + } + c, err := im.RawConfigFile() + if err != nil { + return nil, nil, err + } + rc := io.NopCloser(bytes.NewReader(c)) + if err := writeCacheBlob(rc, mf.Config.Digest, cacheDir); err != nil { + return nil, nil, err + } + cached = append(cached, mf.Config.Digest) + + return cached, skipped, nil +} + +// writeCacheBlob writes blob content from rc into tmpDir with filename equal to +// specified digest. +func writeCacheBlob(rc io.ReadCloser, digest v1.Hash, cacheDir string) error { + path := filepath.Join(cacheDir, digest.String()) + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(f, rc) + if err != nil { + return err + } + + if err := rc.Close(); err != nil { + return err + } + return nil +} + +// readCacheBlob returns a ReadCloser that will read blob content from cacheDir +// with filename equal to specified digest. +func readCacheBlob(digest v1.Hash, cacheDir string) (io.ReadCloser, error) { + path := filepath.Join(cacheDir, digest.String()) + f, err := os.Open(path) + if err != nil { + return nil, err + } + return f, nil +} + +// deleteBlobsExcept removes all OCI.Blob descriptors from fi, except those with +// digests listed in keepDigests. +func deleteBlobsExcept(fi *sif.FileImage, keepDigests []v1.Hash) error { + descs, err := fi.GetDescriptors(sif.WithDataType(sif.DataOCIBlob)) + if err != nil { + return err + } + for _, d := range descs { + dd, err := d.OCIBlobDigest() + if err != nil { + return err + } + if slices.Contains(keepDigests, dd) { + continue + } + if err := fi.DeleteObject(d.ID()); err != nil { + return err + } + } + return nil +} + +// deleteRootIndex removes the RootIndex from a the SIF fi. +func deleteRootIndex(fi *sif.FileImage) error { + desc, err := fi.GetDescriptor(sif.WithDataType(sif.DataOCIRootIndex)) + if err != nil { + return err + } + return fi.DeleteObject(desc.ID()) +} diff --git a/pkg/sif/update_test.go b/pkg/sif/update_test.go new file mode 100644 index 0000000..c5c7730 --- /dev/null +++ b/pkg/sif/update_test.go @@ -0,0 +1,148 @@ +// Copyright 2024 Sylabs Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package sif_test + +import ( + "math/rand" + "os" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + v1mutate "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/sebdah/goldie/v2" + "github.com/sylabs/oci-tools/pkg/mutate" + "github.com/sylabs/oci-tools/pkg/sif" + ssif "github.com/sylabs/sif/v2/pkg/sif" +) + +const randomSeed = 1719306160 + +//nolint:gocognit +func TestUpdate(t *testing.T) { + r := rand.NewSource(randomSeed) + + tests := []struct { + name string + base string + updater func(*testing.T, v1.ImageIndex) v1.ImageIndex + opts []sif.UpdateOpt + }{ + { + name: "AddLayer", + base: "hello-world-docker-v2-manifest", + updater: func(t *testing.T, ii v1.ImageIndex) v1.ImageIndex { + t.Helper() + ih, err := v1.NewHash("sha256:432f982638b3aefab73cc58ab28f5c16e96fdb504e8c134fc58dff4bae8bf338") + if err != nil { + t.Fatal(err) + } + im, err := ii.Image(ih) + if err != nil { + t.Fatal(err) + } + l, err := random.Layer(64, types.DockerLayer, random.WithSource(r)) + if err != nil { + t.Fatal(err) + } + im, err = v1mutate.AppendLayers(im, l) + if err != nil { + t.Fatal(err) + } + return v1mutate.AppendManifests(empty.Index, v1mutate.IndexAddendum{Add: im}) + }, + }, + { + name: "ReplaceLayers", // Replaces many layers with a single layer + base: "many-layers", + updater: func(t *testing.T, ii v1.ImageIndex) v1.ImageIndex { + t.Helper() + ih, err := v1.NewHash("sha256:7c000de5bc837f29d1c9a5e76bba79922d860e5c0f448df3b6fc38431a067c9a") + if err != nil { + t.Fatal(err) + } + im, err := ii.Image(ih) + if err != nil { + t.Fatal(err) + } + l, err := random.Layer(64, types.DockerLayer, random.WithSource(r)) + if err != nil { + t.Fatal(err) + } + im, err = mutate.Apply(im, mutate.ReplaceLayers(l)) + if err != nil { + t.Fatal(err) + } + return v1mutate.AppendManifests(empty.Index, v1mutate.IndexAddendum{Add: im}) + }, + }, + { + name: "AddImage", + base: "hello-world-docker-v2-manifest", + updater: func(t *testing.T, ii v1.ImageIndex) v1.ImageIndex { + t.Helper() + im, err := random.Image(64, 1, random.WithSource(r)) + if err != nil { + t.Fatal(err) + } + if err != nil { + t.Fatal(err) + } + return v1mutate.AppendManifests(ii, v1mutate.IndexAddendum{Add: im}) + }, + }, + { + name: "AddImageIndex", + base: "hello-world-docker-v2-manifest", + updater: func(t *testing.T, ii v1.ImageIndex) v1.ImageIndex { + t.Helper() + addIdx, err := random.Index(64, 1, 1, random.WithSource(r)) + if err != nil { + t.Fatal(err) + } + if err != nil { + t.Fatal(err) + } + return v1mutate.AppendManifests(ii, v1mutate.IndexAddendum{Add: addIdx}) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sifPath := corpus.SIF(t, tt.base, sif.OptWriteWithSpareDescriptorCapacity(8)) + fi, err := ssif.LoadContainerFromPath(sifPath) + if err != nil { + t.Fatal(err) + } + ii, err := sif.ImageIndexFromFileImage(fi) + if err != nil { + t.Fatal(err) + } + + ii = tt.updater(t, ii) + + if err := sif.Update(fi, ii, tt.opts...); err != nil { + t.Fatal(err) + } + + if err := fi.UnloadContainer(); err != nil { + t.Fatal(err) + } + + b, err := os.ReadFile(sifPath) + if err != nil { + t.Fatal(err) + } + + g := goldie.New(t, + goldie.WithTestNameForDir(true), + ) + + g.Assert(t, tt.name, b) + }) + } +} diff --git a/test/images.go b/test/images.go index ddef73b..c815bcd 100644 --- a/test/images.go +++ b/test/images.go @@ -63,12 +63,12 @@ func (c *Corpus) OCILayout(tb testing.TB, name string) string { // SIF returns a temporary SIF for the test to use, populated from the OCI Image Layout with the // specified name in the corpus. The SIF is automatically removed when the test and all its // subtests complete. -func (c *Corpus) SIF(tb testing.TB, name string) string { +func (c *Corpus) SIF(tb testing.TB, name string, opt ...sif.WriteOpt) string { tb.Helper() path := filepath.Join(tb.TempDir(), "image.sif") - if err := sif.Write(path, c.ImageIndex(tb, name)); err != nil { + if err := sif.Write(path, c.ImageIndex(tb, name), opt...); err != nil { tb.Fatalf("failed to write SIF: %v", err) }