From aa4361aabbc18b7eb80d9952d1914a805ccffeaf Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Tue, 20 Feb 2024 10:32:39 -0500 Subject: [PATCH] Support editing ArtifactType, preserve it in lists Add fields to manifest.ListUpdate for adding and updating the ArtifactType field in a list entry. When copying a list or descriptor, preserve the value from the original descriptor for an entry. Signed-off-by: Nalin Dahyabhai --- copy/multiple.go | 10 ++++- internal/manifest/list.go | 9 +++-- internal/manifest/oci_index.go | 45 +++++++++++++---------- internal/manifest/oci_index_test.go | 35 ++++++++++++------ internal/manifest/testdata/oci1index.json | 10 +++++ 5 files changed, 75 insertions(+), 34 deletions(-) diff --git a/copy/multiple.go b/copy/multiple.go index f252e3476f..21c3bda02a 100644 --- a/copy/multiple.go +++ b/copy/multiple.go @@ -32,6 +32,9 @@ type instanceCopy struct { op instanceCopyKind sourceDigest digest.Digest + // Fields which we want to preserve + artifactType string + // Fields which can be used by callers when operation // is `instanceCopyCopy` copyForceCompressionFormat bool @@ -133,6 +136,7 @@ func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest. res = append(res, instanceCopy{ op: instanceCopyCopy, sourceDigest: instanceDigest, + artifactType: instanceDetails.ArtifactType, copyForceCompressionFormat: forceCompressionFormat, }) platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform) @@ -142,6 +146,7 @@ func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest. res = append(res, instanceCopy{ op: instanceCopyClone, sourceDigest: instanceDigest, + artifactType: instanceDetails.ArtifactType, cloneCompressionVariant: compressionVariant, clonePlatform: instanceDetails.ReadOnly.Platform, cloneAnnotations: maps.Clone(instanceDetails.ReadOnly.Annotations), @@ -250,7 +255,9 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte, UpdateDigest: updated.manifestDigest, UpdateSize: int64(len(updated.manifest)), UpdateCompressionAlgorithms: updated.compressionAlgorithms, - UpdateMediaType: updated.manifestMIMEType}) + UpdateMediaType: updated.manifestMIMEType, + UpdateArtifactType: instance.artifactType, + }) case instanceCopyClone: logrus.Debugf("Replicating instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList)) c.Printf("Replicating image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList)) @@ -268,6 +275,7 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte, AddDigest: updated.manifestDigest, AddSize: int64(len(updated.manifest)), AddMediaType: updated.manifestMIMEType, + AddArtifactType: instance.artifactType, AddPlatform: instance.clonePlatform, AddAnnotations: instance.cloneAnnotations, AddCompressionAlgorithms: updated.compressionAlgorithms, diff --git a/internal/manifest/list.go b/internal/manifest/list.go index 189f1a7186..cb556fb9ee 100644 --- a/internal/manifest/list.go +++ b/internal/manifest/list.go @@ -65,9 +65,10 @@ type List interface { // ListUpdate includes the fields which a List's UpdateInstances() method will modify. // This is publicly visible as c/image/manifest.ListUpdate. type ListUpdate struct { - Digest digest.Digest - Size int64 - MediaType string + Digest digest.Digest + Size int64 + MediaType string + ArtifactType string // ReadOnly fields: may be set by Instance(), ignored by UpdateInstance() ReadOnly struct { Platform *imgspecv1.Platform @@ -93,6 +94,7 @@ type ListEdit struct { UpdateDigest digest.Digest UpdateSize int64 UpdateMediaType string + UpdateArtifactType string UpdateAffectAnnotations bool UpdateAnnotations map[string]string UpdateCompressionAlgorithms []compression.Algorithm @@ -101,6 +103,7 @@ type ListEdit struct { AddDigest digest.Digest AddSize int64 AddMediaType string + AddArtifactType string AddPlatform *imgspecv1.Platform AddAnnotations map[string]string AddCompressionAlgorithms []compression.Algorithm diff --git a/internal/manifest/oci_index.go b/internal/manifest/oci_index.go index dd580165a3..caaede8a60 100644 --- a/internal/manifest/oci_index.go +++ b/internal/manifest/oci_index.go @@ -54,9 +54,10 @@ func (index *OCI1IndexPublic) Instance(instanceDigest digest.Digest) (ListUpdate for _, manifest := range index.Manifests { if manifest.Digest == instanceDigest { ret := ListUpdate{ - Digest: manifest.Digest, - Size: manifest.Size, - MediaType: manifest.MediaType, + Digest: manifest.Digest, + Size: manifest.Size, + MediaType: manifest.MediaType, + ArtifactType: manifest.ArtifactType, } ret.ReadOnly.Platform = manifest.Platform ret.ReadOnly.Annotations = manifest.Annotations @@ -73,11 +74,13 @@ func (index *OCI1IndexPublic) UpdateInstances(updates []ListUpdate) error { editInstances := []ListEdit{} for i, instance := range updates { editInstances = append(editInstances, ListEdit{ - UpdateOldDigest: index.Manifests[i].Digest, - UpdateDigest: instance.Digest, - UpdateSize: instance.Size, - UpdateMediaType: instance.MediaType, - ListOperation: ListOpUpdate}) + UpdateOldDigest: index.Manifests[i].Digest, + UpdateDigest: instance.Digest, + UpdateSize: instance.Size, + UpdateMediaType: instance.MediaType, + UpdateArtifactType: instance.ArtifactType, + ListOperation: ListOpUpdate, + }) } return index.editInstances(editInstances) } @@ -138,6 +141,7 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error { return fmt.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had no media type (was %q)", i+1, len(editInstances), index.Manifests[i].MediaType) } index.Manifests[targetIndex].MediaType = editInstance.UpdateMediaType + index.Manifests[targetIndex].ArtifactType = editInstance.UpdateArtifactType if editInstance.UpdateAnnotations != nil { updatedAnnotations = true if editInstance.UpdateAffectAnnotations { @@ -157,11 +161,13 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error { } addCompressionAnnotations(editInstance.AddCompressionAlgorithms, &annotations) addedEntries = append(addedEntries, imgspecv1.Descriptor{ - MediaType: editInstance.AddMediaType, - Size: editInstance.AddSize, - Digest: editInstance.AddDigest, - Platform: editInstance.AddPlatform, - Annotations: annotations}) + MediaType: editInstance.AddMediaType, + ArtifactType: editInstance.AddArtifactType, + Size: editInstance.AddSize, + Digest: editInstance.AddDigest, + Platform: editInstance.AddPlatform, + Annotations: annotations, + }) default: return fmt.Errorf("internal error: invalid operation: %d", editInstance.ListOperation) } @@ -299,12 +305,13 @@ func OCI1IndexPublicFromComponents(components []imgspecv1.Descriptor, annotation platform = &platformCopy } m := imgspecv1.Descriptor{ - MediaType: component.MediaType, - Size: component.Size, - Digest: component.Digest, - URLs: slices.Clone(component.URLs), - Annotations: maps.Clone(component.Annotations), - Platform: platform, + MediaType: component.MediaType, + ArtifactType: component.ArtifactType, + Size: component.Size, + Digest: component.Digest, + URLs: slices.Clone(component.URLs), + Annotations: maps.Clone(component.Annotations), + Platform: platform, } index.Manifests[i] = m } diff --git a/internal/manifest/oci_index_test.go b/internal/manifest/oci_index_test.go index 204af46330..facf29e2d2 100644 --- a/internal/manifest/oci_index_test.go +++ b/internal/manifest/oci_index_test.go @@ -62,11 +62,13 @@ func TestOCI1EditInstances(t *testing.T) { expectedDigests := list.Instances() editInstances := []ListEdit{} editInstances = append(editInstances, ListEdit{ - UpdateOldDigest: list.Instances()[0], - UpdateDigest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - UpdateSize: 32, - UpdateMediaType: "something", - ListOperation: ListOpUpdate}) + UpdateOldDigest: list.Instances()[0], + UpdateDigest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + UpdateSize: 32, + UpdateMediaType: "something", + UpdateArtifactType: "anotherthing", + ListOperation: ListOpUpdate, + }) err = list.EditInstances(editInstances) require.NoError(t, err) @@ -77,6 +79,7 @@ func TestOCI1EditInstances(t *testing.T) { instance, err := list.Instance(list.Instances()[0]) require.NoError(t, err) assert.Equal(t, "something", instance.MediaType) + assert.Equal(t, "anotherthing", instance.ArtifactType) assert.Equal(t, int64(32), instance.Size) // platform must match with what was set in `ociv1.image.index.json` for the first instance assert.Equal(t, &imgspecv1.Platform{Architecture: "ppc64le", OS: "linux", OSVersion: "", OSFeatures: []string(nil), Variant: ""}, instance.ReadOnly.Platform) @@ -98,12 +101,14 @@ func TestOCI1EditInstances(t *testing.T) { ListOperation: ListOpAdd}) // with zstd editInstances = append(editInstances, ListEdit{ - AddDigest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - AddSize: 32, - AddMediaType: "application/vnd.oci.image.manifest.v1+json", - AddPlatform: &imgspecv1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"sse4"}}, - AddAnnotations: annotations, - ListOperation: ListOpAdd}) + AddDigest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + AddSize: 32, + AddMediaType: "application/vnd.oci.image.manifest.v1+json", + AddArtifactType: "application/x-tar", + AddPlatform: &imgspecv1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"sse4"}}, + AddAnnotations: annotations, + ListOperation: ListOpAdd, + }) // with zstd but with compression, annotation must be added automatically editInstances = append(editInstances, ListEdit{ AddDigest: "sha256:hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh", @@ -138,6 +143,8 @@ func TestOCI1EditInstances(t *testing.T) { require.NoError(t, err) // Verify if annotations are preserved and correctly set in ReadOnly field. assert.Equal(t, annotations, instance.ReadOnly.Annotations) + // Verify artifactType is preserved. + assert.Equal(t, "application/x-tar", instance.ArtifactType) // Verify compression of an instance is added to the ReadOnly CompressionAlgorithmNames where compression name // is internally derived from the appropriate annotations. assert.Equal(t, []string{compressionTypes.ZstdAlgorithmName}, instance.ReadOnly.CompressionAlgorithmNames) @@ -157,6 +164,12 @@ func TestOCI1EditInstances(t *testing.T) { // Digest `ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff` should be re-ordered on update. assert.Equal(t, list.Instances(), []digest.Digest{digest.Digest("sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"), digest.Digest("sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270"), digest.Digest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), digest.Digest("sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), digest.Digest("sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), digest.Digest("sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"), digest.Digest("sha256:hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh")}) + instance, err = list.Instance(digest.Digest("sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee")) + require.NoError(t, err) + // Verify if annotations are preserved and correctly set in ReadOnly field. + assert.Equal(t, annotations, instance.ReadOnly.Annotations) + // Verify artifactType is preserved. + assert.Equal(t, "application/x-tar", instance.ArtifactType) } func TestOCI1IndexChooseInstanceByCompression(t *testing.T) { diff --git a/internal/manifest/testdata/oci1index.json b/internal/manifest/testdata/oci1index.json index a85b4d889c..98b93cee73 100644 --- a/internal/manifest/testdata/oci1index.json +++ b/internal/manifest/testdata/oci1index.json @@ -22,6 +22,16 @@ "sse4" ] } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.unknown.artifact.v1", + "size": 8221, + "digest": "sha256:615e8db5ae085b39e9cd24e5bc887d03fbd30ee4f55f5fedf9b697fafea4fbdd", + "platform": { + "architecture": "ard64", + "os": "linux" + } } ], "annotations": {