Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: support OCI artifact manifest with descriptor migration #334

Merged
merged 2 commits into from
Sep 28, 2022
Merged

feat!: support OCI artifact manifest with descriptor migration #334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c739292
refactor!: Migrate `artifactspec.Descriptor` to `ocispec.Descriptor` …
Wwwsylvia Sep 20, 2022
22db027
feat: Support OCI Artifact Manifest (#2)
Wwwsylvia Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions content/file/file_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import (

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
Expand Down Expand Up @@ -2126,17 +2125,14 @@ func TestStore_Predecessors(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down
17 changes: 16 additions & 1 deletion content/graph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) (
return nil, err
}
return index.Manifests, nil
case artifactspec.MediaTypeArtifactManifest:
case artifactspec.MediaTypeArtifactManifest: // TODO: deprecate
content, err := FetchAll(ctx, fetcher, node)
if err != nil {
return nil, err
Expand All @@ -92,6 +92,21 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) (
nodes = append(nodes, descriptor.ArtifactToOCI(blob))
}
return nodes, nil
case ocispec.MediaTypeArtifactManifest:
content, err := FetchAll(ctx, fetcher, node)
if err != nil {
return nil, err
}

var manifest ocispec.Artifact
if err := json.Unmarshal(content, &manifest); err != nil {
return nil, err
}
var nodes []ocispec.Descriptor
if manifest.Subject != nil {
nodes = append(nodes, *manifest.Subject)
}
return append(nodes, manifest.Blobs...), nil
}
return nil, nil
}
13 changes: 4 additions & 9 deletions content/memory/memory_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,11 @@ import (

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/errdef"
"oras.land/oras-go/v2/internal/cas"
"oras.land/oras-go/v2/internal/descriptor"
"oras.land/oras-go/v2/internal/resolver"
)

Expand Down Expand Up @@ -332,17 +330,14 @@ func TestStorePredecessors(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down
24 changes: 8 additions & 16 deletions content/oci/oci_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,12 @@ import (

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/memory"
"oras.land/oras-go/v2/errdef"
"oras.land/oras-go/v2/internal/cas"
"oras.land/oras-go/v2/internal/descriptor"
)

// storageTracker tracks storage API counts.
Expand Down Expand Up @@ -671,17 +669,14 @@ func TestStore_Predecessors(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down Expand Up @@ -786,17 +781,14 @@ func TestStore_ExistingStore(t *testing.T) {
}

generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down
39 changes: 15 additions & 24 deletions example_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import (
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/memory"
"oras.land/oras-go/v2/content/oci"
Expand All @@ -41,29 +40,21 @@ import (
var exampleMemoryStore oras.Target
var remoteHost string
var (
exampleManifest, _ = json.Marshal(artifactspec.Manifest{
MediaType: artifactspec.MediaTypeArtifactManifest,
exampleManifest, _ = json.Marshal(ocispec.Artifact{
MediaType: ocispec.MediaTypeArtifactManifest,
ArtifactType: "example/content"})
exampleManifestDescriptor = ocispec.Descriptor{
MediaType: artifactspec.MediaTypeArtifactManifest,
MediaType: ocispec.MediaTypeArtifactManifest,
Digest: digest.Digest(digest.FromBytes(exampleManifest)),
Size: int64(len(exampleManifest))}
exampleManifestDescriptorArtifactspec = artifactspec.Descriptor{
MediaType: exampleManifestDescriptor.MediaType,
Digest: exampleManifestDescriptor.Digest,
Size: exampleManifestDescriptor.Size}
exampleSignatureManifest, _ = json.Marshal(artifactspec.Manifest{
MediaType: artifactspec.MediaTypeArtifactManifest,
exampleSignatureManifest, _ = json.Marshal(ocispec.Artifact{
MediaType: ocispec.MediaTypeArtifactManifest,
ArtifactType: "example/signature",
Subject: &exampleManifestDescriptorArtifactspec})
Subject: &exampleManifestDescriptor})
exampleSignatureManifestDescriptor = ocispec.Descriptor{
MediaType: artifactspec.MediaTypeArtifactManifest,
MediaType: ocispec.MediaTypeArtifactManifest,
Digest: digest.FromBytes(exampleSignatureManifest),
Size: int64(len(exampleSignatureManifest))}
exampleSignatureManifestDescriptorArtifactspec = artifactspec.Descriptor{
MediaType: exampleSignatureManifestDescriptor.MediaType,
Digest: exampleSignatureManifestDescriptor.Digest,
Size: exampleSignatureManifestDescriptor.Size}
)

func pushBlob(ctx context.Context, mediaType string, blob []byte, target oras.Target) (desc ocispec.Descriptor, err error) {
Expand Down Expand Up @@ -126,15 +117,15 @@ func TestMain(m *testing.M) {
case strings.Contains(p, "/blobs/uploads/"+exampleUploadUUid) && m == "GET":
w.WriteHeader(http.StatusCreated)
case strings.Contains(p, "/manifests/"+string(exampleSignatureManifestDescriptor.Digest)):
w.Header().Set("Content-Type", artifactspec.MediaTypeArtifactManifest)
w.Header().Set("Content-Type", ocispec.MediaTypeArtifactManifest)
w.Header().Set("Docker-Content-Digest", string(exampleSignatureManifestDescriptor.Digest))
w.Header().Set("Content-Length", strconv.Itoa(len(exampleSignatureManifest)))
w.Write(exampleSignatureManifest)
case strings.Contains(p, "/manifests/latest") && m == "PUT":
w.WriteHeader(http.StatusCreated)
case strings.Contains(p, "/manifests/"+string(exampleManifestDescriptor.Digest)),
strings.Contains(p, "/manifests/latest") && m == "HEAD":
w.Header().Set("Content-Type", artifactspec.MediaTypeArtifactManifest)
w.Header().Set("Content-Type", ocispec.MediaTypeArtifactManifest)
w.Header().Set("Docker-Content-Digest", string(exampleManifestDescriptor.Digest))
w.Header().Set("Content-Length", strconv.Itoa(len(exampleManifest)))
if m == "GET" {
Expand All @@ -143,14 +134,14 @@ func TestMain(m *testing.M) {
case strings.Contains(p, "/artifacts/referrers"):
w.Header().Set("ORAS-Api-Version", "oras/1.0")
q := r.URL.Query()
var referrers []artifactspec.Descriptor
var referrers []ocispec.Descriptor
if q.Get("digest") == exampleManifestDescriptor.Digest.String() {
referrers = []artifactspec.Descriptor{exampleSignatureManifestDescriptorArtifactspec}
referrers = []ocispec.Descriptor{exampleSignatureManifestDescriptor}
} else if q.Get("digest") == exampleSignatureManifestDescriptor.Digest.String() {
referrers = []artifactspec.Descriptor{}
referrers = []ocispec.Descriptor{}
}
result := struct {
Referrers []artifactspec.Descriptor `json:"referrers"`
Referrers []ocispec.Descriptor `json:"referrers"`
}{
Referrers: referrers,
}
Expand Down Expand Up @@ -321,7 +312,7 @@ func Example_copyArtifactManifestRemoteToLocal() {
dst := memory.New()
ctx := context.Background()

exampleDigest := "sha256:f9308ac4616a808210c12d049b4eb684754a5acf2c3c8d353a9fa2b3c47c274a"
exampleDigest := "sha256:70c29a81e235dda5c2cebb8ec06eafd3cca346cbd91f15ac74cefd98681c5b3d"
descriptor, err := src.Resolve(ctx, exampleDigest)
if err != nil {
panic(err)
Expand Down Expand Up @@ -362,5 +353,5 @@ func Example_extendedCopyArtifactAndReferrersRemoteToLocal() {

fmt.Println(desc.Digest)
// Output:
// sha256:1f3e679d4fc05dca20a699ae5af5fb2b7d481d5694aff929165d1c8b0f4c8598
// sha256:f396bc4d300934a39ca28ab0d5ac8a3573336d7d63c654d783a68cd1e2057662
}
4 changes: 2 additions & 2 deletions extendedcopy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,11 +287,11 @@ func (opts *ExtendedCopyGraphOptions) FilterArtifactType(regex *regexp.Regexp) {
// findReferrersAndFilter filters the predecessors with Referrers.
func findReferrersAndFilter(rf registry.ReferrerFinder, ctx context.Context, desc ocispec.Descriptor, regex *regexp.Regexp) ([]ocispec.Descriptor, error) {
var predecessors []ocispec.Descriptor
if err := rf.Referrers(ctx, desc, "", func(referrers []artifactspec.Descriptor) error {
if err := rf.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error {
// for each page of the results, do the following:
for _, referrer := range referrers {
if regex.MatchString(referrer.ArtifactType) {
predecessors = append(predecessors, descriptor.ArtifactToOCI(referrer))
predecessors = append(predecessors, referrer)
}
}
return nil
Expand Down
54 changes: 21 additions & 33 deletions extendedcopy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,17 +68,14 @@ func TestExtendedCopy_FullCopy(t *testing.T) {
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down Expand Up @@ -169,17 +166,14 @@ func TestExtendedCopyGraph_FullCopy(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config_1")) // Blob 0
Expand Down Expand Up @@ -377,17 +371,14 @@ func TestExtendedCopyGraph_WithDepthOption(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config_1")) // Blob 0
Expand Down Expand Up @@ -512,17 +503,14 @@ func TestExtendedCopyGraph_WithFindPredecessorsOption(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config_1")) // Blob 0
Expand Down Expand Up @@ -577,7 +565,7 @@ func TestExtendedCopyGraph_WithFindPredecessorsOption(t *testing.T) {
for _, p := range predecessors {
// filter media type
switch p.MediaType {
case artifactspec.MediaTypeArtifactManifest:
case ocispec.MediaTypeArtifactManifest:
filtered = append(filtered, p)
}
}
Expand Down Expand Up @@ -1054,7 +1042,7 @@ func TestExtendedCopyGraph_FilterArtifactTypeByReferrersWithMultipleRegex(t *tes
// generate test content
var blobs [][]byte
var descs []ocispec.Descriptor
var referrerSet []artifactspec.Descriptor
var referrerSet []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
Expand All @@ -1075,7 +1063,7 @@ func TestExtendedCopyGraph_FilterArtifactTypeByReferrersWithMultipleRegex(t *tes
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
}
pushReferrers := func(desc ocispec.Descriptor, artifactType string) {
referrerSet = append(referrerSet, artifactspec.Descriptor{
referrerSet = append(referrerSet, ocispec.Descriptor{
MediaType: desc.MediaType,
ArtifactType: artifactType,
Digest: desc.Digest,
Expand Down Expand Up @@ -1132,12 +1120,12 @@ func TestExtendedCopyGraph_FilterArtifactTypeByReferrersWithMultipleRegex(t *tes
w.Write(blobs[5])
case strings.Contains(p, "referrers"):
q := r.URL.Query()
var referrers []artifactspec.Descriptor
var referrers []ocispec.Descriptor
if q.Get("digest") == descs[0].Digest.String() {
referrers = referrerSet
}
result := struct {
Referrers []artifactspec.Descriptor `json:"referrers"`
Referrers []ocispec.Descriptor `json:"referrers"`
}{
Referrers: referrers,
}
Expand Down
Loading