Merge pull request #3038 from jedevc/attestations-map-merge

Proposal: attestations ref map merge

client/client_test.go

@@ -6329,16 +6329,14 @@ func testExportAttestations(t *testing.T, sb integration.Sandbox) {
 			if err != nil {
 				return nil, err
 			}
-			res.AddAttestation(pk, &gateway.InTotoAttestation{
-				PredicateRef:  refAttest,
+			res.AddInTotoAttestation(pk, &attestation.InTotoAttestation{
 				PredicatePath: "/attestation.json",
 				PredicateType: "https://example.com/attestations/v1.0",
 				Subjects: []attestation.InTotoSubject{
 					&attestation.InTotoSubjectSelf{},
 				},
-			})
-			res.AddAttestation(pk, &gateway.InTotoAttestation{
-				PredicateRef:  refAttest,
+			}, refAttest)
+			res.AddInTotoAttestation(pk, &attestation.InTotoAttestation{
 				PredicatePath: "/attestation2.json",
 				PredicateType: "https://example.com/attestations2/v1.0",
 				Subjects: []attestation.InTotoSubject{
@@ -6347,7 +6345,7 @@ func testExportAttestations(t *testing.T, sb integration.Sandbox) {
 						Digest: []digest.Digest{successDigest},
 					},
 				},
-			})
+			}, refAttest)
 		}
 
 		dt, err := json.Marshal(expPlatforms)

exporter/containerimage/writer.go

@@ -98,8 +98,12 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, sessionI
 		return mfstDesc, nil
 	}
 
-	if len(p.Platforms) != len(inp.Refs) {
-		return nil, errors.Errorf("number of platforms does not match references %d %d", len(p.Platforms), len(inp.Refs))
+	refCount := len(p.Platforms)
+	for _, attests := range inp.Attestations {
+		refCount += len(attests)
+	}
+	if refCount != len(inp.Refs) {
+		return nil, errors.Errorf("number of required refs does not match references %d %d", refCount, len(inp.Refs))
 	}
 
 	refs := make([]cache.ImmutableRef, 0, len(inp.Refs))
@@ -166,7 +170,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, sessionI
 		labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = desc.Digest.String()
 
 		if attestations, ok := inp.Attestations[p.ID]; ok {
-			inTotos, err := ic.extractAttestations(ctx, session.NewGroup(sessionID), desc, attestations...)
+			inTotos, err := ic.extractAttestations(ctx, session.NewGroup(sessionID), desc, inp.Refs, attestations)
 			if err != nil {
 				return nil, err
 			}
@@ -244,16 +248,25 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC
 	return out, err
 }
 
-func (ic *ImageWriter) extractAttestations(ctx context.Context, s session.Group, desc *ocispecs.Descriptor, attestations ...exporter.Attestation) ([]intoto.Statement, error) {
+func (ic *ImageWriter) extractAttestations(ctx context.Context, s session.Group, desc *ocispecs.Descriptor, refs map[string]cache.ImmutableRef, attestations []attestation.Attestation) ([]intoto.Statement, error) {
 	eg, ctx := errgroup.WithContext(ctx)
 	statements := make([]intoto.Statement, len(attestations))
 
+	if len(attestations) > 0 && refs == nil {
+		return nil, errors.Errorf("no refs map provided to lookup attestation keys")
+	}
+
 	for i, att := range attestations {
 		i, att := i, att
 		eg.Go(func() error {
 			switch att := att.(type) {
-			case *exporter.InTotoAttestation:
-				mount, err := att.PredicateRef.Mount(ctx, true, s)
+			case *attestation.InTotoAttestation:
+				ref, ok := refs[att.PredicateRefKey]
+				if !ok {
+					return errors.Errorf("key %s not found in refs map", att.PredicateRefKey)
+				}
+
+				mount, err := ref.Mount(ctx, true, s)
 				if err != nil {
 					return err
 				}
@@ -264,7 +277,6 @@ func (ic *ImageWriter) extractAttestations(ctx context.Context, s session.Group,
 					return err
 				}
 				defer lm.Unmount()
-
 				predicate, err := os.ReadFile(path.Join(src, att.PredicatePath))
 				if err != nil {
 					return err

exporter/exporter.go

@@ -26,18 +26,5 @@ type Source struct {
 	Ref          cache.ImmutableRef
 	Refs         map[string]cache.ImmutableRef
 	Metadata     map[string][]byte
-	Attestations map[string][]Attestation
+	Attestations map[string][]attestation.Attestation
 }
-
-type Attestation interface {
-	isExporterAttestation()
-}
-
-type InTotoAttestation struct {
-	PredicateType string
-	PredicateRef  cache.ImmutableRef
-	PredicatePath string
-	Subjects      []attestation.InTotoSubject
-}
-
-func (a *InTotoAttestation) isExporterAttestation() {}

exporter/local/export.go

@@ -2,17 +2,20 @@ package local
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/exporter"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/progress"
+	"github.com/pkg/errors"
 	"github.com/tonistiigi/fsutil"
 	fstypes "github.com/tonistiigi/fsutil/types"
 	"golang.org/x/sync/errgroup"
@@ -60,6 +63,18 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source,
 
 	isMap := len(inp.Refs) > 0
 
+	platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey]
+	if isMap && !ok {
+		return nil, errors.Errorf("unable to export multiple refs, missing platforms mapping")
+	}
+
+	var p exptypes.Platforms
+	if ok && len(platformsBytes) > 0 {
+		if err := json.Unmarshal(platformsBytes, &p); err != nil {
+			return nil, errors.Wrapf(err, "failed to parse platforms passed to exporter")
+		}
+	}
+
 	export := func(ctx context.Context, k string, ref cache.ImmutableRef) func() error {
 		return func() error {
 			var src string
@@ -130,8 +145,12 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source,
 	eg, ctx := errgroup.WithContext(ctx)
 
 	if isMap {
-		for k, ref := range inp.Refs {
-			eg.Go(export(ctx, k, ref))
+		for _, p := range p.Platforms {
+			r, ok := inp.Refs[p.ID]
+			if !ok {
+				return nil, errors.Errorf("failed to find ref for ID %s", p.ID)
+			}
+			eg.Go(export(ctx, p.ID, r))
 		}
 	} else {
 		eg.Go(export(ctx, "", inp.Ref))

exporter/tar/export.go

@@ -2,6 +2,7 @@ package local
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 	"strconv"
 	"strings"
@@ -10,6 +11,7 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/exporter"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/snapshot"
@@ -131,12 +133,28 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source,
 		}, nil
 	}
 
+	platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey]
+	if len(inp.Refs) > 0 && !ok {
+		return nil, errors.Errorf("unable to export multiple refs, missing platforms mapping")
+	}
+
+	var p exptypes.Platforms
+	if ok && len(platformsBytes) > 0 {
+		if err := json.Unmarshal(platformsBytes, &p); err != nil {
+			return nil, errors.Wrapf(err, "failed to parse platforms passed to exporter")
+		}
+	}
+
 	var fs fsutil.FS
 
 	if len(inp.Refs) > 0 {
-		dirs := make([]fsutil.Dir, 0, len(inp.Refs))
-		for k, ref := range inp.Refs {
-			d, err := getDir(ctx, k, ref)
+		dirs := make([]fsutil.Dir, 0, len(p.Platforms))
+		for _, p := range p.Platforms {
+			r, ok := inp.Refs[p.ID]
+			if !ok {
+				return nil, errors.Errorf("failed to find ref for ID %s", p.ID)
+			}
+			d, err := getDir(ctx, p.ID, r)
 			if err != nil {
 				return nil, err
 			}

frontend/gateway/client/result.go

@@ -4,31 +4,19 @@ import (
 	"context"
 	"sync"
 
+	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/attestation"
 	"github.com/pkg/errors"
 )
 
 type BuildFunc func(context.Context, Client) (*Result, error)
 
-type Attestation interface {
-	isClientAttestation()
-}
-
-type InTotoAttestation struct {
-	PredicateType string
-	PredicateRef  Reference
-	PredicatePath string
-	Subjects      []attestation.InTotoSubject
-}
-
-func (a *InTotoAttestation) isClientAttestation() {}
-
 type Result struct {
 	mu           sync.Mutex
 	Ref          Reference
 	Refs         map[string]Reference
 	Metadata     map[string][]byte
-	Attestations map[string][]Attestation
+	Attestations map[string][]attestation.Attestation
 }
 
 func NewResult() *Result {
@@ -53,11 +41,13 @@ func (r *Result) AddRef(k string, ref Reference) {
 	r.mu.Unlock()
 }
 
-func (r *Result) AddAttestation(k string, v Attestation) {
+func (r *Result) AddInTotoAttestation(k string, v *attestation.InTotoAttestation, predicateRef Reference) {
 	r.mu.Lock()
 	if r.Attestations == nil {
-		r.Attestations = map[string][]Attestation{}
+		r.Attestations = map[string][]attestation.Attestation{}
 	}
+	v.PredicateRefKey = "attestation:" + identity.NewID()
+	r.Refs[v.PredicateRefKey] = predicateRef
 	r.Attestations[k] = append(r.Attestations[k], v)
 	r.mu.Unlock()
 }

frontend/gateway/forwarder/forward.go

@@ -18,7 +18,6 @@ import (
 	llberrdefs "github.com/moby/buildkit/solver/llbsolver/errdefs"
 	opspb "github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
-	"github.com/moby/buildkit/util/attestation"
 	"github.com/moby/buildkit/worker"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -85,18 +84,9 @@ func (c *bridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*cli
 		c.refs = append(c.refs, rr)
 		cRes.SetRef(rr)
 	}
-	for k, as := range res.Attestations {
-		for _, a := range as {
-			att, rrs, err := c.newAttestation(a, session.NewGroup(c.sid))
-			if err != nil {
-				return nil, err
-			}
-			c.refs = append(c.refs, rrs...)
-			cRes.AddAttestation(k, att)
-		}
-	}
 	c.mu.Unlock()
 	cRes.Metadata = res.Metadata
+	cRes.Attestations = res.Attestations
 
 	return cRes, nil
 }
@@ -215,43 +205,7 @@ func (c *bridgeClient) toFrontendResult(r *client.Result) (*frontend.Result, err
 		res.Ref = rr.acquireResultProxy()
 	}
 	res.Metadata = r.Metadata
-	if r.Attestations != nil {
-		res.Attestations = make(map[string][]frontend.Attestation)
-		for k, as := range r.Attestations {
-			for _, a := range as {
-				switch a := a.(type) {
-				case *client.InTotoAttestation:
-					rr, ok := a.PredicateRef.(*ref)
-					if !ok {
-						return nil, errors.Errorf("invalid reference type for forward %T", r)
-					}
-
-					subjects := make([]attestation.InTotoSubject, len(a.Subjects))
-					for i, s := range a.Subjects {
-						switch s := s.(type) {
-						case *attestation.InTotoSubjectSelf:
-							subjects[i] = &attestation.InTotoSubjectSelf{}
-						case *attestation.InTotoSubjectRaw:
-							subjects[i] = &attestation.InTotoSubjectRaw{
-								Name:   s.Name,
-								Digest: s.Digest,
-							}
-						default:
-							return nil, errors.Errorf("unknown attestation subject type %T", s)
-						}
-					}
-					res.Attestations[k] = append(res.Attestations[k], &frontend.InTotoAttestation{
-						PredicateRef:  rr.acquireResultProxy(),
-						PredicatePath: a.PredicatePath,
-						PredicateType: a.PredicateType,
-						Subjects:      subjects,
-					})
-				default:
-					return nil, errors.Errorf("unknown attestation type %T", a)
-				}
-			}
-		}
-	}
+	res.Attestations = r.Attestations
 
 	return res, nil
 }
@@ -360,39 +314,6 @@ func (c *bridgeClient) newRef(r solver.ResultProxy, s session.Group) (*ref, erro
 	return &ref{resultProxy: r, session: s, c: c}, nil
 }
 
-func (c *bridgeClient) newAttestation(a frontend.Attestation, s session.Group) (client.Attestation, []*ref, error) {
-	switch a := a.(type) {
-	case *frontend.InTotoAttestation:
-		rr, err := c.newRef(a.PredicateRef, session.NewGroup(c.sid))
-		if err != nil {
-			return nil, nil, err
-		}
-
-		subjects := make([]attestation.InTotoSubject, len(a.Subjects))
-		for i, subject := range a.Subjects {
-			switch subject := subject.(type) {
-			case *attestation.InTotoSubjectSelf:
-				subjects[i] = &attestation.InTotoSubjectSelf{}
-			case *attestation.InTotoSubjectRaw:
-				subjects[i] = &attestation.InTotoSubjectRaw{
-					Name:   subject.Name,
-					Digest: subject.Digest,
-				}
-			default:
-				return nil, nil, errors.Errorf("unknown attestation subject type %T", s)
-			}
-		}
-		return &client.InTotoAttestation{
-			PredicateType: a.PredicateType,
-			PredicateRef:  rr,
-			PredicatePath: a.PredicatePath,
-			Subjects:      subjects,
-		}, []*ref{rr}, nil
-	default:
-		return nil, nil, errors.Errorf("unknown attestation type %T", a)
-	}
-}
-
 type ref struct {
 	resultProxy       solver.ResultProxy
 	resultProxyClones []solver.ResultProxy