Skip to content

Commit

Permalink
Merge pull request #2684 from crazy-max/fix-buildinfo-deps-sources
Browse files Browse the repository at this point in the history 
buildinfo: merge build sources for deps
  • Loading branch information
@tonistiigi
tonistiigi authored Mar 9, 2022
2 parents bffd728 + 49aa39c commit 068cf68
Show file tree
Hide file tree
Showing 9 changed files with 726 additions and 134 deletions.
74 changes: 43 additions & 31 deletions frontend/dockerfile/builder/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -811,11 +812,11 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
return opts
}

func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) {
return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) {
func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
named, err := reference.ParseNormalizedNamed(name)
if err != nil {
return nil, nil, errors.Wrapf(err, "invalid context name %s", name)
return nil, nil, nil, errors.Wrapf(err, "invalid context name %s", name)
}
name = strings.TrimSuffix(reference.FamiliarString(named), ":latest")

Expand All @@ -825,28 +826,28 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte
}
if p != nil {
name := name + "::" + platforms.Format(platforms.Normalize(*p))
st, img, err := contextByName(ctx, c, name, p)
st, img, bi, err := contextByName(ctx, c, name, p)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
if st != nil {
return st, img, nil
return st, img, bi, nil
}
}
return contextByName(ctx, c, name, p)
}
}

func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) {
func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
opts := c.BuildOpts().Opts
v, ok := opts["context:"+name]
if !ok {
return nil, nil, nil
return nil, nil, nil, nil
}

vv := strings.SplitN(v, ":", 2)
if len(vv) != 2 {
return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name)
return nil, nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name)
}
switch vv[0] {
case "docker-image":
Expand All @@ -859,20 +860,20 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
imgOpt = append(imgOpt, llb.Platform(*platform))
}
st := llb.Image(ref, imgOpt...)
return &st, nil, nil
return &st, nil, nil, nil
case "git":
st, ok := detectGitContext(v, "1")
if !ok {
return nil, nil, errors.Errorf("invalid git context %s", v)
return nil, nil, nil, errors.Errorf("invalid git context %s", v)
}
return st, nil, nil
return st, nil, nil, nil
case "http", "https":
st, ok := detectGitContext(v, "1")
if !ok {
httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
st = &httpst
}
return st, nil, nil
return st, nil, nil, nil
case "local":
st := llb.Local(vv[1],
llb.SessionID(c.BuildOpts().SessionID),
Expand All @@ -883,18 +884,18 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
)
def, err := st.Marshal(ctx)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
res, err := c.Solve(ctx, client.SolveRequest{
Evaluate: true,
Definition: def.ToPB(),
})
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
dt, _ := ref.ReadFile(ctx, client.ReadRequest{
Filename: dockerignoreFilename,
Expand All @@ -903,7 +904,7 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
if len(dt) != 0 {
excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dt))
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
}
st = llb.Local(vv[1],
Expand All @@ -912,38 +913,49 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
llb.SharedKeyHint("context:"+name),
llb.ExcludePatterns(excludes),
)
return &st, nil, nil
return &st, nil, nil, nil
case "input":
inputs, err := c.Inputs(ctx)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
st, ok := inputs[vv[1]]
if !ok {
return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name)
return nil, nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name)
}
md, ok := opts["input-metadata:"+vv[1]]
if ok {
m := make(map[string][]byte)
if err := json.Unmarshal([]byte(md), &m); err != nil {
return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md)
return nil, nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md)
}
dt, ok := m["containerimage.config"]
if ok {
st, err = st.WithImageConfig([]byte(dt))
var bi *binfotypes.BuildInfo
if dtbi, ok := m[exptypes.ExporterBuildInfo]; ok {
var depbi binfotypes.BuildInfo
if err := json.Unmarshal(dtbi, &depbi); err != nil {
return nil, nil, nil, errors.Wrapf(err, "failed to parse buildinfo for %s", name)
}
bi = &binfotypes.BuildInfo{
Deps: map[string]binfotypes.BuildInfo{
strings.SplitN(vv[1], "::", 2)[0]: depbi,
},
}
}
var img *dockerfile2llb.Image
if dtic, ok := m[exptypes.ExporterImageConfigKey]; ok {
st, err = st.WithImageConfig(dtic)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
var img dockerfile2llb.Image
if err := json.Unmarshal(dt, &img); err != nil {
return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name)
if err := json.Unmarshal(dtic, &img); err != nil {
return nil, nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name)
}
return &st, &img, nil
}
return &st, img, bi, nil
}
return &st, nil, nil
return &st, nil, nil, nil
default:
return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name)
return nil, nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name)
}
}

Expand Down
50 changes: 38 additions & 12 deletions frontend/dockerfile/dockerfile2llb/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,18 +70,31 @@ type ConvertOpt struct {
SourceMap *llb.SourceMap
Hostname string
Warn func(short, url string, detail [][]byte, location *parser.Range)
ContextByName func(context.Context, string) (*llb.State, *Image, error)
ContextByName func(context.Context, string) (*llb.State, *Image, *binfotypes.BuildInfo, error)
}

func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
buildInfo := &binfotypes.BuildInfo{}
contextByName := opt.ContextByName
opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, error) {
opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
if contextByName != nil {
return contextByName(ctx, name)
st, img, bi, err := contextByName(ctx, name)
if err != nil {
return nil, nil, nil, err
}
if bi != nil && bi.Deps != nil {
for k := range bi.Deps {
if buildInfo.Deps == nil {
buildInfo.Deps = make(map[string]binfotypes.BuildInfo)
}
buildInfo.Deps[k] = bi.Deps[k]
}
}
return st, img, bi, nil
}
}
return nil, nil, nil
return nil, nil, nil, nil
}

if len(dt) == 0 {
Expand Down Expand Up @@ -152,7 +165,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}

if st.Name != "" {
s, img, err := opt.ContextByName(ctx, st.Name)
s, img, bi, err := opt.ContextByName(ctx, st.Name)
if err != nil {
return nil, nil, nil, err
}
Expand All @@ -162,6 +175,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if img != nil {
ds.image = *img
}
if bi != nil {
ds.buildInfo = *bi
}
allDispatchStates.addState(ds)
continue
}
Expand Down Expand Up @@ -291,7 +307,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
d.stage.BaseName = reference.TagNameOnly(ref).String()

var isScratch bool
st, img, err := opt.ContextByName(ctx, d.stage.BaseName)
st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName)
if err != nil {
return err
}
Expand All @@ -301,6 +317,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
} else {
d.image = emptyImage(platformOpt.targetPlatform)
}
if bi != nil {
d.buildInfo = *bi
}
d.state = *st
d.platform = platform
return nil
Expand Down Expand Up @@ -349,12 +368,12 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if !isScratch {
// if image not scratch set original image name as ref
// and actual reference as alias in binfotypes.Source
d.buildSource = &binfotypes.Source{
d.buildInfo.Sources = append(d.buildInfo.Sources, binfotypes.Source{
Type: binfotypes.SourceTypeDockerImage,
Ref: origName,
Alias: ref.String(),
Pin: dgst.String(),
}
})
}
d.image = img
}
Expand Down Expand Up @@ -382,16 +401,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,

buildContext := &mutableOutput{}
ctxPaths := map[string]struct{}{}
buildInfo := &binfotypes.BuildInfo{}

for _, d := range allDispatchStates.states {
if !isReachable(target, d) {
continue
}

// collect build sources and dependencies
if d.buildSource != nil {
buildInfo.Sources = append(buildInfo.Sources, *d.buildSource)
if len(d.buildInfo.Sources) > 0 {
buildInfo.Sources = append(buildInfo.Sources, d.buildInfo.Sources...)
}
if d.buildInfo.Deps != nil {
for name, bi := range d.buildInfo.Deps {
if buildInfo.Deps == nil {
buildInfo.Deps = make(map[string]binfotypes.BuildInfo)
}
buildInfo.Deps[name] = bi
}
}

if d.base != nil {
Expand Down Expand Up @@ -701,7 +727,7 @@ type dispatchState struct {
cmdIndex int
cmdTotal int
prefixPlatform bool
buildSource *binfotypes.Source
buildInfo binfotypes.BuildInfo
}

type dispatchStates struct {
Expand Down
Loading

0 comments on commit 068cf68

Please sign in to comment.