Skip to content

Commit

Permalink
bake: prototype for composable bake attributes
Browse files Browse the repository at this point in the history 
This allows using either the csv syntax or a map to specify certain
attributes.
  • Loading branch information
@jsternberg
jsternberg committed Oct 30, 2024
1 parent 658ed58 commit 88ff1f4
Show file tree
Hide file tree
Showing 18 changed files with 762 additions and 199 deletions.
230 changes: 151 additions & 79 deletions bake/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package bake

import (
"context"
"encoding"
"io"
"os"
"path"
Expand Down Expand Up @@ -496,7 +497,9 @@ func (c Config) loadLinks(name string, t *Target, m map[string]*Target, o map[st
if err != nil {
return err
}
t2.Outputs = []string{"type=cacheonly"}
t2.Outputs = []*buildflags.ExportEntry{
{Type: "cacheonly"},
}
t2.linked = true
m[target] = t2
}
Expand Down Expand Up @@ -695,59 +698,61 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"`

Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []string `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []*buildflags.ExportEntry `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides/AddOverrides and docs/bake-reference.md.

// linked is a private field to mark a target used as a linked one
linked bool
}

var _ hclparser.WithEvalContexts = &Target{}
var _ hclparser.WithGetName = &Target{}
var _ hclparser.WithEvalContexts = &Group{}
var _ hclparser.WithGetName = &Group{}
var (
_ hclparser.WithEvalContexts = &Target{}
_ hclparser.WithGetName = &Target{}
_ hclparser.WithEvalContexts = &Group{}
_ hclparser.WithGetName = &Group{}
)

func (t *Target) normalize() {
t.Annotations = removeDupes(t.Annotations)
t.Annotations = removeDupesStr(t.Annotations)
t.Attest = removeAttestDupes(t.Attest)
t.Tags = removeDupes(t.Tags)
t.Secrets = removeDupes(t.Secrets)
t.SSH = removeDupes(t.SSH)
t.Platforms = removeDupes(t.Platforms)
t.CacheFrom = removeDupes(t.CacheFrom)
t.CacheTo = removeDupes(t.CacheTo)
t.Tags = removeDupesStr(t.Tags)
t.Secrets = removeDupesStr(t.Secrets)
t.SSH = removeDupesStr(t.SSH)
t.Platforms = removeDupesStr(t.Platforms)
t.CacheFrom = removeDupesStr(t.CacheFrom)
t.CacheTo = removeDupesStr(t.CacheTo)
t.Outputs = removeDupes(t.Outputs)
t.NoCacheFilter = removeDupes(t.NoCacheFilter)
t.Ulimits = removeDupes(t.Ulimits)
t.NoCacheFilter = removeDupesStr(t.NoCacheFilter)
t.Ulimits = removeDupesStr(t.Ulimits)

if t.NetworkMode != nil && *t.NetworkMode == "host" {
t.Entitlements = append(t.Entitlements, "network.host")
}

t.Entitlements = removeDupes(t.Entitlements)
t.Entitlements = removeDupesStr(t.Entitlements)

for k, v := range t.Contexts {
if v == "" {
Expand Down Expand Up @@ -904,7 +909,11 @@ func (t *Target) AddOverrides(overrides map[string]Override) error {
case "platform":
t.Platforms = o.ArrValue
case "output":
t.Outputs = o.ArrValue
outputs, err := parseArrValue[buildflags.ExportEntry](o.ArrValue)
if err != nil {
return errors.Wrap(err, "invalid value for outputs")
}
t.Outputs = outputs
case "entitlements":
t.Entitlements = append(t.Entitlements, o.ArrValue...)
case "annotations":
Expand Down Expand Up @@ -1354,10 +1363,11 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
bo.CacheTo = controllerapi.CreateCaches(cacheExports)

outputs, err := buildflags.ParseExports(t.Outputs)
if err != nil {
return nil, err
outputs := make([]*controllerapi.ExportEntry, len(t.Outputs))
for i, output := range t.Outputs {
outputs[i] = output.ToPB()
}

bo.Exports, err = controllerapi.CreateExports(outputs)
if err != nil {
return nil, err
Expand Down Expand Up @@ -1403,7 +1413,38 @@ func defaultTarget() *Target {
return &Target{}
}

func removeDupes(s []string) []string {
type hasEqual[E any] interface {
Equal(other E) bool
}

func removeDupes[E hasEqual[E]](s []E) []E {
// Move backwards through the slice.
// For each element, any elements after
// the current element are unique.
// If we find our current element conflicts
// with an existing element, then we swap
// the offender with the end of the slice
// and chop it off.

// Start at the second to last element.
// The last element is always unique.
for i := len(s) - 2; i > 0; i-- {
elem := s[i]
// Check for duplicates after our current element.
for j := i + 1; j < len(s); j++ {
if elem.Equal(s[j]) {
// Found a duplicate, exchange the
// duplicate with the last element.
s[j], s[len(s)-1] = s[len(s)-1], s[j]
s = s[:len(s)-1]
break
}
}
}
return s
}

func removeDupesStr(s []string) []string {
i := 0
seen := make(map[string]struct{}, len(s))
for _, v := range s {
Expand Down Expand Up @@ -1464,61 +1505,75 @@ func parseOutputType(str string) string {
return ""
}

func setPushOverride(outputs []string, push bool) []string {
var out []string
func setPushOverride(outputs []*buildflags.ExportEntry, push bool) []*buildflags.ExportEntry {
if !push {
// Disable push for any relevant export types
for i := 0; i < len(outputs); i++ {
output := outputs[i]
if output.Type == "registry" {
// Filter out registry output type
outputs[i], outputs[len(outputs)-1] = outputs[len(outputs)-1], outputs[i]
outputs = outputs[:len(outputs)-1]
} else if output.Type == "image" {
// Override push attribute
output.Attrs["push"] = "false"
}
}
return outputs
}

// Force push to be enabled
setPush := true
for _, output := range outputs {
typ := parseOutputType(output)
if typ == "image" || typ == "registry" {
// no need to set push if image or registry types already defined
if output.Type != "docker" {
// If there is an output type that is not docker, don't set "push"
setPush = false
if typ == "registry" {
if !push {
// don't set registry output if "push" is false
continue
}
// no need to set "push" attribute to true for registry
out = append(out, output)
continue
}
out = append(out, output+",push="+strconv.FormatBool(push))
} else {
if typ != "docker" {
// if there is any output that is not docker, don't set "push"
setPush = false
}
out = append(out, output)
}

// Set push attribute for image
if output.Type == "image" {
output.Attrs["push"] = "true"
}
}
if push && setPush {
out = append(out, "type=image,push=true")

if setPush {
// No existing output that pushes so add one
outputs = append(outputs, &buildflags.ExportEntry{
Type: "image",
Attrs: map[string]string{
"push": "true",
},
})
}
return out
return outputs
}

func setLoadOverride(outputs []string, load bool) []string {
func setLoadOverride(outputs []*buildflags.ExportEntry, load bool) []*buildflags.ExportEntry {
if !load {
return outputs
}

setLoad := true
for _, output := range outputs {
if typ := parseOutputType(output); typ == "docker" {
if v := parseOutput(output); v != nil {
// dest set means we want to output as tar so don't set load
if _, ok := v["dest"]; !ok {
setLoad = false
break
}
switch output.Type {
case "docker":
// if dest is not set, we can reuse this entry and do not need to set load
if output.Destination == "" {
setLoad = false
}
} else if typ != "image" && typ != "registry" && typ != "oci" {
case "image", "registry", "oci":
// Ignore
default:
// if there is any output that is not an image, registry
// or oci, don't set "load" similar to push override
setLoad = false
break
}
}

if setLoad {
outputs = append(outputs, "type=docker")
outputs = append(outputs, &buildflags.ExportEntry{
Type: "docker",
})
}
return outputs
}
Expand Down Expand Up @@ -1558,3 +1613,20 @@ func toNamedContexts(m map[string]string) map[string]build.NamedContext {
}
return m2
}

type arrValue[B any] interface {
encoding.TextUnmarshaler
*B
}

func parseArrValue[T any, PT arrValue[T]](s []string) ([]*T, error) {
outputs := make([]*T, 0, len(s))
for _, text := range s {
output := new(T)
if err := PT(output).UnmarshalText([]byte(text)); err != nil {
return nil, err
}
outputs = append(outputs, output)
}
return outputs, nil
}
13 changes: 10 additions & 3 deletions bake/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/compose-spec/compose-go/v2/dotenv"
"github.com/compose-spec/compose-go/v2/loader"
composetypes "github.com/compose-spec/compose-go/v2/types"
"github.com/docker/buildx/util/buildflags"
dockeropts "github.com/docker/cli/opts"
"github.com/docker/go-units"
"github.com/pkg/errors"
Expand Down Expand Up @@ -292,8 +293,10 @@ type xbake struct {
// https://github.com/docker/docs/blob/main/content/build/bake/compose-file.md#extension-field-with-x-bake
}

type stringMap map[string]string
type stringArray []string
type (
stringMap map[string]string
stringArray []string
)

func (sa *stringArray) UnmarshalYAML(unmarshal func(interface{}) error) error {
var multi []string
Expand Down Expand Up @@ -345,7 +348,11 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
t.Platforms = dedupSlice(append(t.Platforms, xb.Platforms...))
}
if len(xb.Outputs) > 0 {
t.Outputs = dedupSlice(append(t.Outputs, xb.Outputs...))
outputs, err := parseArrValue[buildflags.ExportEntry](xb.Outputs)
if err != nil {
return err
}
t.Outputs = removeDupes(append(t.Outputs, outputs...))
}
if xb.Pull != nil {
t.Pull = xb.Pull
Expand Down
9 changes: 7 additions & 2 deletions bake/hclparser/hclparser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err
}

// decode!
diag = gohcl.DecodeBody(body(), ectx, output.Interface())
diag = decodeBody(body(), ectx, output.Interface())
if diag.HasErrors() {
return diag
}
Expand All @@ -470,7 +470,7 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err
}

// store the result into the evaluation context (so it can be referenced)
outputType, err := gocty.ImpliedType(output.Interface())
outputType, err := ImpliedType(output.Interface())
if err != nil {
return err
}
Expand Down Expand Up @@ -947,3 +947,8 @@ func key(ks ...any) uint64 {
}
return hash.Sum64()
}

func decodeBody(body hcl.Body, ctx *hcl.EvalContext, val interface{}) hcl.Diagnostics {
dec := gohcl.DecodeOptions{ImpliedType: ImpliedType}
return dec.DecodeBody(body, ctx, val)
}
Loading

0 comments on commit 88ff1f4

Please sign in to comment.