Skip to content

Commit

Permalink
Merge pull request #398 from tonistiigi/remote-bake
Browse files Browse the repository at this point in the history 
bake: remote inputs support
  • Loading branch information
@tonistiigi
tonistiigi authored Dec 8, 2020
2 parents 0360668 + 40fad4b commit 080e998
Show file tree
Hide file tree
Showing 19 changed files with 692 additions and 339 deletions.
119 changes: 90 additions & 29 deletions bake/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,63 @@ import (
"io/ioutil"
"os"
"path"
"regexp"
"strconv"
"strings"

"github.com/docker/buildx/build"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/docker/pkg/urlutil"
hcl "github.com/hashicorp/hcl/v2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/pkg/errors"
)

func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]*Target, error) {
var httpPrefix = regexp.MustCompile(`^https?://`)
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)

type File struct {
Name string
Data []byte
}

func defaultFilenames() []string {
return []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
}

func ReadLocalFiles(names []string) ([]File, error) {
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
out := make([]File, 0, len(names))

for _, n := range names {
dt, err := ioutil.ReadFile(n)
if err != nil {
if isDefault && errors.Is(err, os.ErrNotExist) {
continue
}
return nil, err
}
out = append(out, File{Name: n, Data: dt})
}
return out, nil
}

func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
var c Config
for _, f := range files {
cfg, err := ParseFile(f)
cfg, err := ParseFile(f.Data, f.Name)
if err != nil {
return nil, err
}
Expand All @@ -44,12 +86,7 @@ func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[s
return m, nil
}

func ParseFile(fn string) (*Config, error) {
dt, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
}

func ParseFile(dt []byte, fn string) (*Config, error) {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
return ParseCompose(dt)
Expand Down Expand Up @@ -336,20 +373,22 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"`

Context *string `json:"context,omitempty" hcl:"context,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`

// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
}

Expand All @@ -363,10 +402,10 @@ func (t *Target) normalize() {
t.Outputs = removeDupes(t.Outputs)
}

func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
m2 := make(map[string]build.Options, len(m))
for k, v := range m {
bo, err := toBuildOpt(v)
bo, err := toBuildOpt(v, inp)
if err != nil {
return nil, err
}
Expand All @@ -375,7 +414,19 @@ func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
return m2, nil
}

func toBuildOpt(t *Target) (*build.Options, error) {
func updateContext(t *build.Inputs, inp *Input) {
if inp == nil || inp.State == nil {
return
}
if t.ContextPath == "." {
t.ContextPath = inp.URL
return
}
st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath))
t.ContextState = &st
}

func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
if v := t.Context; v != nil && *v == "-" {
return nil, errors.Errorf("context from stdin not allowed in bake")
}
Expand All @@ -387,6 +438,7 @@ func toBuildOpt(t *Target) (*build.Options, error) {
if t.Context != nil {
contextPath = *t.Context
}
contextPath = path.Clean(contextPath)
dockerfilePath := "Dockerfile"
if t.Dockerfile != nil {
dockerfilePath = *t.Dockerfile
Expand All @@ -405,11 +457,17 @@ func toBuildOpt(t *Target) (*build.Options, error) {
pull = *t.Pull
}

bi := build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
}
if t.DockerfileInline != nil {
bi.DockerfileInline = *t.DockerfileInline
}
updateContext(&bi, inp)

bo := &build.Options{
Inputs: build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
},
Inputs: bi,
Tags: t.Tags,
BuildArgs: t.Args,
Labels: t.Labels,
Expand Down Expand Up @@ -473,6 +531,9 @@ func merge(t1, t2 *Target) *Target {
if t2.Dockerfile != nil {
t1.Dockerfile = t2.Dockerfile
}
if t2.DockerfileInline != nil {
t1.DockerfileInline = t2.DockerfileInline
}
for k, v := range t2.Args {
if t1.Args == nil {
t1.Args = map[string]string{}
Expand Down
61 changes: 27 additions & 34 deletions bake/bake_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,18 @@ package bake

import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"
)

func TestReadTargets(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

fp := filepath.Join(tmpdir, "config.hcl")
err = ioutil.WriteFile(fp, []byte(`
fp := File{
Name: "config.hcl",
Data: []byte(`
target "webDEP" {
args = {
VAR_INHERITED = "webDEP"
Expand All @@ -32,13 +28,13 @@ target "webapp" {
VAR_BOTH = "webapp"
}
inherits = ["webDEP"]
}`), 0600)
require.NoError(t, err)
}`),
}

ctx := context.TODO()

t.Run("NoOverrides", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, nil)
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil)
require.NoError(t, err)
require.Equal(t, 1, len(m))

Expand All @@ -50,7 +46,7 @@ target "webapp" {
})

t.Run("InvalidTargetOverrides", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
require.NotNil(t, err)
require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'")
})
Expand All @@ -60,7 +56,7 @@ target "webapp" {
os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv")
defer os.Unsetenv("VAR_FROM_ENV" + t.Name())

m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webapp.args.VAR_UNSET",
"webapp.args.VAR_EMPTY=",
"webapp.args.VAR_SET=bananas",
Expand Down Expand Up @@ -89,7 +85,7 @@ target "webapp" {

// building leaf but overriding parent fields
t.Run("parent", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webDEP.args.VAR_INHERITED=override",
"webDEP.args.VAR_BOTH=override",
})
Expand All @@ -100,23 +96,23 @@ target "webapp" {
})

t.Run("ContextOverride", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"})
require.NotNil(t, err)

m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
require.NoError(t, err)

require.Equal(t, "foo", *m["webapp"].Context)
})

t.Run("NoCacheOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].NoCache)
})

t.Run("PullOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].Pull)
})
Expand Down Expand Up @@ -176,7 +172,7 @@ target "webapp" {
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, test.targets, test.overrides)
m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides)
test.check(t, m, err)
})
}
Expand All @@ -185,14 +181,11 @@ target "webapp" {

func TestReadTargetsCompose(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

fp := filepath.Join(tmpdir, "docker-compose.yml")
err = ioutil.WriteFile(fp, []byte(`
version: "3"

fp := File{
Name: "docker-compose.yml",
Data: []byte(
`version: "3"
services:
db:
build: .
Expand All @@ -203,26 +196,26 @@ services:
dockerfile: Dockerfile.webapp
args:
buildno: 1
`), 0600)
require.NoError(t, err)

fp2 := filepath.Join(tmpdir, "docker-compose2.yml")
err = ioutil.WriteFile(fp2, []byte(`
version: "3"
`),
}

fp2 := File{
Name: "docker-compose2.yml",
Data: []byte(
`version: "3"
services:
newservice:
build: .
webapp:
build:
args:
buildno2: 12
`), 0600)
require.NoError(t, err)
`),
}

ctx := context.TODO()

m, err := ReadTargets(ctx, []string{fp, fp2}, []string{"default"}, nil)
m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil)
require.NoError(t, err)

require.Equal(t, 3, len(m))
Expand Down
32 changes: 24 additions & 8 deletions bake/hcl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,20 @@ type staticConfig struct {
}

func ParseHCL(dt []byte, fn string) (_ *Config, err error) {
if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
return parseHCL(dt, fn)
}
cfg, err := parseHCL(dt, fn+".hcl")
if err != nil {
cfg2, err2 := parseHCL(dt, fn+".json")
if err2 == nil {
return cfg2, nil
}
}
return cfg, err
}

func parseHCL(dt []byte, fn string) (_ *Config, err error) {
defer func() {
err = formatHCLError(dt, err)
}()
Expand Down Expand Up @@ -192,15 +206,17 @@ func formatHCLError(dt []byte, err error) error {
if d.Severity != hcl.DiagError {
continue
}
src := errdefs.Source{
Info: &pb.SourceInfo{
Filename: d.Subject.Filename,
Data: dt,
},
Ranges: []*pb.Range{toErrRange(d.Subject)},
if d.Subject != nil {
src := errdefs.Source{
Info: &pb.SourceInfo{
Filename: d.Subject.Filename,
Data: dt,
},
Ranges: []*pb.Range{toErrRange(d.Subject)},
}
err = errdefs.WithSource(err, src)
break
}
err = errdefs.WithSource(err, src)
break
}
return err
}
Expand Down
Loading

0 comments on commit 080e998

Please sign in to comment.