From 034c35f1fbc60b92733eb9b97cf2f6c405731a70 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 28 Jul 2020 17:08:10 +0200 Subject: [PATCH] vendor: buildkit 4d1f260e8490ec438ab66e08bb105577aca0ce06 full diff: https://github.com/moby/buildkit/compare/df35e9818d1f9066e616e03f4b8d727c97562e5b...4d1f260e8490ec438ab66e08bb105577aca0ce06 - moby/buildkit#1551 session: track sessions with a group construct - moby/buildkit#1534 secrets: allow providing secrets with env - moby/buildkit#1533 git: support for token authentication - moby/buildkit#1549 progressui: fix logs time formatting Signed-off-by: Sebastiaan van Stijn --- cli/command/image/build_buildkit.go | 18 ++-- vendor.conf | 2 +- .../moby/buildkit/client/llb/source.go | 29 +++++- .../frontend/gateway/grpcclient/client.go | 2 +- .../moby/buildkit/session/auth/auth.go | 25 ++++-- .../moby/buildkit/session/context.go | 22 ----- .../github.com/moby/buildkit/session/group.go | 88 +++++++++++++++++++ .../session/secrets/secretsprovider/file.go | 54 ------------ .../session/secrets/secretsprovider/store.go | 86 ++++++++++++++++++ .../moby/buildkit/solver/pb/attr.go | 2 + .../moby/buildkit/solver/pb/caps.go | 13 ++- .../util/progress/progressui/display.go | 9 +- 12 files changed, 253 insertions(+), 97 deletions(-) delete mode 100644 vendor/github.com/moby/buildkit/session/context.go create mode 100644 vendor/github.com/moby/buildkit/session/group.go delete mode 100644 vendor/github.com/moby/buildkit/session/secrets/secretsprovider/file.go create mode 100644 vendor/github.com/moby/buildkit/session/secrets/secretsprovider/store.go diff --git a/cli/command/image/build_buildkit.go b/cli/command/image/build_buildkit.go index d6054f356af3..2465edf5217d 100644 --- a/cli/command/image/build_buildkit.go +++ b/cli/command/image/build_buildkit.go @@ -424,7 +424,7 @@ func (t *tracer) write(msg jsonmessage.JSONMessage) { } func parseSecretSpecs(sl []string) (session.Attachable, error) { - fs := make([]secretsprovider.FileSource, 0, len(sl)) + fs := make([]secretsprovider.Source, 0, len(sl)) for _, v := range sl { s, err := parseSecret(v) if err != nil { @@ -432,22 +432,23 @@ func parseSecretSpecs(sl []string) (session.Attachable, error) { } fs = append(fs, *s) } - store, err := secretsprovider.NewFileStore(fs) + store, err := secretsprovider.NewStore(fs) if err != nil { return nil, err } return secretsprovider.NewSecretProvider(store), nil } -func parseSecret(value string) (*secretsprovider.FileSource, error) { +func parseSecret(value string) (*secretsprovider.Source, error) { csvReader := csv.NewReader(strings.NewReader(value)) fields, err := csvReader.Read() if err != nil { return nil, errors.Wrap(err, "failed to parse csv secret") } - fs := secretsprovider.FileSource{} + fs := secretsprovider.Source{} + var typ string for _, field := range fields { parts := strings.SplitN(field, "=", 2) key := strings.ToLower(parts[0]) @@ -459,17 +460,24 @@ func parseSecret(value string) (*secretsprovider.FileSource, error) { value := parts[1] switch key { case "type": - if value != "file" { + if value != "file" && value != "env" { return nil, errors.Errorf("unsupported secret type %q", value) } + typ = value case "id": fs.ID = value case "source", "src": fs.FilePath = value + case "env": + fs.Env = value default: return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field) } } + if typ == "env" { + fs.Env = fs.FilePath + fs.FilePath = "" + } return &fs, nil } diff --git a/vendor.conf b/vendor.conf index 361a196351c8..ca0b2c082570 100755 --- a/vendor.conf +++ b/vendor.conf @@ -45,7 +45,7 @@ github.com/Microsoft/go-winio 6c72808b55902eae4c5943626030 github.com/Microsoft/hcsshim 5bc557dd210ff2caf615e6e22d398123de77fc11 # v0.8.9 github.com/miekg/pkcs11 210dc1e16747c5ba98a03bcbcf728c38086ea357 # v1.0.3 github.com/mitchellh/mapstructure d16e9488127408e67948eb43b6d3fbb9f222da10 # v1.3.2 -github.com/moby/buildkit df35e9818d1f9066e616e03f4b8d727c97562e5b +github.com/moby/buildkit 4d1f260e8490ec438ab66e08bb105577aca0ce06 github.com/moby/sys 6154f11e6840c0d6b0dbb23f4125a6134b3013c9 # mountinfo/v0.1.3 github.com/moby/term 73f35e472e8f0a3f91347164138ce6bd73b756a9 github.com/modern-go/concurrent bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 # 1.0.3 diff --git a/vendor/github.com/moby/buildkit/client/llb/source.go b/vendor/github.com/moby/buildkit/client/llb/source.go index fd2f44f2697e..51fdcf923598 100644 --- a/vendor/github.com/moby/buildkit/client/llb/source.go +++ b/vendor/github.com/moby/buildkit/client/llb/source.go @@ -215,7 +215,10 @@ func Git(remote, ref string, opts ...GitOption) State { id += "#" + ref } - gi := &GitInfo{} + gi := &GitInfo{ + AuthHeaderSecret: "GIT_AUTH_HEADER", + AuthTokenSecret: "GIT_AUTH_TOKEN", + } for _, o := range opts { o.SetGitOption(gi) } @@ -228,6 +231,14 @@ func Git(remote, ref string, opts ...GitOption) State { attrs[pb.AttrFullRemoteURL] = url addCap(&gi.Constraints, pb.CapSourceGitFullURL) } + if gi.AuthTokenSecret != "" { + attrs[pb.AttrAuthTokenSecret] = gi.AuthTokenSecret + addCap(&gi.Constraints, pb.CapSourceGitHttpAuth) + } + if gi.AuthHeaderSecret != "" { + attrs[pb.AttrAuthHeaderSecret] = gi.AuthHeaderSecret + addCap(&gi.Constraints, pb.CapSourceGitHttpAuth) + } addCap(&gi.Constraints, pb.CapSourceGit) @@ -246,7 +257,9 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) { type GitInfo struct { constraintsWrapper - KeepGitDir bool + KeepGitDir bool + AuthTokenSecret string + AuthHeaderSecret string } func KeepGitDir() GitOption { @@ -255,6 +268,18 @@ func KeepGitDir() GitOption { }) } +func AuthTokenSecret(v string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.AuthTokenSecret = v + }) +} + +func AuthHeaderSecret(v string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.AuthHeaderSecret = v + }) +} + func Scratch() State { return NewState(nil) } diff --git a/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go b/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go index 1ae9ef09c701..5fe33a1963d2 100644 --- a/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go +++ b/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go @@ -32,7 +32,7 @@ type GrpcClient interface { } func New(ctx context.Context, opts map[string]string, session, product string, c pb.LLBBridgeClient, w []client.WorkerInfo) (GrpcClient, error) { - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() resp, err := c.Ping(ctx, &pb.PingRequest{}) if err != nil { diff --git a/vendor/github.com/moby/buildkit/session/auth/auth.go b/vendor/github.com/moby/buildkit/session/auth/auth.go index d40dbce7f7af..6a65eb8dc752 100644 --- a/vendor/github.com/moby/buildkit/session/auth/auth.go +++ b/vendor/github.com/moby/buildkit/session/auth/auth.go @@ -8,19 +8,28 @@ import ( "google.golang.org/grpc/codes" ) -func CredentialsFunc(ctx context.Context, c session.Caller) func(string) (string, string, error) { +func CredentialsFunc(sm *session.Manager, g session.Group) func(string) (string, string, error) { return func(host string) (string, string, error) { - client := NewAuthClient(c.Conn()) + var user, secret string + err := sm.Any(context.TODO(), g, func(ctx context.Context, _ string, c session.Caller) error { + client := NewAuthClient(c.Conn()) - resp, err := client.Credentials(ctx, &CredentialsRequest{ - Host: host, + resp, err := client.Credentials(ctx, &CredentialsRequest{ + Host: host, + }) + if err != nil { + if grpcerrors.Code(err) == codes.Unimplemented { + return nil + } + return err + } + user = resp.Username + secret = resp.Secret + return nil }) if err != nil { - if grpcerrors.Code(err) == codes.Unimplemented { - return "", "", nil - } return "", "", err } - return resp.Username, resp.Secret, nil + return user, secret, nil } } diff --git a/vendor/github.com/moby/buildkit/session/context.go b/vendor/github.com/moby/buildkit/session/context.go deleted file mode 100644 index 31a29f0868a3..000000000000 --- a/vendor/github.com/moby/buildkit/session/context.go +++ /dev/null @@ -1,22 +0,0 @@ -package session - -import "context" - -type contextKeyT string - -var contextKey = contextKeyT("buildkit/session-id") - -func NewContext(ctx context.Context, id string) context.Context { - if id != "" { - return context.WithValue(ctx, contextKey, id) - } - return ctx -} - -func FromContext(ctx context.Context) string { - v := ctx.Value(contextKey) - if v == nil { - return "" - } - return v.(string) -} diff --git a/vendor/github.com/moby/buildkit/session/group.go b/vendor/github.com/moby/buildkit/session/group.go new file mode 100644 index 000000000000..88409bf8be36 --- /dev/null +++ b/vendor/github.com/moby/buildkit/session/group.go @@ -0,0 +1,88 @@ +package session + +import ( + "context" + "time" + + "github.com/pkg/errors" +) + +type Group interface { + SessionIterator() Iterator +} +type Iterator interface { + NextSession() string +} + +func NewGroup(ids ...string) Group { + return &group{ids: ids} +} + +type group struct { + ids []string +} + +func (g *group) SessionIterator() Iterator { + return &group{ids: g.ids} +} + +func (g *group) NextSession() string { + if len(g.ids) == 0 { + return "" + } + v := g.ids[0] + g.ids = g.ids[1:] + return v +} + +func AllSessionIDs(g Group) (out []string) { + if g == nil { + return nil + } + it := g.SessionIterator() + if it == nil { + return nil + } + for { + v := it.NextSession() + if v == "" { + return + } + out = append(out, v) + } +} + +func (sm *Manager) Any(ctx context.Context, g Group, f func(context.Context, string, Caller) error) error { + if g == nil { + return nil + } + + iter := g.SessionIterator() + if iter == nil { + return nil + } + + var lastErr error + for { + id := iter.NextSession() + if id == "" { + if lastErr != nil { + return lastErr + } + return errors.Errorf("no active sessions") + } + + timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + c, err := sm.Get(timeoutCtx, id) + if err != nil { + lastErr = err + continue + } + if err := f(ctx, id, c); err != nil { + lastErr = err + continue + } + return nil + } +} diff --git a/vendor/github.com/moby/buildkit/session/secrets/secretsprovider/file.go b/vendor/github.com/moby/buildkit/session/secrets/secretsprovider/file.go deleted file mode 100644 index 58b3a928af7b..000000000000 --- a/vendor/github.com/moby/buildkit/session/secrets/secretsprovider/file.go +++ /dev/null @@ -1,54 +0,0 @@ -package secretsprovider - -import ( - "context" - "io/ioutil" - "os" - - "github.com/moby/buildkit/session/secrets" - "github.com/pkg/errors" -) - -type FileSource struct { - ID string - FilePath string -} - -func NewFileStore(files []FileSource) (secrets.SecretStore, error) { - m := map[string]FileSource{} - for _, f := range files { - if f.ID == "" { - return nil, errors.Errorf("secret missing ID") - } - if f.FilePath == "" { - f.FilePath = f.ID - } - fi, err := os.Stat(f.FilePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to stat %s", f.FilePath) - } - if fi.Size() > MaxSecretSize { - return nil, errors.Errorf("secret %s too big. max size 500KB", f.ID) - } - m[f.ID] = f - } - return &fileStore{ - m: m, - }, nil -} - -type fileStore struct { - m map[string]FileSource -} - -func (fs *fileStore) GetSecret(ctx context.Context, id string) ([]byte, error) { - v, ok := fs.m[id] - if !ok { - return nil, errors.WithStack(secrets.ErrNotFound) - } - dt, err := ioutil.ReadFile(v.FilePath) - if err != nil { - return nil, err - } - return dt, nil -} diff --git a/vendor/github.com/moby/buildkit/session/secrets/secretsprovider/store.go b/vendor/github.com/moby/buildkit/session/secrets/secretsprovider/store.go new file mode 100644 index 000000000000..db827972f0bd --- /dev/null +++ b/vendor/github.com/moby/buildkit/session/secrets/secretsprovider/store.go @@ -0,0 +1,86 @@ +package secretsprovider + +import ( + "context" + "io/ioutil" + "os" + "runtime" + "strings" + + "github.com/moby/buildkit/session/secrets" + "github.com/pkg/errors" + "github.com/tonistiigi/units" +) + +type Source struct { + ID string + FilePath string + Env string +} + +func NewStore(files []Source) (secrets.SecretStore, error) { + m := map[string]Source{} + for _, f := range files { + if f.ID == "" { + return nil, errors.Errorf("secret missing ID") + } + if f.Env == "" && f.FilePath == "" { + if hasEnv(f.ID) { + f.Env = f.ID + } else { + f.FilePath = f.ID + } + } + if f.FilePath != "" { + fi, err := os.Stat(f.FilePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to stat %s", f.FilePath) + } + if fi.Size() > MaxSecretSize { + return nil, errors.Errorf("secret %s too big. max size %#.f", f.ID, MaxSecretSize*units.B) + } + } + m[f.ID] = f + } + return &fileStore{ + m: m, + }, nil +} + +type fileStore struct { + m map[string]Source +} + +func (fs *fileStore) GetSecret(ctx context.Context, id string) ([]byte, error) { + v, ok := fs.m[id] + if !ok { + return nil, errors.WithStack(secrets.ErrNotFound) + } + if v.Env != "" { + return []byte(os.Getenv(v.Env)), nil + } + dt, err := ioutil.ReadFile(v.FilePath) + if err != nil { + return nil, err + } + return dt, nil +} + +func hasEnv(name string) bool { + for _, entry := range os.Environ() { + idx := strings.IndexRune(entry, '=') + if idx == -1 { + continue + } + if runtime.GOOS == "windows" { + // Environment variable are case-insensitive on Windows. PaTh, path and PATH are equivalent. + if strings.EqualFold(entry[:idx], name) { + return true + } + } + if entry[:idx] == name { + return true + } + } + return false +} diff --git a/vendor/github.com/moby/buildkit/solver/pb/attr.go b/vendor/github.com/moby/buildkit/solver/pb/attr.go index 97d2971cbb00..f22d5d77b0e5 100644 --- a/vendor/github.com/moby/buildkit/solver/pb/attr.go +++ b/vendor/github.com/moby/buildkit/solver/pb/attr.go @@ -2,6 +2,8 @@ package pb const AttrKeepGitDir = "git.keepgitdir" const AttrFullRemoteURL = "git.fullurl" +const AttrAuthHeaderSecret = "git.authheadersecret" +const AttrAuthTokenSecret = "git.authtokensecret" const AttrLocalSessionID = "local.session" const AttrLocalUniqueID = "local.unique" const AttrIncludePatterns = "local.includepattern" diff --git a/vendor/github.com/moby/buildkit/solver/pb/caps.go b/vendor/github.com/moby/buildkit/solver/pb/caps.go index 93c77b3e9add..43a18176be72 100644 --- a/vendor/github.com/moby/buildkit/solver/pb/caps.go +++ b/vendor/github.com/moby/buildkit/solver/pb/caps.go @@ -19,9 +19,10 @@ const ( CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns" CapSourceLocalSharedKeyHint apicaps.CapID = "source.local.sharedkeyhint" - CapSourceGit apicaps.CapID = "source.git" - CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir" - CapSourceGitFullURL apicaps.CapID = "source.git.fullurl" + CapSourceGit apicaps.CapID = "source.git" + CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir" + CapSourceGitFullURL apicaps.CapID = "source.git.fullurl" + CapSourceGitHttpAuth apicaps.CapID = "source.git.httpauth" CapSourceHTTP apicaps.CapID = "source.http" CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum" @@ -131,6 +132,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitHttpAuth, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceHTTP, Enabled: true, diff --git a/vendor/github.com/moby/buildkit/util/progress/progressui/display.go b/vendor/github.com/moby/buildkit/util/progress/progressui/display.go index 93e5b6a211cb..885bcc41c35b 100644 --- a/vendor/github.com/moby/buildkit/util/progress/progressui/display.go +++ b/vendor/github.com/moby/buildkit/util/progress/progressui/display.go @@ -273,7 +273,14 @@ func (t *trace) update(s *client.SolveStatus, termWidth int) { if v.Started != nil { ts = l.Timestamp.Sub(*v.Started) } - v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%#.4g", ts.Seconds())[:5], dt))) + prec := 1 + sec := ts.Seconds() + if sec < 10 { + prec = 3 + } else if sec < 100 { + prec = 2 + } + v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%.[2]*[1]f", sec, prec), dt))) } i++ })