From 2c3cf11fde5910839092199ef99389401a545625 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 5 Mar 2020 11:10:21 -0800 Subject: [PATCH] resolver: update to new registryhosts based config Signed-off-by: Tonis Tiigi --- cache/remotecache/registry/registry.go | 35 +------ cmd/buildkitd/config/config.go | 1 + cmd/buildkitd/main.go | 6 +- cmd/buildkitd/main_containerd_worker.go | 2 +- cmd/buildkitd/main_oci_worker.go | 2 +- exporter/containerimage/export.go | 6 +- source/containerimage/pull.go | 8 +- util/pull/resolver.go | 33 +------ util/push/push.go | 29 +----- util/resolver/resolver.go | 122 ++++++++++++++++++------ util/tracing/tracing.go | 6 ++ worker/base/worker.go | 35 ++++--- 12 files changed, 137 insertions(+), 148 deletions(-) diff --git a/cache/remotecache/registry/registry.go b/cache/remotecache/registry/registry.go index 51e97c783952..a172917a5056 100644 --- a/cache/remotecache/registry/registry.go +++ b/cache/remotecache/registry/registry.go @@ -2,15 +2,12 @@ package registry import ( "context" - "time" "github.com/containerd/containerd/content" - "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/docker/distribution/reference" "github.com/moby/buildkit/cache/remotecache" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/auth" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/resolver" "github.com/opencontainers/go-digest" @@ -34,13 +31,13 @@ const ( attrRef = "ref" ) -func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheExporterFunc { +func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) remotecache.ResolveCacheExporterFunc { return func(ctx context.Context, attrs map[string]string) (remotecache.Exporter, error) { ref, err := canonicalizeRef(attrs[attrRef]) if err != nil { return nil, err } - remote := newRemoteResolver(ctx, resolverOpt, sm, ref) + remote := resolver.New(ctx, hosts, sm) pusher, err := remote.Pusher(ctx, ref) if err != nil { return nil, err @@ -49,13 +46,13 @@ func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveO } } -func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc { +func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, hosts docker.RegistryHosts) remotecache.ResolveCacheImporterFunc { return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) { ref, err := canonicalizeRef(attrs[attrRef]) if err != nil { return nil, specs.Descriptor{}, err } - remote := newRemoteResolver(ctx, resolverOpt, sm, ref) + remote := resolver.New(ctx, hosts, sm) xref, desc, err := remote.Resolve(ctx, ref) if err != nil { return nil, specs.Descriptor{}, err @@ -97,27 +94,3 @@ func (dsl *withDistributionSourceLabel) SetDistributionSourceAnnotation(desc oci desc.Annotations["containerd.io/distribution.source.ref"] = dsl.ref return desc } - -func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver { - opt := resolverOpt(ref) - opt.Credentials = getCredentialsFunc(ctx, sm) - return docker.NewResolver(opt) -} - -func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) { - id := session.FromContext(ctx) - if id == "" { - return nil - } - return func(host string) (string, string, error) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - caller, err := sm.Get(timeoutCtx, id) - if err != nil { - return "", "", err - } - - return auth.CredentialsFunc(context.TODO(), caller)(host) - } -} diff --git a/cmd/buildkitd/config/config.go b/cmd/buildkitd/config/config.go index 97ba3f5b2ab6..6576ad257746 100644 --- a/cmd/buildkitd/config/config.go +++ b/cmd/buildkitd/config/config.go @@ -44,6 +44,7 @@ type GRPCConfig struct { type RegistryConfig struct { Mirrors []string `toml:"mirrors"` PlainHTTP *bool `toml:"http"` + Insecure *bool `toml:"insecure"` } type TLSConfig struct { diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index f9185ef3aa6c..da7cbcd5f5bf 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -18,6 +18,7 @@ import ( "github.com/BurntSushi/toml" "github.com/containerd/containerd/pkg/seed" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/sys" sddaemon "github.com/coreos/go-systemd/v22/daemon" "github.com/docker/docker/pkg/reexec" @@ -619,15 +620,16 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err }) } -func resolverFunc(cfg *config.Config) resolver.ResolveOptionsFunc { +func resolverFunc(cfg *config.Config) docker.RegistryHosts { m := map[string]resolver.RegistryConf{} for k, v := range cfg.Registries { m[k] = resolver.RegistryConf{ Mirrors: v.Mirrors, PlainHTTP: v.PlainHTTP, + Insecure: v.Insecure, } } - return resolver.NewResolveOptionsFunc(m) + return resolver.NewRegistryConfig(m) } func newWorkerController(c *cli.Context, wiOpt workerInitializerOpt) (*worker.Controller, error) { diff --git a/cmd/buildkitd/main_containerd_worker.go b/cmd/buildkitd/main_containerd_worker.go index cf72409b75c1..c578e0e7d15d 100644 --- a/cmd/buildkitd/main_containerd_worker.go +++ b/cmd/buildkitd/main_containerd_worker.go @@ -215,7 +215,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ return nil, err } opt.GCPolicy = getGCPolicy(cfg.GCConfig, common.config.Root) - opt.ResolveOptionsFunc = resolverFunc(common.config) + opt.RegistryHosts = resolverFunc(common.config) if platformsStr := cfg.Platforms; len(platformsStr) != 0 { platforms, err := parsePlatforms(platformsStr) diff --git a/cmd/buildkitd/main_oci_worker.go b/cmd/buildkitd/main_oci_worker.go index 45f96d3d69fd..7710dc8d1d5b 100644 --- a/cmd/buildkitd/main_oci_worker.go +++ b/cmd/buildkitd/main_oci_worker.go @@ -246,7 +246,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker return nil, err } opt.GCPolicy = getGCPolicy(cfg.GCConfig, common.config.Root) - opt.ResolveOptionsFunc = resolverFunc(common.config) + opt.RegistryHosts = resolverFunc(common.config) if platformsStr := cfg.Platforms; len(platformsStr) != 0 { platforms, err := parsePlatforms(platformsStr) diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 30a68975fdb9..c803c0edf590 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -12,6 +12,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/rootfs" "github.com/moby/buildkit/cache/blobs" "github.com/moby/buildkit/exporter" @@ -19,7 +20,6 @@ import ( "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/push" - "github.com/moby/buildkit/util/resolver" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -42,7 +42,7 @@ type Opt struct { SessionManager *session.Manager ImageWriter *ImageWriter Images images.Store - ResolverOpt resolver.ResolveOptionsFunc + RegistryHosts docker.RegistryHosts LeaseManager leases.Manager } @@ -237,7 +237,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) } } if e.push { - if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil { + if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest); err != nil { return nil, err } } diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index d665ccc836fc..1fd08fc662b7 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -11,6 +11,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/remotes/docker" "github.com/docker/distribution/reference" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client/llb" @@ -22,7 +23,6 @@ import ( "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/pull" - "github.com/moby/buildkit/util/resolver" "github.com/moby/buildkit/util/winlayers" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" @@ -39,7 +39,7 @@ type SourceOpt struct { Applier diff.Applier CacheAccessor cache.Accessor ImageStore images.Store // optional - ResolverOpt resolver.ResolveOptionsFunc + RegistryHosts docker.RegistryHosts LeaseManager leases.Manager } @@ -76,7 +76,7 @@ func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt l } res, err := is.g.Do(ctx, key, func(ctx context.Context) (interface{}, error) { - dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.ResolverOpt, sm, is.ImageStore, rm, ref), is.ContentStore, is.LeaseManager, opt.Platform) + dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.RegistryHosts, sm, is.ImageStore, rm, ref), is.ContentStore, is.LeaseManager, opt.Platform) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier, sm *se ContentStore: is.ContentStore, Applier: is.Applier, Src: imageIdentifier.Reference, - Resolver: pull.NewResolver(ctx, is.ResolverOpt, sm, is.ImageStore, imageIdentifier.ResolveMode, imageIdentifier.Reference.String()), + Resolver: pull.NewResolver(ctx, is.RegistryHosts, sm, is.ImageStore, imageIdentifier.ResolveMode, imageIdentifier.Reference.String()), Platform: &platform, } p := &puller{ diff --git a/util/pull/resolver.go b/util/pull/resolver.go index d4fed18354db..ca425c24a7f2 100644 --- a/util/pull/resolver.go +++ b/util/pull/resolver.go @@ -2,7 +2,6 @@ package pull import ( "context" - "net/http" "sync" "sync/atomic" "time" @@ -12,10 +11,8 @@ import ( "github.com/containerd/containerd/remotes/docker" distreference "github.com/docker/distribution/reference" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/auth" "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/resolver" - "github.com/moby/buildkit/util/tracing" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -25,20 +22,12 @@ func init() { cache = newResolverCache() } -func NewResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, sm *session.Manager, imageStore images.Store, mode source.ResolveMode, ref string) remotes.Resolver { +func NewResolver(ctx context.Context, hosts docker.RegistryHosts, sm *session.Manager, imageStore images.Store, mode source.ResolveMode, ref string) remotes.Resolver { if res := cache.Get(ctx, ref); res != nil { return withLocal(res, imageStore, mode) } - opt := docker.ResolverOptions{ - Client: http.DefaultClient, - } - if rfn != nil { - opt = rfn(ref) - } - opt.Credentials = getCredentialsFromSession(ctx, sm) - - r := docker.NewResolver(opt) + r := resolver.New(ctx, hosts, sm) r = cache.Add(ctx, ref, r) return withLocal(r, imageStore, mode) @@ -70,24 +59,6 @@ func withLocal(r remotes.Resolver, imageStore images.Store, mode source.ResolveM return withLocalResolver{Resolver: r, is: imageStore, mode: mode} } -func getCredentialsFromSession(ctx context.Context, sm *session.Manager) func(string) (string, string, error) { - id := session.FromContext(ctx) - if id == "" { - return nil - } - return func(host string) (string, string, error) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - caller, err := sm.Get(timeoutCtx, id) - if err != nil { - return "", "", err - } - - return auth.CredentialsFunc(tracing.ContextWithSpanFromContext(context.TODO(), ctx), caller)(host) - } -} - // A remotes.Resolver which checks the local image store if the real // resolver cannot find the image, essentially falling back to a local // image if one is present. diff --git a/util/push/push.go b/util/push/push.go index 877726efc480..428010c39d71 100644 --- a/util/push/push.go +++ b/util/push/push.go @@ -14,7 +14,6 @@ import ( "github.com/containerd/containerd/remotes/docker" "github.com/docker/distribution/reference" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/auth" "github.com/moby/buildkit/util/imageutil" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/resolver" @@ -24,25 +23,7 @@ import ( "github.com/sirupsen/logrus" ) -func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) { - id := session.FromContext(ctx) - if id == "" { - return nil - } - return func(host string) (string, string, error) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - caller, err := sm.Get(timeoutCtx, id) - if err != nil { - return "", "", err - } - - return auth.CredentialsFunc(context.TODO(), caller)(host) - } -} - -func Push(ctx context.Context, sm *session.Manager, cs content.Store, dgst digest.Digest, ref string, insecure bool, rfn resolver.ResolveOptionsFunc, byDigest bool) error { +func Push(ctx context.Context, sm *session.Manager, cs content.Store, dgst digest.Digest, ref string, insecure bool, hosts docker.RegistryHosts, byDigest bool) error { desc := ocispec.Descriptor{ Digest: dgst, } @@ -60,13 +41,7 @@ func Push(ctx context.Context, sm *session.Manager, cs content.Store, dgst diges ref = reference.TagNameOnly(parsed).String() } - opt := rfn(ref) - opt.Credentials = getCredentialsFunc(ctx, sm) - if insecure { - opt.PlainHTTP = insecure - } - - resolver := docker.NewResolver(opt) + resolver := resolver.New(ctx, hosts, sm) pusher, err := resolver.Pusher(ctx, ref) if err != nil { diff --git a/util/resolver/resolver.go b/util/resolver/resolver.go index 096bfc330b72..8779975cccbc 100644 --- a/util/resolver/resolver.go +++ b/util/resolver/resolver.go @@ -1,54 +1,116 @@ package resolver import ( - "math/rand" - "strings" + "context" + "crypto/tls" + "net/http" + "time" + "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" - "github.com/docker/distribution/reference" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/auth" "github.com/moby/buildkit/util/tracing" ) type RegistryConf struct { Mirrors []string PlainHTTP *bool + Insecure *bool } -type ResolveOptionsFunc func(string) docker.ResolverOptions - -func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc { - return func(ref string) docker.ResolverOptions { - def := docker.ResolverOptions{ - Client: tracing.DefaultClient, +func fillInsecureOpts(host string, c RegistryConf, h *docker.RegistryHost) { + if c.PlainHTTP != nil && *c.PlainHTTP { + h.Scheme = "http" + } else if c.Insecure != nil && *c.Insecure { + h.Client = &http.Client{ + Transport: tracing.NewTransport(&http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}), } - - parsed, err := reference.ParseNormalizedNamed(ref) - if err != nil { - return def + } else if c.PlainHTTP == nil { + if ok, _ := docker.MatchLocalhost(host); ok { + h.Scheme = "http" } - host := reference.Domain(parsed) + } +} - c, ok := m[host] - if !ok { - return def - } +func NewRegistryConfig(m map[string]RegistryConf) docker.RegistryHosts { + return docker.Registries( + func(host string) ([]docker.RegistryHost, error) { + c, ok := m[host] + if !ok { + return nil, nil + } - var mirrorHost string - if len(c.Mirrors) > 0 { - mirrorHost = c.Mirrors[rand.Intn(len(c.Mirrors))] - def.Host = func(string) (string, error) { - return mirrorHost, nil + var out []docker.RegistryHost + + for _, mirror := range c.Mirrors { + h := docker.RegistryHost{ + Scheme: "https", + Client: tracing.DefaultClient, + Host: mirror, + Path: "/v2", + Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve, + } + fillInsecureOpts(mirror, m[mirror], &h) + + out = append(out, h) } - } - if c.PlainHTTP != nil { - def.PlainHTTP = *c.PlainHTTP - } else { - if mirrorHost == "localhost" || strings.HasPrefix(mirrorHost, "localhost:") { - def.PlainHTTP = true + if host == "docker.io" { + host = "registry-1.docker.io" } + + h := docker.RegistryHost{ + Scheme: "https", + Client: tracing.DefaultClient, + Host: host, + Path: "/v2", + Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve, + } + fillInsecureOpts(host, c, &h) + + out = append(out, h) + return out, nil + }, + docker.ConfigureDefaultRegistries(docker.WithClient(tracing.DefaultClient), docker.WithPlainHTTP(docker.MatchLocalhost)), + ) +} + +func New(ctx context.Context, hosts docker.RegistryHosts, sm *session.Manager) remotes.Resolver { + return docker.NewResolver(docker.ResolverOptions{ + Hosts: hostsWithCredentials(ctx, hosts, sm), + }) +} + +func hostsWithCredentials(ctx context.Context, hosts docker.RegistryHosts, sm *session.Manager) docker.RegistryHosts { + id := session.FromContext(ctx) + if id == "" { + return hosts + } + return func(domain string) ([]docker.RegistryHost, error) { + res, err := hosts(domain) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, nil + } + + timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + caller, err := sm.Get(timeoutCtx, id) + if err != nil { + return nil, err } - return def + a := docker.NewDockerAuthorizer( + docker.WithAuthClient(res[0].Client), + docker.WithAuthCreds(auth.CredentialsFunc(context.TODO(), caller)), + ) + for i := range res { + res[i].Authorizer = a + } + return res, nil } } diff --git a/util/tracing/tracing.go b/util/tracing/tracing.go index 6af2b8c55e39..8f2d4dd96213 100644 --- a/util/tracing/tracing.go +++ b/util/tracing/tracing.go @@ -74,6 +74,12 @@ type Transport struct { http.RoundTripper } +func NewTransport(rt http.RoundTripper) http.RoundTripper { + return &Transport{ + RoundTripper: &nethttp.Transport{RoundTripper: rt}, + } +} + func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { span := opentracing.SpanFromContext(req.Context()) if span == nil { // no tracer connected with either request or transport diff --git a/worker/base/worker.go b/worker/base/worker.go index b4c05b485157..96926bbe433a 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -46,7 +46,6 @@ import ( "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/progress" - "github.com/moby/buildkit/util/resolver" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -64,21 +63,21 @@ const labelCreatedAt = "buildkit/createdat" // WorkerOpt is specific to a worker. // See also CommonOpt. type WorkerOpt struct { - ID string - Labels map[string]string - Platforms []specs.Platform - GCPolicy []client.PruneInfo - MetadataStore *metadata.Store - Executor executor.Executor - Snapshotter snapshot.Snapshotter - ContentStore content.Store - Applier diff.Applier - Differ diff.Comparer - ImageStore images.Store // optional - ResolveOptionsFunc resolver.ResolveOptionsFunc - IdentityMapping *idtools.IdentityMapping - LeaseManager leases.Manager - GarbageCollect func(context.Context) (gc.Stats, error) + ID string + Labels map[string]string + Platforms []specs.Platform + GCPolicy []client.PruneInfo + MetadataStore *metadata.Store + Executor executor.Executor + Snapshotter snapshot.Snapshotter + ContentStore content.Store + Applier diff.Applier + Differ diff.Comparer + ImageStore images.Store // optional + RegistryHosts docker.RegistryHosts + IdentityMapping *idtools.IdentityMapping + LeaseManager leases.Manager + GarbageCollect func(context.Context) (gc.Stats, error) } // Worker is a local worker instance with dedicated snapshotter, cache, and so on. @@ -122,7 +121,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) { Applier: opt.Applier, ImageStore: opt.ImageStore, CacheAccessor: cm, - ResolverOpt: opt.ResolveOptionsFunc, + RegistryHosts: opt.RegistryHosts, LeaseManager: opt.LeaseManager, }) if err != nil { @@ -325,7 +324,7 @@ func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter, Images: w.ImageStore, SessionManager: sm, ImageWriter: w.imageWriter, - ResolverOpt: w.ResolveOptionsFunc, + RegistryHosts: w.RegistryHosts, LeaseManager: w.LeaseManager, }) case client.ExporterLocal: