From 326a7a899bd20408f8bd9b617e0d598142302fc9 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Tue, 14 May 2024 14:48:22 +0300 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=EF=B8=8F=20add=20container-proxy=20fl?= =?UTF-8?q?ag=20for=20scanning=20containers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ivan Milchev --- providers/os/config/config.go | 12 ++++++ .../connection/container/image_connection.go | 8 +++- .../container/registry_connection.go | 7 +++- providers/os/connection/shared/shared.go | 2 + providers/os/provider/provider.go | 14 +++++-- .../discovery/container_registry/registry.go | 38 +++++++++++++++++++ .../discovery/container_registry/resolver.go | 22 +++++------ 7 files changed, 86 insertions(+), 17 deletions(-) diff --git a/providers/os/config/config.go b/providers/os/config/config.go index 73b87c4413..e41be8269f 100644 --- a/providers/os/config/config.go +++ b/providers/os/config/config.go @@ -198,6 +198,12 @@ var Config = plugin.Provider{ Default: "false", Desc: "Disable the in-memory cache for images. WARNING: This will slow down scans significantly.", }, + { + Long: "container-proxy", + Type: plugin.FlagType_String, + Default: "", + Desc: "HTTP proxy to use for container pulls", + }, }, }, { @@ -231,6 +237,12 @@ var Config = plugin.Provider{ Default: "false", Desc: "Disable the in-memory cache for images. WARNING: This will slow down scans significantly.", }, + { + Long: "container-proxy", + Type: plugin.FlagType_String, + Default: "", + Desc: "HTTP proxy to use for container pulls", + }, }, }, { diff --git a/providers/os/connection/container/image_connection.go b/providers/os/connection/container/image_connection.go index f399e245ee..75738e6377 100644 --- a/providers/os/connection/container/image_connection.go +++ b/providers/os/connection/container/image_connection.go @@ -11,7 +11,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory" @@ -20,6 +19,7 @@ import ( "go.mondoo.com/cnquery/v11/providers/os/connection/container/image" "go.mondoo.com/cnquery/v11/providers/os/connection/tar" "go.mondoo.com/cnquery/v11/providers/os/id/containerid" + "go.mondoo.com/cnquery/v11/providers/os/resources/discovery/container_registry" ) // NewImageConnection uses a container image reference as input and creates a tar connection @@ -63,7 +63,11 @@ func NewRegistryImage(id uint32, conf *inventory.Config, asset *inventory.Asset) } log.Debug().Str("ref", ref.Name()).Msg("found valid container registry reference") - registryOpts := []remote.Option{auth.TransportOption(conf.Insecure), auth.AuthOption(ref.Name(), conf.Credentials)} + registryOpts, err := container_registry.RemoteOptionsFromConfigOptions(conf) + if err != nil { + return nil, err + } + registryOpts = append(registryOpts, auth.AuthOption(ref.Name(), conf.Credentials)) img, err := image.LoadImageFromRegistry(ref, registryOpts...) if err != nil { return nil, err diff --git a/providers/os/connection/container/registry_connection.go b/providers/os/connection/container/registry_connection.go index 2191b213b3..6b98ffa45f 100644 --- a/providers/os/connection/container/registry_connection.go +++ b/providers/os/connection/container/registry_connection.go @@ -74,7 +74,12 @@ func (r *RegistryConnection) Asset() *inventory.Asset { } func (r *RegistryConnection) DiscoverImages() (*inventory.Inventory, error) { - resolver := container_registry.NewContainerRegistryResolver() + opts, err := container_registry.RemoteOptionsFromConfigOptions(r.asset.Connections[0]) + if err != nil { + return nil, err + } + + resolver := container_registry.NewContainerRegistryResolver(opts...) host := r.asset.Connections[0].Host assets, err := resolver.ListRegistry(host) if err != nil { diff --git a/providers/os/connection/shared/shared.go b/providers/os/connection/shared/shared.go index 87048a657f..d8f3f7aa08 100644 --- a/providers/os/connection/shared/shared.go +++ b/providers/os/connection/shared/shared.go @@ -42,6 +42,8 @@ const ( Type_DockerSnapshot ConnectionType = "docker-snapshot" Type_ContainerRegistry ConnectionType = "container-registry" Type_RegistryImage ConnectionType = "registry-image" + + ContainerProxyOption string = "container-proxy" ) type Connection interface { diff --git a/providers/os/provider/provider.go b/providers/os/provider/provider.go index 784124b6bd..986d49be08 100644 --- a/providers/os/provider/provider.go +++ b/providers/os/provider/provider.go @@ -201,13 +201,21 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error) asset.IdDetector = []string{idDetector} } + if conf.Options == nil { + conf.Options = map[string]string{} + } + if disableCache, ok := flags["disable-cache"]; ok { - if conf.Options == nil { - conf.Options = map[string]string{} - } conf.Options["disable-cache"] = strconv.FormatBool(disableCache.RawData().Value.(bool)) } + if containerProxy, ok := flags[shared.ContainerProxyOption]; ok { + proxyVal := containerProxy.RawData().Value.(string) + if proxyVal != "" { + conf.Options[shared.ContainerProxyOption] = proxyVal + } + } + res := plugin.ParseCLIRes{ Asset: asset, } diff --git a/providers/os/resources/discovery/container_registry/registry.go b/providers/os/resources/discovery/container_registry/registry.go index 17dc5671cd..083cfe4563 100644 --- a/providers/os/resources/discovery/container_registry/registry.go +++ b/providers/os/resources/discovery/container_registry/registry.go @@ -4,8 +4,12 @@ package container_registry import ( + "crypto/tls" "fmt" + "net" + "net/http" "net/url" + "time" "github.com/cockroachdb/errors" "github.com/google/go-containerregistry/pkg/name" @@ -19,6 +23,40 @@ import ( "go.mondoo.com/cnquery/v11/providers/os/id/containerid" ) +func RemoteOptionsFromConfigOptions(cfg *inventory.Config) ([]remote.Option, error) { + opts := []remote.Option{} + + transport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + // We usually are dealing with 2 hosts (at most), split MaxIdleConns between them. + MaxIdleConnsPerHost: 50, + } + if containerProxy, ok := cfg.Options[shared.ContainerProxyOption]; ok { + urlParsed, err := url.Parse(containerProxy) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(urlParsed) + + } + + if cfg.Insecure { + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + opts = append(opts, remote.WithTransport(transport)) + return opts, nil +} + func NewContainerRegistryResolver(opts ...remote.Option) *DockerRegistryImages { return &DockerRegistryImages{ opts: opts, diff --git a/providers/os/resources/discovery/container_registry/resolver.go b/providers/os/resources/discovery/container_registry/resolver.go index 9713a09b5e..6cb923e52b 100644 --- a/providers/os/resources/discovery/container_registry/resolver.go +++ b/providers/os/resources/discovery/container_registry/resolver.go @@ -8,7 +8,6 @@ import ( "errors" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory" "go.mondoo.com/cnquery/v11/providers-sdk/v1/vault" @@ -33,26 +32,27 @@ func (r *Resolver) AvailableDiscoveryTargets() []string { func (r *Resolver) Resolve(ctx context.Context, root *inventory.Asset, conf *inventory.Config, credsResolver vault.Resolver) ([]*inventory.Asset, error) { resolved := []*inventory.Asset{} - imageFetcher := NewContainerRegistryResolver() + opts, err := RemoteOptionsFromConfigOptions(conf) + if err != nil { + return nil, err + } + + imageFetcher := NewContainerRegistryResolver(opts...) // check if the reference is an image // NOTE: we use strict validation here otherwise urls like cr://index.docker.io/mondoo/client are converted // to index.docker.io/mondoo/client:latest - opts := name.StrictValidation + nameOpts := name.StrictValidation if r.NoStrictValidation { - opts = name.WeakValidation + nameOpts = name.WeakValidation } - ref, err := name.ParseReference(conf.Host, opts) + ref, err := name.ParseReference(conf.Host, nameOpts) if err == nil { log.Debug().Str("image", conf.Host).Msg("detected container image in container registry") - remoteOpts := []remote.Option{ - auth.AuthOption(ref.Name(), conf.Credentials), - // to support self-signed certs - auth.TransportOption(conf.Insecure), - } - a, err := imageFetcher.GetImage(ref, conf.Credentials, remoteOpts...) + opts = append(opts, auth.AuthOption(ref.Name(), conf.Credentials)) + a, err := imageFetcher.GetImage(ref, conf.Credentials, opts...) if err != nil { return nil, err }