From e7c012875c11811a7a47abab51ab823d724b15cd Mon Sep 17 00:00:00 2001 From: yeya24 Date: Mon, 5 Aug 2019 15:48:28 +0800 Subject: [PATCH] support dfget flags in each proxy Signed-off-by: yeya24 --- dfdaemon/config/config.go | 36 +++++++++++++++----------- dfdaemon/config/config_test.go | 6 ++--- dfdaemon/proxy/downloader.go | 45 +++++++++++++++++++++++++++++++++ dfdaemon/proxy/proxy.go | 43 +++++++++++++++---------------- dfdaemon/proxy/proxy_test.go | 20 ++++++++++++--- dfdaemon/transport/transport.go | 44 +++++++++++++++++--------------- 6 files changed, 130 insertions(+), 64 deletions(-) create mode 100644 dfdaemon/proxy/downloader.go diff --git a/dfdaemon/config/config.go b/dfdaemon/config/config.go index 15877b330..1d592a429 100644 --- a/dfdaemon/config/config.go +++ b/dfdaemon/config/config.go @@ -79,6 +79,9 @@ type Properties struct { // Proxies is the list of rules for the transparent proxy. If no rules // are provided, all requests will be proxied directly. Request will be // proxied with the first matching rule. + // You can also define custom dfget flags for each proxy. These extra flags + // will append to global dfget flags and extra flags can overwrite global flags + // if they have same parameters. Proxies []*Proxy `yaml:"proxies" json:"proxies"` // HijackHTTPS is the list of hosts whose https requests should be hijacked @@ -97,11 +100,11 @@ type Properties struct { MaxProcs int `yaml:"maxprocs" json:"maxprocs"` // dfget config - DfgetFlags []string `yaml:"dfget_flags" json:"dfget_flags"` - SuperNodes []string `yaml:"supernodes" json:"supernodes"` - RateLimit string `yaml:"ratelimit" json:"ratelimit"` - DFRepo string `yaml:"localrepo" json:"localrepo"` - DFPath string `yaml:"dfpath" json:"dfpath"` + GlobalDfgetFlags []string `yaml:"dfget_flags" json:"dfget_flags"` + SuperNodes []string `yaml:"supernodes" json:"supernodes"` + RateLimit string `yaml:"ratelimit" json:"ratelimit"` + DFRepo string `yaml:"localrepo" json:"localrepo"` + DFPath string `yaml:"dfpath" json:"dfpath"` } // Validate validates the config @@ -138,10 +141,11 @@ func (p *Properties) Validate() error { } // DFGetConfig returns config for dfget downloader -func (p *Properties) DFGetConfig() DFGetConfig { +func (p *Properties) DFGetConfig(extraFlags []string) DFGetConfig { // init DfgetFlags - var dfgetFlags []string - dfgetFlags = append(dfgetFlags, p.DfgetFlags...) + dfgetFlags := make([]string, len(p.GlobalDfgetFlags)) + copy(dfgetFlags, p.GlobalDfgetFlags) + dfgetFlags = append(dfgetFlags, extraFlags...) dfgetFlags = append(dfgetFlags, "--dfdaemon") if p.Verbose { dfgetFlags = append(dfgetFlags, "--verbose") @@ -372,22 +376,24 @@ func certPoolFromFiles(files ...string) (*x509.CertPool, error) { // Proxy describe a regular expression matching rule for how to proxy a request type Proxy struct { - Regx *Regexp `yaml:"regx" json:"regx"` - UseHTTPS bool `yaml:"use_https" json:"use_https"` - Direct bool `yaml:"direct" json:"direct"` + Regx *Regexp `yaml:"regx" json:"regx"` + UseHTTPS bool `yaml:"use_https" json:"use_https"` + Direct bool `yaml:"direct" json:"direct"` + DfgetFlags []string `yaml:"dfget_flags" json:"dfget_flags"` } // NewProxy returns a new proxy rule with given attributes -func NewProxy(regx string, useHTTPS bool, direct bool) (*Proxy, error) { +func NewProxy(regx string, useHTTPS bool, direct bool, dfgetFlags []string) (*Proxy, error) { exp, err := NewRegexp(regx) if err != nil { return nil, errors.Wrap(err, "invalid regexp") } return &Proxy{ - Regx: exp, - UseHTTPS: useHTTPS, - Direct: direct, + Regx: exp, + UseHTTPS: useHTTPS, + Direct: direct, + DfgetFlags: dfgetFlags, }, nil } diff --git a/dfdaemon/config/config_test.go b/dfdaemon/config/config_test.go index 622bc8d9f..c7812c43f 100644 --- a/dfdaemon/config/config_test.go +++ b/dfdaemon/config/config_test.go @@ -234,7 +234,7 @@ func (ts *configTestSuite) TestProxyNew() { validRegexp := ".*" useHTTPS := false direct := false - p, err := NewProxy(validRegexp, useHTTPS, direct) + p, err := NewProxy(validRegexp, useHTTPS, direct, nil) r.Nil(err) r.NotNil(p) r.Equal(useHTTPS, p.UseHTTPS) @@ -243,7 +243,7 @@ func (ts *configTestSuite) TestProxyNew() { } { - p, err := NewProxy(`\K`, false, false) + p, err := NewProxy(`\K`, false, false, nil) r.Nil(p) r.NotNil(err) r.True(strings.HasPrefix(err.Error(), "invalid regexp:")) @@ -252,7 +252,7 @@ func (ts *configTestSuite) TestProxyNew() { func (ts *configTestSuite) TestProxyMatch() { r := ts.Require() - p, err := NewProxy("blobs/sha256.*", false, false) + p, err := NewProxy("blobs/sha256.*", false, false, nil) r.Nil(err) r.NotNil(p) diff --git a/dfdaemon/proxy/downloader.go b/dfdaemon/proxy/downloader.go new file mode 100644 index 000000000..1610f3c2e --- /dev/null +++ b/dfdaemon/proxy/downloader.go @@ -0,0 +1,45 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package proxy + +import ( + "github.com/dragonflyoss/Dragonfly/dfdaemon/config" + "github.com/dragonflyoss/Dragonfly/dfdaemon/downloader" + "github.com/dragonflyoss/Dragonfly/dfdaemon/downloader/dfget" +) + +// proxyDownloader consists of proxy rule and its downloader +type proxyDownloader struct { + *config.Proxy + downloader downloader.Interface +} + +func newProxyDownloaders(p *config.Properties) []*proxyDownloader { + pds := make([]*proxyDownloader, len(p.Proxies)) + for i, proxy := range p.Proxies { + pds[i] = &proxyDownloader{ + Proxy: proxy, + downloader: dfget.NewGetter(p.DFGetConfig(proxy.DfgetFlags)), + } + } + return pds +} + +// Download is used to implement Download interface. +func (pd *proxyDownloader) Download(url string, header map[string][]string, name string) (string, error) { + return pd.downloader.Download(url, header, name) +} diff --git a/dfdaemon/proxy/proxy.go b/dfdaemon/proxy/proxy.go index 949604657..5fa2fb559 100644 --- a/dfdaemon/proxy/proxy.go +++ b/dfdaemon/proxy/proxy.go @@ -88,9 +88,12 @@ func WithDirectHandler(h *http.ServeMux) Option { } } -// WithRules sets the proxy rules -func WithRules(rules []*config.Proxy) Option { - return func(p *Proxy) error { return p.SetRules(rules) } +// WithProxyDownloaders sets the proxy downloaders +func WithProxyDownloaders(proxyDownloaders []*proxyDownloader) Option { + return func(p *Proxy) error { + p.proxyDownloaders = proxyDownloaders + return nil + } } // WithDownloaderFactory sets the factory function to get a downloader @@ -119,10 +122,10 @@ func New(opts ...Option) (*Proxy, error) { // NewFromConfig returns a new transparent proxy from the given properties func NewFromConfig(c config.Properties) (*Proxy, error) { opts := []Option{ - WithRules(c.Proxies), + WithProxyDownloaders(newProxyDownloaders(&c)), WithRegistryMirror(c.RegistryMirror), WithDownloaderFactory(func() downloader.Interface { - return dfget.NewGetter(c.DFGetConfig()) + return dfget.NewGetter(c.DFGetConfig(nil)) }), } @@ -162,8 +165,8 @@ func NewFromConfig(c config.Properties) (*Proxy, error) { type Proxy struct { // reverse proxy upstream url for the default registry registry *config.RegistryMirror - // proxy rules - rules []*config.Proxy + // downloaders for each proxy + proxyDownloaders []*proxyDownloader // httpsHosts is the list of hosts whose https requests will be hijacked httpsHosts []*config.HijackHost // cert is the certificate used to hijack https proxy requests @@ -202,12 +205,6 @@ func (proxy *Proxy) remoteConfig(host string) *tls.Config { return nil } -// SetRules change the rule lists of the proxy to the given rules -func (proxy *Proxy) SetRules(rules []*config.Proxy) error { - proxy.rules = rules - return nil -} - // ServeHTTP implements http.Handler.ServeHTTP func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodConnect { @@ -245,23 +242,23 @@ func (proxy *Proxy) roundTripper(tlsConfig *tls.Config) http.RoundTripper { return rt } -// shouldUseDfget returns whether we should use dfget to proxy a request. It -// also change the scheme of the given request if the matched rule has -// UseHTTPS = true -func (proxy *Proxy) shouldUseDfget(req *http.Request) bool { +// shouldUseDfget returns whether we should use dfget to proxy a request +// and the downloader instance. It also change the scheme of the given +// request if the matched rule has UseHTTPS = true. +func (proxy *Proxy) shouldUseDfget(req *http.Request) (downloader.Interface, bool) { if req.Method != http.MethodGet { - return false + return nil, false } - for _, rule := range proxy.rules { - if rule.Match(req.URL.String()) { - if rule.UseHTTPS { + for _, proxyDownloader := range proxy.proxyDownloaders { + if proxyDownloader.Match(req.URL.String()) { + if proxyDownloader.UseHTTPS { req.URL.Scheme = "https" } - return !rule.Direct + return proxyDownloader, !proxyDownloader.Direct } } - return false + return nil, false } // tunnelHTTPS handles a CONNECT request and proxy an https request through an diff --git a/dfdaemon/proxy/proxy_test.go b/dfdaemon/proxy/proxy_test.go index d5c505248..6553eb3dc 100644 --- a/dfdaemon/proxy/proxy_test.go +++ b/dfdaemon/proxy/proxy_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/dragonflyoss/Dragonfly/dfdaemon/config" + "github.com/stretchr/testify/assert" ) @@ -47,7 +48,7 @@ func (tc *testCase) WithRule(regx string, direct bool, useHTTPS bool) *testCase } var r *config.Proxy - r, tc.Error = config.NewProxy(regx, useHTTPS, direct) + r, tc.Error = config.NewProxy(regx, useHTTPS, direct, nil) tc.Rules = append(tc.Rules, r) return tc } @@ -62,7 +63,7 @@ func (tc *testCase) Test(t *testing.T) { if !a.Nil(tc.Error) { return } - tp, err := New(WithRules(tc.Rules)) + tp, err := New(WithProxyDownloaders(newProxyDownloaderWithRules(tc.Rules))) if !a.Nil(err) { return } @@ -71,7 +72,8 @@ func (tc *testCase) Test(t *testing.T) { if !a.Nil(err) { continue } - if !a.Equal(tp.shouldUseDfget(req), !item.Direct) { + _, useDfget := tp.shouldUseDfget(req) + if !a.Equal(useDfget, !item.Direct) { fmt.Println(item.URL) } if item.UseHTTPS { @@ -102,3 +104,15 @@ func TestMatch(t *testing.T) { WithTest("http://h/a/e", false, false). // should match /a, not /a/e Test(t) } + +// This function is for test utils. +func newProxyDownloaderWithRules(rules []*config.Proxy) []*proxyDownloader { + pds := make([]*proxyDownloader, len(rules)) + for i, r := range rules { + pds[i] = &proxyDownloader{ + Proxy: r, + downloader: nil, + } + } + return pds +} diff --git a/dfdaemon/transport/transport.go b/dfdaemon/transport/transport.go index 9a9c300f8..462e4be85 100644 --- a/dfdaemon/transport/transport.go +++ b/dfdaemon/transport/transport.go @@ -24,12 +24,12 @@ import ( "regexp" "time" + "github.com/dragonflyoss/Dragonfly/dfdaemon/downloader" + "github.com/dragonflyoss/Dragonfly/dfdaemon/exception" + "github.com/pborman/uuid" "github.com/pkg/errors" "github.com/sirupsen/logrus" - - "github.com/dragonflyoss/Dragonfly/dfdaemon/downloader" - "github.com/dragonflyoss/Dragonfly/dfdaemon/exception" ) var ( @@ -41,10 +41,10 @@ var ( // It uses http.fileTransport to serve requests that need to use dfget, // and uses http.Transport to serve the other requests. type DFRoundTripper struct { - Round *http.Transport - Round2 http.RoundTripper - ShouldUseDfget func(req *http.Request) bool - Downloader downloader.Interface + Round *http.Transport + Round2 http.RoundTripper + ShouldUseDfget func(req *http.Request) (downloader.Interface, bool) + GlobalDownloader downloader.Interface } // New return the default DFRoundTripper. @@ -61,8 +61,8 @@ func New(opts ...Option) (*DFRoundTripper, error) { } } - if rt.Downloader == nil { - return nil, errors.Errorf("nil downloader") + if rt.GlobalDownloader == nil { + return nil, errors.Errorf("global downloader doesn't exist") } return rt, nil @@ -96,16 +96,16 @@ func defaultHTTPTransport(cfg *tls.Config) *http.Transport { } } -// WithDownloader sets the downloader for the roundTripper +// WithDownloader sets the global downloader for the roundTripper func WithDownloader(d downloader.Interface) Option { return func(rt *DFRoundTripper) error { - rt.Downloader = d + rt.GlobalDownloader = d return nil } } // WithCondition configures how to decide whether to use dfget or not -func WithCondition(c func(r *http.Request) bool) Option { +func WithCondition(c func(r *http.Request) (downloader.Interface, bool)) Option { return func(rt *DFRoundTripper) error { rt.ShouldUseDfget = c return nil @@ -115,12 +115,16 @@ func WithCondition(c func(r *http.Request) bool) Option { // RoundTrip only process first redirect at present // fix resource release func (roundTripper *DFRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - if roundTripper.ShouldUseDfget(req) { + if d, useDfget := roundTripper.ShouldUseDfget(req); useDfget { // delete the Accept-Encoding header to avoid returning the same cached // result for different requests req.Header.Del("Accept-Encoding") logrus.Debugf("round trip with dfget: %s", req.URL.String()) - if res, err := roundTripper.download(req, req.URL.String()); err == nil || !exception.IsNotAuth(err) { + // if it is not downloaded by proxy downloader, just use global downloader + if d == nil { + d = roundTripper.GlobalDownloader + } + if res, err := roundTripper.download(d, req, req.URL.String()); err == nil || !exception.IsNotAuth(err) { return res, err } } @@ -132,8 +136,8 @@ func (roundTripper *DFRoundTripper) RoundTrip(req *http.Request) (*http.Response } // download uses dfget to download -func (roundTripper *DFRoundTripper) download(req *http.Request, urlString string) (*http.Response, error) { - dstPath, err := roundTripper.downloadByGetter(urlString, req.Header, uuid.New()) +func (roundTripper *DFRoundTripper) download(downloader downloader.Interface, req *http.Request, urlString string) (*http.Response, error) { + dstPath, err := downloadByGetter(downloader, urlString, req.Header, uuid.New()) if err != nil { logrus.Errorf("download fail: %v", err) return nil, err @@ -155,13 +159,13 @@ func (roundTripper *DFRoundTripper) download(req *http.Request, urlString string } // downloadByGetter is to download file by DFGetter -func (roundTripper *DFRoundTripper) downloadByGetter(url string, header map[string][]string, name string) (string, error) { +func downloadByGetter(downloader downloader.Interface, url string, header map[string][]string, name string) (string, error) { logrus.Infof("start download url:%s to %s in repo", url, name) - return roundTripper.Downloader.Download(url, header, name) + return downloader.Download(url, header, name) } // needUseGetter is the default value for ShouldUseDfget, which downloads all // images layers with dfget. -func needUseGetter(req *http.Request) bool { - return req.Method == http.MethodGet && layerReg.MatchString(req.URL.Path) +func needUseGetter(req *http.Request) (downloader.Interface, bool) { + return nil, req.Method == http.MethodGet && layerReg.MatchString(req.URL.Path) }