support dfget flags in each proxy

Signed-off-by: yeya24 <[email protected]>

dfdaemon/config/config.go

@@ -97,11 +97,11 @@ type Properties struct {
 	MaxProcs int `yaml:"maxprocs" json:"maxprocs"`
 
 	// dfget config
-	DfgetFlags []string `yaml:"dfget_flags" json:"dfget_flags"`
-	SuperNodes []string `yaml:"supernodes" json:"supernodes"`
-	RateLimit  string   `yaml:"ratelimit" json:"ratelimit"`
-	DFRepo     string   `yaml:"localrepo" json:"localrepo"`
-	DFPath     string   `yaml:"dfpath" json:"dfpath"`
+	GlobalDfgetFlags []string `yaml:"dfget_flags" json:"dfget_flags"`
+	SuperNodes       []string `yaml:"supernodes" json:"supernodes"`
+	RateLimit        string   `yaml:"ratelimit" json:"ratelimit"`
+	DFRepo           string   `yaml:"localrepo" json:"localrepo"`
+	DFPath           string   `yaml:"dfpath" json:"dfpath"`
 }
 
 // Validate validates the config
@@ -138,10 +138,10 @@ func (p *Properties) Validate() error {
 }
 
 // DFGetConfig returns config for dfget downloader
-func (p *Properties) DFGetConfig() DFGetConfig {
+func (p *Properties) DFGetConfig(flags []string) DFGetConfig {
 	// init DfgetFlags
 	var dfgetFlags []string
-	dfgetFlags = append(dfgetFlags, p.DfgetFlags...)
+	dfgetFlags = append(dfgetFlags, flags...)
 	dfgetFlags = append(dfgetFlags, "--dfdaemon")
 	if p.Verbose {
 		dfgetFlags = append(dfgetFlags, "--verbose")
@@ -372,22 +372,24 @@ func certPoolFromFiles(files ...string) (*x509.CertPool, error) {
 
 // Proxy describe a regular expression matching rule for how to proxy a request
 type Proxy struct {
-	Regx     *Regexp `yaml:"regx" json:"regx"`
-	UseHTTPS bool    `yaml:"use_https" json:"use_https"`
-	Direct   bool    `yaml:"direct" json:"direct"`
+	Regx       *Regexp  `yaml:"regx" json:"regx"`
+	UseHTTPS   bool     `yaml:"use_https" json:"use_https"`
+	Direct     bool     `yaml:"direct" json:"direct"`
+	DfgetFlags []string `yaml:"dfget_flags" json:"dfget_flags"`
 }
 
 // NewProxy returns a new proxy rule with given attributes
-func NewProxy(regx string, useHTTPS bool, direct bool) (*Proxy, error) {
+func NewProxy(regx string, useHTTPS bool, direct bool, dfgetFlags []string) (*Proxy, error) {
 	exp, err := NewRegexp(regx)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid regexp")
 	}
 
 	return &Proxy{
-		Regx:     exp,
-		UseHTTPS: useHTTPS,
-		Direct:   direct,
+		Regx:       exp,
+		UseHTTPS:   useHTTPS,
+		Direct:     direct,
+		DfgetFlags: dfgetFlags,
 	}, nil
 }
 

dfdaemon/config/config_test.go

@@ -234,7 +234,7 @@ func (ts *configTestSuite) TestProxyNew() {
 		validRegexp := ".*"
 		useHTTPS := false
 		direct := false
-		p, err := NewProxy(validRegexp, useHTTPS, direct)
+		p, err := NewProxy(validRegexp, useHTTPS, direct, nil)
 		r.Nil(err)
 		r.NotNil(p)
 		r.Equal(useHTTPS, p.UseHTTPS)
@@ -243,7 +243,7 @@ func (ts *configTestSuite) TestProxyNew() {
 	}
 
 	{
-		p, err := NewProxy(`\K`, false, false)
+		p, err := NewProxy(`\K`, false, false, nil)
 		r.Nil(p)
 		r.NotNil(err)
 		r.True(strings.HasPrefix(err.Error(), "invalid regexp:"))
@@ -252,7 +252,7 @@ func (ts *configTestSuite) TestProxyNew() {
 
 func (ts *configTestSuite) TestProxyMatch() {
 	r := ts.Require()
-	p, err := NewProxy("blobs/sha256.*", false, false)
+	p, err := NewProxy("blobs/sha256.*", false, false, nil)
 	r.Nil(err)
 	r.NotNil(p)
 

dfdaemon/proxy/downloader.go

@@ -0,0 +1,45 @@
+/*
+ * Copyright The Dragonfly Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package proxy
+
+import (
+	"github.com/dragonflyoss/Dragonfly/dfdaemon/config"
+	"github.com/dragonflyoss/Dragonfly/dfdaemon/downloader"
+	"github.com/dragonflyoss/Dragonfly/dfdaemon/downloader/dfget"
+)
+
+// proxyDownloader consists of proxy rule and its downloader
+type proxyDownloader struct {
+	*config.Proxy
+	downloader downloader.Interface
+}
+
+func newProxyDownloaders(p *config.Properties) []*proxyDownloader {
+	pds := make([]*proxyDownloader, len(p.Proxies))
+	for i, proxy := range p.Proxies {
+		pds[i] = &proxyDownloader{
+			Proxy:      proxy,
+			downloader: dfget.NewGetter(p.DFGetConfig(proxy.DfgetFlags)),
+		}
+	}
+	return pds
+}
+
+// Download is used to implement Download interface.
+func (pd *proxyDownloader) Download(url string, header map[string][]string, name string) (string, error) {
+	return pd.downloader.Download(url, header, name)
+}

dfdaemon/proxy/proxy.go

@@ -88,9 +88,12 @@ func WithDirectHandler(h *http.ServeMux) Option {
 	}
 }
 
-// WithRules sets the proxy rules
-func WithRules(rules []*config.Proxy) Option {
-	return func(p *Proxy) error { return p.SetRules(rules) }
+// WithProxyDownloaders sets the proxy downloaders
+func WithProxyDownloaders(proxyDownloaders []*proxyDownloader) Option {
+	return func(p *Proxy) error {
+		p.proxyDownloaders = proxyDownloaders
+		return nil
+	}
 }
 
 // WithDownloaderFactory sets the factory function to get a downloader
@@ -119,10 +122,10 @@ func New(opts ...Option) (*Proxy, error) {
 // NewFromConfig returns a new transparent proxy from the given properties
 func NewFromConfig(c config.Properties) (*Proxy, error) {
 	opts := []Option{
-		WithRules(c.Proxies),
+		WithProxyDownloaders(newProxyDownloaders(&c)),
 		WithRegistryMirror(c.RegistryMirror),
 		WithDownloaderFactory(func() downloader.Interface {
-			return dfget.NewGetter(c.DFGetConfig())
+			return dfget.NewGetter(c.DFGetConfig(c.GlobalDfgetFlags))
 		}),
 	}
 
@@ -162,8 +165,8 @@ func NewFromConfig(c config.Properties) (*Proxy, error) {
 type Proxy struct {
 	// reverse proxy upstream url for the default registry
 	registry *config.RegistryMirror
-	// proxy rules
-	rules []*config.Proxy
+	// downloaders for each proxy
+	proxyDownloaders []*proxyDownloader
 	// httpsHosts is the list of hosts whose https requests will be hijacked
 	httpsHosts []*config.HijackHost
 	// cert is the certificate used to hijack https proxy requests
@@ -202,12 +205,6 @@ func (proxy *Proxy) remoteConfig(host string) *tls.Config {
 	return nil
 }
 
-// SetRules change the rule lists of the proxy to the given rules
-func (proxy *Proxy) SetRules(rules []*config.Proxy) error {
-	proxy.rules = rules
-	return nil
-}
-
 // ServeHTTP implements http.Handler.ServeHTTP
 func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if r.Method == http.MethodConnect {
@@ -245,23 +242,23 @@ func (proxy *Proxy) roundTripper(tlsConfig *tls.Config) http.RoundTripper {
 	return rt
 }
 
-// shouldUseDfget returns whether we should use dfget to proxy a request. It
-// also change the scheme of the given request if the matched rule has
-// UseHTTPS = true
-func (proxy *Proxy) shouldUseDfget(req *http.Request) bool {
+// shouldUseDfget returns whether we should use dfget to proxy a request
+// and the downloader instance. It also change the scheme of the given
+// request if the matched rule has UseHTTPS = true.
+func (proxy *Proxy) shouldUseDfget(req *http.Request) (downloader.Interface, bool) {
 	if req.Method != http.MethodGet {
-		return false
+		return nil, false
 	}
 
-	for _, rule := range proxy.rules {
-		if rule.Match(req.URL.String()) {
-			if rule.UseHTTPS {
+	for _, proxyDownloader := range proxy.proxyDownloaders {
+		if proxyDownloader.Match(req.URL.String()) {
+			if proxyDownloader.UseHTTPS {
 				req.URL.Scheme = "https"
 			}
-			return !rule.Direct
+			return proxyDownloader, !proxyDownloader.Direct
 		}
 	}
-	return false
+	return nil, false
 }
 
 // tunnelHTTPS handles a CONNECT request and proxy an https request through an

dfdaemon/proxy/proxy_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/dragonflyoss/Dragonfly/dfdaemon/config"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -47,7 +48,7 @@ func (tc *testCase) WithRule(regx string, direct bool, useHTTPS bool) *testCase
 	}
 
 	var r *config.Proxy
-	r, tc.Error = config.NewProxy(regx, useHTTPS, direct)
+	r, tc.Error = config.NewProxy(regx, useHTTPS, direct, nil)
 	tc.Rules = append(tc.Rules, r)
 	return tc
 }
@@ -62,7 +63,7 @@ func (tc *testCase) Test(t *testing.T) {
 	if !a.Nil(tc.Error) {
 		return
 	}
-	tp, err := New(WithRules(tc.Rules))
+	tp, err := New(WithProxyDownloaders(newProxyDownloaderWithRules(tc.Rules)))
 	if !a.Nil(err) {
 		return
 	}
@@ -71,7 +72,8 @@ func (tc *testCase) Test(t *testing.T) {
 		if !a.Nil(err) {
 			continue
 		}
-		if !a.Equal(tp.shouldUseDfget(req), !item.Direct) {
+		_, useDfget := tp.shouldUseDfget(req)
+		if !a.Equal(useDfget, !item.Direct) {
 			fmt.Println(item.URL)
 		}
 		if item.UseHTTPS {
@@ -102,3 +104,15 @@ func TestMatch(t *testing.T) {
 		WithTest("http://h/a/e", false, false). // should match /a, not /a/e
 		Test(t)
 }
+
+// This function is for test utils.
+func newProxyDownloaderWithRules(rules []*config.Proxy) []*proxyDownloader {
+	pds := make([]*proxyDownloader, len(rules))
+	for i, r := range rules {
+		pds[i] = &proxyDownloader{
+			Proxy:      r,
+			downloader: nil,
+		}
+	}
+	return pds
+}

dfdaemon/transport/transport.go

@@ -24,12 +24,12 @@ import (
 	"regexp"
 	"time"
 
+	"github.com/dragonflyoss/Dragonfly/dfdaemon/downloader"
+	"github.com/dragonflyoss/Dragonfly/dfdaemon/exception"
+
 	"github.com/pborman/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-
-	"github.com/dragonflyoss/Dragonfly/dfdaemon/downloader"
-	"github.com/dragonflyoss/Dragonfly/dfdaemon/exception"
 )
 
 var (
@@ -41,10 +41,10 @@ var (
 // It uses http.fileTransport to serve requests that need to use dfget,
 // and uses http.Transport to serve the other requests.
 type DFRoundTripper struct {
-	Round          *http.Transport
-	Round2         http.RoundTripper
-	ShouldUseDfget func(req *http.Request) bool
-	Downloader     downloader.Interface
+	Round            *http.Transport
+	Round2           http.RoundTripper
+	ShouldUseDfget   func(req *http.Request) (downloader.Interface, bool)
+	GlobalDownloader downloader.Interface
 }
 
 // New return the default DFRoundTripper.
@@ -61,8 +61,8 @@ func New(opts ...Option) (*DFRoundTripper, error) {
 		}
 	}
 
-	if rt.Downloader == nil {
-		return nil, errors.Errorf("nil downloader")
+	if rt.GlobalDownloader == nil {
+		return nil, errors.Errorf("global downloader doesn't exist")
 	}
 
 	return rt, nil
@@ -96,16 +96,16 @@ func defaultHTTPTransport(cfg *tls.Config) *http.Transport {
 	}
 }
 
-// WithDownloader sets the downloader for the roundTripper
+// WithDownloader sets the global downloader for the roundTripper
 func WithDownloader(d downloader.Interface) Option {
 	return func(rt *DFRoundTripper) error {
-		rt.Downloader = d
+		rt.GlobalDownloader = d
 		return nil
 	}
 }
 
 // WithCondition configures how to decide whether to use dfget or not
-func WithCondition(c func(r *http.Request) bool) Option {
+func WithCondition(c func(r *http.Request) (downloader.Interface, bool)) Option {
 	return func(rt *DFRoundTripper) error {
 		rt.ShouldUseDfget = c
 		return nil
@@ -115,12 +115,16 @@ func WithCondition(c func(r *http.Request) bool) Option {
 // RoundTrip only process first redirect at present
 // fix resource release
 func (roundTripper *DFRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	if roundTripper.ShouldUseDfget(req) {
+	if d, useDfget := roundTripper.ShouldUseDfget(req); useDfget {
 		// delete the Accept-Encoding header to avoid returning the same cached
 		// result for different requests
 		req.Header.Del("Accept-Encoding")
 		logrus.Debugf("round trip with dfget: %s", req.URL.String())
-		if res, err := roundTripper.download(req, req.URL.String()); err == nil || !exception.IsNotAuth(err) {
+		// if it is not downloaded by proxy downloader, just use global downloader
+		if d == nil {
+			d = roundTripper.GlobalDownloader
+		}
+		if res, err := roundTripper.download(d, req, req.URL.String()); err == nil || !exception.IsNotAuth(err) {
 			return res, err
 		}
 	}
@@ -132,8 +136,8 @@ func (roundTripper *DFRoundTripper) RoundTrip(req *http.Request) (*http.Response
 }
 
 // download uses dfget to download
-func (roundTripper *DFRoundTripper) download(req *http.Request, urlString string) (*http.Response, error) {
-	dstPath, err := roundTripper.downloadByGetter(urlString, req.Header, uuid.New())
+func (roundTripper *DFRoundTripper) download(downloader downloader.Interface, req *http.Request, urlString string) (*http.Response, error) {
+	dstPath, err := downloadByGetter(downloader, urlString, req.Header, uuid.New())
 	if err != nil {
 		logrus.Errorf("download fail: %v", err)
 		return nil, err
@@ -155,13 +159,13 @@ func (roundTripper *DFRoundTripper) download(req *http.Request, urlString string
 }
 
 // downloadByGetter is to download file by DFGetter
-func (roundTripper *DFRoundTripper) downloadByGetter(url string, header map[string][]string, name string) (string, error) {
+func downloadByGetter(downloader downloader.Interface, url string, header map[string][]string, name string) (string, error) {
 	logrus.Infof("start download url:%s to %s in repo", url, name)
-	return roundTripper.Downloader.Download(url, header, name)
+	return downloader.Download(url, header, name)
 }
 
 // needUseGetter is the default value for ShouldUseDfget, which downloads all
 // images layers with dfget.
-func needUseGetter(req *http.Request) bool {
-	return req.Method == http.MethodGet && layerReg.MatchString(req.URL.Path)
+func needUseGetter(req *http.Request) (downloader.Interface, bool) {
+	return nil, req.Method == http.MethodGet && layerReg.MatchString(req.URL.Path)
 }