Merge pull request #2447 from tifayuki/cloudfront-s3-filter

add s3 region filters for cloudfront

.gitignore

@@ -35,3 +35,4 @@ bin/*
 # Editor/IDE specific files.
 *.sublime-project
 *.sublime-workspace
+.idea/*

context/logger.go

@@ -39,6 +39,8 @@ type Logger interface {
 	Warn(args ...interface{})
 	Warnf(format string, args ...interface{})
 	Warnln(args ...interface{})
+
+	WithError(err error) *logrus.Entry
 }
 
 type loggerKey struct{}

docs/configuration.md

@@ -183,6 +183,10 @@ middleware:
         privatekey: /path/to/pem
         keypairid: cloudfrontkeypairid
         duration: 3000s
+        ipfilteredby: awsregion
+        awsregion: us-east-1, use-east-2
+        updatefrenquency: 12h
+        iprangesurl: https://ip-ranges.amazonaws.com/ip-ranges.json
   storage:
     - name: redirect
       options:
@@ -636,6 +640,10 @@ middleware:
         privatekey: /path/to/pem
         keypairid: cloudfrontkeypairid
         duration: 3000s
+        ipfilteredby: awsregion
+        awsregion: us-east-1, use-east-2
+        updatefrenquency: 12h
+        iprangesurl: https://ip-ranges.amazonaws.com/ip-ranges.json
 ```
 
 Each middleware entry has `name` and `options` entries. The `name` must
@@ -655,6 +663,14 @@ interpretation of the options.
 | `privatekey` | yes   | The private key for Cloudfront, provided by AWS.        |
 | `keypairid` | yes    | The key pair ID provided by AWS.                         |
 | `duration` | no      | An integer and unit for the duration of the Cloudfront session. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `3000s` is valid, but `3000 s` is not. If you do not specify a `duration` or you specify an integer without a time unit, the duration defaults to `20m` (20 minutes).|
+|`ipfilteredby`|no     | A string with the following value `none|aws|awsregion`. |
+|`awsregion`|no        | A comma separated string of AWS regions, only available when `ipfilteredby` is `awsregion`. For example, `us-east-1, us-west-2`|
+|`updatefrenquency`|no | The frequency to update AWS IP regions, default: `12h`|
+|`iprangesurl`|no      | The URL contains the AWS IP ranges information, default: `https://ip-ranges.amazonaws.com/ip-ranges.json`|
+Then value of ipfilteredby:
+`none`: default, do not filter by IP
+`aws`: IP from AWS goes to S3 directly
+`awsregion`: IP from certain AWS regions goes to S3 directly, use together with `awsregion`
 
 ### `redirect`
 

registry/storage/driver/middleware/cloudfront/middleware.go

@@ -16,14 +16,15 @@ import (
 	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
 	dcontext "github.com/docker/distribution/context"
 	storagedriver "github.com/docker/distribution/registry/storage/driver"
-	storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
+	"github.com/docker/distribution/registry/storage/driver/middleware"
 )
 
 // cloudFrontStorageMiddleware provides a simple implementation of layerHandler that
 // constructs temporary signed CloudFront URLs from the storagedriver layer URL,
 // then issues HTTP Temporary Redirects to this CloudFront content URL.
 type cloudFrontStorageMiddleware struct {
 	storagedriver.StorageDriver
+	awsIPs    *awsIPs
 	urlSigner *sign.URLSigner
 	baseURL   string
 	duration  time.Duration
@@ -34,7 +35,13 @@ var _ storagedriver.StorageDriver = &cloudFrontStorageMiddleware{}
 // newCloudFrontLayerHandler constructs and returns a new CloudFront
 // LayerHandler implementation.
 // Required options: baseurl, privatekey, keypairid
+
+// Optional options: ipFilteredBy, awsregion
+// ipfilteredby: valid value "none|aws|awsregion". "none", do not filter any IP, default value. "aws", only aws IP goes
+//               to S3 directly. "awsregion", only regions listed in awsregion options goes to S3 directly
+// awsregion: a comma separated string of AWS regions.
 func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) {
+	// parse baseurl
 	base, ok := options["baseurl"]
 	if !ok {
 		return nil, fmt.Errorf("no baseurl provided")
@@ -52,6 +59,8 @@ func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, o
 	if _, err := url.Parse(baseURL); err != nil {
 		return nil, fmt.Errorf("invalid baseurl: %v", err)
 	}
+
+	// parse privatekey to get pkPath
 	pk, ok := options["privatekey"]
 	if !ok {
 		return nil, fmt.Errorf("no privatekey provided")
@@ -60,6 +69,8 @@ func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, o
 	if !ok {
 		return nil, fmt.Errorf("privatekey must be a string")
 	}
+
+	// parse keypairid
 	kpid, ok := options["keypairid"]
 	if !ok {
 		return nil, fmt.Errorf("no keypairid provided")
@@ -69,6 +80,7 @@ func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, o
 		return nil, fmt.Errorf("keypairid must be a string")
 	}
 
+	// get urlSigner from the file specified in pkPath
 	pkBytes, err := ioutil.ReadFile(pkPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read privatekey file: %s", err)
@@ -82,12 +94,11 @@ func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, o
 	if err != nil {
 		return nil, err
 	}
-
 	urlSigner := sign.NewURLSigner(keypairID, privateKey)
 
+	// parse duration
 	duration := 20 * time.Minute
-	d, ok := options["duration"]
-	if ok {
+	if d, ok := options["duration"]; ok {
 		switch d := d.(type) {
 		case time.Duration:
 			duration = d
@@ -100,11 +111,62 @@ func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, o
 		}
 	}
 
+	// parse updatefrenquency
+	updateFrequency := defaultUpdateFrequency
+	if u, ok := options["updatefrenquency"]; ok {
+		switch u := u.(type) {
+		case time.Duration:
+			updateFrequency = u
+		case string:
+			updateFreq, err := time.ParseDuration(u)
+			if err != nil {
+				return nil, fmt.Errorf("invalid updatefrenquency: %s", err)
+			}
+			duration = updateFreq
+		}
+	}
+
+	// parse iprangesurl
+	ipRangesURL := defaultIPRangesURL
+	if i, ok := options["iprangesurl"]; ok {
+		if iprangeurl, ok := i.(string); ok {
+			ipRangesURL = iprangeurl
+		} else {
+			return nil, fmt.Errorf("iprangesurl must be a string")
+		}
+	}
+
+	// parse ipfilteredby
+	var awsIPs *awsIPs
+	if ipFilteredBy := options["ipfilteredby"].(string); ok {
+		switch strings.ToLower(strings.TrimSpace(ipFilteredBy)) {
+		case "", "none":
+			awsIPs = nil
+		case "aws":
+			newAWSIPs(ipRangesURL, updateFrequency, nil)
+		case "awsregion":
+			var awsRegion []string
+			if regions, ok := options["awsregion"].(string); ok {
+				for _, awsRegions := range strings.Split(regions, ",") {
+					awsRegion = append(awsRegion, strings.ToLower(strings.TrimSpace(awsRegions)))
+				}
+				awsIPs = newAWSIPs(ipRangesURL, updateFrequency, awsRegion)
+			} else {
+				return nil, fmt.Errorf("awsRegion must be a comma separated string of valid aws regions")
+			}
+		default:
+			return nil, fmt.Errorf("ipfilteredby only allows a string the following value: none|aws|awsregion")
+		}
+	} else {
+		return nil, fmt.Errorf("ipfilteredby only allows a string with the following value: none|aws|awsregion")
+	}
+
 	return &cloudFrontStorageMiddleware{
 		StorageDriver: storageDriver,
 		urlSigner:     urlSigner,
 		baseURL:       baseURL,
 		duration:      duration,
+		awsIPs:        awsIPs,
 	}, nil
 }
 
@@ -114,8 +176,8 @@ type S3BucketKeyer interface {
 	S3BucketKey(path string) string
 }
 
-// Resolve returns an http.Handler which can serve the contents of the given
-// Layer, or an error if not supported by the storagedriver.
+// URLFor attempts to find a url which may be used to retrieve the file at the given path.
+// Returns an error if the file cannot be found.
 func (lh *cloudFrontStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
 	// TODO(endophage): currently only supports S3
 	keyer, ok := lh.StorageDriver.(S3BucketKeyer)
@@ -124,6 +186,11 @@ func (lh *cloudFrontStorageMiddleware) URLFor(ctx context.Context, path string,
 		return lh.StorageDriver.URLFor(ctx, path, options)
 	}
 
+	if eligibleForS3(ctx, lh.awsIPs) {
+		return lh.StorageDriver.URLFor(ctx, path, options)
+	}
+
+	// Get signed cloudfront url.
 	cfURL, err := lh.urlSigner.Sign(lh.baseURL+keyer.S3BucketKey(path), time.Now().Add(lh.duration))
 	if err != nil {
 		return "", err