feat: use google/go-cloud to fetch rules and credentials from object …

…storage (#562) Closes #518 As proposed in #518, I've used go-cloud library to fetch credentials and rules files from an object storage (S3, Azure Blob Storage or GCS).
ory · Nov 2, 2020 · 666b951 · 666b951
1 parent a9f8630
commit 666b951
Show file tree

Hide file tree

Showing 19 changed files with 1,457 additions and 170 deletions.
diff --git a/credentials/fetcher_default.go b/credentials/fetcher_default.go
@@ -32,13 +32,20 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ory/oathkeeper/internal/cloudstorage"
+
 	"github.com/pkg/errors"
 	"gopkg.in/square/go-jose.v2"
 
 	"github.com/ory/x/logrusx"
 
 	"github.com/ory/herodot"
 	"github.com/ory/x/httpx"
+
+	"gocloud.dev/blob"
+	_ "gocloud.dev/blob/azureblob"
+	_ "gocloud.dev/blob/gcsblob"
+	_ "gocloud.dev/blob/s3blob"
 )
 
 type reasoner interface {
@@ -56,6 +63,7 @@ type FetcherDefault struct {
 	keys        map[string]jose.JSONWebKeySet
 	fetchedAt   map[string]time.Time
 	l           *logrusx.Logger
+	mux         *blob.URLMux
 }
 
 // NewFetcherDefault returns a new JWKS Fetcher with:
@@ -71,6 +79,7 @@ func NewFetcherDefault(l *logrusx.Logger, cancelAfter time.Duration, ttl time.Du
 		keys:        make(map[string]jose.JSONWebKeySet),
 		fetchedAt:   make(map[string]time.Time),
 		client:      httpx.NewResilientClientLatencyToleranceHigh(nil),
+		mux:         cloudstorage.NewURLMux(),
 	}
 }
 
@@ -196,6 +205,41 @@ func (s *FetcherDefault) resolve(wg *sync.WaitGroup, errs chan error, location u
 	var reader io.Reader
 
 	switch location.Scheme {
+	case "azblob":
+		fallthrough
+	case "gs":
+		fallthrough
+	case "s3":
+		ctx := context.Background()
+		bucket, err := s.mux.OpenBucket(ctx, location.Scheme+"://"+location.Host)
+		if err != nil {
+			errs <- errors.WithStack(herodot.
+				ErrInternalServerError.
+				WithReasonf(
+					`Unable to fetch JSON Web Keys from location "%s" because "%s".`,
+					location.String(),
+					err,
+				),
+			)
+			return
+		}
+		defer bucket.Close()
+
+		r, err := bucket.NewReader(ctx, location.Path[1:], nil)
+		if err != nil {
+			errs <- errors.WithStack(herodot.
+				ErrInternalServerError.
+				WithReasonf(
+					`Unable to fetch JSON Web Keys from location "%s" because "%s".`,
+					location.String(),
+					err,
+				),
+			)
+			return
+		}
+		defer r.Close()
+
+		reader = r
 	case "file":
 		f, err := os.Open(strings.Replace(location.String(), "file://", "", 1))
 		if err != nil {

diff --git a/credentials/fetcher_default_test.go b/credentials/fetcher_default_test.go
@@ -17,6 +17,8 @@ import (
 
 	"github.com/ory/herodot"
 	"github.com/ory/x/urlx"
+
+	"github.com/ory/oathkeeper/internal/cloudstorage"
 )
 
 var sets = [...]json.RawMessage{
@@ -28,6 +30,9 @@ var sets = [...]json.RawMessage{
 
 func TestFetcherDefault(t *testing.T) {
 	const maxWait = time.Millisecond * 100
+	t.Cleanup(func() {
+		cloudstorage.SetCurrentTest(nil)
+	})
 
 	l := logrusx.New("", "", logrusx.ForceLevel(logrus.DebugLevel))
 	w := herodot.NewJSONWriter(l.Logger)
@@ -146,4 +151,43 @@ func TestFetcherDefault(t *testing.T) {
 		assert.True(t, check("f4190122-ae96-4c29-8b79-56024e459d80"))
 		assert.True(t, check("8e884167-1300-4f58-8cc1-81af68f878a8"))
 	})
+
+	t.Run("name=should fetch from s3 object storage", func(t *testing.T) {
+		ctx := context.Background()
+		cloudstorage.SetCurrentTest(t)
+
+		s := NewFetcherDefault(l, maxWait, maxWait*7)
+
+		key, err := s.ResolveKey(ctx, []url.URL{
+			*urlx.ParseOrPanic("s3://oathkeeper-test-bucket/path/prefix/jwks.json"),
+		}, "81be3441-5303-4c52-b00d-bbdfadc75633", "sig")
+		require.NoError(t, err)
+		assert.Equal(t, "81be3441-5303-4c52-b00d-bbdfadc75633", key.KeyID)
+	})
+
+	t.Run("name=should fetch from gs object storage", func(t *testing.T) {
+		ctx := context.Background()
+		cloudstorage.SetCurrentTest(t)
+
+		s := NewFetcherDefault(l, maxWait, maxWait*7)
+
+		key, err := s.ResolveKey(ctx, []url.URL{
+			*urlx.ParseOrPanic("gs://oathkeeper-test-bucket/path/prefix/jwks.json"),
+		}, "81be3441-5303-4c52-b00d-bbdfadc75633", "sig")
+		require.NoError(t, err)
+		assert.Equal(t, "81be3441-5303-4c52-b00d-bbdfadc75633", key.KeyID)
+	})
+
+	t.Run("name=should fetch from azure object storage", func(t *testing.T) {
+		ctx := context.Background()
+		cloudstorage.SetCurrentTest(t)
+
+		s := NewFetcherDefault(l, maxWait, maxWait*7)
+
+		jwkKey, err := s.ResolveKey(ctx, []url.URL{
+			*urlx.ParseOrPanic("azblob://path/prefix/jwks.json"),
+		}, "81be3441-5303-4c52-b00d-bbdfadc75633", "sig")
+		require.NoError(t, err)
+		assert.Equal(t, "81be3441-5303-4c52-b00d-bbdfadc75633", jwkKey.KeyID)
+	})
 }
diff --git a/...als/testdata/TestFetcherDefault/name=should_fetch_from_azure_object_storage-azblob.replay b/...als/testdata/TestFetcherDefault/name=should_fetch_from_azure_object_storage-azblob.replay
@@ -0,0 +1,120 @@
+{
+  "Initial": "AQAAAA7XLQcUD9nUsAA8",
+  "Version": "0.2",
+  "Converter": {
+    "ClearHeaders": [
+      "^X-Goog-.*Encryption-Key$",
+      "^X-Ms-Date$",
+      "^User-Agent$"
+    ],
+    "RemoveRequestHeaders": [
+      "^Authorization$",
+      "^Proxy-Authorization$",
+      "^Connection$",
+      "^Content-Type$",
+      "^Date$",
+      "^Host$",
+      "^Transfer-Encoding$",
+      "^Via$",
+      "^X-Forwarded-.*$",
+      "^X-Cloud-Trace-Context$",
+      "^X-Goog-Api-Client$",
+      "^X-Google-.*$",
+      "^X-Gfe-.*$"
+    ],
+    "RemoveResponseHeaders": [
+      "^X-Google-.*$",
+      "^X-Gfe-.*$"
+    ],
+    "ClearParams": null,
+    "RemoveParams": [
+      "^se$",
+      "^sig$",
+      "^X-Ms-Date$"
+    ]
+  },
+  "Entries": [
+    {
+      "ID": "1fb66968f4225198",
+      "Request": {
+        "Method": "GET",
+        "URL": "https://oathkeepertestbucket.blob.core.windows.net/path/prefix/jwks.json",
+        "Header": {
+          "Accept-Encoding": [
+            "gzip"
+          ],
+          "User-Agent": [
+            "CLEARED"
+          ],
+          "X-Ms-Date": [
+            "CLEARED"
+          ],
+          "X-Ms-Version": [
+            "2018-11-09"
+          ]
+        },
+        "MediaType": "",
+        "BodyParts": [
+          ""
+        ]
+      },
+      "Response": {
+        "StatusCode": 200,
+        "Proto": "HTTP/1.1",
+        "ProtoMajor": 1,
+        "ProtoMinor": 1,
+        "Header": {
+          "Accept-Ranges": [
+            "bytes"
+          ],
+          "Content-Disposition": [
+            ""
+          ],
+          "Content-Length": [
+            "1811"
+          ],
+          "Content-Md5": [
+            "Yu3yvund6SMtT06kBFdUjw=="
+          ],
+          "Content-Type": [
+            "application/json"
+          ],
+          "Date": [
+            "Thu, 29 Oct 2020 18:55:17 GMT"
+          ],
+          "Etag": [
+            "\"0x8D87C392EBEBAD7\""
+          ],
+          "Last-Modified": [
+            "Thu, 29 Oct 2020 18:33:51 GMT"
+          ],
+          "Server": [
+            "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0"
+          ],
+          "X-Ms-Blob-Type": [
+            "BlockBlob"
+          ],
+          "X-Ms-Creation-Time": [
+            "Thu, 29 Oct 2020 18:33:51 GMT"
+          ],
+          "X-Ms-Lease-State": [
+            "available"
+          ],
+          "X-Ms-Lease-Status": [
+            "unlocked"
+          ],
+          "X-Ms-Request-Id": [
+            "bf519152-b01e-0093-7f25-ae09a5000000"
+          ],
+          "X-Ms-Server-Encrypted": [
+            "true"
+          ],
+          "X-Ms-Version": [
+            "2018-11-09"
+          ]
+        },
+        "Body": "ewogICJrZXlzIjogWwogICAgewogICAgICAiYWxnIjogIlJTMjU2IiwKICAgICAgInAiOiAiMFNkM3JQTmtfbVV4VERGRmV1V1ZSVl9hQXlmZEFfZ1NfMlpnMWxXNlVuMlVXUWNiekJzWXUwV29ZUGgzNXZFcHVYMHJTLVNkdmNWbHF0N0ZJVFZUQ0NzVC1VM0xtUEJ6eVRzM296dXJzLTNseHk5NGxhSHRYc0hqOUpvNjhlQXNHZ2tfeWlJd0RDaE52eEFJSGt4VFRGbHRUTm9SSzlNampEempNYmlCZmowIiwKICAgICAgImt0eSI6ICJSU0EiLAogICAgICAicSI6ICJvWk1WaE9jLTRzbDRseUNWcVBmQmJ6TGhPeWVPclVodlY3VkllZDk1akFlcTFyWHJEcjl3OTdscGFOWDMtam9INXBnR0hIVlFxTWp5TUJPQTA3VFpsb0NRdUF0VUY4WGpnSzA0cGZZXzFGX2RzX0w0ODZpdUFtMkV2QTNtMEg4enFzNnAwaVdNcm1YQzl1a2JvY19CVlZzQ2ZoMDhlUGN6MXlBVmxYOTJPLWsiLAogICAgICAiZCI6ICJZd3h6UVJHT0luXzNyN2lQQzRiTEdfWjRvQTYtdUZTd2Nrby1lSkpUdEZLRVFGYmFPMjNveVd3QzFSNXl4U1NGdnYxZGV2UnNSdGQ4OVFBTFExUm1jclpQR1F3aC1aRXdYTlFaYlNodGtUVVdGV1llZkwwcVkxWEticm5ub0JsWllzVXU5MkJNbjJ0eG5saWtiS3h4Y1g2WFNBR0ItTDl1bWRGVGQ0enpILWFtYXFMS0V5b29LWDR0YllzRGY3OG1yUDFmNUR4RnhhU2dobFMzdjRJY2FlRDR0TEZVdzJfZXpQVlZfQlc5eEc1c3JHa0RQRlJ1enhvcVdEcG4wWVY3YWc5OGYxbEU5aXhYREp6aml0TC1jckdoeFY0Qi1GVGFCR1VZYXdLcFFrZVpEa1pmQTZkd2psUU9acVpBamUtZmJmOXBjU0k5Y0g2LVZjTUJ2OGdFUVEiLAogICAgICAiZSI6ICJBUUFCIiwKICAgICAgInVzZSI6ICJzaWciLAogICAgICAia2lkIjogIjgxYmUzNDQxLTUzMDMtNGM1Mi1iMDBkLWJiZGZhZGM3NTYzMyIsCiAgICAgICJxaSI6ICJVV2NPMThsQVF0blZiS1FEY3RiYTlGdGRROWVHVnlQbXE2MVNpeWVmcnBoU0hfZjlsbFZMeWM3ZFlDNXJhNTZOd0xPZmtrU3YyU3ZqdFJqSjF1WXJxMm5jZVZhbU5EMWJuNEY2dUMya3FTaERRNU9FdGVGZE56LXdJdE5FazlSd3lBLXdhS1VGZnFvVnVMdktBTU9Dd3RHQVp3dTdsbDFIdVd6MmdWZmw3S2ciLAogICAgICAiZHAiOiAidGNDOVg5OFcyWGNvVXRUU1V3emxIYWRBOWRNVFhocWFyN1RRM0JFZFhBZmlfVWFQSWx3OXJxZkd2WlhONURPWUZhZXZFMEYwem0tV2ZOa2ZwMDhHZTZlcnNNNm82WnV5a3FGU0pSaDZpbDVtZWlSSGZvaDdsUWVheDNtclZYdF85NVFvQ2NpUnhrN1QyUF9lZmkzSGlTYUJNX0tnVDF3b25QcGdXRm5FdGVVIiwKICAgICAgImRxIjogIlg5WlhpRXFGMFkxQld4cjN5bzZTalB1M194ZDZtS3dEWnhuZlVITE9yTkdMY3VuYThCZDcwV1NIOHFnZi02dGFMbHYwSEFNMkFiZVhlQTdKZGttcVdvd055eXpsQjJqWWhlVzVXUy1VTXN0ZVBMT2RHU0h2ZlA4cklVUjVxU2dhSkhHSzUxeGdjcjFuaEg5R1paZE9qT3Q3c2dSemJ5anA5ZXRSQWlqbG42ayIsCiAgICAgICJuIjogImhBSDlnazk3S2ZYLXVtV2l0WGhURDMycVlvUGZRb1RzZU9CblRVQTJWX2pobEtCeEMydzJqakRGYXVMTE10YVFtWEtRMVNvYzBRMFE1cmJtRDNoeDV4a3lNV1A3UFhxQjNubktZbnlGWFpEUklLOHhHNFJpVldaejgyYzUtVnk2QllMTFZCMWg5eEVjd2NJbGNzd0F4OVBRdzRxbGFyaFBXSk1XTVVyb1VTX1lnSnJHbERSQ2FHTzU2NDl6UXBhakpPcVBaWndKRGxYMGF1bmpsSjRVNmFyY0xpZEdhZWR6UE5BNmsyanh4REJaczdRbzVhaWRaM0JIYVFsWlFUOExJX0FjSGdEUl94NEV4RXhZTDlIU1I0bWlZZkgzQm54SDRVM0tLQVNMYVplOWl6NzI0cmpvbXlWY2xGTFdfR0NBS25DNkhLRHhWLUlVbEFHWU9SSDBoUSIKICAgIH0KICBdCn0="
+      }
+    }
+  ]
+}
diff --git a/credentials/testdata/TestFetcherDefault/name=should_fetch_from_gs_object_storage-gs.replay b/credentials/testdata/TestFetcherDefault/name=should_fetch_from_gs_object_storage-gs.replay
@@ -0,0 +1,117 @@
+{
+  "Initial": "AQAAAA7XLQYEIQ6ZyAA8",
+  "Version": "0.2",
+  "Converter": {
+    "ClearHeaders": [
+      "^X-Goog-.*Encryption-Key$",
+      "^Expires$",
+      "^Signature$"
+    ],
+    "RemoveRequestHeaders": [
+      "^Authorization$",
+      "^Proxy-Authorization$",
+      "^Connection$",
+      "^Content-Type$",
+      "^Date$",
+      "^Host$",
+      "^Transfer-Encoding$",
+      "^Via$",
+      "^X-Forwarded-.*$",
+      "^X-Cloud-Trace-Context$",
+      "^X-Goog-Api-Client$",
+      "^X-Google-.*$",
+      "^X-Gfe-.*$"
+    ],
+    "RemoveResponseHeaders": [
+      "^X-Google-.*$",
+      "^X-Gfe-.*$"
+    ],
+    "ClearParams": [
+      "^Expires$",
+      "^Signature$"
+    ],
+    "RemoveParams": null
+  },
+  "Entries": [
+    {
+      "ID": "17621eb301ff5978",
+      "Request": {
+        "Method": "GET",
+        "URL": "https://storage.googleapis.com/oathkeeper-test-bucket/path/prefix/jwks.json",
+        "Header": {
+          "Accept-Encoding": [
+            "gzip"
+          ],
+          "User-Agent": [
+            "go-cloud/blob/0.1.0"
+          ]
+        },
+        "MediaType": "",
+        "BodyParts": [
+          ""
+        ]
+      },
+      "Response": {
+        "StatusCode": 200,
+        "Proto": "HTTP/1.1",
+        "ProtoMajor": 1,
+        "ProtoMinor": 1,
+        "Header": {
+          "Accept-Ranges": [
+            "bytes"
+          ],
+          "Alt-Svc": [
+            "h3-Q050=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\""
+          ],
+          "Cache-Control": [
+            "private, max-age=0"
+          ],
+          "Content-Length": [
+            "1811"
+          ],
+          "Content-Type": [
+            "application/json"
+          ],
+          "Date": [
+            "Thu, 29 Oct 2020 18:50:45 GMT"
+          ],
+          "Etag": [
+            "\"62edf2bee9dde9232d4f4ea40457548f\""
+          ],
+          "Expires": [
+            "CLEARED"
+          ],
+          "Last-Modified": [
+            "Thu, 29 Oct 2020 17:18:55 GMT"
+          ],
+          "Server": [
+            "UploadServer"
+          ],
+          "X-Goog-Generation": [
+            "1603991935010126"
+          ],
+          "X-Goog-Hash": [
+            "crc32c=l52Lxg==",
+            "md5=Yu3yvund6SMtT06kBFdUjw=="
+          ],
+          "X-Goog-Metageneration": [
+            "1"
+          ],
+          "X-Goog-Storage-Class": [
+            "STANDARD"
+          ],
+          "X-Goog-Stored-Content-Encoding": [
+            "identity"
+          ],
+          "X-Goog-Stored-Content-Length": [
+            "1811"
+          ],
+          "X-Guploader-Uploadid": [
+            "ABg5-UxYhQqahmJdafg-9J_c6Z3luhTgHnt2Dk9xHa6mOyh7OVlcTm4Zfyc7oHbBNDDRRWMwuo4nkZaWlHfAohthIOtzPDWxDw"
+          ]
+        },
+        "Body": "ewogICJrZXlzIjogWwogICAgewogICAgICAiYWxnIjogIlJTMjU2IiwKICAgICAgInAiOiAiMFNkM3JQTmtfbVV4VERGRmV1V1ZSVl9hQXlmZEFfZ1NfMlpnMWxXNlVuMlVXUWNiekJzWXUwV29ZUGgzNXZFcHVYMHJTLVNkdmNWbHF0N0ZJVFZUQ0NzVC1VM0xtUEJ6eVRzM296dXJzLTNseHk5NGxhSHRYc0hqOUpvNjhlQXNHZ2tfeWlJd0RDaE52eEFJSGt4VFRGbHRUTm9SSzlNampEempNYmlCZmowIiwKICAgICAgImt0eSI6ICJSU0EiLAogICAgICAicSI6ICJvWk1WaE9jLTRzbDRseUNWcVBmQmJ6TGhPeWVPclVodlY3VkllZDk1akFlcTFyWHJEcjl3OTdscGFOWDMtam9INXBnR0hIVlFxTWp5TUJPQTA3VFpsb0NRdUF0VUY4WGpnSzA0cGZZXzFGX2RzX0w0ODZpdUFtMkV2QTNtMEg4enFzNnAwaVdNcm1YQzl1a2JvY19CVlZzQ2ZoMDhlUGN6MXlBVmxYOTJPLWsiLAogICAgICAiZCI6ICJZd3h6UVJHT0luXzNyN2lQQzRiTEdfWjRvQTYtdUZTd2Nrby1lSkpUdEZLRVFGYmFPMjNveVd3QzFSNXl4U1NGdnYxZGV2UnNSdGQ4OVFBTFExUm1jclpQR1F3aC1aRXdYTlFaYlNodGtUVVdGV1llZkwwcVkxWEticm5ub0JsWllzVXU5MkJNbjJ0eG5saWtiS3h4Y1g2WFNBR0ItTDl1bWRGVGQ0enpILWFtYXFMS0V5b29LWDR0YllzRGY3OG1yUDFmNUR4RnhhU2dobFMzdjRJY2FlRDR0TEZVdzJfZXpQVlZfQlc5eEc1c3JHa0RQRlJ1enhvcVdEcG4wWVY3YWc5OGYxbEU5aXhYREp6aml0TC1jckdoeFY0Qi1GVGFCR1VZYXdLcFFrZVpEa1pmQTZkd2psUU9acVpBamUtZmJmOXBjU0k5Y0g2LVZjTUJ2OGdFUVEiLAogICAgICAiZSI6ICJBUUFCIiwKICAgICAgInVzZSI6ICJzaWciLAogICAgICAia2lkIjogIjgxYmUzNDQxLTUzMDMtNGM1Mi1iMDBkLWJiZGZhZGM3NTYzMyIsCiAgICAgICJxaSI6ICJVV2NPMThsQVF0blZiS1FEY3RiYTlGdGRROWVHVnlQbXE2MVNpeWVmcnBoU0hfZjlsbFZMeWM3ZFlDNXJhNTZOd0xPZmtrU3YyU3ZqdFJqSjF1WXJxMm5jZVZhbU5EMWJuNEY2dUMya3FTaERRNU9FdGVGZE56LXdJdE5FazlSd3lBLXdhS1VGZnFvVnVMdktBTU9Dd3RHQVp3dTdsbDFIdVd6MmdWZmw3S2ciLAogICAgICAiZHAiOiAidGNDOVg5OFcyWGNvVXRUU1V3emxIYWRBOWRNVFhocWFyN1RRM0JFZFhBZmlfVWFQSWx3OXJxZkd2WlhONURPWUZhZXZFMEYwem0tV2ZOa2ZwMDhHZTZlcnNNNm82WnV5a3FGU0pSaDZpbDVtZWlSSGZvaDdsUWVheDNtclZYdF85NVFvQ2NpUnhrN1QyUF9lZmkzSGlTYUJNX0tnVDF3b25QcGdXRm5FdGVVIiwKICAgICAgImRxIjogIlg5WlhpRXFGMFkxQld4cjN5bzZTalB1M194ZDZtS3dEWnhuZlVITE9yTkdMY3VuYThCZDcwV1NIOHFnZi02dGFMbHYwSEFNMkFiZVhlQTdKZGttcVdvd055eXpsQjJqWWhlVzVXUy1VTXN0ZVBMT2RHU0h2ZlA4cklVUjVxU2dhSkhHSzUxeGdjcjFuaEg5R1paZE9qT3Q3c2dSemJ5anA5ZXRSQWlqbG42ayIsCiAgICAgICJuIjogImhBSDlnazk3S2ZYLXVtV2l0WGhURDMycVlvUGZRb1RzZU9CblRVQTJWX2pobEtCeEMydzJqakRGYXVMTE10YVFtWEtRMVNvYzBRMFE1cmJtRDNoeDV4a3lNV1A3UFhxQjNubktZbnlGWFpEUklLOHhHNFJpVldaejgyYzUtVnk2QllMTFZCMWg5eEVjd2NJbGNzd0F4OVBRdzRxbGFyaFBXSk1XTVVyb1VTX1lnSnJHbERSQ2FHTzU2NDl6UXBhakpPcVBaWndKRGxYMGF1bmpsSjRVNmFyY0xpZEdhZWR6UE5BNmsyanh4REJaczdRbzVhaWRaM0JIYVFsWlFUOExJX0FjSGdEUl94NEV4RXhZTDlIU1I0bWlZZkgzQm54SDRVM0tLQVNMYVplOWl6NzI0cmpvbXlWY2xGTFdfR0NBS25DNkhLRHhWLUlVbEFHWU9SSDBoUSIKICAgIH0KICBdCn0="
+      }
+    }
+  ]
+}