redirect AWS client blob requests to s3 based on client IP -> AWS region

cmd/archeio/app/buckets.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+func regionToBucket(region string) string {
+	// for now always return @ameukam's test bucket
+	switch region {
+	default:
+		return "https://painfully-really-suddenly-many-raccoon-image-layers.s3.us-west-2.amazonaws.com"
+	}
+}

cmd/archeio/app/buckets_test.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"testing"
+
+	"sigs.k8s.io/oci-proxy/pkg/net/cidrs/aws"
+)
+
+func TestRegionToBucket(t *testing.T) {
+	// ensure all known regions return a configured bucket
+	regions := aws.Regions()
+	for region := range regions {
+		bucket := regionToBucket(region)
+		if bucket == "" {
+			t.Fatalf("received empty string for known region %q bucket", region)
+		}
+	}
+}

cmd/archeio/app/clientip.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/netip"
+	"strings"
+)
+
+// getClientIP gets the client IP for an http.Request
+//
+// NOTE: currently only two scenarios are supported:
+// 1. no loadbalancer, local testing
+// 2. behind Google Cloud LoadBalancer
+//
+// At this time we have no need to complicate it further
+func getClientIP(r *http.Request) (netip.Addr, error) {
+	// Upstream docs:
+	// https://cloud.google.com/load-balancing/docs/https#x-forwarded-for_header
+	//
+	// If there is no X-Forwarded-For header on the incoming request,
+	// these two IP addresses are the entire header value:
+	// X-Forwarded-For: <client-ip>,<load-balancer-ip>
+	//
+	// If the request includes an X-Forwarded-For header, the load balancer
+	// preserves the supplied value before the <client-ip>,<load-balancer-ip>:
+	// X-Forwarded-For: [<supplied-value>,]<client-ip>,<load-balancer-ip>
+	//
+	// Caution: The load balancer does not verify any IP addresses that
+	// precede <client-ip>,<load-balancer-ip> in this header.
+	// The preceding IP addresses might contain other characters, including spaces.
+	rawXFwdFor := r.Header.Get("X-Forwarded-For")
+
+	// clearly we are not in cloud if this header is not set, we can use
+	// r.RemoteAddr in that case to support local testing
+	// Go http server will always set this value for us
+	if rawXFwdFor == "" {
+		host, _, err := net.SplitHostPort(r.RemoteAddr)
+		if err != nil {
+			return netip.Addr{}, err
+		}
+		return netip.ParseAddr(host)
+	}
+
+	// assume we are in cloud run, get <client-ip> from load balancer header
+	// local tests with direct connection to the server can also fake this
+	// header which is fine + useful
+	//
+	// we want the contents between the second to last comma and the last comma
+	// or if only one comma between the start of the string and the last comma
+	lastComma := strings.LastIndex(rawXFwdFor, ",")
+	if lastComma == -1 {
+		// we should have had at least one comma, something is wrong
+		return netip.Addr{}, fmt.Errorf("invalid X-Forwarded-For value: %s", rawXFwdFor)
+	}
+	secondLastComma := strings.LastIndex(rawXFwdFor[:lastComma], ",")
+	return netip.ParseAddr(rawXFwdFor[secondLastComma+1 : lastComma])
+}

cmd/archeio/app/clientip_test.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"net/http"
+	"net/netip"
+	"testing"
+)
+
+func TestGetClientIP(t *testing.T) {
+	testCases := []struct {
+		Name        string
+		Request     http.Request
+		ExpectedIP  netip.Addr
+		ExpectError bool
+	}{
+		{
+			Name: "NO X-Forwarded-For",
+			Request: http.Request{
+				RemoteAddr: "127.0.0.1:8888",
+			},
+			ExpectedIP: netip.MustParseAddr("127.0.0.1"),
+		},
+		{
+			Name: "NO X-Forwarded-For, somehow bogus RemoteAddr ??? gotta pump code coverage 🤷",
+			Request: http.Request{
+				RemoteAddr: "127.0.0.1asd;lfkj8888",
+			},
+			ExpectError: true,
+		},
+		{
+			Name: "X-Forwarded-For without client-supplied",
+			Request: http.Request{
+				Header: http.Header{
+					"X-Forwarded-For": []string{"8.8.8.8,8.8.8.9"},
+				},
+				RemoteAddr: "127.0.0.1:8888",
+			},
+			ExpectedIP: netip.MustParseAddr("8.8.8.8"),
+		},
+		{
+			Name: "X-Forwarded-For with clean client-supplied",
+			Request: http.Request{
+				Header: http.Header{
+					"X-Forwarded-For": []string{"127.0.0.1,8.8.8.8,8.8.8.9"},
+				},
+				RemoteAddr: "127.0.0.1:8888",
+			},
+			ExpectedIP: netip.MustParseAddr("8.8.8.8"),
+		},
+		{
+			Name: "X-Forwarded-For with garbage client-supplied",
+			Request: http.Request{
+				Header: http.Header{
+					"X-Forwarded-For": []string{"asd;lfkjaasdf;lk,,8.8.8.8,8.8.8.9"},
+				},
+				RemoteAddr: "127.0.0.1:8888",
+			},
+			ExpectedIP: netip.MustParseAddr("8.8.8.8"),
+		},
+		{
+			Name: "Bogus crafted non-cloud X-Forwarded-For with no commas",
+			Request: http.Request{
+				Header: http.Header{
+					"X-Forwarded-For": []string{"8.8.8.8"},
+				},
+				RemoteAddr: "127.0.0.1:8888",
+			},
+			ExpectError: true,
+		},
+	}
+	for i := range testCases {
+		tc := testCases[i]
+		t.Run(tc.Name, func(t *testing.T) {
+			t.Parallel()
+			ip, err := getClientIP(&tc.Request)
+			if err != nil {
+				if !tc.ExpectError {
+					t.Fatalf("unexpted error: %v", err)
+				}
+			} else if tc.ExpectError {
+				t.Fatal("expected error but err was nil")
+			} else if ip != tc.ExpectedIP {
+				t.Fatalf("IP does not match expected IP got: %q, expected: %q", ip, tc.ExpectedIP)
+			}
+		})
+	}
+}

cmd/archeio/app/handlers.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"net/http"
+	"regexp"
+	"strings"
+
+	"k8s.io/klog/v2"
+
+	"sigs.k8s.io/oci-proxy/pkg/net/cidrs/aws"
+)
+
+// MakeHandler returns the root archeio HTTP handler
+//
+// upstream registry should be the url to the primary registry
+// archeio is fronting.
+func MakeHandler(upstreamRegistry string) http.Handler {
+	doV2 := makeV2Handler(upstreamRegistry)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// all valid registry requests should be at /v2/
+		// v1 API is super old and not supported by GCR anymore.
+		path := r.URL.Path
+		switch {
+		case strings.HasPrefix(path, "/v2/"):
+			doV2(w, r)
+		default:
+			klog.V(2).InfoS("unknown request", "path", path)
+			http.NotFound(w, r)
+		}
+	})
+}
+
+func makeV2Handler(upstreamRegistry string) func(w http.ResponseWriter, r *http.Request) {
+	// matches blob requests, captures the requested blob hash
+	reBlob := regexp.MustCompile("^/v2/.*/blobs/sha256:([0-9a-f]{64})$")
+	// initialize map of clientIP to AWS region
+	regionMapper := aws.NewAWSRegionMapper()
+	// capture these in a http handler lambda
+	return func(w http.ResponseWriter, r *http.Request) {
+		path := r.URL.Path
+		klog.V(2).InfoS("v2 request", "path", path)
+
+		// check if blob request
+		matches := reBlob.FindStringSubmatch(path)
+		if len(matches) != 2 {
+			// doesn't match so just forward it to the main upstream registry
+			http.Redirect(w, r, upstreamRegistry+path, http.StatusPermanentRedirect)
+			return
+		}
+
+		// for matches, identify the appropriate backend
+		clientIP, err := getClientIP(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+		}
+
+		region, matched := regionMapper.GetIP(clientIP)
+		if !matched {
+			// no region match, redirect to main upstream registry
+			http.Redirect(w, r, upstreamRegistry+path, http.StatusPermanentRedirect)
+			return
+		}
+
+		bucket := regionToBucket(region)
+		hash := matches[1]
+		// blobs are in the buckets are stored at /containers/images/sha256:$hash
+		// this matches the GCS bucket backing GCR
+		http.Redirect(w, r, bucket+"/containers/images/sha256%3A"+hash, http.StatusPermanentRedirect)
+	}
+}