From 21c70fcb1979a30778be93a0075c6e17c9cc31e5 Mon Sep 17 00:00:00 2001 From: Antonin Bas Date: Fri, 14 Apr 2023 18:53:31 -0700 Subject: [PATCH] Ability to proxy arbitrary K8s APIs We add a new API endpoint which proxies GET requests to the K8s api-server. This removes the need to define custom APIs for each use case that requires directly retrieving data from the K8s API. At the moment we can use this API for the web client "Summary" page, which displays information from the AntreaAgentInfo and AntreaControllerInfo CRs. As a result, the specialized "/info" endpoint is now deprecated, and will be removed in the near future (July 2023). The list of API resources being proxied is determined by: 1) a global variable in the backend code; if a client attempts to access an API that is not being proxied, the "Not Found" status code will be returned. Same if the HTTP verb being used is not GET. 2) K8s RBAC permissions for the antrea-ui backend; the list / get verbs must be granted for the resources being proxied. We use the Go standard library reverse proxy implementation. Note that the implementation was greatly improved in Go 1.20, but we currently use Go 1.19 for all Antrea projects. Fixes #10 Signed-off-by: Antonin Bas --- client/web/antrea-ui/src/api/info.tsx | 8 +-- cmd/server/main.go | 18 +++++-- pkg/handlers/k8sproxy/handler.go | 44 +++++++++++++++ pkg/handlers/k8sproxy/handler_test.go | 53 ++++++++++++++++++ pkg/k8s/client.go | 22 +++++++- pkg/server/info.go | 4 +- pkg/server/info_test.go | 9 ++++ pkg/server/k8s.go | 64 ++++++++++++++++++++++ pkg/server/k8s_test.go | 77 +++++++++++++++++++++++++++ pkg/server/server.go | 13 +++++ pkg/server/server_test.go | 28 +++++++++- 11 files changed, 327 insertions(+), 13 deletions(-) create mode 100644 pkg/handlers/k8sproxy/handler.go create mode 100644 pkg/handlers/k8sproxy/handler_test.go create mode 100644 pkg/server/k8s.go create mode 100644 pkg/server/k8s_test.go diff --git a/client/web/antrea-ui/src/api/info.tsx b/client/web/antrea-ui/src/api/info.tsx index 47b0b163..d9e9fd7c 100644 --- a/client/web/antrea-ui/src/api/info.tsx +++ b/client/web/antrea-ui/src/api/info.tsx @@ -79,7 +79,7 @@ export interface AgentInfo { export const controllerInfoAPI = { fetch: async (): Promise => { return api.get( - `info/controller`, + `k8s/apis/crd.antrea.io/v1beta1/antreacontrollerinfos/antrea-controller` ).then((response) => response.data as ControllerInfo).catch((error) => { console.error("Unable to fetch Controller Info"); handleError(error); @@ -90,8 +90,8 @@ export const controllerInfoAPI = { export const agentInfoAPI = { fetchAll: async (): Promise => { return api.get( - `info/agents`, - ).then((response) => response.data as AgentInfo[]).catch((error) => { + `k8s/apis/crd.antrea.io/v1beta1/antreaagentinfos`, + ).then((response) => response.data.items as AgentInfo[]).catch((error) => { console.error("Unable to fetch Agent Infos"); handleError(error); }); @@ -99,7 +99,7 @@ export const agentInfoAPI = { fetch: async (name: string): Promise => { return api.get( - `info/agents/${name}`, + `k8s/apis/crd.antrea.io/v1beta1/antreaagentinfos/${name}`, ).then((response) => response.data as AgentInfo).catch((error) => { console.error("Unable to fetch Agent Info"); handleError(error); diff --git a/cmd/server/main.go b/cmd/server/main.go index 10a9e357..a3b433f4 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -20,6 +20,7 @@ import ( "flag" "fmt" "net/http" + "net/url" "os" "time" @@ -32,6 +33,7 @@ import ( "antrea.io/antrea-ui/pkg/auth" "antrea.io/antrea-ui/pkg/env" + "antrea.io/antrea-ui/pkg/handlers/k8sproxy" traceflowhandler "antrea.io/antrea-ui/pkg/handlers/traceflow" "antrea.io/antrea-ui/pkg/k8s" "antrea.io/antrea-ui/pkg/password" @@ -98,13 +100,18 @@ func ginLogger(logger logr.Logger, level int) gin.HandlerFunc { func run() error { logger.Info("Starting Antrea UI backend", "version", version.GetFullVersionWithRuntimeInfo()) - k8sClient, err := k8s.DynamicClient() + k8sRESTConfig, k8sHTTPClient, k8sDynamicClient, err := k8s.Client() if err != nil { - return fmt.Errorf("failed to create K8s dynamic client: %w", err) + return fmt.Errorf("failed to create K8s client: %w", err) + } + k8sServerURL, err := url.Parse(k8sRESTConfig.Host) + if err != nil { + return fmt.Errorf("failed to parse K8s server URL '%s': %w", k8sRESTConfig.Host, err) } - traceflowHandler := traceflowhandler.NewRequestsHandler(logger, k8sClient) - passwordStore := password.NewStore(passwordrw.NewK8sSecret(env.GetNamespace(), "antrea-ui-passwd", k8sClient), passwordhasher.NewArgon2id()) + traceflowHandler := traceflowhandler.NewRequestsHandler(logger, k8sDynamicClient) + k8sProxyHandler := k8sproxy.NewK8sProxyHandler(logger, k8sServerURL, k8sHTTPClient.Transport) + passwordStore := password.NewStore(passwordrw.NewK8sSecret(env.GetNamespace(), "antrea-ui-passwd", k8sDynamicClient), passwordhasher.NewArgon2id()) if err := passwordStore.Init(context.Background()); err != nil { return err } @@ -125,8 +132,9 @@ func run() error { s := server.NewServer( logger, - k8sClient, + k8sDynamicClient, traceflowHandler, + k8sProxyHandler, passwordStore, tokenManager, server.SetCookieSecure(cookieSecure), diff --git a/pkg/handlers/k8sproxy/handler.go b/pkg/handlers/k8sproxy/handler.go new file mode 100644 index 00000000..5bc76731 --- /dev/null +++ b/pkg/handlers/k8sproxy/handler.go @@ -0,0 +1,44 @@ +// Copyright 2023 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8sproxy + +import ( + "net/http" + "net/http/httputil" + "net/url" + + "github.com/go-logr/logr" +) + +type transportWrapper struct { + logger logr.Logger + t http.RoundTripper +} + +func (w *transportWrapper) RoundTrip(r *http.Request) (*http.Response, error) { + w.logger.V(4).Info("Proxying request", "url", r.URL) + return w.t.RoundTrip(r) +} + +func NewK8sProxyHandler(logger logr.Logger, k8sServerURL *url.URL, k8sHTTPTransport http.RoundTripper) http.Handler { + // TODO: the httputil.ReverseProxy is much improved in Go v1.20, but we currently use Go + // v1.19. When we upgrade, we should revisit this code. + k8sReverseProxy := httputil.NewSingleHostReverseProxy(k8sServerURL) + k8sReverseProxy.Transport = &transportWrapper{ + logger: logger, + t: k8sHTTPTransport, + } + return k8sReverseProxy +} diff --git a/pkg/handlers/k8sproxy/handler_test.go b/pkg/handlers/k8sproxy/handler_test.go new file mode 100644 index 00000000..0e1db0a5 --- /dev/null +++ b/pkg/handlers/k8sproxy/handler_test.go @@ -0,0 +1,53 @@ +// Copyright 2023 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8sproxy + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestK8sProxyHandler(t *testing.T) { + var capturedReq *http.Request + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedReq = r + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + logger := testr.New(t) + serverURL, err := url.Parse(ts.URL) + require.NoError(t, err) + h := NewK8sProxyHandler(logger, serverURL, http.DefaultTransport) + + req, err := http.NewRequest("GET", "/api/v1/k8s/api/v1/pods", nil) + req.RemoteAddr = "127.0.0.1:32167" + require.NoError(t, err) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + require.Equal(t, http.StatusOK, rr.Code) + require.NotNil(t, capturedReq) + assert.Equal(t, "GET", capturedReq.Method) + assert.Equal(t, "/api/v1/k8s/api/v1/pods", capturedReq.URL.String()) + // TODO: after we improve the reverse proxy, we need to do more validation + header := capturedReq.Header + assert.Equal(t, "127.0.0.1", header.Get("X-Forwarded-For")) +} diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go index 582b1cd3..a8eafb18 100644 --- a/pkg/k8s/client.go +++ b/pkg/k8s/client.go @@ -16,9 +16,11 @@ package k8s import ( "flag" + "net/http" "os" "k8s.io/client-go/dynamic" + _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -32,7 +34,7 @@ func inCluster() bool { return inCluster } -func DynamicClient() (dynamic.Interface, error) { +func restConfig() (*rest.Config, error) { var config *rest.Config if inCluster() { var err error @@ -48,7 +50,23 @@ func DynamicClient() (dynamic.Interface, error) { return nil, err } } - return dynamic.NewForConfig(config) + return config, nil +} + +func Client() (*rest.Config, *http.Client, *dynamic.DynamicClient, error) { + config, err := restConfig() + if err != nil { + return nil, nil, nil, err + } + httpClient, err := rest.HTTPClientFor(config) + if err != nil { + return nil, nil, nil, err + } + client, err := dynamic.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, nil, nil, err + } + return config, httpClient, client, nil } func init() { diff --git a/pkg/server/info.go b/pkg/server/info.go index 0e9388ac..4808b5c3 100644 --- a/pkg/server/info.go +++ b/pkg/server/info.go @@ -17,6 +17,7 @@ package server import ( "fmt" "net/http" + "time" "github.com/gin-gonic/gin" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -101,7 +102,8 @@ func (s *server) GetAgentInfo(c *gin.Context) { func (s *server) AddInfoRoutes(r *gin.RouterGroup) { r = r.Group("/info") - r.Use(s.checkBearerToken) + removalDate := time.Date(2023, 7, 1, 0, 0, 0, 0, time.UTC) + r.Use(s.checkBearerToken, announceDeprecationMiddleware(removalDate, "use /k8s instead")) r.GET("/controller", s.GetControllerInfo) r.GET("/agents", s.GetAgentInfos) r.GET("/agents/:name", s.GetAgentInfo) diff --git a/pkg/server/info_test.go b/pkg/server/info_test.go index c2b9fef3..58cdc95b 100644 --- a/pkg/server/info_test.go +++ b/pkg/server/info_test.go @@ -44,6 +44,11 @@ func createTestControllerInfo(ctx context.Context, k8sClient dynamic.Interface, return err } +func checkInfoDeprecationHeaders(t *testing.T, header http.Header) { + assert.Equal(t, `299 - "Deprecated API: use /k8s instead"`, header.Get("Warning")) + assert.Equal(t, "Sat, 01 Jul 2023 00:00:00 GMT", header.Get("Sunset")) +} + func TestGetControllerInfo(t *testing.T) { ctx := context.Background() ts := newTestServer(t) @@ -56,6 +61,7 @@ func TestGetControllerInfo(t *testing.T) { ts.router.ServeHTTP(rr, req) assert.Equal(t, http.StatusOK, rr.Code) assert.Equal(t, "antrea-controller", gjson.GetBytes(rr.Body.Bytes(), "metadata.name").String()) + checkInfoDeprecationHeaders(t, rr.Result().Header) } func createTestAgentInfo(ctx context.Context, k8sClient dynamic.Interface, name string) error { @@ -87,6 +93,7 @@ func TestGetAgentInfo(t *testing.T) { ts.router.ServeHTTP(rr, req) assert.Equal(t, http.StatusOK, rr.Code) assert.Equal(t, "node-A", gjson.GetBytes(rr.Body.Bytes(), "metadata.name").String()) + checkInfoDeprecationHeaders(t, rr.Result().Header) }) t.Run("invalid name", func(t *testing.T) { @@ -96,6 +103,7 @@ func TestGetAgentInfo(t *testing.T) { rr := httptest.NewRecorder() ts.router.ServeHTTP(rr, req) assert.Equal(t, http.StatusNotFound, rr.Code) + checkInfoDeprecationHeaders(t, rr.Result().Header) }) } @@ -112,4 +120,5 @@ func TestGetAgentInfos(t *testing.T) { ts.router.ServeHTTP(rr, req) assert.Equal(t, http.StatusOK, rr.Code) assert.Len(t, gjson.ParseBytes(rr.Body.Bytes()).Array(), 2) + checkInfoDeprecationHeaders(t, rr.Result().Header) } diff --git a/pkg/server/k8s.go b/pkg/server/k8s.go new file mode 100644 index 00000000..59584e38 --- /dev/null +++ b/pkg/server/k8s.go @@ -0,0 +1,64 @@ +// Copyright 2023 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" +) + +// allowedPaths contains the K8s api paths that we are proxying. +// Note the leading slash, since the Gin "catch-all" parameter ("/*path") will include it. +var allowedPaths = []string{ + "/apis/crd.antrea.io/v1beta1/antreaagentinfos", + "/apis/crd.antrea.io/v1beta1/antreacontrollerinfos", +} + +func (s *server) GetK8s(c *gin.Context) { + // we need to strip the beginning of the path (/api/v1/k8s) before proxying + path := c.Param("path") + request := c.Request + request.URL.Path = path + // we also ensure that the Bearer Token is removed + request.Header.Del("Authorization") + s.k8sProxyHandler.ServeHTTP(c.Writer, c.Request) +} + +func (s *server) checkK8sPath(c *gin.Context) { + if sError := func() *serverError { + path := c.Param("path") + for _, allowedPath := range allowedPaths { + if strings.HasPrefix(path, allowedPath) { + return nil + } + } + return &serverError{ + code: http.StatusNotFound, + message: "This K8s API path is not being proxied", + } + }(); sError != nil { + s.HandleError(c, sError) + c.Abort() + return + } +} + +func (s *server) AddK8sRoutes(r *gin.RouterGroup) { + r = r.Group("/k8s") + r.Use(s.checkBearerToken) + r.GET("/*path", s.checkK8sPath, s.GetK8s) +} diff --git a/pkg/server/k8s_test.go b/pkg/server/k8s_test.go new file mode 100644 index 00000000..e3507c37 --- /dev/null +++ b/pkg/server/k8s_test.go @@ -0,0 +1,77 @@ +// Copyright 2023 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestK8sProxyRequest(t *testing.T) { + testCases := []struct { + name string + path string + expectedStatusCode int + expectedMessage string + }{ + { + name: "allowed path 1", + path: "/apis/crd.antrea.io/v1beta1/antreaagentinfos/node=A", + expectedStatusCode: http.StatusOK, + }, + { + name: "allowed path 2", + path: "/apis/crd.antrea.io/v1beta1/antreacontrollerinfos", + expectedStatusCode: http.StatusOK, + }, + { + name: "forbidden path", + path: "/api/v1/pods", + expectedStatusCode: http.StatusNotFound, + expectedMessage: "This K8s API path is not being proxied", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + ts := newTestServer(t) + path, err := url.JoinPath("/api/v1/k8s", tc.path) + require.NoError(t, err) + req, err := http.NewRequest("GET", path, nil) + require.NoError(t, err) + ts.authorizeRequest(req) + rr := httptest.NewRecorder() + ts.router.ServeHTTP(rr, req) + assert.Equal(t, tc.expectedStatusCode, rr.Code) + if rr.Code == http.StatusOK { + assert.Equal(t, tc.path, ts.k8sProxyHandler.request.URL.Path) + } + resp := rr.Result() + if tc.expectedMessage != "" { + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(body), tc.expectedMessage) + } + assert.Empty(t, resp.Header.Get("Authorization")) + }) + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 78e1e615..100a95c8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -15,8 +15,10 @@ package server import ( + "fmt" "net/http" "strings" + "time" "github.com/gin-gonic/gin" "github.com/go-logr/logr" @@ -32,6 +34,7 @@ type server struct { logger logr.Logger k8sClient dynamic.Interface traceflowRequestsHandler traceflow.RequestsHandler + k8sProxyHandler http.Handler passwordStore password.Store tokenManager auth.TokenManager config serverConfig @@ -68,6 +71,7 @@ func NewServer( logger logr.Logger, k8sClient dynamic.Interface, traceflowRequestsHandler traceflow.RequestsHandler, + k8sProxyHandler http.Handler, passwordStore password.Store, tokenManager auth.TokenManager, options ...ServerOptions, @@ -85,6 +89,7 @@ func NewServer( logger: logger, k8sClient: k8sClient, traceflowRequestsHandler: traceflowRequestsHandler, + k8sProxyHandler: k8sProxyHandler, passwordStore: passwordStore, tokenManager: tokenManager, config: config, @@ -122,6 +127,13 @@ func (s *server) checkBearerToken(c *gin.Context) { } } +func announceDeprecationMiddleware(removalDate time.Time, message string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Header("Warning", fmt.Sprintf(`299 - "Deprecated API: %s"`, message)) + c.Header("Sunset", removalDate.UTC().Format(http.TimeFormat)) + } +} + func (s *server) AddRoutes(router *gin.Engine) { router.GET("/healthz", func(c *gin.Context) { c.Status(http.StatusOK) @@ -134,4 +146,5 @@ func (s *server) AddRoutes(router *gin.Engine) { s.AddInfoRoutes(apiv1) s.AddAccountRoutes(apiv1) s.AddAuthRoutes(apiv1) + s.AddK8sRoutes(apiv1) } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 7a814299..e11c5425 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -18,6 +18,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/http/httputil" "testing" "time" @@ -42,11 +43,26 @@ func init() { gin.SetMode(gin.ReleaseMode) } +type testk8sProxyHandler struct { + request *http.Request +} + +func (h *testk8sProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.request = r + b, err := httputil.DumpRequest(r, false) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } else { + w.Write(b) + } +} + type testServer struct { s *server router *gin.Engine k8sClient *dynamicfake.FakeDynamicClient traceflowRequestsHandler *traceflowhandlertesting.MockRequestsHandler + k8sProxyHandler *testk8sProxyHandler passwordStore *passwordtesting.MockStore tokenManager *authtesting.MockTokenManager } @@ -58,9 +74,18 @@ func newTestServer(t *testing.T, options ...ServerOptions) *testServer { k8sClient := dynamicfake.NewSimpleDynamicClient(scheme) ctrl := gomock.NewController(t) traceflowRequestsHandler := traceflowhandlertesting.NewMockRequestsHandler(ctrl) + k8sProxyHandler := &testk8sProxyHandler{} passwordStore := passwordtesting.NewMockStore(ctrl) tokenManager := authtesting.NewMockTokenManager(ctrl) - s := NewServer(logger, k8sClient, traceflowRequestsHandler, passwordStore, tokenManager, options...) + s := NewServer( + logger, + k8sClient, + traceflowRequestsHandler, + k8sProxyHandler, + passwordStore, + tokenManager, + options..., + ) router := gin.Default() s.AddRoutes(router) return &testServer{ @@ -68,6 +93,7 @@ func newTestServer(t *testing.T, options ...ServerOptions) *testServer { router: router, k8sClient: k8sClient, traceflowRequestsHandler: traceflowRequestsHandler, + k8sProxyHandler: k8sProxyHandler, passwordStore: passwordStore, tokenManager: tokenManager, }