diff --git a/.golangci.yaml b/.golangci.yaml index 576bdc94108..aba409ccaa2 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -3,6 +3,7 @@ # options for analysis running run: + go: '1.16' # default concurrency is a available CPU number concurrency: 4 @@ -44,4 +45,4 @@ linters: - gci - ineffassign - misspell - - vet \ No newline at end of file + - vet diff --git a/Makefile b/Makefile index bf0edbb52ee..84896ff1087 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ vet: # make release WHAT="yurthub yurt-controller-manager yurtctl-servant" ARCH="arm64 arm" REGION=cn # # # compile all components with all architectures (i.e., amd64, arm64, arm) -# make relase +# make release release: bash hack/make-rules/release-images.sh diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index b3bda16ba8d..d111ac7f113 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -123,9 +123,10 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { if err != nil { return nil, err } - registerInformers(sharedFactory, yurtSharedFactory, workingMode, serviceTopologyFilterEnabled(options), options.NodePoolName, options.NodeName) - + tenantNs := util.ParseTenantNs(options.YurtHubCertOrganizations) + registerInformers(sharedFactory, yurtSharedFactory, workingMode, serviceTopologyFilterEnabled(options), options.NodePoolName, options.NodeName, tenantNs) filterManager, err := createFilterManager(options, sharedFactory, yurtSharedFactory, serializerManager, storageWrapper, us[0].Host, proxySecureServerDummyAddr, proxySecureServerAddr) + if err != nil { klog.Errorf("could not create filter manager, %v", err) return nil, err @@ -226,7 +227,8 @@ func registerInformers(informerFactory informers.SharedInformerFactory, yurtInformerFactory yurtinformers.SharedInformerFactory, workingMode util.WorkingMode, serviceTopologyFilterEnabled bool, - nodePoolName, nodeName string) { + nodePoolName, nodeName string, + tenantNs string) { // skip construct node/nodePool informers if service topology filter disabled if serviceTopologyFilterEnabled { if workingMode == util.WorkingModeCloud { @@ -258,6 +260,15 @@ func registerInformers(informerFactory informers.SharedInformerFactory, return coreinformers.NewFilteredConfigMapInformer(client, util.YurtHubNamespace, resyncPeriod, nil, tweakListOptions) } informerFactory.InformerFor(&corev1.ConfigMap{}, newConfigmapInformer) + + if tenantNs != "" { + newSecretInformer := func(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + + return coreinformers.NewFilteredSecretInformer(client, tenantNs, resyncPeriod, nil, nil) + } + informerFactory.InformerFor(&corev1.Secret{}, newSecretInformer) + } + } // registerAllFilters by order, the front registered filter will be diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 05ab84ee4bd..f3dd6a0e5ef 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -36,6 +36,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/network" "github.com/openyurtio/openyurt/pkg/yurthub/proxy" "github.com/openyurtio/openyurt/pkg/yurthub/server" + "github.com/openyurtio/openyurt/pkg/yurthub/tenant" "github.com/openyurtio/openyurt/pkg/yurthub/transport" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -154,8 +155,13 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { } trace++ + klog.Infof("%d. new tenant sa manager", trace) + tenantMgr := tenant.New(cfg.YurtHubCertOrganizations, cfg.SharedFactory, stopCh) + trace++ + klog.Infof("%d. new reverse proxy handler for remote servers", trace) - yurtProxyHandler, err := proxy.NewYurtReverseProxyHandler(cfg, cacheMgr, transportManager, healthChecker, certManager, stopCh) + yurtProxyHandler, err := proxy.NewYurtReverseProxyHandler(cfg, cacheMgr, transportManager, healthChecker, certManager, tenantMgr, stopCh) + if err != nil { return fmt.Errorf("could not create reverse proxy handler, %v", err) } diff --git a/go.mod b/go.mod index 44476a2b058..222f03a118c 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 google.golang.org/grpc v1.27.1 gopkg.in/cheggaaa/pb.v1 v1.0.25 + gopkg.in/square/go-jose.v2 v2.2.2 k8s.io/api v0.22.3 k8s.io/apimachinery v0.22.3 k8s.io/apiserver v0.20.11 diff --git a/go.sum b/go.sum index 58643322d15..b52c1f400eb 100644 --- a/go.sum +++ b/go.sum @@ -902,4 +902,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= \ No newline at end of file diff --git a/pkg/yurthub/proxy/proxy.go b/pkg/yurthub/proxy/proxy.go index ca85af697e1..2d69157b6ab 100644 --- a/pkg/yurthub/proxy/proxy.go +++ b/pkg/yurthub/proxy/proxy.go @@ -24,6 +24,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/filters" apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server" + "k8s.io/klog/v2" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" @@ -32,6 +33,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/proxy/local" "github.com/openyurtio/openyurt/pkg/yurthub/proxy/remote" "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" + "github.com/openyurtio/openyurt/pkg/yurthub/tenant" "github.com/openyurtio/openyurt/pkg/yurthub/transport" hubutil "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -43,6 +45,7 @@ type yurtReverseProxy struct { localProxy *local.LocalProxy cacheMgr cachemanager.CacheManager maxRequestsInFlight int + tenantMgr tenant.Interface stopCh <-chan struct{} } @@ -54,6 +57,7 @@ func NewYurtReverseProxyHandler( transportMgr transport.Interface, healthChecker healthchecker.HealthChecker, certManager interfaces.YurtCertificateManager, + tenantMgr tenant.Interface, stopCh <-chan struct{}) (http.Handler, error) { cfg := &server.Config{ LegacyAPIGroupPrefixes: sets.NewString(server.DefaultLegacyAPIPrefix), @@ -87,6 +91,7 @@ func NewYurtReverseProxyHandler( localProxy: localProxy, cacheMgr: cacheMgr, maxRequestsInFlight: yurtHubCfg.MaxRequestInFlight, + tenantMgr: tenantMgr, stopCh: stopCh, } @@ -105,7 +110,15 @@ func (p *yurtReverseProxy) buildHandlerChain(handler http.Handler) http.Handler } handler = util.WithMaxInFlightLimit(handler, p.maxRequestsInFlight) handler = util.WithRequestClientComponent(handler) + + if p.tenantMgr != nil && p.tenantMgr.GetTenantNs() != "" { + handler = util.WithSaTokenSubstitute(handler, p.tenantMgr) + } else { + klog.V(2).Info("tenant ns is empty, no need to substitute ") + } + handler = filters.WithRequestInfo(handler, p.resolver) + return handler } diff --git a/pkg/yurthub/proxy/util/util.go b/pkg/yurthub/proxy/util/util.go index aa9c208c2a7..cfde7c29308 100644 --- a/pkg/yurthub/proxy/util/util.go +++ b/pkg/yurthub/proxy/util/util.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The OpenYurt Authors. +Copyright 2022 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,20 +18,24 @@ package util import ( "context" + "fmt" "net/http" "strings" "time" + "gopkg.in/square/go-jose.v2/jwt" "k8s.io/apimachinery/pkg/api/errors" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apiserver/pkg/authentication/serviceaccount" apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/yurthub/metrics" + "github.com/openyurtio/openyurt/pkg/yurthub/tenant" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -147,6 +151,7 @@ func WithRequestClientComponent(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { ctx := req.Context() if info, ok := apirequest.RequestInfoFrom(ctx); ok { + if info.IsResourceRequest { var comp string userAgent := strings.ToLower(req.Header.Get("User-Agent")) @@ -295,3 +300,36 @@ func WithRequestTimeout(handler http.Handler) http.Handler { handler.ServeHTTP(w, req) }) } + +func WithSaTokenSubstitute(handler http.Handler, tenantMgr tenant.Interface) http.Handler { + + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + + if oldToken := util.ParseBearerToken(req.Header.Get("Authorization")); oldToken != "" { //bearer token is not empty&valid + + if jsonWebToken, err := jwt.ParseSigned(oldToken); err != nil { + + klog.Errorf("invaled bearer token %s, err: %v", oldToken, err) + } else { + oldClaim := jwt.Claims{} + + if err := jsonWebToken.UnsafeClaimsWithoutVerification(&oldClaim); err == nil { + + if tenantNs, _, err := serviceaccount.SplitUsername(oldClaim.Subject); err == nil { + + if tenantMgr.GetTenantNs() != tenantNs && tenantNs == "kube-system" && tenantMgr.WaitForCacheSync() { // token is not from tenant's namespace + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tenantMgr.GetTenantToken())) + klog.V(2).Infof("replace token, old: %s, new: %s", oldToken, tenantMgr.GetTenantToken()) + } + + } else { + klog.Errorf("failed to parse tenant ns from token, token %s, sub: %s", oldToken, oldClaim.Subject) + } + } + } + + } + + handler.ServeHTTP(w, req) + }) +} diff --git a/pkg/yurthub/proxy/util/util_test.go b/pkg/yurthub/proxy/util/util_test.go index 836b2c5b567..c6adbc1f1b9 100644 --- a/pkg/yurthub/proxy/util/util_test.go +++ b/pkg/yurthub/proxy/util/util_test.go @@ -18,16 +18,19 @@ package util import ( "context" + "fmt" "net/http" "net/http/httptest" "sync" "testing" "time" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/endpoints/filters" "k8s.io/apiserver/pkg/endpoints/request" + "github.com/openyurtio/openyurt/pkg/yurthub/tenant" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -380,3 +383,191 @@ func TestWithListRequestSelector(t *testing.T) { }) } } + +func TestWithSaTokenSubsitute(t *testing.T) { + //jwt token with algorithm RS256 + tenantToken := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjpbeyJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L25hbWVzcGFjZSI6ImlvdC10ZXN0In0seyJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGVmYXVsdCJ9XSwiaWF0IjoxNjQ4NzkzNTI3LCJleHAiOjM3MzE1ODcxOTksImF1ZCI6IiIsImlzcyI6Imt1YmVybmV0ZXMvc2VydmljZWFjY291bnQiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.9N5ChVgM67BbUDmW2B5ziRyW5JTJYxLKPfFd57wbC-c" + + testcases := map[string]struct { + Verb string + Path string + Token string + NeedSubsitute bool + }{ + "1.no token, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/pods?resourceVersion=1494416105", + Token: "", + NeedSubsitute: false, + }, + "2.iot-test, no token, GET, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/iot-test/pods?resourceVersion=1494416105&fieldSelector=metadata.name=test", + NeedSubsitute: false, + }, + "3.iot-test, tenant token, LIST, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/iot-test/pods?resourceVersion=1494416105", + Token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJpb3QtdGVzdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.TYA_QK5OUN1Hmnurf27zPj-Xmh6Fxe67EzEtNI0OouElA_6FEYfuD98g2xBaUcSFZrc97ILC102gtRYX5a_IPvAgeke9WuqwoaxaA-DxMj_cUt5FUri1PEcSmtIUNM3XPgL3UebZxFn_bG_sZwYePIb7ryq4E_1XfaEA3uYO27BwuDbMxhmU6Hwsz4yKQfJDts-2SRnmG8uEc70svtgfqSBhv7EZim1S7lFY87je28sES2w-WXvWTszaUx8707QdVJjntqcxAvFUGskXQoO_hEI88xnz_-F4NX2Wiv1Mew52Srmpyh2vwTRW3TWn9_-4Lh0X9OBqnlWV0ZjElvJZig", + NeedSubsitute: false, + }, + "4.kube-system, GET, invalid token, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&fieldSelector=metadata.name=test", + Token: "invalidToken", + NeedSubsitute: false, + }, + "5.kube-system, tenantNs iot-test001, LIST, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105", + Token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJpb3QtdGVzdDAwMSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.HrjxSSuvb-MncngvIL1rh4FnGWVZYtNfB-l8rvysP9nqGcTbKnOw5KF0SDiCvoZEK_SNYi2gJH84onsOnG7Wh7ZIjv0KbptQpVrG0dFSW6qElH_5wr2LL1_YLUalHYMmFl9jq9cD7YmXBh9B38ApuCyBIbRxOlk3QiB_ZEoSSNJX-oivHPDmoXFM2ehxaJA9cMl_i-8OSaFKaW8ptn4hN5LobI14LG2QDTNspmJqeIS5SIucl4cBJ5rRtmY6SVatGqUDsUekL-KfK0RrX4H30cTaDDJF2yLRoUvHt7fa6hDZFwvg-dh3af2aYg1_C0vGqAuLc26V12DKYPp_EIoGrg", + NeedSubsitute: false, + }, + "6.kube-system, WATCH, tenantNs iot-test001, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&watch=true", + Token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJpb3QtdGVzdDAwMSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.HrjxSSuvb-MncngvIL1rh4FnGWVZYtNfB-l8rvysP9nqGcTbKnOw5KF0SDiCvoZEK_SNYi2gJH84onsOnG7Wh7ZIjv0KbptQpVrG0dFSW6qElH_5wr2LL1_YLUalHYMmFl9jq9cD7YmXBh9B38ApuCyBIbRxOlk3QiB_ZEoSSNJX-oivHPDmoXFM2ehxaJA9cMl_i-8OSaFKaW8ptn4hN5LobI14LG2QDTNspmJqeIS5SIucl4cBJ5rRtmY6SVatGqUDsUekL-KfK0RrX4H30cTaDDJF2yLRoUvHt7fa6hDZFwvg-dh3af2aYg1_C0vGqAuLc26V12DKYPp_EIoGrg", + NeedSubsitute: false, + }, + "7.kube-system, WATCH, tenantNs kube-system, need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&watch=true", + Token: "eyJhbGciOiJSUzI1NiIsImtpZCI6InVfTVZpZWIySUFUTzQ4NjlkM0VwTlBRb0xJOWVKUGg1ZXVzbEdaY0ZxckEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06ZGVmYXVsdCJ9.sFpHHg4o88Z0CBJseMBvBeP00bS5isLBmQJpAOiYs3BTkEAD63YLTnDURt0r3I9QjtcP0DZAb5wSOccGChMAFVtxMIoIoZC6Mk4FSB720kawRxFVujNFR1T7uVV_dbpEU-wsxSb9-Y4ILVknuJR9t35x6lUbRkUE9tN1wDy4DH296C3gEGNJf8sbJMERZzOckc82_BamlCzaieo1nX396KafxdQGVIgxstx88hm_rgpjDy3LA1GNsx6x2pqXdzZ8mufQt7sTljRorXUk-rNU6y9wX2RvIMO8tNiPClNkdIpgpmeQo-g7XZivpEeq3VzoeExphRbusgCtO9T9tgU64w", + NeedSubsitute: true, + }, + } + + resolver := newTestRequestInfoResolver() + orgs := []string{"system:bootstrappers:openyurt:tenant:myspace"} + + stopCh := make(<-chan struct{}) + tenantMgr := tenant.New(orgs, nil, stopCh) + + data := make(map[string][]byte) + data["token"] = []byte(tenantToken) + secret := v1.Secret{ + Data: data, + } + tenantMgr.SetSecret(&secret) + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + req, _ := http.NewRequest(tc.Verb, tc.Path, nil) + req.RemoteAddr = "127.0.0.1" + if tc.Token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tc.Token)) + + } + + var needSubsitute bool + var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + rToken := req.Header.Get("Authorization") + if rToken == fmt.Sprintf("Bearer %s", tenantToken) { + needSubsitute = true + } + + }) + + handler = WithSaTokenSubstitute(handler, tenantMgr) + handler = filters.WithRequestInfo(handler, resolver) + + handler.ServeHTTP(httptest.NewRecorder(), req) + + if tc.NeedSubsitute != needSubsitute { + t.Errorf("expect needSubsited %v, but got %v", tc.NeedSubsitute, needSubsitute) + } + + }) + } +} + +func TestWithSaTokenSubsituteTenantTokenEmpty(t *testing.T) { + + //jwt token with algorithm RS256 + tenantToken := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjpbeyJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L25hbWVzcGFjZSI6ImlvdC10ZXN0In0seyJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGVmYXVsdCJ9XSwiaWF0IjoxNjQ4NzkzNTI3LCJleHAiOjM3MzE1ODcxOTksImF1ZCI6IiIsImlzcyI6Imt1YmVybmV0ZXMvc2VydmljZWFjY291bnQiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.9N5ChVgM67BbUDmW2B5ziRyW5JTJYxLKPfFd57wbC-c" + testcases := map[string]struct { + Verb string + Path string + Token string + NeedSubsitute bool + }{ + "no token, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/pods?resourceVersion=1494416105", + Token: "", + NeedSubsitute: false, + }, + "iot-test, no token, GET, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/iot-test/pods?resourceVersion=1494416105&fieldSelector=metadata.name=test", + NeedSubsitute: false, + }, + "iot-test, tenant token, LIST, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/iot-test/pods?resourceVersion=1494416105", + Token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJpb3QtdGVzdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.TYA_QK5OUN1Hmnurf27zPj-Xmh6Fxe67EzEtNI0OouElA_6FEYfuD98g2xBaUcSFZrc97ILC102gtRYX5a_IPvAgeke9WuqwoaxaA-DxMj_cUt5FUri1PEcSmtIUNM3XPgL3UebZxFn_bG_sZwYePIb7ryq4E_1XfaEA3uYO27BwuDbMxhmU6Hwsz4yKQfJDts-2SRnmG8uEc70svtgfqSBhv7EZim1S7lFY87je28sES2w-WXvWTszaUx8707QdVJjntqcxAvFUGskXQoO_hEI88xnz_-F4NX2Wiv1Mew52Srmpyh2vwTRW3TWn9_-4Lh0X9OBqnlWV0ZjElvJZig", + NeedSubsitute: false, + }, + "kube-system, GET, invalid token, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&fieldSelector=metadata.name=test", + Token: "invalidToken", + NeedSubsitute: false, + }, + "kube-system, tenantNs iot-test001, LIST, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105", + Token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJpb3QtdGVzdDAwMSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.HrjxSSuvb-MncngvIL1rh4FnGWVZYtNfB-l8rvysP9nqGcTbKnOw5KF0SDiCvoZEK_SNYi2gJH84onsOnG7Wh7ZIjv0KbptQpVrG0dFSW6qElH_5wr2LL1_YLUalHYMmFl9jq9cD7YmXBh9B38ApuCyBIbRxOlk3QiB_ZEoSSNJX-oivHPDmoXFM2ehxaJA9cMl_i-8OSaFKaW8ptn4hN5LobI14LG2QDTNspmJqeIS5SIucl4cBJ5rRtmY6SVatGqUDsUekL-KfK0RrX4H30cTaDDJF2yLRoUvHt7fa6hDZFwvg-dh3af2aYg1_C0vGqAuLc26V12DKYPp_EIoGrg", + NeedSubsitute: false, + }, + "kube-system, WATCH, tenantNs iot-test001, no need to subsitute bearer token": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&watch=true", + Token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJpb3QtdGVzdDAwMSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW90LXRlc3Q6ZGVmYXVsdCJ9.HrjxSSuvb-MncngvIL1rh4FnGWVZYtNfB-l8rvysP9nqGcTbKnOw5KF0SDiCvoZEK_SNYi2gJH84onsOnG7Wh7ZIjv0KbptQpVrG0dFSW6qElH_5wr2LL1_YLUalHYMmFl9jq9cD7YmXBh9B38ApuCyBIbRxOlk3QiB_ZEoSSNJX-oivHPDmoXFM2ehxaJA9cMl_i-8OSaFKaW8ptn4hN5LobI14LG2QDTNspmJqeIS5SIucl4cBJ5rRtmY6SVatGqUDsUekL-KfK0RrX4H30cTaDDJF2yLRoUvHt7fa6hDZFwvg-dh3af2aYg1_C0vGqAuLc26V12DKYPp_EIoGrg", + NeedSubsitute: false, + }, + } + + resolver := newTestRequestInfoResolver() + orgs := []string{"system:bootstrappers:openyurt:tenant:myspace"} + + stopCh := make(<-chan struct{}) + tenantMgr := tenant.New(orgs, nil, stopCh) + + data := make(map[string][]byte) + data["token"] = []byte(tenantToken) + secret := v1.Secret{ + Data: data, + } + tenantMgr.SetSecret(&secret) + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + req, _ := http.NewRequest(tc.Verb, tc.Path, nil) + req.RemoteAddr = "127.0.0.1" + if tc.Token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tc.Token)) + + } + + var needSubsitute bool + var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + rToken := req.Header.Get("Authorization") + if rToken == fmt.Sprintf("Bearer %s", tenantToken) { + needSubsitute = true + } + + }) + + handler = WithSaTokenSubstitute(handler, tenantMgr) + handler = filters.WithRequestInfo(handler, resolver) + + handler.ServeHTTP(httptest.NewRecorder(), req) + + if tc.NeedSubsitute != needSubsitute { + t.Errorf("expect needSubsited %v, but got %v", tc.NeedSubsitute, needSubsitute) + } + + }) + } +} diff --git a/pkg/yurthub/tenant/jwt/jwt_test.go b/pkg/yurthub/tenant/jwt/jwt_test.go new file mode 100644 index 00000000000..fe30eb604f9 --- /dev/null +++ b/pkg/yurthub/tenant/jwt/jwt_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package jwt + +import ( + "testing" + + "gopkg.in/square/go-jose.v2/jwt" + "k8s.io/apiserver/pkg/authentication/serviceaccount" +) + +func TestJwt(t *testing.T) { + + bearerToken := "eyJhbGciOiJSUzI1NiIsImtpZCI6InVfTVZpZWIySUFUTzQ4NjlkM0VwTlBRb0xJOWVKUGg1ZXVzbEdaY0ZxckEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXF3c2ZtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4M2EwMzc4ZS1mY2UxLTRmZDEtOGI1NC00MTE2MjUzYzNkYWMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06ZGVmYXVsdCJ9.sFpHHg4o88Z0CBJseMBvBeP00bS5isLBmQJpAOiYs3BTkEAD63YLTnDURt0r3I9QjtcP0DZAb5wSOccGChMAFVtxMIoIoZC6Mk4FSB720kawRxFVujNFR1T7uVV_dbpEU-wsxSb9-Y4ILVknuJR9t35x6lUbRkUE9tN1wDy4DH296C3gEGNJf8sbJMERZzOckc82_BamlCzaieo1nX396KafxdQGVIgxstx88hm_rgpjDy3LA1GNsx6x2pqXdzZ8mufQt7sTljRorXUk-rNU6y9wX2RvIMO8tNiPClNkdIpgpmeQo-g7XZivpEeq3VzoeExphRbusgCtO9T9tgU64w" + var bearerClaims = jwt.Claims{} + + if token, err := jwt.ParseSigned(bearerToken); err == nil { + if err := token.UnsafeClaimsWithoutVerification(&bearerClaims); err == nil { + + if tenantNs, username, err := serviceaccount.SplitUsername(bearerClaims.Subject); err == nil { + t.Logf("succeed to parse toke, ns: %s, username: %s", tenantNs, username) + } else { + t.Errorf("failed to parse jwt token, %v", err) + } + } else { + t.Errorf("failed to parse jwt token, %v", err) + } + } else { + t.Errorf("failed to parse jwt token, %v", err) + } +} diff --git a/pkg/yurthub/tenant/tenant.go b/pkg/yurthub/tenant/tenant.go new file mode 100644 index 00000000000..431ca08e4f5 --- /dev/null +++ b/pkg/yurthub/tenant/tenant.go @@ -0,0 +1,166 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package tenant + +import ( + "sync" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +type Interface interface { + GetTenantNs() string + + GetTenantToken() string + + WaitForCacheSync() bool + + SetSecret(sec *v1.Secret) +} + +type tenantManager struct { + secretSynced cache.InformerSynced + + TenantSecret *v1.Secret + + TenantNs string + + StopCh <-chan struct{} + + IsSynced bool + + mutex sync.Mutex +} + +func (mgr *tenantManager) WaitForCacheSync() bool { + + if mgr.IsSynced || mgr.TenantSecret != nil { //try to do sync for just one timeļ¼Œ fast return + return true + } + + mgr.mutex.Lock() + defer mgr.mutex.Unlock() + + mgr.IsSynced = cache.WaitForCacheSync(mgr.StopCh, mgr.secretSynced) + + return mgr.IsSynced +} + +func New(orgs []string, factory informers.SharedInformerFactory, stopCh <-chan struct{}) Interface { + + tenantNs := util.ParseTenantNsFromOrgs(orgs) + klog.Infof("parse tenant ns: %s", tenantNs) + if tenantNs == "" { + return nil + } + + tenantMgr := &tenantManager{TenantNs: tenantNs, StopCh: stopCh} + + if factory != nil { + informer := factory.InformerFor(&v1.Secret{}, nil) //get registered secret informer + + tenantMgr.secretSynced = informer.HasSynced + //add handlers + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: tenantMgr.addSecret, + UpdateFunc: tenantMgr.updateSecret, + DeleteFunc: tenantMgr.deleteSecret}) + } + + return tenantMgr + +} + +func (mgr *tenantManager) GetTenantToken() string { + + if mgr.TenantSecret == nil { + + return "" + } + + return string(mgr.TenantSecret.Data["token"]) +} + +func (mgr *tenantManager) GetTenantNs() string { + + return mgr.TenantNs +} + +func (mgr *tenantManager) InformerSynced() cache.InformerSynced { + + return mgr.secretSynced +} + +func (mgr *tenantManager) SetSecret(sec *v1.Secret) { + mgr.TenantSecret = sec +} + +func (mgr *tenantManager) addSecret(sec interface{}) { + + secret, ok := sec.(*v1.Secret) + if !ok { + klog.Errorf("failed to convert to *v1.Secret") + return + } + + klog.Infof("secret added %s", secret.GetName()) + + if IsDefaultSASecret(secret) { //found it + + mgr.TenantSecret = secret + } + +} + +func (mgr *tenantManager) deleteSecret(sec interface{}) { + secret, ok := sec.(*v1.Secret) + if !ok { + klog.Errorf("failed to convert to *v1.Secret") + return + } + + klog.Infof("secret deleted %s", secret.GetName()) + + if IsDefaultSASecret(secret) { //found it + mgr.TenantSecret = nil + } +} + +func (mgr *tenantManager) updateSecret(oldSec interface{}, newSec interface{}) { + + secret, ok := newSec.(*v1.Secret) + if !ok { + klog.Errorf("failed to convert to *v1.Secret") + return + } + + klog.Infof("secret updated %s", secret.GetName()) + if IsDefaultSASecret(secret) { //found it + + mgr.TenantSecret = secret + } +} + +func IsDefaultSASecret(secret *v1.Secret) bool { + + return secret.Type == v1.SecretTypeServiceAccountToken && + (len(secret.Annotations) != 0 && secret.Annotations[v1.ServiceAccountNameKey] == "default") +} diff --git a/pkg/yurthub/util/util.go b/pkg/yurthub/util/util.go index 50be9683a2e..bc320faf309 100644 --- a/pkg/yurthub/util/util.go +++ b/pkg/yurthub/util/util.go @@ -473,3 +473,46 @@ func NewGZipReaderCloser(header http.Header, body io.ReadCloser, req *http.Reque body: body, }, true } + +func ParseTenantNs(certOrg string) string { + + if !strings.Contains(certOrg, "openyurt:tenant:") { + return "" + } + + return strings.TrimPrefix(certOrg, "openyurt:tenant:") +} + +func ParseTenantNsFromOrgs(orgs []string) string { + + if len(orgs) == 0 { + + return "" + } + + ns := "" + for _, v := range orgs { + + tns := ParseTenantNs(v) + + if tns != "" { + ns = tns + break + } + } + + return ns +} + +func ParseBearerToken(token string) string { + + if token == "" { + return "" + } + + if !strings.HasPrefix(token, "Bearer ") { //not invalid bearer token + return "" + } + + return strings.TrimPrefix(token, "Bearer ") +} diff --git a/pkg/yurthub/util/util_test.go b/pkg/yurthub/util/util_test.go index e5aa7e056a8..39ab50cebb3 100644 --- a/pkg/yurthub/util/util_test.go +++ b/pkg/yurthub/util/util_test.go @@ -18,6 +18,7 @@ package util import ( "bytes" + "encoding/base64" "io" "testing" ) @@ -217,3 +218,25 @@ func TestSplitKey(t *testing.T) { }) } } + +func TestParseTenantNs(t *testing.T) { + + testCases := map[string]string{ + "a": "", + "openyurt:tenant:myspace": "myspace", + } + + for k, v := range testCases { + + ns := ParseTenantNs(k) + if v != ns { + t.Errorf("%s is not equal to %s", v, ns) + } + + } + + token := "ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNkluVmZUVlpwWldJeVNVRlVUelE0Tmpsa00wVndUbEJSYjB4Sk9XVktVR2cxWlhWemJFZGFZMFp4Y2tFaWZRLmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUpwYjNRdGRHVnpkQ0lzSW10MVltVnlibVYwWlhNdWFXOHZjMlZ5ZG1salpXRmpZMjkxYm5RdmMyVmpjbVYwTG01aGJXVWlPaUprWldaaGRXeDBMWFJ2YTJWdUxYRjNjMlp0SWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXpaWEoyYVdObExXRmpZMjkxYm5RdWJtRnRaU0k2SW1SbFptRjFiSFFpTENKcmRXSmxjbTVsZEdWekxtbHZMM05sY25acFkyVmhZMk52ZFc1MEwzTmxjblpwWTJVdFlXTmpiM1Z1ZEM1MWFXUWlPaUk0TTJFd016YzRaUzFtWTJVeExUUm1aREV0T0dJMU5DMDBNVEUyTWpVell6TmtZV01pTENKemRXSWlPaUp6ZVhOMFpXMDZjMlZ5ZG1salpXRmpZMjkxYm5RNmFXOTBMWFJsYzNRNlpHVm1ZWFZzZENKOS5QM2xuc3NWSTZvVUg2U19CX2thYU1QWmV5Vm8xM2xCQU50aGVvb3ByY1ZnQWlIWXpNOVdBcUFPTi12c2h5RTBBcUFHTFl3Q1FsY0FReXhKNDZqbEd0TXJxUlhpRWIyMldobXVtRkswc3NNTGJkbHBOWmJjNzc5WmxoeXUyVDJnRTlKSExFMHUyUFkwQm5sQUlQZmtGYzZPZk9veklybDBGZUxGWklFY1MzQi1yTlUwYUZDekJZNEpsMThYdUpKOEhubHA4N3V1Q2FlLUZzWHJWajFIZUd4MWw4S2JzZVJwSkFrN0Q0aklPNDFndXRlSHV5MnE3SldHLUwyWWZ0VG1peWdEb2pqMlhFTkEyTkxrRXFLbG5NQ3BlSjFwUl82UjRKZ21OaTUzLWktTE5mTVNGWXNnckNMUWNNTkhiZkg1MEpBOXp0cHd1Y2xmWUl3WjBPZkdPOWc=" + + out, _ := base64.StdEncoding.DecodeString(token) + t.Logf("token: %s", string(out)) +} diff --git a/test/integration/yurttunnel_test.go b/test/integration/yurttunnel_test.go index ff954122bb5..fcc87597f35 100644 --- a/test/integration/yurttunnel_test.go +++ b/test/integration/yurttunnel_test.go @@ -79,10 +79,10 @@ func startDummyServer(t *testing.T) { w.WriteHeader(http.StatusOK) n, err := w.Write([]byte(DummyServerResponse)) if err != nil { - t.Fatalf("fail to write response: %v", err) + t.Errorf("fail to write response: %v", err) } if n != len([]byte(DummyServerResponse)) { - t.Fatalf("fail to write response: write %d of the %d bytes", + t.Errorf("fail to write response: write %d of the %d bytes", n, len([]byte(DummyServerResponse))) } }) @@ -100,7 +100,7 @@ func startDummyServer(t *testing.T) { klog.Infof("[TEST] dummy-server is listening on :%d", DummyServerPort) if err := s.ListenAndServeTLS("", ""); err != nil { - t.Fatalf("the dummy-server failed: %v", err) + t.Errorf("the dummy-server failed: %v", err) } } @@ -122,24 +122,24 @@ func startDummyClient(t *testing.T, wg *sync.WaitGroup) { r, err := http.NewRequest(http.MethodGet, "https://127.0.0.1:9515/"+HTTPPath, nil) if err != nil { - t.Fatalf("fail to generate http request: %v", err) + t.Errorf("fail to generate http request: %v", err) } rep, err := c.Do(r) if err != nil { - t.Fatalf("fail to send request to the server: %v", err) + t.Errorf("fail to send request to the server: %v", err) } if rep.StatusCode != http.StatusOK { - t.Fatalf("the response status code is incorrect, expect: %d, get: %d", + t.Errorf("the response status code is incorrect, expect: %d, get: %d", http.StatusOK, r.Response.StatusCode) } defer rep.Body.Close() content, err := io.ReadAll(rep.Body) if err != nil { - t.Fatalf("fail to read from the response body: %v", err) + t.Errorf("fail to read from the response body: %v", err) } if string(content) != DummyServerResponse { - t.Fatalf("fail to read from the response body: expect %s, get %s", + t.Errorf("fail to read from the response body: expect %s, get %s", DummyServerResponse, string(content)) }