diff --git a/go.mod b/go.mod index ba29b8574e8e..d5bdcc5e7053 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ replace ( github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.1.1 github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.25.0-k3s1 - github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.4 + github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.5 github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.10.1 go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.3-k3s1 @@ -101,8 +101,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/rancher/dynamiclistener v0.3.5 github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc - github.com/rancher/remotedialer v0.2.6-0.20220624190122-ea57207bf2b8 - github.com/rancher/wharfie v0.5.1 + github.com/rancher/remotedialer v0.3.0 + github.com/rancher/wharfie v0.5.3 github.com/rancher/wrangler v1.1.1 github.com/robfig/cron/v3 v3.0.1 github.com/rootless-containers/rootlesskit v1.0.1 @@ -236,7 +236,7 @@ require ( github.com/google/cel-go v0.12.6 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/go-containerregistry v0.6.1-0.20211111182346-7a6ee45528a9 // indirect + github.com/google/go-containerregistry v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/gax-go/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index 5243ceb3508e..598e75ca7945 100644 --- a/go.sum +++ b/go.sum @@ -85,7 +85,6 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -203,7 +202,7 @@ github.com/containerd/nri v0.1.0 h1:6QioHRlThlKh2RkRTR4kIT3PKAcrLo3gIWnjkM4dQmQ= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/stargz-snapshotter v0.13.0 h1:3zr1/IkW1aEo6cMYTQeZ4L2jSuCN+F4kgGfjnuowe4U= github.com/containerd/stargz-snapshotter v0.13.0/go.mod h1:01uOvoNzN1T4kV+8HeVt9p29esO5/61x8+VP/KU4fvQ= -github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= +github.com/containerd/stargz-snapshotter/estargz v0.10.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= github.com/containerd/stargz-snapshotter/estargz v0.13.0 h1:fD7AwuVV+B40p0d9qVkH/Au1qhp8hn/HWJHIYjpEcfw= github.com/containerd/stargz-snapshotter/estargz v0.13.0/go.mod h1:m+9VaGJGlhCnrcEUod8mYumTmRgblwd3rC5UCEh2Yp0= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= @@ -262,7 +261,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v20.10.9+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.10+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.21+incompatible h1:qVkgyYUnOLQ98LtXBrwd/duVqPT2X4SHndOuGsfwyhU= github.com/docker/cli v20.10.21+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= @@ -442,8 +441,8 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.6.1-0.20211111182346-7a6ee45528a9 h1:m5Yu3eJF1gVvzA51M9Ll2TZH+fi8P38bBVDL2jdQzoU= -github.com/google/go-containerregistry v0.6.1-0.20211111182346-7a6ee45528a9/go.mod h1:VGc0QpDDhK6zwYGlQb3s0LDGbTCGMFiifm0UgG9v1oQ= +github.com/google/go-containerregistry v0.7.0 h1:u0onUUOcyoCDHEiJoyR1R1gx5er1+r06V5DBhUU5ndk= +github.com/google/go-containerregistry v0.7.0/go.mod h1:2zaoelrL0d08gGbpdP3LqyUuBmhWbpD6IOe2s9nLS2k= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -853,8 +852,8 @@ github.com/opencontainers/image-spec v1.0.2-0.20210730191737-8e42a01fb1b7/go.mod github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg= -github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= +github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w= @@ -929,10 +928,10 @@ github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S github.com/rancher/dynamiclistener v0.3.5/go.mod h1:dW/YF6/m2+uEyJ5VtEcd9THxda599HP6N9dSXk81+k0= github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc h1:29VHrInLV4qSevvcvhBj5UhQWkPShxrxv4AahYg2Scw= github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc/go.mod h1:dEfC9eFQigj95lv/JQ8K5e7+qQCacWs1aIA6nLxKzT8= -github.com/rancher/remotedialer v0.2.6-0.20220624190122-ea57207bf2b8 h1:leqh0chjBsXhKWebxxFd5QPcoQLu51EpaHo04ce0o+8= -github.com/rancher/remotedialer v0.2.6-0.20220624190122-ea57207bf2b8/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q= -github.com/rancher/wharfie v0.5.1 h1:TUqZyNj6BaGe2+tqhwAGwZouuwx02mvAMMjNuyejc5I= -github.com/rancher/wharfie v0.5.1/go.mod h1:5AHZRFBAOWYPDNCwj/y5Dpj+MMwXLoitPwxjYAIbcxQ= +github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcpvX1sM= +github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q= +github.com/rancher/wharfie v0.5.3 h1:6hiO26H7YTgChbLAE6JppxFRjaH3tbKfMItv/LqV0Q0= +github.com/rancher/wharfie v0.5.3/go.mod h1:Ebpai7digxegLroBseeC54XRBt5we3DgFS6kAE2ho+o= github.com/rancher/wrangler v1.1.1 h1:wmqUwqc2M7ADfXnBCJTFkTB5ZREWpD78rnZMzmxwMvM= github.com/rancher/wrangler v1.1.1/go.mod h1:ioVbKupzcBOdzsl55MvEDN0R1wdGggj8iNCYGFI5JvM= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= diff --git a/manifests/coredns.yaml b/manifests/coredns.yaml index 024b5f20ff5d..abe733390d5f 100644 --- a/manifests/coredns.yaml +++ b/manifests/coredns.yaml @@ -119,7 +119,7 @@ spec: k8s-app: kube-dns containers: - name: coredns - image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-coredns-coredns:1.9.4 + image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-coredns-coredns:1.10.1 imagePullPolicy: IfNotPresent resources: limits: diff --git a/manifests/local-storage.yaml b/manifests/local-storage.yaml index 915e67084218..cb6fcc5dc64b 100644 --- a/manifests/local-storage.yaml +++ b/manifests/local-storage.yaml @@ -67,7 +67,7 @@ spec: effect: "NoSchedule" containers: - name: local-path-provisioner - image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/local-path-provisioner:v0.0.23 + image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/local-path-provisioner:v0.0.24 imagePullPolicy: IfNotPresent command: - local-path-provisioner diff --git a/pkg/agent/loadbalancer/loadbalancer.go b/pkg/agent/loadbalancer/loadbalancer.go index efd64e445b0c..f47f4c38a38f 100644 --- a/pkg/agent/loadbalancer/loadbalancer.go +++ b/pkg/agent/loadbalancer/loadbalancer.go @@ -14,10 +14,25 @@ import ( "inet.af/tcpproxy" ) +// server tracks the connections to a server, so that they can be closed when the server is removed. +type server struct { + mutex sync.Mutex + connections map[net.Conn]struct{} +} + +// serverConn wraps a net.Conn so that it can be removed from the server's connection map when closed. +type serverConn struct { + server *server + net.Conn +} + +// LoadBalancer holds data for a local listener which forwards connections to a +// pool of remote servers. It is not a proper load-balancer in that it does not +// actually balance connections, but instead fails over to a new server only +// when a connection attempt to the currently selected server fails. type LoadBalancer struct { - mutex sync.Mutex - dialer *net.Dialer - proxy *tcpproxy.Proxy + mutex sync.Mutex + proxy *tcpproxy.Proxy serviceName string configFile string @@ -27,6 +42,7 @@ type LoadBalancer struct { ServerURL string ServerAddresses []string randomServers []string + servers map[string]*server currentServerAddress string nextServerIndex int Listener net.Listener @@ -40,6 +56,8 @@ var ( ETCDServerServiceName = version.Program + "-etcd-server-load-balancer" ) +// New contstructs a new LoadBalancer instance. The default server URL, and +// currently active servers, are stored in a file within the dataDir. func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPort int, isIPv6 bool) (_lb *LoadBalancer, _err error) { config := net.ListenConfig{Control: reusePort} var localAddress string @@ -76,11 +94,11 @@ func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPo lb := &LoadBalancer{ serviceName: serviceName, - dialer: &net.Dialer{}, configFile: filepath.Join(dataDir, "etc", serviceName+".json"), localAddress: localAddress, localServerURL: localServerURL, defaultServerAddress: defaultServerAddress, + servers: make(map[string]*server), ServerURL: serverURL, } @@ -103,14 +121,28 @@ func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPo if err := lb.proxy.Start(); err != nil { return nil, err } - logrus.Infof("Running load balancer %s %s -> %v", serviceName, lb.localAddress, lb.randomServers) + logrus.Infof("Running load balancer %s %s -> %v [default: %s]", serviceName, lb.localAddress, lb.ServerAddresses, lb.defaultServerAddress) return lb, nil } func (lb *LoadBalancer) SetDefault(serverAddress string) { - logrus.Infof("Updating load balancer %s default server address -> %s", lb.serviceName, serverAddress) + lb.mutex.Lock() + defer lb.mutex.Unlock() + + _, hasOriginalServer := sortServers(lb.ServerAddresses, lb.defaultServerAddress) + // if the old default server is not currently in use, remove it from the server map + if server := lb.servers[lb.defaultServerAddress]; server != nil && !hasOriginalServer { + defer server.closeAll() + delete(lb.servers, lb.defaultServerAddress) + } + // if the new default server doesn't have an entry in the map, add one + if _, ok := lb.servers[serverAddress]; !ok { + lb.servers[serverAddress] = &server{connections: make(map[net.Conn]struct{})} + } + lb.defaultServerAddress = serverAddress + logrus.Infof("Updated load balancer %s default server address -> %s", lb.serviceName, serverAddress) } func (lb *LoadBalancer) Update(serverAddresses []string) { @@ -120,7 +152,7 @@ func (lb *LoadBalancer) Update(serverAddresses []string) { if !lb.setServers(serverAddresses) { return } - logrus.Infof("Updating load balancer %s server addresses -> %v", lb.serviceName, lb.randomServers) + logrus.Infof("Updated load balancer %s server addresses -> %v [default: %s]", lb.serviceName, lb.ServerAddresses, lb.defaultServerAddress) if err := lb.writeConfig(); err != nil { logrus.Warnf("Error updating load balancer %s config: %s", lb.serviceName, err) @@ -139,18 +171,23 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, address string for { targetServer := lb.currentServerAddress - conn, err := lb.dialer.DialContext(ctx, network, targetServer) - if err == nil { - return conn, nil + server := lb.servers[targetServer] + if server == nil || targetServer == "" { + logrus.Debugf("Nil server for load balancer %s: %s", lb.serviceName, targetServer) + } else { + conn, err := server.dialContext(ctx, network, targetServer) + if err == nil { + return conn, nil + } + logrus.Debugf("Dial error from load balancer %s: %s", lb.serviceName, err) } - logrus.Debugf("Dial error from load balancer %s: %s", lb.serviceName, err) newServer, err := lb.nextServer(targetServer) if err != nil { return nil, err } if targetServer != newServer { - logrus.Debugf("Dial server in load balancer %s failed over to %s", lb.serviceName, newServer) + logrus.Debugf("Failed over to new server for load balancer %s: %s", lb.serviceName, newServer) } if ctx.Err() != nil { return nil, ctx.Err() @@ -167,7 +204,7 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, address string } func onDialError(src net.Conn, dstDialErr error) { - logrus.Debugf("Incoming conn %v, error dialing load balancer servers: %v", src.RemoteAddr().String(), dstDialErr) + logrus.Debugf("Incoming conn %s, error dialing load balancer servers: %v", src.RemoteAddr(), dstDialErr) src.Close() } diff --git a/pkg/agent/loadbalancer/loadbalancer_test.go b/pkg/agent/loadbalancer/loadbalancer_test.go index d36c503e5e1d..1cb26736e07c 100644 --- a/pkg/agent/loadbalancer/loadbalancer_test.go +++ b/pkg/agent/loadbalancer/loadbalancer_test.go @@ -3,30 +3,33 @@ package loadbalancer import ( "bufio" "context" - "errors" "fmt" "net" "net/url" - "os" "strings" "testing" "time" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/sirupsen/logrus" ) -type server struct { +func init() { + logrus.SetLevel(logrus.DebugLevel) +} + +type testServer struct { listener net.Listener conns []net.Conn prefix string } -func createServer(prefix string) (*server, error) { +func createServer(prefix string) (*testServer, error) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return nil, err } - s := &server{ + s := &testServer{ prefix: prefix, listener: listener, } @@ -34,7 +37,7 @@ func createServer(prefix string) (*server, error) { return s, nil } -func (s *server) serve() { +func (s *testServer) serve() { for { conn, err := s.listener.Accept() if err != nil { @@ -45,14 +48,14 @@ func (s *server) serve() { } } -func (s *server) close() { +func (s *testServer) close() { s.listener.Close() for _, conn := range s.conns { conn.Close() } } -func (s *server) echo(conn net.Conn) { +func (s *testServer) echo(conn net.Conn) { for { result, err := bufio.NewReader(conn).ReadString('\n') if err != nil { @@ -71,33 +74,17 @@ func ping(conn net.Conn) (string, error) { return strings.TrimSpace(result), nil } -func assertEqual(t *testing.T, a interface{}, b interface{}) { - if a != b { - t.Fatalf("[ %v != %v ]", a, b) - } -} - -func assertNotEqual(t *testing.T, a interface{}, b interface{}) { - if a == b { - t.Fatalf("[ %v == %v ]", a, b) - } -} - func Test_UnitFailOver(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "lb-test") - if err != nil { - assertEqual(t, err, nil) - } - defer os.RemoveAll(tmpDir) + tmpDir := t.TempDir() ogServe, err := createServer("og") if err != nil { - assertEqual(t, err, nil) + t.Fatalf("createServer(og) failed: %v", err) } lbServe, err := createServer("lb") if err != nil { - assertEqual(t, err, nil) + t.Fatalf("createServer(lb) failed: %v", err) } cfg := cmds.Agent{ @@ -107,12 +94,12 @@ func Test_UnitFailOver(t *testing.T) { lb, err := New(context.TODO(), cfg.DataDir, SupervisorServiceName, cfg.ServerURL, RandomPort, false) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("New() failed: %v", err) } parsedURL, err := url.Parse(lb.LoadBalancerServerURL()) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("url.Parse failed: %v", err) } localAddress := parsedURL.Host @@ -120,36 +107,39 @@ func Test_UnitFailOver(t *testing.T) { conn1, err := net.Dial("tcp", localAddress) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("net.Dial failed: %v", err) } result1, err := ping(conn1) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("ping(conn1) failed: %v", err) + } + if result1 != "lb:ping" { + t.Fatalf("Unexpected ping result: %v", result1) } - assertEqual(t, result1, "lb:ping") lbServe.close() _, err = ping(conn1) - assertNotEqual(t, err, nil) + if err == nil { + t.Fatal("Unexpected successful ping on closed connection conn1") + } conn2, err := net.Dial("tcp", localAddress) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("net.Dial failed: %v", err) + } result2, err := ping(conn2) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("ping(conn2) failed: %v", err) + } + if result2 != "og:ping" { + t.Fatalf("Unexpected ping result: %v", result2) } - assertEqual(t, result2, "og:ping") } func Test_UnitFailFast(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "lb-test") - if err != nil { - assertEqual(t, err, nil) - } - defer os.RemoveAll(tmpDir) + tmpDir := t.TempDir() cfg := cmds.Agent{ ServerURL: "http://127.0.0.1:0/", @@ -158,12 +148,12 @@ func Test_UnitFailFast(t *testing.T) { lb, err := New(context.TODO(), cfg.DataDir, SupervisorServiceName, cfg.ServerURL, RandomPort, false) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("New() failed: %v", err) } conn, err := net.Dial("tcp", lb.localAddress) if err != nil { - assertEqual(t, err, nil) + t.Fatalf("net.Dial failed: %v", err) } done := make(chan error) @@ -175,8 +165,10 @@ func Test_UnitFailFast(t *testing.T) { select { case err := <-done: - assertNotEqual(t, err, nil) + if err == nil { + t.Fatal("Unexpected successful ping from invalid address") + } case <-timeout: - t.Fatal(errors.New("time out")) + t.Fatal("Test timed out") } } diff --git a/pkg/agent/loadbalancer/servers.go b/pkg/agent/loadbalancer/servers.go index e0de81034b7c..e8e9f83a1502 100644 --- a/pkg/agent/loadbalancer/servers.go +++ b/pkg/agent/loadbalancer/servers.go @@ -1,11 +1,17 @@ package loadbalancer import ( + "context" "errors" "math/rand" - "reflect" + "net" + + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/sets" ) +var defaultDialer = &net.Dialer{} + func (lb *LoadBalancer) setServers(serverAddresses []string) bool { serverAddresses, hasOriginalServer := sortServers(serverAddresses, lb.defaultServerAddress) if len(serverAddresses) == 0 { @@ -15,10 +21,32 @@ func (lb *LoadBalancer) setServers(serverAddresses []string) bool { lb.mutex.Lock() defer lb.mutex.Unlock() - if reflect.DeepEqual(serverAddresses, lb.ServerAddresses) { + newAddresses := sets.NewString(serverAddresses...) + curAddresses := sets.NewString(lb.ServerAddresses...) + if newAddresses.Equal(curAddresses) { return false } + for addedServer := range newAddresses.Difference(curAddresses) { + logrus.Infof("Adding server to load balancer %s: %s", lb.serviceName, addedServer) + lb.servers[addedServer] = &server{connections: make(map[net.Conn]struct{})} + } + + for removedServer := range curAddresses.Difference(newAddresses) { + server := lb.servers[removedServer] + if server != nil { + logrus.Infof("Removing server from load balancer %s: %s", lb.serviceName, removedServer) + // Defer closing connections until after the new server list has been put into place. + // Closing open connections ensures that anything stuck retrying on a stale server is forced + // over to a valid endpoint. + defer server.closeAll() + // Don't delete the default server from the server map, in case we need to fall back to it. + if removedServer != lb.defaultServerAddress { + delete(lb.servers, removedServer) + } + } + } + lb.ServerAddresses = serverAddresses lb.randomServers = append([]string{}, lb.ServerAddresses...) rand.Shuffle(len(lb.randomServers), func(i, j int) { @@ -55,3 +83,41 @@ func (lb *LoadBalancer) nextServer(failedServer string) (string, error) { return lb.currentServerAddress, nil } + +// dialContext dials a new connection, and adds its wrapped connection to the map +func (s *server) dialContext(ctx context.Context, network, address string) (net.Conn, error) { + conn, err := defaultDialer.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + // don't lock until adding the connection to the map, otherwise we may block + // while waiting for the dial to time out + s.mutex.Lock() + defer s.mutex.Unlock() + + conn = &serverConn{server: s, Conn: conn} + s.connections[conn] = struct{}{} + return conn, nil +} + +// closeAll closes all connections to the server, and removes their entries from the map +func (s *server) closeAll() { + s.mutex.Lock() + defer s.mutex.Unlock() + + logrus.Debugf("Closing %d connections to load balancer server", len(s.connections)) + for conn := range s.connections { + // Close the connection in a goroutine so that we don't hold the lock while doing so. + go conn.Close() + } +} + +// Close removes the connection entry from the server's connection map, and +// closes the wrapped connection. +func (sc *serverConn) Close() error { + sc.server.mutex.Lock() + defer sc.server.mutex.Unlock() + + delete(sc.server.connections, sc) + return sc.Conn.Close() +} diff --git a/pkg/agent/tunnel/tunnel.go b/pkg/agent/tunnel/tunnel.go index 5a77ea550ada..701fdb5fe1e4 100644 --- a/pkg/agent/tunnel/tunnel.go +++ b/pkg/agent/tunnel/tunnel.go @@ -35,6 +35,10 @@ import ( "k8s.io/kubernetes/pkg/cluster/ports" ) +var ( + endpointDebounceDelay = time.Second +) + type agentTunnel struct { client kubernetes.Interface cidrs cidranger.Ranger @@ -306,9 +310,14 @@ func (a *agentTunnel) watchEndpoints(ctx context.Context, apiServerReady <-chan <-done }() + var cancelUpdate context.CancelFunc + for { select { case <-ctx.Done(): + if cancelUpdate != nil { + cancelUpdate() + } return case ev, ok := <-watch.ResultChan(): endpoint, ok := ev.Object.(*v1.Endpoints) @@ -317,28 +326,49 @@ func (a *agentTunnel) watchEndpoints(ctx context.Context, apiServerReady <-chan continue } - newAddresses := util.GetAddresses(endpoint) - if reflect.DeepEqual(newAddresses, proxy.SupervisorAddresses()) { - continue + if cancelUpdate != nil { + cancelUpdate() } - proxy.Update(newAddresses) - validEndpoint := map[string]bool{} + var debounceCtx context.Context + debounceCtx, cancelUpdate = context.WithCancel(ctx) + + // When joining the cluster, the apiserver adds, removes, and then readds itself to + // the endpoint list several times. This causes a bit of thrashing if we react to + // endpoint changes immediately. Instead, perform the endpoint update in a + // goroutine that sleeps for a short period before checking for changes and updating + // the proxy addresses. If another update occurs, the previous update operation + // will be cancelled and a new one queued. + go func() { + select { + case <-time.After(endpointDebounceDelay): + case <-debounceCtx.Done(): + return + } - for _, address := range proxy.SupervisorAddresses() { - validEndpoint[address] = true - if _, ok := disconnect[address]; !ok { - disconnect[address] = a.connect(ctx, nil, address, tlsConfig) + newAddresses := util.GetAddresses(endpoint) + if reflect.DeepEqual(newAddresses, proxy.SupervisorAddresses()) { + return } - } + proxy.Update(newAddresses) - for address, cancel := range disconnect { - if !validEndpoint[address] { - cancel() - delete(disconnect, address) - logrus.Infof("Stopped tunnel to %s", address) + validEndpoint := map[string]bool{} + + for _, address := range proxy.SupervisorAddresses() { + validEndpoint[address] = true + if _, ok := disconnect[address]; !ok { + disconnect[address] = a.connect(ctx, nil, address, tlsConfig) + } } - } + + for address, cancel := range disconnect { + if !validEndpoint[address] { + cancel() + delete(disconnect, address) + logrus.Infof("Stopped tunnel to %s", address) + } + } + }() } } } diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 69d5155f1ea6..592ca0f71a21 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -273,31 +273,17 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, } defer storageClient.Close() - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - RETRY: - for { - value, c.saveBootstrap, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) - if err != nil { - if strings.Contains(err.Error(), "not supported for learner") { - for range ticker.C { - continue RETRY - } - - } - return err - } - if value == nil { - return nil - } - - dbRawData, err = decrypt(normalizedToken, value.Data) - if err != nil { - return err - } + value, c.saveBootstrap, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) + if err != nil { + return err + } + if value == nil { + return nil + } - break + dbRawData, err = decrypt(normalizedToken, value.Data) + if err != nil { + return err } buf = bytes.NewReader(dbRawData) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 2cf6fcc444a1..56c7a999a565 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -6,20 +6,27 @@ import ( "errors" "os" "path/filepath" - "strings" + "time" "github.com/k3s-io/k3s/pkg/bootstrap" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/kine/pkg/client" "github.com/sirupsen/logrus" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + "k8s.io/apimachinery/pkg/util/wait" ) +// maxBootstrapWaitAttempts is the number of iterations to wait for another node to populate an empty bootstrap key. +// After this many attempts, the lock is deleted and the counter reset. +const maxBootstrapWaitAttempts = 5 + // Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token. // This is used when bootstrapping a cluster from a managed database or external etcd cluster. // This is NOT used with embedded etcd, which bootstraps over HTTP. func Save(ctx context.Context, config *config.Control, override bool) error { + logrus.Info("Saving cluster bootstrap data to datastore") buf := &bytes.Buffer{} if err := bootstrap.ReadFromDisk(buf, &config.Runtime.ControlRuntimeBootstrap); err != nil { return err @@ -48,13 +55,19 @@ func Save(ctx context.Context, config *config.Control, override bool) error { } defer storageClient.Close() - if _, _, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil { + currentKey, _, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) + if err != nil { return err } + // If there's an empty bootstrap key, then we've locked it and can override. + if currentKey != nil && len(currentKey.Data) == 0 { + logrus.Info("Bootstrap key lock is held") + override = true + } + if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil { if err.Error() == "key exists" { - logrus.Warn("bootstrap key already exists") if override { bsd, err := bootstrapKeyData(ctx, storageClient) if err != nil { @@ -62,9 +75,10 @@ func Save(ctx context.Context, config *config.Control, override bool) error { } return storageClient.Update(ctx, storageKey(normalizedToken), bsd.Modified, data) } + logrus.Warn("Bootstrap key already exists") return nil - } else if strings.Contains(err.Error(), "not supported for learner") { - logrus.Debug("skipping bootstrap data save on learner") + } else if errors.Is(err, rpctypes.ErrGPRCNotSupportedForLearner) { + logrus.Debug("Skipping bootstrap data save on learner") return nil } return err @@ -89,9 +103,12 @@ func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client return &bootstrapList[0], nil } -// storageBootstrap loads data from the datastore into the ControlRuntimeBootstrap struct. -// The storage key and encryption passphrase are both derived from the join token. -// token is either passed. +// storageBootstrap loads data from the datastore's bootstrap key into the +// ControlRuntimeBootstrap struct. The storage key and encryption passphrase are both derived +// from the join token. If no bootstrap key exists, indicating that data needs to be written +// back to the datastore, this function will set c.saveBootstrap to true and create an empty +// bootstrap key as a lock. This function will not return successfully until either the +// bootstrap key has been locked, or data is read into the struct. func (c *Cluster) storageBootstrap(ctx context.Context) error { if err := c.startStorage(ctx); err != nil { return err @@ -110,7 +127,12 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { return err } if tokenFromFile == "" { - // at this point this is a fresh start in a non managed environment + // No token on disk or from CLI, but we don't know if there's data in the datastore. + // Return here and generate new CA certs and tokens. Note that startup will fail + // later when saving to the datastore if there's already a bootstrap key - but + // that's AFTER generating CA certs and tokens. If the config is updated to set the + // matching key, further startups will still be blocked pending cleanup of the + // "newer" files as per the bootstrap reconciliation code. c.saveBootstrap = true return nil } @@ -121,34 +143,81 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { return err } - value, saveBootstrap, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) - c.saveBootstrap = saveBootstrap - if err != nil { - return err - } - if value == nil { - return nil - } + attempts := 0 + tokenKey := storageKey(normalizedToken) + return wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (bool, error) { + attempts++ + value, saveBootstrap, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) + c.saveBootstrap = saveBootstrap + if err != nil { + return false, err + } - data, err := decrypt(normalizedToken, value.Data) - if err != nil { - return err - } + if value == nil { + // No bootstrap keys found in the datastore - create an empty bootstrap key as a lock to + // ensure that no other node races us to populate it. If we fail to create the key, then + // some other node beat us to it and we should just wait for them to finish. + if err := storageClient.Create(ctx, tokenKey, []byte{}); err != nil { + if err.Error() == "key exists" { + logrus.Info("Bootstrap key already locked - waiting for data to be populated by another server") + return false, nil + } + return false, err + } + logrus.Info("Bootstrap key locked for initial create") + return true, nil + } + + if len(value.Data) == 0 { + // Empty (locked) bootstrap key found - check to see if we should continue waiting, or + // delete it and attempt to retake the lock on the next iteration (assuming that the + // other node failed while holding the lock). + if attempts >= maxBootstrapWaitAttempts { + logrus.Info("Bootstrap key lock timed out - deleting lock and retrying") + attempts = 0 + if err := storageClient.Delete(ctx, tokenKey, value.Modified); err != nil { + return false, err + } + } else { + logrus.Infof("Bootstrap key is locked - waiting for data to be populated by another server") + } + return false, nil + } + + data, err := decrypt(normalizedToken, value.Data) + if err != nil { + return false, err + } - return c.ReconcileBootstrapData(ctx, bytes.NewReader(data), &c.config.Runtime.ControlRuntimeBootstrap, false) + return true, c.ReconcileBootstrapData(ctx, bytes.NewReader(data), &c.config.Runtime.ControlRuntimeBootstrap, false) + }) } // getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and will check for any key that is hashed by different token than the one // passed to it, it will return error if it finds a key that is hashed with different token and will return -// value if it finds the key hashed by passed token or empty string +// value if it finds the key hashed by passed token or empty string. +// Upon receiving a "not supported for learner" error from etcd, this function will retry until the context is cancelled. func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, bool, error) { emptyStringKey := storageKey("") tokenKey := storageKey(normalizedToken) - bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0) - if err != nil { + + var bootstrapList []client.Value + var err error + + if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) { + bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0) + if err != nil { + if errors.Is(err, rpctypes.ErrGPRCNotSupportedForLearner) { + return false, nil + } + return false, err + } + return true, nil + }); err != nil { return nil, false, err } + if len(bootstrapList) == 0 { return nil, true, nil } @@ -248,7 +317,7 @@ func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue c if err := storageClient.Create(ctx, newTokenKey, encryptedData); err != nil { if err.Error() == "key exists" { logrus.Warn("bootstrap key exists") - } else if strings.Contains(err.Error(), "not supported for learner") { + } else if errors.Is(err, rpctypes.ErrGPRCNotSupportedForLearner) { logrus.Debug("skipping bootstrap data save on learner") return nil } else { diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 54f53505cb50..d81b6384ccc0 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -111,7 +111,7 @@ func ccmYaml() (*asset, error) { return a, nil } -var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x57\x6f\x6f\xdb\xb6\x13\x7e\xef\x4f\x41\x08\xc8\x9b\x1f\x7e\x72\xe2\x65\xed\x52\xbe\x4b\x63\xb7\x0d\x96\xb8\x86\xed\x14\x28\x86\x21\xa0\xa9\xb3\xc5\x85\xe2\x71\x24\xe5\xc4\xeb\xf2\xdd\x07\xea\x9f\x45\x5b\x4e\x93\xac\xf3\x1b\x4b\x3a\xde\x73\xe4\xc3\xe3\x73\x47\xa6\xc5\x17\x30\x56\xa0\xa2\x64\x3d\xe8\xdd\x09\x95\x50\x32\x03\xb3\x16\x1c\xce\x39\xc7\x5c\xb9\x5e\x06\x8e\x25\xcc\x31\xda\x23\x44\xb1\x0c\x28\xe1\x68\x20\x51\xb6\x7a\xb7\x9a\x71\xa0\xe4\x2e\x5f\x40\x6c\x37\xd6\x41\xd6\x8b\xe3\xb8\xd7\x86\x36\x0b\xc6\xfb\x2c\x77\x29\x1a\xf1\x17\x73\x02\x55\xff\xee\xcc\xf6\x05\x1e\x37\x41\x2f\x64\x6e\x1d\x98\x29\x4a\x08\x22\x4a\xb6\x00\x69\xfd\x13\x29\x42\x18\x05\x0e\x0a\xd7\x05\xa2\xb3\xce\x30\xad\x85\x5a\x95\x31\xe2\x04\x96\x2c\x97\xce\x36\x53\x2d\x27\x44\xeb\x19\x9b\x5c\x82\xa5\xbd\x98\x30\x2d\x3e\x1a\xcc\x75\x81\x1c\x93\x28\xea\x11\x62\xc0\x62\x6e\x38\x54\xdf\x40\x25\x1a\x85\x2a\xc0\x62\x62\x4b\x52\xca\x17\x8d\x49\xf9\xd0\xac\xdf\xbf\xae\xc1\x2c\x2a\x5f\x29\xac\x2b\x1e\xee\x99\xe3\xe9\x7e\xbc\x44\x58\x8e\x6b\x30\x9b\x8a\x87\x27\xa2\x4b\xf1\x5d\xf4\x7f\xc5\xf6\x7b\xa1\x12\xa1\x56\x01\xe9\x4c\x29\x74\x85\x67\xc5\x7c\x17\x64\xb0\x19\x2c\x77\x98\xeb\x84\x39\xa0\x24\x72\x26\x87\xe8\xc7\xef\x1d\x4a\x98\xc2\xb2\x98\x5f\xc5\xe6\x13\x6b\xed\x11\xb2\x9f\x58\x07\x90\x6d\xbe\xf8\x03\xb8\x2b\x12\xa3\xf3\x08\xbc\x3a\xf1\xb7\x84\xa3\x5a\x8a\xd5\x35\xd3\xaf\x39\x4e\xf5\xf0\x0b\x34\xb0\x14\x12\x28\xf9\xbb\xe0\xb4\x4f\xdf\x9c\x92\x6f\xc5\xa3\xff\x81\x31\x68\x6c\xf3\x9a\x02\x93\x2e\x6d\x5e\x0d\xb0\x64\xd3\xbc\x6d\xb7\x83\x1c\x7d\xbb\xb8\xba\x99\xcd\x47\xd3\xdb\xe1\xe7\xeb\xf3\xcb\xf1\xe3\x11\x11\x2a\x66\x49\x62\xfa\xcc\x68\x46\x84\x7e\x5b\x3e\x6c\x23\x91\xe2\x04\x10\xa1\x2c\xf0\xdc\x40\xeb\xfb\x92\x49\xe9\x52\x83\xf9\x2a\xed\x46\x69\xc6\x3e\x6e\x27\x8a\xd6\x59\x72\x0c\x8e\x1f\x57\x54\x1c\x8f\x31\x81\x4f\xc5\xe7\x76\x50\xe7\x24\x79\x7b\xd2\xfa\x60\x40\x22\x4b\xc8\xe0\x8d\xed\x9e\x42\x47\x30\x6d\x30\x03\x97\x42\x6e\x09\x7d\x37\x78\x73\xda\x18\x96\x68\xee\x99\x49\x48\xbf\x9c\x89\x3f\x8e\x72\xdd\xe7\xa8\x96\xcd\x10\xce\x78\x0a\xe4\x74\x3b\x03\x89\xa8\x7b\xe1\x64\x5a\x36\x96\x2c\x98\x64\x8a\x97\xfc\x94\x53\x10\x99\x46\xe3\xc2\xc5\xf2\xdc\x3a\xcc\x8e\xff\xd7\xf7\x1a\x03\x66\x2f\x89\x98\xd6\x76\x7b\x74\x87\xa0\x25\x6e\x32\x78\x9d\x32\xef\x1c\xca\x33\x1b\x33\xad\xab\x21\xa5\xe3\xee\x51\x2d\x81\x23\x9f\x7b\xc3\xf1\x2c\xea\x59\x0d\x9c\x16\x7a\xb5\x16\x7e\x7e\x9f\x84\x75\x68\x36\x57\x22\x13\x8e\x12\xcf\x8d\x3f\xd8\x0e\x56\x9b\x32\x86\xdb\x68\xa0\x64\x8a\x52\x0a\xb5\xba\x29\x24\xa2\x94\x94\xf6\x17\x5a\xd1\x96\xb1\x87\x1b\xc5\xd6\x4c\x48\xb6\xf0\x79\x3e\xf0\x70\x20\x81\x3b\x34\xe5\x98\xcc\x4b\xde\x55\x6b\x0d\xdd\xab\x70\x90\x69\xd9\x00\xb7\x89\x2a\xf6\x26\xf0\x3f\xc4\x43\xbd\xd2\x32\x6d\x04\x1a\xe1\x36\x17\x92\x59\x3b\x2e\x29\x29\x29\x8d\x79\x29\x30\x31\x37\xc2\x09\xce\x64\x54\xb9\xd8\x40\x43\xc6\x3b\xfb\x53\x50\x83\x12\x4c\x5b\x66\xfd\x2f\x26\x77\xb0\xf1\x84\x57\x70\xe7\x49\x82\xca\x7e\x56\x72\x13\xb5\x92\x1c\xb5\xf7\x44\x43\x49\x34\x7a\x10\xd6\xd9\x68\x0f\x40\x61\x02\xb1\x17\xcd\x1d\xa9\xe6\xa8\x9c\x41\x19\x6b\xc9\x14\x3c\x13\x93\x10\x58\x2e\x81\x3b\x4a\xa2\x31\xce\x78\x0a\x49\x2e\xe1\xf9\x21\x33\xe6\x19\xfa\x11\xb1\x7c\x84\x59\x90\x10\xfb\x19\x8b\x96\x12\x29\x54\xfe\xd0\xd0\xac\x51\xe2\x6a\x33\xd3\x5e\x03\x2f\x50\xf9\x04\xf5\xa5\xb5\x4d\x7a\xc6\x1e\x66\x77\x70\x5f\xa6\x5c\xfd\xab\x3d\x7f\xf5\xab\x0b\x83\x78\xd1\xf2\x47\xa3\x35\xfa\x3e\x05\x75\xa3\x2c\x73\xc2\x2e\x45\x99\xbf\x43\x1c\xa3\xab\xd7\xd0\x1a\x5a\x24\xe0\xfe\x3a\x0e\x24\xf8\xd3\x69\x4a\x88\xdf\x51\x26\x14\x98\xc6\x23\xde\xd3\x83\xf2\x27\x32\xb6\x02\x4a\x8e\xbe\xcd\xbe\xce\xe6\xa3\xeb\xdb\xe1\xe8\xc3\xf9\xcd\xd5\xfc\x76\x3a\xfa\x78\x39\x9b\x4f\xbf\x3e\x1e\x19\xa6\x78\x0a\xe6\x38\x13\xbe\x9a\x40\x12\x57\x10\xf5\x3f\x1d\xf4\xdf\xf5\x7f\x0e\x01\x27\xb9\x94\x13\x94\x82\x6f\x28\xb9\x5c\x8e\xd1\x4d\x0c\x58\x28\xea\x66\xf9\x0b\x7a\x9b\x86\x03\xaf\x18\x3b\x6b\xcc\x20\x43\xb3\xa1\x64\xf0\xcb\xc9\xb5\x08\x84\xfe\xcf\x1c\xec\xee\x68\xae\x73\x4a\x06\x27\x27\x59\x27\x46\x00\xc1\xcc\xca\x52\xf2\x1b\x89\x62\xaf\xe8\xd1\xff\x49\x14\x48\x70\x5d\x59\x23\xf2\x7b\xe3\xb2\x46\x99\x67\x70\xed\x0f\x6f\x90\x29\x35\xb3\xbe\xa0\xc7\xe5\xa0\x56\xfc\xcc\x8f\x9f\x30\x97\xd2\x40\xe4\x83\xb5\xb0\xc4\x1f\x67\x4a\x7c\x9f\xb4\x0f\x5c\x54\x83\xf8\x85\xf8\x55\x11\xf9\x7e\x18\x5f\x7e\x82\xe5\x34\xc9\x33\x41\xe3\x28\x69\x55\xc4\xba\xa8\x84\xd3\xd7\x06\x1d\x72\x94\x94\xdc\x0c\x27\x2f\xc5\x89\x1d\xd7\x9d\x58\xf3\x8b\x27\xb0\x82\x3a\x5d\xa3\x65\xe0\x8c\xe0\xdd\x33\x6b\xa3\x15\x2d\x8a\x57\x6e\x54\x0e\x1e\x5c\x3b\x83\x98\x94\x78\x3f\x31\x62\x2d\x24\xac\x60\x64\x39\x93\x85\x1a\x53\xdf\x43\xd8\x36\xeb\x9c\x69\xb6\x10\x52\x38\x01\x3b\x39\xc8\x92\x24\xfc\x10\x93\xf1\x68\x7e\xfb\xfe\x72\x3c\xbc\x9d\x8d\xa6\x5f\x2e\x2f\x46\x81\x39\x31\xa8\x77\x1d\x98\x94\x1d\x1b\x37\x45\x74\x1f\x84\x84\xaa\x59\x0d\xb7\x51\x8a\x35\x28\xb0\x76\x62\x70\x01\x6d\xbc\xd4\x39\xfd\x11\x5c\x18\x42\x97\xf9\xb2\xd3\x11\x92\x2a\x1d\x28\x39\x3b\x39\x3b\x09\x3e\x5b\x9e\x82\x27\xf9\xd3\x7c\x3e\x69\x19\x84\x12\x4e\x30\x39\x04\xc9\x36\x33\xe0\xa8\x12\x4b\xc3\x8e\x4c\x83\x11\x98\x34\xb6\x41\xdb\xe6\x44\x06\x98\xbb\xad\xb1\x65\xb3\x39\xe7\x60\xed\x3c\x35\x60\x53\x94\x49\x68\x5d\x32\x21\x73\x03\x2d\xeb\x69\xd0\xd7\x8a\x17\x53\x11\x76\xc3\x2d\x26\x06\x67\x83\x57\x33\xf1\x04\x11\x3f\xfd\xc7\x3c\x24\xca\xd6\x0a\x3c\x2c\xef\x51\x95\xa1\x14\x90\x17\x08\x18\xaf\x6f\x2a\x21\x6f\xdd\xf5\xa4\xa0\xc2\x41\x66\x77\x53\xba\xe8\x07\x6a\x55\xdd\x29\x63\xe5\x16\x74\x1a\x2b\xc7\xa6\xfd\xef\xf4\xdc\xb7\x3e\x53\x3b\x9f\xb3\xb4\x78\x4f\x48\x7d\xb3\xe2\x55\x81\xc9\xea\x0c\x1e\xbc\xe4\x55\xb7\xc6\x8e\xbe\xbc\x55\xb0\x0f\x36\xe6\x7b\x97\xee\xed\x55\xc5\x37\x1c\x65\x7e\x46\x5e\x0b\xa3\x0e\xb3\xe5\x86\xe9\x83\x97\xef\x67\xf4\xf9\x75\x1b\x5b\xb5\xad\x2d\xa4\xe7\xde\x08\xc2\x46\xbd\x2b\x66\x15\xe3\x72\x42\xdb\xb7\xce\xf1\xec\xf1\xa8\xd7\xaa\x4c\xf1\x4e\xdd\xd1\xed\x82\xb2\x5b\x7e\xe2\x8e\xe2\x72\xc0\xa1\xac\x0a\x71\x47\xfd\xd0\x61\x99\x09\x5d\xfe\x09\x00\x00\xff\xff\xc4\xd5\x15\x5f\x24\x13\x00\x00") +var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x57\x51\x6f\xdb\x38\x12\x7e\xf7\xaf\x20\x04\xe4\xe5\x70\x72\xe2\x0b\xda\xcb\xf1\x2d\x8d\xdd\x36\xb8\xc4\x35\x6c\xa7\x40\xb1\x58\x04\x34\x35\xb6\xb8\xa1\x38\x5c\x92\x72\xe2\xed\xe6\xbf\x2f\x28\xc9\x32\x69\x2b\x69\x92\xed\xfa\xc5\x92\x86\xf3\x0d\xf9\x71\xf8\xcd\x90\x69\xf1\x15\x8c\x15\xa8\x28\x59\x0f\x7a\x77\x42\x65\x94\xcc\xc0\xac\x05\x87\x73\xce\xb1\x54\xae\x57\x80\x63\x19\x73\x8c\xf6\x08\x51\xac\x00\x4a\x38\x1a\xc8\x94\x6d\xde\xad\x66\x1c\x28\xb9\x2b\x17\x90\xda\x8d\x75\x50\xf4\xd2\x34\xed\x85\xd0\x66\xc1\x78\x9f\x95\x2e\x47\x23\xfe\x60\x4e\xa0\xea\xdf\x9d\xd9\xbe\xc0\xe3\x36\xe8\x85\x2c\xad\x03\x33\x45\x09\x51\x44\xc9\x16\x20\xad\x7f\x22\x55\x08\xa3\xc0\x41\xe5\xba\x40\x74\xd6\x19\xa6\xb5\x50\xab\x3a\x46\x9a\xc1\x92\x95\xd2\xd9\x76\xaa\xf5\x84\xe8\x76\xc6\xa6\x94\x60\x69\x2f\x25\x4c\x8b\x4f\x06\x4b\x5d\x21\xa7\x24\x49\x7a\x84\x18\xb0\x58\x1a\x0e\xcd\x37\x50\x99\x46\xa1\x2a\xb0\x94\xd8\x9a\x94\xfa\x45\x63\x56\x3f\xb4\xeb\xf7\xaf\x6b\x30\x8b\xc6\x57\x0a\xeb\xaa\x87\x7b\xe6\x78\x7e\x18\x2f\x13\x96\xe3\x1a\xcc\xa6\xe1\xe1\x99\xe8\x52\xfc\x10\xfd\x6f\xb1\xfd\x41\xa8\x4c\xa8\x55\x44\x3a\x53\x0a\x5d\xe5\xd9\x30\xdf\x05\x19\x6d\x06\x2b\x1d\x96\x3a\x63\x0e\x28\x49\x9c\x29\x21\xf9\xf9\x7b\x87\x12\xa6\xb0\xac\xe6\xd7\xb0\xf9\xcc\x5a\x7b\x84\x1c\x26\xd6\x13\xc8\xb6\x5c\xfc\x06\xdc\x55\x89\xd1\x79\x04\xde\x9c\xf8\x3b\xc2\x51\x2d\xc5\xea\x9a\xe9\xb7\x1c\xa7\xed\xf0\x0b\x34\xb0\x14\x12\x28\xf9\xb3\xe2\xb4\x4f\xdf\x9d\x92\xef\xd5\xa3\xff\x81\x31\x68\x6c\xfb\x9a\x03\x93\x2e\x6f\x5f\x0d\xb0\x6c\xd3\xbe\xed\xb6\x83\x1c\x7d\xbf\xb8\xba\x99\xcd\x47\xd3\xdb\xe1\x97\xeb\xf3\xcb\xf1\xe3\x11\x11\x2a\x65\x59\x66\xfa\xcc\x68\x46\x84\x7e\x5f\x3f\xec\x22\x91\xea\x04\x10\xa1\x2c\xf0\xd2\x40\xf0\x7d\xc9\xa4\x74\xb9\xc1\x72\x95\x77\xa3\xb4\x63\x1f\x77\x13\x45\xeb\x2c\x39\x06\xc7\x8f\x1b\x2a\x8e\xc7\x98\xc1\xe7\xea\x73\x18\xd4\x39\x49\xde\x9f\x04\x1f\x0c\x48\x64\x19\x19\xbc\xb3\xdd\x53\xe8\x08\xa6\x0d\x16\xe0\x72\x28\x2d\xa1\xff\x1b\xbc\x3b\x6d\x0d\x4b\x34\xf7\xcc\x64\xa4\x5f\xcf\xc4\x1f\x47\xb9\xee\x73\x54\xcb\x76\x08\x67\x3c\x07\x72\xba\x9b\x81\x44\xd4\xbd\x78\x32\x81\x8d\x65\x0b\x26\x99\xe2\x35\x3f\xf5\x14\x44\xa1\xd1\xb8\x78\xb1\xbc\xb4\x0e\x8b\xe3\x7f\xf5\xbd\xc6\x80\x39\x48\x22\xa6\xb5\xdd\x1d\xdd\x21\x68\x89\x9b\x02\xde\xa6\xcc\x7b\x87\xf2\xcc\xa6\x4c\xeb\x66\x48\xed\xb8\x7f\x54\x6b\xe0\xc4\xe7\xde\x70\x3c\x4b\x7a\x56\x03\xa7\x95\x5e\xad\x85\x9f\xdf\x67\x61\x1d\x9a\xcd\x95\x28\x84\xa3\xc4\x73\xe3\x0f\xb6\x83\xd5\xa6\x8e\xe1\x36\x1a\x28\x99\xa2\x94\x42\xad\x6e\x2a\x89\xa8\x25\x25\xfc\x42\x1b\xda\x0a\xf6\x70\xa3\xd8\x9a\x09\xc9\x16\x3e\xcf\x07\x1e\x0e\x24\x70\x87\xa6\x1e\x53\x78\xc9\xbb\x0a\xd6\xd0\xbd\x0a\x07\x85\x96\x2d\x70\x48\x54\xb5\x37\x91\xff\x53\x3c\x6c\x57\x5a\xa7\x8d\x40\x23\xdc\xe6\x42\x32\x6b\xc7\x35\x25\x35\xa5\x29\xaf\x05\x26\xe5\x46\x38\xc1\x99\x4c\x1a\x17\x1b\x69\xc8\x78\x6f\x7f\x2a\x6a\x50\x82\x09\x65\xd6\xff\x52\x72\x07\x1b\x4f\x78\x03\x77\x9e\x65\xa8\xec\x17\x25\x37\x49\x90\xe4\xa8\xbd\x27\x1a\x4a\x92\xd1\x83\xb0\xce\x26\x07\x00\x0a\x33\x48\xbd\x68\xee\x49\x35\x47\xe5\x0c\xca\x54\x4b\xa6\xe0\x85\x98\x84\xc0\x72\x09\xdc\x51\x92\x8c\x71\xc6\x73\xc8\x4a\x09\x2f\x0f\x59\x30\xcf\xd0\xcf\x88\xe5\x23\xcc\xa2\x84\x38\xcc\x58\xb4\x94\x48\xa1\xca\x87\x96\x66\x8d\x12\x57\x9b\x99\xf6\x1a\x78\x81\xca\x27\xa8\x2f\xad\x21\xe9\x05\x7b\x98\xdd\xc1\x7d\x9d\x72\xdb\xdf\xd6\xf3\xff\x7e\x75\x71\x10\x2f\x5a\xfe\x68\x04\xa3\xef\x73\x50\x37\xca\x32\x27\xec\x52\xd4\xf9\x3b\xc4\x31\xba\xed\x1a\x82\xa1\x55\x02\x1e\xae\xe3\x89\x04\x7f\x3e\x4d\x09\xf1\x3b\xca\x84\x02\xd3\x7a\xa4\x07\x7a\x50\xff\x44\xc1\x56\x40\xc9\xd1\xf7\xd9\xb7\xd9\x7c\x74\x7d\x3b\x1c\x7d\x3c\xbf\xb9\x9a\xdf\x4e\x47\x9f\x2e\x67\xf3\xe9\xb7\xc7\x23\xc3\x14\xcf\xc1\x1c\x17\xc2\x57\x13\xc8\xd2\x06\x62\xfb\x4f\x07\xfd\xc1\x49\x7f\x10\x23\x4e\x4a\x29\x27\x28\x05\xdf\x50\x72\xb9\x1c\xa3\x9b\x18\xb0\x50\x15\xce\xfa\x17\x35\x37\x2d\x09\x5e\x32\xf6\x16\x59\x40\x81\x66\x43\xc9\xe0\xbf\x27\xd7\x22\x52\xfa\xdf\x4b\xb0\xfb\xa3\xb9\x2e\x29\x19\x9c\x9c\x14\x9d\x18\x11\x04\x33\x2b\x4b\xc9\x2f\x24\x49\xbd\xa4\x27\xff\x26\x49\xa4\xc1\xdb\xd2\x9a\x90\x5f\x5b\x97\x35\xca\xb2\x80\x6b\x7f\x7a\xa3\x54\xd9\x52\xeb\x2b\x7a\x5a\x0f\x0a\xe2\x17\x7e\xfc\x84\xb9\x9c\x46\x2a\x1f\xad\x85\x65\xfe\x3c\x53\xe2\x1b\xa5\x43\xe0\xaa\x1c\xa4\xaf\xc4\x6f\xaa\xc8\x8f\xc3\xf8\xfa\x13\x2d\xa7\xcd\x9e\x09\x1a\x47\x49\x50\x12\xb7\x55\x25\x9e\xbe\x36\xe8\x90\xa3\xa4\xe4\x66\x38\x79\x2d\x4e\xea\xb8\xee\xc4\x9a\x5f\x3c\x83\x15\x15\xea\x2d\x5a\x01\xce\x08\xde\x3d\xb3\x10\xad\xea\x51\xbc\x74\xa3\x72\xf0\xe0\xc2\x0c\x62\x52\xe2\xfd\xc4\x88\xb5\x90\xb0\x82\x91\xe5\x4c\x56\x72\x4c\x7d\x13\x61\x43\xd6\x39\xd3\x6c\x21\xa4\x70\x02\xf6\x72\x90\x65\x59\xfc\x21\x25\xe3\xd1\xfc\xf6\xc3\xe5\x78\x78\x3b\x1b\x4d\xbf\x5e\x5e\x8c\x22\x73\x66\x50\xef\x3b\x30\x29\x3b\x36\x6e\x8a\xe8\x3e\x0a\x09\x4d\xb7\x1a\x6f\xa3\x14\x6b\x50\x60\xed\xc4\xe0\x02\x42\xbc\xdc\x39\xfd\x09\x5c\x1c\x42\xd7\xf9\xb2\xd7\x12\x92\x26\x1d\x28\x39\x3b\x39\x3b\x89\x3e\x5b\x9e\x83\x27\xf9\xf3\x7c\x3e\x09\x0c\x42\x09\x27\x98\x1c\x82\x64\x9b\x19\x70\x54\x99\xa5\x71\x4b\xa6\xc1\x08\xcc\x5a\xdb\x20\xb4\x39\x51\x00\x96\x6e\x67\x0c\x6c\xb6\xe4\x1c\xac\x9d\xe7\x06\x6c\x8e\x32\x8b\xad\x4b\x26\x64\x69\x20\xb0\x9e\x46\x8d\xad\x78\x35\x15\x71\x3b\x1c\x30\x31\x38\x1b\xbc\x99\x89\x67\x88\xf8\xcf\x3f\xcc\x43\xa6\xec\x56\x81\x87\xf5\x45\xaa\x31\xd4\x02\xf2\x0a\x01\xe3\xdb\xab\x4a\xcc\x5b\x77\x41\xa9\xa8\x70\x50\xd8\xfd\x94\xae\x1a\x82\xad\xaa\xee\xd5\xb1\x7a\x0b\x3a\x8d\x8d\x63\xdb\xff\x77\x7a\x1e\x5a\x5f\xa8\x9d\x2f\x59\x5a\x7a\x20\xa4\xbe\x5b\xf1\xaa\xc0\x64\x73\x06\x9f\xbc\xe5\x35\xd7\xc6\x8e\xc6\x3c\xa8\xd8\x4f\x76\xe6\x07\xb7\xee\xdd\x5d\xc5\x77\x1c\x75\x7e\x26\x5e\x0b\x93\x0e\xb3\xe5\x86\xe9\x27\x6f\xdf\x2f\x68\xf4\xb7\x7d\x6c\xd3\xb7\x06\x48\x2f\xbd\x12\xc4\x9d\x7a\x57\xcc\x26\xc6\xe5\x84\x86\xd7\xce\xf1\xec\xf1\xa8\x17\x54\xa6\x74\xaf\xee\xe8\xb0\xa0\xec\x97\x9f\xb4\xa3\xb8\x3c\xe1\x50\x57\x85\xb4\xa3\x7e\xe8\xb8\xcc\xc4\x2e\x7f\x05\x00\x00\xff\xff\xfd\x41\xe7\x07\x25\x13\x00\x00") func corednsYamlBytes() ([]byte, error) { return bindataRead( @@ -131,7 +131,7 @@ func corednsYaml() (*asset, error) { return a, nil } -var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\x5f\x6f\xdb\xb6\x16\x7f\xd7\xa7\x38\x57\xb7\x79\xb8\x17\xa5\x9d\xac\x03\x32\xb0\xd8\x83\x9b\x38\x69\x80\xc4\x36\x6c\x77\x43\x51\x14\x06\x2d\x1d\xdb\x6c\x28\x92\x20\x29\xb7\x6a\x96\xef\x3e\x90\x94\x1d\xc9\x71\x13\x07\xdb\xde\xa6\x17\x81\x87\xe7\xef\xef\xfc\x23\xd3\xfc\x37\x34\x96\x2b\x49\x61\x7d\x92\xdc\x72\x99\x53\x98\xa0\x59\xf3\x0c\x7b\x59\xa6\x4a\xe9\x92\x02\x1d\xcb\x99\x63\x34\x01\x90\xac\x40\x0a\x42\x65\x4c\x10\xcd\xdc\x8a\x68\xa3\xd6\xdc\xcb\xa3\x21\x36\xca\x11\x56\x0b\x46\x76\xab\x59\x86\x14\x6e\xcb\x39\x12\x5b\x59\x87\x45\x42\x08\x49\x9a\x96\xcd\x9c\x65\x1d\x56\xba\x95\x32\xfc\x3b\x73\x5c\xc9\xce\xed\x2f\xb6\xc3\x55\x77\xeb\xd3\x99\x28\xad\x43\x33\x56\x02\x0f\x77\xc8\x78\x6e\x53\x0a\xb4\x34\x21\xc0\x34\xbf\x34\xaa\xd4\x96\xc2\xa7\x34\xfd\x9c\x00\x18\xb4\xaa\x34\x19\x06\x8a\x54\x39\xda\xf4\x35\xa4\xda\xbb\x65\x1d\x4a\xb7\x56\xa2\x2c\x30\x13\x8c\x17\xe1\x26\x53\x72\xc1\x97\x05\xd3\x36\x88\xaf\xd1\xcc\x83\xe8\x12\x9d\xbf\x16\xdc\x86\xff\x57\xe6\xb2\x55\xfa\xf9\x79\x93\x28\x73\xad\xb8\x74\x7b\xcd\x46\xa2\xca\x77\x6c\xfd\xff\x20\xc5\x6b\xf4\x5a\x5b\x82\x99\x41\xe6\x30\x28\xdd\xef\x9f\x75\xca\xb0\x25\xd6\xd0\x3f\x56\x5a\xdf\x67\x82\x59\x8b\x07\x22\xf0\x97\x12\xfd\x8e\xcb\x9c\xcb\xe5\xe1\xf9\x9e\x73\x99\x27\x3e\xe9\x63\x5c\x78\xe6\x4d\x78\x4f\x18\x4e\x00\x1e\x17\xd8\x21\x65\x65\xcb\xf9\x17\xcc\x5c\xa8\xac\xbd\x6d\xf3\x4f\x35\x0b\xd3\xda\x3e\xc0\x75\x8e\x5a\xa8\xaa\xc0\x17\xf4\xe9\x8f\x4d\x59\x8d\x19\x0d\x69\x8f\xbc\xef\xb9\xcf\x79\x75\xcd\x0b\xee\x28\x1c\x27\x00\xd6\x19\xe6\x70\x59\x79\x2e\x00\x57\x69\xa4\x30\x56\x42\x70\xb9\xfc\xa0\x73\xe6\x30\xd0\x4d\x93\x12\x59\x01\x0a\xf6\xed\x83\x64\x6b\xc6\x05\x9b\x0b\xa4\x70\xe2\xd5\xa1\xc0\xcc\x29\x13\x79\x0a\x5f\x35\xd7\x6c\x8e\xc2\x6e\x84\x98\xd6\x4f\x84\xe1\xb0\xd0\x62\x6b\xa2\x19\xbf\xff\x44\x4b\xd3\x73\xba\x00\x36\xd1\xfb\x4f\x1b\xae\x0c\x77\xd5\x99\x2f\xf6\x41\x00\x33\x8d\x20\x11\x3f\x27\x48\x66\xb8\xe3\x19\x13\x69\xcd\x6f\x5b\xb9\x1f\xbc\x2c\xf1\x01\x4a\x25\xd0\x84\xc2\x6c\x78\x0c\x40\xe0\x16\x2b\x0a\xe9\x59\x6d\xaf\x97\xe7\x4a\xda\xa1\x14\x55\xda\xe0\x02\x50\xda\x4b\x2b\x43\x21\xed\x7f\xe3\xd6\xd9\x74\x8f\x92\xe0\xb9\x2f\xde\x8e\x4f\xba\x91\xe8\x30\xf4\x5e\xa6\xa4\x33\x4a\x10\x2d\x98\xc4\x17\xe8\x05\xc0\xc5\x02\x33\x47\x21\x1d\xa8\x49\xb6\xc2\xbc\x14\xf8\x12\xc3\x05\xf3\x2d\xf7\x77\x59\xf4\x61\x30\x2e\xd1\x6c\x11\x24\xcf\xf5\x41\xfc\x78\xc1\x96\x48\xe1\xe8\x6e\xf2\x71\x32\xed\xdf\xcc\xce\xfb\x17\xbd\x0f\xd7\xd3\xd9\xb8\x7f\x79\x35\x99\x8e\x3f\xde\x1f\x19\x26\xb3\x15\x9a\xee\x7e\x45\x74\x7d\xdc\x39\xee\xfc\xf4\xa6\xad\x70\x54\x0a\x31\x52\x82\x67\x15\x85\xab\xc5\x40\xb9\x91\x41\x8b\xdb\x84\x7b\x7f\x8b\x82\xc9\xfc\x21\xdd\xe4\x39\x47\x09\x58\xc7\x8c\x6b\x9c\x09\x89\x3b\xa9\x41\xea\xa2\xcb\xba\x91\x5a\xff\x3a\x5f\xac\x92\x5b\x8e\xb8\x5d\x6e\x7c\xed\xd9\xa6\xed\x08\x55\x94\x20\x91\xa9\x81\x7c\xe1\xf9\x47\xcc\xad\x68\xcb\xc0\x96\x03\xe5\xfa\xb1\xb2\xd1\xf0\x7c\x36\xe8\xdd\xf4\x27\xa3\xde\x59\xbf\xa1\x6c\xcd\x44\x89\x17\x46\x15\xb4\x95\xdb\x05\x47\x91\xd7\xa3\xfb\x11\x3d\xda\xde\xf4\x78\x67\x3b\xc1\x92\x66\x54\x2f\x08\x28\xd2\x6f\x98\x6e\x5b\x7b\x54\x30\x35\xbe\xbb\x53\xb8\xbd\x2c\x1f\xe6\xf1\x24\xd2\xc3\xdc\x78\x72\x22\xfb\xf5\x24\xa5\x72\xcd\x9e\x6f\x6e\xd8\x9d\x56\xe1\x96\xe4\xb8\x60\xa5\x70\x24\x5c\x53\x48\x9d\x29\x31\x4d\x9a\x75\x08\x75\x9d\x7a\x81\x86\xa5\x18\x7b\xbd\x4d\x6f\x54\x8e\x14\x7e\x67\xdc\x5d\x28\x73\xc1\x8d\x75\x67\x4a\xda\xb2\x40\x93\x98\xf8\xd4\xd9\x14\xed\x39\x0a\x74\x18\x22\xaf\x57\xe4\x06\xb2\x64\xe7\xd9\xf8\xe4\xe6\xd9\x16\xe8\x0f\x96\xce\x46\xb0\x51\xab\x14\xfe\x20\x01\x90\xbb\x3a\x37\x61\x82\xf8\x0a\xb8\x61\x3a\xa5\x9f\x6a\xea\xdd\x36\x73\xe1\x3e\xa5\xe9\xa6\x73\x47\xbd\xe9\xfb\xd9\xc5\x70\x3c\x1b\x0c\x07\xb3\xeb\xab\xc9\xb4\x7f\x3e\x1b\x0c\xcf\xfb\x93\xf4\xf5\x83\x8c\xf7\xce\xa6\xf4\x53\x7a\x74\xb7\x91\xbb\x1e\x9e\xf5\xae\x67\x93\xe9\x70\xdc\xbb\xec\x07\x2d\xf7\x47\xe1\xa1\xe3\xbf\xfb\xfa\x1f\xcf\xf7\x61\x7d\x39\xff\xb8\xa8\x9d\xfd\xef\x7f\xba\x73\x2e\xbb\x76\x15\x4e\x5f\x57\x5c\x20\x2c\xd1\x29\xed\x2c\xa4\x05\xb5\x54\xd3\x14\x94\x8e\xed\x9b\xab\x87\x39\xc0\x2c\xc2\x2b\xa5\x1d\x70\xd9\xaa\x45\xfd\xbf\xd6\x91\xcd\xad\x12\xa5\x0b\x38\xfc\xfa\x6a\x38\x9a\xf6\xc6\x97\x2d\x86\xb7\x6f\x5b\x47\xdb\x16\xb7\xfc\x3b\x5e\xc9\x77\x95\x43\x7b\x88\x74\xd1\x96\x5e\x2b\xe1\x2b\xe7\x39\x49\xb4\x2c\xab\xe3\x93\xb1\xdb\x8a\xdb\x9c\x1b\x20\x05\x1c\x9f\x9e\x9e\x02\xd1\xf0\xea\xae\x19\x48\x04\x35\x5b\x15\x2a\x87\xd3\xe3\x93\xdd\xdb\x6e\xa7\x13\xf6\x3c\x33\xb9\xfa\x2a\xff\x85\xfa\x49\xa8\x4d\x01\xc4\x2c\xf6\x00\xbc\x42\xa1\xd1\x8c\x54\xde\xa9\x58\x21\xb6\x28\xee\x74\xb1\x27\xc5\x46\x1f\xa9\x7c\xef\x8b\x2a\xf6\x76\xd4\x46\x74\xcd\xd4\x7c\x36\xfd\x78\x05\xef\x08\xc1\x8b\xd6\x6e\xc1\x8d\x51\x06\x73\x22\xf8\xdc\x30\x53\x91\x79\x69\xab\xb9\xfa\x46\x4f\x3a\x6f\x7e\xee\x9c\x1c\xb8\x77\xff\x0c\x00\x00\xff\xff\x7b\xe8\x43\xdf\xec\x0e\x00\x00") +var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\x5f\x6f\xdb\xb6\x16\x7f\xd7\xa7\x38\x57\xb7\x79\xb8\x17\xa5\x9d\x6c\x05\x32\xb0\xd8\x83\x9b\x38\x69\x80\xc4\x36\x6c\x77\x43\x51\x14\x06\x2d\x1d\xdb\x6c\x28\x92\x20\x29\xb7\x6a\x96\xef\x3e\x90\x94\x1d\xc9\x71\x13\x07\xdb\xde\xa6\x17\x81\x87\xe7\xef\xef\xfc\x23\xd3\xfc\x37\x34\x96\x2b\x49\x61\x7d\x92\xdc\x72\x99\x53\x98\xa0\x59\xf3\x0c\x7b\x59\xa6\x4a\xe9\x92\x02\x1d\xcb\x99\x63\x34\x01\x90\xac\x40\x0a\x42\x65\x4c\x10\xcd\xdc\x8a\x68\xa3\xd6\xdc\xcb\xa3\x21\x36\xca\x11\x56\x0b\x46\x76\xab\x59\x86\x14\x6e\xcb\x39\x12\x5b\x59\x87\x45\x42\x08\x49\x9a\x96\xcd\x9c\x65\x1d\x56\xba\x95\x32\xfc\x3b\x73\x5c\xc9\xce\xed\x2f\xb6\xc3\x55\x77\xeb\xd3\x99\x28\xad\x43\x33\x56\x02\x0f\x77\xc8\x78\x6e\x53\x0a\xb4\x34\x21\xc0\x34\xbf\x34\xaa\xd4\x96\xc2\xa7\x34\xfd\x9c\x00\x18\xb4\xaa\x34\x19\x06\x8a\x54\x39\xda\xf4\x35\xa4\xda\xbb\x65\x1d\x4a\xb7\x56\xa2\x2c\x30\x13\x8c\x17\xe1\x26\x53\x72\xc1\x97\x05\xd3\x36\x88\xaf\xd1\xcc\x83\xe8\x12\x9d\xbf\x16\xdc\x86\xff\x57\xe6\xb2\x55\xfa\xf9\x79\x93\x28\x73\xad\xb8\x74\x7b\xcd\x46\xa2\xca\x77\x6c\xfd\xff\x20\xc5\x6b\xf4\x5a\x5b\x82\x99\x41\xe6\x30\x28\xdd\xef\x9f\x75\xca\xb0\x25\xd6\xd0\x3f\x56\x5a\xdf\x67\x82\x59\x8b\x07\x22\xf0\x97\x12\xfd\x8e\xcb\x9c\xcb\xe5\xe1\xf9\x9e\x73\x99\x27\x3e\xe9\x63\x5c\x78\xe6\x4d\x78\x4f\x18\x4e\x00\x1e\x17\xd8\x21\x65\x65\xcb\xf9\x17\xcc\x5c\xa8\xac\xbd\x6d\xf3\x4f\x35\x0b\xd3\xda\x3e\xc0\x75\x8e\x5a\xa8\xaa\xc0\x17\xf4\xe9\x8f\x4d\x59\x8d\x19\x0d\x69\x8f\xbc\xef\xb9\xcf\x79\x75\xcd\x0b\xee\x28\x1c\x27\x00\xd6\x19\xe6\x70\x59\x79\x2e\x00\x57\x69\xa4\x30\x56\x42\x70\xb9\xfc\xa0\x73\xe6\x30\xd0\x4d\x93\x12\x59\x01\x0a\xf6\xed\x83\x64\x6b\xc6\x05\x9b\x0b\xa4\x70\xe2\xd5\xa1\xc0\xcc\x29\x13\x79\x0a\x5f\x35\xd7\x6c\x8e\xc2\x6e\x84\x98\xd6\x4f\x84\xe1\xb0\xd0\x62\x6b\xa2\x19\xbf\xff\x44\x4b\xd3\x73\xba\x00\x36\xd1\xfb\x4f\x1b\xae\x0c\x77\xd5\x99\x2f\xf6\x41\x00\x33\x8d\x20\x11\x3f\x27\x48\x66\xb8\xe3\x19\x13\x69\xcd\x6f\x5b\xb9\x1f\xbc\x2c\xf1\x01\x4a\x25\xd0\x84\xc2\x6c\x78\x0c\x40\xe0\x16\x2b\x0a\xe9\x59\x6d\xaf\x97\xe7\x4a\xda\xa1\x14\x55\xda\xe0\x02\x50\xda\x4b\x2b\x43\x21\xed\x7f\xe3\xd6\xd9\x74\x8f\x92\xe0\xb9\x2f\xde\x8e\x4f\xba\x91\xe8\x30\xf4\x5e\xa6\xa4\x33\x4a\x10\x2d\x98\xc4\x17\xe8\x05\xc0\xc5\x02\x33\x47\x21\x1d\xa8\x49\xb6\xc2\xbc\x14\xf8\x12\xc3\x05\xf3\x2d\xf7\x77\x59\xf4\x61\x30\x2e\xd1\x6c\x11\x24\xcf\xf5\x41\xfc\x78\xc1\x96\x48\xe1\xe8\x6e\xf2\x71\x32\xed\xdf\xcc\xce\xfb\x17\xbd\x0f\xd7\xd3\xd9\xb8\x7f\x79\x35\x99\x8e\x3f\xde\x1f\x19\x26\xb3\x15\x9a\xee\x7e\x45\x74\x7d\xdc\x39\xee\xfc\xf4\xa6\xad\x70\x54\x0a\x31\x52\x82\x67\x15\x85\xab\xc5\x40\xb9\x91\x41\x8b\xdb\x84\x7b\x7f\x8b\x82\xc9\xfc\x21\xdd\xe4\x39\x47\x09\x58\xc7\x8c\x6b\x9c\x09\x89\x3b\xa9\x41\xea\xa2\xcb\xba\x91\x5a\xff\x3a\x5f\xac\x92\x5b\x8e\xb8\x5d\x6e\x7c\xed\xd9\xa6\xed\x08\x55\x94\x20\x91\xa9\x81\x7c\xe1\xf9\x47\xcc\xad\x68\xcb\xc0\x96\x03\xe5\xfa\xb1\xb2\xd1\xf0\x7c\x36\xe8\xdd\xf4\x27\xa3\xde\x59\xbf\xa1\x6c\xcd\x44\x89\x17\x46\x15\xb4\x95\xdb\x05\x47\x91\xd7\xa3\xfb\x11\x3d\xda\xde\xf4\x78\x67\x3b\xc1\x92\x66\x54\x2f\x08\x28\xd2\x6f\x98\x6e\x5b\x7b\x54\x30\x35\xbe\xbb\x53\xb8\xbd\x2c\x1f\xe6\xf1\x24\xd2\xc3\xdc\x78\x72\x22\xfb\xf5\x24\xa5\x72\xcd\x9e\x6f\x6e\xd8\x9d\x56\xe1\x96\xe4\xb8\x60\xa5\x70\x24\x5c\x53\x48\x9d\x29\x31\x4d\x9a\x75\x08\x75\x9d\x7a\x81\x86\xa5\x18\x7b\xbd\x4d\x6f\x54\x8e\x14\x7e\x67\xdc\x5d\x28\x73\xc1\x8d\x75\x67\x4a\xda\xb2\x40\x93\x98\xf8\xd4\xd9\x14\xed\x39\x0a\x74\x18\x22\xaf\x57\xe4\x06\xb2\x64\xe7\xd9\xf8\xe4\xe6\xd9\x16\xe8\x0f\x96\xce\x46\xb0\x51\xab\x14\xfe\x20\x01\x90\xbb\x3a\x37\x61\x82\xf8\x0a\xb8\x61\x3a\xa5\x9f\x6a\xea\xdd\x36\x73\xe1\x3e\xa5\xe9\xa6\x73\x47\xbd\xe9\xfb\xd9\xc5\x70\x3c\x1b\x0c\x07\xb3\xeb\xab\xc9\xb4\x7f\x3e\x1b\x0c\xcf\xfb\x93\xf4\xf5\x83\x8c\xf7\xce\xa6\xf4\x53\x7a\x74\xb7\x91\xbb\x1e\x9e\xf5\xae\x67\x93\xe9\x70\xdc\xbb\xec\x07\x2d\xf7\x47\xe1\xa1\xe3\xbf\xfb\xfa\x1f\xcf\xf7\x61\x7d\x39\xff\xb8\xa8\x9d\xfd\xef\x7f\xba\x73\x2e\xbb\x76\x15\x4e\x5f\x57\x5c\x20\x2c\xd1\x29\xed\x2c\xa4\x05\xb5\x54\xd3\x14\x94\x8e\xed\x9b\xab\x87\x39\xc0\x2c\xc2\x2b\xa5\x1d\x70\xd9\xaa\x45\xfd\xbf\xd6\x91\xcd\xad\x12\xa5\x0b\x38\xfc\xfa\x6a\x38\x9a\xf6\xc6\x97\x2d\x86\xb7\x6f\x5b\x47\xdb\x16\xb7\xfc\x3b\x5e\xc9\x77\x95\x43\x7b\x88\x74\xd1\x96\x5e\x2b\xe1\x2b\xe7\x39\x49\xb4\x2c\xab\xe3\x93\xb1\xdb\x8a\xdb\x9c\x1b\x20\x05\x1c\x9f\x9e\x9e\x02\xd1\xf0\xea\xae\x19\x48\x04\x35\x5b\x15\x2a\x87\xd3\xe3\x93\xdd\xdb\x6e\xa7\x13\xf6\x3c\x33\xb9\xfa\x2a\xff\x85\xfa\x49\xa8\x4d\x01\xc4\x2c\xf6\x00\xbc\x42\xa1\xd1\x8c\x54\xde\xa9\x58\x21\xb6\x28\xee\x74\xb1\x27\xc5\x46\x1f\xa9\x7c\xef\x8b\x2a\xf6\x76\xd4\x46\x74\xcd\xd4\x7c\x36\xfd\x78\x05\xef\x08\xc1\x8b\xd6\x6e\xc1\x8d\x51\x06\x73\x22\xf8\xdc\x30\x53\x91\x79\x69\xab\xb9\xfa\x46\x4f\x3a\x3f\xbf\xe9\x9c\x1c\xb8\x77\xff\x0c\x00\x00\xff\xff\x46\xbb\x1e\xf6\xec\x0e\x00\x00") func localStorageYamlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 6389641b2536..0942330ea9f7 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -182,7 +182,7 @@ func (e *ETCD) SetControlConfig(ctx context.Context, config *config.Control) err e.client.Close() }() - address, err := GetAdvertiseAddress(config.PrivateIP) + address, err := getAdvertiseAddress(config.PrivateIP) if err != nil { return err } @@ -537,7 +537,7 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, handler htt e.client.Close() }() - address, err := GetAdvertiseAddress(config.PrivateIP) + address, err := getAdvertiseAddress(config.PrivateIP) if err != nil { return nil, err } @@ -708,7 +708,7 @@ func toTLSConfig(runtime *config.ControlRuntime) (*tls.Config, error) { } // getAdvertiseAddress returns the IP address best suited for advertising to clients -func GetAdvertiseAddress(advertiseIP string) (string, error) { +func getAdvertiseAddress(advertiseIP string) (string, error) { ip := advertiseIP if ip == "" { ipAddr, err := utilnet.ChooseHostInterface() @@ -809,9 +809,19 @@ func (e *ETCD) clientURL() string { return fmt.Sprintf("https://%s", net.JoinHostPort(e.address, "2379")) } +// advertiseClientURLs returns the advertised addresses for the local node. +// During cluster reset/restore we only listen on loopback to avoid having apiservers +// on other nodes connect mid-process. +func (e *ETCD) advertiseClientURLs(reset bool) string { + if reset { + return fmt.Sprintf("https://%s", net.JoinHostPort(e.config.Loopback(true), "2379")) + } + return e.clientURL() +} + // listenClientURLs returns a list of URLs to bind to for client connections. -// During cluster reset/restore, we only listen on loopback to avoid having the apiserver -// connect mid-process. +// During cluster reset/restore, we only listen on loopback to avoid having apiservers +// on other nodes connect mid-process. func (e *ETCD) listenClientURLs(reset bool) string { clientURLs := fmt.Sprintf("https://%s:2379", e.config.Loopback(true)) if !reset { @@ -839,7 +849,7 @@ func (e *ETCD) cluster(ctx context.Context, reset bool, options executor.Initial ListenClientURLs: e.listenClientURLs(reset), ListenMetricsURLs: e.listenMetricsURLs(reset), ListenPeerURLs: e.listenPeerURLs(reset), - AdvertiseClientURLs: e.clientURL(), + AdvertiseClientURLs: e.advertiseClientURLs(reset), DataDir: DBDir(e.config), ServerTrust: executor.ServerTrust{ CertFile: e.config.Runtime.ServerETCDCert, @@ -941,7 +951,7 @@ func (e *ETCD) RemovePeer(ctx context.Context, name, address string, allowSelfRe } logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address) _, err := e.client.MemberRemove(ctx, member.ID) - if err == rpctypes.ErrGRPCMemberNotFound { + if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) { return nil } return err @@ -1142,7 +1152,7 @@ func ClientURLs(ctx context.Context, clientAccessInfo *clientaccess.Info, selfIP if err := json.Unmarshal(resp, &memberList); err != nil { return nil, memberList, err } - ip, err := GetAdvertiseAddress(selfIP) + ip, err := getAdvertiseAddress(selfIP) if err != nil { return nil, memberList, err } diff --git a/scripts/airgap/image-list.txt b/scripts/airgap/image-list.txt index ab72c7d9c9d2..b09729ed8ffa 100644 --- a/scripts/airgap/image-list.txt +++ b/scripts/airgap/image-list.txt @@ -1,7 +1,7 @@ docker.io/rancher/klipper-helm:v0.7.4-build20221121 docker.io/rancher/klipper-lb:v0.4.0 -docker.io/rancher/local-path-provisioner:v0.0.23 -docker.io/rancher/mirrored-coredns-coredns:1.9.4 +docker.io/rancher/local-path-provisioner:v0.0.24 +docker.io/rancher/mirrored-coredns-coredns:1.10.1 docker.io/rancher/mirrored-library-busybox:1.34.1 docker.io/rancher/mirrored-library-traefik:2.9.4 docker.io/rancher/mirrored-metrics-server:v0.6.2