From e898f9bbd2198258c30562f04a85f5373af7c26c Mon Sep 17 00:00:00 2001 From: Chris Hein Date: Thu, 3 Jun 2021 11:25:44 -0700 Subject: [PATCH] adding vn-agent healthcheck & ability to boot from kconfig Signed-off-by: Chris Hein --- .../cmd/vn-agent/app/options/options.go | 4 ++++ virtualcluster/cmd/vn-agent/app/server.go | 11 ++++++++- virtualcluster/go.sum | 1 + .../pkg/controller/controllers/suite_test.go | 4 ++-- virtualcluster/pkg/vn-agent/server/server.go | 23 +++++++++++++++---- .../pkg/vn-agent/server/test/go.sum | 1 + .../pkg/vn-agent/server/test/server_test.go | 3 ++- 7 files changed, 38 insertions(+), 9 deletions(-) diff --git a/virtualcluster/cmd/vn-agent/app/options/options.go b/virtualcluster/cmd/vn-agent/app/options/options.go index 621dbac0..d19f8f95 100644 --- a/virtualcluster/cmd/vn-agent/app/options/options.go +++ b/virtualcluster/cmd/vn-agent/app/options/options.go @@ -51,6 +51,9 @@ type ServerOption struct { // Port is the vn-agent server listening on. Port uint + // Kubeconfig is the supercluster Kubeconfig to connect to + Kubeconfig string + // FeatureGates enabled by the user. FeatureGates map[string]bool } @@ -84,6 +87,7 @@ func (o *Options) Flags() cliflag.NamedFlagSets { serverFS.StringVar(&o.CertDirectory, "cert-dir", o.CertDirectory, "CertDirectory is the directory where the TLS certs are located") serverFS.StringVar(&o.TLSCertFile, "tls-cert-file", o.TLSCertFile, "TLSCertFile is the file containing x509 Certificate for HTTPS") serverFS.StringVar(&o.TLSPrivateKeyFile, "tls-private-key-file", o.TLSPrivateKeyFile, "TLSPrivateKeyFile is the file containing x509 private key matching tlsCertFile") + serverFS.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") serverFS.UintVar(&o.Port, "port", 10550, "Port is the server listening on") serverFS.Var(cliflag.NewMapStringBool(&o.ServerOption.FeatureGates), "feature-gates", "A set of key=value pairs that describe featuregate gates for various features.") diff --git a/virtualcluster/cmd/vn-agent/app/server.go b/virtualcluster/cmd/vn-agent/app/server.go index fe72d825..82f9fbbf 100644 --- a/virtualcluster/cmd/vn-agent/app/server.go +++ b/virtualcluster/cmd/vn-agent/app/server.go @@ -30,6 +30,7 @@ import ( "k8s.io/component-base/cli/globalflag" "k8s.io/component-base/term" "k8s.io/klog" + "k8s.io/kubernetes/pkg/healthz" "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options" utilflag "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/util/flag" @@ -90,7 +91,7 @@ func NewVnAgentCommand(stopChan <-chan struct{}) *cobra.Command { // Run start the vn-agent server. func Run(c *config.Config, serverOption *options.ServerOption, stopCh <-chan struct{}) error { - handler, err := server.NewServer(c) + handler, err := server.NewServer(c, serverOption) if err != nil { return errors.Wrapf(err, "create server") } @@ -131,6 +132,14 @@ func Run(c *config.Config, serverOption *options.ServerOption, stopCh <-chan str errCh <- err }() + go func() { + // start a health http server. + mux := http.NewServeMux() + healthz.InstallHandler(mux) + klog.Fatal(http.ListenAndServe(":8080", mux)) + errCh <- err + }() + select { case <-stopCh: klog.Infof("closing server...") diff --git a/virtualcluster/go.sum b/virtualcluster/go.sum index 3a190bb2..9acf2c01 100644 --- a/virtualcluster/go.sum +++ b/virtualcluster/go.sum @@ -1082,6 +1082,7 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhD k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/kubernetes v0.20.2 h1:KxQftTwyg3DRlwKDcI6wla6eLygmMJ21FXlE4o7QcvU= k8s.io/kubernetes v0.20.2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= diff --git a/virtualcluster/pkg/controller/controllers/suite_test.go b/virtualcluster/pkg/controller/controllers/suite_test.go index 4930e95b..fbfa1446 100644 --- a/virtualcluster/pkg/controller/controllers/suite_test.go +++ b/virtualcluster/pkg/controller/controllers/suite_test.go @@ -44,8 +44,8 @@ var cfg *rest.Config var cli client.Client var testEnv *envtest.Environment -const timeout = time.Second * 10 -const longTimeout = time.Second * 20 +const timeout = time.Second * 20 +const longTimeout = time.Second * 60 const interval = time.Millisecond * 250 func TestAPIs(t *testing.T) { diff --git a/virtualcluster/pkg/vn-agent/server/server.go b/virtualcluster/pkg/vn-agent/server/server.go index 6c7f0737..b96e12e2 100644 --- a/virtualcluster/pkg/vn-agent/server/server.go +++ b/virtualcluster/pkg/vn-agent/server/server.go @@ -18,14 +18,17 @@ package server import ( "crypto/tls" + "crypto/x509" "net/http" "net/url" "github.com/emicklei/go-restful" "github.com/pkg/errors" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" certutil "k8s.io/client-go/util/cert" + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options" "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/config" ) @@ -44,7 +47,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // NewServer initializes and configures a vn-agent.Server object to handle HTTP requests. -func NewServer(cfg *config.Config) (*Server, error) { +func NewServer(cfg *config.Config, serverOption *options.ServerOption) (*Server, error) { u, err := url.Parse(cfg.KubeletServerHost) if err != nil { return nil, errors.Wrap(err, "parse kubelet server url") @@ -66,9 +69,20 @@ func NewServer(cfg *config.Config) (*Server, error) { }, } } else { - restConfig, err := rest.InClusterConfig() - if err != nil { - return nil, errors.Wrapf(err, "failed to get in cluster config") + var restConfig *rest.Config + var caCrtPool *x509.CertPool + if len(serverOption.Kubeconfig) == 0 { + restConfig, err = rest.InClusterConfig() + if err != nil { + return nil, errors.Wrapf(err, "failed to get in cluster config") + } + caCrtPool, err = certutil.NewPool(restConfig.TLSClientConfig.CAFile) + } else { + // This creates a client, first loading any specified kubeconfig\ + restConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: serverOption.Kubeconfig}, + &clientcmd.ConfigOverrides{}).ClientConfig() + caCrtPool, err = certutil.NewPoolFromBytes(restConfig.TLSClientConfig.CAData) } server.restConfig = restConfig superHttpsUrl, err := url.Parse(restConfig.Host) @@ -76,7 +90,6 @@ func NewServer(cfg *config.Config) (*Server, error) { return nil, errors.Wrapf(err, "unable to parse apiserver address") } server.superAPIServerAddress = superHttpsUrl - caCrtPool, err := certutil.NewPool(restConfig.TLSClientConfig.CAFile) if err != nil { return nil, errors.Wrapf(err, "unable to parse ca file") } diff --git a/virtualcluster/pkg/vn-agent/server/test/go.sum b/virtualcluster/pkg/vn-agent/server/test/go.sum index 49b03a12..d8de0aa1 100644 --- a/virtualcluster/pkg/vn-agent/server/test/go.sum +++ b/virtualcluster/pkg/vn-agent/server/test/go.sum @@ -548,6 +548,7 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= diff --git a/virtualcluster/pkg/vn-agent/server/test/server_test.go b/virtualcluster/pkg/vn-agent/server/test/server_test.go index cc90ad72..20ef186b 100644 --- a/virtualcluster/pkg/vn-agent/server/test/server_test.go +++ b/virtualcluster/pkg/vn-agent/server/test/server_test.go @@ -61,6 +61,7 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/utils/pointer" + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options" "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/config" "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/server" "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/testcerts" @@ -124,7 +125,7 @@ func newServerTestWithDebug(enableDebugging bool, streamingServer streaming.Serv server, err := server.NewServer(&config.Config{ KubeletClientCert: &kubeletClientCert, KubeletServerHost: fv.kubeletServer.testHTTPServer.URL, - }) + }, &options.ServerOption{}) if err != nil { panic(errors.Wrap(err, "new server")) }