Merge pull request #157 from River-sh/add_healthy_check

add healthy check and some logs

charts/raven-agent/templates/daemonset.yaml

@@ -20,6 +20,26 @@ spec:
       - image: {{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         name: raven-agent
+        livenessProbe:
+          failureThreshold: 10
+          httpGet:
+            path: /healthz
+            port: 10275
+            scheme: HTTP
+          initialDelaySeconds: 20
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 15
+        readinessProbe:
+          failureThreshold: 10
+          httpGet:
+            path: /readyz
+            port: 10275
+            scheme: HTTP
+          initialDelaySeconds: 20
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 15
         securityContext:
           {{- toYaml .Values.securityContext | nindent 12 }}
         {{- with .Values.containerEnv }}
@@ -32,6 +52,7 @@ spec:
           - --forward-node-ip={{.Values.vpn.forwardNodeIP}}
           - --nat-traversal={{.Values.vpn.natTraversal}}
           - --metric-bind-addr={{.Values.vpn.metricBindAddr}}
+          - --health-probe-addr={{.Values.healthyBindAddr}}
           - --vpn-bind-port={{.Values.vpn.tunnelAddr}}
           - --proxy-metric-bind-addr={{.Values.proxy.metricsBindAddr}}
           - --proxy-internal-secure-addr={{.Values.proxy.internalSecureAddr}}

charts/raven-agent/values.yaml

@@ -11,6 +11,7 @@ image:
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
+healthyBindAddr: ":10275"
 
 serviceAccount:
   # Specifies whether a service account should be created

cmd/agent/app/config/config.go

@@ -26,10 +26,12 @@ type Config struct {
 	NodeName           string
 	NodeIP             string
 	MetricsBindAddress string
-	KubeConfig         *rest.Config
-	Manager            manager.Manager
-	Tunnel             *TunnelConfig
-	Proxy              *ProxyConfig
+	HealthProbeAddr    string
+
+	KubeConfig *rest.Config
+	Manager    manager.Manager
+	Tunnel     *TunnelConfig
+	Proxy      *ProxyConfig
 }
 
 type TunnelConfig struct {

cmd/agent/app/options/options.go

@@ -3,7 +3,6 @@ package options
 import (
 	"errors"
 	"fmt"
-
 	"net"
 	"os"
 	"strconv"
@@ -16,6 +15,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	"github.com/openyurtio/api/raven/v1beta1"
@@ -28,6 +28,7 @@ import (
 const (
 	DefaultTunnelMetricsPort = 10265
 	DefaultProxyMetricsPort  = 10266
+	DefaultHealthyProbeAddr  = 10275
 )
 
 // AgentOptions has the information that required by the raven agent
@@ -38,6 +39,7 @@ type AgentOptions struct {
 	NodeIP             string
 	Kubeconfig         string
 	MetricsBindAddress string
+	HealthProbeAddr    string
 }
 
 type TunnelOptions struct {
@@ -87,6 +89,8 @@ func (o *AgentOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&o.ForwardNodeIP, "forward-node-ip", o.ForwardNodeIP, `Forward node IP or not. (default "false")`)
 	fs.BoolVar(&o.NATTraversal, "nat-traversal", o.NATTraversal, `Enable NAT Traversal or not. (default "false")`)
 	fs.StringVar(&o.MetricsBindAddress, "metric-bind-addr", o.MetricsBindAddress, `Binding address of tunnel metrics. (default ":10265")`)
+	fs.StringVar(&o.HealthProbeAddr, "health-probe-addr", o.HealthProbeAddr, `The address the healthz/readyz endpoint binds to.. (default ":10275")`)
+
 	fs.StringVar(&o.VPNPort, "vpn-bind-port", o.VPNPort, `Binding port of vpn. (default ":4500")`)
 	fs.StringVar(&o.ProxyMetricsAddress, "proxy-metric-bind-addr", o.ProxyMetricsAddress, `Binding address of proxy metrics. (default ":10266")`)
 	fs.StringVar(&o.InternalSecureAddress, "proxy-internal-secure-addr", o.InternalSecureAddress, `Binding secure address of proxy server. (default ":10263")`)
@@ -114,7 +118,8 @@ func (o *AgentOptions) Config() (*config.Config, error) {
 	cfg = restclient.AddUserAgent(cfg, "raven-agent-ds")
 	c.KubeConfig = cfg
 	c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, c.NodeIP, strconv.Itoa(DefaultTunnelMetricsPort))
-	c.Manager, err = newMgr(cfg, c.MetricsBindAddress)
+	c.HealthProbeAddr = resolveAddress(c.HealthProbeAddr, c.NodeIP, strconv.Itoa(DefaultHealthyProbeAddr))
+	c.Manager, err = newMgr(cfg, c.MetricsBindAddress, c.HealthProbeAddr)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create manager: %s", err)
 	}
@@ -171,25 +176,39 @@ func (o *AgentOptions) Config() (*config.Config, error) {
 	return c, err
 }
 
-func newMgr(cfg *restclient.Config, metricsBindAddress string) (manager.Manager, error) {
+func newMgr(cfg *restclient.Config, metricsBindAddress, healthyProbeAddress string) (manager.Manager, error) {
 	scheme := runtime.NewScheme()
 	_ = v1.AddToScheme(scheme)
 	_ = v1beta1.AddToScheme(scheme)
 
 	opt := ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsBindAddress,
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsBindAddress,
+		HealthProbeBindAddress: healthyProbeAddress,
 	}
 
 	mgr, err := ctrl.NewManager(cfg, opt)
 	if err != nil {
 		klog.ErrorS(err, "failed to new manager for raven agent controller")
 		return nil, err
 	}
+
+	if err = mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
+		klog.ErrorS(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err = mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
+		klog.ErrorS(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
 	return mgr, nil
 }
 
 func resolveAddress(srcAddr, defaultHost, defaultPort string) string {
+	if srcAddr == "" {
+		return net.JoinHostPort(defaultHost, defaultPort)
+	}
 	host, port, err := net.SplitHostPort(srcAddr)
 	if err != nil {
 		return net.JoinHostPort(defaultHost, defaultPort)

pkg/networkengine/vpndriver/libreswan/libreswan.go

@@ -433,7 +433,7 @@ func whackCmdFn(args ...string) error {
 		time.Sleep(1 * time.Second)
 	}
 	if err != nil {
-		return fmt.Errorf("error whacking with %v: %v", args, err)
+		return fmt.Errorf("error whacking with %v: %v, error %s", args, err, string(output))
 	}
 	return nil
 }

pkg/proxyengine/proxyclient/proxyclient.go

@@ -78,6 +78,7 @@ func (c *ProxyClient) Start(ctx context.Context) error {
 		return fmt.Errorf("failed to new cert manager factory for proxy client %s, error %s", c.name, err.Error())
 	}
 	clientCertManager.Start()
+	defer clientCertManager.Stop()
 	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
 		if clientCertManager.Current() != nil {
 			return true, nil

pkg/proxyengine/proxyserver/manageheader.go

@@ -159,17 +159,16 @@ func (h *headerManger) getAPIServerRequestDestAddress(r *http.Request) (name, ip
 }
 
 func (h *headerManger) getNormalRequestDestAddress(r *http.Request) (name, ip, port string, err error) {
-	var nodeName string
-	nodeName, port, err = net.SplitHostPort(r.Host)
-	if err != nil {
-		return "", "", "", err
-	}
+	nodeName := r.Header.Get(utils.RavenProxyHostHeaderKey)
 	if nodeName == "" {
-		nodeName = r.Header.Get(utils.RavenProxyHostHeaderKey)
+		nodeName, port, err = net.SplitHostPort(r.Host)
+		if err != nil {
+			return "", "", "", err
+		}
 	}
 	ipAddress := net.ParseIP(nodeName)
 	if ipAddress != nil {
-		klog.Warning(utils.FormatProxyServer("raven proxy server not support request.Host is %s", r.Host))
+		klog.Warning(utils.FormatProxyServer("raven proxy server not support dest address %s and request.URL is %s", ipAddress, r.URL.String()))
 		return "", "", "", nil
 	}
 	var node v1.Node