Feature: yurt-tunnel and yurthub support working on ipv6 network (#842)

Co-authored-by: tydra-wang <[email protected]>

cmd/yurt-tunnel-agent/app/options/options.go

@@ -25,10 +25,12 @@ import (
 
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
+	utilnet "k8s.io/utils/net"
 	"sigs.k8s.io/apiserver-network-proxy/pkg/agent"
 
 	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app/config"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	utilip "github.com/openyurtio/openyurt/pkg/util/ip"
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
 	kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes"
 )
@@ -52,7 +54,6 @@ type AgentOptions struct {
 // NewAgentOptions creates a new AgentOptions with a default config.
 func NewAgentOptions() *AgentOptions {
 	o := &AgentOptions{
-		MetaHost: "127.0.0.1",
 		MetaPort: constants.YurttunnelAgentMetaPort,
 	}
 
@@ -75,6 +76,10 @@ func (o *AgentOptions) Validate() error {
 		}
 	}
 
+	if o.MetaHost == "" {
+		o.MetaHost = utilip.MustGetLoopbackIP(utilnet.IsIPv6String(o.NodeIP))
+	}
+
 	if !agentIdentifiersAreValid(o.AgentIdentifiers) {
 		return errors.New("--agent-identifiers are invalid, format should be host={node-name}")
 	}
@@ -134,7 +139,11 @@ func (o *AgentOptions) Config() (*config.Config, error) {
 	}
 
 	if len(c.AgentIdentifiers) == 0 {
-		c.AgentIdentifiers = fmt.Sprintf("ipv4=%s&host=%s", o.NodeIP, o.NodeName)
+		ipFamily := "ipv4"
+		if utilnet.IsIPv6String(o.NodeIP) {
+			ipFamily = "ipv6"
+		}
+		c.AgentIdentifiers = fmt.Sprintf("%s=%s&host=%s", ipFamily, o.NodeIP, o.NodeName)
 	}
 	klog.Infof("%s is set for agent identifies", c.AgentIdentifiers)
 

cmd/yurt-tunnel-server/app/config/config.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	"github.com/openyurtio/openyurt/pkg/util/iptables"
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
 )
 
@@ -34,6 +35,7 @@ type Config struct {
 	EnableIptables              bool
 	EnableDNSController         bool
 	IptablesSyncPeriod          int
+	IPFamily                    iptables.Protocol
 	DNSSyncPeriod               int
 	CertDNSNames                []string
 	CertIPs                     []net.IP
@@ -72,3 +74,7 @@ func (c *Config) Complete() *CompletedConfig {
 	}
 	return &CompletedConfig{&cc}
 }
+
+func (c *Config) IsIPv6() bool {
+	return c.IPFamily == iptables.ProtocolIpv6
+}

cmd/yurt-tunnel-server/app/options/options.go

@@ -25,11 +25,14 @@ import (
 	"github.com/spf13/pflag"
 	"k8s.io/client-go/informers"
 	"k8s.io/klog/v2"
+	utilnet "k8s.io/utils/net"
 	"sigs.k8s.io/apiserver-network-proxy/pkg/server"
 
 	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/config"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/util/certmanager"
+	utilip "github.com/openyurtio/openyurt/pkg/util/ip"
+	"github.com/openyurtio/openyurt/pkg/util/iptables"
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
 	kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes"
 )
@@ -60,7 +63,6 @@ type ServerOptions struct {
 func NewServerOptions() *ServerOptions {
 	o := &ServerOptions{
 		BindAddr:               "0.0.0.0",
-		InsecureBindAddr:       "127.0.0.1",
 		EnableIptables:         true,
 		EnableDNSController:    true,
 		IptablesSyncPeriod:     60,
@@ -81,6 +83,9 @@ func (o *ServerOptions) Validate() error {
 		return fmt.Errorf("%s's bind address can't be empty",
 			projectinfo.GetServerName())
 	}
+	if len(o.InsecureBindAddr) == 0 {
+		o.InsecureBindAddr = utilip.MustGetLoopbackIP(utilnet.IsIPv6String(o.BindAddr))
+	}
 	return nil
 }
 
@@ -136,6 +141,11 @@ func (o *ServerOptions) Config() (*config.Config, error) {
 		}
 	}
 
+	if utilnet.IsIPv6String(o.BindAddr) {
+		cfg.IPFamily = iptables.ProtocolIpv6
+	} else {
+		cfg.IPFamily = iptables.ProtocolIpv4
+	}
 	cfg.ListenAddrForAgent = net.JoinHostPort(o.BindAddr, o.TunnelAgentConnectPort)
 	cfg.ListenAddrForMaster = net.JoinHostPort(o.BindAddr, o.SecurePort)
 	cfg.ListenInsecureAddrForMaster = net.JoinHostPort(o.InsecureBindAddr, o.InsecurePort)

cmd/yurt-tunnel-server/app/start.go

@@ -93,11 +93,12 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 	}
 	// 1. start the IP table manager
 	if cfg.EnableIptables {
-		iptablesMgr := iptables.NewIptablesManager(cfg.Client,
+		iptablesMgr := iptables.NewIptablesManagerWithIPFamily(cfg.Client,
 			cfg.SharedInformerFactory.Core().V1().Nodes(),
 			cfg.ListenAddrForMaster,
 			cfg.ListenInsecureAddrForMaster,
-			cfg.IptablesSyncPeriod)
+			cfg.IptablesSyncPeriod,
+			cfg.IPFamily)
 		if iptablesMgr == nil {
 			return fmt.Errorf("fail to create a new IptableManager")
 		}
@@ -121,7 +122,7 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 
 	// 4. create handler wrappers
 	mInitializer := initializer.NewMiddlewareInitializer(cfg.SharedInformerFactory)
-	wrappers, err := wraphandler.InitHandlerWrappers(mInitializer)
+	wrappers, err := wraphandler.InitHandlerWrappers(mInitializer, cfg.IsIPv6())
 	if err != nil {
 		klog.Errorf("failed to init handler wrappers, %v", err)
 		return err

cmd/yurthub/app/config/config.go

@@ -91,6 +91,7 @@ type YurtHubConfiguration struct {
 	WorkingMode                       util.WorkingMode
 	KubeletHealthGracePeriod          time.Duration
 	FilterManager                     *filter.Manager
+	CertIPs                           []net.IP
 }
 
 // Complete converts *options.YurtHubOptions to *YurtHubConfiguration
@@ -135,6 +136,12 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 		return nil, err
 	}
 
+	// use dummy ip and bind ip as cert IP SANs
+	certIPs := []net.IP{
+		net.ParseIP(options.HubAgentDummyIfIP),
+		net.ParseIP(options.YurtHubHost),
+	}
+
 	cfg := &YurtHubConfiguration{
 		LBMode:                            options.LBMode,
 		RemoteServers:                     us,
@@ -167,6 +174,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 		YurtSharedFactory:                 yurtSharedFactory,
 		KubeletHealthGracePeriod:          options.KubeletHealthGracePeriod,
 		FilterManager:                     filterManager,
+		CertIPs:                           certIPs,
 	}
 
 	return cfg, nil

cmd/yurthub/app/options/options.go

@@ -23,15 +23,19 @@ import (
 	"time"
 
 	"github.com/spf13/pflag"
+	"k8s.io/klog/v2"
+	utilnet "k8s.io/utils/net"
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/yurthub/storage/disk"
 	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
 
 const (
-	DummyIfCIDR   = "169.254.0.0/16"
-	ExclusiveCIDR = "169.254.31.0/24"
+	DefaultDummyIfIP4 = "169.254.2.1"
+	DefaultDummyIfIP6 = "fd00::2:1"
+	DummyIfCIDR4      = "169.254.0.0/16"
+	ExclusiveCIDR     = "169.254.31.0/24"
 )
 
 // YurtHubOptions is the main settings for the yurthub
@@ -90,7 +94,6 @@ func NewYurtHubOptions() *YurtHubOptions {
 		EnableProfiling:           true,
 		EnableDummyIf:             true,
 		EnableIptables:            true,
-		HubAgentDummyIfIP:         "169.254.2.1",
 		HubAgentDummyIfName:       fmt.Sprintf("%s-dummy0", projectinfo.GetHubName()),
 		DiskCachePath:             disk.CacheBaseDir,
 		AccessServerThroughHub:    true,
@@ -103,8 +106,8 @@ func NewYurtHubOptions() *YurtHubOptions {
 	return o
 }
 
-// ValidateOptions validates YurtHubOptions
-func ValidateOptions(options *YurtHubOptions) error {
+// Validate validates YurtHubOptions
+func (options *YurtHubOptions) Validate() error {
 	if len(options.NodeName) == 0 {
 		return fmt.Errorf("node name is empty")
 	}
@@ -125,7 +128,7 @@ func ValidateOptions(options *YurtHubOptions) error {
 		return fmt.Errorf("working mode %s is not supported", options.WorkingMode)
 	}
 
-	if err := verifyDummyIP(options.HubAgentDummyIfIP); err != nil {
+	if err := options.verifyDummyIP(); err != nil {
 		return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err)
 	}
 
@@ -168,21 +171,35 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&o.EnableNodePool, "enable-node-pool", o.EnableNodePool, "enable list/watch nodepools resource or not for filters(only used for testing)")
 }
 
-// verifyDummyIP verify the specified ip is valid or not
-func verifyDummyIP(dummyIP string) error {
-	//169.254.2.1/32
+// verifyDummyIP verify the specified ip is valid or not and set the default ip if empty
+func (o *YurtHubOptions) verifyDummyIP() error {
+	if o.HubAgentDummyIfIP == "" {
+		if utilnet.IsIPv6String(o.YurtHubHost) {
+			o.HubAgentDummyIfIP = DefaultDummyIfIP6
+		} else {
+			o.HubAgentDummyIfIP = DefaultDummyIfIP4
+		}
+		klog.Infof("dummy ip not set, will use %s as default", o.HubAgentDummyIfIP)
+		return nil
+	}
+
+	dummyIP := o.HubAgentDummyIfIP
 	dip := net.ParseIP(dummyIP)
 	if dip == nil {
 		return fmt.Errorf("dummy ip %s is invalid", dummyIP)
 	}
 
-	_, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR)
+	if utilnet.IsIPv6(dip) {
+		return nil
+	}
+
+	_, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR4)
 	if err != nil {
-		return fmt.Errorf("cidr(%s) is invalid, %w", DummyIfCIDR, err)
+		return fmt.Errorf("cidr(%s) is invalid, %w", DummyIfCIDR4, err)
 	}
 
 	if !dummyIfIPNet.Contains(dip) {
-		return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR)
+		return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR4)
 	}
 
 	_, exclusiveIPNet, err := net.ParseCIDR(ExclusiveCIDR)

cmd/yurthub/app/start.go

@@ -59,7 +59,7 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command {
 			cmd.Flags().VisitAll(func(flag *pflag.Flag) {
 				klog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value)
 			})
-			if err := options.ValidateOptions(yurtHubOptions); err != nil {
+			if err := yurtHubOptions.Validate(); err != nil {
 				klog.Fatalf("validate options: %v", err)
 			}
 
@@ -125,7 +125,8 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	trace++
 
 	klog.Infof("%d. create tls config for secure servers ", trace)
-	cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.YurtHubProxyServerSecureDummyAddr, stopCh)
+	cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(
+		restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.CertIPs, stopCh)
 	if err != nil {
 		return fmt.Errorf("could not create tls config, %w", err)
 	}

pkg/util/certmanager/certmanager.go

@@ -38,6 +38,7 @@ import (
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/util/certmanager/store"
+	utilip "github.com/openyurtio/openyurt/pkg/util/ip"
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr"
 )
@@ -69,7 +70,7 @@ func NewYurttunnelServerCertManager(
 	)
 
 	// the ips and dnsNames should be acquired through api-server at the first time, because the informer factory has not started yet.
-	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
+	werr := wait.PollUntil(5*time.Second, func() (bool, error) {
 		dnsNames, ips, err = serveraddr.GetYurttunelServerDNSandIP(clientset)
 		if err != nil {
 			klog.Errorf("failed to get yurt tunnel server dns and ip, %v", err)
@@ -94,10 +95,14 @@ func NewYurttunnelServerCertManager(
 
 		return true, nil
 	}, stopCh)
-	// add user specified DNS names and IP addresses
+	if werr != nil {
+		return nil, werr
+	}
+
+	// add user specified DNS anems and IP addresses
 	dnsNames = append(dnsNames, clCertNames...)
 	ips = append(ips, clIPs...)
-	klog.Infof("subject of tunnel server certificate, ips=%#+v, dnsNames=%#+v", ips, dnsNames)
+	klog.Infof("subject of tunnel server certificate, ips=%s, dnsNames=%#+v", utilip.JoinIPStrings(ips), dnsNames)
 
 	// the dynamic ip acquire func
 	getIPs := func() ([]net.IP, error) {
@@ -180,13 +185,8 @@ func NewYurttunnelAgentCertManager(
 func NewYurtHubServerCertManager(
 	clientset kubernetes.Interface,
 	certDir,
-	nodeName,
-	proxyServerSecureDummyAddr string) (certificate.Manager, error) {
-
-	host, _, err := net.SplitHostPort(proxyServerSecureDummyAddr)
-	if err != nil {
-		return nil, err
-	}
+	nodeName string,
+	certIPs []net.IP) (certificate.Manager, error) {
 
 	return newCertManager(
 		clientset,
@@ -201,7 +201,7 @@ func NewYurtHubServerCertManager(
 			certificatesv1.UsageDigitalSignature,
 			certificatesv1.UsageServerAuth,
 		},
-		[]net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(host)},
+		certIPs,
 		nil)
 }