diff --git a/cmd/yurt-tunnel-agent/app/options/options.go b/cmd/yurt-tunnel-agent/app/options/options.go index 9231a67ebc9..6541c5f382b 100644 --- a/cmd/yurt-tunnel-agent/app/options/options.go +++ b/cmd/yurt-tunnel-agent/app/options/options.go @@ -25,10 +25,12 @@ import ( "github.com/spf13/pflag" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "sigs.k8s.io/apiserver-network-proxy/pkg/agent" "github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app/config" "github.com/openyurtio/openyurt/pkg/projectinfo" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes" ) @@ -52,7 +54,6 @@ type AgentOptions struct { // NewAgentOptions creates a new AgentOptions with a default config. func NewAgentOptions() *AgentOptions { o := &AgentOptions{ - MetaHost: "127.0.0.1", MetaPort: constants.YurttunnelAgentMetaPort, } @@ -75,6 +76,10 @@ func (o *AgentOptions) Validate() error { } } + if o.MetaHost == "" { + o.MetaHost = utilip.MustGetLoopbackIP(utilnet.IsIPv6String(o.NodeIP)) + } + if !agentIdentifiersAreValid(o.AgentIdentifiers) { return errors.New("--agent-identifiers are invalid, format should be host={node-name}") } @@ -134,7 +139,11 @@ func (o *AgentOptions) Config() (*config.Config, error) { } if len(c.AgentIdentifiers) == 0 { - c.AgentIdentifiers = fmt.Sprintf("ipv4=%s&host=%s", o.NodeIP, o.NodeName) + ipFamily := "ipv4" + if utilnet.IsIPv6String(o.NodeIP) { + ipFamily = "ipv6" + } + c.AgentIdentifiers = fmt.Sprintf("%s=%s&host=%s", ipFamily, o.NodeIP, o.NodeName) } klog.Infof("%s is set for agent identifies", c.AgentIdentifiers) diff --git a/cmd/yurt-tunnel-server/app/config/config.go b/cmd/yurt-tunnel-server/app/config/config.go index 84e7e2f298e..73d5b03c977 100644 --- a/cmd/yurt-tunnel-server/app/config/config.go +++ b/cmd/yurt-tunnel-server/app/config/config.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -34,6 +35,7 @@ type Config struct { EnableIptables bool EnableDNSController bool IptablesSyncPeriod int + IPFamily iptables.Protocol DNSSyncPeriod int CertDNSNames []string CertIPs []net.IP @@ -72,3 +74,7 @@ func (c *Config) Complete() *CompletedConfig { } return &CompletedConfig{&cc} } + +func (c *Config) IsIPv6() bool { + return c.IPFamily == iptables.ProtocolIpv6 +} diff --git a/cmd/yurt-tunnel-server/app/options/options.go b/cmd/yurt-tunnel-server/app/options/options.go index 1dfb23603ea..1833b576770 100644 --- a/cmd/yurt-tunnel-server/app/options/options.go +++ b/cmd/yurt-tunnel-server/app/options/options.go @@ -25,11 +25,14 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/informers" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "sigs.k8s.io/apiserver-network-proxy/pkg/server" "github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/config" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/certmanager" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" + "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes" ) @@ -60,7 +63,6 @@ type ServerOptions struct { func NewServerOptions() *ServerOptions { o := &ServerOptions{ BindAddr: "0.0.0.0", - InsecureBindAddr: "127.0.0.1", EnableIptables: true, EnableDNSController: true, IptablesSyncPeriod: 60, @@ -81,6 +83,9 @@ func (o *ServerOptions) Validate() error { return fmt.Errorf("%s's bind address can't be empty", projectinfo.GetServerName()) } + if len(o.InsecureBindAddr) == 0 { + o.InsecureBindAddr = utilip.MustGetLoopbackIP(utilnet.IsIPv6String(o.BindAddr)) + } return nil } @@ -136,6 +141,11 @@ func (o *ServerOptions) Config() (*config.Config, error) { } } + if utilnet.IsIPv6String(o.BindAddr) { + cfg.IPFamily = iptables.ProtocolIpv6 + } else { + cfg.IPFamily = iptables.ProtocolIpv4 + } cfg.ListenAddrForAgent = net.JoinHostPort(o.BindAddr, o.TunnelAgentConnectPort) cfg.ListenAddrForMaster = net.JoinHostPort(o.BindAddr, o.SecurePort) cfg.ListenInsecureAddrForMaster = net.JoinHostPort(o.InsecureBindAddr, o.InsecurePort) diff --git a/cmd/yurt-tunnel-server/app/start.go b/cmd/yurt-tunnel-server/app/start.go index bc4fb7b43f2..9e83504dfda 100644 --- a/cmd/yurt-tunnel-server/app/start.go +++ b/cmd/yurt-tunnel-server/app/start.go @@ -93,11 +93,12 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { } // 1. start the IP table manager if cfg.EnableIptables { - iptablesMgr := iptables.NewIptablesManager(cfg.Client, + iptablesMgr := iptables.NewIptablesManagerWithIPFamily(cfg.Client, cfg.SharedInformerFactory.Core().V1().Nodes(), cfg.ListenAddrForMaster, cfg.ListenInsecureAddrForMaster, - cfg.IptablesSyncPeriod) + cfg.IptablesSyncPeriod, + cfg.IPFamily) if iptablesMgr == nil { return fmt.Errorf("fail to create a new IptableManager") } @@ -115,7 +116,7 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { // 3. create handler wrappers mInitializer := initializer.NewMiddlewareInitializer(cfg.SharedInformerFactory) - wrappers, err := wraphandler.InitHandlerWrappers(mInitializer) + wrappers, err := wraphandler.InitHandlerWrappers(mInitializer, cfg.IsIPv6()) if err != nil { klog.Errorf("failed to init handler wrappers, %v", err) return err diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index cee8d1d10a6..3cc34a96c02 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -17,12 +17,14 @@ limitations under the License. package options import ( + "errors" "fmt" "net" "path/filepath" "time" "github.com/spf13/pflag" + utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" @@ -125,6 +127,10 @@ func ValidateOptions(options *YurtHubOptions) error { return fmt.Errorf("working mode %s is not supported", options.WorkingMode) } + if options.EnableDummyIf && utilnet.IsIPv6String(options.YurtHubHost) { + return errors.New("dummy ip not supported in ipv6") + } + if err := verifyDummyIP(options.HubAgentDummyIfIP); err != nil { return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err) } diff --git a/pkg/util/certmanager/certmanager.go b/pkg/util/certmanager/certmanager.go index b230da613d0..04726facab6 100644 --- a/pkg/util/certmanager/certmanager.go +++ b/pkg/util/certmanager/certmanager.go @@ -38,6 +38,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/certmanager/store" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" "github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr" ) @@ -64,7 +65,7 @@ func NewYurttunnelServerCertManager( ) // the ips and dnsNames should be acquired through api-server at the first time, because the informer factory has not started yet. - _ = wait.PollUntil(5*time.Second, func() (bool, error) { + werr := wait.PollUntil(5*time.Second, func() (bool, error) { dnsNames, ips, err = serveraddr.GetYurttunelServerDNSandIP(clientset) if err != nil { klog.Errorf("failed to get yurt tunnel server dns and ip, %v", err) @@ -89,10 +90,14 @@ func NewYurttunnelServerCertManager( return true, nil }, stopCh) - // add user specified DNS names and IP addresses + if werr != nil { + return nil, werr + } + + // add user specified DNS anems and IP addresses dnsNames = append(dnsNames, clCertNames...) ips = append(ips, clIPs...) - klog.Infof("subject of tunnel server certificate, ips=%#+v, dnsNames=%#+v", ips, dnsNames) + klog.Infof("subject of tunnel server certificate, ips=%s, dnsNames=%#+v", utilip.JoinIPStrings(ips), dnsNames) // the dynamic ip acquire func getIPs := func() ([]net.IP, error) { diff --git a/pkg/util/ip/ip.go b/pkg/util/ip/ip.go new file mode 100644 index 00000000000..2703c6e690b --- /dev/null +++ b/pkg/util/ip/ip.go @@ -0,0 +1,52 @@ +package ip + +import ( + "net" + "strings" + + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" +) + +const ( + DefaultLoopbackIP4 = "127.0.0.1" + DefaultLoopbackIP6 = "::1" +) + +// MustGetLoopbackIP is a wrapper for GetLoopbackIP. If any error occurs or loopback interface is not found, +// will fall back to 127.0.0.1 for ipv4 or ::1 for ipv6. +func MustGetLoopbackIP(wantIPv6 bool) string { + ip, err := GetLoopbackIP(wantIPv6) + if err != nil { + klog.Errorf("failed to get loopback addr: %v", err) + } + if ip != "" { + return ip + } + if wantIPv6 { + return DefaultLoopbackIP6 + } + return DefaultLoopbackIP4 +} + +// GetLoopbackIP returns the ip address of local loopback interface. +func GetLoopbackIP(wantIPv6 bool) (string, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", err + } + for _, address := range addrs { + if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsLoopback() && wantIPv6 == utilnet.IsIPv6(ipnet.IP) { + return ipnet.IP.String(), nil + } + } + return "", nil +} + +func JoinIPStrings(ips []net.IP) string { + var strs []string + for _, ip := range ips { + strs = append(strs, ip.String()) + } + return strings.Join(strs, ",") +} diff --git a/pkg/util/ip/ip_test.go b/pkg/util/ip/ip_test.go new file mode 100644 index 00000000000..527d6f4caf3 --- /dev/null +++ b/pkg/util/ip/ip_test.go @@ -0,0 +1,28 @@ +package ip + +import ( + "testing" +) + +func TestGetLoopbackIP(t *testing.T) { + lo4, err := GetLoopbackIP(false) + if err != nil { + t.Errorf("failed to get ipv4 loopback address: %v", err) + } + t.Logf("got ipv4 loopback address: %s", lo4) + if lo4 != "127.0.0.1" { + t.Errorf("got ipv4 loopback addr: '%s', expect: '127.0.0.1'", lo4) + } + + lo6, err := GetLoopbackIP(true) + if err != nil { + t.Errorf("failed to get ipv6 loopback address: %v", err) + } + if lo6 != "" { + // dual stack env + t.Logf("got ipv6 loopback address: %s", lo6) + if lo6 != "::1" { + t.Errorf("got ipv6 loopback addr: '%s', expect: '::1'", lo6) + } + } +} diff --git a/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go b/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go index 7b72b5edc22..7897c0b87a3 100644 --- a/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go +++ b/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go @@ -35,6 +35,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" hw "github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper" "github.com/openyurtio/openyurt/pkg/yurttunnel/util" @@ -47,11 +48,13 @@ type localHostProxyMiddleware struct { localhostPorts map[string]struct{} nodeInformerSynced cache.InformerSynced cmInformerSynced cache.InformerSynced + loopbackAddr string } -func NewLocalHostProxyMiddleware() hw.Middleware { +func NewLocalHostProxyMiddleware(isIPv6 bool) hw.Middleware { return &localHostProxyMiddleware{ localhostPorts: make(map[string]struct{}), + loopbackAddr: utilip.MustGetLoopbackIP(isIPv6), } } @@ -99,7 +102,7 @@ func (plm *localHostProxyMiddleware) WrapHandler(handler http.Handler) http.Hand req.Header.Set(constants.ProxyHostHeaderKey, nodeName) } - proxyDest = fmt.Sprintf("127.0.0.1:%s", port) + proxyDest = net.JoinHostPort(plm.loopbackAddr, port) oldHost := req.URL.Host req.Host = proxyDest req.Header.Set("Host", proxyDest) diff --git a/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go b/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go index d90ab7ea00e..c5919642bd7 100644 --- a/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go +++ b/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go @@ -27,7 +27,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/tracerequest" ) -func InitHandlerWrappers(mi initializer.MiddlewareInitializer) (hw.HandlerWrappers, error) { +func InitHandlerWrappers(mi initializer.MiddlewareInitializer, isIPv6 bool) (hw.HandlerWrappers, error) { wrappers := make(hw.HandlerWrappers, 0) // register all of middleware here // @@ -42,7 +42,7 @@ func InitHandlerWrappers(mi initializer.MiddlewareInitializer) (hw.HandlerWrappe // // then the middleware m2 will be called before the mw1 wrappers = append(wrappers, tracerequest.NewTraceReqMiddleware()) - wrappers = append(wrappers, localhostproxy.NewLocalHostProxyMiddleware()) + wrappers = append(wrappers, localhostproxy.NewLocalHostProxyMiddleware(isIPv6)) // init all of wrappers for i := range wrappers { diff --git a/pkg/yurttunnel/server/interceptor.go b/pkg/yurttunnel/server/interceptor.go index b5087f587c0..3e4baf4841a 100644 --- a/pkg/yurttunnel/server/interceptor.go +++ b/pkg/yurttunnel/server/interceptor.go @@ -88,7 +88,7 @@ func NewRequestInterceptor(udsSockFile string, cfg *tls.Config) *RequestIntercep } } - fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s%s\r\n\r\n", addr, "127.0.0.1", connectHeaders) + fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: localhost%s\r\n\r\n", addr, connectHeaders) br := newBufioReader(proxyConn) defer putBufioReader(br) res, err := http.ReadResponse(br, nil) diff --git a/pkg/yurttunnel/server/serveraddr/addr.go b/pkg/yurttunnel/server/serveraddr/addr.go index bfc2eca94c7..9024b7b3887 100644 --- a/pkg/yurttunnel/server/serveraddr/addr.go +++ b/pkg/yurttunnel/server/serveraddr/addr.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "net" + "strconv" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,6 +31,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -57,7 +59,7 @@ func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) { for _, tmpIP := range ips { // we use the first non-loopback IP address. - if tmpIP.String() != "127.0.0.1" { + if s := tmpIP.String(); s != utilip.DefaultLoopbackIP4 && s != utilip.DefaultLoopbackIP6 { ip = tmpIP break } @@ -86,7 +88,7 @@ func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) { return "", errors.New("fail to get the port number") } - return fmt.Sprintf("%s:%d", host, tcpPort), nil + return net.JoinHostPort(host, strconv.Itoa(int(tcpPort))), nil } // GetYurttunelServerDNSandIP gets DNS names and IPS for generating tunnel server certificate. @@ -231,7 +233,7 @@ func extractTunnelServerDNSandIPs(svc *corev1.Service, eps []*corev1.Endpoints, if svc.Spec.ClusterIP != "None" { ips = append(ips, net.ParseIP(svc.Spec.ClusterIP)) } - ips = append(ips, net.ParseIP("127.0.0.1")) + ips = append(ips, net.ParseIP(utilip.DefaultLoopbackIP4), net.ParseIP(utilip.DefaultLoopbackIP6)) // 3. extract dns and ip from the endpoint for _, eps := range eps { diff --git a/pkg/yurttunnel/trafficforward/iptables/iptables.go b/pkg/yurttunnel/trafficforward/iptables/iptables.go index 23230fc1287..0fccfd2763e 100644 --- a/pkg/yurttunnel/trafficforward/iptables/iptables.go +++ b/pkg/yurttunnel/trafficforward/iptables/iptables.go @@ -34,13 +34,13 @@ import ( utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/server/metrics" "github.com/openyurtio/openyurt/pkg/yurttunnel/util" ) const ( - loopbackAddr = "127.0.0.1" reqReturnComment = "return request to access node directly" dnatToTunnelComment = "dnat to tunnel for access node" yurttunnelServerPortChain = "TUNNEL-PORT" @@ -84,6 +84,7 @@ type iptablesManager struct { nodeInformer coreinformer.NodeInformer iptables iptables.Interface execer exec.Interface + loopbackAddr string conntrackPath string secureDnatDest string insecureDnatDest string @@ -92,18 +93,18 @@ type iptablesManager struct { syncPeriod int } -// NewIptablesManager creates an IptablesManager; deletes old chains, if any; +// NewIptablesManagerWithIPFamily creates an IptablesManager; deletes old chains, if any; // generates new dnat rules based on IPs of current active nodes; and // appends the rules to the iptable. -func NewIptablesManager(client clientset.Interface, +func NewIptablesManagerWithIPFamily(client clientset.Interface, nodeInformer coreinformer.NodeInformer, listenAddr string, listenInsecureAddr string, - syncPeriod int) IptablesManager { + syncPeriod int, + ipFamily iptables.Protocol) IptablesManager { - protocol := iptables.ProtocolIpv4 execer := exec.New() - iptInterface := iptables.New(execer, protocol) + iptInterface := iptables.New(execer, ipFamily) if syncPeriod < defaultSyncPeriod { syncPeriod = defaultSyncPeriod @@ -121,6 +122,8 @@ func NewIptablesManager(client clientset.Interface, syncPeriod: syncPeriod, } + im.loopbackAddr = ip.MustGetLoopbackIP(ipFamily == iptables.ProtocolIpv6) + // conntrack setting conntrackPath, err := im.execer.LookPath("conntrack") if err != nil { @@ -135,6 +138,15 @@ func NewIptablesManager(client clientset.Interface, return im } +// NewIptablesManager creates an IptablesManager with ipv4 protocol +func NewIptablesManager(client clientset.Interface, + nodeInformer coreinformer.NodeInformer, + listenAddr string, + listenInsecureAddr string, + syncPeriod int) IptablesManager { + return NewIptablesManagerWithIPFamily(client, nodeInformer, listenAddr, listenInsecureAddr, syncPeriod, iptables.ProtocolIpv4) +} + // Run starts the iptablesManager that will updates dnat rules periodically func (im *iptablesManager) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) { defer wg.Done() @@ -361,7 +373,7 @@ func (im *iptablesManager) ensurePortIptables(port string, currentIPs, deletedIP iptables.Prepend, iptables.TableNAT, portChain, reqReturnPortIptablesArgs...) if err != nil { - klog.Errorf("could not ensure -j RETURN iptables rule for %s:%s: %v", ip, port, err) + klog.Errorf("could not ensure -j RETURN iptables rule for %s: %v", net.JoinHostPort(ip, port), err) return err } } @@ -382,7 +394,7 @@ func (im *iptablesManager) ensurePortIptables(port string, currentIPs, deletedIP err = im.iptables.DeleteRule(iptables.TableNAT, portChain, deletedIPIptablesArgs...) if err != nil { - klog.Errorf("could not delete old iptables rules for %s:%s: %v", ip, port, err) + klog.Errorf("could not delete old iptables rules for %s: %v", net.JoinHostPort(ip, port), err) return err } } @@ -509,7 +521,7 @@ func (im *iptablesManager) syncIptableSetting() { // check if there are new nodes nodesIP := im.getIPOfNodesWithoutAgent() nodesChanged, addedNodesIP, deletedNodesIP := im.getAddedAndDeletedNodes(nodesIP) - currentNodesIP := append(nodesIP, loopbackAddr) + currentNodesIP := append(nodesIP, im.loopbackAddr) // update the iptables setting if necessary err = im.ensurePortsIptables(currentDnatPorts, deletedDnatPorts, currentNodesIP, deletedNodesIP, portMappings)