diff --git a/cmd/yurt-controller-manager/controller-manager.go b/cmd/yurt-controller-manager/controller-manager.go index 6dcb942f9bd..f707272d61e 100644 --- a/cmd/yurt-controller-manager/controller-manager.go +++ b/cmd/yurt-controller-manager/controller-manager.go @@ -26,10 +26,8 @@ import ( "time" "k8s.io/component-base/logs" - // load all the prometheus client-go plugin _ "k8s.io/component-base/metrics/prometheus/clientgo" - // for version metric registration _ "k8s.io/component-base/metrics/prometheus/version" diff --git a/cmd/yurt-tunnel-agent/app/options/options.go b/cmd/yurt-tunnel-agent/app/options/options.go index 9231a67ebc9..30725062da9 100644 --- a/cmd/yurt-tunnel-agent/app/options/options.go +++ b/cmd/yurt-tunnel-agent/app/options/options.go @@ -25,6 +25,7 @@ import ( "github.com/spf13/pflag" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "sigs.k8s.io/apiserver-network-proxy/pkg/agent" "github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app/config" @@ -134,7 +135,11 @@ func (o *AgentOptions) Config() (*config.Config, error) { } if len(c.AgentIdentifiers) == 0 { - c.AgentIdentifiers = fmt.Sprintf("ipv4=%s&host=%s", o.NodeIP, o.NodeName) + ipFamily := "ipv4" + if utilnet.IsIPv6String(o.NodeIP) { + ipFamily = "ipv6" + } + c.AgentIdentifiers = fmt.Sprintf("%s=%s&host=%s", ipFamily, o.NodeIP, o.NodeName) } klog.Infof("%s is set for agent identifies", c.AgentIdentifiers) diff --git a/cmd/yurt-tunnel-server/app/config/config.go b/cmd/yurt-tunnel-server/app/config/config.go index 84e7e2f298e..fe2c1927e94 100644 --- a/cmd/yurt-tunnel-server/app/config/config.go +++ b/cmd/yurt-tunnel-server/app/config/config.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -34,6 +35,7 @@ type Config struct { EnableIptables bool EnableDNSController bool IptablesSyncPeriod int + IPFamily iptables.Protocol DNSSyncPeriod int CertDNSNames []string CertIPs []net.IP diff --git a/cmd/yurt-tunnel-server/app/options/options.go b/cmd/yurt-tunnel-server/app/options/options.go index 1dfb23603ea..eb654097094 100644 --- a/cmd/yurt-tunnel-server/app/options/options.go +++ b/cmd/yurt-tunnel-server/app/options/options.go @@ -25,11 +25,13 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/informers" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "sigs.k8s.io/apiserver-network-proxy/pkg/server" "github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/config" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/certmanager" + "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes" ) @@ -136,6 +138,11 @@ func (o *ServerOptions) Config() (*config.Config, error) { } } + if utilnet.IsIPv6String(o.BindAddr) { + cfg.IPFamily = iptables.ProtocolIpv6 + } else { + cfg.IPFamily = iptables.ProtocolIpv4 + } cfg.ListenAddrForAgent = net.JoinHostPort(o.BindAddr, o.TunnelAgentConnectPort) cfg.ListenAddrForMaster = net.JoinHostPort(o.BindAddr, o.SecurePort) cfg.ListenInsecureAddrForMaster = net.JoinHostPort(o.InsecureBindAddr, o.InsecurePort) diff --git a/cmd/yurt-tunnel-server/app/start.go b/cmd/yurt-tunnel-server/app/start.go index bc4fb7b43f2..39ff885c4f1 100644 --- a/cmd/yurt-tunnel-server/app/start.go +++ b/cmd/yurt-tunnel-server/app/start.go @@ -93,11 +93,12 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { } // 1. start the IP table manager if cfg.EnableIptables { - iptablesMgr := iptables.NewIptablesManager(cfg.Client, + iptablesMgr := iptables.NewIptablesManagerWithIPFamily(cfg.Client, cfg.SharedInformerFactory.Core().V1().Nodes(), cfg.ListenAddrForMaster, cfg.ListenInsecureAddrForMaster, - cfg.IptablesSyncPeriod) + cfg.IptablesSyncPeriod, + cfg.IPFamily) if iptablesMgr == nil { return fmt.Errorf("fail to create a new IptableManager") } diff --git a/pkg/util/certmanager/certmanager.go b/pkg/util/certmanager/certmanager.go index b230da613d0..4b179025618 100644 --- a/pkg/util/certmanager/certmanager.go +++ b/pkg/util/certmanager/certmanager.go @@ -64,7 +64,7 @@ func NewYurttunnelServerCertManager( ) // the ips and dnsNames should be acquired through api-server at the first time, because the informer factory has not started yet. - _ = wait.PollUntil(5*time.Second, func() (bool, error) { + werr := wait.PollUntil(5*time.Second, func() (bool, error) { dnsNames, ips, err = serveraddr.GetYurttunelServerDNSandIP(clientset) if err != nil { klog.Errorf("failed to get yurt tunnel server dns and ip, %v", err) @@ -89,7 +89,10 @@ func NewYurttunnelServerCertManager( return true, nil }, stopCh) - // add user specified DNS names and IP addresses + if werr != nil { + return nil, werr + } + // add user specified DNS anems and IP addresses dnsNames = append(dnsNames, clCertNames...) ips = append(ips, clIPs...) klog.Infof("subject of tunnel server certificate, ips=%#+v, dnsNames=%#+v", ips, dnsNames) diff --git a/pkg/yurthub/network/iptables.go b/pkg/yurthub/network/iptables.go index 379c69ffbc6..9f00b1162bd 100644 --- a/pkg/yurthub/network/iptables.go +++ b/pkg/yurthub/network/iptables.go @@ -22,6 +22,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/klog/v2" "k8s.io/utils/exec" + utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/util/iptables" ) @@ -40,6 +41,9 @@ type IptablesManager struct { func NewIptablesManager(dummyIfIP, dummyIfPort string) *IptablesManager { protocol := iptables.ProtocolIpv4 + if utilnet.IsIPv6String(dummyIfIP) { + protocol = iptables.ProtocolIpv6 + } execer := exec.New() iptInterface := iptables.New(execer, protocol) @@ -52,6 +56,10 @@ func NewIptablesManager(dummyIfIP, dummyIfPort string) *IptablesManager { } func makeupIptablesRules(ifIP, ifPort string) []iptablesRule { + loopbackAddr := "127.0.0.1" + if utilnet.IsIPv6String(ifIP) { + loopbackAddr = "::1" + } return []iptablesRule{ // skip connection track for traffic from container to 169.254.2.1:10261 {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", ifIP, "-j", "NOTRACK"}}, @@ -63,16 +71,16 @@ func makeupIptablesRules(ifIP, ifPort string) []iptablesRule { {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "NOTRACK"}}, // accept traffic from 169.254.2.1:10261 {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "ACCEPT"}}, - // skip connection track for traffic from container to 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, - // skip connection track for traffic from host network to 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, - // accept traffic to 127.0.0.1:10261 - {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "ACCEPT"}}, - // skip connection track for traffic from 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "NOTRACK"}}, - // accept traffic from 127.0.0.1:10261 - {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "ACCEPT"}}, + // skip connection track for traffic from container to localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", loopbackAddr, "-j", "NOTRACK"}}, + // skip connection track for traffic from host network to localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", loopbackAddr, "-j", "NOTRACK"}}, + // accept traffic to localhost:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "--dport", ifPort, "--destination", loopbackAddr, "-j", "ACCEPT"}}, + // skip connection track for traffic from localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", loopbackAddr, "-j", "NOTRACK"}}, + // accept traffic from localhost:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", loopbackAddr, "-j", "ACCEPT"}}, } } diff --git a/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go b/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go index 7b72b5edc22..09b96b0d4bd 100644 --- a/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go +++ b/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go @@ -99,7 +99,15 @@ func (plm *localHostProxyMiddleware) WrapHandler(handler http.Handler) http.Hand req.Header.Set(constants.ProxyHostHeaderKey, nodeName) } - proxyDest = fmt.Sprintf("127.0.0.1:%s", port) + loopbackAddr := "127.0.0.1" + isIPv6, err := util.IsIPv6Request(req) + if err != nil { + klog.Errorf("failed to parse remote ip: %v", err) + } else if isIPv6 { + loopbackAddr = "::1" + } + + proxyDest = net.JoinHostPort(loopbackAddr, port) oldHost := req.URL.Host req.Host = proxyDest req.Header.Set("Host", proxyDest) diff --git a/pkg/yurttunnel/server/interceptor.go b/pkg/yurttunnel/server/interceptor.go index b5087f587c0..b2c3e5e40b0 100644 --- a/pkg/yurttunnel/server/interceptor.go +++ b/pkg/yurttunnel/server/interceptor.go @@ -34,6 +34,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" + "github.com/openyurtio/openyurt/pkg/yurttunnel/util" ) var ( @@ -64,7 +65,7 @@ func putBufioReader(br *bufio.Reader) { // prometheus and metric server, setup proxy tunnel to kubelet, sends requests // through the tunnel and sends responses back to the master type RequestInterceptor struct { - contextDialer func(addr string, header http.Header, isTLS bool) (net.Conn, error) + contextDialer func(addr string, header http.Header, isTLS, isIPv6 bool) (net.Conn, error) } // NewRequestInterceptor creates a interceptor object that intercept request from kube-apiserver @@ -74,7 +75,7 @@ func NewRequestInterceptor(udsSockFile string, cfg *tls.Config) *RequestIntercep } cfg.InsecureSkipVerify = true - contextDialer := func(addr string, header http.Header, isTLS bool) (net.Conn, error) { + contextDialer := func(addr string, header http.Header, isTLS, isIPv6 bool) (net.Conn, error) { klog.V(4).Infof("Sending request to %q.", addr) proxyConn, err := net.Dial("unix", udsSockFile) if err != nil { @@ -88,7 +89,12 @@ func NewRequestInterceptor(udsSockFile string, cfg *tls.Config) *RequestIntercep } } - fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s%s\r\n\r\n", addr, "127.0.0.1", connectHeaders) + loopbackAddr := "127.0.0.1" + if isIPv6 { + loopbackAddr = "::1" + } + + fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s%s\r\n\r\n", addr, loopbackAddr, connectHeaders) br := newBufioReader(proxyConn) defer putBufioReader(br) res, err := http.ReadResponse(br, nil) @@ -140,8 +146,12 @@ func klogAndHTTPError(w http.ResponseWriter, errCode int, format string, i ...in // ServeHTTP will proxy the request to the tunnel and return response from tunnel back // to the client func (ri *RequestInterceptor) ServeHTTP(w http.ResponseWriter, r *http.Request) { + isIPv6, err := util.IsIPv6Request(r) + if err != nil { + klog.Errorf("failed to check if request from IPv6 client: %v", err) + } // 1. setup the tunnel - tunnelConn, err := ri.contextDialer(r.Host, r.Header, r.TLS != nil) + tunnelConn, err := ri.contextDialer(r.Host, r.Header, r.TLS != nil, isIPv6) if err != nil { klogAndHTTPError(w, http.StatusServiceUnavailable, "fail to setup the tunnel: %s", err) diff --git a/pkg/yurttunnel/server/serveraddr/addr.go b/pkg/yurttunnel/server/serveraddr/addr.go index bfc2eca94c7..3df07bff3e6 100644 --- a/pkg/yurttunnel/server/serveraddr/addr.go +++ b/pkg/yurttunnel/server/serveraddr/addr.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "net" + "strconv" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,7 +58,7 @@ func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) { for _, tmpIP := range ips { // we use the first non-loopback IP address. - if tmpIP.String() != "127.0.0.1" { + if s := tmpIP.String(); s != "127.0.0.1" && s != "::1" { ip = tmpIP break } @@ -86,7 +87,7 @@ func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) { return "", errors.New("fail to get the port number") } - return fmt.Sprintf("%s:%d", host, tcpPort), nil + return net.JoinHostPort(host, strconv.Itoa(int(tcpPort))), nil } // GetYurttunelServerDNSandIP gets DNS names and IPS for generating tunnel server certificate. diff --git a/pkg/yurttunnel/trafficforward/iptables/iptables.go b/pkg/yurttunnel/trafficforward/iptables/iptables.go index 23230fc1287..f6093791e60 100644 --- a/pkg/yurttunnel/trafficforward/iptables/iptables.go +++ b/pkg/yurttunnel/trafficforward/iptables/iptables.go @@ -40,7 +40,6 @@ import ( ) const ( - loopbackAddr = "127.0.0.1" reqReturnComment = "return request to access node directly" dnatToTunnelComment = "dnat to tunnel for access node" yurttunnelServerPortChain = "TUNNEL-PORT" @@ -84,6 +83,7 @@ type iptablesManager struct { nodeInformer coreinformer.NodeInformer iptables iptables.Interface execer exec.Interface + loopbackAddr string conntrackPath string secureDnatDest string insecureDnatDest string @@ -92,18 +92,18 @@ type iptablesManager struct { syncPeriod int } -// NewIptablesManager creates an IptablesManager; deletes old chains, if any; +// NewIptablesManagerWithIPFamily creates an IptablesManager; deletes old chains, if any; // generates new dnat rules based on IPs of current active nodes; and // appends the rules to the iptable. -func NewIptablesManager(client clientset.Interface, +func NewIptablesManagerWithIPFamily(client clientset.Interface, nodeInformer coreinformer.NodeInformer, listenAddr string, listenInsecureAddr string, - syncPeriod int) IptablesManager { + syncPeriod int, + ipFamily iptables.Protocol) IptablesManager { - protocol := iptables.ProtocolIpv4 execer := exec.New() - iptInterface := iptables.New(execer, protocol) + iptInterface := iptables.New(execer, ipFamily) if syncPeriod < defaultSyncPeriod { syncPeriod = defaultSyncPeriod @@ -121,6 +121,12 @@ func NewIptablesManager(client clientset.Interface, syncPeriod: syncPeriod, } + if ipFamily == iptables.ProtocolIpv6 { + im.loopbackAddr = "::1" + } else { + im.loopbackAddr = "127.0.0.1" + } + // conntrack setting conntrackPath, err := im.execer.LookPath("conntrack") if err != nil { @@ -135,6 +141,15 @@ func NewIptablesManager(client clientset.Interface, return im } +// NewIptablesManager creates an IptablesManager with ipv4 protocol +func NewIptablesManager(client clientset.Interface, + nodeInformer coreinformer.NodeInformer, + listenAddr string, + listenInsecureAddr string, + syncPeriod int) IptablesManager { + return NewIptablesManagerWithIPFamily(client, nodeInformer, listenAddr, listenInsecureAddr, syncPeriod, iptables.ProtocolIpv4) +} + // Run starts the iptablesManager that will updates dnat rules periodically func (im *iptablesManager) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) { defer wg.Done() @@ -361,7 +376,7 @@ func (im *iptablesManager) ensurePortIptables(port string, currentIPs, deletedIP iptables.Prepend, iptables.TableNAT, portChain, reqReturnPortIptablesArgs...) if err != nil { - klog.Errorf("could not ensure -j RETURN iptables rule for %s:%s: %v", ip, port, err) + klog.Errorf("could not ensure -j RETURN iptables rule for %s: %v", net.JoinHostPort(ip, port), err) return err } } @@ -382,7 +397,7 @@ func (im *iptablesManager) ensurePortIptables(port string, currentIPs, deletedIP err = im.iptables.DeleteRule(iptables.TableNAT, portChain, deletedIPIptablesArgs...) if err != nil { - klog.Errorf("could not delete old iptables rules for %s:%s: %v", ip, port, err) + klog.Errorf("could not delete old iptables rules for %s: %v", net.JoinHostPort(ip, port), err) return err } } @@ -509,7 +524,7 @@ func (im *iptablesManager) syncIptableSetting() { // check if there are new nodes nodesIP := im.getIPOfNodesWithoutAgent() nodesChanged, addedNodesIP, deletedNodesIP := im.getAddedAndDeletedNodes(nodesIP) - currentNodesIP := append(nodesIP, loopbackAddr) + currentNodesIP := append(nodesIP, im.loopbackAddr) // update the iptables setting if necessary err = im.ensurePortsIptables(currentDnatPorts, deletedDnatPorts, currentNodesIP, deletedNodesIP, portMappings) diff --git a/pkg/yurttunnel/util/util.go b/pkg/yurttunnel/util/util.go index 0810247e907..ef6638353c1 100644 --- a/pkg/yurttunnel/util/util.go +++ b/pkg/yurttunnel/util/util.go @@ -31,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/profile" "github.com/openyurtio/openyurt/pkg/projectinfo" @@ -176,3 +177,12 @@ func resolvePorts(portsStr, insecurePort string) []string { return ports } + +// IsIPv6Request returns if request send by an IPv6 client. +func IsIPv6Request(req *http.Request) (bool, error) { + remoteIP, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + return false, err + } + return utilnet.IsIPv6String(remoteIP), nil +}