diff --git a/cmd/yurt-tunnel-agent/app/options/options.go b/cmd/yurt-tunnel-agent/app/options/options.go index 9231a67ebc9..6541c5f382b 100644 --- a/cmd/yurt-tunnel-agent/app/options/options.go +++ b/cmd/yurt-tunnel-agent/app/options/options.go @@ -25,10 +25,12 @@ import ( "github.com/spf13/pflag" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "sigs.k8s.io/apiserver-network-proxy/pkg/agent" "github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app/config" "github.com/openyurtio/openyurt/pkg/projectinfo" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes" ) @@ -52,7 +54,6 @@ type AgentOptions struct { // NewAgentOptions creates a new AgentOptions with a default config. func NewAgentOptions() *AgentOptions { o := &AgentOptions{ - MetaHost: "127.0.0.1", MetaPort: constants.YurttunnelAgentMetaPort, } @@ -75,6 +76,10 @@ func (o *AgentOptions) Validate() error { } } + if o.MetaHost == "" { + o.MetaHost = utilip.MustGetLoopbackIP(utilnet.IsIPv6String(o.NodeIP)) + } + if !agentIdentifiersAreValid(o.AgentIdentifiers) { return errors.New("--agent-identifiers are invalid, format should be host={node-name}") } @@ -134,7 +139,11 @@ func (o *AgentOptions) Config() (*config.Config, error) { } if len(c.AgentIdentifiers) == 0 { - c.AgentIdentifiers = fmt.Sprintf("ipv4=%s&host=%s", o.NodeIP, o.NodeName) + ipFamily := "ipv4" + if utilnet.IsIPv6String(o.NodeIP) { + ipFamily = "ipv6" + } + c.AgentIdentifiers = fmt.Sprintf("%s=%s&host=%s", ipFamily, o.NodeIP, o.NodeName) } klog.Infof("%s is set for agent identifies", c.AgentIdentifiers) diff --git a/cmd/yurt-tunnel-server/app/config/config.go b/cmd/yurt-tunnel-server/app/config/config.go index 84e7e2f298e..73d5b03c977 100644 --- a/cmd/yurt-tunnel-server/app/config/config.go +++ b/cmd/yurt-tunnel-server/app/config/config.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -34,6 +35,7 @@ type Config struct { EnableIptables bool EnableDNSController bool IptablesSyncPeriod int + IPFamily iptables.Protocol DNSSyncPeriod int CertDNSNames []string CertIPs []net.IP @@ -72,3 +74,7 @@ func (c *Config) Complete() *CompletedConfig { } return &CompletedConfig{&cc} } + +func (c *Config) IsIPv6() bool { + return c.IPFamily == iptables.ProtocolIpv6 +} diff --git a/cmd/yurt-tunnel-server/app/options/options.go b/cmd/yurt-tunnel-server/app/options/options.go index 1dfb23603ea..1833b576770 100644 --- a/cmd/yurt-tunnel-server/app/options/options.go +++ b/cmd/yurt-tunnel-server/app/options/options.go @@ -25,11 +25,14 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/informers" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "sigs.k8s.io/apiserver-network-proxy/pkg/server" "github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/config" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/certmanager" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" + "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes" ) @@ -60,7 +63,6 @@ type ServerOptions struct { func NewServerOptions() *ServerOptions { o := &ServerOptions{ BindAddr: "0.0.0.0", - InsecureBindAddr: "127.0.0.1", EnableIptables: true, EnableDNSController: true, IptablesSyncPeriod: 60, @@ -81,6 +83,9 @@ func (o *ServerOptions) Validate() error { return fmt.Errorf("%s's bind address can't be empty", projectinfo.GetServerName()) } + if len(o.InsecureBindAddr) == 0 { + o.InsecureBindAddr = utilip.MustGetLoopbackIP(utilnet.IsIPv6String(o.BindAddr)) + } return nil } @@ -136,6 +141,11 @@ func (o *ServerOptions) Config() (*config.Config, error) { } } + if utilnet.IsIPv6String(o.BindAddr) { + cfg.IPFamily = iptables.ProtocolIpv6 + } else { + cfg.IPFamily = iptables.ProtocolIpv4 + } cfg.ListenAddrForAgent = net.JoinHostPort(o.BindAddr, o.TunnelAgentConnectPort) cfg.ListenAddrForMaster = net.JoinHostPort(o.BindAddr, o.SecurePort) cfg.ListenInsecureAddrForMaster = net.JoinHostPort(o.InsecureBindAddr, o.InsecurePort) diff --git a/cmd/yurt-tunnel-server/app/start.go b/cmd/yurt-tunnel-server/app/start.go index e5169f0becd..01947a2bdcd 100644 --- a/cmd/yurt-tunnel-server/app/start.go +++ b/cmd/yurt-tunnel-server/app/start.go @@ -93,11 +93,12 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { } // 1. start the IP table manager if cfg.EnableIptables { - iptablesMgr := iptables.NewIptablesManager(cfg.Client, + iptablesMgr := iptables.NewIptablesManagerWithIPFamily(cfg.Client, cfg.SharedInformerFactory.Core().V1().Nodes(), cfg.ListenAddrForMaster, cfg.ListenInsecureAddrForMaster, - cfg.IptablesSyncPeriod) + cfg.IptablesSyncPeriod, + cfg.IPFamily) if iptablesMgr == nil { return fmt.Errorf("fail to create a new IptableManager") } @@ -121,7 +122,7 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { // 4. create handler wrappers mInitializer := initializer.NewMiddlewareInitializer(cfg.SharedInformerFactory) - wrappers, err := wraphandler.InitHandlerWrappers(mInitializer) + wrappers, err := wraphandler.InitHandlerWrappers(mInitializer, cfg.IsIPv6()) if err != nil { klog.Errorf("failed to init handler wrappers, %v", err) return err diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 82c3a13c447..76f2cca31ce 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -91,6 +91,7 @@ type YurtHubConfiguration struct { WorkingMode util.WorkingMode KubeletHealthGracePeriod time.Duration FilterManager *filter.Manager + CertIPs []net.IP } // Complete converts *options.YurtHubOptions to *YurtHubConfiguration @@ -135,6 +136,12 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { return nil, err } + // use dummy ip and bind ip as cert IP SANs + certIPs := []net.IP{ + net.ParseIP(options.HubAgentDummyIfIP), + net.ParseIP(options.YurtHubHost), + } + cfg := &YurtHubConfiguration{ LBMode: options.LBMode, RemoteServers: us, @@ -167,6 +174,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { YurtSharedFactory: yurtSharedFactory, KubeletHealthGracePeriod: options.KubeletHealthGracePeriod, FilterManager: filterManager, + CertIPs: certIPs, } return cfg, nil diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index cee8d1d10a6..5ec07e4cc82 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -23,6 +23,8 @@ import ( "time" "github.com/spf13/pflag" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" @@ -30,8 +32,10 @@ import ( ) const ( - DummyIfCIDR = "169.254.0.0/16" - ExclusiveCIDR = "169.254.31.0/24" + DefaultDummyIfIP4 = "169.254.2.1" + DefaultDummyIfIP6 = "fd00::2:1" + DummyIfCIDR4 = "169.254.0.0/16" + ExclusiveCIDR = "169.254.31.0/24" ) // YurtHubOptions is the main settings for the yurthub @@ -90,7 +94,6 @@ func NewYurtHubOptions() *YurtHubOptions { EnableProfiling: true, EnableDummyIf: true, EnableIptables: true, - HubAgentDummyIfIP: "169.254.2.1", HubAgentDummyIfName: fmt.Sprintf("%s-dummy0", projectinfo.GetHubName()), DiskCachePath: disk.CacheBaseDir, AccessServerThroughHub: true, @@ -103,8 +106,8 @@ func NewYurtHubOptions() *YurtHubOptions { return o } -// ValidateOptions validates YurtHubOptions -func ValidateOptions(options *YurtHubOptions) error { +// Validate validates YurtHubOptions +func (options *YurtHubOptions) Validate() error { if len(options.NodeName) == 0 { return fmt.Errorf("node name is empty") } @@ -125,7 +128,7 @@ func ValidateOptions(options *YurtHubOptions) error { return fmt.Errorf("working mode %s is not supported", options.WorkingMode) } - if err := verifyDummyIP(options.HubAgentDummyIfIP); err != nil { + if err := options.verifyDummyIP(); err != nil { return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err) } @@ -168,21 +171,35 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.EnableNodePool, "enable-node-pool", o.EnableNodePool, "enable list/watch nodepools resource or not for filters(only used for testing)") } -// verifyDummyIP verify the specified ip is valid or not -func verifyDummyIP(dummyIP string) error { - //169.254.2.1/32 +// verifyDummyIP verify the specified ip is valid or not and set the default ip if empty +func (o *YurtHubOptions) verifyDummyIP() error { + if o.HubAgentDummyIfIP == "" { + if utilnet.IsIPv6String(o.YurtHubHost) { + o.HubAgentDummyIfIP = DefaultDummyIfIP6 + } else { + o.HubAgentDummyIfIP = DefaultDummyIfIP4 + } + klog.Infof("dummy ip not set, will use %s as default", o.HubAgentDummyIfIP) + return nil + } + + dummyIP := o.HubAgentDummyIfIP dip := net.ParseIP(dummyIP) if dip == nil { return fmt.Errorf("dummy ip %s is invalid", dummyIP) } - _, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR) + if utilnet.IsIPv6(dip) { + return nil + } + + _, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR4) if err != nil { - return fmt.Errorf("cidr(%s) is invalid, %w", DummyIfCIDR, err) + return fmt.Errorf("cidr(%s) is invalid, %w", DummyIfCIDR4, err) } if !dummyIfIPNet.Contains(dip) { - return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR) + return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR4) } _, exclusiveIPNet, err := net.ParseCIDR(ExclusiveCIDR) diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 23c4838afd7..9245a7b7635 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -59,7 +59,7 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command { cmd.Flags().VisitAll(func(flag *pflag.Flag) { klog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value) }) - if err := options.ValidateOptions(yurtHubOptions); err != nil { + if err := yurtHubOptions.Validate(); err != nil { klog.Fatalf("validate options: %v", err) } @@ -125,7 +125,8 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { trace++ klog.Infof("%d. create tls config for secure servers ", trace) - cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.YurtHubProxyServerSecureDummyAddr, stopCh) + cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig( + restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.CertIPs, stopCh) if err != nil { return fmt.Errorf("could not create tls config, %w", err) } diff --git a/pkg/util/certmanager/certmanager.go b/pkg/util/certmanager/certmanager.go index 054c2e24a35..acc377ef921 100644 --- a/pkg/util/certmanager/certmanager.go +++ b/pkg/util/certmanager/certmanager.go @@ -38,6 +38,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/certmanager/store" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" "github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr" ) @@ -69,7 +70,7 @@ func NewYurttunnelServerCertManager( ) // the ips and dnsNames should be acquired through api-server at the first time, because the informer factory has not started yet. - _ = wait.PollUntil(5*time.Second, func() (bool, error) { + werr := wait.PollUntil(5*time.Second, func() (bool, error) { dnsNames, ips, err = serveraddr.GetYurttunelServerDNSandIP(clientset) if err != nil { klog.Errorf("failed to get yurt tunnel server dns and ip, %v", err) @@ -94,10 +95,14 @@ func NewYurttunnelServerCertManager( return true, nil }, stopCh) - // add user specified DNS names and IP addresses + if werr != nil { + return nil, werr + } + + // add user specified DNS anems and IP addresses dnsNames = append(dnsNames, clCertNames...) ips = append(ips, clIPs...) - klog.Infof("subject of tunnel server certificate, ips=%#+v, dnsNames=%#+v", ips, dnsNames) + klog.Infof("subject of tunnel server certificate, ips=%s, dnsNames=%#+v", utilip.JoinIPStrings(ips), dnsNames) // the dynamic ip acquire func getIPs := func() ([]net.IP, error) { @@ -180,13 +185,8 @@ func NewYurttunnelAgentCertManager( func NewYurtHubServerCertManager( clientset kubernetes.Interface, certDir, - nodeName, - proxyServerSecureDummyAddr string) (certificate.Manager, error) { - - host, _, err := net.SplitHostPort(proxyServerSecureDummyAddr) - if err != nil { - return nil, err - } + nodeName string, + certIPs []net.IP) (certificate.Manager, error) { return newCertManager( clientset, @@ -201,7 +201,7 @@ func NewYurtHubServerCertManager( certificatesv1.UsageDigitalSignature, certificatesv1.UsageServerAuth, }, - []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(host)}, + certIPs, nil) } diff --git a/pkg/util/ip/ip.go b/pkg/util/ip/ip.go new file mode 100644 index 00000000000..2d1c9b239a5 --- /dev/null +++ b/pkg/util/ip/ip.go @@ -0,0 +1,68 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ip + +import ( + "net" + "strings" + + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" +) + +const ( + DefaultLoopbackIP4 = "127.0.0.1" + DefaultLoopbackIP6 = "::1" +) + +// MustGetLoopbackIP is a wrapper for GetLoopbackIP. If any error occurs or loopback interface is not found, +// will fall back to 127.0.0.1 for ipv4 or ::1 for ipv6. +func MustGetLoopbackIP(wantIPv6 bool) string { + ip, err := GetLoopbackIP(wantIPv6) + if err != nil { + klog.Errorf("failed to get loopback addr: %v", err) + } + if ip != "" { + return ip + } + if wantIPv6 { + return DefaultLoopbackIP6 + } + return DefaultLoopbackIP4 +} + +// GetLoopbackIP returns the ip address of local loopback interface. +func GetLoopbackIP(wantIPv6 bool) (string, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", err + } + for _, address := range addrs { + if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsLoopback() && wantIPv6 == utilnet.IsIPv6(ipnet.IP) { + return ipnet.IP.String(), nil + } + } + return "", nil +} + +func JoinIPStrings(ips []net.IP) string { + var strs []string + for _, ip := range ips { + strs = append(strs, ip.String()) + } + return strings.Join(strs, ",") +} diff --git a/pkg/util/ip/ip_test.go b/pkg/util/ip/ip_test.go new file mode 100644 index 00000000000..efcef4195d3 --- /dev/null +++ b/pkg/util/ip/ip_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ip + +import ( + "testing" +) + +func TestGetLoopbackIP(t *testing.T) { + lo4, err := GetLoopbackIP(false) + if err != nil { + t.Errorf("failed to get ipv4 loopback address: %v", err) + } + t.Logf("got ipv4 loopback address: %s", lo4) + if lo4 != "127.0.0.1" { + t.Errorf("got ipv4 loopback addr: '%s', expect: '127.0.0.1'", lo4) + } + + lo6, err := GetLoopbackIP(true) + if err != nil { + t.Errorf("failed to get ipv6 loopback address: %v", err) + } + if lo6 != "" { + // dual stack env + t.Logf("got ipv6 loopback address: %s", lo6) + if lo6 != "::1" { + t.Errorf("got ipv6 loopback addr: '%s', expect: '::1'", lo6) + } + } +} diff --git a/pkg/yurthub/network/iptables.go b/pkg/yurthub/network/iptables.go index 379c69ffbc6..0c5a8519125 100644 --- a/pkg/yurthub/network/iptables.go +++ b/pkg/yurthub/network/iptables.go @@ -22,6 +22,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/klog/v2" "k8s.io/utils/exec" + utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/util/iptables" ) @@ -40,6 +41,9 @@ type IptablesManager struct { func NewIptablesManager(dummyIfIP, dummyIfPort string) *IptablesManager { protocol := iptables.ProtocolIpv4 + if utilnet.IsIPv6String(dummyIfIP) { + protocol = iptables.ProtocolIpv6 + } execer := exec.New() iptInterface := iptables.New(execer, protocol) @@ -63,16 +67,16 @@ func makeupIptablesRules(ifIP, ifPort string) []iptablesRule { {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "NOTRACK"}}, // accept traffic from 169.254.2.1:10261 {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "ACCEPT"}}, - // skip connection track for traffic from container to 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, - // skip connection track for traffic from host network to 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, - // accept traffic to 127.0.0.1:10261 - {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "ACCEPT"}}, - // skip connection track for traffic from 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "NOTRACK"}}, - // accept traffic from 127.0.0.1:10261 - {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "ACCEPT"}}, + // skip connection track for traffic from container to localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", "localhost", "-j", "NOTRACK"}}, + // skip connection track for traffic from host network to localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "localhost", "-j", "NOTRACK"}}, + // accept traffic to localhost:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "localhost", "-j", "ACCEPT"}}, + // skip connection track for traffic from localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "localhost", "-j", "NOTRACK"}}, + // accept traffic from localhost:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "localhost", "-j", "ACCEPT"}}, } } diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 0c7038e2bad..9c2bb62cd3a 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -169,7 +169,12 @@ func healthz(w http.ResponseWriter, _ *http.Request) { } // GenUseCertMgrAndTLSConfig create a certificate manager for the yurthub server and generate a TLS configuration -func GenUseCertMgrAndTLSConfig(restConfigMgr *rest.RestConfigManager, certificateMgr interfaces.YurtCertificateManager, certDir, nodeName, proxyServerSecureDummyAddr string, stopCh <-chan struct{}) (*tls.Config, error) { +func GenUseCertMgrAndTLSConfig( + restConfigMgr *rest.RestConfigManager, + certificateMgr interfaces.YurtCertificateManager, + certDir, nodeName string, + certIPs []net.IP, + stopCh <-chan struct{}) (*tls.Config, error) { cfg := restConfigMgr.GetRestConfig(false) if cfg == nil { return nil, fmt.Errorf("failed to prepare rest config based ong hub agent client certificate") @@ -180,7 +185,7 @@ func GenUseCertMgrAndTLSConfig(restConfigMgr *rest.RestConfigManager, certificat return nil, err } // create a certificate manager for the yurthub server and run the csr approver for both yurthub - serverCertMgr, err := certmanager.NewYurtHubServerCertManager(clientSet, certDir, nodeName, proxyServerSecureDummyAddr) + serverCertMgr, err := certmanager.NewYurtHubServerCertManager(clientSet, certDir, nodeName, certIPs) if err != nil { return nil, err } diff --git a/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go b/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go index 7b72b5edc22..7897c0b87a3 100644 --- a/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go +++ b/pkg/yurttunnel/handlerwrapper/localhostproxy/handler.go @@ -35,6 +35,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" hw "github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper" "github.com/openyurtio/openyurt/pkg/yurttunnel/util" @@ -47,11 +48,13 @@ type localHostProxyMiddleware struct { localhostPorts map[string]struct{} nodeInformerSynced cache.InformerSynced cmInformerSynced cache.InformerSynced + loopbackAddr string } -func NewLocalHostProxyMiddleware() hw.Middleware { +func NewLocalHostProxyMiddleware(isIPv6 bool) hw.Middleware { return &localHostProxyMiddleware{ localhostPorts: make(map[string]struct{}), + loopbackAddr: utilip.MustGetLoopbackIP(isIPv6), } } @@ -99,7 +102,7 @@ func (plm *localHostProxyMiddleware) WrapHandler(handler http.Handler) http.Hand req.Header.Set(constants.ProxyHostHeaderKey, nodeName) } - proxyDest = fmt.Sprintf("127.0.0.1:%s", port) + proxyDest = net.JoinHostPort(plm.loopbackAddr, port) oldHost := req.URL.Host req.Host = proxyDest req.Header.Set("Host", proxyDest) diff --git a/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go b/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go index d90ab7ea00e..c5919642bd7 100644 --- a/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go +++ b/pkg/yurttunnel/handlerwrapper/wraphandler/wraphandler.go @@ -27,7 +27,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/tracerequest" ) -func InitHandlerWrappers(mi initializer.MiddlewareInitializer) (hw.HandlerWrappers, error) { +func InitHandlerWrappers(mi initializer.MiddlewareInitializer, isIPv6 bool) (hw.HandlerWrappers, error) { wrappers := make(hw.HandlerWrappers, 0) // register all of middleware here // @@ -42,7 +42,7 @@ func InitHandlerWrappers(mi initializer.MiddlewareInitializer) (hw.HandlerWrappe // // then the middleware m2 will be called before the mw1 wrappers = append(wrappers, tracerequest.NewTraceReqMiddleware()) - wrappers = append(wrappers, localhostproxy.NewLocalHostProxyMiddleware()) + wrappers = append(wrappers, localhostproxy.NewLocalHostProxyMiddleware(isIPv6)) // init all of wrappers for i := range wrappers { diff --git a/pkg/yurttunnel/server/interceptor.go b/pkg/yurttunnel/server/interceptor.go index b5087f587c0..3e4baf4841a 100644 --- a/pkg/yurttunnel/server/interceptor.go +++ b/pkg/yurttunnel/server/interceptor.go @@ -88,7 +88,7 @@ func NewRequestInterceptor(udsSockFile string, cfg *tls.Config) *RequestIntercep } } - fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s%s\r\n\r\n", addr, "127.0.0.1", connectHeaders) + fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: localhost%s\r\n\r\n", addr, connectHeaders) br := newBufioReader(proxyConn) defer putBufioReader(br) res, err := http.ReadResponse(br, nil) diff --git a/pkg/yurttunnel/server/serveraddr/addr.go b/pkg/yurttunnel/server/serveraddr/addr.go index bfc2eca94c7..9024b7b3887 100644 --- a/pkg/yurttunnel/server/serveraddr/addr.go +++ b/pkg/yurttunnel/server/serveraddr/addr.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "net" + "strconv" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,6 +31,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" + utilip "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -57,7 +59,7 @@ func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) { for _, tmpIP := range ips { // we use the first non-loopback IP address. - if tmpIP.String() != "127.0.0.1" { + if s := tmpIP.String(); s != utilip.DefaultLoopbackIP4 && s != utilip.DefaultLoopbackIP6 { ip = tmpIP break } @@ -86,7 +88,7 @@ func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) { return "", errors.New("fail to get the port number") } - return fmt.Sprintf("%s:%d", host, tcpPort), nil + return net.JoinHostPort(host, strconv.Itoa(int(tcpPort))), nil } // GetYurttunelServerDNSandIP gets DNS names and IPS for generating tunnel server certificate. @@ -231,7 +233,7 @@ func extractTunnelServerDNSandIPs(svc *corev1.Service, eps []*corev1.Endpoints, if svc.Spec.ClusterIP != "None" { ips = append(ips, net.ParseIP(svc.Spec.ClusterIP)) } - ips = append(ips, net.ParseIP("127.0.0.1")) + ips = append(ips, net.ParseIP(utilip.DefaultLoopbackIP4), net.ParseIP(utilip.DefaultLoopbackIP6)) // 3. extract dns and ip from the endpoint for _, eps := range eps { diff --git a/pkg/yurttunnel/server/serveraddr/addr_test.go b/pkg/yurttunnel/server/serveraddr/addr_test.go index b668ca70312..53cea5aa368 100644 --- a/pkg/yurttunnel/server/serveraddr/addr_test.go +++ b/pkg/yurttunnel/server/serveraddr/addr_test.go @@ -447,6 +447,7 @@ func TestExtractTunnelServerDNSandIPs(t *testing.T) { net.ParseIP("192.168.1.1"), net.ParseIP("10.10.102.1"), net.ParseIP("127.0.0.1"), + net.ParseIP("::1"), net.ParseIP("192.168.1.2"), net.ParseIP("192.168.1.3"), }, @@ -520,6 +521,7 @@ func TestExtractTunnelServerDNSandIPs(t *testing.T) { ips: []net.IP{ net.ParseIP("10.10.102.1"), net.ParseIP("127.0.0.1"), + net.ParseIP("::1"), net.ParseIP("192.168.1.2"), net.ParseIP("192.168.1.3"), }, @@ -612,6 +614,7 @@ func TestExtractTunnelServerDNSandIPs(t *testing.T) { net.ParseIP("192.168.1.5"), net.ParseIP("10.10.102.1"), net.ParseIP("127.0.0.1"), + net.ParseIP("::1"), net.ParseIP("192.168.1.2"), net.ParseIP("192.168.1.3"), }, diff --git a/pkg/yurttunnel/trafficforward/iptables/iptables.go b/pkg/yurttunnel/trafficforward/iptables/iptables.go index 23230fc1287..0fccfd2763e 100644 --- a/pkg/yurttunnel/trafficforward/iptables/iptables.go +++ b/pkg/yurttunnel/trafficforward/iptables/iptables.go @@ -34,13 +34,13 @@ import ( utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/util/iptables" "github.com/openyurtio/openyurt/pkg/yurttunnel/server/metrics" "github.com/openyurtio/openyurt/pkg/yurttunnel/util" ) const ( - loopbackAddr = "127.0.0.1" reqReturnComment = "return request to access node directly" dnatToTunnelComment = "dnat to tunnel for access node" yurttunnelServerPortChain = "TUNNEL-PORT" @@ -84,6 +84,7 @@ type iptablesManager struct { nodeInformer coreinformer.NodeInformer iptables iptables.Interface execer exec.Interface + loopbackAddr string conntrackPath string secureDnatDest string insecureDnatDest string @@ -92,18 +93,18 @@ type iptablesManager struct { syncPeriod int } -// NewIptablesManager creates an IptablesManager; deletes old chains, if any; +// NewIptablesManagerWithIPFamily creates an IptablesManager; deletes old chains, if any; // generates new dnat rules based on IPs of current active nodes; and // appends the rules to the iptable. -func NewIptablesManager(client clientset.Interface, +func NewIptablesManagerWithIPFamily(client clientset.Interface, nodeInformer coreinformer.NodeInformer, listenAddr string, listenInsecureAddr string, - syncPeriod int) IptablesManager { + syncPeriod int, + ipFamily iptables.Protocol) IptablesManager { - protocol := iptables.ProtocolIpv4 execer := exec.New() - iptInterface := iptables.New(execer, protocol) + iptInterface := iptables.New(execer, ipFamily) if syncPeriod < defaultSyncPeriod { syncPeriod = defaultSyncPeriod @@ -121,6 +122,8 @@ func NewIptablesManager(client clientset.Interface, syncPeriod: syncPeriod, } + im.loopbackAddr = ip.MustGetLoopbackIP(ipFamily == iptables.ProtocolIpv6) + // conntrack setting conntrackPath, err := im.execer.LookPath("conntrack") if err != nil { @@ -135,6 +138,15 @@ func NewIptablesManager(client clientset.Interface, return im } +// NewIptablesManager creates an IptablesManager with ipv4 protocol +func NewIptablesManager(client clientset.Interface, + nodeInformer coreinformer.NodeInformer, + listenAddr string, + listenInsecureAddr string, + syncPeriod int) IptablesManager { + return NewIptablesManagerWithIPFamily(client, nodeInformer, listenAddr, listenInsecureAddr, syncPeriod, iptables.ProtocolIpv4) +} + // Run starts the iptablesManager that will updates dnat rules periodically func (im *iptablesManager) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) { defer wg.Done() @@ -361,7 +373,7 @@ func (im *iptablesManager) ensurePortIptables(port string, currentIPs, deletedIP iptables.Prepend, iptables.TableNAT, portChain, reqReturnPortIptablesArgs...) if err != nil { - klog.Errorf("could not ensure -j RETURN iptables rule for %s:%s: %v", ip, port, err) + klog.Errorf("could not ensure -j RETURN iptables rule for %s: %v", net.JoinHostPort(ip, port), err) return err } } @@ -382,7 +394,7 @@ func (im *iptablesManager) ensurePortIptables(port string, currentIPs, deletedIP err = im.iptables.DeleteRule(iptables.TableNAT, portChain, deletedIPIptablesArgs...) if err != nil { - klog.Errorf("could not delete old iptables rules for %s:%s: %v", ip, port, err) + klog.Errorf("could not delete old iptables rules for %s: %v", net.JoinHostPort(ip, port), err) return err } } @@ -509,7 +521,7 @@ func (im *iptablesManager) syncIptableSetting() { // check if there are new nodes nodesIP := im.getIPOfNodesWithoutAgent() nodesChanged, addedNodesIP, deletedNodesIP := im.getAddedAndDeletedNodes(nodesIP) - currentNodesIP := append(nodesIP, loopbackAddr) + currentNodesIP := append(nodesIP, im.loopbackAddr) // update the iptables setting if necessary err = im.ensurePortsIptables(currentDnatPorts, deletedDnatPorts, currentNodesIP, deletedNodesIP, portMappings)