From 309bfd933be82e6a81790c34a6066f9ca6fe4536 Mon Sep 17 00:00:00 2001 From: tydra-wang Date: Wed, 25 May 2022 23:19:34 +0800 Subject: [PATCH] Feature: yurthub supports setting ipv6 dummy if ip --- cmd/yurthub/app/config/config.go | 8 ++++++ cmd/yurthub/app/options/options.go | 41 ++++++++++++++++++++--------- cmd/yurthub/app/start.go | 5 ++-- pkg/util/certmanager/certmanager.go | 11 +++----- pkg/util/ip/ip.go | 16 +++++++++++ pkg/util/ip/ip_test.go | 16 +++++++++++ pkg/yurthub/network/iptables.go | 24 ++++++++++------- pkg/yurthub/server/server.go | 9 +++++-- 8 files changed, 96 insertions(+), 34 deletions(-) diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 82c3a13c447..76f2cca31ce 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -91,6 +91,7 @@ type YurtHubConfiguration struct { WorkingMode util.WorkingMode KubeletHealthGracePeriod time.Duration FilterManager *filter.Manager + CertIPs []net.IP } // Complete converts *options.YurtHubOptions to *YurtHubConfiguration @@ -135,6 +136,12 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { return nil, err } + // use dummy ip and bind ip as cert IP SANs + certIPs := []net.IP{ + net.ParseIP(options.HubAgentDummyIfIP), + net.ParseIP(options.YurtHubHost), + } + cfg := &YurtHubConfiguration{ LBMode: options.LBMode, RemoteServers: us, @@ -167,6 +174,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { YurtSharedFactory: yurtSharedFactory, KubeletHealthGracePeriod: options.KubeletHealthGracePeriod, FilterManager: filterManager, + CertIPs: certIPs, } return cfg, nil diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index cee8d1d10a6..5ec07e4cc82 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -23,6 +23,8 @@ import ( "time" "github.com/spf13/pflag" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" @@ -30,8 +32,10 @@ import ( ) const ( - DummyIfCIDR = "169.254.0.0/16" - ExclusiveCIDR = "169.254.31.0/24" + DefaultDummyIfIP4 = "169.254.2.1" + DefaultDummyIfIP6 = "fd00::2:1" + DummyIfCIDR4 = "169.254.0.0/16" + ExclusiveCIDR = "169.254.31.0/24" ) // YurtHubOptions is the main settings for the yurthub @@ -90,7 +94,6 @@ func NewYurtHubOptions() *YurtHubOptions { EnableProfiling: true, EnableDummyIf: true, EnableIptables: true, - HubAgentDummyIfIP: "169.254.2.1", HubAgentDummyIfName: fmt.Sprintf("%s-dummy0", projectinfo.GetHubName()), DiskCachePath: disk.CacheBaseDir, AccessServerThroughHub: true, @@ -103,8 +106,8 @@ func NewYurtHubOptions() *YurtHubOptions { return o } -// ValidateOptions validates YurtHubOptions -func ValidateOptions(options *YurtHubOptions) error { +// Validate validates YurtHubOptions +func (options *YurtHubOptions) Validate() error { if len(options.NodeName) == 0 { return fmt.Errorf("node name is empty") } @@ -125,7 +128,7 @@ func ValidateOptions(options *YurtHubOptions) error { return fmt.Errorf("working mode %s is not supported", options.WorkingMode) } - if err := verifyDummyIP(options.HubAgentDummyIfIP); err != nil { + if err := options.verifyDummyIP(); err != nil { return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err) } @@ -168,21 +171,35 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.EnableNodePool, "enable-node-pool", o.EnableNodePool, "enable list/watch nodepools resource or not for filters(only used for testing)") } -// verifyDummyIP verify the specified ip is valid or not -func verifyDummyIP(dummyIP string) error { - //169.254.2.1/32 +// verifyDummyIP verify the specified ip is valid or not and set the default ip if empty +func (o *YurtHubOptions) verifyDummyIP() error { + if o.HubAgentDummyIfIP == "" { + if utilnet.IsIPv6String(o.YurtHubHost) { + o.HubAgentDummyIfIP = DefaultDummyIfIP6 + } else { + o.HubAgentDummyIfIP = DefaultDummyIfIP4 + } + klog.Infof("dummy ip not set, will use %s as default", o.HubAgentDummyIfIP) + return nil + } + + dummyIP := o.HubAgentDummyIfIP dip := net.ParseIP(dummyIP) if dip == nil { return fmt.Errorf("dummy ip %s is invalid", dummyIP) } - _, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR) + if utilnet.IsIPv6(dip) { + return nil + } + + _, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR4) if err != nil { - return fmt.Errorf("cidr(%s) is invalid, %w", DummyIfCIDR, err) + return fmt.Errorf("cidr(%s) is invalid, %w", DummyIfCIDR4, err) } if !dummyIfIPNet.Contains(dip) { - return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR) + return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR4) } _, exclusiveIPNet, err := net.ParseCIDR(ExclusiveCIDR) diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 23c4838afd7..9245a7b7635 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -59,7 +59,7 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command { cmd.Flags().VisitAll(func(flag *pflag.Flag) { klog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value) }) - if err := options.ValidateOptions(yurtHubOptions); err != nil { + if err := yurtHubOptions.Validate(); err != nil { klog.Fatalf("validate options: %v", err) } @@ -125,7 +125,8 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { trace++ klog.Infof("%d. create tls config for secure servers ", trace) - cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.YurtHubProxyServerSecureDummyAddr, stopCh) + cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig( + restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.CertIPs, stopCh) if err != nil { return fmt.Errorf("could not create tls config, %w", err) } diff --git a/pkg/util/certmanager/certmanager.go b/pkg/util/certmanager/certmanager.go index c3927ced59f..acc377ef921 100644 --- a/pkg/util/certmanager/certmanager.go +++ b/pkg/util/certmanager/certmanager.go @@ -185,13 +185,8 @@ func NewYurttunnelAgentCertManager( func NewYurtHubServerCertManager( clientset kubernetes.Interface, certDir, - nodeName, - proxyServerSecureDummyAddr string) (certificate.Manager, error) { - - host, _, err := net.SplitHostPort(proxyServerSecureDummyAddr) - if err != nil { - return nil, err - } + nodeName string, + certIPs []net.IP) (certificate.Manager, error) { return newCertManager( clientset, @@ -206,7 +201,7 @@ func NewYurtHubServerCertManager( certificatesv1.UsageDigitalSignature, certificatesv1.UsageServerAuth, }, - []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(host)}, + certIPs, nil) } diff --git a/pkg/util/ip/ip.go b/pkg/util/ip/ip.go index 2703c6e690b..2d1c9b239a5 100644 --- a/pkg/util/ip/ip.go +++ b/pkg/util/ip/ip.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package ip import ( diff --git a/pkg/util/ip/ip_test.go b/pkg/util/ip/ip_test.go index 527d6f4caf3..efcef4195d3 100644 --- a/pkg/util/ip/ip_test.go +++ b/pkg/util/ip/ip_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package ip import ( diff --git a/pkg/yurthub/network/iptables.go b/pkg/yurthub/network/iptables.go index 379c69ffbc6..0c5a8519125 100644 --- a/pkg/yurthub/network/iptables.go +++ b/pkg/yurthub/network/iptables.go @@ -22,6 +22,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/klog/v2" "k8s.io/utils/exec" + utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/util/iptables" ) @@ -40,6 +41,9 @@ type IptablesManager struct { func NewIptablesManager(dummyIfIP, dummyIfPort string) *IptablesManager { protocol := iptables.ProtocolIpv4 + if utilnet.IsIPv6String(dummyIfIP) { + protocol = iptables.ProtocolIpv6 + } execer := exec.New() iptInterface := iptables.New(execer, protocol) @@ -63,16 +67,16 @@ func makeupIptablesRules(ifIP, ifPort string) []iptablesRule { {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "NOTRACK"}}, // accept traffic from 169.254.2.1:10261 {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "ACCEPT"}}, - // skip connection track for traffic from container to 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, - // skip connection track for traffic from host network to 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, - // accept traffic to 127.0.0.1:10261 - {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "ACCEPT"}}, - // skip connection track for traffic from 127.0.0.1:10261 - {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "NOTRACK"}}, - // accept traffic from 127.0.0.1:10261 - {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "ACCEPT"}}, + // skip connection track for traffic from container to localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", "localhost", "-j", "NOTRACK"}}, + // skip connection track for traffic from host network to localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "localhost", "-j", "NOTRACK"}}, + // accept traffic to localhost:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "localhost", "-j", "ACCEPT"}}, + // skip connection track for traffic from localhost:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "localhost", "-j", "NOTRACK"}}, + // accept traffic from localhost:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "localhost", "-j", "ACCEPT"}}, } } diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 0c7038e2bad..9c2bb62cd3a 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -169,7 +169,12 @@ func healthz(w http.ResponseWriter, _ *http.Request) { } // GenUseCertMgrAndTLSConfig create a certificate manager for the yurthub server and generate a TLS configuration -func GenUseCertMgrAndTLSConfig(restConfigMgr *rest.RestConfigManager, certificateMgr interfaces.YurtCertificateManager, certDir, nodeName, proxyServerSecureDummyAddr string, stopCh <-chan struct{}) (*tls.Config, error) { +func GenUseCertMgrAndTLSConfig( + restConfigMgr *rest.RestConfigManager, + certificateMgr interfaces.YurtCertificateManager, + certDir, nodeName string, + certIPs []net.IP, + stopCh <-chan struct{}) (*tls.Config, error) { cfg := restConfigMgr.GetRestConfig(false) if cfg == nil { return nil, fmt.Errorf("failed to prepare rest config based ong hub agent client certificate") @@ -180,7 +185,7 @@ func GenUseCertMgrAndTLSConfig(restConfigMgr *rest.RestConfigManager, certificat return nil, err } // create a certificate manager for the yurthub server and run the csr approver for both yurthub - serverCertMgr, err := certmanager.NewYurtHubServerCertManager(clientSet, certDir, nodeName, proxyServerSecureDummyAddr) + serverCertMgr, err := certmanager.NewYurtHubServerCertManager(clientSet, certDir, nodeName, certIPs) if err != nil { return nil, err }