diff --git a/cmd/agent/app/config/config.go b/cmd/agent/app/config/config.go index d5310f3..b9b4a3b 100644 --- a/cmd/agent/app/config/config.go +++ b/cmd/agent/app/config/config.go @@ -38,6 +38,7 @@ type TunnelConfig struct { VPNDriver string VPNPort string RouteDriver string + MACPrefix string ForwardNodeIP bool NATTraversal bool KeepAliveInterval int diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index 3b09d91..19017e7 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -5,7 +5,9 @@ import ( "fmt" "net" "os" + "regexp" "strconv" + "strings" "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" @@ -35,6 +37,7 @@ const ( DefaultTunnelMetricsPort = 10265 DefaultProxyMetricsPort = 10266 DefaultHealthyProbeAddr = 10275 + DefaultMACPrefix = "aa:0f" ) // AgentOptions has the information that required by the raven agent @@ -52,6 +55,7 @@ type TunnelOptions struct { VPNDriver string VPNPort string RouteDriver string + MACPrefix string ForwardNodeIP bool NATTraversal bool KeepAliveInterval int @@ -77,6 +81,15 @@ func (o *AgentOptions) Validate() error { return errors.New("currently only supports libreswan and wireguard VPN drivers") } } + if o.MACPrefix != "" { + reg := regexp.MustCompile(`^[0-9a-fA-F]+$`) + strs := strings.Split(o.MACPrefix, ":") + for i := range strs { + if !reg.MatchString(strings.ToLower(strs[i])) { + return errors.New(fmt.Sprintf("Mac prefix %s is nonstandard", o.MACPrefix)) + } + } + } return nil } @@ -95,6 +108,7 @@ func (o *AgentOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.ForwardNodeIP, "forward-node-ip", o.ForwardNodeIP, `Forward node IP or not. (default "false")`) fs.IntVar(&o.KeepAliveInterval, "keep-alive-interval", o.KeepAliveInterval, `Interval for sending keepalive packets in the VPN tunnel, (default "0", closed)`) fs.IntVar(&o.KeepAliveTimeout, "keep-alive-timeout", o.KeepAliveTimeout, `Timeout for sending keepalive packets in the VPN tunnel, (default "0", closed)`) + fs.StringVar(&o.MACPrefix, "customized-mac-prefix", o.MACPrefix, `Customized MAC address prefix for vxlan link, (default "aa:0f")`) fs.StringVar(&o.ProxyMetricsAddress, "proxy-metric-bind-addr", o.ProxyMetricsAddress, `Binding address of proxy metrics. (default ":10266")`) fs.StringVar(&o.InternalSecureAddress, "proxy-internal-secure-addr", o.InternalSecureAddress, `Binding secure address of proxy server. (default ":10263")`) @@ -145,6 +159,7 @@ func (o *AgentOptions) Config() (*config.Config, error) { VPNPort: port, VPNDriver: o.VPNDriver, RouteDriver: o.RouteDriver, + MACPrefix: o.MACPrefix, ForwardNodeIP: o.ForwardNodeIP, NATTraversal: o.NATTraversal, KeepAliveInterval: o.KeepAliveInterval, @@ -171,6 +186,9 @@ func (o *AgentOptions) Config() (*config.Config, error) { if c.Tunnel.VPNPort == "" { c.Tunnel.VPNPort = vpndriver.DefaultVPNPort } + if c.Tunnel.MACPrefix == "" { + c.Tunnel.MACPrefix = DefaultMACPrefix + } if c.Proxy.ProxyClientCertDir == "" { c.Proxy.ProxyClientCertDir = utils.RavenProxyClientCertDir } diff --git a/pkg/networkengine/routedriver/vxlan/vxlan.go b/pkg/networkengine/routedriver/vxlan/vxlan.go index 2390c97..dbe5f60 100644 --- a/pkg/networkengine/routedriver/vxlan/vxlan.go +++ b/pkg/networkengine/routedriver/vxlan/vxlan.go @@ -22,6 +22,8 @@ import ( "math" "net" "os" + "strconv" + "strings" "syscall" "github.com/vdobler/ht/errorlist" @@ -67,6 +69,7 @@ func init() { type vxlan struct { vxlanIface netlink.Link nodeName types.NodeName + macPrefix string iptables iptablesutil.IPTablesInterface ipset ipsetutil.IPSetInterface @@ -137,7 +140,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error if vx.isGatewayRole(network) { desiredRoutes = vx.calRouteOnGateway(network) - desiredFDBs = vx.calFDBOnGateway(network) + desiredFDBs, err = vx.calFDBOnGateway(network) + if err != nil { + return fmt.Errorf("error calculate gateway fdb: %s", err) + } err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, nonGatewayChainRuleSpec) if err != nil { @@ -149,8 +155,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error } } else { desiredRoutes = vx.calRouteOnNonGateway(network) - desiredFDBs = vx.calFDBOnNonGateway(network) - + desiredFDBs, err = vx.calFDBOnNonGateway(network) + if err != nil { + return fmt.Errorf("error calculate non gateway fdb: %s", err) + } err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, gatewayChainRuleSpec) if err != nil { return fmt.Errorf("error deleting gateway chain rule: %s", err) @@ -218,7 +226,8 @@ func (vx *vxlan) nodeInfo(network *types.Network) *v1beta1.NodeInfo { func New(cfg *config.Config) (routedriver.Driver, error) { return &vxlan{ - nodeName: types.NodeName(cfg.NodeName), + nodeName: types.NodeName(cfg.NodeName), + macPrefix: cfg.Tunnel.MACPrefix, }, nil } @@ -274,13 +283,17 @@ func (vx *vxlan) ensureVxlanLink(network *types.Network, vpnDriverMTUFn func() ( VxlanId: vxlanID, Age: 300, Port: vxlanPort, - Learning: true, + Learning: false, } if !vx.isGatewayRole(network) { vxlanLink.Group = net.ParseIP(network.LocalEndpoint.PrivateIP) } localIP := net.ParseIP(vx.nodeInfo(network).PrivateIP) + vxlanLink.HardwareAddr, err = vx.ipAddrToHardwareAddr(localIP) + if err != nil { + return fmt.Errorf("error convert vxlan ip to mac address: %s", err) + } nl, err := ensureVxlanLink(vxlanLink, vxlanIP(localIP)) if err != nil { return fmt.Errorf("error ensuring vxlan link: %s", err) @@ -383,13 +396,17 @@ func (vx *vxlan) calRulesOnNode() map[string]*netlink.Rule { // calFDBOnGateway calculates and returns the desired FDB entries on gateway node. // The FDB entries format are equivalent to the following `bridge fdb append` command: // -// bridge fdb append 00:00:00:00:00:00 dev raven0 dst {non_gateway_nodeN_private_ip} self permanent -func (vx *vxlan) calFDBOnGateway(network *types.Network) map[string]*netlink.Neigh { +// bridge fdb append fixed mac address dev raven0 dst {non_gateway_nodeN_private_ip} self permanent +func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Neigh, error) { fdbs := make(map[string]*netlink.Neigh) for k, v := range network.LocalNodeInfo { if vx.nodeName == k { continue } + HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(v.PrivateIP)) + if err != nil { + return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error()) + } fdbs[v.PrivateIP] = &netlink.Neigh{ LinkIndex: vx.vxlanIface.Attrs().Index, State: netlink.NUD_PERMANENT | netlink.NUD_NOARP, @@ -397,17 +414,21 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) map[string]*netlink.Nei Family: syscall.AF_BRIDGE, Flags: netlink.NTF_SELF, IP: net.ParseIP(v.PrivateIP), - HardwareAddr: networkutil.AllZeroMAC, + HardwareAddr: HardwareAddr, } } - return fdbs + return fdbs, nil } // calFDBOnNonGateway calculates and returns the desired FDB entries on non-gateway node. // The FDB entries format are equivalent to the following `bridge fdb append` command: // -// bridge fdb append 00:00:00:00:00:00 dev raven0 dst {gateway_node_private_ip} self permanent -func (vx *vxlan) calFDBOnNonGateway(network *types.Network) map[string]*netlink.Neigh { +// bridge fdb append fixed mac address dev raven0 dst {gateway_node_private_ip} self permanent +func (vx *vxlan) calFDBOnNonGateway(network *types.Network) (map[string]*netlink.Neigh, error) { + HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(network.LocalEndpoint.PrivateIP)) + if err != nil { + return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error()) + } return map[string]*netlink.Neigh{ network.LocalEndpoint.PrivateIP: { LinkIndex: vx.vxlanIface.Attrs().Index, @@ -416,9 +437,9 @@ func (vx *vxlan) calFDBOnNonGateway(network *types.Network) map[string]*netlink. Family: syscall.AF_BRIDGE, Flags: netlink.NTF_SELF, IP: net.ParseIP(network.LocalEndpoint.PrivateIP), - HardwareAddr: networkutil.AllZeroMAC, + HardwareAddr: HardwareAddr, }, - } + }, nil } // calIPSetOnNonGateway calculates and returns the desired ip set entries on non-gateway node. @@ -522,3 +543,21 @@ func (vx *vxlan) isGatewayRole(network *types.Network) bool { network.LocalEndpoint != nil && network.LocalEndpoint.NodeName == vx.nodeName } + +func (vx *vxlan) ipAddrToHardwareAddr(ip net.IP) (net.HardwareAddr, error) { + ips := strings.Split(vxlanIP(ip).String(), ".") + mac := []string{vx.macPrefix} + for idx := range ips { + num, err := strconv.ParseInt(ips[idx], 10, 64) + if err != nil { + return nil, fmt.Errorf("parse ip %s error %s", ip, err.Error()) + } + mac = append(mac, fmt.Sprintf("%02x", num)) + } + macStr := strings.Join(mac, ":") + macAddr, err := net.ParseMAC(macStr) + if err != nil { + return nil, fmt.Errorf("parse mac %s error %s", macStr, err.Error()) + } + return macAddr, nil +}