Skip to content

Commit

Permalink
Merge pull request #169 from River-sh/bugfix/raven_l3_and_l7
Browse files Browse the repository at this point in the history 
fix some bug for raven l3 and ravne l7
  • Loading branch information
@BSWANG
BSWANG authored Apr 2, 2024
2 parents a826b83 + ab59000 commit 78cb6f2
Show file tree
Hide file tree
Showing 5 changed files with 208 additions and 71 deletions.
14 changes: 12 additions & 2 deletions cmd/agent/app/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const (
DefaultTunnelMetricsPort = 10265
DefaultProxyMetricsPort = 10266
DefaultHealthyProbeAddr = 10275
DefaultLocalHost = "127.0.0.1"
DefaultMACPrefix = "aa:0f"
)

Expand Down Expand Up @@ -144,7 +145,7 @@ func (o *AgentOptions) Config() (*config.Config, error) {
NodeIP: o.NodeIP,
}
c.KubeConfig = cfg
c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, c.NodeIP, strconv.Itoa(DefaultTunnelMetricsPort))
c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, resolveLocalHost(), strconv.Itoa(DefaultTunnelMetricsPort))
c.HealthProbeAddr = resolveAddress(c.HealthProbeAddr, c.NodeIP, strconv.Itoa(DefaultHealthyProbeAddr))
c.Manager, err = newMgr(cfg, c.MetricsBindAddress, c.HealthProbeAddr)
if err != nil {
Expand Down Expand Up @@ -202,7 +203,7 @@ func (o *AgentOptions) Config() (*config.Config, error) {
c.Proxy.InternalInsecureAddress = resolveAddress(c.Proxy.InternalInsecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerInsecurePort))
c.Proxy.InternalSecureAddress = resolveAddress(c.Proxy.InternalSecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerSecurePort))
c.Proxy.ExternalAddress = resolveAddress(c.Proxy.ExternalAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerExposedPort))
c.Proxy.ProxyMetricsAddress = resolveAddress(c.Proxy.ProxyMetricsAddress, c.NodeIP, strconv.Itoa(DefaultProxyMetricsPort))
c.Proxy.ProxyMetricsAddress = resolveAddress(c.Proxy.ProxyMetricsAddress, resolveLocalHost(), strconv.Itoa(DefaultProxyMetricsPort))

return c, nil
}
Expand Down Expand Up @@ -310,6 +311,15 @@ func getGatewayAPIGroupResource() *restmapper.APIGroupResources {
}
}

func resolveLocalHost() string {
ipv4Addr, err := net.ResolveIPAddr("ip4", "localhost")
if err != nil {
klog.Warningf("can not get localhost addr, error %s, using default address %s", err.Error(), DefaultLocalHost)
return DefaultLocalHost
}
return ipv4Addr.String()
}

func resolveAddress(srcAddr, defaultHost, defaultPort string) string {
if srcAddr == "" {
return net.JoinHostPort(defaultHost, defaultPort)
Expand Down
14 changes: 10 additions & 4 deletions pkg/engine/proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,16 +97,22 @@ func (p *ProxyEngine) processNextWorkItem() bool {
}

func (p *ProxyEngine) handler(gw *v1beta1.Gateway) error {
proxyStatus := enableProxy(gw)
p.option.SetProxyStatus(proxyStatus)
specServer, specClient := p.getRole(proxyStatus)
var err error
p.gateway, err = utils.GetOwnGateway(p.client, p.nodeName)
if err != nil {
klog.Errorf(utils.FormatProxyServer("failed get gateway for %s, can not start proxy server", p.nodeName))
return err
}

proxyStatus := p.option.GetProxyStatus()
if p.gateway != nil && gw.GetName() == p.gateway.GetName() {
proxyStatus = enableProxy(gw)
} else {
if gw.Spec.ExposeType != "" {
proxyStatus = enableProxy(gw)
}
}
p.option.SetProxyStatus(proxyStatus)
specServer, specClient := p.getRole(proxyStatus)
switch JudgeType(p.proxyOption.GetServerStatus(), specServer) {
case StartType:
srcAddr := getSrcAddressForProxyServer(p.client, p.nodeName)
Expand Down
85 changes: 75 additions & 10 deletions pkg/networkengine/routedriver/vxlan/vxlan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,13 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
var desiredRules, currentRules map[string]*netlink.Rule

// The desired and current FDB entries calculated from given network.
// The key is netlink.Neigh.IP
// The key is NeighKey()
var desiredFDBs, currentFDBs map[string]*netlink.Neigh

// The desired and current ARP entries calculated from given network.
// The key is NeighKey()
var desiredARPs, currentARPs map[string]*netlink.Neigh

// The desired and current ipset entries calculated from given network.
// The key is ip set entry
var desiredSet, currentSet map[string]*netlink.IPSetEntry
Expand Down Expand Up @@ -129,6 +133,11 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
return fmt.Errorf("error listing fdb on node: %s", err)
}

currentARPs, err = networkutil.ListARPsOnNode(vx.vxlanIface)
if err != nil {
return fmt.Errorf("error listing arp on node: %s", err)
}

currentSet, err = networkutil.ListIPSetOnNode(vx.ipset)
if err != nil {
return fmt.Errorf("error listing ip set on node: %s", err)
Expand All @@ -143,7 +152,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
if err != nil {
return fmt.Errorf("error calculate gateway fdb: %s", err)
}

desiredARPs, err = vx.calARPOnGateway(network)
if err != nil {
return fmt.Errorf("error calculate gateway arp: %s", err)
}
err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, nonGatewayChainRuleSpec)
if err != nil {
return fmt.Errorf("error deleting non gateway chain rule: %s", err)
Expand All @@ -158,6 +170,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
if err != nil {
return fmt.Errorf("error calculate non gateway fdb: %s", err)
}
desiredARPs, err = vx.calARPOnNonGateway(network)
if err != nil {
return fmt.Errorf("error calculate non gateway arp: %s", err)
}
err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, gatewayChainRuleSpec)
if err != nil {
return fmt.Errorf("error deleting gateway chain rule: %s", err)
Expand All @@ -180,6 +196,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
if err != nil {
return fmt.Errorf("error applying fdb: %s", err)
}
err = networkutil.ApplyARPs(currentARPs, desiredARPs)
if err != nil {
return fmt.Errorf("error applying arp: %s", err)
}
err = networkutil.ApplyIPSet(vx.ipset, currentSet, desiredSet)
if err != nil {
return fmt.Errorf("error applying ip set: %s", err)
Expand Down Expand Up @@ -406,7 +426,7 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Ne
if err != nil {
return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error())
}
fdbs[v.PrivateIP] = &netlink.Neigh{
nh := &netlink.Neigh{
LinkIndex: vx.vxlanIface.Attrs().Index,
State: netlink.NUD_PERMANENT | netlink.NUD_NOARP,
Type: netlink.NDA_DST,
Expand All @@ -415,6 +435,7 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Ne
IP: net.ParseIP(v.PrivateIP),
HardwareAddr: HardwareAddr,
}
fdbs[networkutil.NeighKey(nh)] = nh
}
return fdbs, nil
}
Expand All @@ -428,17 +449,61 @@ func (vx *vxlan) calFDBOnNonGateway(network *types.Network) (map[string]*netlink
if err != nil {
return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error())
}
return map[string]*netlink.Neigh{
network.LocalEndpoint.PrivateIP: {
nh := &netlink.Neigh{
LinkIndex: vx.vxlanIface.Attrs().Index,
State: netlink.NUD_PERMANENT | netlink.NUD_NOARP,
Type: netlink.NDA_DST,
Family: syscall.AF_BRIDGE,
Flags: netlink.NTF_SELF,
IP: net.ParseIP(network.LocalEndpoint.PrivateIP),
HardwareAddr: HardwareAddr,
}
return map[string]*netlink.Neigh{networkutil.NeighKey(nh): nh}, nil
}

// calARPOnGateway calculates and returns the desired ARP entries on gateway node.
// The ARP entries format are equivalent to the following `ip neigh` command:
func (vx *vxlan) calARPOnGateway(network *types.Network) (map[string]*netlink.Neigh, error) {
arps := make(map[string]*netlink.Neigh)
for k, v := range network.LocalNodeInfo {
if vx.nodeName == k {
continue
}
HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(v.PrivateIP))
if err != nil {
return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error())
}
nh := &netlink.Neigh{
LinkIndex: vx.vxlanIface.Attrs().Index,
State: netlink.NUD_PERMANENT | netlink.NUD_NOARP,
State: netlink.NUD_PERMANENT,
Type: netlink.NDA_DST,
Family: syscall.AF_BRIDGE,
Family: syscall.AF_INET,
Flags: netlink.NTF_SELF,
IP: net.ParseIP(network.LocalEndpoint.PrivateIP),
IP: vxlanIP(net.ParseIP(v.PrivateIP)),
HardwareAddr: HardwareAddr,
},
}, nil
}
arps[networkutil.NeighKey(nh)] = nh
}
return arps, nil
}

// calARPOnNonGateway calculates and returns the desired ARP entries on non-gateway node.
// The ARP entries format are equivalent to the following `ip neigh` command:
func (vx *vxlan) calARPOnNonGateway(network *types.Network) (map[string]*netlink.Neigh, error) {
HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(network.LocalEndpoint.PrivateIP))
if err != nil {
return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error())
}
nh := &netlink.Neigh{
LinkIndex: vx.vxlanIface.Attrs().Index,
State: netlink.NUD_PERMANENT,
Type: netlink.NDA_DST,
Family: syscall.AF_INET,
Flags: netlink.NTF_SELF,
IP: vxlanIP(net.ParseIP(network.LocalEndpoint.PrivateIP)),
HardwareAddr: HardwareAddr,
}
return map[string]*netlink.Neigh{networkutil.NeighKey(nh): nh}, nil
}

// calIPSetOnNonGateway calculates and returns the desired ip set entries on non-gateway node.
Expand Down
25 changes: 18 additions & 7 deletions pkg/networkengine/util/netlink/netlink.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ var (

XfrmPolicyFlush = xfrmPolicyFlush

NeighAppend = neighAppend
NeighList = neighList
NeighDel = neighDel
NeighAdd = neighAdd
NeighReplace = neighReplace
NeighList = neighList
NeighDel = neighDel

LinkByName = linkByName
LinkByIndex = linkByIndex
Expand Down Expand Up @@ -158,13 +159,23 @@ func ruleDel(rule *netlink.Rule) (err error) {
return
}

func neighAppend(neigh *netlink.Neigh) (err error) {
err = netlink.NeighAppend(neigh)
func neighAdd(neigh *netlink.Neigh) (err error) {
err = netlink.NeighAdd(neigh)
if err != nil {
klog.ErrorS(err, "error on netlink.NeighAppend")
klog.ErrorS(err, "error on netlink.NeighSet")
return
}
klog.V(5).InfoS("netlink.NeighAppend succeeded")
klog.V(5).InfoS("netlink.NeighAdd succeeded")
return
}

func neighReplace(neigh *netlink.Neigh) (err error) {
err = netlink.NeighSet(neigh)
if err != nil {
klog.ErrorS(err, "error on netlink.NeighSet")
return
}
klog.V(5).InfoS("netlink.NeighSet succeeded")
return
}

Expand Down
Loading

0 comments on commit 78cb6f2

Please sign in to comment.