From a070b58228ba65b6764d766cada76853cdd1fab9 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Fri, 5 Jan 2024 02:06:54 +0800
Subject: [PATCH 01/13] Handle transparent UDP socket in an independent netns

---
 control/anyfrom_pool.go |   6 +-
 control/netns_utils.go  | 134 ++++++++++++++++++++++++++++++++++++++++
 control/udp.go          |   5 +-
 control/utils.go        |   4 ++
 4 files changed, 147 insertions(+), 2 deletions(-)
 create mode 100644 control/netns_utils.go

diff --git a/control/anyfrom_pool.go b/control/anyfrom_pool.go
index f7206178d..b10354ff5 100644
--- a/control/anyfrom_pool.go
+++ b/control/anyfrom_pool.go
@@ -190,7 +190,11 @@ func (p *AnyfromPool) GetOrCreate(lAddr string, ttl time.Duration) (conn *Anyfro
 			},
 			KeepAlive: 0,
 		}
-		pc, err := d.ListenPacket(context.Background(), "udp", lAddr)
+		var pc net.PacketConn
+		err := WithIndieNetns(func() (err error) {
+			pc, err = d.ListenPacket(context.Background(), "udp", lAddr)
+			return err
+		})
 		if err != nil {
 			return nil, true, err
 		}
diff --git a/control/netns_utils.go b/control/netns_utils.go
new file mode 100644
index 000000000..99ed50ca8
--- /dev/null
+++ b/control/netns_utils.go
@@ -0,0 +1,134 @@
+package control
+
+import (
+	"net"
+	"os"
+	"path"
+	"runtime"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
+)
+
+var indieNetns netns.NsHandle
+
+func WithIndieNetns(f func() error) (err error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	hostNetns, err := netns.Get()
+	if err != nil {
+		return
+	}
+	defer netns.Set(hostNetns)
+
+	ns, err := GetIndieNetns()
+	if err != nil {
+		return
+	}
+	if err = netns.Set(ns); err != nil {
+		return
+	}
+
+	return f()
+}
+
+func GetIndieNetns() (_ netns.NsHandle, err error) {
+	if indieNetns != 0 {
+		return indieNetns, nil
+	}
+
+	// Setup a new netns
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	hostNetns, err := netns.Get()
+	if err != nil {
+		return
+	}
+	defer netns.Set(hostNetns)
+
+	// ip netns a daens
+	DeleteNamedNetns("daens")
+	indieNetns, err = netns.NewNamed("daens")
+	if err != nil {
+		return
+	}
+	if err = netns.Set(hostNetns); err != nil {
+		return
+	}
+	// ip l a dae0 type veth peer name dae0peer
+	DeleteLink("dae0")
+	if err = netlink.LinkAdd(&netlink.Veth{
+		LinkAttrs: netlink.LinkAttrs{
+			Name: "dae0",
+		},
+		PeerName: "dae0peer",
+	}); err != nil {
+		return
+	}
+	dae0, err := netlink.LinkByName("dae0")
+	if err != nil {
+		return
+	}
+	dae0peer, err := netlink.LinkByName("dae0peer")
+	if err != nil {
+		return
+	}
+	// ip l s dae0 up
+	if err = netlink.LinkSetUp(dae0); err != nil {
+		return
+	}
+	// sysctl net.ipv4.conf.{dae0,all}.rp_filter=0
+	if err = SetRpFilter("dae0", "0"); err != nil {
+		return
+	}
+	if err = SetRpFilter("all", "0"); err != nil {
+		return
+	}
+	// ip l s dae0peer netns daens
+	if err = netlink.LinkSetNsFd(dae0peer, int(indieNetns)); err != nil {
+		return
+	}
+	// ip net e daens
+	if err = netns.Set(indieNetns); err != nil {
+		return
+	}
+	// (ip net e daens) ip l s dae0peer up
+	if err = netlink.LinkSetUp(dae0peer); err != nil {
+		return
+	}
+	// (ip net e daens) ip a a 169.254.0.1 dev dae0peer
+	ip, ipNet, err := net.ParseCIDR("169.254.0.1/24")
+	ipNet.IP = ip
+	if err != nil {
+		return
+	}
+	if err = netlink.AddrAdd(dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
+		return
+	}
+	// (ip net e daens) ip r a default dev dae0peer
+	if err = netlink.RouteAdd(&netlink.Route{
+		LinkIndex: dae0peer.Attrs().Index,
+		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
+		Gw:        nil,
+	}); err != nil {
+		return
+	}
+	return indieNetns, err
+}
+
+func DeleteNamedNetns(name string) error {
+	namedPath := path.Join("/run/netns", name)
+	unix.Unmount(namedPath, unix.MNT_DETACH)
+	return os.Remove(namedPath)
+}
+
+func DeleteLink(name string) error {
+	link, err := netlink.LinkByName(name)
+	if err == nil {
+		return netlink.LinkDel(link)
+	}
+	return err
+}
diff --git a/control/udp.go b/control/udp.go
index cefa7c033..e0ed524c2 100644
--- a/control/udp.go
+++ b/control/udp.go
@@ -97,7 +97,10 @@ func sendPkt(data []byte, from netip.AddrPort, realTo, to netip.AddrPort, lConn
 		}
 		return err
 	}
-	_, err = uConn.WriteToUDPAddrPort(data, realTo)
+	err = WithIndieNetns(func() (err error) {
+		_, err = uConn.WriteToUDPAddrPort(data, realTo)
+		return
+	})
 	return err
 }
 
diff --git a/control/utils.go b/control/utils.go
index 8a99edbfa..caa4f5999 100644
--- a/control/utils.go
+++ b/control/utils.go
@@ -132,6 +132,10 @@ func SetAcceptLocal(ifname, val string) error {
 	return os.WriteFile(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/accept_local", ifname), []byte(val), 0644)
 }
 
+func SetRpFilter(ifname, val string) error {
+	return os.WriteFile(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/rp_filter", ifname), []byte(val), 0644)
+}
+
 func checkSendRedirects(ifname string, ipversion consts.IpVersionStr) error {
 	path := fmt.Sprintf("/proc/sys/net/ipv%v/conf/%v/send_redirects", ipversion, ifname)
 	b, err := os.ReadFile(path)

From 559b8e51e658ac9b6b7529a09dcd48bc1212bd21 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Fri, 5 Jan 2024 20:08:10 +0800
Subject: [PATCH 02/13] Use netns only if EADDRINUSE occurred

---
 control/anyfrom_pool.go |  6 +-----
 control/udp.go          | 17 ++++++++---------
 2 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/control/anyfrom_pool.go b/control/anyfrom_pool.go
index b10354ff5..f7206178d 100644
--- a/control/anyfrom_pool.go
+++ b/control/anyfrom_pool.go
@@ -190,11 +190,7 @@ func (p *AnyfromPool) GetOrCreate(lAddr string, ttl time.Duration) (conn *Anyfro
 			},
 			KeepAlive: 0,
 		}
-		var pc net.PacketConn
-		err := WithIndieNetns(func() (err error) {
-			pc, err = d.ListenPacket(context.Background(), "udp", lAddr)
-			return err
-		})
+		pc, err := d.ListenPacket(context.Background(), "udp", lAddr)
 		if err != nil {
 			return nil, true, err
 		}
diff --git a/control/udp.go b/control/udp.go
index e0ed524c2..d3c380ef0 100644
--- a/control/udp.go
+++ b/control/udp.go
@@ -90,17 +90,16 @@ func sendPkt(data []byte, from netip.AddrPort, realTo, to netip.AddrPort, lConn
 	}
 
 	uConn, _, err := DefaultAnyfromPool.GetOrCreate(from.String(), AnyfromTimeout)
-	if err != nil {
-		if errors.Is(err, syscall.EADDRINUSE) {
-			// Port collision, use traditional method.
-			return sendPktWithHdrWithFlag(data, from, lConn, to, lanWanFlag)
-		}
-		return err
+	if err != nil && errors.Is(err, syscall.EADDRINUSE) {
+		err = WithIndieNetns(func() (err error) {
+			uConn, _, err = DefaultAnyfromPool.GetOrCreate(from.String(), AnyfromTimeout)
+			return err
+		})
 	}
-	err = WithIndieNetns(func() (err error) {
-		_, err = uConn.WriteToUDPAddrPort(data, realTo)
+	if err != nil {
 		return
-	})
+	}
+	_, err = uConn.WriteToUDPAddrPort(data, realTo)
 	return err
 }
 

From 91e6e8795732fdba64cc055c519310aa739a4b5a Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Sat, 6 Jan 2024 00:08:35 +0800
Subject: [PATCH 03/13] Concurrency control for indieNetns setup

---
 control/netns_utils.go | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index 99ed50ca8..36225ce5f 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -5,13 +5,17 @@ import (
 	"os"
 	"path"
 	"runtime"
+	"sync"
 
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 	"golang.org/x/sys/unix"
 )
 
-var indieNetns netns.NsHandle
+var (
+	indieNetns netns.NsHandle
+	once       sync.Once
+)
 
 func WithIndieNetns(f func() error) (err error) {
 	runtime.LockOSThread()
@@ -39,7 +43,13 @@ func GetIndieNetns() (_ netns.NsHandle, err error) {
 		return indieNetns, nil
 	}
 
-	// Setup a new netns
+	once.Do(func() {
+		err = setupIndieNetns()
+	})
+	return indieNetns, err
+}
+
+func setupIndieNetns() (err error) {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
@@ -116,7 +126,7 @@ func GetIndieNetns() (_ netns.NsHandle, err error) {
 	}); err != nil {
 		return
 	}
-	return indieNetns, err
+	return
 }
 
 func DeleteNamedNetns(name string) error {

From 8c8c3711f7417ccfda99f7168527d6fc4ac0ffc8 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Sat, 6 Jan 2024 01:08:37 +0800
Subject: [PATCH 04/13] Add error details for setupIndieNetns

---
 control/netns_utils.go | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index 36225ce5f..1d1f01b0f 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -1,6 +1,7 @@
 package control
 
 import (
+	"fmt"
 	"net"
 	"os"
 	"path"
@@ -55,7 +56,7 @@ func setupIndieNetns() (err error) {
 
 	hostNetns, err := netns.Get()
 	if err != nil {
-		return
+		return fmt.Errorf("Failed to get host netns: %v", err)
 	}
 	defer netns.Set(hostNetns)
 
@@ -63,10 +64,10 @@ func setupIndieNetns() (err error) {
 	DeleteNamedNetns("daens")
 	indieNetns, err = netns.NewNamed("daens")
 	if err != nil {
-		return
+		return fmt.Errorf("Failed to create netns: %v", err)
 	}
 	if err = netns.Set(hostNetns); err != nil {
-		return
+		return fmt.Errorf("Failed to switch to host netns: %v", err)
 	}
 	// ip l a dae0 type veth peer name dae0peer
 	DeleteLink("dae0")
@@ -76,47 +77,47 @@ func setupIndieNetns() (err error) {
 		},
 		PeerName: "dae0peer",
 	}); err != nil {
-		return
+		return fmt.Errorf("Failed to add veth pair: %v", err)
 	}
 	dae0, err := netlink.LinkByName("dae0")
 	if err != nil {
-		return
+		return fmt.Errorf("Failed to get link dae0: %v", err)
 	}
 	dae0peer, err := netlink.LinkByName("dae0peer")
 	if err != nil {
-		return
+		return fmt.Errorf("Failed to get link dae0peer: %v", err)
 	}
 	// ip l s dae0 up
 	if err = netlink.LinkSetUp(dae0); err != nil {
-		return
+		return fmt.Errorf("Failed to set link dae0 up: %v", err)
 	}
 	// sysctl net.ipv4.conf.{dae0,all}.rp_filter=0
 	if err = SetRpFilter("dae0", "0"); err != nil {
-		return
+		return fmt.Errorf("Failed to set rp_filter for dae0: %v", err)
 	}
 	if err = SetRpFilter("all", "0"); err != nil {
-		return
+		return fmt.Errorf("Failed to set rp_filter for all: %v", err)
 	}
 	// ip l s dae0peer netns daens
 	if err = netlink.LinkSetNsFd(dae0peer, int(indieNetns)); err != nil {
-		return
+		return fmt.Errorf("Failed to move dae0peer to daens: %v", err)
 	}
 	// ip net e daens
 	if err = netns.Set(indieNetns); err != nil {
-		return
+		return fmt.Errorf("Failed to switch to daens: %v", err)
 	}
 	// (ip net e daens) ip l s dae0peer up
 	if err = netlink.LinkSetUp(dae0peer); err != nil {
-		return
+		return fmt.Errorf("Failed to set link dae0peer up: %v", err)
 	}
 	// (ip net e daens) ip a a 169.254.0.1 dev dae0peer
 	ip, ipNet, err := net.ParseCIDR("169.254.0.1/24")
 	ipNet.IP = ip
 	if err != nil {
-		return
+		return fmt.Errorf("Failed to parse ip: %v", err)
 	}
 	if err = netlink.AddrAdd(dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
-		return
+		return fmt.Errorf("Failed to add addr to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a default dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
@@ -124,14 +125,14 @@ func setupIndieNetns() (err error) {
 		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
 		Gw:        nil,
 	}); err != nil {
-		return
+		return fmt.Errorf("Failed to add route to dae0peer: %v", err)
 	}
 	return
 }
 
 func DeleteNamedNetns(name string) error {
 	namedPath := path.Join("/run/netns", name)
-	unix.Unmount(namedPath, unix.MNT_DETACH)
+	unix.Unmount(namedPath, unix.MNT_DETACH|unix.MNT_FORCE)
 	return os.Remove(namedPath)
 }
 

From 27baf38d0e6220ac82d9e85180e61bab6a20b122 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Sat, 6 Jan 2024 19:38:34 +0800
Subject: [PATCH 05/13] Support IPv6

---
 control/netns_utils.go | 45 +++++++++++++++++++++++++++++++++++++-----
 control/udp.go         |  4 ++++
 control/utils.go       |  8 ++++++++
 3 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index 1d1f01b0f..5cdc159ed 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -24,7 +24,7 @@ func WithIndieNetns(f func() error) (err error) {
 
 	hostNetns, err := netns.Get()
 	if err != nil {
-		return
+		return fmt.Errorf("Failed to get host netns: %v", err)
 	}
 	defer netns.Set(hostNetns)
 
@@ -33,7 +33,7 @@ func WithIndieNetns(f func() error) (err error) {
 		return
 	}
 	if err = netns.Set(ns); err != nil {
-		return
+		return fmt.Errorf("Failed to switch to daens: %v", err)
 	}
 
 	return f()
@@ -91,13 +91,39 @@ func setupIndieNetns() (err error) {
 	if err = netlink.LinkSetUp(dae0); err != nil {
 		return fmt.Errorf("Failed to set link dae0 up: %v", err)
 	}
-	// sysctl net.ipv4.conf.{dae0,all}.rp_filter=0
+	// sysctl net.ipv4.conf.dae0.rp_filter=0
 	if err = SetRpFilter("dae0", "0"); err != nil {
 		return fmt.Errorf("Failed to set rp_filter for dae0: %v", err)
 	}
+	// sysctl net.ipv4.conf.all.rp_filter=0
 	if err = SetRpFilter("all", "0"); err != nil {
 		return fmt.Errorf("Failed to set rp_filter for all: %v", err)
 	}
+	// sysctl net.ipv4.conf.dae0.arp_filter=0
+	if err = SetArpFilter("dae0", "0"); err != nil {
+		return fmt.Errorf("Failed to set arp_filter for dae0: %v", err)
+	}
+	// sysctl net.ipv4.conf.all.arp_filter=0
+	if err = SetArpFilter("all", "0"); err != nil {
+		return fmt.Errorf("Failed to set arp_filter for all: %v", err)
+	}
+	// sysctl net.ipv6.conf.dae0.disable_ipv6=0
+	if err = SetDisableIpv6("dae0", "0"); err != nil {
+		return fmt.Errorf("Failed to set disable_ipv6 for dae0: %v", err)
+	}
+	// sysctl net.ipv6.conf.dae0.forwarding=1
+	SetForwarding("dae0", "1")
+	// sysctl net.ipv6.conf.all.forwarding=1
+	SetForwarding("all", "1")
+	// ip -6 a a fe80::ecee:eeff:feee:eeee dev dae0 scope link
+	if err = netlink.AddrAdd(dae0, &netlink.Addr{
+		IPNet: &net.IPNet{
+			IP:   net.ParseIP("fe80::ecee:eeff:feee:eeee"),
+			Mask: net.CIDRMask(128, 128),
+		},
+	}); err != nil {
+		return fmt.Errorf("Failed to add v6 addr to dae0: %v", err)
+	}
 	// ip l s dae0peer netns daens
 	if err = netlink.LinkSetNsFd(dae0peer, int(indieNetns)); err != nil {
 		return fmt.Errorf("Failed to move dae0peer to daens: %v", err)
@@ -117,7 +143,7 @@ func setupIndieNetns() (err error) {
 		return fmt.Errorf("Failed to parse ip: %v", err)
 	}
 	if err = netlink.AddrAdd(dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
-		return fmt.Errorf("Failed to add addr to dae0peer: %v", err)
+		return fmt.Errorf("Failed to add v4 addr to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a default dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
@@ -125,8 +151,17 @@ func setupIndieNetns() (err error) {
 		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
 		Gw:        nil,
 	}); err != nil {
-		return fmt.Errorf("Failed to add route to dae0peer: %v", err)
+		return fmt.Errorf("Failed to add v4 route to dae0peer: %v", err)
+	}
+	// (ip net e daens) ip -6 r a default via fe80::ecee:eeff:feee:eeee dev dae0peer
+	if err = netlink.RouteAdd(&netlink.Route{
+		LinkIndex: dae0peer.Attrs().Index,
+		Dst:       &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)},
+		Gw:        net.ParseIP("fe80::ecee:eeff:feee:eeee"),
+	}); err != nil {
+		return fmt.Errorf("Failed to add v6 route to dae0peer: %v", err)
 	}
+
 	return
 }
 
diff --git a/control/udp.go b/control/udp.go
index d3c380ef0..0eee005a8 100644
--- a/control/udp.go
+++ b/control/udp.go
@@ -91,6 +91,10 @@ func sendPkt(data []byte, from netip.AddrPort, realTo, to netip.AddrPort, lConn
 
 	uConn, _, err := DefaultAnyfromPool.GetOrCreate(from.String(), AnyfromTimeout)
 	if err != nil && errors.Is(err, syscall.EADDRINUSE) {
+		logrus.WithField("from", from).
+			WithField("to", to).
+			WithField("realTo", realTo).
+			Trace("Port in use, fallback to use netns.")
 		err = WithIndieNetns(func() (err error) {
 			uConn, _, err = DefaultAnyfromPool.GetOrCreate(from.String(), AnyfromTimeout)
 			return err
diff --git a/control/utils.go b/control/utils.go
index caa4f5999..98888454f 100644
--- a/control/utils.go
+++ b/control/utils.go
@@ -136,6 +136,14 @@ func SetRpFilter(ifname, val string) error {
 	return os.WriteFile(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/rp_filter", ifname), []byte(val), 0644)
 }
 
+func SetArpFilter(ifname, val string) error {
+	return os.WriteFile(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/arp_filter", ifname), []byte(val), 0644)
+}
+
+func SetDisableIpv6(ifname, val string) error {
+	return os.WriteFile(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/disable_ipv6", ifname), []byte(val), 0644)
+}
+
 func checkSendRedirects(ifname string, ipversion consts.IpVersionStr) error {
 	path := fmt.Sprintf("/proc/sys/net/ipv%v/conf/%v/send_redirects", ipversion, ifname)
 	b, err := os.ReadFile(path)

From e1eba412bb44e2d2554f05a7d9a8dc7e3d4684dd Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Sun, 7 Jan 2024 20:39:35 +0800
Subject: [PATCH 06/13] sysctl net.ipv4.conf.dae0.accept_local=1

---
 control/netns_utils.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index 5cdc159ed..a81f97472 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -107,6 +107,10 @@ func setupIndieNetns() (err error) {
 	if err = SetArpFilter("all", "0"); err != nil {
 		return fmt.Errorf("Failed to set arp_filter for all: %v", err)
 	}
+	// sysctl net.ipv4.conf.dae0.accept_local=1
+	if err = SetAcceptLocal("dae0", "1"); err != nil {
+		return fmt.Errorf("Failed to set accept_local for dae0: %v", err)
+	}
 	// sysctl net.ipv6.conf.dae0.disable_ipv6=0
 	if err = SetDisableIpv6("dae0", "0"); err != nil {
 		return fmt.Errorf("Failed to set disable_ipv6 for dae0: %v", err)

From c1c967a509fc7f93c93f5030177716668545f210 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Mon, 8 Jan 2024 15:19:23 +0800
Subject: [PATCH 07/13] Add permanent ARP entry to avoid L2 resolution failure

---
 control/netns_utils.go | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index a81f97472..bc0e33858 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -140,8 +140,8 @@ func setupIndieNetns() (err error) {
 	if err = netlink.LinkSetUp(dae0peer); err != nil {
 		return fmt.Errorf("Failed to set link dae0peer up: %v", err)
 	}
-	// (ip net e daens) ip a a 169.254.0.1 dev dae0peer
-	ip, ipNet, err := net.ParseCIDR("169.254.0.1/24")
+	// (ip net e daens) ip a a 169.254.0.11 dev dae0peer
+	ip, ipNet, err := net.ParseCIDR("169.254.0.11/32")
 	ipNet.IP = ip
 	if err != nil {
 		return fmt.Errorf("Failed to parse ip: %v", err)
@@ -149,13 +149,31 @@ func setupIndieNetns() (err error) {
 	if err = netlink.AddrAdd(dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
 		return fmt.Errorf("Failed to add v4 addr to dae0peer: %v", err)
 	}
-	// (ip net e daens) ip r a default dev dae0peer
+	// (ip net e daens) ip r a 169.254.0.1 dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
 		LinkIndex: dae0peer.Attrs().Index,
-		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
+		Dst:       &net.IPNet{IP: net.ParseIP("169.254.0.1"), Mask: net.CIDRMask(32, 32)},
 		Gw:        nil,
+		Scope:     netlink.SCOPE_LINK,
+	}); err != nil {
+		return fmt.Errorf("Failed to add v4 route1 to dae0peer: %v", err)
+	}
+	// (ip net e daens) ip r a default via 169.254.0.1 dev dae0peer
+	if err = netlink.RouteAdd(&netlink.Route{
+		LinkIndex: dae0peer.Attrs().Index,
+		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
+		Gw:        net.ParseIP("169.254.0.1"),
+	}); err != nil {
+		return fmt.Errorf("Failed to add v4 route2 to dae0peer: %v", err)
+	}
+	// (ip net e daens) ip n r 169.254.0.1 dev dae0peer lladdr $mac_dae0 nud permanent
+	if err = netlink.NeighAdd(&netlink.Neigh{
+		IP:           net.ParseIP("169.254.0.1"),
+		HardwareAddr: dae0.Attrs().HardwareAddr,
+		LinkIndex:    dae0peer.Attrs().Index,
+		State:        netlink.NUD_PERMANENT,
 	}); err != nil {
-		return fmt.Errorf("Failed to add v4 route to dae0peer: %v", err)
+		return fmt.Errorf("Failed to add neigh to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip -6 r a default via fe80::ecee:eeff:feee:eeee dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{

From 7a60facc5362b9a08c83577e6ca25877c97a87f2 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Mon, 8 Jan 2024 15:33:41 +0800
Subject: [PATCH 08/13] setupNetns can retry on errors

---
 control/netns_utils.go | 78 +++++++++++++++++++++++-------------------
 control/udp.go         |  2 +-
 2 files changed, 43 insertions(+), 37 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index bc0e33858..b2e344171 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -8,17 +8,18 @@ import (
 	"runtime"
 	"sync"
 
+	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 	"golang.org/x/sys/unix"
 )
 
 var (
-	indieNetns netns.NsHandle
-	once       sync.Once
+	daeNetns netns.NsHandle
+	once     sync.Once
 )
 
-func WithIndieNetns(f func() error) (err error) {
+func WithDaeNetns(f func() error) (err error) {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
@@ -28,7 +29,7 @@ func WithIndieNetns(f func() error) (err error) {
 	}
 	defer netns.Set(hostNetns)
 
-	ns, err := GetIndieNetns()
+	ns, err := GetDaeNetns()
 	if err != nil {
 		return
 	}
@@ -39,35 +40,40 @@ func WithIndieNetns(f func() error) (err error) {
 	return f()
 }
 
-func GetIndieNetns() (_ netns.NsHandle, err error) {
-	if indieNetns != 0 {
-		return indieNetns, nil
+func GetDaeNetns() (_ netns.NsHandle, err error) {
+	if daeNetns != 0 {
+		return daeNetns, nil
 	}
 
 	once.Do(func() {
-		err = setupIndieNetns()
+		daeNetns, err = setupDaeNetns()
+		if err != nil {
+			once = sync.Once{}
+		}
 	})
-	return indieNetns, err
+	return daeNetns, err
 }
 
-func setupIndieNetns() (err error) {
+func setupDaeNetns() (ns netns.NsHandle, err error) {
+	logrus.Trace("Setting up dae netns")
+
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
 	hostNetns, err := netns.Get()
 	if err != nil {
-		return fmt.Errorf("Failed to get host netns: %v", err)
+		return 0, fmt.Errorf("Failed to get host netns: %v", err)
 	}
 	defer netns.Set(hostNetns)
 
 	// ip netns a daens
 	DeleteNamedNetns("daens")
-	indieNetns, err = netns.NewNamed("daens")
+	ns, err = netns.NewNamed("daens")
 	if err != nil {
-		return fmt.Errorf("Failed to create netns: %v", err)
+		return 0, fmt.Errorf("Failed to create netns: %v", err)
 	}
 	if err = netns.Set(hostNetns); err != nil {
-		return fmt.Errorf("Failed to switch to host netns: %v", err)
+		return 0, fmt.Errorf("Failed to switch to host netns: %v", err)
 	}
 	// ip l a dae0 type veth peer name dae0peer
 	DeleteLink("dae0")
@@ -77,43 +83,43 @@ func setupIndieNetns() (err error) {
 		},
 		PeerName: "dae0peer",
 	}); err != nil {
-		return fmt.Errorf("Failed to add veth pair: %v", err)
+		return 0, fmt.Errorf("Failed to add veth pair: %v", err)
 	}
 	dae0, err := netlink.LinkByName("dae0")
 	if err != nil {
-		return fmt.Errorf("Failed to get link dae0: %v", err)
+		return 0, fmt.Errorf("Failed to get link dae0: %v", err)
 	}
 	dae0peer, err := netlink.LinkByName("dae0peer")
 	if err != nil {
-		return fmt.Errorf("Failed to get link dae0peer: %v", err)
+		return 0, fmt.Errorf("Failed to get link dae0peer: %v", err)
 	}
 	// ip l s dae0 up
 	if err = netlink.LinkSetUp(dae0); err != nil {
-		return fmt.Errorf("Failed to set link dae0 up: %v", err)
+		return 0, fmt.Errorf("Failed to set link dae0 up: %v", err)
 	}
 	// sysctl net.ipv4.conf.dae0.rp_filter=0
 	if err = SetRpFilter("dae0", "0"); err != nil {
-		return fmt.Errorf("Failed to set rp_filter for dae0: %v", err)
+		return 0, fmt.Errorf("Failed to set rp_filter for dae0: %v", err)
 	}
 	// sysctl net.ipv4.conf.all.rp_filter=0
 	if err = SetRpFilter("all", "0"); err != nil {
-		return fmt.Errorf("Failed to set rp_filter for all: %v", err)
+		return 0, fmt.Errorf("Failed to set rp_filter for all: %v", err)
 	}
 	// sysctl net.ipv4.conf.dae0.arp_filter=0
 	if err = SetArpFilter("dae0", "0"); err != nil {
-		return fmt.Errorf("Failed to set arp_filter for dae0: %v", err)
+		return 0, fmt.Errorf("Failed to set arp_filter for dae0: %v", err)
 	}
 	// sysctl net.ipv4.conf.all.arp_filter=0
 	if err = SetArpFilter("all", "0"); err != nil {
-		return fmt.Errorf("Failed to set arp_filter for all: %v", err)
+		return 0, fmt.Errorf("Failed to set arp_filter for all: %v", err)
 	}
 	// sysctl net.ipv4.conf.dae0.accept_local=1
 	if err = SetAcceptLocal("dae0", "1"); err != nil {
-		return fmt.Errorf("Failed to set accept_local for dae0: %v", err)
+		return 0, fmt.Errorf("Failed to set accept_local for dae0: %v", err)
 	}
 	// sysctl net.ipv6.conf.dae0.disable_ipv6=0
 	if err = SetDisableIpv6("dae0", "0"); err != nil {
-		return fmt.Errorf("Failed to set disable_ipv6 for dae0: %v", err)
+		return 0, fmt.Errorf("Failed to set disable_ipv6 for dae0: %v", err)
 	}
 	// sysctl net.ipv6.conf.dae0.forwarding=1
 	SetForwarding("dae0", "1")
@@ -126,28 +132,28 @@ func setupIndieNetns() (err error) {
 			Mask: net.CIDRMask(128, 128),
 		},
 	}); err != nil {
-		return fmt.Errorf("Failed to add v6 addr to dae0: %v", err)
+		return 0, fmt.Errorf("Failed to add v6 addr to dae0: %v", err)
 	}
 	// ip l s dae0peer netns daens
-	if err = netlink.LinkSetNsFd(dae0peer, int(indieNetns)); err != nil {
-		return fmt.Errorf("Failed to move dae0peer to daens: %v", err)
+	if err = netlink.LinkSetNsFd(dae0peer, int(ns)); err != nil {
+		return 0, fmt.Errorf("Failed to move dae0peer to daens: %v", err)
 	}
 	// ip net e daens
-	if err = netns.Set(indieNetns); err != nil {
-		return fmt.Errorf("Failed to switch to daens: %v", err)
+	if err = netns.Set(ns); err != nil {
+		return 0, fmt.Errorf("Failed to switch to daens: %v", err)
 	}
 	// (ip net e daens) ip l s dae0peer up
 	if err = netlink.LinkSetUp(dae0peer); err != nil {
-		return fmt.Errorf("Failed to set link dae0peer up: %v", err)
+		return 0, fmt.Errorf("Failed to set link dae0peer up: %v", err)
 	}
 	// (ip net e daens) ip a a 169.254.0.11 dev dae0peer
 	ip, ipNet, err := net.ParseCIDR("169.254.0.11/32")
 	ipNet.IP = ip
 	if err != nil {
-		return fmt.Errorf("Failed to parse ip: %v", err)
+		return 0, fmt.Errorf("Failed to parse ip: %v", err)
 	}
 	if err = netlink.AddrAdd(dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
-		return fmt.Errorf("Failed to add v4 addr to dae0peer: %v", err)
+		return 0, fmt.Errorf("Failed to add v4 addr to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a 169.254.0.1 dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
@@ -156,7 +162,7 @@ func setupIndieNetns() (err error) {
 		Gw:        nil,
 		Scope:     netlink.SCOPE_LINK,
 	}); err != nil {
-		return fmt.Errorf("Failed to add v4 route1 to dae0peer: %v", err)
+		return 0, fmt.Errorf("Failed to add v4 route1 to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a default via 169.254.0.1 dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
@@ -164,7 +170,7 @@ func setupIndieNetns() (err error) {
 		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
 		Gw:        net.ParseIP("169.254.0.1"),
 	}); err != nil {
-		return fmt.Errorf("Failed to add v4 route2 to dae0peer: %v", err)
+		return 0, fmt.Errorf("Failed to add v4 route2 to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip n r 169.254.0.1 dev dae0peer lladdr $mac_dae0 nud permanent
 	if err = netlink.NeighAdd(&netlink.Neigh{
@@ -173,7 +179,7 @@ func setupIndieNetns() (err error) {
 		LinkIndex:    dae0peer.Attrs().Index,
 		State:        netlink.NUD_PERMANENT,
 	}); err != nil {
-		return fmt.Errorf("Failed to add neigh to dae0peer: %v", err)
+		return 0, fmt.Errorf("Failed to add neigh to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip -6 r a default via fe80::ecee:eeff:feee:eeee dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
@@ -181,7 +187,7 @@ func setupIndieNetns() (err error) {
 		Dst:       &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)},
 		Gw:        net.ParseIP("fe80::ecee:eeff:feee:eeee"),
 	}); err != nil {
-		return fmt.Errorf("Failed to add v6 route to dae0peer: %v", err)
+		return 0, fmt.Errorf("Failed to add v6 route to dae0peer: %v", err)
 	}
 
 	return
diff --git a/control/udp.go b/control/udp.go
index 0eee005a8..44073c0d6 100644
--- a/control/udp.go
+++ b/control/udp.go
@@ -95,7 +95,7 @@ func sendPkt(data []byte, from netip.AddrPort, realTo, to netip.AddrPort, lConn
 			WithField("to", to).
 			WithField("realTo", realTo).
 			Trace("Port in use, fallback to use netns.")
-		err = WithIndieNetns(func() (err error) {
+		err = WithDaeNetns(func() (err error) {
 			uConn, _, err = DefaultAnyfromPool.GetOrCreate(from.String(), AnyfromTimeout)
 			return err
 		})

From 818bb961c5e904ca4d13ab1d1c10de3360c0b0e0 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Wed, 10 Jan 2024 02:56:47 +0800
Subject: [PATCH 09/13] Monitor lladdr changes of dae0

---
 control/netns_utils.go | 275 +++++++++++++++++++++++++++--------------
 control/udp.go         |   2 +-
 2 files changed, 184 insertions(+), 93 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index b2e344171..abb6e3792 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -1,6 +1,7 @@
 package control
 
 import (
+	"bytes"
 	"fmt"
 	"net"
 	"os"
@@ -14,185 +15,275 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-var (
-	daeNetns netns.NsHandle
-	once     sync.Once
+const (
+	NsName       = "daens"
+	HostVethName = "dae0"
+	NsVethName   = "dae0peer"
 )
 
-func WithDaeNetns(f func() error) (err error) {
-	runtime.LockOSThread()
-	defer runtime.UnlockOSThread()
+var (
+	daeNetns *DaeNetns
+)
 
-	hostNetns, err := netns.Get()
-	if err != nil {
-		return fmt.Errorf("Failed to get host netns: %v", err)
-	}
-	defer netns.Set(hostNetns)
+type DaeNetns struct {
+	once sync.Once
 
-	ns, err := GetDaeNetns()
-	if err != nil {
-		return
-	}
-	if err = netns.Set(ns); err != nil {
-		return fmt.Errorf("Failed to switch to daens: %v", err)
-	}
+	dae0, dae0peer netlink.Link
+	hostNs, daeNs  netns.NsHandle
+}
 
-	return f()
+func init() {
+	daeNetns = &DaeNetns{}
 }
 
-func GetDaeNetns() (_ netns.NsHandle, err error) {
-	if daeNetns != 0 {
-		return daeNetns, nil
+func (ns *DaeNetns) Setup() (err error) {
+	if ns.daeNs != 0 && ns.hostNs != 0 {
+		return nil
 	}
 
-	once.Do(func() {
-		daeNetns, err = setupDaeNetns()
-		if err != nil {
-			once = sync.Once{}
+	ns.once.Do(func() {
+		if err = ns.setup(); err != nil {
+			logrus.Fatal("Failed to setup dae netns: %v", err)
 		}
 	})
-	return daeNetns, err
+	return err
+}
+
+func (ns *DaeNetns) With(f func() error) (err error) {
+	if err = daeNetns.Setup(); err != nil {
+		return fmt.Errorf("Failed to setup dae netns: %v", err)
+	}
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	if err = netns.Set(ns.daeNs); err != nil {
+		return fmt.Errorf("Failed to switch to daens: %v", err)
+	}
+	defer netns.Set(ns.hostNs)
+
+	if err = f(); err != nil {
+		return fmt.Errorf("Failed to run func in dae netns: %v", err)
+	}
+	return
 }
 
-func setupDaeNetns() (ns netns.NsHandle, err error) {
+func (ns *DaeNetns) setup() (err error) {
 	logrus.Trace("Setting up dae netns")
 
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
-	hostNetns, err := netns.Get()
-	if err != nil {
-		return 0, fmt.Errorf("Failed to get host netns: %v", err)
+	if ns.hostNs, err = netns.Get(); err != nil {
+		return fmt.Errorf("Failed to get host netns: %v", err)
 	}
-	defer netns.Set(hostNetns)
+	defer netns.Set(ns.hostNs)
 
-	// ip netns a daens
-	DeleteNamedNetns("daens")
-	ns, err = netns.NewNamed("daens")
-	if err != nil {
-		return 0, fmt.Errorf("Failed to create netns: %v", err)
+	if err = ns.setupVeth(); err != nil {
+		return
+	}
+	if err = ns.setupSysctl(); err != nil {
+		return
+	}
+	if err = ns.setupNetns(); err != nil {
+		return
 	}
-	if err = netns.Set(hostNetns); err != nil {
-		return 0, fmt.Errorf("Failed to switch to host netns: %v", err)
+	if err = ns.setupIPv4Datapath(); err != nil {
+		return
+	}
+	if err = ns.setupIPv6Datapath(); err != nil {
+		return
 	}
+	go ns.monitorDae0LinkAddr()
+	return
+}
+
+func (ns *DaeNetns) setupVeth() (err error) {
 	// ip l a dae0 type veth peer name dae0peer
-	DeleteLink("dae0")
+	DeleteLink(HostVethName)
 	if err = netlink.LinkAdd(&netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{
-			Name: "dae0",
+			Name: HostVethName,
 		},
-		PeerName: "dae0peer",
+		PeerName: NsVethName,
 	}); err != nil {
-		return 0, fmt.Errorf("Failed to add veth pair: %v", err)
+		return fmt.Errorf("Failed to add veth pair: %v", err)
 	}
-	dae0, err := netlink.LinkByName("dae0")
-	if err != nil {
-		return 0, fmt.Errorf("Failed to get link dae0: %v", err)
+	if ns.dae0, err = netlink.LinkByName(HostVethName); err != nil {
+		return fmt.Errorf("Failed to get link dae0: %v", err)
 	}
-	dae0peer, err := netlink.LinkByName("dae0peer")
-	if err != nil {
-		return 0, fmt.Errorf("Failed to get link dae0peer: %v", err)
+	if ns.dae0peer, err = netlink.LinkByName(NsVethName); err != nil {
+		return fmt.Errorf("Failed to get link dae0peer: %v", err)
 	}
 	// ip l s dae0 up
-	if err = netlink.LinkSetUp(dae0); err != nil {
-		return 0, fmt.Errorf("Failed to set link dae0 up: %v", err)
+	if err = netlink.LinkSetUp(ns.dae0); err != nil {
+		return fmt.Errorf("Failed to set link dae0 up: %v", err)
 	}
+	return
+}
+
+func (ns *DaeNetns) setupSysctl() (err error) {
 	// sysctl net.ipv4.conf.dae0.rp_filter=0
-	if err = SetRpFilter("dae0", "0"); err != nil {
-		return 0, fmt.Errorf("Failed to set rp_filter for dae0: %v", err)
+	if err = SetRpFilter(HostVethName, "0"); err != nil {
+		return fmt.Errorf("Failed to set rp_filter for dae0: %v", err)
 	}
 	// sysctl net.ipv4.conf.all.rp_filter=0
 	if err = SetRpFilter("all", "0"); err != nil {
-		return 0, fmt.Errorf("Failed to set rp_filter for all: %v", err)
+		return fmt.Errorf("Failed to set rp_filter for all: %v", err)
 	}
 	// sysctl net.ipv4.conf.dae0.arp_filter=0
-	if err = SetArpFilter("dae0", "0"); err != nil {
-		return 0, fmt.Errorf("Failed to set arp_filter for dae0: %v", err)
+	if err = SetArpFilter(HostVethName, "0"); err != nil {
+		return fmt.Errorf("Failed to set arp_filter for dae0: %v", err)
 	}
 	// sysctl net.ipv4.conf.all.arp_filter=0
 	if err = SetArpFilter("all", "0"); err != nil {
-		return 0, fmt.Errorf("Failed to set arp_filter for all: %v", err)
+		return fmt.Errorf("Failed to set arp_filter for all: %v", err)
 	}
 	// sysctl net.ipv4.conf.dae0.accept_local=1
-	if err = SetAcceptLocal("dae0", "1"); err != nil {
-		return 0, fmt.Errorf("Failed to set accept_local for dae0: %v", err)
+	if err = SetAcceptLocal(HostVethName, "1"); err != nil {
+		return fmt.Errorf("Failed to set accept_local for dae0: %v", err)
 	}
 	// sysctl net.ipv6.conf.dae0.disable_ipv6=0
-	if err = SetDisableIpv6("dae0", "0"); err != nil {
-		return 0, fmt.Errorf("Failed to set disable_ipv6 for dae0: %v", err)
+	if err = SetDisableIpv6(HostVethName, "0"); err != nil {
+		return fmt.Errorf("Failed to set disable_ipv6 for dae0: %v", err)
 	}
 	// sysctl net.ipv6.conf.dae0.forwarding=1
-	SetForwarding("dae0", "1")
+	SetForwarding(HostVethName, "1")
 	// sysctl net.ipv6.conf.all.forwarding=1
 	SetForwarding("all", "1")
-	// ip -6 a a fe80::ecee:eeff:feee:eeee dev dae0 scope link
-	if err = netlink.AddrAdd(dae0, &netlink.Addr{
-		IPNet: &net.IPNet{
-			IP:   net.ParseIP("fe80::ecee:eeff:feee:eeee"),
-			Mask: net.CIDRMask(128, 128),
-		},
-	}); err != nil {
-		return 0, fmt.Errorf("Failed to add v6 addr to dae0: %v", err)
+	return
+}
+
+func (ns *DaeNetns) setupNetns() (err error) {
+	// ip netns a daens
+	DeleteNamedNetns(NsName)
+	ns.daeNs, err = netns.NewNamed(NsName)
+	if err != nil {
+		return fmt.Errorf("Failed to create netns: %v", err)
+	}
+	// NewNamed() will switch to the new netns, switch back to host netns
+	if err = netns.Set(ns.hostNs); err != nil {
+		return fmt.Errorf("Failed to switch to host netns: %v", err)
 	}
 	// ip l s dae0peer netns daens
-	if err = netlink.LinkSetNsFd(dae0peer, int(ns)); err != nil {
-		return 0, fmt.Errorf("Failed to move dae0peer to daens: %v", err)
+	if err = netlink.LinkSetNsFd(ns.dae0peer, int(ns.daeNs)); err != nil {
+		return fmt.Errorf("Failed to move dae0peer to daens: %v", err)
 	}
-	// ip net e daens
-	if err = netns.Set(ns); err != nil {
-		return 0, fmt.Errorf("Failed to switch to daens: %v", err)
+	return
+}
+
+func (ns *DaeNetns) setupIPv4Datapath() (err error) {
+	if err = netns.Set(ns.daeNs); err != nil {
+		return fmt.Errorf("Failed to switch to daens: %v", err)
 	}
+	defer netns.Set(ns.hostNs)
+
 	// (ip net e daens) ip l s dae0peer up
-	if err = netlink.LinkSetUp(dae0peer); err != nil {
-		return 0, fmt.Errorf("Failed to set link dae0peer up: %v", err)
+	if err = netlink.LinkSetUp(ns.dae0peer); err != nil {
+		return fmt.Errorf("Failed to set link dae0peer up: %v", err)
 	}
 	// (ip net e daens) ip a a 169.254.0.11 dev dae0peer
+	// Although transparent UDP socket doesn't use this IP, it's still needed to make proper L3 header
 	ip, ipNet, err := net.ParseCIDR("169.254.0.11/32")
 	ipNet.IP = ip
 	if err != nil {
-		return 0, fmt.Errorf("Failed to parse ip: %v", err)
+		return fmt.Errorf("Failed to parse ip 169.254.0.11: %v", err)
 	}
-	if err = netlink.AddrAdd(dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
-		return 0, fmt.Errorf("Failed to add v4 addr to dae0peer: %v", err)
+	if err = netlink.AddrAdd(ns.dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
+		return fmt.Errorf("Failed to add v4 addr to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a 169.254.0.1 dev dae0peer
+	// 169.254.0.1 is the link-local address used for ARP caching
 	if err = netlink.RouteAdd(&netlink.Route{
-		LinkIndex: dae0peer.Attrs().Index,
+		LinkIndex: ns.dae0peer.Attrs().Index,
 		Dst:       &net.IPNet{IP: net.ParseIP("169.254.0.1"), Mask: net.CIDRMask(32, 32)},
 		Gw:        nil,
 		Scope:     netlink.SCOPE_LINK,
 	}); err != nil {
-		return 0, fmt.Errorf("Failed to add v4 route1 to dae0peer: %v", err)
+		return fmt.Errorf("Failed to add v4 route1 to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a default via 169.254.0.1 dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
-		LinkIndex: dae0peer.Attrs().Index,
+		LinkIndex: ns.dae0peer.Attrs().Index,
 		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
 		Gw:        net.ParseIP("169.254.0.1"),
 	}); err != nil {
-		return 0, fmt.Errorf("Failed to add v4 route2 to dae0peer: %v", err)
+		return fmt.Errorf("Failed to add v4 route2 to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip n r 169.254.0.1 dev dae0peer lladdr $mac_dae0 nud permanent
-	if err = netlink.NeighAdd(&netlink.Neigh{
-		IP:           net.ParseIP("169.254.0.1"),
-		HardwareAddr: dae0.Attrs().HardwareAddr,
-		LinkIndex:    dae0peer.Attrs().Index,
-		State:        netlink.NUD_PERMANENT,
+	return
+}
+
+func (ns *DaeNetns) setupIPv6Datapath() (err error) {
+	// ip -6 a a fe80::ecee:eeff:feee:eeee/128 dev dae0 scope link
+	// fe80::ecee:eeff:feee:eeee/128 is the link-local address used for L2 NDP addressing
+	if err = netlink.AddrAdd(ns.dae0, &netlink.Addr{
+		IPNet: &net.IPNet{
+			IP:   net.ParseIP("fe80::ecee:eeff:feee:eeee"),
+			Mask: net.CIDRMask(128, 128),
+		},
 	}); err != nil {
-		return 0, fmt.Errorf("Failed to add neigh to dae0peer: %v", err)
+		return fmt.Errorf("Failed to add v6 addr to dae0: %v", err)
 	}
+
+	if err = netns.Set(ns.daeNs); err != nil {
+		return fmt.Errorf("Failed to switch to daens: %v", err)
+	}
+	defer netns.Set(ns.hostNs)
+
 	// (ip net e daens) ip -6 r a default via fe80::ecee:eeff:feee:eeee dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
-		LinkIndex: dae0peer.Attrs().Index,
+		LinkIndex: ns.dae0peer.Attrs().Index,
 		Dst:       &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)},
 		Gw:        net.ParseIP("fe80::ecee:eeff:feee:eeee"),
 	}); err != nil {
-		return 0, fmt.Errorf("Failed to add v6 route to dae0peer: %v", err)
+		return fmt.Errorf("Failed to add v6 route to dae0peer: %v", err)
+	}
+	return
+}
+
+// updateNeigh() isn't named as setupNeigh() because it requires runtime.LockOSThread()
+func (ns *DaeNetns) updateNeigh() (err error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	if err = netns.Set(ns.daeNs); err != nil {
+		return fmt.Errorf("Failed to switch to daens: %v", err)
 	}
+	defer netns.Set(ns.hostNs)
 
+	if err = netlink.NeighSet(&netlink.Neigh{
+		IP:           net.ParseIP("169.254.0.1"),
+		HardwareAddr: ns.dae0.Attrs().HardwareAddr,
+		LinkIndex:    ns.dae0peer.Attrs().Index,
+		State:        netlink.NUD_PERMANENT,
+	}); err != nil {
+		return fmt.Errorf("Failed to add neigh to dae0peer: %v", err)
+	}
 	return
 }
 
+func (ns *DaeNetns) monitorDae0LinkAddr() {
+	ch := make(chan netlink.LinkUpdate)
+	done := make(chan struct{})
+	defer close(done)
+
+	err := netlink.LinkSubscribe(ch, done)
+	if err != nil {
+		logrus.Errorf("Failed to subscribe link updates: %v", err)
+	}
+	if err = ns.updateNeigh(); err != nil {
+		logrus.Errorf("Failed to update neigh: %v", err)
+	}
+	for msg := range ch {
+		if msg.Link.Attrs().Name == HostVethName && !bytes.Equal(msg.Link.Attrs().HardwareAddr, ns.dae0.Attrs().HardwareAddr) {
+			logrus.WithField("old addr", ns.dae0.Attrs().HardwareAddr).WithField("new addr", msg.Link.Attrs().HardwareAddr).Info("dae0 link addr changed")
+			ns.dae0 = msg.Link
+			ns.updateNeigh()
+		}
+	}
+}
+
 func DeleteNamedNetns(name string) error {
 	namedPath := path.Join("/run/netns", name)
 	unix.Unmount(namedPath, unix.MNT_DETACH|unix.MNT_FORCE)
diff --git a/control/udp.go b/control/udp.go
index 44073c0d6..9224172ae 100644
--- a/control/udp.go
+++ b/control/udp.go
@@ -95,7 +95,7 @@ func sendPkt(data []byte, from netip.AddrPort, realTo, to netip.AddrPort, lConn
 			WithField("to", to).
 			WithField("realTo", realTo).
 			Trace("Port in use, fallback to use netns.")
-		err = WithDaeNetns(func() (err error) {
+		err = daeNetns.With(func() (err error) {
 			uConn, _, err = DefaultAnyfromPool.GetOrCreate(from.String(), AnyfromTimeout)
 			return err
 		})

From a3a194070160fed04d7ebbd0776bff4413fdf077 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Wed, 10 Jan 2024 19:34:01 +0800
Subject: [PATCH 10/13] Control concurrency using sync.Mutex

---
 control/netns_utils.go | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index abb6e3792..eb736c742 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -8,6 +8,7 @@ import (
 	"path"
 	"runtime"
 	"sync"
+	"sync/atomic"
 
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -26,7 +27,8 @@ var (
 )
 
 type DaeNetns struct {
-	once sync.Once
+	setupDone atomic.Bool
+	mu        sync.Mutex
 
 	dae0, dae0peer netlink.Link
 	hostNs, daeNs  netns.NsHandle
@@ -37,16 +39,20 @@ func init() {
 }
 
 func (ns *DaeNetns) Setup() (err error) {
-	if ns.daeNs != 0 && ns.hostNs != 0 {
-		return nil
+	if ns.setupDone.Load() {
+		return
 	}
 
-	ns.once.Do(func() {
-		if err = ns.setup(); err != nil {
-			logrus.Fatal("Failed to setup dae netns: %v", err)
-		}
-	})
-	return err
+	ns.mu.Lock()
+	defer ns.mu.Unlock()
+	if ns.setupDone.Load() {
+		return
+	}
+	if err = ns.setup(); err != nil {
+		return
+	}
+	ns.setupDone.Store(true)
+	return nil
 }
 
 func (ns *DaeNetns) With(f func() error) (err error) {

From 0e0fe1ebd7984a304ff81141418b9df8dfa5d39d Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Wed, 10 Jan 2024 19:53:22 +0800
Subject: [PATCH 11/13] minor: adjust code style

---
 control/netns_utils.go | 62 +++++++++++++++++++++---------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index eb736c742..529d49f7f 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -57,31 +57,31 @@ func (ns *DaeNetns) Setup() (err error) {
 
 func (ns *DaeNetns) With(f func() error) (err error) {
 	if err = daeNetns.Setup(); err != nil {
-		return fmt.Errorf("Failed to setup dae netns: %v", err)
+		return fmt.Errorf("failed to setup dae netns: %v", err)
 	}
 
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
 	if err = netns.Set(ns.daeNs); err != nil {
-		return fmt.Errorf("Failed to switch to daens: %v", err)
+		return fmt.Errorf("failed to switch to daens: %v", err)
 	}
 	defer netns.Set(ns.hostNs)
 
 	if err = f(); err != nil {
-		return fmt.Errorf("Failed to run func in dae netns: %v", err)
+		return fmt.Errorf("failed to run func in dae netns: %v", err)
 	}
 	return
 }
 
 func (ns *DaeNetns) setup() (err error) {
-	logrus.Trace("Setting up dae netns")
+	logrus.Trace("setting up dae netns")
 
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
 	if ns.hostNs, err = netns.Get(); err != nil {
-		return fmt.Errorf("Failed to get host netns: %v", err)
+		return fmt.Errorf("failed to get host netns: %v", err)
 	}
 	defer netns.Set(ns.hostNs)
 
@@ -113,17 +113,17 @@ func (ns *DaeNetns) setupVeth() (err error) {
 		},
 		PeerName: NsVethName,
 	}); err != nil {
-		return fmt.Errorf("Failed to add veth pair: %v", err)
+		return fmt.Errorf("failed to add veth pair: %v", err)
 	}
 	if ns.dae0, err = netlink.LinkByName(HostVethName); err != nil {
-		return fmt.Errorf("Failed to get link dae0: %v", err)
+		return fmt.Errorf("failed to get link dae0: %v", err)
 	}
 	if ns.dae0peer, err = netlink.LinkByName(NsVethName); err != nil {
-		return fmt.Errorf("Failed to get link dae0peer: %v", err)
+		return fmt.Errorf("failed to get link dae0peer: %v", err)
 	}
 	// ip l s dae0 up
 	if err = netlink.LinkSetUp(ns.dae0); err != nil {
-		return fmt.Errorf("Failed to set link dae0 up: %v", err)
+		return fmt.Errorf("failed to set link dae0 up: %v", err)
 	}
 	return
 }
@@ -131,27 +131,27 @@ func (ns *DaeNetns) setupVeth() (err error) {
 func (ns *DaeNetns) setupSysctl() (err error) {
 	// sysctl net.ipv4.conf.dae0.rp_filter=0
 	if err = SetRpFilter(HostVethName, "0"); err != nil {
-		return fmt.Errorf("Failed to set rp_filter for dae0: %v", err)
+		return fmt.Errorf("failed to set rp_filter for dae0: %v", err)
 	}
 	// sysctl net.ipv4.conf.all.rp_filter=0
 	if err = SetRpFilter("all", "0"); err != nil {
-		return fmt.Errorf("Failed to set rp_filter for all: %v", err)
+		return fmt.Errorf("failed to set rp_filter for all: %v", err)
 	}
 	// sysctl net.ipv4.conf.dae0.arp_filter=0
 	if err = SetArpFilter(HostVethName, "0"); err != nil {
-		return fmt.Errorf("Failed to set arp_filter for dae0: %v", err)
+		return fmt.Errorf("failed to set arp_filter for dae0: %v", err)
 	}
 	// sysctl net.ipv4.conf.all.arp_filter=0
 	if err = SetArpFilter("all", "0"); err != nil {
-		return fmt.Errorf("Failed to set arp_filter for all: %v", err)
+		return fmt.Errorf("failed to set arp_filter for all: %v", err)
 	}
 	// sysctl net.ipv4.conf.dae0.accept_local=1
 	if err = SetAcceptLocal(HostVethName, "1"); err != nil {
-		return fmt.Errorf("Failed to set accept_local for dae0: %v", err)
+		return fmt.Errorf("failed to set accept_local for dae0: %v", err)
 	}
 	// sysctl net.ipv6.conf.dae0.disable_ipv6=0
 	if err = SetDisableIpv6(HostVethName, "0"); err != nil {
-		return fmt.Errorf("Failed to set disable_ipv6 for dae0: %v", err)
+		return fmt.Errorf("failed to set disable_ipv6 for dae0: %v", err)
 	}
 	// sysctl net.ipv6.conf.dae0.forwarding=1
 	SetForwarding(HostVethName, "1")
@@ -165,38 +165,38 @@ func (ns *DaeNetns) setupNetns() (err error) {
 	DeleteNamedNetns(NsName)
 	ns.daeNs, err = netns.NewNamed(NsName)
 	if err != nil {
-		return fmt.Errorf("Failed to create netns: %v", err)
+		return fmt.Errorf("failed to create netns: %v", err)
 	}
 	// NewNamed() will switch to the new netns, switch back to host netns
 	if err = netns.Set(ns.hostNs); err != nil {
-		return fmt.Errorf("Failed to switch to host netns: %v", err)
+		return fmt.Errorf("failed to switch to host netns: %v", err)
 	}
 	// ip l s dae0peer netns daens
 	if err = netlink.LinkSetNsFd(ns.dae0peer, int(ns.daeNs)); err != nil {
-		return fmt.Errorf("Failed to move dae0peer to daens: %v", err)
+		return fmt.Errorf("failed to move dae0peer to daens: %v", err)
 	}
 	return
 }
 
 func (ns *DaeNetns) setupIPv4Datapath() (err error) {
 	if err = netns.Set(ns.daeNs); err != nil {
-		return fmt.Errorf("Failed to switch to daens: %v", err)
+		return fmt.Errorf("failed to switch to daens: %v", err)
 	}
 	defer netns.Set(ns.hostNs)
 
 	// (ip net e daens) ip l s dae0peer up
 	if err = netlink.LinkSetUp(ns.dae0peer); err != nil {
-		return fmt.Errorf("Failed to set link dae0peer up: %v", err)
+		return fmt.Errorf("failed to set link dae0peer up: %v", err)
 	}
 	// (ip net e daens) ip a a 169.254.0.11 dev dae0peer
 	// Although transparent UDP socket doesn't use this IP, it's still needed to make proper L3 header
 	ip, ipNet, err := net.ParseCIDR("169.254.0.11/32")
 	ipNet.IP = ip
 	if err != nil {
-		return fmt.Errorf("Failed to parse ip 169.254.0.11: %v", err)
+		return fmt.Errorf("failed to parse ip 169.254.0.11: %v", err)
 	}
 	if err = netlink.AddrAdd(ns.dae0peer, &netlink.Addr{IPNet: ipNet}); err != nil {
-		return fmt.Errorf("Failed to add v4 addr to dae0peer: %v", err)
+		return fmt.Errorf("failed to add v4 addr to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a 169.254.0.1 dev dae0peer
 	// 169.254.0.1 is the link-local address used for ARP caching
@@ -206,7 +206,7 @@ func (ns *DaeNetns) setupIPv4Datapath() (err error) {
 		Gw:        nil,
 		Scope:     netlink.SCOPE_LINK,
 	}); err != nil {
-		return fmt.Errorf("Failed to add v4 route1 to dae0peer: %v", err)
+		return fmt.Errorf("failed to add v4 route1 to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip r a default via 169.254.0.1 dev dae0peer
 	if err = netlink.RouteAdd(&netlink.Route{
@@ -214,7 +214,7 @@ func (ns *DaeNetns) setupIPv4Datapath() (err error) {
 		Dst:       &net.IPNet{IP: net.IPv4(0, 0, 0, 0), Mask: net.CIDRMask(0, 32)},
 		Gw:        net.ParseIP("169.254.0.1"),
 	}); err != nil {
-		return fmt.Errorf("Failed to add v4 route2 to dae0peer: %v", err)
+		return fmt.Errorf("failed to add v4 route2 to dae0peer: %v", err)
 	}
 	// (ip net e daens) ip n r 169.254.0.1 dev dae0peer lladdr $mac_dae0 nud permanent
 	return
@@ -229,11 +229,11 @@ func (ns *DaeNetns) setupIPv6Datapath() (err error) {
 			Mask: net.CIDRMask(128, 128),
 		},
 	}); err != nil {
-		return fmt.Errorf("Failed to add v6 addr to dae0: %v", err)
+		return fmt.Errorf("failed to add v6 addr to dae0: %v", err)
 	}
 
 	if err = netns.Set(ns.daeNs); err != nil {
-		return fmt.Errorf("Failed to switch to daens: %v", err)
+		return fmt.Errorf("failed to switch to daens: %v", err)
 	}
 	defer netns.Set(ns.hostNs)
 
@@ -243,7 +243,7 @@ func (ns *DaeNetns) setupIPv6Datapath() (err error) {
 		Dst:       &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)},
 		Gw:        net.ParseIP("fe80::ecee:eeff:feee:eeee"),
 	}); err != nil {
-		return fmt.Errorf("Failed to add v6 route to dae0peer: %v", err)
+		return fmt.Errorf("failed to add v6 route to dae0peer: %v", err)
 	}
 	return
 }
@@ -254,7 +254,7 @@ func (ns *DaeNetns) updateNeigh() (err error) {
 	defer runtime.UnlockOSThread()
 
 	if err = netns.Set(ns.daeNs); err != nil {
-		return fmt.Errorf("Failed to switch to daens: %v", err)
+		return fmt.Errorf("failed to switch to daens: %v", err)
 	}
 	defer netns.Set(ns.hostNs)
 
@@ -264,7 +264,7 @@ func (ns *DaeNetns) updateNeigh() (err error) {
 		LinkIndex:    ns.dae0peer.Attrs().Index,
 		State:        netlink.NUD_PERMANENT,
 	}); err != nil {
-		return fmt.Errorf("Failed to add neigh to dae0peer: %v", err)
+		return fmt.Errorf("failed to add neigh to dae0peer: %v", err)
 	}
 	return
 }
@@ -276,10 +276,10 @@ func (ns *DaeNetns) monitorDae0LinkAddr() {
 
 	err := netlink.LinkSubscribe(ch, done)
 	if err != nil {
-		logrus.Errorf("Failed to subscribe link updates: %v", err)
+		logrus.Errorf("failed to subscribe link updates: %v", err)
 	}
 	if err = ns.updateNeigh(); err != nil {
-		logrus.Errorf("Failed to update neigh: %v", err)
+		logrus.Errorf("failed to update neigh: %v", err)
 	}
 	for msg := range ch {
 		if msg.Link.Attrs().Name == HostVethName && !bytes.Equal(msg.Link.Attrs().HardwareAddr, ns.dae0.Attrs().HardwareAddr) {

From 52252eb0c0a54edf19ee5947af38c263d1a39460 Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Wed, 10 Jan 2024 20:45:27 +0800
Subject: [PATCH 12/13] Cleanup daens and veth for graceful termination

---
 cmd/run.go             |  3 ++-
 control/netns_utils.go | 10 ++++++++++
 control/udp.go         |  2 +-
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/cmd/run.go b/cmd/run.go
index c56d42220..c3e24d963 100644
--- a/cmd/run.go
+++ b/cmd/run.go
@@ -268,10 +268,11 @@ loop:
 			break loop
 		}
 	}
+	defer os.Remove(PidFilePath)
+	defer control.GetDaeNetns().Close()
 	if e := c.Close(); e != nil {
 		return fmt.Errorf("close control plane: %w", e)
 	}
-	_ = os.Remove(PidFilePath)
 	return nil
 }
 
diff --git a/control/netns_utils.go b/control/netns_utils.go
index 529d49f7f..40ea527fc 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -38,6 +38,10 @@ func init() {
 	daeNetns = &DaeNetns{}
 }
 
+func GetDaeNetns() *DaeNetns {
+	return daeNetns
+}
+
 func (ns *DaeNetns) Setup() (err error) {
 	if ns.setupDone.Load() {
 		return
@@ -55,6 +59,12 @@ func (ns *DaeNetns) Setup() (err error) {
 	return nil
 }
 
+func (ns *DaeNetns) Close() (err error) {
+	DeleteNamedNetns(NsName)
+	DeleteLink(HostVethName)
+	return
+}
+
 func (ns *DaeNetns) With(f func() error) (err error) {
 	if err = daeNetns.Setup(); err != nil {
 		return fmt.Errorf("failed to setup dae netns: %v", err)
diff --git a/control/udp.go b/control/udp.go
index 9224172ae..54f07ade8 100644
--- a/control/udp.go
+++ b/control/udp.go
@@ -95,7 +95,7 @@ func sendPkt(data []byte, from netip.AddrPort, realTo, to netip.AddrPort, lConn
 			WithField("to", to).
 			WithField("realTo", realTo).
 			Trace("Port in use, fallback to use netns.")
-		err = daeNetns.With(func() (err error) {
+		err = GetDaeNetns().With(func() (err error) {
 			uConn, _, err = DefaultAnyfromPool.GetOrCreate(from.String(), AnyfromTimeout)
 			return err
 		})

From b3f6ea41e4e9f47b9ed26d3e6fb442cb4fad79ff Mon Sep 17 00:00:00 2001
From: Gray Liang <gray.liang@isovalent.com>
Date: Thu, 11 Jan 2024 21:16:02 +0800
Subject: [PATCH 13/13] Subscribe link events properly

---
 control/netns_utils.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/control/netns_utils.go b/control/netns_utils.go
index 40ea527fc..a33f5fbf8 100644
--- a/control/netns_utils.go
+++ b/control/netns_utils.go
@@ -288,6 +288,9 @@ func (ns *DaeNetns) monitorDae0LinkAddr() {
 	if err != nil {
 		logrus.Errorf("failed to subscribe link updates: %v", err)
 	}
+	if ns.dae0, err = netlink.LinkByName(HostVethName); err != nil {
+		logrus.Errorf("failed to get link dae0: %v", err)
+	}
 	if err = ns.updateNeigh(); err != nil {
 		logrus.Errorf("failed to update neigh: %v", err)
 	}