From 4817ac0c5a8ac749aaff87c0a8541e17d2283908 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 3 Jan 2024 23:03:56 +0900 Subject: [PATCH 1/5] Initial support for ipvs-compat --- api/loxinlp/nlp.go | 6 +++++- go.mod | 4 +++- go.sum | 8 ++++++++ loxinet/loxinet.go | 2 +- options/options.go | 1 + 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/api/loxinlp/nlp.go b/api/loxinlp/nlp.go index ee25862cb..420e3553b 100644 --- a/api/loxinlp/nlp.go +++ b/api/loxinlp/nlp.go @@ -1536,7 +1536,7 @@ func LbSessionGet(done bool) int { return 0 } -func NlpInit(bgpPeerMode bool, blackList string) *NlH { +func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH { nNl = new(NlH) @@ -1601,5 +1601,9 @@ func NlpInit(bgpPeerMode bool, blackList string) *NlH { go LbSessionGet(done) + if ipvsCompat { + IpVSInit() + } + return nNl } diff --git a/go.mod b/go.mod index 8da5159d5..358dfdae8 100644 --- a/go.mod +++ b/go.mod @@ -25,10 +25,12 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/loxilb-io/ipvs v0.1.0 // indirect github.com/loxilb-io/sctp v0.0.0-20230519081703-6d1baec82fd4 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect ) require ( @@ -44,7 +46,7 @@ require ( github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/prometheus/client_golang v1.14.0 - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + github.com/vishvananda/netns v0.0.2 // indirect go.mongodb.org/mongo-driver v1.11.6 // indirect golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect golang.org/x/text v0.13.0 // indirect diff --git a/go.sum b/go.sum index d5b46893c..87c0ef274 100644 --- a/go.sum +++ b/go.sum @@ -233,6 +233,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/loxilb-io/ipvs v0.1.0 h1:TpTkwh5CLgJ7YW86rvWyqJPEpQFqs2TNbRG/IECeq+w= +github.com/loxilb-io/ipvs v0.1.0/go.mod h1:EKjimnzyVL9AXMMNfPWeokxF1uNeuDrEGF5gPFMdmIo= github.com/loxilb-io/loxilib v0.8.8 h1:nW6RvLXDQxr5Pe2Ygg7qyYm4QG5y5cG+/jQ4m/DckP4= github.com/loxilb-io/loxilib v0.8.8/go.mod h1:LoQCxBz+N0fO9rGwRmPHrQPHol/jUf4MNpph63Cydkg= github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f h1:q745LqB2G97bOpf6MCa34VyNJEuvVx4rTFVut3kz9oo= @@ -309,6 +311,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -328,6 +332,8 @@ github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA85 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.2 h1:Cn05BRLm+iRP/DZxyVSsfVyrzgjDbwHwkVt38qvXnNI= +github.com/vishvananda/netns v0.0.2/go.mod h1:yitZXdAVI+yPFSb4QUe+VW3vOVl4PZPNcBgbPxAtJxw= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -489,6 +495,8 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/loxinet/loxinet.go b/loxinet/loxinet.go index 352991644..89a55c970 100644 --- a/loxinet/loxinet.go +++ b/loxinet/loxinet.go @@ -277,7 +277,7 @@ func loxiNetInit() { // Initialize the nlp subsystem if !opts.Opts.NoNlp { nlp.NlpRegister(NetAPIInit(opts.Opts.BgpPeerMode)) - nlp.NlpInit(opts.Opts.BgpPeerMode, opts.Opts.BlackList) + nlp.NlpInit(opts.Opts.BgpPeerMode, opts.Opts.BlackList, opts.Opts.IPVSCompat) } // Initialize the Prometheus subsystem diff --git a/options/options.go b/options/options.go index 875c4c345..aab36230e 100644 --- a/options/options.go +++ b/options/options.go @@ -28,4 +28,5 @@ var Opts struct { BgpPeerMode bool `short:"r" long:"peer" description:"Run loxilb with goBGP only, no Datapath"` BlackList string `long:"blacklist" description:"Regex string of blacklisted ports" default:"none"` Rpc string `long:"rpc" description:"RPC mode for syncing - netrpc or grpc" default:"netrpc"` + IPVSCompat bool `long:"ipvs-compat" description:"Enable ipvs-compat(experimental)"` } From 84a1168ce3a01ff8800b0e364cd42cd3f62e1886 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 3 Jan 2024 23:41:23 +0900 Subject: [PATCH 2/5] Initial support for ipvs-compat --- .golangci.yml | 8 +++ api/loxinlp/ipvs.go | 167 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 .golangci.yml create mode 100644 api/loxinlp/ipvs.go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..22e099215 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,8 @@ +linters: + disable-all: true + enable: + - gofmt + - govet + - ineffassign + - misspell + - revive diff --git a/api/loxinlp/ipvs.go b/api/loxinlp/ipvs.go new file mode 100644 index 000000000..e2683c757 --- /dev/null +++ b/api/loxinlp/ipvs.go @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2023 NetLOX Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loxinlp + +import ( + "github.com/loxilb-io/ipvs" + cmn "github.com/loxilb-io/loxilb/common" + tk "github.com/loxilb-io/loxilib" + "os" + "reflect" + "time" +) + +type ipVSKey struct { + Address string + Protocol string + Port uint16 +} + +type ipvsEndPoint struct { + EpIP string + EpPort uint16 + Weight uint8 +} + +type ipVSEntry struct { + Key ipVSKey + Type string + InValid bool + EndPoints []ipvsEndPoint +} + +type IpVSH struct { + RMap map[ipVSKey]*ipVSEntry + ticker *time.Ticker + tDone chan bool + handle *ipvs.Handle +} + +var ipVSCtx *IpVSH + +func (ctx *IpVSH) BuildIpVSDB() []*ipVSEntry { + + var ipVSList []*ipVSEntry + svcs, err := ctx.handle.GetServices() + if err != nil { + tk.LogIt(tk.LogError, "[ipvs] failed to get services\n") + return nil + } + + for _, svc := range svcs { + var newEntry ipVSEntry + + endPoints, err := ctx.handle.GetDestinations(svc) + if err != nil { + continue + } + + newEntry.Type = svc.SchedName + if svc.SchedName != "rr" { + continue + } + + proto := "" + if svc.Protocol == 1 { + proto = "icmp" + } else if svc.Protocol == 6 { + proto = "tcp" + } else if svc.Protocol == 17 { + proto = "udp" + } else if svc.Protocol == 132 { + proto = "sctp" + } else { + continue + } + + key := ipVSKey{Address: svc.Address.String(), Protocol: proto, Port: svc.Port} + for _, endPoint := range endPoints { + newEntry.EndPoints = append(newEntry.EndPoints, ipvsEndPoint{EpIP: endPoint.Address.String(), EpPort: endPoint.Port, Weight: uint8(endPoint.Weight)}) + } + + if len(newEntry.EndPoints) != 0 { + if eEnt := ctx.RMap[key]; eEnt != nil { + if reflect.DeepEqual(eEnt.EndPoints, newEntry.EndPoints) { + eEnt.InValid = false + continue + } + } + + newEntry.Key = key + ipVSList = append(ipVSList, &newEntry) + } + } + return ipVSList +} + +func IpVSSync() { + for { + select { + case <-ipVSCtx.tDone: + return + case <-ipVSCtx.ticker.C: + + for _, ent := range ipVSCtx.RMap { + ent.InValid = true + } + + tk.LogIt(tk.LogInfo, "IPVS BuildDB\n") + ipVSList := ipVSCtx.BuildIpVSDB() + + for _, ent := range ipVSCtx.RMap { + if ent.InValid { + lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: ent.Key.Address, ServPort: ent.Key.Port, Proto: ent.Key.Protocol, Sel: cmn.LbSelRr}} + _, err := hooks.NetLbRuleDel(&lbrule) + if err != nil { + tk.LogIt(tk.LogError, "IPVS LB %v delete failed\n", ent.Key) + } + tk.LogIt(tk.LogInfo, "IPVS ent %v deleted\n", ent.Key) + delete(ipVSCtx.RMap, ent.Key) + } + } + + for _, newEnt := range ipVSList { + lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: newEnt.Key.Address, ServPort: newEnt.Key.Port, Proto: newEnt.Key.Protocol, Sel: cmn.LbSelRr}} + for _, ep := range newEnt.EndPoints { + lbrule.Eps = append(lbrule.Eps, cmn.LbEndPointArg{EpIP: ep.EpIP, EpPort: ep.EpPort, Weight: 1}) + } + + _, err := hooks.NetLbRuleAdd(&lbrule) + if err != nil { + tk.LogIt(tk.LogError, "IPVS LB %v add failed\n", newEnt.Key) + continue + } + ipVSCtx.RMap[newEnt.Key] = newEnt + tk.LogIt(tk.LogError, "IPVS ent %v added\n", newEnt.Key) + } + } + } +} + +func IpVSInit() { + ipVSCtx = new(IpVSH) + ipVSCtx.ticker = time.NewTicker(10 * time.Second) + ipVSCtx.RMap = make(map[ipVSKey]*ipVSEntry) + ipVSCtx.tDone = make(chan bool) + handle, err := ipvs.New("") + if err != nil { + tk.LogIt(tk.LogError, "ipvs.New: %s\n", err) + os.Exit(1) + } + ipVSCtx.handle = handle + go IpVSSync() +} From ead101a38f765992b7dd49d90a5c8c7983733b30 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 3 Jan 2024 23:47:13 +0900 Subject: [PATCH 3/5] Initial support for ipvs-compat --- api/loxinlp/ipvs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/loxinlp/ipvs.go b/api/loxinlp/ipvs.go index e2683c757..b2a63e06b 100644 --- a/api/loxinlp/ipvs.go +++ b/api/loxinlp/ipvs.go @@ -119,7 +119,6 @@ func IpVSSync() { ent.InValid = true } - tk.LogIt(tk.LogInfo, "IPVS BuildDB\n") ipVSList := ipVSCtx.BuildIpVSDB() for _, ent := range ipVSCtx.RMap { From 375f5c6c5dd1af240cb0f5a5990aa2fb8eba6313 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Fri, 5 Jan 2024 01:21:37 +0900 Subject: [PATCH 4/5] Initial support for ipvs-compat --- api/loxinlp/ipvs.go | 14 +++++++++----- api/loxinlp/nlp.go | 29 +++++++++++++++++++---------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/api/loxinlp/ipvs.go b/api/loxinlp/ipvs.go index b2a63e06b..ab5f950ab 100644 --- a/api/loxinlp/ipvs.go +++ b/api/loxinlp/ipvs.go @@ -17,12 +17,14 @@ package loxinlp import ( - "github.com/loxilb-io/ipvs" - cmn "github.com/loxilb-io/loxilb/common" - tk "github.com/loxilb-io/loxilib" + "fmt" "os" "reflect" "time" + + "github.com/loxilb-io/ipvs" + cmn "github.com/loxilb-io/loxilb/common" + tk "github.com/loxilb-io/loxilib" ) type ipVSKey struct { @@ -123,7 +125,8 @@ func IpVSSync() { for _, ent := range ipVSCtx.RMap { if ent.InValid { - lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: ent.Key.Address, ServPort: ent.Key.Port, Proto: ent.Key.Protocol, Sel: cmn.LbSelRr}} + name := fmt.Sprintf("ipvs_%s:%d-%s", ent.Key.Address, ent.Key.Port, ent.Key.Protocol) + lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: ent.Key.Address, ServPort: ent.Key.Port, Proto: ent.Key.Protocol, Sel: cmn.LbSelRr, Name: name}} _, err := hooks.NetLbRuleDel(&lbrule) if err != nil { tk.LogIt(tk.LogError, "IPVS LB %v delete failed\n", ent.Key) @@ -134,7 +137,8 @@ func IpVSSync() { } for _, newEnt := range ipVSList { - lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: newEnt.Key.Address, ServPort: newEnt.Key.Port, Proto: newEnt.Key.Protocol, Sel: cmn.LbSelRr}} + name := fmt.Sprintf("ipvs_%s:%d-%s", newEnt.Key.Address, newEnt.Key.Port, newEnt.Key.Protocol) + lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: newEnt.Key.Address, ServPort: newEnt.Key.Port, Proto: newEnt.Key.Protocol, Sel: cmn.LbSelRr, Name: name}} for _, ep := range newEnt.EndPoints { lbrule.Eps = append(lbrule.Eps, cmn.LbEndPointArg{EpIP: ep.EpIP, EpPort: ep.EpPort, Weight: 1}) } diff --git a/api/loxinlp/nlp.go b/api/loxinlp/nlp.go index 420e3553b..744f45f1e 100644 --- a/api/loxinlp/nlp.go +++ b/api/loxinlp/nlp.go @@ -19,7 +19,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net" "os" "os/exec" @@ -118,7 +117,7 @@ func applyLoadBalancerConfig() bool { var resp struct { Attr []cmn.LbRuleMod `json:"lbAttr"` } - byteBuf, err := ioutil.ReadFile("/etc/loxilb/lbconfig.txt") + byteBuf, err := os.ReadFile("/etc/loxilb/lbconfig.txt") if err != nil { fmt.Println(err.Error()) return false @@ -139,7 +138,7 @@ func applySessionConfig() bool { var resp struct { Attr []cmn.SessionMod `json:"sessionAttr"` } - byteBuf, err := ioutil.ReadFile("/etc/loxilb/sessionconfig.txt") + byteBuf, err := os.ReadFile("/etc/loxilb/sessionconfig.txt") if err != nil { fmt.Println(err.Error()) return false @@ -160,7 +159,7 @@ func applyUlClConfig() bool { var resp struct { Attr []cmn.SessionUlClMod `json:"ulclAttr"` } - byteBuf, err := ioutil.ReadFile("/etc/loxilb/sessionulclconfig.txt") + byteBuf, err := os.ReadFile("/etc/loxilb/sessionulclconfig.txt") if err != nil { fmt.Println(err.Error()) return false @@ -181,7 +180,7 @@ func applyFWConfig() bool { var resp struct { Attr []cmn.FwRuleMod `json:"fwAttr"` } - byteBuf, err := ioutil.ReadFile("/etc/loxilb/FWconfig.txt") + byteBuf, err := os.ReadFile("/etc/loxilb/FWconfig.txt") if err != nil { fmt.Println(err.Error()) return false @@ -202,7 +201,7 @@ func applyEPConfig() bool { var resp struct { Attr []cmn.EndPointMod `json:"Attr"` } - byteBuf, err := ioutil.ReadFile("/etc/loxilb/EPconfig.txt") + byteBuf, err := os.ReadFile("/etc/loxilb/EPconfig.txt") if err != nil { fmt.Println(err.Error()) return false @@ -719,8 +718,12 @@ func ModLink(link nlp.Link, add bool) int { vid, _ = strconv.Atoi(strings.Join(re.FindAllString(name, -1), " ")) // Dirty hack to support docker0 bridge - if vid == 0 && name == "docker0" { - vid = 4090 + if vid == 0 { + if name == "docker0" { + vid = 4090 + } else if name == "cni0" { + vid = 4091 + } } if add { ret, err = hooks.NetVlanAdd(&cmn.VlanMod{Vid: vid, Dev: name, LinkIndex: idx, @@ -739,6 +742,8 @@ func ModLink(link nlp.Link, add bool) int { if (add && (err != nil)) || !add { applyConfigMap(name, state, add) } + + return ret } /* Get bridge detail */ @@ -750,8 +755,12 @@ func ModLink(link nlp.Link, add bool) int { } vid, _ = strconv.Atoi(strings.Join(re.FindAllString(brLink.Attrs().Name, -1), " ")) // Dirty hack to support docker bridge - if vid == 0 && brLink.Attrs().Name == "docker0" { - vid = 4090 + if vid == 0 { + if brLink.Attrs().Name == "docker0" { + vid = 4090 + } else if brLink.Attrs().Name == "cni0" { + vid = 4091 + } } } From 320643ca494a75ecab01e274a1111d15ac38f526 Mon Sep 17 00:00:00 2001 From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com> Date: Fri, 5 Jan 2024 18:59:18 +0900 Subject: [PATCH 5/5] Update nlp.go --- api/loxinlp/nlp.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/loxinlp/nlp.go b/api/loxinlp/nlp.go index 744f45f1e..fef0f07c9 100644 --- a/api/loxinlp/nlp.go +++ b/api/loxinlp/nlp.go @@ -742,8 +742,6 @@ func ModLink(link nlp.Link, add bool) int { if (add && (err != nil)) || !add { applyConfigMap(name, state, add) } - - return ret } /* Get bridge detail */