diff --git a/.github/workflows/advanced-lb-sanity.yml b/.github/workflows/advanced-lb-sanity.yml index 48e5e38a2..23549a666 100644 --- a/.github/workflows/advanced-lb-sanity.yml +++ b/.github/workflows/advanced-lb-sanity.yml @@ -101,3 +101,15 @@ jobs: ./validation.sh ./rmconfig.sh cd - + - run: | + cd cicd/ipmasquerade/ + ./config.sh + ./validation.sh + ./rmconfig.sh + cd - + #- run: | + # cd cicd/httpsproxy/ + # ./config.sh + # ./validation.sh + # ./rmconfig.sh + # cd - diff --git a/README.md b/README.md index 3159fc449..d6263a003 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Additionally, loxilb can also support cluster-ip and node-port services, thereby loxilb works as a L4 load-balancer/service-proxy by default. Although it provides great performance, at times, L7 load-balancing might become necessary in K8s. There are many good L7 proxies already available for K8s. Still, we are working on providing a great L7 solution natively in eBPF. It is a tough endeavor one which should reap great benefits once completed. Please keep an eye for updates on this. ## Telco-Cloud with loxilb -For deploying telco-cloud with cloud-native functions, loxilb can be used as a SCP(service communication proxy). SCP is nothing but a communication hub for telco micro-services running in Kubernetes. But telco-cloud requires load-balancing and communication across various interfaces/standards like N2, N4, E2(ORAN), S6x, 5GLAN, GTP etc. Each of these present its own unique challenges which loxilb aims to solve e.g. +For deploying telco-cloud with cloud-native functions, loxilb can be used as a SCP(service communication proxy). SCP is a communication proxy defined by [3GPP](https://www.etsi.org/deliver/etsi_ts/129500_129599/129500/16.04.00_60/ts_129500v160400p.pdf) and aimed at telco micro-services running in cloud-native environment. Telco-cloud requires load-balancing and communication across various interfaces/standards like N2, N4, E2(ORAN), S6x, 5GLAN, GTP etc. Each of these present its own unique challenges which loxilb aims to solve e.g.: - N4 requires PFCP level session-intelligence - N2 requires NGAP parsing capability - S6x requires Diameter/SCTP multi-homing LB support @@ -66,20 +66,11 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - E2 might require SCTP-LB with OpenVPN bundled together - SIP support is needed to enable cloud-native VOIP -## How-To Guides -- [How-To : Deploy loxilb in K8s with kube-loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/kube-loxilb.md) -- [How-To : Run in K8s with in-cluster mode](https://www.loxilb.io/post/k8s-nuances-of-in-cluster-external-service-lb-with-loxilb) -- [How-To : High-availability with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ha-deploy.md) -- [How-To : Run loxilb in standalone mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/standalone.md) -- [How-To : Manual build/run](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/run.md) -- [How-To : Standalone configuration](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd.md) -- [How-To : Debug loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md) -- [How-To : Access end-points outside K8s](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ext-ep.md) -- [How-To : Deploy multi-server K3s HA with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s-multi-master.md) -- [How-To : Deploy loxilb with multi-AZ HA support in AWS](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/aws-multi-az.md) - -## Getting started with different K8s distributions/tools +## Architectural Considerations +- [Understanding loxilb modes and deployment in K8s with kube-loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/kube-loxilb.md) +- [Understanding High-availability with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ha-deploy.md) +## Getting Started #### loxilb as ext-cluster pod - [K3s : loxilb with default flannel](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s_quick_start_flannel.md) - [K3s : loxilb with calico](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s_quick_start_calico.md) @@ -97,6 +88,16 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - [K3s : loxilb service-proxy with flannel](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/service-proxy-flannel.md) - [K3s : loxilb service-proxy with calico](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/service-proxy-calico.md) +#### loxilb in standalone mode +- [Run loxilb standalone](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/standalone.md) + +## Advanced Guides +- [How-To : Service-group zones with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/service-zones.md) +- [How-To : Access end-points outside K8s](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ext-ep.md) +- [How-To : Deploy multi-server K3s HA with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s-multi-master.md) +- [How-To : Deploy loxilb with multi-AZ HA support in AWS](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/aws-multi-az.md) +- [How-To : Deploy loxilb with Ingress](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/loxilb-nginx-ingress.md) + ## Knowledge-Base - [What is eBPF](ebpf.md) - [What is k8s service - load-balancer](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/lb.md) @@ -105,6 +106,9 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - [eBPF internals of loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/loxilbebpf.md) - [What are loxilb NAT Modes](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/nat.md) - [loxilb load-balancer algorithms](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/lb-algo.md) +- [Manual steps to build/run](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/run.md) +- [Debugging loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md) +- [loxicmd command-line tool usage](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd.md) - [Developer's guide to loxicmd](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd-dev.md) - [Developer's guide to loxilb API](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/api-dev.md) - [API Reference - loxilb web-Api](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/api.md) diff --git a/api/models/firewall_option_entry.go b/api/models/firewall_option_entry.go index a3ac936fd..0a26bd90e 100644 --- a/api/models/firewall_option_entry.go +++ b/api/models/firewall_option_entry.go @@ -23,6 +23,9 @@ type FirewallOptionEntry struct { // traffic counters Counter string `json:"counter,omitempty"` + // Do SNAT on matching rule + DoSnat bool `json:"doSnat,omitempty"` + // Drop any matching rule Drop bool `json:"drop,omitempty"` @@ -38,6 +41,12 @@ type FirewallOptionEntry struct { // Redirect any matching rule RedirectPortName string `json:"redirectPortName,omitempty"` + // Modify to given IP in CIDR notation + ToIP string `json:"toIP,omitempty"` + + // Modify to given Port (Zero if port is not to be modified) + ToPort int64 `json:"toPort,omitempty"` + // Trap anything matching rule Trap bool `json:"trap,omitempty"` } diff --git a/api/models/loadbalance_entry.go b/api/models/loadbalance_entry.go index e55f0feff..79cb9f189 100644 --- a/api/models/loadbalance_entry.go +++ b/api/models/loadbalance_entry.go @@ -365,6 +365,9 @@ type LoadbalanceEntryServiceArguments struct { // value for load balance algorithim Sel int64 `json:"sel,omitempty"` + + // snat rule + Snat bool `json:"snat,omitempty"` } // Validate validates this loadbalance entry service arguments diff --git a/api/restapi/embedded_spec.go b/api/restapi/embedded_spec.go index 8782fe145..ffc261a15 100644 --- a/api/restapi/embedded_spec.go +++ b/api/restapi/embedded_spec.go @@ -4750,6 +4750,10 @@ func init() { "description": "traffic counters", "type": "string" }, + "doSnat": { + "description": "Do SNAT on matching rule", + "type": "boolean" + }, "drop": { "description": "Drop any matching rule", "type": "boolean" @@ -4770,6 +4774,14 @@ func init() { "description": "Redirect any matching rule", "type": "string" }, + "toIP": { + "description": "Modify to given IP in CIDR notation", + "type": "string" + }, + "toPort": { + "description": "Modify to given Port (Zero if port is not to be modified)", + "type": "integer" + }, "trap": { "description": "Trap anything matching rule", "type": "boolean" @@ -4981,6 +4993,10 @@ func init() { "sel": { "description": "value for load balance algorithim", "type": "integer" + }, + "snat": { + "description": "snat rule", + "type": "boolean" } } } @@ -10797,6 +10813,10 @@ func init() { "description": "traffic counters", "type": "string" }, + "doSnat": { + "description": "Do SNAT on matching rule", + "type": "boolean" + }, "drop": { "description": "Drop any matching rule", "type": "boolean" @@ -10817,6 +10837,14 @@ func init() { "description": "Redirect any matching rule", "type": "string" }, + "toIP": { + "description": "Modify to given IP in CIDR notation", + "type": "string" + }, + "toPort": { + "description": "Modify to given Port (Zero if port is not to be modified)", + "type": "integer" + }, "trap": { "description": "Trap anything matching rule", "type": "boolean" @@ -11002,6 +11030,10 @@ func init() { "sel": { "description": "value for load balance algorithim", "type": "integer" + }, + "snat": { + "description": "snat rule", + "type": "boolean" } } } @@ -11125,6 +11157,10 @@ func init() { "sel": { "description": "value for load balance algorithim", "type": "integer" + }, + "snat": { + "description": "snat rule", + "type": "boolean" } } }, diff --git a/api/restapi/handler/firewall.go b/api/restapi/handler/firewall.go index dae392677..d090bec0f 100644 --- a/api/restapi/handler/firewall.go +++ b/api/restapi/handler/firewall.go @@ -56,6 +56,9 @@ func ConfigPostFW(params operations.PostConfigFirewallParams) middleware.Respond Opts.Trap = params.Attr.Opts.Trap Opts.Record = params.Attr.Opts.Record Opts.Mark = uint32(params.Attr.Opts.FwMark) + Opts.DoSnat = params.Attr.Opts.DoSnat + Opts.ToIP = params.Attr.Opts.ToIP + Opts.ToPort = uint16(params.Attr.Opts.ToPort) FW.Rule = Rules FW.Opts = Opts @@ -156,6 +159,9 @@ func ConfigGetFW(params operations.GetConfigFirewallAllParams) middleware.Respon tmpOpts.Trap = FW.Opts.Trap tmpOpts.Record = FW.Opts.Record tmpOpts.FwMark = int64(FW.Opts.Mark) + tmpOpts.DoSnat = FW.Opts.DoSnat + tmpOpts.ToIP = FW.Opts.ToIP + tmpOpts.ToPort = int64(FW.Opts.ToPort) tmpOpts.Counter = FW.Opts.Counter tmpResult.RuleArguments = &tmpRule diff --git a/api/restapi/handler/loadbalancer.go b/api/restapi/handler/loadbalancer.go index 1001881b9..c1110b0ae 100644 --- a/api/restapi/handler/loadbalancer.go +++ b/api/restapi/handler/loadbalancer.go @@ -132,6 +132,7 @@ func ConfigGetLoadbalancer(params operations.GetConfigLoadbalancerAllParams) mid tmpSvc.Probetype = lb.Serv.ProbeType tmpSvc.Probeport = lb.Serv.ProbePort tmpSvc.Name = lb.Serv.Name + tmpSvc.Snat = lb.Serv.Snat tmpLB.ServiceArguments = &tmpSvc diff --git a/api/swagger.yml b/api/swagger.yml index 67667561d..adca60e68 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -2894,6 +2894,9 @@ definitions: name: type: string description: service name + snat: + type: boolean + description: snat rule oper: type: integer format: int32 @@ -3614,6 +3617,15 @@ definitions: fwMark: type: integer description: Set a fwmark for any matching rule + doSnat: + type: boolean + description: Do SNAT on matching rule + toIP: + type: string + description: Modify to given IP in CIDR notation + toPort: + type: integer + description: Modify to given Port (Zero if port is not to be modified) counter: type: string description: traffic counters @@ -3949,4 +3961,3 @@ definitions: type: integer format: uint8 description: Retry Count to detect failure - \ No newline at end of file diff --git a/cicd/httpsproxy/config.sh b/cicd/httpsproxy/config.sh new file mode 100755 index 000000000..0d3ce4b2d --- /dev/null +++ b/cicd/httpsproxy/config.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +source ../common.sh + +echo "#########################################" +echo "Spawning all hosts" +echo "#########################################" + +spawn_docker_host --dock-type loxilb --dock-name llb1 +spawn_docker_host --dock-type host --dock-name l3h1 +spawn_docker_host --dock-type host --dock-name l3ep1 +spawn_docker_host --dock-type host --dock-name l3ep2 +spawn_docker_host --dock-type host --dock-name l3ep3 + +echo "#########################################" +echo "Connecting and configuring hosts" +echo "#########################################" + + +connect_docker_hosts l3h1 llb1 +connect_docker_hosts l3ep1 llb1 +connect_docker_hosts l3ep2 llb1 +connect_docker_hosts l3ep3 llb1 + +sleep 5 + +#L3 config +config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254 +config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254 +config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254 +config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254 +config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24 +config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24 +config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24 +config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24 + +$dexec llb1 ip addr add 10.10.10.3/32 dev lo +./minica -ip-addresses 10.10.10.254 + +docker cp minica.pem llb1:/opt/loxilb/cert/rootCA.crt +docker cp 10.10.10.254/cert.pem llb1:/opt/loxilb/cert/server.crt +docker cp 10.10.10.254/key.pem llb1:/opt/loxilb/cert/server.key + +sleep 5 +create_lb_rule llb1 10.10.10.254 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullproxy --security=https diff --git a/cicd/httpsproxy/minica b/cicd/httpsproxy/minica new file mode 100755 index 000000000..a152b1664 Binary files /dev/null and b/cicd/httpsproxy/minica differ diff --git a/cicd/httpsproxy/rmconfig.sh b/cicd/httpsproxy/rmconfig.sh new file mode 100755 index 000000000..9ea80577f --- /dev/null +++ b/cicd/httpsproxy/rmconfig.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source ../common.sh + +disconnect_docker_hosts l3h1 llb1 +disconnect_docker_hosts l3ep1 llb1 +disconnect_docker_hosts l3ep2 llb1 +disconnect_docker_hosts l3ep3 llb1 + +delete_docker_host llb1 +delete_docker_host l3h1 +delete_docker_host l3ep1 +delete_docker_host l3ep2 +delete_docker_host l3ep3 + +echo "#########################################" +echo "Deleted testbed" +echo "#########################################" diff --git a/cicd/httpsproxy/validation.sh b/cicd/httpsproxy/validation.sh new file mode 100755 index 000000000..0443884d3 --- /dev/null +++ b/cicd/httpsproxy/validation.sh @@ -0,0 +1,64 @@ +#!/bin/bash +source ../common.sh +echo SCENARIO-https-tcplb +$hexec l3ep1 node ../common/tcp_server.js server1 & +$hexec l3ep2 node ../common/tcp_server.js server2 & +$hexec l3ep3 node ../common/tcp_server.js server3 & + +sleep 5 +code=0 +servIP=( "10.10.10.254" ) +servArr=( "server1" "server2" "server3" ) +ep=( "31.31.31.1" "32.32.32.1" "33.33.33.1" ) +j=0 +waitCount=0 +while [ $j -le 2 ] +do + res=$($hexec l3h1 curl --max-time 10 -s ${ep[j]}:8080) + #echo $res + if [[ $res == "${servArr[j]}" ]] + then + echo "$res UP" + j=$(( $j + 1 )) + else + echo "Waiting for ${servArr[j]}(${ep[j]})" + waitCount=$(( $waitCount + 1 )) + if [[ $waitCount == 10 ]]; + then + echo "All Servers are not UP" + echo SCENARIO-tcplb [FAILED] + sudo killall -9 node 2>&1 > /dev/null + exit 1 + fi + fi + sleep 1 +done + +for k in {0..0} +do +echo "Testing Service IP: ${servIP[k]}" +lcode=0 +for i in {1..4} +do +for j in {0..2} +do + res=$($hexec l3h1 curl --max-time 10 --insecure -s https://${servIP[k]}:2020) + echo $res + if [[ $res != "${servArr[j]}" ]] + then + lcode=1 + fi + sleep 1 +done +done +if [[ $lcode == 0 ]] +then + echo SCENARIO-https-tcplb with ${servIP[k]} [OK] +else + echo SCENARIO-https-tcplb with ${servIP[k]} [FAILED] + code=1 +fi +done + +sudo killall -9 node 2>&1 > /dev/null +exit $code diff --git a/cicd/ipmasquerade/config.sh b/cicd/ipmasquerade/config.sh new file mode 100755 index 000000000..a894d53ac --- /dev/null +++ b/cicd/ipmasquerade/config.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +source ../common.sh + +echo "#########################################" +echo "Spawning all hosts" +echo "#########################################" + +spawn_docker_host --dock-type loxilb --dock-name llb1 +spawn_docker_host --dock-type host --dock-name l3h1 +spawn_docker_host --dock-type host --dock-name l3ep1 +spawn_docker_host --dock-type host --dock-name l3ep2 +spawn_docker_host --dock-type host --dock-name l3ep3 + +echo "#########################################" +echo "Connecting and configuring hosts" +echo "#########################################" + +connect_docker_hosts l3h1 llb1 +connect_docker_hosts l3ep1 llb1 +connect_docker_hosts l3ep2 llb1 +connect_docker_hosts l3ep3 llb1 + +sleep 5 + +#L3 config +config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254 +config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254 +config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254 +config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254 +config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24 +config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24 +config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24 +config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24 + +sleep 5 + +$dexec llb1 loxicmd create firewall --firewallRule="portName:ellb1l3ep1" --snat=10.10.10.254 diff --git a/cicd/ipmasquerade/rmconfig.sh b/cicd/ipmasquerade/rmconfig.sh new file mode 100755 index 000000000..9ea80577f --- /dev/null +++ b/cicd/ipmasquerade/rmconfig.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source ../common.sh + +disconnect_docker_hosts l3h1 llb1 +disconnect_docker_hosts l3ep1 llb1 +disconnect_docker_hosts l3ep2 llb1 +disconnect_docker_hosts l3ep3 llb1 + +delete_docker_host llb1 +delete_docker_host l3h1 +delete_docker_host l3ep1 +delete_docker_host l3ep2 +delete_docker_host l3ep3 + +echo "#########################################" +echo "Deleted testbed" +echo "#########################################" diff --git a/cicd/ipmasquerade/validation.sh b/cicd/ipmasquerade/validation.sh new file mode 100755 index 000000000..fdff2d202 --- /dev/null +++ b/cicd/ipmasquerade/validation.sh @@ -0,0 +1,39 @@ +#!/bin/bash +source ../common.sh +echo SCENARIO-masquerade +$hexec l3h1 node ../common/tcp_server.js server1 & + +sleep 15 +code=0 +servIP=( "10.10.10.1" ) +servArr=( "server1" ) +clientArr=( "l3ep1" "l3ep2" "l3ep3" ) +j=0 +waitCount=0 + +echo "Testing Service IP: ${servIP[0]}" +lcode=0 +for i in {0..2} +do +for j in {0..2} +do + res=$($hexec ${clientArr[i]} curl --max-time 10 -s ${servIP[0]}:8080) + echo $res + if [[ $res != "${servArr[0]}" ]] + then + lcode=1 + fi + sleep 1 +done +done + +if [[ $lcode == 0 ]] +then + echo SCENARIO-masquerade with ${servIP[0]} [OK] +else + echo SCENARIO-masquerade with ${servIP[0]} [FAILED] + code=1 +fi + +sudo killall -9 node 2>&1 > /dev/null +exit $code diff --git a/common/common.go b/common/common.go index 9d4c90cbc..e4395d4b0 100644 --- a/common/common.go +++ b/common/common.go @@ -398,6 +398,10 @@ type FwOptArg struct { Allow bool `json:"allow"` // Mark - Mark the matching rule Mark uint32 `json:"fwMark"` + // DoSnat - Do snat on matching rule + DoSnat bool `json:"doSnat"` + ToIP string `json:"toIP"` + ToPort uint16 `json:"toPort"` // Counter - Traffic counter Counter string `json:"counter"` } @@ -561,6 +565,8 @@ type LbServiceArg struct { Name string `json:"name"` // PersistTimeout - Persistence timeout in seconds PersistTimeout uint32 `json:"persistTimeout"` + // Snat - Do SNAT + Snat bool `json:"snat"` } // LbEndPointArg - Information related to load-balancer end-point diff --git a/loxilb-ebpf b/loxilb-ebpf index 5d12c46ef..ce219cbe4 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit 5d12c46ef7a63f889267bdb7cf8813a0184b0e86 +Subproject commit ce219cbe43efb89bb39e6f063f751bfbe0a48903 diff --git a/pkg/loxinet/apiclient.go b/pkg/loxinet/apiclient.go index 3941d7d15..f82f8bfd5 100644 --- a/pkg/loxinet/apiclient.go +++ b/pkg/loxinet/apiclient.go @@ -764,7 +764,6 @@ func (na *NetAPIStruct) NetGoBGPPolicyApplyAdd(param *cmn.GoBGPPolicyApply) (int } - // NetGoBGPPolicyApplyDel - Del bgp neigh to gobgp func (na *NetAPIStruct) NetGoBGPPolicyApplyDel(param *cmn.GoBGPPolicyApply) (int, error) { if mh.bgp != nil { diff --git a/pkg/loxinet/dpebpf_linux.go b/pkg/loxinet/dpebpf_linux.go index 20d0e2476..243d8688d 100644 --- a/pkg/loxinet/dpebpf_linux.go +++ b/pkg/loxinet/dpebpf_linux.go @@ -922,18 +922,24 @@ func DpNatLbRuleMod(w *NatDpWorkQ) int { key := new(natKey) - key.daddr = [4]C.uint{0, 0, 0, 0} - if tk.IsNetIPv4(w.ServiceIP.String()) { - key.daddr[0] = C.uint(tk.IPtonl(w.ServiceIP)) - key.v6 = 0 + key.mark = C.ushort(w.BlockNum) + + if w.NatType == DpSnat { + key.mark |= 0x1000 } else { - convNetIP2DPv6Addr(unsafe.Pointer(&key.daddr[0]), w.ServiceIP) - key.v6 = 1 + key.daddr = [4]C.uint{0, 0, 0, 0} + if tk.IsNetIPv4(w.ServiceIP.String()) { + key.daddr[0] = C.uint(tk.IPtonl(w.ServiceIP)) + key.v6 = 0 + } else { + convNetIP2DPv6Addr(unsafe.Pointer(&key.daddr[0]), w.ServiceIP) + key.v6 = 1 + } + key.mark = C.ushort(w.BlockNum) + key.dport = C.ushort(tk.Htons(w.L4Port)) + key.l4proto = C.uchar(w.Proto) + key.zone = C.ushort(w.ZoneNum) } - key.mark = C.ushort(w.BlockNum) - key.dport = C.ushort(tk.Htons(w.L4Port)) - key.l4proto = C.uchar(w.Proto) - key.zone = C.ushort(w.ZoneNum) if w.Work == DpCreate { dat := new(natActs) @@ -1500,7 +1506,7 @@ func (e *DpEbpfH) DpTableGet(w *TableDpWorkQ) (DpRetT, error) { for e, proxyCt := range proxyCtInfo { ePCT := ctMap[proxyCt.Key()] if ePCT != nil { - if e > 1 { + if e > 0 { ePCT.CAct += " " } ePCT.CAct += proxyCt.CAct diff --git a/pkg/loxinet/rules.go b/pkg/loxinet/rules.go index da0cd7b27..1776cef2a 100644 --- a/pkg/loxinet/rules.go +++ b/pkg/loxinet/rules.go @@ -233,10 +233,12 @@ type ruleNatActs struct { } type ruleFwOpt struct { - rdrMirr string - rdrPort string - fwMark uint32 - record bool + rdrMirr string + rdrPort string + fwMark uint32 + record bool + snatIP string + snatPort uint16 } type ruleFwOpts struct { @@ -689,6 +691,10 @@ func (a *ruleAct) String() string { } } } + case *ruleFwOpts: + if a.actType == RtActSnat { + ks += fmt.Sprintf("%s:%d", na.opt.snatIP, na.opt.snatPort) + } } } @@ -779,6 +785,9 @@ func (R *RuleH) GetNatLbRule() ([]cmn.LbRuleMod, error) { ret.Serv.ProbeReq = data.hChk.prbReq ret.Serv.ProbeResp = data.hChk.prbResp ret.Serv.Name = data.name + if data.act.actType == RtActSnat { + ret.Serv.Snat = true + } for _, sip := range data.secIP { ret.SecIPs = append(ret.SecIPs, cmn.LbSecIPArg{SecIP: sip.sIP.String()}) @@ -1243,7 +1252,7 @@ func (R *RuleH) unFoldRecursiveEPs(r *ruleEnt) { // addVIPSys - system specific operations for VIPs of a LB rule func (R *RuleH) addVIPSys(r *ruleEnt) { - if !strings.Contains(r.name, "ipvs") && !strings.Contains(r.name, "static") { + if r.act.actType != RtActSnat && !strings.Contains(r.name, "ipvs") && !strings.Contains(r.name, "static") { if !r.tuples.l3Dst.addr.IP.IsUnspecified() { R.vipMap[r.tuples.l3Dst.addr.IP.String()]++ @@ -1441,9 +1450,9 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, return a < b }) - if sNetAddr.IP.IsUnspecified() && serv.Mode != cmn.LBModeHostOneArm { - serv.Mode = cmn.LBModeHostOneArm - tk.LogIt(tk.LogInfo, "nat lb-rule %s-%v-%s updated to hostOneArm\n", serv.ServIP, serv.ServPort, serv.Proto) + if serv.Mode == cmn.LBModeHostOneArm && !sNetAddr.IP.IsUnspecified() { + tk.LogIt(tk.LogInfo, "nat lb-rule %s-%v-%s hostarm needs unspec VIP\n", serv.ServIP, serv.ServPort, serv.Proto) + return RuleArgsErr, errors.New("hostarm-args error") } natActs.sel = serv.Sel @@ -1534,8 +1543,10 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, // Managed flag can't be modified on the fly // eRule.managed = serv.Managed - R.modNatEpHost(eRule, retEps, true, activateProbe) - R.electEPSrc(eRule) + if !serv.Snat { + R.modNatEpHost(eRule, retEps, true, activateProbe) + R.electEPSrc(eRule) + } eRule.sT = time.Now() eRule.iTO = serv.InactiveTimeout @@ -1552,7 +1563,9 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, r.tuples = rt r.zone = R.zone r.name = serv.Name - if serv.Mode == cmn.LBModeFullNAT || serv.Mode == cmn.LBModeOneArm || serv.Mode == cmn.LBModeHostOneArm { + if serv.Snat { + r.act.actType = RtActSnat + } else if serv.Mode == cmn.LBModeFullNAT || serv.Mode == cmn.LBModeOneArm || serv.Mode == cmn.LBModeHostOneArm { r.act.actType = RtActFullNat } else if serv.Mode == cmn.LBModeFullProxy { r.act.actType = RtActFullProxy @@ -1572,6 +1585,7 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, r.hChk.prbRetries = serv.ProbeRetries r.hChk.prbTimeo = serv.ProbeTimeout r.hChk.actChk = serv.Monitor + r.act.action = &natActs r.ruleNum, err = R.tables[RtLB].Mark.GetCounter() if err != nil { @@ -1592,11 +1606,13 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, } r.locIPs = make(map[string]struct{}) - R.foldRecursiveEPs(r) - R.modNatEpHost(r, natActs.endPoints, true, activateProbe) - R.electEPSrc(r) - if serv.Mode == cmn.LBModeHostOneArm { - R.mkHostAssocs(r) + if !serv.Snat { + R.foldRecursiveEPs(r) + R.modNatEpHost(r, natActs.endPoints, true, activateProbe) + R.electEPSrc(r) + if serv.Mode == cmn.LBModeHostOneArm { + R.mkHostAssocs(r) + } } tk.LogIt(tk.LogDebug, "nat lb-rule added - %d:%s-%s\n", r.ruleNum, r.tuples.String(), r.act.String()) @@ -1614,7 +1630,7 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, // deleteVIPSys - system specific operations for deleting VIPs of a LB rule func (R *RuleH) deleteVIPSys(r *ruleEnt) { - if !strings.Contains(r.name, "ipvs") && !strings.Contains(r.name, "static") { + if r.act.actType != RtActSnat && !strings.Contains(r.name, "ipvs") && !strings.Contains(r.name, "static") { if !r.tuples.l3Dst.addr.IP.IsUnspecified() { R.vipMap[r.tuples.l3Dst.addr.IP.String()]-- @@ -1700,8 +1716,10 @@ func (R *RuleH) DeleteNatLbRule(serv cmn.LbServiceArg) (int, error) { if rule.act.action.(*ruleNatActs).mode == cmn.LBModeOneArm || rule.act.action.(*ruleNatActs).mode == cmn.LBModeFullNAT || rule.act.action.(*ruleNatActs).mode == cmn.LBModeHostOneArm || rule.hChk.actChk { activatedProbe = true } - R.modNatEpHost(rule, eEps, false, activatedProbe) - R.unFoldRecursiveEPs(rule) + if rule.act.actType != RtActSnat { + R.modNatEpHost(rule, eEps, false, activatedProbe) + R.unFoldRecursiveEPs(rule) + } delete(R.tables[RtLB].eMap, rt.ruleKey()) if rule.ruleNum < RtMaximumLbs { @@ -1754,8 +1772,14 @@ func (R *RuleH) GetFwRule() ([]cmn.FwRuleMod, error) { ret.Opts.RdrPort = fwOpts.opt.rdrPort } else if fwOpts.op == RtActTrap { ret.Opts.Trap = true + } else if fwOpts.op == RtActSnat { + ret.Opts.DoSnat = true + ret.Opts.ToIP = fwOpts.opt.snatIP + ret.Opts.ToPort = uint16(fwOpts.opt.snatPort) + } + if fwOpts.op != RtActSnat { + ret.Opts.Mark = fwOpts.opt.fwMark } - ret.Opts.Mark = fwOpts.opt.fwMark ret.Opts.Record = fwOpts.opt.record data.Fw2DP(DpStatsGetImm) @@ -1847,6 +1871,19 @@ func (R *RuleH) AddFwRule(fwRule cmn.FwRuleArg, fwOptArgs cmn.FwOptArg) (int, er } else if fwOptArgs.Trap { r.act.actType = RtActTrap fwOpts.op = RtActTrap + } else if fwOptArgs.DoSnat { + r.act.actType = RtActSnat + fwOpts.op = RtActSnat + fwOpts.opt.snatIP = fwOptArgs.ToIP + fwOpts.opt.snatPort = fwOptArgs.ToPort + + if sIP := net.ParseIP(fwOptArgs.ToIP); sIP == nil { + return RuleArgsErr, errors.New("malformed-args error") + } + + if fwOpts.opt.fwMark != 0 { + return RuleArgsErr, errors.New("malformed-args fwmark !=0 for snat-error") + } } r.act.action = &fwOpts @@ -1857,11 +1894,36 @@ func (R *RuleH) AddFwRule(fwRule cmn.FwRuleArg, fwOptArgs cmn.FwOptArg) (int, er } r.sT = time.Now() + if fwOptArgs.DoSnat { + // Create SNAT Rule + var servArg cmn.LbServiceArg + servArg.ServIP = "0.0.0.0" + servArg.ServPort = 0 + servArg.Proto = "none" + servArg.BlockNum = uint16(r.ruleNum) | 0x1000 + servArg.Sel = cmn.LbSelRr + servArg.Mode = cmn.LBModeDefault + servArg.Snat = true + servArg.InactiveTimeout = LbDefaultInactiveTimeout + servArg.Name = fmt.Sprintf("%s:%s:%d", "snat", fwOpts.opt.snatIP, fwOpts.opt.snatPort) + + snatEP := []cmn.LbEndPointArg{{EpIP: fwOpts.opt.snatIP, EpPort: fwOpts.opt.snatPort}} + + _, err := R.AddNatLbRule(servArg, nil, snatEP) + if err != nil { + tk.LogIt(tk.LogError, "fw-rule - %s:%s (%s) snat create error\n", r.tuples.String(), r.act.String(), err) + return RuleArgsErr, errors.New("rule-snat error") + } + + fwOpts.opt.fwMark = uint32(uint16((r.ruleNum) | 0x1000)) + + } + tk.LogIt(tk.LogDebug, "fw-rule added - %d:%s-%s\n", r.ruleNum, r.tuples.String(), r.act.String()) R.tables[RtFw].eMap[rt.ruleKey()] = r - r.DP(DpCreate) + r.Fw2DP(DpCreate) return 0, nil } @@ -1920,13 +1982,36 @@ func (R *RuleH) DeleteFwRule(fwRule cmn.FwRuleArg) (int, error) { return RuleNotExistsErr, errors.New("no-rule error") } + if rule.act.actType == RtActSnat { + // Delete implicit SNAT Rule + + var servArg cmn.LbServiceArg + servArg.ServIP = "0.0.0.0" + servArg.ServPort = 0 + servArg.Proto = "none" + servArg.BlockNum = uint16(rule.ruleNum) | 0x1000 + servArg.Sel = cmn.LbSelRr + servArg.Mode = cmn.LBModeDefault + servArg.Snat = true + + switch fwOpts := rule.act.action.(type) { + case *ruleFwOpts: + servArg.Name = fmt.Sprintf("%s:%s:%d", "Masq", fwOpts.opt.snatIP, fwOpts.opt.snatPort) + } + + _, err := R.DeleteNatLbRule(servArg) + if err != nil { + tk.LogIt(tk.LogError, "fw-rule - %s:%s snat delete error\n", rule.tuples.String(), rule.act.String()) + } + } + defer R.tables[RtFw].Mark.PutCounter(rule.ruleNum) delete(R.tables[RtFw].eMap, rt.ruleKey()) tk.LogIt(tk.LogDebug, "fw-rule deleted %s-%s\n", rule.tuples.String(), rule.act.String()) - rule.DP(DpRemove) + rule.Fw2DP(DpRemove) return 0, nil } @@ -2377,15 +2462,15 @@ func (R *RuleH) RulesSync() { } for _, rule := range R.tables[RtFw].eMap { - ruleKeys := rule.tuples.String() - ruleActs := rule.act.String() + //ruleKeys := rule.tuples.String() + //ruleActs := rule.act.String() if rule.sync != 0 { - rule.DP(DpCreate) + rule.Fw2DP(DpCreate) } - rule.DP(DpStatsGet) - tk.LogIt(-1, "%d:%s,%s pc %v bc %v \n", - rule.ruleNum, ruleKeys, ruleActs, - rule.stat.packets, rule.stat.bytes) + //rule.DP(DpStatsGet) + //tk.LogIt(-1, "%d:%s,%s pc %v bc %v \n", + // rule.ruleNum, ruleKeys, ruleActs, + // rule.stat.packets, rule.stat.bytes) } } @@ -2720,6 +2805,8 @@ func (r *ruleEnt) Fw2DP(work DpWorkT) int { nWork.FwVal1 = uint16(port.PortNo) case RtActTrap: nWork.FwType = DpFwTrap + case RtActSnat: + nWork.FwType = DpFwFwd default: nWork.FwType = DpFwDrop }