diff --git a/Dockerfile b/Dockerfile index 11d94d35b..c233df155 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,9 @@ ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64/" # Install loxilb related packages RUN mkdir -p /opt/loxilb && \ mkdir -p /root/loxilb-io/loxilb/ && \ + mkdir -p /usr/lib64/ && \ mkdir -p /opt/loxilb/cert/ && \ + mkdir -p /etc/loxilb/certs/ && \ mkdir -p /etc/bash_completion.d/ && \ # Update Ubuntu Software repository apt-get update && apt-get install -y wget && \ @@ -22,6 +24,14 @@ RUN mkdir -p /opt/loxilb && \ apt-get install -y clang llvm libelf-dev libpcap-dev vim net-tools \ elfutils dwarves git libbsd-dev bridge-utils wget unzip build-essential \ bison flex sudo iproute2 pkg-config tcpdump iputils-ping curl bash-completion && \ + # Install openssl-3.0.0 + wget https://www.openssl.org/source/openssl-3.0.0.tar.gz && tar -xvzf openssl-3.0.0.tar.gz && \ + cd openssl-3.0.0 && ./Configure enable-ktls '-Wl,-rpath,$(LIBRPATH)' --prefix=/usr/local/build && \ + make -j$(nproc) && make install_dev install_modules && cd - && \ + cp -a /usr/local/build/include/openssl /usr/include/ && \ + if [ -d /usr/local/build/lib64 ] ; then mv /usr/local/build/lib64 /usr/local/build/lib; fi && \ + cp -fr /usr/local/build/lib/* /usr/lib/ && ldconfig && \ + rm -fr openssl-3.0.0* && \ # Install loxilb's custom ntc tool wget https://github.com/loxilb-io/iproute2/archive/refs/heads/main.zip && \ unzip main.zip && cd iproute2-main/ && rm -fr libbpf && wget https://github.com/loxilb-io/libbpf/archive/refs/heads/main.zip && \ @@ -31,10 +41,10 @@ RUN mkdir -p /opt/loxilb && \ LIBBPF_FORCE=on LIBBPF_DIR=`pwd`/libbpf/src/build ./configure && make && \ cp -f tc/tc /usr/local/sbin/ntc && cd .. && rm -fr main.zip iproute2-main && \ # Install bpftool - git clone --recurse-submodules https://github.com/libbpf/bpftool.git && cd bpftool/src/ && \ - git switch --detach v7.2.0 && \ + wget https://github.com/libbpf/bpftool/releases/download/v7.2.0/bpftool-libbpf-v7.2.0-sources.tar.gz && \ + tar -xvzf bpftool-libbpf-v7.2.0-sources.tar.gz && cd bpftool/src/ && \ make clean && make -j $(nproc) && cp -f ./bpftool /usr/local/sbin/bpftool && \ - cd - && rm -fr bpftool && \ + cd - && rm -fr bpftool* && \ # Install loxicmd git clone https://github.com/loxilb-io/loxicmd.git && cd loxicmd && go get . && \ make && cp ./loxicmd /usr/local/sbin/loxicmd && cd - && rm -fr loxicmd && \ @@ -85,6 +95,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends sudo \ rm -rf /var/lib/apt/lists/* && apt clean COPY --from=build /usr/lib64/libbpf* /usr/lib64/ +COPY --from=build /usr/local/build/lib/* /usr/lib64 COPY --from=build /usr/local/go/bin /usr/local/go/bin COPY --from=build /usr/local/sbin/mkllb_bpffs /usr/local/sbin/mkllb_bpffs COPY --from=build /usr/local/sbin/mkllb_cgroup /usr/local/sbin/mkllb_cgroup diff --git a/api/models/loadbalance_entry.go b/api/models/loadbalance_entry.go index eea7547a6..e55f0feff 100644 --- a/api/models/loadbalance_entry.go +++ b/api/models/loadbalance_entry.go @@ -360,6 +360,9 @@ type LoadbalanceEntryServiceArguments struct { // value for access protocol Protocol string `json:"protocol,omitempty"` + // value for Security mode (0-Plain, 1-HTTPs) + Security int32 `json:"security,omitempty"` + // value for load balance algorithim Sel int64 `json:"sel,omitempty"` } diff --git a/api/restapi/embedded_spec.go b/api/restapi/embedded_spec.go index fc7362b54..09a294506 100644 --- a/api/restapi/embedded_spec.go +++ b/api/restapi/embedded_spec.go @@ -4911,6 +4911,11 @@ func init() { "description": "value for access protocol", "type": "string" }, + "security": { + "description": "value for Security mode (0-Plain, 1-HTTPs)", + "type": "integer", + "format": "int32" + }, "sel": { "description": "value for load balance algorithim", "type": "integer" @@ -10861,6 +10866,11 @@ func init() { "description": "value for access protocol", "type": "string" }, + "security": { + "description": "value for Security mode (0-Plain, 1-HTTPs)", + "type": "integer", + "format": "int32" + }, "sel": { "description": "value for load balance algorithim", "type": "integer" @@ -10979,6 +10989,11 @@ func init() { "description": "value for access protocol", "type": "string" }, + "security": { + "description": "value for Security mode (0-Plain, 1-HTTPs)", + "type": "integer", + "format": "int32" + }, "sel": { "description": "value for load balance algorithim", "type": "integer" diff --git a/api/restapi/handler/loadbalancer.go b/api/restapi/handler/loadbalancer.go index 03cacc29f..1001881b9 100644 --- a/api/restapi/handler/loadbalancer.go +++ b/api/restapi/handler/loadbalancer.go @@ -36,6 +36,7 @@ func ConfigPostLoadbalancer(params operations.PostConfigLoadbalancerParams) midd lbRules.Serv.Bgp = params.Attr.ServiceArguments.Bgp lbRules.Serv.Monitor = params.Attr.ServiceArguments.Monitor lbRules.Serv.Mode = cmn.LBMode(params.Attr.ServiceArguments.Mode) + lbRules.Serv.Security = cmn.LBSec(params.Attr.ServiceArguments.Security) lbRules.Serv.InactiveTimeout = uint32(params.Attr.ServiceArguments.InactiveTimeOut) lbRules.Serv.Managed = params.Attr.ServiceArguments.Managed lbRules.Serv.ProbeType = params.Attr.ServiceArguments.Probetype @@ -124,6 +125,7 @@ func ConfigGetLoadbalancer(params operations.GetConfigLoadbalancerAllParams) mid tmpSvc.Block = uint16(lb.Serv.BlockNum) tmpSvc.Sel = int64(lb.Serv.Sel) tmpSvc.Mode = int32(lb.Serv.Mode) + tmpSvc.Security = int32(lb.Serv.Security) tmpSvc.InactiveTimeOut = int32(lb.Serv.InactiveTimeout) tmpSvc.Monitor = lb.Serv.Monitor tmpSvc.Managed = lb.Serv.Managed diff --git a/api/swagger.yml b/api/swagger.yml index 91c6dd7a2..ab1a10e6c 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -2831,6 +2831,10 @@ definitions: type: integer format: int32 description: value for NAT mode (0-DNAT, 1-oneArm, 2-fullNAT) + security: + type: integer + format: int32 + description: value for Security mode (0-Plain, 1-HTTPs) block: type: integer format: uint16 diff --git a/common/common.go b/common/common.go index 229ee0de2..74002738e 100644 --- a/common/common.go +++ b/common/common.go @@ -475,6 +475,8 @@ const ( LbSelRrPersist // LbSelLeastConnections - select client based on least connections LbSelLeastConnections + // LbSelN2 - select client based on N2 SCTP interface + LbSelN2 ) // LBMode - Variable to define LB mode @@ -507,6 +509,16 @@ const ( LBOPDetach ) +// LBSec - Variable to define LB front-end security +type LBSec int32 + +const ( + // LBServPlain - Plain mode + LBServPlain LBSec = iota + // LBServHttps - HTTPS termination + LBServHttps +) + // LbServiceArg - Information related to load-balancer service type LbServiceArg struct { // ServIP - the service ip or vip of the load-balancer rule @@ -525,6 +537,8 @@ type LbServiceArg struct { Monitor bool `json:"monitor"` // Oper - Attach/Detach if the LB already exists Oper LBOp `json:"oper"` + // Security - Security mode if any + Security LBSec `json:"lbsec"` // Mode - NAT mode Mode LBMode `json:"mode"` // InactiveTimeout - Forced session reset after inactive timeout diff --git a/loxilb-ebpf b/loxilb-ebpf index 996b0ee92..5d12c46ef 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit 996b0ee9278f14e4b8a08d11a4720cddcb9a1a39 +Subproject commit 5d12c46ef7a63f889267bdb7cf8813a0184b0e86 diff --git a/pkg/loxinet/dpbroker.go b/pkg/loxinet/dpbroker.go index da8378da1..0818a20d6 100644 --- a/pkg/loxinet/dpbroker.go +++ b/pkg/loxinet/dpbroker.go @@ -263,6 +263,7 @@ const ( EpPrio EpRRPersist EpLeastConn + EpN2 ) // NatEP - a nat end-point @@ -284,6 +285,7 @@ type NatDpWorkQ struct { BlockNum uint16 DsrMode bool CsumDis bool + TermHTTPs bool Proto uint8 Mark int NatType NatT diff --git a/pkg/loxinet/dpebpf_linux.go b/pkg/loxinet/dpebpf_linux.go index 306670145..20d0e2476 100644 --- a/pkg/loxinet/dpebpf_linux.go +++ b/pkg/loxinet/dpebpf_linux.go @@ -37,9 +37,10 @@ package loxinet int bpf_map_get_next_key(int fd, const void *key, void *next_key); int bpf_map_lookup_elem(int fd, const void *key, void *value); extern void goMapNotiHandler(struct ll_dp_map_notif *); +extern void goProxyEntCollector(struct dp_proxy_ct_ent *); extern void goLinuxArpResolver(unsigned int); #cgo CFLAGS: -I./../../loxilb-ebpf/libbpf/src/ -I./../../loxilb-ebpf/common -#cgo LDFLAGS: -L. -L/lib64 -L./../../loxilb-ebpf/kernel -L./../../loxilb-ebpf/libbpf/src/build/usr/lib64/ -Wl,-rpath=/lib64/ -l:./../../loxilb-ebpf/kernel/libloxilbdp.a -l:./../../loxilb-ebpf/libbpf/src/libbpf.a -lelf -lz +#cgo LDFLAGS: -L. -L/lib64 -L./../../loxilb-ebpf/kernel -L./../../loxilb-ebpf/libbpf/src/build/usr/lib64/ -Wl,-rpath=/lib64/ -l:./../../loxilb-ebpf/kernel/libloxilbdp.a -l:./../../loxilb-ebpf/libbpf/src/libbpf.a -lelf -lz -lssl -lcrypto */ import "C" import ( @@ -140,6 +141,11 @@ type ( mapNoti C.struct_ll_dp_map_notif vipKey C.struct_sock_rwr_key vipAct C.struct_sock_rwr_action + proxtCT C.struct_dp_proxy_ct_ent +) + +var ( + proxyCtInfo []*DpCtInfo ) // DpEbpfH - context container @@ -965,6 +971,8 @@ func DpNatLbRuleMod(w *NatDpWorkQ) int { dat.sel_type = C.NAT_LB_SEL_RR_PERSIST case w.EpSel == EpLeastConn: dat.sel_type = C.NAT_LB_SEL_LC + case w.EpSel == EpN2: + dat.sel_type = C.NAT_LB_SEL_N2 /* Currently not implemented in DP */ /*case w.EpSel == EP_PRIO: dat.sel_type = C.NAT_LB_SEL_PRIO*/ @@ -1015,6 +1023,10 @@ func DpNatLbRuleMod(w *NatDpWorkQ) int { dat.cdis = 0 } + if w.TermHTTPs { + dat.sec_mode = C.SEC_MODE_HTTPS + } + ret := C.llb_add_map_elem(C.LL_DP_NAT_MAP, unsafe.Pointer(key), unsafe.Pointer(dat)) @@ -1347,6 +1359,90 @@ func (ct *DpCtInfo) convDPCt2GoObj(ctKey *C.struct_dp_ct_key, ctDat *C.struct_dp return ct.convDPCt2GoObjFixup(ctKey, ctDat, false) } +func (ct *DpCtInfo) convDPCtKey2GoObj(ctKey *C.struct_dp_ct_key) *DpCtInfo { + if ctKey.v6 == 0 { + ct.DIP = tk.NltoIP(uint32(ctKey.daddr[0])) + ct.SIP = tk.NltoIP(uint32(ctKey.saddr[0])) + } else { + ct.SIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.saddr[0])) + ct.DIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.daddr[0])) + } + ct.Dport = tk.Ntohs(uint16(ctKey.dport)) + ct.Sport = tk.Ntohs(uint16(ctKey.sport)) + + p := uint8(ctKey.l4proto) + switch { + case p == 1 || p == 58: + if p == 1 { + ct.Proto = "icmp" + } else { + ct.Proto = "icmp6" + } + case p == 6: + ct.Proto = "tcp" + case p == 17: + ct.Proto = "udp" + case p == 132: + ct.Proto = "sctp" + default: + ct.Proto = fmt.Sprintf("%d", p) + } + return ct +} + +func (ct *DpCtInfo) convDPCtProxy2ActString(ctKey *C.struct_dp_ct_key) { + var DIP net.IP + var SIP net.IP + + if ctKey.v6 == 0 { + DIP = tk.NltoIP(uint32(ctKey.daddr[0])) + SIP = tk.NltoIP(uint32(ctKey.saddr[0])) + } else { + SIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.saddr[0])) + DIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.daddr[0])) + } + Dport := tk.Ntohs(uint16(ctKey.dport)) + Sport := tk.Ntohs(uint16(ctKey.sport)) + Proto := "" + + p := uint8(ctKey.l4proto) + switch { + case p == 1 || p == 58: + if p == 1 { + Proto = "icmp" + } else { + Proto = "icmp6" + } + case p == 6: + Proto = "tcp" + case p == 17: + Proto = "udp" + case p == 132: + Proto = "sctp" + default: + Proto = fmt.Sprintf("%d", p) + } + + ct.CAct = fmt.Sprintf("fp|%s:%d->%s:%d|%s", SIP.String(), Sport, DIP.String(), Dport, Proto) +} + +//export goProxyEntCollector +func goProxyEntCollector(e *proxtCT) { + + proxyCt := new(DpCtInfo) + proxyCt.convDPCtKey2GoObj(&e.ct_in) + proxyCt.convDPCtProxy2ActString(&e.ct_out) + proxyCt.Bytes = uint64(e.st_out.bytes) + proxyCt.Bytes += uint64(e.st_in.bytes) + + proxyCt.Packets = uint64(e.st_out.packets) + proxyCt.Packets += uint64(e.st_in.packets) + proxyCt.RuleID = uint32(e.rid) + proxyCt.CState = "est" + + proxyCtInfo = append(proxyCtInfo, proxyCt) +} + // DpTableGet - routine to work on a ebpf map get request func (e *DpEbpfH) DpTableGet(w *TableDpWorkQ) (DpRetT, error) { var tbl int @@ -1364,12 +1460,12 @@ func (e *DpEbpfH) DpTableGet(w *TableDpWorkQ) (DpRetT, error) { if tbl == C.LL_DP_CT_MAP { ctMap := make(map[string]*DpCtInfo) - var n int = 0 - var key *C.struct_dp_ct_key = nil + var key *C.struct_dp_ct_key nextKey := new(C.struct_dp_ct_key) var tact C.struct_dp_ct_tact var act *C.struct_dp_ct_dat + n := 0 fd := C.llb_map2fd(C.int(tbl)) for C.bpf_map_get_next_key(C.int(fd), (unsafe.Pointer)(key), (unsafe.Pointer)(nextKey)) == 0 { @@ -1398,6 +1494,24 @@ func (e *DpEbpfH) DpTableGet(w *TableDpWorkQ) (DpRetT, error) { key = nextKey n++ } + + proxyCtInfo = nil + C.llb_trigger_get_proxy_entries() + for e, proxyCt := range proxyCtInfo { + ePCT := ctMap[proxyCt.Key()] + if ePCT != nil { + if e > 1 { + ePCT.CAct += " " + } + ePCT.CAct += proxyCt.CAct + ePCT.Bytes += proxyCt.Bytes + ePCT.Packets += proxyCt.Packets + } else { + ctMap[proxyCt.Key()] = proxyCt + } + } + proxyCtInfo = nil + return ctMap, nil } diff --git a/pkg/loxinet/rules.go b/pkg/loxinet/rules.go index 10cfd57ea..da0cd7b27 100644 --- a/pkg/loxinet/rules.go +++ b/pkg/loxinet/rules.go @@ -283,6 +283,7 @@ type ruleEnt struct { secIP []ruleNatSIP stat ruleStat name string + secMode cmn.LBSec locIPs map[string]struct{} } @@ -772,6 +773,7 @@ func (R *RuleH) GetNatLbRule() ([]cmn.LbRuleMod, error) { ret.Serv.Bgp = data.bgp ret.Serv.BlockNum = data.tuples.pref ret.Serv.Managed = data.managed + ret.Serv.Security = data.secMode ret.Serv.ProbeType = data.hChk.prbType ret.Serv.ProbePort = data.hChk.prbPort ret.Serv.ProbeReq = data.hChk.prbReq @@ -1509,6 +1511,15 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, return R.DeleteNatLbRule(serv) } + if eRule.act.action.(*ruleNatActs).mode == cmn.LBModeFullProxy && natActs.mode != cmn.LBModeFullProxy || + eRule.act.action.(*ruleNatActs).mode != cmn.LBModeFullProxy && natActs.mode == cmn.LBModeFullProxy { + return RuleExistsErr, errors.New("lbrule-exist error: cant modify fullproxy rule mode") + } + + if eRule.act.action.(*ruleNatActs).mode == cmn.LBModeFullProxy { + eRule.DP(DpRemove) + } + // Update the rule eRule.hChk.prbType = serv.ProbeType eRule.hChk.prbPort = serv.ProbePort @@ -1550,6 +1561,7 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg, } r.managed = serv.Managed r.secIP = nSecIP + r.secMode = serv.Security // Per LB end-point health-check is supposed to be handled at kube-loxilb/CCM, // but it certain cases like stand-alone mode, loxilb can do its own // lb end-point health monitoring @@ -2477,6 +2489,9 @@ func (r *ruleEnt) Nat2DP(work DpWorkT) int { nWork.Work = work nWork.Status = &r.sync nWork.ZoneNum = r.zone.ZoneNum + if r.secMode == cmn.LBServHttps { + nWork.TermHTTPs = true + } nWork.ServiceIP = r.tuples.l3Dst.addr.IP.Mask(r.tuples.l3Dst.addr.Mask) nWork.L4Port = r.tuples.l4Dst.val nWork.Proto = r.tuples.l4Prot.val @@ -2517,6 +2532,8 @@ func (r *ruleEnt) Nat2DP(work DpWorkT) int { nWork.EpSel = EpRRPersist case at.sel == cmn.LbSelLeastConnections: nWork.EpSel = EpLeastConn + case at.sel == cmn.LbSelN2: + nWork.EpSel = EpN2 default: nWork.EpSel = EpRR }