Merge pull request #709 from TrekkieCoder/main

gh-48 - Initial support for L7 HTTPs proxy

Dockerfile

@@ -11,7 +11,9 @@ ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64/"
 # Install loxilb related packages
 RUN mkdir -p /opt/loxilb && \
     mkdir -p /root/loxilb-io/loxilb/ && \
+    mkdir -p /usr/lib64/ && \
     mkdir -p /opt/loxilb/cert/ && \
+    mkdir -p /etc/loxilb/certs/ && \
     mkdir -p /etc/bash_completion.d/ && \
     # Update Ubuntu Software repository
     apt-get update && apt-get install -y wget && \
@@ -22,6 +24,14 @@ RUN mkdir -p /opt/loxilb && \
     apt-get install -y clang llvm libelf-dev libpcap-dev vim net-tools \
     elfutils dwarves git libbsd-dev bridge-utils wget unzip build-essential \
     bison flex sudo iproute2 pkg-config tcpdump iputils-ping curl bash-completion && \
+    # Install openssl-3.0.0
+    wget https://www.openssl.org/source/openssl-3.0.0.tar.gz && tar -xvzf openssl-3.0.0.tar.gz && \
+    cd openssl-3.0.0 && ./Configure enable-ktls '-Wl,-rpath,$(LIBRPATH)' --prefix=/usr/local/build && \
+    make -j$(nproc) && make install_dev install_modules && cd - && \
+    cp -a /usr/local/build/include/openssl /usr/include/ && \
+    if [ -d /usr/local/build/lib64  ] ; then mv /usr/local/build/lib64  /usr/local/build/lib; fi && \
+    cp -fr /usr/local/build/lib/* /usr/lib/ && ldconfig && \
+    rm -fr openssl-3.0.0*  && \
     # Install loxilb's custom ntc tool
     wget https://github.com/loxilb-io/iproute2/archive/refs/heads/main.zip && \
     unzip main.zip && cd iproute2-main/ && rm -fr libbpf && wget https://github.com/loxilb-io/libbpf/archive/refs/heads/main.zip && \
@@ -31,10 +41,10 @@ RUN mkdir -p /opt/loxilb && \
     LIBBPF_FORCE=on LIBBPF_DIR=`pwd`/libbpf/src/build ./configure && make && \
     cp -f tc/tc /usr/local/sbin/ntc && cd .. && rm -fr main.zip iproute2-main && \
     # Install bpftool
-    git clone --recurse-submodules https://github.com/libbpf/bpftool.git && cd bpftool/src/ && \
-    git switch --detach v7.2.0 && \
+    wget https://github.com/libbpf/bpftool/releases/download/v7.2.0/bpftool-libbpf-v7.2.0-sources.tar.gz && \
+    tar -xvzf bpftool-libbpf-v7.2.0-sources.tar.gz && cd bpftool/src/ && \
     make clean && 	make -j $(nproc) && cp -f ./bpftool /usr/local/sbin/bpftool && \
-    cd - && rm -fr bpftool && \
+    cd - && rm -fr bpftool* && \
     # Install loxicmd
     git clone https://github.com/loxilb-io/loxicmd.git && cd loxicmd && go get . && \
     make && cp ./loxicmd /usr/local/sbin/loxicmd && cd - && rm -fr loxicmd && \
@@ -85,6 +95,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends sudo \
     rm -rf /var/lib/apt/lists/* && apt clean
 
 COPY --from=build /usr/lib64/libbpf* /usr/lib64/
+COPY --from=build /usr/local/build/lib/* /usr/lib64
 COPY --from=build /usr/local/go/bin /usr/local/go/bin
 COPY --from=build /usr/local/sbin/mkllb_bpffs /usr/local/sbin/mkllb_bpffs
 COPY --from=build /usr/local/sbin/mkllb_cgroup /usr/local/sbin/mkllb_cgroup

api/restapi/handler/loadbalancer.go

@@ -36,6 +36,7 @@ func ConfigPostLoadbalancer(params operations.PostConfigLoadbalancerParams) midd
 	lbRules.Serv.Bgp = params.Attr.ServiceArguments.Bgp
 	lbRules.Serv.Monitor = params.Attr.ServiceArguments.Monitor
 	lbRules.Serv.Mode = cmn.LBMode(params.Attr.ServiceArguments.Mode)
+	lbRules.Serv.Security = cmn.LBSec(params.Attr.ServiceArguments.Security)
 	lbRules.Serv.InactiveTimeout = uint32(params.Attr.ServiceArguments.InactiveTimeOut)
 	lbRules.Serv.Managed = params.Attr.ServiceArguments.Managed
 	lbRules.Serv.ProbeType = params.Attr.ServiceArguments.Probetype
@@ -124,6 +125,7 @@ func ConfigGetLoadbalancer(params operations.GetConfigLoadbalancerAllParams) mid
 		tmpSvc.Block = uint16(lb.Serv.BlockNum)
 		tmpSvc.Sel = int64(lb.Serv.Sel)
 		tmpSvc.Mode = int32(lb.Serv.Mode)
+		tmpSvc.Security = int32(lb.Serv.Security)
 		tmpSvc.InactiveTimeOut = int32(lb.Serv.InactiveTimeout)
 		tmpSvc.Monitor = lb.Serv.Monitor
 		tmpSvc.Managed = lb.Serv.Managed

api/swagger.yml

@@ -2831,6 +2831,10 @@ definitions:
             type: integer
             format: int32
             description: value for NAT mode (0-DNAT, 1-oneArm, 2-fullNAT)
+          security:
+            type: integer
+            format: int32
+            description: value for Security mode (0-Plain, 1-HTTPs)
           block:
             type: integer
             format: uint16

common/common.go

@@ -475,6 +475,8 @@ const (
 	LbSelRrPersist
 	// LbSelLeastConnections - select client based on least connections
 	LbSelLeastConnections
+	// LbSelN2 - select client based on N2 SCTP interface
+	LbSelN2
 )
 
 // LBMode - Variable to define LB mode
@@ -507,6 +509,16 @@ const (
 	LBOPDetach
 )
 
+// LBSec - Variable to define LB front-end security
+type LBSec int32
+
+const (
+	// LBServPlain - Plain mode
+	LBServPlain LBSec = iota
+	// LBServHttps - HTTPS termination
+	LBServHttps
+)
+
 // LbServiceArg - Information related to load-balancer service
 type LbServiceArg struct {
 	// ServIP - the service ip or vip  of the load-balancer rule
@@ -525,6 +537,8 @@ type LbServiceArg struct {
 	Monitor bool `json:"monitor"`
 	// Oper - Attach/Detach if the LB already exists
 	Oper LBOp `json:"oper"`
+	// Security - Security mode if any
+	Security LBSec `json:"lbsec"`
 	// Mode - NAT mode
 	Mode LBMode `json:"mode"`
 	// InactiveTimeout - Forced session reset after inactive timeout

pkg/loxinet/dpbroker.go

@@ -263,6 +263,7 @@ const (
 	EpPrio
 	EpRRPersist
 	EpLeastConn
+	EpN2
 )
 
 // NatEP - a nat end-point
@@ -284,6 +285,7 @@ type NatDpWorkQ struct {
 	BlockNum  uint16
 	DsrMode   bool
 	CsumDis   bool
+	TermHTTPs bool
 	Proto     uint8
 	Mark      int
 	NatType   NatT

pkg/loxinet/dpebpf_linux.go

@@ -37,9 +37,10 @@ package loxinet
 int bpf_map_get_next_key(int fd, const void *key, void *next_key);
 int bpf_map_lookup_elem(int fd, const void *key, void *value);
 extern void goMapNotiHandler(struct ll_dp_map_notif *);
+extern void goProxyEntCollector(struct dp_proxy_ct_ent *);
 extern void goLinuxArpResolver(unsigned int);
 #cgo CFLAGS:  -I./../../loxilb-ebpf/libbpf/src/ -I./../../loxilb-ebpf/common
-#cgo LDFLAGS: -L. -L/lib64 -L./../../loxilb-ebpf/kernel -L./../../loxilb-ebpf/libbpf/src/build/usr/lib64/ -Wl,-rpath=/lib64/ -l:./../../loxilb-ebpf/kernel/libloxilbdp.a -l:./../../loxilb-ebpf/libbpf/src/libbpf.a -lelf -lz
+#cgo LDFLAGS: -L. -L/lib64 -L./../../loxilb-ebpf/kernel -L./../../loxilb-ebpf/libbpf/src/build/usr/lib64/ -Wl,-rpath=/lib64/ -l:./../../loxilb-ebpf/kernel/libloxilbdp.a -l:./../../loxilb-ebpf/libbpf/src/libbpf.a -lelf -lz -lssl -lcrypto
 */
 import "C"
 import (
@@ -140,6 +141,11 @@ type (
 	mapNoti    C.struct_ll_dp_map_notif
 	vipKey     C.struct_sock_rwr_key
 	vipAct     C.struct_sock_rwr_action
+	proxtCT    C.struct_dp_proxy_ct_ent
+)
+
+var (
+	proxyCtInfo []*DpCtInfo
 )
 
 // DpEbpfH - context container
@@ -965,6 +971,8 @@ func DpNatLbRuleMod(w *NatDpWorkQ) int {
 			dat.sel_type = C.NAT_LB_SEL_RR_PERSIST
 		case w.EpSel == EpLeastConn:
 			dat.sel_type = C.NAT_LB_SEL_LC
+		case w.EpSel == EpN2:
+			dat.sel_type = C.NAT_LB_SEL_N2
 		/* Currently not implemented in DP */
 		/*case w.EpSel == EP_PRIO:
 		  dat.sel_type = C.NAT_LB_SEL_PRIO*/
@@ -1015,6 +1023,10 @@ func DpNatLbRuleMod(w *NatDpWorkQ) int {
 			dat.cdis = 0
 		}
 
+		if w.TermHTTPs {
+			dat.sec_mode = C.SEC_MODE_HTTPS
+		}
+
 		ret := C.llb_add_map_elem(C.LL_DP_NAT_MAP,
 			unsafe.Pointer(key),
 			unsafe.Pointer(dat))
@@ -1347,6 +1359,90 @@ func (ct *DpCtInfo) convDPCt2GoObj(ctKey *C.struct_dp_ct_key, ctDat *C.struct_dp
 	return ct.convDPCt2GoObjFixup(ctKey, ctDat, false)
 }
 
+func (ct *DpCtInfo) convDPCtKey2GoObj(ctKey *C.struct_dp_ct_key) *DpCtInfo {
+	if ctKey.v6 == 0 {
+		ct.DIP = tk.NltoIP(uint32(ctKey.daddr[0]))
+		ct.SIP = tk.NltoIP(uint32(ctKey.saddr[0]))
+	} else {
+		ct.SIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.saddr[0]))
+		ct.DIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.daddr[0]))
+	}
+	ct.Dport = tk.Ntohs(uint16(ctKey.dport))
+	ct.Sport = tk.Ntohs(uint16(ctKey.sport))
+
+	p := uint8(ctKey.l4proto)
+	switch {
+	case p == 1 || p == 58:
+		if p == 1 {
+			ct.Proto = "icmp"
+		} else {
+			ct.Proto = "icmp6"
+		}
+	case p == 6:
+		ct.Proto = "tcp"
+	case p == 17:
+		ct.Proto = "udp"
+	case p == 132:
+		ct.Proto = "sctp"
+	default:
+		ct.Proto = fmt.Sprintf("%d", p)
+	}
+	return ct
+}
+
+func (ct *DpCtInfo) convDPCtProxy2ActString(ctKey *C.struct_dp_ct_key) {
+	var DIP net.IP
+	var SIP net.IP
+
+	if ctKey.v6 == 0 {
+		DIP = tk.NltoIP(uint32(ctKey.daddr[0]))
+		SIP = tk.NltoIP(uint32(ctKey.saddr[0]))
+	} else {
+		SIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.saddr[0]))
+		DIP = convDPv6Addr2NetIP(unsafe.Pointer(&ctKey.daddr[0]))
+	}
+	Dport := tk.Ntohs(uint16(ctKey.dport))
+	Sport := tk.Ntohs(uint16(ctKey.sport))
+	Proto := ""
+
+	p := uint8(ctKey.l4proto)
+	switch {
+	case p == 1 || p == 58:
+		if p == 1 {
+			Proto = "icmp"
+		} else {
+			Proto = "icmp6"
+		}
+	case p == 6:
+		Proto = "tcp"
+	case p == 17:
+		Proto = "udp"
+	case p == 132:
+		Proto = "sctp"
+	default:
+		Proto = fmt.Sprintf("%d", p)
+	}
+
+	ct.CAct = fmt.Sprintf("fp|%s:%d->%s:%d|%s", SIP.String(), Sport, DIP.String(), Dport, Proto)
+}
+
+//export goProxyEntCollector
+func goProxyEntCollector(e *proxtCT) {
+
+	proxyCt := new(DpCtInfo)
+	proxyCt.convDPCtKey2GoObj(&e.ct_in)
+	proxyCt.convDPCtProxy2ActString(&e.ct_out)
+	proxyCt.Bytes = uint64(e.st_out.bytes)
+	proxyCt.Bytes += uint64(e.st_in.bytes)
+
+	proxyCt.Packets = uint64(e.st_out.packets)
+	proxyCt.Packets += uint64(e.st_in.packets)
+	proxyCt.RuleID = uint32(e.rid)
+	proxyCt.CState = "est"
+
+	proxyCtInfo = append(proxyCtInfo, proxyCt)
+}
+
 // DpTableGet - routine to work on a ebpf map get request
 func (e *DpEbpfH) DpTableGet(w *TableDpWorkQ) (DpRetT, error) {
 	var tbl int
@@ -1364,12 +1460,12 @@ func (e *DpEbpfH) DpTableGet(w *TableDpWorkQ) (DpRetT, error) {
 
 	if tbl == C.LL_DP_CT_MAP {
 		ctMap := make(map[string]*DpCtInfo)
-		var n int = 0
-		var key *C.struct_dp_ct_key = nil
+		var key *C.struct_dp_ct_key
 		nextKey := new(C.struct_dp_ct_key)
 		var tact C.struct_dp_ct_tact
 		var act *C.struct_dp_ct_dat
 
+		n := 0
 		fd := C.llb_map2fd(C.int(tbl))
 
 		for C.bpf_map_get_next_key(C.int(fd), (unsafe.Pointer)(key), (unsafe.Pointer)(nextKey)) == 0 {
@@ -1398,6 +1494,24 @@ func (e *DpEbpfH) DpTableGet(w *TableDpWorkQ) (DpRetT, error) {
 			key = nextKey
 			n++
 		}
+
+		proxyCtInfo = nil
+		C.llb_trigger_get_proxy_entries()
+		for e, proxyCt := range proxyCtInfo {
+			ePCT := ctMap[proxyCt.Key()]
+			if ePCT != nil {
+				if e > 1 {
+					ePCT.CAct += " "
+				}
+				ePCT.CAct += proxyCt.CAct
+				ePCT.Bytes += proxyCt.Bytes
+				ePCT.Packets += proxyCt.Packets
+			} else {
+				ctMap[proxyCt.Key()] = proxyCt
+			}
+		}
+		proxyCtInfo = nil
+
 		return ctMap, nil
 	}