Merge pull request #8893 from sridhartigera/auto-pick-of-#8876-upstre…

…am-release-v3.27

[release-v3.27] Auto pick #8876: Support for service loop prevention - ebpf

felix/bpf-gpl/reasons.h

@@ -22,6 +22,7 @@ enum calico_reason {
 	CALI_REASON_DECAP_FAIL,
 	CALI_REASON_UNAUTH_SOURCE,
 	CALI_REASON_RT_UNKNOWN,
+	CALI_REASON_BLACK_HOLE,
 	CALI_REASON_ACCEPTED_BY_XDP, // Not used by counters map
 	CALI_REASON_WEP_NOT_READY,
 	CALI_REASON_NATIFACE,

felix/bpf-gpl/routes.h

@@ -30,6 +30,8 @@ enum cali_rt_flags {
 	CALI_RT_SAME_SUBNET = 0x20,
 	CALI_RT_TUNNELED    = 0x40,
 	CALI_RT_NO_DSR      = 0x80,
+	CALI_RT_BLACKHOLE_DROP  = 0x100,
+	CALI_RT_BLACKHOLE_REJECT  = 0x200,
 };
 
 struct cali_rt {
@@ -76,6 +78,8 @@ static CALI_BPF_INLINE enum cali_rt_flags cali_rt_lookup_flags(ipv46_addr_t *add
 #define cali_rt_is_host(rt)	((rt)->flags & CALI_RT_HOST)
 #define cali_rt_is_workload(rt)	((rt)->flags & CALI_RT_WORKLOAD)
 #define cali_rt_is_tunneled(rt)	((rt)->flags & CALI_RT_TUNNELED)
+#define cali_rt_is_blackhole_drop(rt) ((rt)->flags & CALI_RT_BLACKHOLE_DROP)
+#define cali_rt_is_blackhole_reject(rt) ((rt)->flags & CALI_RT_BLACKHOLE_REJECT)
 
 #define cali_rt_flags_host(t) (((t) & CALI_RT_HOST) == CALI_RT_HOST)
 #define cali_rt_flags_local_host(t) (((t) & (CALI_RT_LOCAL | CALI_RT_HOST)) == (CALI_RT_LOCAL | CALI_RT_HOST))

felix/bpf-gpl/tc.c

@@ -428,6 +428,7 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx)
 			 * usecase of this is node-local-dns.
 			 */
 			ctx->state->flags |= CALI_ST_SKIP_FIB;
+			ctx->state->flags |= CALI_ST_NAT_EXCLUDE;
 		}
 	}
 
@@ -541,8 +542,12 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx)
 		ctx->state->flags |= CALI_ST_SRC_IS_HOST;
 	}
 
-	struct cali_rt *dest_rt = cali_rt_lookup(&ctx->state->post_nat_ip_dst);
-
+	struct cali_rt *dest_rt = NULL;
+	// Route lookup is not done for those packets which are nat excluded, where there
+	// is a nat hit, but we don't resolve (such as node local DNS cache).
+	if (!(ctx->state->flags & CALI_ST_NAT_EXCLUDE)) {
+		dest_rt = cali_rt_lookup(&ctx->state->post_nat_ip_dst);
+	}
 	if (!dest_rt) {
 		CALI_DEBUG("No route for post DNAT dest %x\n", debug_ip(ctx->state->post_nat_ip_dst));
 		if (CALI_F_FROM_HEP) {
@@ -558,6 +563,32 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx)
 		goto do_policy;
 	}
 
+	/* If the dest route is a blackhole route, drop/reject the packet.
+	 * This is based on the service loop prevention configuration.
+	 * If ServiceLoopPrevention = Drop, route is a blackhole drop route.
+	 * If ServiceLoopPrevention = Reject, route is a blackhole reject route.
+	 * If ServiceLoopPrevention = Disabled, these routes are not programmed.
+	 */
+	if (CALI_F_TO_HOST) {
+		if (cali_rt_is_blackhole_drop(dest_rt)) {
+			CALI_DEBUG("Packet hit a black hole route: DROP\n");
+			deny_reason(ctx, CALI_REASON_BLACK_HOLE);
+			goto deny;
+		}
+		if (cali_rt_is_blackhole_reject(dest_rt)) {
+			CALI_DEBUG("Packet hit a black hole route: REJECT\n");
+			deny_reason(ctx, CALI_REASON_BLACK_HOLE);
+#ifdef IPVER6
+			ctx->state->icmp_type = ICMPV6_DEST_UNREACH;
+			ctx->state->icmp_code = ICMPV6_PORT_UNREACH;
+#else
+			ctx->state->icmp_type = ICMP_DEST_UNREACH;
+			ctx->state->icmp_code = ICMP_PORT_UNREACH;
+#endif
+			goto icmp_send_reply;
+		}
+	}
+
 	if (cali_rt_flags_local_host(dest_rt->flags)) {
 		CALI_DEBUG("Post-NAT dest IP is local host.\n");
 		if (CALI_F_FROM_HEP && is_failsafe_in(ctx->state->ip_proto, ctx->state->post_nat_dport, ctx->state->ip_src)) {

felix/bpf-gpl/types.h

@@ -142,6 +142,8 @@ enum cali_state_flags {
 	CALI_ST_CT_NP_LOOP	  = 0x80,
 	/* CALI_ST_CT_NP_REMOTE is set when host is accessing a remote nodeport. */
 	CALI_ST_CT_NP_REMOTE	  = 0x100,
+	/* CALI_ST_NAT_EXCLUDE is set when there is a NAT hit, but we don't want to resolve (such as node local DNS). */
+	CALI_ST_NAT_EXCLUDE       = 0x200,
 };
 
 struct fwd {

felix/bpf/counters/counters.go

@@ -72,6 +72,7 @@ const (
 	DroppedFailedDecap
 	DroppedUnauthSource
 	DroppedUnknownRoute
+	DroppedBlackholeRoute
 )
 
 type Description struct {
@@ -150,6 +151,10 @@ var descriptions DescList = DescList{
 		Counter:  DroppedUnknownRoute,
 		Category: "Dropped", Caption: "packets with unknown route",
 	},
+	{
+		Counter:  DroppedBlackholeRoute,
+		Category: "Dropped", Caption: "packets hitting blackhole route",
+	},
 }
 
 func Descriptions() DescList {

felix/bpf/routes/map.go

@@ -72,14 +72,16 @@ func (k Key) AsBytes() []byte {
 type Flags uint32
 
 const (
-	FlagInIPAMPool  Flags = 0x01
-	FlagNATOutgoing Flags = 0x02
-	FlagWorkload    Flags = 0x04
-	FlagLocal       Flags = 0x08
-	FlagHost        Flags = 0x10
-	FlagSameSubnet  Flags = 0x20
-	FlagTunneled    Flags = 0x40
-	FlagNoDSR       Flags = 0x80
+	FlagInIPAMPool      Flags = 0x01
+	FlagNATOutgoing     Flags = 0x02
+	FlagWorkload        Flags = 0x04
+	FlagLocal           Flags = 0x08
+	FlagHost            Flags = 0x10
+	FlagSameSubnet      Flags = 0x20
+	FlagTunneled        Flags = 0x40
+	FlagNoDSR           Flags = 0x80
+	FlagBlackHoleDrop   Flags = 0x100
+	FlagBlackHoleReject Flags = 0x200
 
 	FlagsUnknown            Flags = 0
 	FlagsRemoteWorkload           = FlagWorkload
@@ -136,7 +138,7 @@ func (v Value) String() string {
 
 	if typeFlags&FlagLocal != 0 {
 		parts = append(parts, "local")
-	} else {
+	} else if typeFlags&FlagBlackHoleDrop == 0 && typeFlags&FlagBlackHoleReject == 0 {
 		parts = append(parts, "remote")
 	}
 
@@ -174,6 +176,14 @@ func (v Value) String() string {
 		parts = append(parts, "nh", fmt.Sprint(v.NextHop()))
 	}
 
+	if typeFlags&FlagBlackHoleDrop != 0 {
+		parts = append(parts, "blackhole-drop")
+	}
+
+	if typeFlags&FlagBlackHoleReject != 0 {
+		parts = append(parts, "blackhole-reject")
+	}
+
 	if len(parts) == 0 {
 		return fmt.Sprintf("unknown type (%d)", typeFlags)
 	}

felix/bpf/routes/map6.go

@@ -77,7 +77,7 @@ func (v ValueV6) String() string {
 
 	if typeFlags&FlagLocal != 0 {
 		parts = append(parts, "local")
-	} else {
+	} else if typeFlags&FlagBlackHoleDrop == 0 && typeFlags&FlagBlackHoleReject == 0 {
 		parts = append(parts, "remote")
 	}
 
@@ -107,6 +107,14 @@ func (v ValueV6) String() string {
 		parts = append(parts, "tunneled")
 	}
 
+	if typeFlags&FlagBlackHoleDrop != 0 {
+		parts = append(parts, "blackhole-drop")
+	}
+
+	if typeFlags&FlagBlackHoleReject != 0 {
+		parts = append(parts, "blackhole-reject")
+	}
+
 	if typeFlags&FlagLocal != 0 && typeFlags&FlagWorkload != 0 {
 		parts = append(parts, "idx", fmt.Sprint(v.IfaceIndex()))
 	}

felix/bpf/ut/bpf_prog_test.go

@@ -717,7 +717,7 @@ func objLoad(fname, bpfFsDir, ipFamily string, topts testOpts, polProg, hasHostC
 					globals.Jumps[i] = uint32(i)
 				}
 
-				log.WithField("globals", globals).Debugf("configure program")
+				log.WithField("globals", globals).Debugf("configure program v6")
 
 				if err := tc.ConfigureProgramV6(m, ifaceLog, &globals); err != nil {
 					return nil, fmt.Errorf("failed to configure tc program: %w", err)

felix/bpf/ut/icmp_port_unreachable_test.go

@@ -22,7 +22,9 @@ import (
 	"github.com/google/gopacket/layers"
 	. "github.com/onsi/gomega"
 
+	"github.com/projectcalico/calico/felix/bpf/counters"
 	"github.com/projectcalico/calico/felix/bpf/nat"
+	"github.com/projectcalico/calico/felix/bpf/routes"
 	tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs"
 )
 
@@ -198,3 +200,101 @@ func checkICMPv6PortUnreachable(pktR gopacket.Packet, ipv6 *layers.IPv6) {
 		gopacket.NewPacket(cpkt.Bytes(), layers.LayerTypeEthernet, gopacket.Default).
 			Layer(layers.LayerTypeICMPv6).(*layers.ICMPv6).Checksum))
 }
+
+func TestSVCLoopPrevention(t *testing.T) {
+	RegisterTestingT(t)
+
+	iphdr := *ipv4Default
+
+	_, ipv4, _, _, pktBytesV4, err := testPacketV4(nil, &iphdr, nil, nil)
+	Expect(err).NotTo(HaveOccurred())
+	_ = ipv4
+	rtKey := routes.NewKey(dstV4CIDR).AsBytes()
+	rtVal := routes.NewValueWithIfIndex(routes.FlagBlackHoleDrop, 1).AsBytes()
+	err = rtMap.Update(rtKey, rtVal)
+	Expect(err).NotTo(HaveOccurred())
+	_, ipv6, _, _, pktBytesV6, err := testPacketV6(nil, ipv6Default, nil, nil)
+	Expect(err).NotTo(HaveOccurred())
+
+	rtKeyV6 := routes.NewKeyV6(dstV6CIDR).AsBytes()
+	rtValV6 := routes.NewValueV6WithIfIndex(routes.FlagBlackHoleReject, 1).AsBytes()
+	err = rtMapV6.Update(rtKeyV6, rtValV6)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Insert a reverse route for the source workload.
+	rtKeyW := routes.NewKey(srcV4CIDR).AsBytes()
+	rtValW := routes.NewValueWithIfIndex(routes.FlagsLocalWorkload|routes.FlagInIPAMPool, 1).AsBytes()
+	err = rtMap.Update(rtKeyW, rtValW)
+	Expect(err).NotTo(HaveOccurred())
+
+	defer func() {
+		err := rtMap.Delete(rtKey)
+		Expect(err).NotTo(HaveOccurred())
+		err = rtMapV6.Delete(rtKeyV6)
+		Expect(err).NotTo(HaveOccurred())
+		err = rtMap.Delete(rtKeyW)
+		Expect(err).NotTo(HaveOccurred())
+	}()
+
+	// Test with action = drop
+	skbMark = 0
+	runBpfTest(t, "calico_from_host_ep", nil, func(bpfrun bpfProgRunFn) {
+		res, err := bpfrun(pktBytesV4)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.RetvalStr()).To(Equal("TC_ACT_SHOT"), "expected program to return TC_ACT_SHOT")
+
+		pktR := gopacket.NewPacket(res.dataOut, layers.LayerTypeEthernet, gopacket.Default)
+		fmt.Printf("pktR = %+v\n", pktR)
+		bpfCounters, err := counters.Read(countersMap, 1, 0)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(int(bpfCounters[counters.DroppedBlackholeRoute])).To(Equal(1))
+	})
+
+	skbMark = 0
+	runBpfTest(t, "calico_from_workload_ep", nil, func(bpfrun bpfProgRunFn) {
+		res, err := bpfrun(pktBytesV4)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.RetvalStr()).To(Equal("TC_ACT_SHOT"), "expected program to return TC_ACT_SHOT")
+
+		pktR := gopacket.NewPacket(res.dataOut, layers.LayerTypeEthernet, gopacket.Default)
+		fmt.Printf("pktR = %+v\n", pktR)
+		bpfCounters, err := counters.Read(countersMap, 1, 0)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(int(bpfCounters[counters.DroppedBlackholeRoute])).To(Equal(2))
+	})
+
+	rtVal = routes.NewValueWithIfIndex(routes.FlagBlackHoleReject, 1).AsBytes()
+	err = rtMap.Update(rtKey, rtVal)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Test with action = reject
+	skbMark = 0
+	runBpfTest(t, "calico_from_host_ep", nil, func(bpfrun bpfProgRunFn) {
+		res, err := bpfrun(pktBytesV4)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.RetvalStr()).To(Equal("TC_ACT_UNSPEC"), "expected program to return TC_ACT_UNSPEC")
+
+		pktR := gopacket.NewPacket(res.dataOut, layers.LayerTypeEthernet, gopacket.Default)
+		fmt.Printf("pktR = %+v\n", pktR)
+
+		checkICMPPortUnreachable(pktR, ipv4)
+		bpfCounters, err := counters.Read(countersMap, 1, 0)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(int(bpfCounters[counters.DroppedBlackholeRoute])).To(Equal(3))
+	})
+
+	skbMark = 0
+	runBpfTest(t, "calico_from_host_ep", nil, func(bpfrun bpfProgRunFn) {
+		res, err := bpfrun(pktBytesV6)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.RetvalStr()).To(Equal("TC_ACT_UNSPEC"), "expected program to return TC_ACT_UNSPEC")
+
+		pktR := gopacket.NewPacket(res.dataOut, layers.LayerTypeEthernet, gopacket.Default)
+		fmt.Printf("pktR = %+v\n", pktR)
+
+		checkICMPv6PortUnreachable(pktR, ipv6)
+		bpfCounters, err := counters.Read(countersMap, 1, 0)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(int(bpfCounters[counters.DroppedBlackholeRoute])).To(Equal(4))
+	}, withIPv6())
+}

felix/dataplane/driver.go

@@ -379,6 +379,7 @@ func StartDataplaneDriver(configParams *config.Config,
 			RouteTableManager:                  routeTableIndexAllocator,
 			MTUIfacePattern:                    configParams.MTUIfacePattern,
 			BPFExcludeCIDRsFromNAT:             configParams.BPFExcludeCIDRsFromNAT,
+			ServiceLoopPrevention:              configParams.ServiceLoopPrevention,
 
 			KubeClientSet: k8sClientSet,