feat(capture): Add --backend=cgroup-skb support for cgroup-based pa…

…cket capture (#208) * feat(cli): Add `--backend=cgroup-skb` support for cgroup-based packet capture * improve docs * skip inject pcap filter to cgroup programs when backend is cgroup-skb * fix typo * fix ci * fix ci
mozillazg · Dec 7, 2024 · 0308649 · 0308649
1 parent 1d3d114
commit 0308649
Show file tree

Hide file tree

Showing 32 changed files with 586 additions and 100 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -79,7 +79,10 @@ jobs:
       - run:
           name: e2e (test netns)
           command: |
-            sudo bash testdata/test_netns.sh ./ptcpdump
+            for i in {1..5}; do
+              sudo bash testdata/test_netns.sh ./ptcpdump && exit 0 || sleep 1
+            done
+            exit 1
 
       - run:
           name: e2e (test netns newly)
@@ -274,6 +277,30 @@ jobs:
           command: |
             sudo bash testdata/run_test_k8s_filter_by_pod_2.sh
 
+  test-backend:
+    machine:
+      image: ubuntu-2204:2024.04.4
+    resource_class: medium
+    steps:
+      - checkout
+
+      - run:
+          name: build
+          command: |
+            make build-via-docker
+
+            echo '========== info =========='
+            uname -a
+            cat /etc/issue
+            file ./ptcpdump
+      - run:
+          name: test cgroup-skb
+          command: |
+            set -ex
+            sudo bash testdata/test_cgroup_skb_base_pcap.sh ./ptcpdump
+            sudo bash testdata/test_cgroup_skb_base_pcapng.sh ./ptcpdump
+            sudo bash testdata/test_cgroup_skb_filter_ifindex.sh ./ptcpdump
+
 workflows:
   e2e:
     jobs:
@@ -287,3 +314,4 @@ workflows:
       - docker-e2e
       - containerd-e2e
       - k8s-1-30-e2e
+      - test-backend
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -47,6 +47,27 @@ jobs:
         run: |
             bash testdata/test_run_with_docker.sh "quay.io/ptcpdump/ptcpdump:latest"
 
+  test-backend:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 5
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - name: Retrieve stored ptcpdump executable
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        with:
+          name: ptcpdump
+          path: ptcpdump
+
+      - name: test cgroup-skb
+        run: |
+          set -ex
+          chmod +x ./ptcpdump/ptcpdump
+          sudo bash testdata/test_cgroup_skb_base_pcap.sh ./ptcpdump/ptcpdump
+          sudo bash testdata/test_cgroup_skb_base_pcapng.sh ./ptcpdump/ptcpdump
+          sudo bash testdata/test_cgroup_skb_filter_ifindex.sh ./ptcpdump/ptcpdump
+
   e2e-test:
     name: e2e-test
     needs: build

diff --git a/README.md b/README.md
@@ -7,8 +7,8 @@
 English | [中文](README.zh-CN.md)
 
 
-ptcpdump is the tcpdump(8) implementation using eBPF, with an extra feature:
-it adds process info as packet comments for each Packet when possible.
+ptcpdump is an eBPF-based implementation of tcpdump that includes an additional feature:
+it adds process information as comments for each packet when available.
 Inspired by [jschwinger233/skbdump](https://github.com/jschwinger233/skbdump).
 
 ![](./docs/wireshark.png)
@@ -23,6 +23,7 @@ Table of Contents
     * [Example commands](#example-commands)
     * [Example output](#example-output)
     * [Running with Docker](#running-with-docker)
+    * [Backend](#backend)
     * [Flags](#flags)
 * [Compare with tcpdump](#compare-with-tcpdump)
 * [Developing](#developing)
@@ -229,6 +230,24 @@ docker run --privileged --rm -t --net=host --pid=host \
 <p align="right"><a href="#top">🔝</a></p>
 
 
+### Backend
+
+
+ptcpdump supports specifying a particular eBPF technology for packet capture through the
+`--backend` flag.
+
+| --backend    | eBPF Program Type          | Include the Layer 2 data |
+|--------------|----------------------------|--------------------------|
+| `tc`         | `BPF_PROG_TYPE_SCHED_ACT`  | ✅                        |
+| `cgroup-skb` | `BPF_PROG_TYPE_CGROUP_SKB` | ❌                        |
+
+
+If this flag isn't specified, it defaults to `tc`.
+
+
+<p align="right"><a href="#top">🔝</a></p>
+
+
 ### Flags
 
 
@@ -252,6 +271,7 @@ Examples:
 Expression: see "man 7 pcap-filter"
 
 Flags:
+      --backend string                               Specify the backend to use for capturing packets. Possible values are "tc" and "cgroup-skb" (default "tc")
       --container-id string                          Filter by container id (only TCP and UDP packets are supported)
       --container-name string                        Filter by container name (only TCP and UDP packets are supported)
       --containerd-address string                    Address of containerd service (default "/run/containerd/containerd.sock")

diff --git a/README.zh-CN.md b/README.zh-CN.md
@@ -25,6 +25,7 @@ Table of Contents
     * [Example commands](#example-commands)
     * [Example output](#example-output)
     * [Running with Docker](#running-with-docker)
+    * [Backend](#backend)
     * [Flags](#flags)
 * [Compare with tcpdump](#compare-with-tcpdump)
 * [Developing](#developing)
@@ -235,6 +236,23 @@ docker run --privileged --rm -t --net=host --pid=host \
 <p align="right"><a href="#top">🔝</a></p>
 
 
+### Backend
+
+
+ptcpdump 支持通过 --backend 参数指定使用特定的 eBPF 技术进行抓包。
+
+| --backend    | eBPF Program Type          | 结果包含 L2 链路层数据 |
+|--------------|----------------------------|---------------|
+| `tc`         | `BPF_PROG_TYPE_SCHED_ACT`  | ✅             |
+| `cgroup-skb` | `BPF_PROG_TYPE_CGROUP_SKB` | ❌             |
+
+
+该参数未指定时的默认值是 `tc`。
+
+
+<p align="right"><a href="#top">🔝</a></p>
+
+
 ### Flags
 
 
@@ -258,6 +276,7 @@ Examples:
 Expression: see "man 7 pcap-filter"
 
 Flags:
+      --backend string                               Specify the backend to use for capturing packets. Possible values are "tc" and "cgroup-skb" (default "tc")
       --container-id string                          Filter by container id (only TCP and UDP packets are supported)
       --container-name string                        Filter by container name (only TCP and UDP packets are supported)
       --containerd-address string                    Address of containerd service (default "/run/containerd/containerd.sock")

diff --git a/bpf/bpf.go b/bpf/bpf.go
@@ -49,10 +49,12 @@ type Options struct {
 	mntnsIds       []uint32
 	pidnsIds       []uint32
 	netnsIds       []uint32
+	ifindexes      []uint32
 	maxPayloadSize uint32
 	hookMount      bool
 	hookNetDev     bool
 	kernelTypes    *btf.Spec
+	backend        types.NetHookBackend
 }
 
 func NewBPF() (*BPF, error) {
@@ -99,13 +101,17 @@ func (b *BPF) Load(opts Options) error {
 	log.Infof("load with opts: %#v", opts)
 	var err error
 
+	b.opts = opts
 	config := BpfGconfigT{
 		HaveFilter:        opts.haveFilter,
 		FilterFollowForks: opts.followForks,
 		FilterComm:        opts.comm,
 		FilterCommEnable:  opts.filterComm,
 		MaxPayloadSize:    opts.maxPayloadSize,
 	}
+	if len(opts.ifindexes) > 0 {
+		config.FilterIfindexEnable = 1
+	}
 	if !b.isLegacyKernel {
 		log.Infof("rewrite constants with %+v", config)
 		err = b.spec.RewriteConstants(map[string]interface{}{
@@ -117,28 +123,8 @@ func (b *BPF) Load(opts Options) error {
 	}
 
 	if opts.pcapFilter != "" {
-		for _, progName := range []string{"tc_ingress", "tc_egress", "tcx_ingress", "tcx_egress"} {
-			prog, ok := b.spec.Programs[progName]
-			if !ok {
-				log.Infof("program %s not found", progName)
-				continue
-			}
-			if prog == nil {
-				log.Infof("program %s is nil", progName)
-				continue
-			}
-			prog.Instructions, err = elibpcap.Inject(
-				opts.pcapFilter,
-				prog.Instructions,
-				elibpcap.Options{
-					AtBpf2Bpf:  "pcap_filter",
-					DirectRead: true,
-					L2Skb:      true,
-				},
-			)
-			if err != nil {
-				return fmt.Errorf("inject pcap filter: %w", err)
-			}
+		if err := b.injectPcapFilter(); err != nil {
+			return fmt.Errorf(": %w", err)
 		}
 	}
 
@@ -155,8 +141,6 @@ func (b *BPF) Load(opts Options) error {
 		return fmt.Errorf("bpf load and assign: %w", err)
 	}
 
-	b.opts = opts
-
 	if b.isLegacyKernel {
 		log.Infof("update config map with %+v", config)
 		key := uint32(0)
@@ -171,6 +155,43 @@ func (b *BPF) Load(opts Options) error {
 	return nil
 }
 
+func (b *BPF) injectPcapFilter() error {
+	var err error
+	for _, progName := range []string{"tc_ingress", "tc_egress", "tcx_ingress", "tcx_egress",
+		"cgroup_skb__ingress", "cgroup_skb__egress"} {
+		prog, ok := b.spec.Programs[progName]
+		if !ok {
+			log.Infof("program %s not found", progName)
+			continue
+		}
+		if prog == nil {
+			log.Infof("program %s is nil", progName)
+			continue
+		}
+		l2skb := true
+		if strings.Contains(progName, "cgroup_skb") {
+			l2skb = false
+			if b.opts.backend != types.NetHookBackendCgroupSkb {
+				continue
+			}
+		}
+		log.Infof("inject pcap filter to %s", progName)
+		prog.Instructions, err = elibpcap.Inject(
+			b.opts.pcapFilter,
+			prog.Instructions,
+			elibpcap.Options{
+				AtBpf2Bpf:  "pcap_filter",
+				DirectRead: true,
+				L2Skb:      l2skb,
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("inject pcap filter to %s: %w", progName, err)
+		}
+	}
+	return nil
+}
+
 func (b *BPF) Close() {
 	for _, lk := range b.links {
 		if err := lk.Close(); err != nil {
@@ -507,6 +528,14 @@ func (b *BPF) applyFilters() error {
 		}
 	}
 
+	log.Infof("start to update FilterIfindexMap with %+v", opts.ifindexes)
+	for _, id := range opts.ifindexes {
+		id := id
+		if err := b.objs.BpfMaps.FilterIfindexMap.Update(id, value, ebpf.UpdateAny); err != nil {
+			return fmt.Errorf("update FilterIfindexMap: %w", err)
+		}
+	}
+
 	return nil
 }
 
@@ -608,3 +637,18 @@ func (opts *Options) WithKernelTypes(spec *btf.Spec) *Options {
 	opts.kernelTypes = spec
 	return opts
 }
+
+func (opts *Options) WithBackend(backend types.NetHookBackend) *Options {
+	opts.backend = backend
+	return opts
+}
+
+func (opts *Options) WithIfindexes(ifindexes []uint32) *Options {
+	for _, id := range ifindexes {
+		if id == 0 {
+			continue
+		}
+		opts.ifindexes = append(opts.ifindexes, id)
+	}
+	return opts
+}
diff --git a/bpf/bpf_arm64_bpfel.go b/bpf/bpf_arm64_bpfel.go
diff --git a/bpf/bpf_arm64_bpfel.o b/bpf/bpf_arm64_bpfel.o