Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BPF] detach cali programs from devices no longer in regex #7008

Merged
merged 5 commits into from
Nov 25, 2022
Merged

[BPF] detach cali programs from devices no longer in regex #7008

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bde28a9
[BPF] cleaning up maps is for both TC and XDP
tomastigera Nov 17, 2022
1794d15
[BPF] bpfEndpointManager.ensureStarted should exec once in tests too.
tomastigera Nov 17, 2022
a3dee7d
[BPF] add ListCalicoAttached() to list calico programs
tomastigera Nov 17, 2022
ff48445
[BPF] detach cali programs from devices no longer in regex
tomastigera Nov 18, 2022
05ea9a9
[BPF] fv test for prog cleanups
tomastigera Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions felix/bpf/attach.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,73 @@ func sha256OfFile(name string) (string, error) {
}
return hex.EncodeToString(hasher.Sum(nil)), nil
}

// EPAttachInfo tells what programs are attached to an endpoint.
type EPAttachInfo struct {
TCId int
XDPId int
XDPMode string
}

// ListCalicoAttached list all programs that are attached to TC or XDP and are
// related to Calico. That is, they have jumpmap pinned in our dir hierarchy.
func ListCalicoAttached() (map[string]EPAttachInfo, error) {
aTC, aXDP, err := ListTcXDPAttachedProgs()
if err != nil {
return nil, err
}

attachedProgIDs := set.New[int]()

for _, p := range aTC {
attachedProgIDs.Add(p.ID)
}

for _, p := range aXDP {
attachedProgIDs.Add(p.ID)
}

maps, err := ListPerEPMaps()
if err != nil {
return nil, err
}

allProgs, err := GetAllProgs()
if err != nil {
return nil, err
}

caliProgs := set.New[int]()

for _, p := range allProgs {
if !attachedProgIDs.Contains(p.Id) {
continue
}

for _, m := range p.MapIds {
if _, ok := maps[m]; ok {
caliProgs.Add(p.Id)
break
}
}
}

ai := make(map[string]EPAttachInfo)

for _, p := range aTC {
if caliProgs.Contains(p.ID) {
ai[p.DevName] = EPAttachInfo{TCId: p.ID}
}
}

for _, p := range aXDP {
if caliProgs.Contains(p.ID) {
info := ai[p.DevName]
info.XDPId = p.ID
info.XDPMode = p.Mode
ai[p.DevName] = info
}
}

return ai, nil
}
189 changes: 183 additions & 6 deletions felix/bpf/bpf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package bpf

import (
"bufio"
"bytes"
"encoding/binary"
"encoding/json"
"errors"
Expand All @@ -45,6 +46,7 @@ import (
"github.com/projectcalico/calico/felix/environment"
"github.com/projectcalico/calico/felix/labelindex"
"github.com/projectcalico/calico/felix/proto"
"github.com/projectcalico/calico/libcalico-go/lib/set"
)

// Hook is the hook to which a BPF program should be attached. This is relative to
Expand Down Expand Up @@ -528,7 +530,7 @@ type perCpuMapEntry []struct {
} `json:"values"`
}

type progInfo struct {
type ProgInfo struct {
Id int `json:"id"`
Type string `json:"type"`
Tag string `json:"tag"`
Expand Down Expand Up @@ -1096,7 +1098,7 @@ func (b *BPFLib) GetXDPTag(ifName string) (string, error) {
return "", fmt.Errorf("failed to show XDP program (%s): %s\n%s", progPath, err, output)
}

p := progInfo{}
p := ProgInfo{}
err = json.Unmarshal(output, &p)
if err != nil {
return "", fmt.Errorf("cannot parse json output: %v\n%s", err, output)
Expand Down Expand Up @@ -1186,7 +1188,7 @@ func (b *BPFLib) GetMapsFromXDP(ifName string) ([]int, error) {
if err != nil {
return nil, fmt.Errorf("failed to show XDP program (%s): %s\n%s", progPath, err, output)
}
p := progInfo{}
p := ProgInfo{}
err = json.Unmarshal(output, &p)
if err != nil {
return nil, fmt.Errorf("cannot parse json output: %v\n%s", err, output)
Expand Down Expand Up @@ -1602,7 +1604,11 @@ func (b *BPFLib) getSkMsgID() (int, error) {
return -1, nil
}

func getAllProgs() ([]progInfo, error) {
func GetAllProgs() ([]ProgInfo, error) {
return getAllProgs()
}

func getAllProgs() ([]ProgInfo, error) {
prog := "bpftool"
args := []string{
"--json",
Expand All @@ -1617,7 +1623,7 @@ func getAllProgs() ([]progInfo, error) {
return nil, fmt.Errorf("failed to get progs: %s\n%s", err, output)
}

var progs []progInfo
var progs []ProgInfo
err = json.Unmarshal(output, &progs)
if err != nil {
return nil, fmt.Errorf("cannot parse json output: %v\n%s", err, output)
Expand Down Expand Up @@ -1672,7 +1678,7 @@ func (b *BPFLib) getSockMapID(progID int) (int, error) {
return -1, fmt.Errorf("failed to get sockmap ID for prog %d: %s\n%s", progID, err, output)
}

p := progInfo{}
p := ProgInfo{}
err = json.Unmarshal(output, &p)
if err != nil {
return -1, fmt.Errorf("cannot parse json output: %v\n%s", err, output)
Expand Down Expand Up @@ -2312,3 +2318,174 @@ func MapPinPath(typ int, name, iface string, hook Hook) string {
}
return path.Join(PinBaseDir, subDir, name)
}

type TcList []struct {
DevName string `json:"devname"`
ID int `json:"id"`
}

type XDPList []struct {
DevName string `json:"devname"`
IfIndex int `json:"ifindex"`
Mode string `json:"mode"`
ID int `json:"id"`
}

// ListTcXDPAttachedProgs returns all programs attached to TC or XDP hooks.
func ListTcXDPAttachedProgs() (TcList, XDPList, error) {
// Find all the programs that are attached to interfaces.
out, err := exec.Command("bpftool", "net", "-j").Output()
if err != nil {
return nil, nil, fmt.Errorf("failed to list attached bpf programs: %w", err)
}

var attached []struct {
TC TcList `json:"tc"`
XDP XDPList `json:"xdp"`
}

err = json.Unmarshal(out, &attached)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse list of attached BPF programs: %w\n%s", err, out)
}

return attached[0].TC, attached[0].XDP, nil
}

func ListPerEPMaps() (map[int]string, error) {
mapIDToPath := make(map[int]string)
err := filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if strings.HasPrefix(info.Name(), JumpMapName()) ||
strings.HasPrefix(info.Name(), CountersMapName()) {
log.WithField("path", p).Debug("Examining map")

out, err := exec.Command("bpftool", "map", "show", "pinned", p).Output()
if err != nil {
log.WithError(err).Panic("Failed to show map")
}
log.WithField("dump", string(out)).Debug("Map show before deletion")
idStr := string(bytes.Split(out, []byte(":"))[0])
id, err := strconv.Atoi(idStr)
if err != nil {
log.WithError(err).WithField("dump", string(out)).Error("Failed to parse bpftool output.")
return err
}
mapIDToPath[id] = p
}
return nil
})

return mapIDToPath, err
}

// pinDirRegex matches tc's and xdp's auto-created directory names, directories created when using libbpf
// so we can clean them up when removing maps without accidentally removing other user-created dirs..
var pinDirRegex = regexp.MustCompile(`([0-9a-f]{40})|(.*_(igr|egr|xdp))`)

// CleanUpMaps scans for cali_jump maps that are still pinned to the filesystem but no longer referenced by
// our BPF programs.
func CleanUpMaps() {
// Find the maps we care about by walking the BPF filesystem.
mapIDToPath, err := ListPerEPMaps()
if os.IsNotExist(err) {
log.WithError(err).Warn("tc directory missing from BPF file system?")
return
}
if err != nil {
log.WithError(err).Error("Error while looking for maps.")
return
}

aTc, aXdp, err := ListTcXDPAttachedProgs()
if err != nil {
log.WithError(err).Warn("Failed to list attached programs.")
return
}
log.WithFields(log.Fields{"tc": aTc, "xdp": aXdp}).Debug("Attached BPF programs")

attachedProgs := set.New[int]()
for _, prog := range aTc {
log.WithField("prog", prog).Debug("Adding TC prog to attached set")
attachedProgs.Add(prog.ID)
}
for _, prog := range aXdp {
log.WithField("prog", prog).Debug("Adding XDP prog to attached set")
attachedProgs.Add(prog.ID)
}

// Find all the maps that the attached programs refer to and remove them from consideration.
progsJSON, err := exec.Command("bpftool", "prog", "list", "--json").Output()
if err != nil {
log.WithError(err).Info("Failed to list BPF programs, assuming there's nothing to clean up.")
return
}
var progs []struct {
ID int `json:"id"`
Name string `json:"name"`
Maps []int `json:"map_ids"`
}
err = json.Unmarshal(progsJSON, &progs)
if err != nil {
log.WithError(err).Info("Failed to parse bpftool output. Assuming nothing to clean up.")
return
}
for _, p := range progs {
if !attachedProgs.Contains(p.ID) {
log.WithField("prog", p).Debug("Prog is not in the attached set, skipping")
continue
}
for _, id := range p.Maps {
log.WithField("mapID", id).WithField("prog", p).Debugf("Map is still in use: %v", mapIDToPath[id])
delete(mapIDToPath, id)
}
}

// Remove the pins.
for id, p := range mapIDToPath {
log.WithFields(log.Fields{"id": id, "path": p}).Debug("Removing stale BPF map pin.")
err := os.Remove(p)
if err != nil {
log.WithError(err).Warn("Removed stale BPF map pin.")
}
log.WithFields(log.Fields{"id": id, "path": p}).Info("Removed stale BPF map pin.")
}

// Look for empty dirs.
emptyAutoDirs := set.New[string]()
err = filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() && pinDirRegex.MatchString(info.Name()) {
p := path.Clean(p)
log.WithField("path", p).Debug("Found tc auto-created dir.")
emptyAutoDirs.Add(p)
} else {
dirPath := path.Clean(path.Dir(p))
if emptyAutoDirs.Contains(dirPath) {
log.WithField("path", dirPath).Debug("tc dir is not empty.")
emptyAutoDirs.Discard(dirPath)
}
}
return nil
})
if os.IsNotExist(err) {
log.WithError(err).Warn("tc directory missing from BPF file system?")
return
}
if err != nil {
log.WithError(err).Error("Error while looking for maps.")
}

emptyAutoDirs.Iter(func(p string) error {
log.WithField("path", p).Debug("Removing empty dir.")
err := os.Remove(p)
if err != nil {
log.WithError(err).Error("Error while removing empty dir.")
}
return nil
})
}
Loading