From 23346cfb9823141c9f9acabaf391c2119f1d4073 Mon Sep 17 00:00:00 2001 From: utam0k Date: Mon, 28 Mar 2022 15:49:13 +0900 Subject: [PATCH] try using netlink instead of cmd. Co-authored-by: Prince Rachit Sinha --- components/workspacekit/cmd/rings.go | 2 +- .../ws-daemon-api/workspace_daemon.proto | 2 +- components/ws-daemon/nsinsider/go.mod | 3 +- components/ws-daemon/nsinsider/go.sum | 8 +- components/ws-daemon/nsinsider/main.go | 148 ++++++------------ components/ws-daemon/pkg/iws/iws.go | 14 +- 6 files changed, 73 insertions(+), 104 deletions(-) diff --git a/components/workspacekit/cmd/rings.go b/components/workspacekit/cmd/rings.go index 2e4ce8cffb5971..a422e3be08d030 100644 --- a/components/workspacekit/cmd/rings.go +++ b/components/workspacekit/cmd/rings.go @@ -502,7 +502,7 @@ var ring1Cmd = &cobra.Command{ } _, err = client.SetupPairVeths(ctx, &daemonapi.SetupPairVethsRequest{Pid: int64(cmd.Process.Pid)}) if err != nil { - log.WithError(err).Error("can not set up pair of veths") + log.WithError(err).Error("cannot setup pair of veths") return } client.Close() diff --git a/components/ws-daemon-api/workspace_daemon.proto b/components/ws-daemon-api/workspace_daemon.proto index 7f2a67fd88401b..7802e5d7ee4a0a 100644 --- a/components/ws-daemon-api/workspace_daemon.proto +++ b/components/ws-daemon-api/workspace_daemon.proto @@ -46,7 +46,7 @@ service InWorkspaceService { // when the workspace is about to shut down, e.g. using the PreStop hook of a Kubernetes container. rpc Teardown(TeardownRequest) returns (TeardownResponse) {} - // Set up a pair of veths that interconnect the specified PID and the workspace container's network namename. + // Set up a pair of veths that interconnect the specified PID and the workspace container's network namespace. rpc SetupPairVeths(SetupPairVethsRequest) returns (SetupPairVethsResponse) {} } diff --git a/components/ws-daemon/nsinsider/go.mod b/components/ws-daemon/nsinsider/go.mod index 48fd0ab0b0eec3..8f9212c914596c 100644 --- a/components/ws-daemon/nsinsider/go.mod +++ b/components/ws-daemon/nsinsider/go.mod @@ -5,9 +5,9 @@ go 1.17 replace github.com/seccomp/libseccomp-golang => github.com/gitpod-io/libseccomp-golang v0.9.2-0.20220203100026-45179215fdb1 // leeway indirect from components/workspacekit:lib require ( - github.com/coreos/go-iptables v0.6.0 github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000 github.com/urfave/cli/v2 v2.3.0 + github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ) @@ -18,6 +18,7 @@ require ( github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/stretchr/testify v1.7.0 // indirect + github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/components/ws-daemon/nsinsider/go.sum b/components/ws-daemon/nsinsider/go.sum index 9c4bef2b6d2734..652bc825eddc1e 100644 --- a/components/ws-daemon/nsinsider/go.sum +++ b/components/ws-daemon/nsinsider/go.sum @@ -59,8 +59,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -265,6 +263,10 @@ github.com/uber/jaeger-client-go v2.29.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -399,6 +401,7 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -410,6 +413,7 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/components/ws-daemon/nsinsider/main.go b/components/ws-daemon/nsinsider/main.go index 57809d9f228d9d..a32defe8aca11e 100644 --- a/components/ws-daemon/nsinsider/main.go +++ b/components/ws-daemon/nsinsider/main.go @@ -20,6 +20,7 @@ import ( "github.com/gitpod-io/gitpod/common-go/log" _ "github.com/gitpod-io/gitpod/common-go/nsenter" + "github.com/vishvananda/netlink" ) func main() { @@ -263,112 +264,47 @@ func main() { Name: "target-pid", Required: true, }, - &cli.StringFlag{ - Name: "name", - Required: true, - }, }, Action: func(c *cli.Context) error { containerIf, vethIf, cethIf := "eth0", "veth0", "ceth0" mask := net.IPv4Mask(255, 255, 255, 0) - vethAddr := net.IPNet{ + vethIp := net.IPNet{ IP: net.IPv4(10, 0, 5, 1), Mask: mask, } - cethAddr := net.IPNet{ - IP: net.IPv4(10, 0, 5, 2), - Mask: mask, - } masqueradeAddr := net.IPNet{ - IP: vethAddr.IP.Mask(mask), + IP: vethIp.IP.Mask(mask), Mask: mask, } - ipCmd, iptablesCmd := "/usr/sbin/ip", "/usr/sbin/iptables" - netns := "workspace-ns" - - cmd := exec.Command(ipCmd, "link", "add", vethIf, "type", "veth", "peer", "name", cethIf) - out, err := cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("create a veth pair (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) - } - - path := "/var/run/netns" - if err := os.MkdirAll(path, 0755); err != nil { - return xerrors.Errorf("create a dir %s failed: %v", path, err) + iptablesCmd := "/usr/sbin/iptables" + targetPid := c.Int("target-pid") + + veth := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: vethIf, + Flags: net.FlagUp, + MTU: 1500, + }, + PeerName: cethIf, + PeerNamespace: netlink.NsPid(targetPid), } - if err := os.Symlink(fmt.Sprintf("/proc/%d/ns/net", c.Int("target-pid")), filepath.Join(path, netns)); err != nil { - return xerrors.Errorf("create a symlink to netns (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) - } - cmd = exec.Command(ipCmd, "link", "set", cethIf, "netns", netns) - out, err = cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("link cethIf to netns (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) + if err := netlink.LinkAdd(veth); err != nil { + return xerrors.Errorf("link %q-%q netns failed: %v", vethIf, cethIf, err) } - cmd = exec.Command(ipCmd, "addr", "add", vethAddr.String(), "dev", vethIf) - out, err = cmd.CombinedOutput() + vethLink, err := netlink.LinkByName(vethIf) if err != nil { - return xerrors.Errorf("assign IP address to the vethIf (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) + return xerrors.Errorf("cannot found %q netns failed: %v", vethIf, err) } - - cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "addr", "add", cethAddr.String(), "dev", cethIf) - out, err = cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("assign IP address to the cethIf (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) + if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIp}); err != nil { + return xerrors.Errorf("failed to add IP address to %q: %v", vethIf, err) } - - cmd = exec.Command(ipCmd, "link", "set", vethIf, "up") - out, err = cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("bring up the vethIf (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) - } - - cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "link", "set", cethIf, "up") - out, err = cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("bring up the cethIf (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) - } - - cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "link", "set", "lo", "up") - out, err = cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("bring up the lo (%v) failed: %q\n%v", - cmd.Args, - string(out), - err, - ) + if err := netlink.LinkSetUp(vethLink); err != nil { + return xerrors.Errorf("failed to enable %q: %v", vethIf, err) } - cmd = exec.Command(iptablesCmd, "-A", "FORWARD", "-o", containerIf, "-i", vethIf, "-j", "ACCEPT") - out, err = cmd.CombinedOutput() + cmd := exec.Command(iptablesCmd, "-A", "FORWARD", "-o", containerIf, "-i", vethIf, "-j", "ACCEPT") + out, err := cmd.CombinedOutput() if err != nil { return xerrors.Errorf("add a forwarding rule for iptable: vethIf -> eth0 (%v) failed: %q\n%v", cmd.Args, @@ -397,17 +333,33 @@ func main() { ) } - // TODO(toru) if we will implement the dynamically ports exporsing, these are commented out. - // cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "route", "replace", "default", "via", vethAddr.IP.String()) - // out, err = cmd.CombinedOutput() - // if err != nil { - // return xerrors.Errorf("change up a default (%v) failed: %q\n%v", - // cmd.Args, - // string(out), - // err, - // ) - // } + return nil + }, + }, + { + Name: "setup-peer-veth", + Usage: "set up a peer veth", + Action: func(c *cli.Context) error { + cethIf := "ceth0" + mask := net.IPv4Mask(255, 255, 255, 0) + cethIp := net.IPNet{ + IP: net.IPv4(10, 0, 5, 2), + Mask: mask, + } + + cethLink, err := netlink.LinkByName(cethIf) + if err != nil { + return xerrors.Errorf("cannot found %q netns failed: %v", cethIf, err) + } + if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIp}); err != nil { + return xerrors.Errorf("failed to add IP address to %q: %v", cethIf, err) + } + if err := netlink.LinkSetUp(cethLink); err != nil { + return xerrors.Errorf("failed to enable %q: %v", cethIf, err) + } + // TODO(toru): when we will complete implementing the dynamically ports exporsing, + // we have to implement changing default gw here. return nil }, }, diff --git a/components/ws-daemon/pkg/iws/iws.go b/components/ws-daemon/pkg/iws/iws.go index b819c0e79ac6c7..a2dd71002d9742 100644 --- a/components/ws-daemon/pkg/iws/iws.go +++ b/components/ws-daemon/pkg/iws/iws.go @@ -325,13 +325,25 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap } err = nsinsider(wbs.Session.InstanceID, int(containerPID), func(c *exec.Cmd) { - c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid)), "--name", wbs.Session.InstanceID) + c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid))) }, enterMountNS(true), enterPidNS(true), enterNetNS(true)) if err != nil { log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a pair of veths") return nil, status.Errorf(codes.Internal, "cannot setup a pair of veths") } + pid, err := wbs.Uidmapper.findHostPID(containerPID, uint64(req.Pid)) + if err != nil { + return nil, xerrors.Errorf("cannot map in-container PID %d (container PID: %d): %w", req.Pid, containerPID, err) + } + err = nsinsider(wbs.Session.InstanceID, int(pid), func(c *exec.Cmd) { + c.Args = append(c.Args, "setup-peer-veth") + }, enterMountNS(true), enterPidNS(true), enterNetNS(true)) + if err != nil { + log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a peer veths") + return nil, status.Errorf(codes.Internal, "cannot setup a peer veths") + } + return &api.SetupPairVethsResponse{}, nil }