Skip to content

Commit

Permalink
try using netlink instead of cmd.
Browse files Browse the repository at this point in the history 
Co-authored-by: Prince Rachit Sinha <[email protected]>
  • Loading branch information
utam0k and Prince Rachit Sinha committed Mar 28, 2022
1 parent 06b6656 commit 23346cf
Show file tree
Hide file tree
Showing 6 changed files with 73 additions and 104 deletions.
2 changes: 1 addition & 1 deletion components/workspacekit/cmd/rings.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ var ring1Cmd = &cobra.Command{
}
_, err = client.SetupPairVeths(ctx, &daemonapi.SetupPairVethsRequest{Pid: int64(cmd.Process.Pid)})
if err != nil {
log.WithError(err).Error("can not set up pair of veths")
log.WithError(err).Error("cannot setup pair of veths")
return
}
client.Close()
Expand Down
2 changes: 1 addition & 1 deletion components/ws-daemon-api/workspace_daemon.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ service InWorkspaceService {
// when the workspace is about to shut down, e.g. using the PreStop hook of a Kubernetes container.
rpc Teardown(TeardownRequest) returns (TeardownResponse) {}

// Set up a pair of veths that interconnect the specified PID and the workspace container's network namename.
// Set up a pair of veths that interconnect the specified PID and the workspace container's network namespace.
rpc SetupPairVeths(SetupPairVethsRequest) returns (SetupPairVethsResponse) {}
}

Expand Down
3 changes: 2 additions & 1 deletion components/ws-daemon/nsinsider/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ go 1.17
replace github.com/seccomp/libseccomp-golang => github.com/gitpod-io/libseccomp-golang v0.9.2-0.20220203100026-45179215fdb1 // leeway indirect from components/workspacekit:lib

require (
github.com/coreos/go-iptables v0.6.0
github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
github.com/urfave/cli/v2 v2.3.0
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
)
Expand All @@ -18,6 +18,7 @@ require (
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

Expand Down
8 changes: 6 additions & 2 deletions components/ws-daemon/nsinsider/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
Expand Down Expand Up @@ -265,6 +263,10 @@ github.com/uber/jaeger-client-go v2.29.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA=
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down Expand Up @@ -399,6 +401,7 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand All @@ -410,6 +413,7 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
148 changes: 50 additions & 98 deletions components/ws-daemon/nsinsider/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

"github.com/gitpod-io/gitpod/common-go/log"
_ "github.com/gitpod-io/gitpod/common-go/nsenter"
"github.com/vishvananda/netlink"
)

func main() {
Expand Down Expand Up @@ -263,112 +264,47 @@ func main() {
Name: "target-pid",
Required: true,
},
&cli.StringFlag{
Name: "name",
Required: true,
},
},
Action: func(c *cli.Context) error {
containerIf, vethIf, cethIf := "eth0", "veth0", "ceth0"
mask := net.IPv4Mask(255, 255, 255, 0)
vethAddr := net.IPNet{
vethIp := net.IPNet{
IP: net.IPv4(10, 0, 5, 1),
Mask: mask,
}
cethAddr := net.IPNet{
IP: net.IPv4(10, 0, 5, 2),
Mask: mask,
}
masqueradeAddr := net.IPNet{
IP: vethAddr.IP.Mask(mask),
IP: vethIp.IP.Mask(mask),
Mask: mask,
}
ipCmd, iptablesCmd := "/usr/sbin/ip", "/usr/sbin/iptables"
netns := "workspace-ns"

cmd := exec.Command(ipCmd, "link", "add", vethIf, "type", "veth", "peer", "name", cethIf)
out, err := cmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("create a veth pair (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
}

path := "/var/run/netns"
if err := os.MkdirAll(path, 0755); err != nil {
return xerrors.Errorf("create a dir %s failed: %v", path, err)
iptablesCmd := "/usr/sbin/iptables"
targetPid := c.Int("target-pid")

veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{
Name: vethIf,
Flags: net.FlagUp,
MTU: 1500,
},
PeerName: cethIf,
PeerNamespace: netlink.NsPid(targetPid),
}
if err := os.Symlink(fmt.Sprintf("/proc/%d/ns/net", c.Int("target-pid")), filepath.Join(path, netns)); err != nil {
return xerrors.Errorf("create a symlink to netns (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
}
cmd = exec.Command(ipCmd, "link", "set", cethIf, "netns", netns)
out, err = cmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("link cethIf to netns (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
if err := netlink.LinkAdd(veth); err != nil {
return xerrors.Errorf("link %q-%q netns failed: %v", vethIf, cethIf, err)
}

cmd = exec.Command(ipCmd, "addr", "add", vethAddr.String(), "dev", vethIf)
out, err = cmd.CombinedOutput()
vethLink, err := netlink.LinkByName(vethIf)
if err != nil {
return xerrors.Errorf("assign IP address to the vethIf (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
return xerrors.Errorf("cannot found %q netns failed: %v", vethIf, err)
}

cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "addr", "add", cethAddr.String(), "dev", cethIf)
out, err = cmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("assign IP address to the cethIf (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIp}); err != nil {
return xerrors.Errorf("failed to add IP address to %q: %v", vethIf, err)
}

cmd = exec.Command(ipCmd, "link", "set", vethIf, "up")
out, err = cmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("bring up the vethIf (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
}

cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "link", "set", cethIf, "up")
out, err = cmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("bring up the cethIf (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
}

cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "link", "set", "lo", "up")
out, err = cmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("bring up the lo (%v) failed: %q\n%v",
cmd.Args,
string(out),
err,
)
if err := netlink.LinkSetUp(vethLink); err != nil {
return xerrors.Errorf("failed to enable %q: %v", vethIf, err)
}

cmd = exec.Command(iptablesCmd, "-A", "FORWARD", "-o", containerIf, "-i", vethIf, "-j", "ACCEPT")
out, err = cmd.CombinedOutput()
cmd := exec.Command(iptablesCmd, "-A", "FORWARD", "-o", containerIf, "-i", vethIf, "-j", "ACCEPT")
out, err := cmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("add a forwarding rule for iptable: vethIf -> eth0 (%v) failed: %q\n%v",
cmd.Args,
Expand Down Expand Up @@ -397,17 +333,33 @@ func main() {
)
}

// TODO(toru) if we will implement the dynamically ports exporsing, these are commented out.
// cmd = exec.Command(ipCmd, "netns", "exec", netns, "ip", "route", "replace", "default", "via", vethAddr.IP.String())
// out, err = cmd.CombinedOutput()
// if err != nil {
// return xerrors.Errorf("change up a default (%v) failed: %q\n%v",
// cmd.Args,
// string(out),
// err,
// )
// }
return nil
},
},
{
Name: "setup-peer-veth",
Usage: "set up a peer veth",
Action: func(c *cli.Context) error {
cethIf := "ceth0"
mask := net.IPv4Mask(255, 255, 255, 0)
cethIp := net.IPNet{
IP: net.IPv4(10, 0, 5, 2),
Mask: mask,
}

cethLink, err := netlink.LinkByName(cethIf)
if err != nil {
return xerrors.Errorf("cannot found %q netns failed: %v", cethIf, err)
}
if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIp}); err != nil {
return xerrors.Errorf("failed to add IP address to %q: %v", cethIf, err)
}
if err := netlink.LinkSetUp(cethLink); err != nil {
return xerrors.Errorf("failed to enable %q: %v", cethIf, err)
}

// TODO(toru): when we will complete implementing the dynamically ports exporsing,
// we have to implement changing default gw here.
return nil
},
},
Expand Down
14 changes: 13 additions & 1 deletion components/ws-daemon/pkg/iws/iws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,13 +325,25 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
}

err = nsinsider(wbs.Session.InstanceID, int(containerPID), func(c *exec.Cmd) {
c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid)), "--name", wbs.Session.InstanceID)
c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid)))
}, enterMountNS(true), enterPidNS(true), enterNetNS(true))
if err != nil {
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a pair of veths")
return nil, status.Errorf(codes.Internal, "cannot setup a pair of veths")
}

pid, err := wbs.Uidmapper.findHostPID(containerPID, uint64(req.Pid))
if err != nil {
return nil, xerrors.Errorf("cannot map in-container PID %d (container PID: %d): %w", req.Pid, containerPID, err)
}
err = nsinsider(wbs.Session.InstanceID, int(pid), func(c *exec.Cmd) {
c.Args = append(c.Args, "setup-peer-veth")
}, enterMountNS(true), enterPidNS(true), enterNetNS(true))
if err != nil {
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a peer veths")
return nil, status.Errorf(codes.Internal, "cannot setup a peer veths")
}

return &api.SetupPairVethsResponse{}, nil
}

Expand Down

0 comments on commit 23346cf

Please sign in to comment.