Skip to content
This repository has been archived by the owner on May 2, 2023. It is now read-only.

Commit

Permalink
Merge pull request #35 from nabla-containers/k8s-network
Browse files Browse the repository at this point in the history
Added networking hack for CNI namespaced veth pairs
  • Loading branch information
ricarkol authored Oct 16, 2018
2 parents 1e8765d + ae1743f commit 61d33ec
Show file tree
Hide file tree
Showing 7 changed files with 80 additions and 28 deletions.
8 changes: 8 additions & 0 deletions hack/copy_binaries.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,17 @@
# PERFORMANCE OF THIS SOFTWARE.

BIN_PATH=/usr/local/bin/

# We add binaries like runnc-cont and nabla-run to /opt/X since they are not
# to be consumed directly by the user.
BIN_PATH2=/opt/runnc/bin/

COPY_BINS=("build/runnc" "build/runnc-cont" "build/nabla-run")

mkdir -p ${BIN_PATH2}

for i in ${COPY_BINS[@]}; do
echo $i
cp $i ${BIN_PATH}/
cp $i ${BIN_PATH2}/
done
3 changes: 3 additions & 0 deletions libcontainer/configs/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ type Config struct {

// Labels are user defined metadata that is stored in the config and populated on the state
Labels []string `json:"labels"`

// Network namespace
NetnsPath string `json:"netnspath"`
}

// HostUID returns the UID to run the nabla container as. Default is root.
Expand Down
22 changes: 16 additions & 6 deletions libcontainer/configs/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,23 @@ func ParseSpec(s *specs.Spec) (*Config, error) {
labels = append(labels, fmt.Sprintf("%s=%s", k, v))
}

netnsPath := ""
if s.Linux != nil {
for _, v := range s.Linux.Namespaces {
if v.Type == specs.NetworkNamespace {
netnsPath = v.Path
}
}
}

cfg := Config{
Args: s.Process.Args,
Rootfs: s.Root.Path,
Env: s.Process.Env,
Cwd: s.Process.Cwd,
Version: s.Version,
Labels: labels,
Args: s.Process.Args,
Rootfs: s.Root.Path,
Env: s.Process.Env,
Cwd: s.Process.Cwd,
Version: s.Version,
NetnsPath: netnsPath,
Labels: labels,
}

return &cfg, nil
Expand Down
13 changes: 7 additions & 6 deletions libcontainer/container_nabla.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,12 +202,13 @@ func (c *nablaContainer) start(p *Process) error {

defer parentPipe.Close()
config := initConfig{
Root: c.config.Rootfs,
Args: c.config.Args,
FsPath: c.fsPath,
Cwd: c.config.Cwd,
Env: c.config.Env,
TapName: nablaTapName(c.id),
Root: c.config.Rootfs,
Args: c.config.Args,
FsPath: c.fsPath,
Cwd: c.config.Cwd,
Env: c.config.Env,
TapName: nablaTapName(c.id),
NetnsPath: c.config.NetnsPath,
}

enc := json.NewEncoder(parentPipe)
Expand Down
30 changes: 24 additions & 6 deletions libcontainer/init_nabla.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package libcontainer
import (
"encoding/json"
"fmt"
"github.com/vishvananda/netns"
"os"
"path/filepath"
"strconv"
Expand Down Expand Up @@ -46,6 +47,10 @@ func nablaRunArgs(cfg *initConfig) ([]string, error) {
"-volume", cfg.FsPath + ":/",
"-unikernel", filepath.Join(cfg.Root, cfg.Args[0])}

if cfg.NetnsPath != "" {
args = append(args, "-k8s")
}

for _, e := range cfg.Env {
args = append(args, "-env", e)
}
Expand All @@ -58,12 +63,13 @@ func nablaRunArgs(cfg *initConfig) ([]string, error) {
}

type initConfig struct {
Root string `json:"root"`
Args []string `json:"args"`
FsPath string `json:"fspath"`
Cwd string `json:"cwd"`
Env []string `json:"env"`
TapName string `json:"tap"`
Root string `json:"root"`
Args []string `json:"args"`
FsPath string `json:"fspath"`
Cwd string `json:"cwd"`
Env []string `json:"env"`
TapName string `json:"tap"`
NetnsPath string `json:"netnspath"`
}

func initNabla() error {
Expand Down Expand Up @@ -108,6 +114,18 @@ func initNabla() error {
syscall.Close(fd)
syscall.Close(rootfd)

// Go into network namespace for temporary hack for CNI plugin using veth pairs
if config.NetnsPath != "" {
nsh, err := netns.GetFromPath(config.NetnsPath)
if err != nil {
return newSystemErrorWithCause(err, "unable to get netns handle")
}

if err := netns.Set(nsh); err != nil {
return newSystemErrorWithCause(err, "unable to get set netns")
}
}

// Check if it is a pause container, if it is, just pause
if len(config.Args) == 1 && config.Args[0] == pauseNablaName {
select {}
Expand Down
16 changes: 8 additions & 8 deletions nabla-lib/network/network_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,12 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (

netHandle, err := netlink.NewHandle()
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to create netlink handler")
}

err = SetupTunDev()
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to setup tun dev")
}

masterLink, err := netlink.LinkByName(master)
Expand All @@ -175,13 +175,13 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (

macvtapLink, name, tapNameTmp, err := createMacvtapInterface(netHandle, masterLink)
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to create Macvtapint")
}
*tapName = tapNameTmp

addrs, err := netlink.AddrList(masterLink, netlink.FAMILY_V4)
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to get address list")
}
if len(addrs) == 0 {
return nil, nil, nil, "", fmt.Errorf("master should have an IP")
Expand All @@ -192,7 +192,7 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (

routes, err := netlink.RouteList(masterLink, netlink.FAMILY_V4)
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to get route list")
}
if len(routes) == 0 {
return nil, nil, nil, "",
Expand All @@ -204,17 +204,17 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (
// ip addr del $INET_STR dev master
err = netlink.AddrDel(masterLink, &masterAddr)
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to delete address of master")
}

err = netlink.LinkSetUp(macvtapLink)
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to set up tap link")
}

err = netlink.LinkSetUp(masterLink)
if err != nil {
return nil, nil, nil, "", err
return nil, nil, nil, "", errors.Wrap(err, "Unable to set up master link")
}

// The HardwareAddr Attr doesn't automatically get updated
Expand Down
16 changes: 14 additions & 2 deletions runnc-cont/runnc_cont.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ func main() {
"Max memory size in MBs (defaults to 512)")
inDocker := flag.Bool("docker", false,
"Is this running in a Docker container")
inK8s := flag.Bool("k8s", false,
"Is this running in a K8s cluster")
volume := flag.String("volume", ":",
"'--volume <SRC>:<DST>'. "+
"<SRC> is the directory or device to mount, and <DST> "+
Expand All @@ -111,12 +113,12 @@ func main() {
cmdargs := strings.Join(flag.Args(), " ")

os.Exit(run(*nablarun, *unikernel, *tap, ip, ipNet.Mask, gw,
*inDocker, vol, cmdargs, envVars, *cwd, *mem))
*inDocker, *inK8s, vol, cmdargs, envVars, *cwd, *mem))
}

func run(nablarun string, unikernel string, tapName string,
ip net.IP, mask net.IPMask, gw net.IP,
inDocker bool, volume []string,
inDocker bool, inK8s bool, volume []string,
cmdargs string, envVars []string, cwd string, mem int) int {

disk, err := setupDisk(volume[0])
Expand All @@ -135,6 +137,16 @@ func run(nablarun string, unikernel string, tapName string,
fmt.Fprintf(os.Stderr, "Could not create %s: %v\n", tapName, err)
return 1
}
} else if inK8s {
// The tap device will get the IP assigned to the k8s nabla
// container veth pair.
// XXX: This is a workaround due to an error with MacvTap, error was :
// Could not create /dev/tap8863: open /sys/devices/virtual/net/macvtap8863/tap8863/dev: no such file or directory
ip, gw, mask, err = network.CreateTapInterfaceDocker(tapName, "eth0")
if err != nil {
fmt.Fprintf(os.Stderr, "Could not create %s: %v\n", tapName, err)
return 1
}
} else {
err = network.CreateTapInterface(tapName, &gw, &mask)
if err != nil {
Expand Down

0 comments on commit 61d33ec

Please sign in to comment.