add hostIface command line parameter

user can now override deafult network interface selection using hostIface argument in vhive binary Signed-off-by: Shyam Jesal <[email protected]>
NiuJ1ao · Feb 25, 2021 · 562ceba · 562ceba
1 parent dedd18b
commit 562ceba
Show file tree

Hide file tree

Showing 13 changed files with 62 additions and 37 deletions.
diff --git a/ctriface/bench_test.go b/ctriface/bench_test.go
@@ -56,7 +56,7 @@ func TestBenchmarkStart(t *testing.T) {
 	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout)
 	defer cancel()
 
-	orch := NewOrchestrator("devmapper", WithTestModeOn(true), WithUPF(*isUPFEnabled))
+	orch := NewOrchestrator("devmapper", "", WithTestModeOn(true), WithUPF(*isUPFEnabled))
 
 	images := getAllImages()
 	benchCount := 10

diff --git a/ctriface/failing_test.go b/ctriface/failing_test.go
@@ -51,7 +51,7 @@ func TestStartSnapStop(t *testing.T) {
 	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout)
 	defer cancel()
 
-	orch := NewOrchestrator("devmapper", WithTestModeOn(true))
+	orch := NewOrchestrator("devmapper", "", WithTestModeOn(true))
 
 	vmID := "2"
 

diff --git a/ctriface/iface.go b/ctriface/iface.go
@@ -75,7 +75,7 @@ func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (_ *
 	logger := log.WithFields(log.Fields{"vmID": vmID, "image": imageName})
 	logger.Debug("StartVM: Received StartVM")
 
-	vm, err := o.vmPool.Allocate(vmID)
+	vm, err := o.vmPool.Allocate(vmID, o.hostIface)
 	if err != nil {
 		logger.Error("failed to allocate VM in VM pool")
 		return nil, nil, err
@@ -290,9 +290,7 @@ func (o *Orchestrator) getImage(ctx context.Context, imageName string) (*contain
 	return &image, nil
 }
 
-func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest {
-	kernelArgs := "ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1"
-
+func getK8sDNS() []string {
 	//using googleDNS as a backup
 	dnsIPs := []string{"8.8.8.8"}
 	//get k8s DNS clusterIP
@@ -307,6 +305,11 @@ func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest {
 		//adding k8s DNS clusterIP to the list
 		dnsIPs = []string{strings.TrimSpace(string(stdoutStderr)), dnsIPs[0]}
 	}
+	return dnsIPs
+}
+
+func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest {
+	kernelArgs := "ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1"
 
 	return &proto.CreateVMRequest{
 		VMID:           vm.ID,
@@ -323,7 +326,7 @@ func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest {
 				IPConfig: &proto.IPConfiguration{
 					PrimaryAddr: vm.Ni.PrimaryAddress + vm.Ni.Subnet,
 					GatewayAddr: vm.Ni.GatewayAddress,
-					Nameservers: dnsIPs,
+					Nameservers: getK8sDNS(),
 				},
 			},
 		}},
@@ -496,7 +499,7 @@ func (o *Orchestrator) Offload(ctx context.Context, vmID string) error {
 		return err
 	}
 
-	if err := o.vmPool.RecreateTap(vmID); err != nil {
+	if err := o.vmPool.RecreateTap(vmID, o.hostIface); err != nil {
 		logger.Error("Failed to recreate tap upon offloading")
 		return err
 	}

diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go
@@ -60,6 +60,7 @@ func TestPauseSnapResume(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),
@@ -102,6 +103,7 @@ func TestStartStopSerial(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),
@@ -135,6 +137,7 @@ func TestPauseResumeSerial(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),
@@ -175,6 +178,7 @@ func TestStartStopParallel(t *testing.T) {
 	vmNum := 10
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),
@@ -233,6 +237,7 @@ func TestPauseResumeParallel(t *testing.T) {
 	vmNum := 10
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),

diff --git a/ctriface/manual_cleanup_test.go b/ctriface/manual_cleanup_test.go
@@ -55,6 +55,7 @@ func TestSnapLoad(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),
@@ -104,6 +105,7 @@ func TestSnapLoadMultiple(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),
@@ -165,6 +167,7 @@ func TestParallelSnapLoad(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),
@@ -226,6 +229,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
+		"",
 		WithTestModeOn(true),
 		WithUPF(*isUPFEnabled),
 		WithLazyMode(*isLazyMode),

diff --git a/ctriface/orch.go b/ctriface/orch.go
@@ -65,19 +65,21 @@ type Orchestrator struct {
 	isLazyMode       bool
 	snapshotsDir     string
 	isMetricsMode    bool
+	hostIface        string
 
 	memoryManager *manager.MemoryManager
 }
 
 // NewOrchestrator Initializes a new orchestrator
-func NewOrchestrator(snapshotter string, opts ...OrchestratorOption) *Orchestrator {
+func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption) *Orchestrator {
 	var err error
 
 	o := new(Orchestrator)
 	o.vmPool = misc.NewVMPool()
 	o.cachedImages = make(map[string]containerd.Image)
 	o.snapshotter = snapshotter
 	o.snapshotsDir = "/fccd/snapshots"
+	o.hostIface = hostIface
 
 	for _, opt := range opts {
 		opt(o)

diff --git a/ctriface/orch_options.go b/ctriface/orch_options.go
@@ -72,3 +72,11 @@ func WithMetricsMode(isMetricsMode bool) OrchestratorOption {
 		o.isMetricsMode = isMetricsMode
 	}
 }
+
+// WithCustomHostIface Sets the custom host net interface
+// for the VMs to link to
+func WithCustomHostIface(hostIface string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.hostIface = hostIface
+	}
+}
diff --git a/misc/misc_test.go b/misc/misc_test.go
@@ -53,7 +53,7 @@ func TestAllocateFreeVMs(t *testing.T) {
 	vmIDs := [2]string{"test1", "test2"}
 
 	for _, vmID := range vmIDs {
-		_, err := vmPool.Allocate(vmID)
+		_, err := vmPool.Allocate(vmID, "")
 		require.NoError(t, err, "Failed to allocate VM")
 	}
 
@@ -76,7 +76,7 @@ func TestAllocateFreeVMsParallel(t *testing.T) {
 		go func(i int) {
 			defer vmGroup.Done()
 			vmID := fmt.Sprintf("test_%d", i)
-			_, err := vmPool.Allocate(vmID)
+			_, err := vmPool.Allocate(vmID, "")
 			require.NoError(t, err, "Failed to allocate VM")
 		}(i)
 	}
@@ -108,7 +108,7 @@ func TestRecreateParallel(t *testing.T) {
 		go func(i int) {
 			defer vmGroup.Done()
 			vmID := fmt.Sprintf("test_%d", i)
-			_, err := vmPool.Allocate(vmID)
+			_, err := vmPool.Allocate(vmID, "")
 			require.NoError(t, err, "Failed to allocate VM")
 		}(i)
 	}
@@ -123,7 +123,7 @@ func TestRecreateParallel(t *testing.T) {
 		go func(i int) {
 			defer vmGroupRecreate.Done()
 			vmID := fmt.Sprintf("test_%d", i)
-			err := vmPool.RecreateTap(vmID)
+			err := vmPool.RecreateTap(vmID, "")
 			require.NoError(t, err, "Failed to recreate tap")
 		}(i)
 	}

diff --git a/misc/vm_pool.go b/misc/vm_pool.go
@@ -37,7 +37,7 @@ func NewVMPool() *VMPool {
 }
 
 // Allocate Initializes a VM, activates it and then adds it to VM map
-func (p *VMPool) Allocate(vmID string) (*VM, error) {
+func (p *VMPool) Allocate(vmID, hostIface string) (*VM, error) {
 
 	logger := log.WithFields(log.Fields{"vmID": vmID})
 
@@ -50,7 +50,7 @@ func (p *VMPool) Allocate(vmID string) (*VM, error) {
 	vm := NewVM(vmID)
 
 	var err error
-	vm.Ni, err = p.tapManager.AddTap(vmID + "_tap")
+	vm.Ni, err = p.tapManager.AddTap(vmID+"_tap", hostIface)
 	if err != nil {
 		logger.Warn("Ni allocation failed")
 		return nil, err
@@ -84,7 +84,7 @@ func (p *VMPool) Free(vmID string) error {
 }
 
 // RecreateTap Deletes and creates the tap for a VM
-func (p *VMPool) RecreateTap(vmID string) error {
+func (p *VMPool) RecreateTap(vmID, hostIface string) error {
 	logger := log.WithFields(log.Fields{"vmID": vmID})
 
 	logger.Debug("Recreating tap")
@@ -100,7 +100,7 @@ func (p *VMPool) RecreateTap(vmID string) error {
 		return err
 	}
 
-	_, err := p.tapManager.AddTap(vmID + "_tap")
+	_, err := p.tapManager.AddTap(vmID+"_tap", hostIface)
 	if err != nil {
 		logger.Error("Failed to add tap")
 		return err

diff --git a/taps/tapManager.go b/taps/tapManager.go
@@ -105,25 +105,24 @@ func createBridge(bridgeName, gatewayAddr string) {
 }
 
 //ConfigIPtables Configures IP tables for internet access inside VM
-func ConfigIPtables(tapName string) error {
+func ConfigIPtables(tapName, hostIface string) error {
 
-	var hostIface = ""
-
-	out, err := exec.Command(
-		"route",
-	).Output()
-	if err != nil {
-		log.Warnf("Failed to fetch host net interfaces %v\n%s\n", err, out)
-		return err
-	}
-	scanner := bufio.NewScanner(bytes.NewReader(out))
-	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, "default") {
-			hostIface = line[strings.LastIndex(line, " ")+1:]
+	if hostIface == "" {
+		out, err := exec.Command(
+			"route",
+		).Output()
+		if err != nil {
+			log.Warnf("Failed to fetch host net interfaces %v\n%s\n", err, out)
+			return err
+		}
+		scanner := bufio.NewScanner(bytes.NewReader(out))
+		for scanner.Scan() {
+			line := scanner.Text()
+			if strings.Contains(line, "default") {
+				hostIface = line[strings.LastIndex(line, " ")+1:]
+			}
 		}
 	}
-
 	cmd := exec.Command(
 		"sudo", "iptables", "-t", "nat", "-A", "POSTROUTING", "-o", hostIface, "-j", "MASQUERADE",
 	)
@@ -160,7 +159,7 @@ func ConfigIPtables(tapName string) error {
 }
 
 // AddTap Creates a new tap and returns the corresponding network interface
-func (tm *TapManager) AddTap(tapName string) (*NetworkInterface, error) {
+func (tm *TapManager) AddTap(tapName, hostIface string) (*NetworkInterface, error) {
 	tm.Lock()
 
 	if ni, ok := tm.createdTaps[tapName]; ok {
@@ -179,7 +178,7 @@ func (tm *TapManager) AddTap(tapName string) (*NetworkInterface, error) {
 				tm.Lock()
 				tm.createdTaps[tapName] = ni
 				tm.Unlock()
-				err := ConfigIPtables(tapName)
+				err := ConfigIPtables(tapName, hostIface)
 				if err != nil {
 					return nil, err
 				}

diff --git a/taps/taps_test.go b/taps/taps_test.go
@@ -66,7 +66,7 @@ func TestCreateRemoveTaps(t *testing.T) {
 			wg.Add(1)
 			go func(i int) {
 				defer wg.Done()
-				tm.AddTap(fmt.Sprintf("tap_%d", i))
+				tm.AddTap(fmt.Sprintf("tap_%d", i), "")
 			}(i)
 		}
 		wg.Wait()
@@ -91,7 +91,7 @@ func TestCreateRemoveExtra(t *testing.T) {
 	defer tm.RemoveBridges()
 
 	for i := 0; i < tapsNum; i++ {
-		_, err := tm.AddTap(fmt.Sprintf("tap_%d", i))
+		_, err := tm.AddTap(fmt.Sprintf("tap_%d", i), "")
 		if i < tm.numBridges*TapsPerBridge {
 			require.NoError(t, err, "Failed to create tap")
 		} else {

diff --git a/vhive.go b/vhive.go
@@ -60,6 +60,7 @@ var (
 	servedThreshold    *uint64
 	pinnedFuncNum      *int
 	criSock            *string
+	hostIface          *string
 )
 
 func main() {
@@ -78,6 +79,7 @@ func main() {
 	pinnedFuncNum = flag.Int("hn", 0, "Number of functions pinned in memory (IDs from 0 to X)")
 	isLazyMode = flag.Bool("lazy", false, "Enable lazy serving mode when UPFs are enabled")
 	criSock = flag.String("criSock", "/etc/firecracker-containerd/fccd-cri.sock", "Socket address for CRI service")
+	hostIface = flag.String("hostIface", "", "Host net-interface for the VMs to bind to for internet access")
 
 	flag.Parse()
 
@@ -119,6 +121,7 @@ func main() {
 
 	orch = ctriface.NewOrchestrator(
 		*snapshotter,
+		*hostIface,
 		ctriface.WithTestModeOn(testModeOn),
 		ctriface.WithSnapshots(*isSnapshotsEnabled),
 		ctriface.WithUPF(*isUPFEnabled),

diff --git a/vhive_test.go b/vhive_test.go
@@ -74,6 +74,7 @@ func TestMain(m *testing.M) {
 
 	orch = ctriface.NewOrchestrator(
 		"devmapper",
+		"",
 		ctriface.WithTestModeOn(true),
 		ctriface.WithSnapshots(*isSnapshotsEnabledTest),
 		ctriface.WithUPF(*isUPFEnabledTest),