get a ip lease of lxd bridge on sandbox creation and assign it as sta…

…tic address to the nic device

cri/runtime.go

@@ -15,7 +15,6 @@ import (
 	"github.com/lxc/lxd/shared/logger"
 	"github.com/lxc/lxe/lxf"
 	"github.com/lxc/lxe/lxf/device"
-	"github.com/lxc/lxe/network"
 	"golang.org/x/net/context"
 	utilNet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/client-go/tools/remotecommand"
@@ -165,11 +164,15 @@ func (s RuntimeServer) RunPodSandbox(ctx context.Context,
 		sb.NetworkConfig.Mode = lxf.NetworkCNI
 	} else {
 		// default is to use the predefined lxd bridge managed by lxe
+		randIP, err := s.lxf.FindFreeIP(LXEBridge)
+		if err != nil {
+			logger.Errorf("unable to find a free ip: %v", err)
+			return nil, err
+		}
 		sb.NetworkConfig.Mode = lxf.NetworkBridged
 		sb.NetworkConfig.ModeData = map[string]string{
 			"bridge":            LXEBridge,
-			"interface-name":    network.DefaultInterface,
-			"interface-address": "dhcp",
+			"interface-address": randIP.String(),
 		}
 	}
 
@@ -408,7 +411,7 @@ func (s RuntimeServer) PodSandboxStatus(ctx context.Context, req *rtApi.PodSandb
 			State: rtApi.PodSandboxState(
 				rtApi.PodSandboxState_value["SANDBOX_"+strings.ToUpper(sb.State.String())]),
 			Network: &rtApi.PodSandboxNetworkStatus{
-				Ip: "127.0.0.1",
+				Ip: "",
 			},
 		},
 	}
@@ -430,7 +433,6 @@ func (s RuntimeServer) PodSandboxStatus(ctx context.Context, req *rtApi.PodSandb
 		}
 	}
 
-	// TODO: these should be encapsulated inside lxf - something like sandbox.GetIP()?
 	ip, err := s.lxf.GetSandboxIP(sb)
 	if err != nil {
 		logger.Errorf("could not look up sandbox ip: %v", err)
@@ -922,11 +924,11 @@ func (s RuntimeServer) ListContainerStats(ctx context.Context,
 // UpdateRuntimeConfig updates the runtime configuration based on the given request.
 func (s RuntimeServer) UpdateRuntimeConfig(ctx context.Context,
 	req *rtApi.UpdateRuntimeConfigRequest) (*rtApi.UpdateRuntimeConfigResponse, error) {
-
+	//logger.Infof("UpdateRuntimeConfig called: PodCIDR %v", req.GetRuntimeConfig().GetNetworkConfig().GetPodCidr())
 	logger.Debugf("UpdateRuntimeConfig triggered: %v", req)
 
 	podCIDR := req.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()
-	err := s.lxf.EnsureBridge(LXEBridge, podCIDR, true)
+	err := s.lxf.EnsureBridge(LXEBridge, podCIDR, true, false)
 	if err != nil {
 		logger.Errorf("UpdateRuntimeConfig: %v", err)
 		return nil, err

cri/server.go

@@ -78,6 +78,13 @@ func NewServer(criConfig *LXEConfig) *Server {
 		os.Exit(shared.ExitCodeSchemaMigrationFailure)
 	}
 
+	// Initialize lxd bridge for lxe is created with new generated cidr if missing
+	err = lxf.EnsureBridge(LXEBridge, "", true, true)
+	if err != nil {
+		logger.Critf("Unable to setup bridge %v: %v", LXEBridge, err)
+		os.Exit(shared.ExitCodeUnspecified)
+	}
+
 	// for now we bind the http on every interface
 	streamServerAddr := ":" + criConfig.LXEStreamingPort
 	runtimeServer, err := NewRuntimeServer(criConfig, streamServerAddr, lxf)

doc/examples/single-master-with-lxe/setup.sh

@@ -5,6 +5,7 @@ set -ex
 #cluster_domain="cluster.local"
 #cluster_servicesubnet="10.96.0.0/12"
 #cluster_podsubnet="10.244.0.0/16"
+kubernetes_version=1.12.4-00
 
 # remove existing lxd and lxc packages
 apt-get purge lxd* lxc* liblxc* -y
@@ -38,7 +39,7 @@ apt-get install cri-tools
 crictl version
 
 # install and configure kubelet and create certificates and credentials
-apt-get install kubeadm kubelet kubectl -o Dpkg::Options::="--force-confold" --force-yes -y
+apt-get install kubeadm="$kubernetes_version" kubelet="$kubernetes_version" kubectl="$kubernetes_version" -o Dpkg::Options::="--force-confold" --force-yes -y
 kubeadm alpha phase certs all --config /etc/kubernetes/kubeadm.conf
 kubeadm alpha phase kubeconfig all --config /etc/kubernetes/kubeadm.conf
 mkdir -p ~/.kube

lxf/device/nic.go

@@ -4,18 +4,20 @@ const nicType = "nic"
 
 // Nic device
 type Nic struct {
-	Name    string
-	NicType string
-	Parent  string
+	Name        string
+	NicType     string
+	Parent      string
+	IPv4Address string
 }
 
 // ToMap serializes itself into a lxd device map entry
 func (b Nic) ToMap() (map[string]string, error) {
 	return map[string]string{
-		"type":    nicType,
-		"name":    b.Name,
-		"nictype": b.NicType,
-		"parent":  b.Parent,
+		"type":         nicType,
+		"name":         b.Name,
+		"nictype":      b.NicType,
+		"parent":       b.Parent,
+		"ipv4.address": b.IPv4Address,
 	}, nil
 }
 
@@ -27,8 +29,9 @@ func (b Nic) GetName() string {
 // NicFromMap create a new nic from map entries
 func NicFromMap(dev map[string]string) (Nic, error) {
 	return Nic{
-		Name:    dev["name"],
-		NicType: dev["nictype"],
-		Parent:  dev["parent"],
+		Name:        dev["name"],
+		NicType:     dev["nictype"],
+		Parent:      dev["parent"],
+		IPv4Address: dev["ipv4.address"],
 	}, nil
 }

lxf/networking.go

@@ -1,30 +1,43 @@
 package lxf
 
 import (
+	"encoding/binary"
+	"fmt"
+	"math/rand"
 	"net"
 	"strconv"
 
 	"github.com/lxc/lxd/shared/api"
 )
 
 // EnsureBridge ensures the bridge exists with the defined options
-func (l *LXF) EnsureBridge(name, cidr string, nat bool) error {
-	// Always use first address in range for the bridge
-	_, net, err := net.ParseCIDR(cidr)
-	if err != nil {
-		return err
+// cidr is an expected ipv4 cidr or can be empty to automatically assign a cidr
+func (l *LXF) EnsureBridge(name, cidr string, nat, createOnly bool) error {
+	var address string
+	if cidr == "" {
+		address = "auto"
+	} else {
+		// Always use first address in range for the bridge
+		_, net, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return err
+		}
+		net.IP[3]++
+		address = net.String()
 	}
-	net.IP[3]++
-
-	// TODO: disable dnsmasq sending dns server via dhcp, we don't need/want it. keyword: raw.dnsmasq in networking
 
 	put := api.NetworkPut{
 		Description: "managed by LXE, default bridge",
 		Config: map[string]string{
-			"ipv4.address": net.String(),
+			"ipv4.address": address,
 			"ipv4.dhcp":    strconv.FormatBool(true),
 			"ipv4.nat":     strconv.FormatBool(true),
 			"ipv6.address": "none",
+			// We don't need to recieve a DNS in DHCP, Kubernetes' DNS is always set
+			// disables dns (option -p: https://linux.die.net/man/8/dnsmasq)
+			// > Listen on <port> instead of the standard DNS port (53). Setting this to
+			// > zero completely disables DNS function, leaving only DHCP and/or TFTP.
+			"raw.dnsmasq": `port=0`,
 		},
 	}
 
@@ -41,9 +54,70 @@ func (l *LXF) EnsureBridge(name, cidr string, nat bool) error {
 
 		return err
 	}
+	if network.Type != "bridge" {
+		return fmt.Errorf("Expected %v to be a bridge, but is %v", name, network.Type)
+	}
+
+	// don't update when only creation is requested
+	if createOnly {
+		return nil
+	}
 
 	for k, v := range put.Config {
 		network.Config[k] = v
 	}
 	return l.server.UpdateNetwork(name, network.Writable(), ETag)
 }
+
+// FindFreeIP generates a IP within the range of the provided lxd managed bridge which does
+// not exist in the current leases
+func (l *LXF) FindFreeIP(bridge string) (net.IP, error) {
+	network, _, err := l.server.GetNetwork(bridge)
+	if err != nil {
+		return nil, err
+	}
+	if network.Config["ipv4.dhcp.ranges"] != "" {
+		return nil, fmt.Errorf("Not yet implemented to find an IP with explicitly set ip ranges `ipv4.dhcp.ranges` in bridge %v", bridge)
+	}
+
+	leases, err := l.server.GetNetworkLeases(bridge)
+	if err != nil {
+		return nil, err
+	}
+
+	bridgeIP, bridgeNet, err := net.ParseCIDR(network.Config["ipv4.address"])
+	if err != nil {
+		return nil, err
+	}
+
+	broadcastIP := make(net.IP, 4)
+	for i := range broadcastIP {
+		broadcastIP[i] = bridgeNet.IP[i] | ^bridgeNet.Mask[i]
+	}
+
+	var ip net.IP
+	// Until a usable IP is found...
+	for {
+		// select randomly an ip address within the specified range
+		randIP := make(net.IP, 4)
+		binary.LittleEndian.PutUint32(randIP, rand.Uint32())
+		for i, v := range randIP {
+			randIP[i] = bridgeNet.IP[i] + (v &^ bridgeNet.Mask[i])
+		}
+
+		// not allowed to be the bridge ip, network address or broadcast address
+		if randIP.String() == bridgeIP.String() || randIP.String() == bridgeNet.IP.String() || randIP.String() == broadcastIP.String() {
+			continue
+		}
+		// not allowed to exist in current leases
+		for _, lease := range leases {
+			if randIP.String() == lease.Address {
+				continue
+			}
+		}
+		// if reached here, ip is fine
+		ip = randIP
+		break
+	}
+	return ip, nil
+}

lxf/sandbox.go

@@ -182,9 +182,10 @@ func (l *LXF) CreateSandbox(s *Sandbox) error {
 	switch s.NetworkConfig.Mode {
 	case NetworkBridged:
 		s.Nics = append(s.Nics, device.Nic{
-			Name:    network.DefaultInterface,
-			NicType: "bridged",
-			Parent:  s.NetworkConfig.ModeData["bridge"],
+			Name:        network.DefaultInterface,
+			NicType:     "bridged",
+			Parent:      s.NetworkConfig.ModeData["bridge"],
+			IPv4Address: s.NetworkConfig.ModeData["interface-address"],
 		})
 	default:
 		// do nothing
@@ -304,15 +305,15 @@ func (l *LXF) saveSandbox(s *Sandbox) error {
 				},
 			},
 		}
-		if s.NetworkConfig.ModeData["interface-name"] != "" {
+		if s.NetworkConfig.Mode == NetworkBridged && s.NetworkConfig.ModeData["interface-address"] != "" {
 			data.Config = append(data.Config, NetworkConfigEntryPhysical{
 				NetworkConfigEntry: NetworkConfigEntry{
 					Type: "physical",
 				},
-				Name: s.NetworkConfig.ModeData["interface-name"],
+				Name: network.DefaultInterface,
 				Subnets: []NetworkConfigEntryPhysicalSubnet{
 					NetworkConfigEntryPhysicalSubnet{
-						Type: s.NetworkConfig.ModeData["interface-address"],
+						Type: "dhcp",
 					},
 				},
 			})
@@ -486,6 +487,10 @@ func (l *LXF) GetSandboxIP(s *Sandbox) (string, error) {
 	case NetworkNone:
 		return "", nil
 	case NetworkBridged:
+		// if statically assigned ip exists, return that
+		if s.NetworkConfig.ModeData["interface-address"] != "" {
+			return s.NetworkConfig.ModeData["interface-address"], nil
+		}
 		fallthrough
 	case NetworkCNI:
 		fallthrough