From 3ecd5fb82c2a9c377b4f7fd5193ac8fd97508d05 Mon Sep 17 00:00:00 2001 From: Denis Manente Date: Fri, 4 Jan 2019 17:28:21 +0100 Subject: [PATCH] get a ip lease of lxd bridge on sandbox creation and assign it as static address to the nic device --- cri/runtime.go | 16 ++-- cri/server.go | 7 ++ doc/examples/single-master-with-lxe/setup.sh | 3 +- lxf/device/nic.go | 23 ++--- lxf/networking.go | 92 ++++++++++++++++++-- lxf/sandbox.go | 17 ++-- 6 files changed, 125 insertions(+), 33 deletions(-) diff --git a/cri/runtime.go b/cri/runtime.go index 848fe3a..22aba19 100644 --- a/cri/runtime.go +++ b/cri/runtime.go @@ -15,7 +15,6 @@ import ( "github.com/lxc/lxd/shared/logger" "github.com/lxc/lxe/lxf" "github.com/lxc/lxe/lxf/device" - "github.com/lxc/lxe/network" "golang.org/x/net/context" utilNet "k8s.io/apimachinery/pkg/util/net" "k8s.io/client-go/tools/remotecommand" @@ -165,11 +164,15 @@ func (s RuntimeServer) RunPodSandbox(ctx context.Context, sb.NetworkConfig.Mode = lxf.NetworkCNI } else { // default is to use the predefined lxd bridge managed by lxe + randIP, err := s.lxf.FindFreeIP(LXEBridge) + if err != nil { + logger.Errorf("unable to find a free ip: %v", err) + return nil, err + } sb.NetworkConfig.Mode = lxf.NetworkBridged sb.NetworkConfig.ModeData = map[string]string{ "bridge": LXEBridge, - "interface-name": network.DefaultInterface, - "interface-address": "dhcp", + "interface-address": randIP.String(), } } @@ -408,7 +411,7 @@ func (s RuntimeServer) PodSandboxStatus(ctx context.Context, req *rtApi.PodSandb State: rtApi.PodSandboxState( rtApi.PodSandboxState_value["SANDBOX_"+strings.ToUpper(sb.State.String())]), Network: &rtApi.PodSandboxNetworkStatus{ - Ip: "127.0.0.1", + Ip: "", }, }, } @@ -430,7 +433,6 @@ func (s RuntimeServer) PodSandboxStatus(ctx context.Context, req *rtApi.PodSandb } } - // TODO: these should be encapsulated inside lxf - something like sandbox.GetIP()? ip, err := s.lxf.GetSandboxIP(sb) if err != nil { logger.Errorf("could not look up sandbox ip: %v", err) @@ -922,11 +924,11 @@ func (s RuntimeServer) ListContainerStats(ctx context.Context, // UpdateRuntimeConfig updates the runtime configuration based on the given request. func (s RuntimeServer) UpdateRuntimeConfig(ctx context.Context, req *rtApi.UpdateRuntimeConfigRequest) (*rtApi.UpdateRuntimeConfigResponse, error) { - + //logger.Infof("UpdateRuntimeConfig called: PodCIDR %v", req.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()) logger.Debugf("UpdateRuntimeConfig triggered: %v", req) podCIDR := req.GetRuntimeConfig().GetNetworkConfig().GetPodCidr() - err := s.lxf.EnsureBridge(LXEBridge, podCIDR, true) + err := s.lxf.EnsureBridge(LXEBridge, podCIDR, true, false) if err != nil { logger.Errorf("UpdateRuntimeConfig: %v", err) return nil, err diff --git a/cri/server.go b/cri/server.go index b069e1c..f209a60 100644 --- a/cri/server.go +++ b/cri/server.go @@ -78,6 +78,13 @@ func NewServer(criConfig *LXEConfig) *Server { os.Exit(shared.ExitCodeSchemaMigrationFailure) } + // Initialize lxd bridge for lxe is created with new generated cidr if missing + err = lxf.EnsureBridge(LXEBridge, "", true, true) + if err != nil { + logger.Critf("Unable to setup bridge %v: %v", LXEBridge, err) + os.Exit(shared.ExitCodeUnspecified) + } + // for now we bind the http on every interface streamServerAddr := ":" + criConfig.LXEStreamingPort runtimeServer, err := NewRuntimeServer(criConfig, streamServerAddr, lxf) diff --git a/doc/examples/single-master-with-lxe/setup.sh b/doc/examples/single-master-with-lxe/setup.sh index a9bc234..da765f4 100755 --- a/doc/examples/single-master-with-lxe/setup.sh +++ b/doc/examples/single-master-with-lxe/setup.sh @@ -5,6 +5,7 @@ set -ex #cluster_domain="cluster.local" #cluster_servicesubnet="10.96.0.0/12" #cluster_podsubnet="10.244.0.0/16" +kubernetes_version=1.12.4-00 # remove existing lxd and lxc packages apt-get purge lxd* lxc* liblxc* -y @@ -38,7 +39,7 @@ apt-get install cri-tools crictl version # install and configure kubelet and create certificates and credentials -apt-get install kubeadm kubelet kubectl -o Dpkg::Options::="--force-confold" --force-yes -y +apt-get install kubeadm="$kubernetes_version" kubelet="$kubernetes_version" kubectl="$kubernetes_version" -o Dpkg::Options::="--force-confold" --force-yes -y kubeadm alpha phase certs all --config /etc/kubernetes/kubeadm.conf kubeadm alpha phase kubeconfig all --config /etc/kubernetes/kubeadm.conf mkdir -p ~/.kube diff --git a/lxf/device/nic.go b/lxf/device/nic.go index e65c2f0..0efb4b0 100644 --- a/lxf/device/nic.go +++ b/lxf/device/nic.go @@ -4,18 +4,20 @@ const nicType = "nic" // Nic device type Nic struct { - Name string - NicType string - Parent string + Name string + NicType string + Parent string + IPv4Address string } // ToMap serializes itself into a lxd device map entry func (b Nic) ToMap() (map[string]string, error) { return map[string]string{ - "type": nicType, - "name": b.Name, - "nictype": b.NicType, - "parent": b.Parent, + "type": nicType, + "name": b.Name, + "nictype": b.NicType, + "parent": b.Parent, + "ipv4.address": b.IPv4Address, }, nil } @@ -27,8 +29,9 @@ func (b Nic) GetName() string { // NicFromMap create a new nic from map entries func NicFromMap(dev map[string]string) (Nic, error) { return Nic{ - Name: dev["name"], - NicType: dev["nictype"], - Parent: dev["parent"], + Name: dev["name"], + NicType: dev["nictype"], + Parent: dev["parent"], + IPv4Address: dev["ipv4.address"], }, nil } diff --git a/lxf/networking.go b/lxf/networking.go index 4dc7e95..b387fde 100644 --- a/lxf/networking.go +++ b/lxf/networking.go @@ -1,6 +1,9 @@ package lxf import ( + "encoding/binary" + "fmt" + "math/rand" "net" "strconv" @@ -8,23 +11,33 @@ import ( ) // EnsureBridge ensures the bridge exists with the defined options -func (l *LXF) EnsureBridge(name, cidr string, nat bool) error { - // Always use first address in range for the bridge - _, net, err := net.ParseCIDR(cidr) - if err != nil { - return err +// cidr is an expected ipv4 cidr or can be empty to automatically assign a cidr +func (l *LXF) EnsureBridge(name, cidr string, nat, createOnly bool) error { + var address string + if cidr == "" { + address = "auto" + } else { + // Always use first address in range for the bridge + _, net, err := net.ParseCIDR(cidr) + if err != nil { + return err + } + net.IP[3]++ + address = net.String() } - net.IP[3]++ - - // TODO: disable dnsmasq sending dns server via dhcp, we don't need/want it. keyword: raw.dnsmasq in networking put := api.NetworkPut{ Description: "managed by LXE, default bridge", Config: map[string]string{ - "ipv4.address": net.String(), + "ipv4.address": address, "ipv4.dhcp": strconv.FormatBool(true), "ipv4.nat": strconv.FormatBool(true), "ipv6.address": "none", + // We don't need to recieve a DNS in DHCP, Kubernetes' DNS is always set + // disables dns (option -p: https://linux.die.net/man/8/dnsmasq) + // > Listen on instead of the standard DNS port (53). Setting this to + // > zero completely disables DNS function, leaving only DHCP and/or TFTP. + "raw.dnsmasq": `port=0`, }, } @@ -41,9 +54,70 @@ func (l *LXF) EnsureBridge(name, cidr string, nat bool) error { return err } + if network.Type != "bridge" { + return fmt.Errorf("Expected %v to be a bridge, but is %v", name, network.Type) + } + + // don't update when only creation is requested + if createOnly { + return nil + } for k, v := range put.Config { network.Config[k] = v } return l.server.UpdateNetwork(name, network.Writable(), ETag) } + +// FindFreeIP generates a IP within the range of the provided lxd managed bridge which does +// not exist in the current leases +func (l *LXF) FindFreeIP(bridge string) (net.IP, error) { + network, _, err := l.server.GetNetwork(bridge) + if err != nil { + return nil, err + } + if network.Config["ipv4.dhcp.ranges"] != "" { + return nil, fmt.Errorf("Not yet implemented to find an IP with explicitly set ip ranges `ipv4.dhcp.ranges` in bridge %v", bridge) + } + + leases, err := l.server.GetNetworkLeases(bridge) + if err != nil { + return nil, err + } + + bridgeIP, bridgeNet, err := net.ParseCIDR(network.Config["ipv4.address"]) + if err != nil { + return nil, err + } + + broadcastIP := make(net.IP, 4) + for i := range broadcastIP { + broadcastIP[i] = bridgeNet.IP[i] | ^bridgeNet.Mask[i] + } + + var ip net.IP + // Until a usable IP is found... + for { + // select randomly an ip address within the specified range + randIP := make(net.IP, 4) + binary.LittleEndian.PutUint32(randIP, rand.Uint32()) + for i, v := range randIP { + randIP[i] = bridgeNet.IP[i] + (v &^ bridgeNet.Mask[i]) + } + + // not allowed to be the bridge ip, network address or broadcast address + if randIP.String() == bridgeIP.String() || randIP.String() == bridgeNet.IP.String() || randIP.String() == broadcastIP.String() { + continue + } + // not allowed to exist in current leases + for _, lease := range leases { + if randIP.String() == lease.Address { + continue + } + } + // if reached here, ip is fine + ip = randIP + break + } + return ip, nil +} diff --git a/lxf/sandbox.go b/lxf/sandbox.go index 3a021d0..3daf536 100644 --- a/lxf/sandbox.go +++ b/lxf/sandbox.go @@ -182,9 +182,10 @@ func (l *LXF) CreateSandbox(s *Sandbox) error { switch s.NetworkConfig.Mode { case NetworkBridged: s.Nics = append(s.Nics, device.Nic{ - Name: network.DefaultInterface, - NicType: "bridged", - Parent: s.NetworkConfig.ModeData["bridge"], + Name: network.DefaultInterface, + NicType: "bridged", + Parent: s.NetworkConfig.ModeData["bridge"], + IPv4Address: s.NetworkConfig.ModeData["interface-address"], }) default: // do nothing @@ -304,15 +305,15 @@ func (l *LXF) saveSandbox(s *Sandbox) error { }, }, } - if s.NetworkConfig.ModeData["interface-name"] != "" { + if s.NetworkConfig.Mode == NetworkBridged && s.NetworkConfig.ModeData["interface-address"] != "" { data.Config = append(data.Config, NetworkConfigEntryPhysical{ NetworkConfigEntry: NetworkConfigEntry{ Type: "physical", }, - Name: s.NetworkConfig.ModeData["interface-name"], + Name: network.DefaultInterface, Subnets: []NetworkConfigEntryPhysicalSubnet{ NetworkConfigEntryPhysicalSubnet{ - Type: s.NetworkConfig.ModeData["interface-address"], + Type: "dhcp", }, }, }) @@ -486,6 +487,10 @@ func (l *LXF) GetSandboxIP(s *Sandbox) (string, error) { case NetworkNone: return "", nil case NetworkBridged: + // if statically assigned ip exists, return that + if s.NetworkConfig.ModeData["interface-address"] != "" { + return s.NetworkConfig.ModeData["interface-address"], nil + } fallthrough case NetworkCNI: fallthrough