Skip to content

Commit

Permalink
Changes include:
Browse files Browse the repository at this point in the history 
* Create vlan for pod requesting unique security group.
* Adding packet verifier binary to validate the packet flow as part of integration tests.
  • Loading branch information
@SaranBalaji90
SaranBalaji90 committed Aug 7, 2020
1 parent a96bfe0 commit 2fa7981
Show file tree
Hide file tree
Showing 16 changed files with 918 additions and 175 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ export GOPROXY = direct
# LDFLAGS is the set of flags used when building golang executables.
LDFLAGS = -X main.version=$(VERSION)
# ALLPKGS is the set of packages provided in source.
ALLPKGS = $(shell go list ./...)
ALLPKGS = $(shell go list ./... | grep -v cmd/packet-verifier)
# BINS is the set of built command executables.
BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper
# Plugin binaries
Expand Down Expand Up @@ -240,7 +240,7 @@ lint:

# Run go vet on source code.
vet:
go vet ./...
go vet $(ALLPKGS)

# Run go vet inside of a container.
docker-vet: build-docker-test
Expand Down
31 changes: 23 additions & 8 deletions cmd/routed-eni-cni-plugin/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,20 +173,26 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
return errors.New("add cmd: failed to assign an IP address to container")
}

log.Infof("Received add network response for container %s interface %s: %s, table %d, external-SNAT: %v, vpcCIDR: %v",
args.ContainerID, args.IfName,
r.IPv4Addr, r.DeviceNumber, r.UseExternalSNAT, r.VPCcidrs)
log.Infof("Received add network response for container %s interface %s: %+v",
args.ContainerID, args.IfName, r)

addr := &net.IPNet{
IP: net.ParseIP(r.IPv4Addr),
Mask: net.IPv4Mask(255, 255, 255, 255),
}

// build hostVethName
// Note: the maximum length for linux interface name is 15
hostVethName := generateHostVethName(conf.VethPrefix, conf.Name, args.ContainerID, args.IfName)
if r.PodVlanId != 0 {
hostVethName := generateHostVethName("vlan", conf.Name, args.ContainerID, args.IfName)

err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu, log)
err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, addr, int(r.PodVlanId), r.PodENIMAC,
r.PodENISubnetGW, int(r.ParentIfIndex), mtu, log)
} else {
// build hostVethName
// Note: the maximum length for linux interface name is 15
hostVethName := generateHostVethName(conf.VethPrefix, conf.Name, args.ContainerID, args.IfName)

err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu, log)
}

if err != nil {
log.Errorf("Failed SetupPodNetwork for container %s: %v",
Expand Down Expand Up @@ -299,13 +305,22 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
return errors.New("del cmd: failed to process delete request")
}

log.Infof("Received del network response for pod %s namespace %s sandbox %s: %+v", string(k8sArgs.K8S_POD_NAME),
string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), r)

deletedPodIP := net.ParseIP(r.IPv4Addr)
if deletedPodIP != nil {
addr := &net.IPNet{
IP: deletedPodIP,
Mask: net.IPv4Mask(255, 255, 255, 255),
}
err = driverClient.TeardownNS(addr, int(r.DeviceNumber), log)

if r.PodVlanId != 0 {
err = driverClient.TeardownPodENINetwork(int(r.PodVlanId), log)
} else {
err = driverClient.TeardownNS(addr, int(r.DeviceNumber), log)
}

if err != nil {
log.Errorf("Failed on TeardownPodNetwork for container ID %s: %v",
args.ContainerID, err)
Expand Down
65 changes: 65 additions & 0 deletions cmd/routed-eni-cni-plugin/cni_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,3 +265,68 @@ func TestCmdDelErrTeardown(t *testing.T) {
err := del(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
assert.Error(t, err)
}

func TestCmdAddForPodENINetwork(t *testing.T) {
ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t)
defer ctrl.Finish()

stdinData, _ := json.Marshal(netConf)

cmdArgs := &skel.CmdArgs{ContainerID: containerID,
Netns: netNS,
IfName: ifName,
StdinData: stdinData}

mocksTypes.EXPECT().LoadArgs(gomock.Any(), gomock.Any()).Return(nil)

conn, _ := grpc.Dial(ipamdAddress, grpc.WithInsecure())

mocksGRPC.EXPECT().Dial(gomock.Any(), gomock.Any()).Return(conn, nil)
mockC := mock_rpc.NewMockCNIBackendClient(ctrl)
mocksRPC.EXPECT().NewCNIBackendClient(conn).Return(mockC)

addNetworkReply := &rpc.AddNetworkReply{Success: true, IPv4Addr: ipAddr, PodENISubnetGW: "10.0.0.1", PodVlanId: 1,
PodENIMAC: "eniHardwareAddr", ParentIfIndex: 2}
mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, nil)

addr := &net.IPNet{
IP: net.ParseIP(addNetworkReply.IPv4Addr),
Mask: net.IPv4Mask(255, 255, 255, 255),
}
mocksNetwork.EXPECT().SetupPodENINetwork(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns, addr, 1, "eniHardwareAddr",
"10.0.0.1", 2, gomock.Any(), gomock.Any()).Return(nil)

mocksTypes.EXPECT().PrintResult(gomock.Any(), gomock.Any()).Return(nil)

err := add(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
assert.Nil(t, err)
}

func TestCmdDelForPodENINetwork(t *testing.T) {
ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t)
defer ctrl.Finish()

stdinData, _ := json.Marshal(netConf)

cmdArgs := &skel.CmdArgs{ContainerID: containerID,
Netns: netNS,
IfName: ifName,
StdinData: stdinData}

mocksTypes.EXPECT().LoadArgs(gomock.Any(), gomock.Any()).Return(nil)

conn, _ := grpc.Dial(ipamdAddress, grpc.WithInsecure())

mocksGRPC.EXPECT().Dial(gomock.Any(), gomock.Any()).Return(conn, nil)
mockC := mock_rpc.NewMockCNIBackendClient(ctrl)
mocksRPC.EXPECT().NewCNIBackendClient(conn).Return(mockC)

delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, PodVlanId: 1}

mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, nil)

mocksNetwork.EXPECT().TeardownPodENINetwork(1, gomock.Any()).Return(nil)

err := del(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
assert.Nil(t, err)
}
217 changes: 180 additions & 37 deletions cmd/routed-eni-cni-plugin/driver/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import (
)

const (
// vlan rule priority
vlanRulePriority = 1
// IP rules priority, leaving a 512 gap for the future
toContainerRulePriority = 512
// 1024 is reserved for (IP rule not to <VPC's subnet> table main)
Expand All @@ -46,6 +48,9 @@ const (
type NetworkAPIs interface {
SetupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, table int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error
TeardownNS(addr *net.IPNet, table int, log logger.Logger) error
SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, vlanId int, eniMAC string,
subnetGW string, parentIfIndex int, mtu int, log logger.Logger) error
TeardownPodENINetwork(vlanId int, log logger.Logger) error
}

type linuxNetwork struct {
Expand Down Expand Up @@ -175,45 +180,10 @@ func (os *linuxNetwork) SetupNS(hostVethName string, contVethName string, netnsP

func setupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, table int, vpcCIDRs []string, useExternalSNAT bool,
netLink netlinkwrapper.NetLink, ns nswrapper.NS, mtu int, log logger.Logger, procSys procsyswrapper.ProcSys) error {
// Clean up if hostVeth exists.
if oldHostVeth, err := netLink.LinkByName(hostVethName); err == nil {
if err = netLink.LinkDel(oldHostVeth); err != nil {
return errors.Wrapf(err, "setupNS network: failed to delete old hostVeth %q", hostVethName)
}
log.Debugf("Clean up old hostVeth: %v\n", hostVethName)
}

createVethContext := newCreateVethPairContext(contVethName, hostVethName, addr, mtu)
if err := ns.WithNetNSPath(netnsPath, createVethContext.run); err != nil {
log.Errorf("Failed to setup NS network %v", err)
return errors.Wrap(err, "setupNS network: failed to setup NS network")
}

hostVeth, err := netLink.LinkByName(hostVethName)
hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, addr, netLink, ns, mtu, procSys, log)
if err != nil {
return errors.Wrapf(err, "setupNS network: failed to find link %q", hostVethName)
}

// NB: Must be set after move to host namespace, or kernel will reset to defaults.
if err := procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", hostVethName), "0"); err != nil {
if !os.IsNotExist(err) {
return errors.Wrapf(err, "setup NS network: failed to disable IPv6 router advertisements")
}
log.Debugf("SetupNS: Ignoring '%s' writing to accept_ra: Assuming kernel lacks IPv6 support", err)
}

if err := procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_redirects", hostVethName), "0"); err != nil {
if !os.IsNotExist(err) {
return errors.Wrapf(err, "setup NS network: failed to disable IPv6 ICMP redirects")
}
log.Debugf("SetupNS: Ignoring '%s' writing to accept_redirects: Assuming kernel lacks IPv6 support", err)
}
log.Debugf("SetupNS: disabled IPv6 RA and ICMP redirects on %s", hostVethName)

// Explicitly set the veth to UP state, because netlink doesn't always do that on all the platforms with net.FlagUp.
// veth won't get a link local address unless it's set to UP state.
if err = netLink.LinkSetUp(hostVeth); err != nil {
return errors.Wrapf(err, "setupNS network: failed to set link %q up", hostVethName)
return errors.Wrapf(err, "setupNS network: failed to setup veth pair.")
}

log.Debugf("Setup host route outgoing hostVeth, LinkIndex %d", hostVeth.Attrs().Index)
Expand Down Expand Up @@ -282,6 +252,151 @@ func setupNS(hostVethName string, contVethName string, netnsPath string, addr *n
return nil
}

// setupVeth sets up veth for the pod.
func setupVeth(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, netLink netlinkwrapper.NetLink,
ns nswrapper.NS, mtu int, procSys procsyswrapper.ProcSys, log logger.Logger) (netlink.Link, error) {
// Clean up if hostVeth exists.
if oldHostVeth, err := netLink.LinkByName(hostVethName); err == nil {
if err = netLink.LinkDel(oldHostVeth); err != nil {
return nil, errors.Wrapf(err, "setupVeth network: failed to delete old hostVeth %q", hostVethName)
}
log.Debugf("Cleaned up old hostVeth: %v\n", hostVethName)
}

createVethContext := newCreateVethPairContext(contVethName, hostVethName, addr, mtu)
if err := ns.WithNetNSPath(netnsPath, createVethContext.run); err != nil {
log.Errorf("Failed to setup veth network %v", err)
return nil, errors.Wrap(err, "setupVeth network: failed to setup veth network")
}

hostVeth, err := netLink.LinkByName(hostVethName)
if err != nil {
return nil, errors.Wrapf(err, "setupVeth network: failed to find link %q", hostVethName)
}

// NB: Must be set after move to host namespace, or kernel will reset to defaults.
if err := procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", hostVethName), "0"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "setupVeth network: failed to disable IPv6 router advertisements")
}
log.Debugf("setupVeth network: Ignoring '%s' writing to accept_ra: Assuming kernel lacks IPv6 support", err)
}

if err := procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_redirects", hostVethName), "0"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "setupVeth network: failed to disable IPv6 ICMP redirects")
}
log.Debugf("setupVeth network: Ignoring '%s' writing to accept_redirects: Assuming kernel lacks IPv6 support", err)
}
log.Debugf("setupVeth network: disabled IPv6 RA and ICMP redirects on %s", hostVethName)

// Explicitly set the veth to UP state, because netlink doesn't always do that on all the platforms with net.FlagUp.
// veth won't get a link local address unless it's set to UP state.
if err = netLink.LinkSetUp(hostVeth); err != nil {
return nil, errors.Wrapf(err, "setupVeth network: failed to set link %q up", hostVethName)
}
return hostVeth, nil
}

// SetupPodENINetwork sets up the network ns for pods requesting its own security group
func (os *linuxNetwork) SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet,
vlanId int, eniMAC string, subnetGW string, parentIfIndex int, mtu int, log logger.Logger) error {

hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, addr, os.netLink, os.ns, mtu, os.procSys, log)
if err != nil {
return errors.Wrapf(err, "SetupPodENINetwork failed to setup veth pair.")
}

vlanTableId := vlanId + 100
vlanLink := buildVlanLink(vlanId, parentIfIndex, eniMAC)

// 1. clean up if vlan already exists (necessary when trunk ENI changes).
if oldVlan, err := os.netLink.LinkByName(vlanLink.Name); err == nil {
if err = os.netLink.LinkDel(oldVlan); err != nil {
return errors.Wrapf(err, "SetupPodENINetwork: failed to delete old vlan %s", vlanLink.Name)
}
log.Debugf("Cleaned up old vlan: %s", vlanLink.Name)
}

// 2. add new vlan link
err = os.netLink.LinkAdd(vlanLink)
if err != nil {
return errors.Wrapf(err, "SetupPodENINetwork: failed to add vlan link.")
}

// 3. bring up the vlan
if err = os.netLink.LinkSetUp(vlanLink); err != nil {
return errors.Wrapf(err, "SetupPodENINetwork: failed to set link %q up", vlanLink.Name)
}

// 4. create default routes for vlan
routes := buildRoutesForVlan(vlanTableId, vlanLink.Index, net.ParseIP(subnetGW))
for _, r := range routes {
if err := os.netLink.RouteReplace(&r); err != nil {
return errors.Wrapf(err, "SetupPodENINetwork: unable to replace route entry %s via %s", r.Dst.IP.String(), subnetGW)
}
}

// 5. create route entry for hostveth.
route := netlink.Route{
LinkIndex: hostVeth.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: addr,
Table: vlanTableId,
}
if err := os.netLink.RouteReplace(&route); err != nil {
return errors.Wrapf(err, "SetupPodENINetwork: unable to add or replace route entry for %s", route.Dst.IP.String())
}

log.Debugf("Successfully set host route to be %s/0", route.Dst.IP.String())

// 6. Add ip rules for the pod.
vlanRule := os.netLink.NewRule()
vlanRule.Table = vlanTableId
vlanRule.Priority = vlanRulePriority
vlanRule.IifName = vlanLink.Name
err = os.netLink.RuleAdd(vlanRule)
if err != nil && !isRuleExistsError(err) {
return errors.Wrapf(err, "SetupPodENINetwork: unable to add ip rule for vlan link %s ", vlanLink.Name)
}

vlanRule.IifName = hostVeth.Attrs().Name
err = os.netLink.RuleAdd(vlanRule)
if err != nil && !isRuleExistsError(err) {
return errors.Wrapf(err, "SetupPodENINetwork: unable to add ip rule for host veth %s", hostVethName)
}
return nil
}

// buildRoutesForVlan builds routes required for the vlan link.
func buildRoutesForVlan(vlanTableId int, vlanIndex int, gw net.IP) []netlink.Route {
return []netlink.Route{
// Add a direct link route for the pod vlan link only.
{
LinkIndex: vlanIndex,
Dst: &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)},
Scope: netlink.SCOPE_LINK,
Table: vlanTableId,
},
{
LinkIndex: vlanIndex,
Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
Scope: netlink.SCOPE_UNIVERSE,
Gw: gw,
Table: vlanTableId,
},
}
}

// buildVlanLink builds vlan link for the pod.
func buildVlanLink(vlanId int, parentIfIndex int, eniMAC string) *netlink.Vlan {
la := netlink.NewLinkAttrs()
la.Name = fmt.Sprintf("vlan.eth.%d", vlanId)
la.ParentIndex = parentIfIndex
la.HardwareAddr, _ = net.ParseMAC(eniMAC)
return &netlink.Vlan{LinkAttrs: la, VlanId: vlanId}
}

func addContainerRule(netLink netlinkwrapper.NetLink, isToContainer bool, addr *net.IPNet, table int) error {
if addr == nil {
return errors.New("can't add container rules without an IP address")
Expand Down Expand Up @@ -356,6 +471,34 @@ func tearDownNS(addr *net.IPNet, table int, netLink netlinkwrapper.NetLink, log
return nil
}

// TeardownPodENINetwork tears down the vlan and corresponding ip rules.
func (os *linuxNetwork) TeardownPodENINetwork(vlanId int, log logger.Logger) error {
log.Infof("Tear down of pod ENI namespace")

// 1. delete vlan
if vlan, err := os.netLink.LinkByName(fmt.Sprintf("vlan.eth.%d",
vlanId)); err == nil {
err := os.netLink.LinkDel(vlan)
if err != nil {
return errors.Wrapf(err, "TeardownPodENINetwork: failed to delete vlan link for %d", vlanId)
}
}

// 2. delete two ip rules associated with the vlan
vlanRule := os.netLink.NewRule()
vlanRule.Table = vlanId + 100
vlanRule.Priority = vlanRulePriority

for i := 0; i < 2; i++ {
if err := os.netLink.RuleDel(vlanRule); err != nil {
if !containsNoSuchRule(err) {
return errors.Wrapf(err, "TeardownPodENINetwork: failed to delete container rule for %d", vlanId)
}
}
}
return nil
}

func deleteRuleListBySrc(src net.IPNet) error {
networkClient := networkutils.New()
return networkClient.DeleteRuleListBySrc(src)
Expand Down
Loading

0 comments on commit 2fa7981

Please sign in to comment.