diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 4bb740b078..7f958647e8 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -395,6 +395,11 @@ func (c *IPAMContext) nodeInit() error { if err != nil { return errors.Wrap(err, "ipamd init: failed to set up host network") } + err = c.networkClient.CleanUpStaleAWSChains(c.enableIPv4, c.enableIPv6) + if err != nil { + // We should not error if clean up fails since these chains don't affect the rules + log.Debugf("Failed to clean up stale AWS chains: %v", err) + } metadataResult, err := c.awsClient.DescribeAllENIs() if err != nil { diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 1cf0386b66..cd6bee4a70 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -150,6 +150,7 @@ func TestNodeInit(t *testing.T) { m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) m.awsutils.EXPECT().GetPrimaryENImac().Return("") m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, true, false).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().RefreshSGIDs(gomock.Any()).AnyTimes().Return(nil) @@ -234,6 +235,7 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) m.awsutils.EXPECT().GetPrimaryENImac().Return("") m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, true, false).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().RefreshSGIDs(gomock.Any()).AnyTimes().Return(nil) @@ -308,6 +310,7 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { primaryIP := net.ParseIP(ipaddr01) m.network.EXPECT().SetupHostNetwork(cidrs, eni1.MAC, &primaryIP, false, false, true).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(false, true).Return(nil) m.awsutils.EXPECT().GetIPv6PrefixesFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv6Prefixes, nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().GetPrimaryENImac().Return(eni1.MAC) diff --git a/pkg/networkutils/mocks/network_mocks.go b/pkg/networkutils/mocks/network_mocks.go index a4c9016777..e68b213f44 100644 --- a/pkg/networkutils/mocks/network_mocks.go +++ b/pkg/networkutils/mocks/network_mocks.go @@ -50,6 +50,20 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder { return m.recorder } +// CleanUpStaleAWSChains mocks base method. +func (m *MockNetworkAPIs) CleanUpStaleAWSChains(arg0, arg1 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanUpStaleAWSChains", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanUpStaleAWSChains indicates an expected call of CleanUpStaleAWSChains. +func (mr *MockNetworkAPIsMockRecorder) CleanUpStaleAWSChains(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanUpStaleAWSChains", reflect.TypeOf((*MockNetworkAPIs)(nil).CleanUpStaleAWSChains), arg0, arg1) +} + // GetExcludeSNATCIDRs mocks base method. func (m *MockNetworkAPIs) GetExcludeSNATCIDRs() []string { m.ctrl.T.Helper() diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index c003eab987..39ddb042b8 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -152,6 +152,7 @@ type NetworkAPIs interface { SetupENINetwork(eniIP string, mac string, deviceNumber int, subnetCIDR string) error // UpdateHostIptablesRules updates the nat table iptables rules on the host UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, v6Enabled bool) error + CleanUpStaleAWSChains(v4Enabled, v6Enabled bool) error UseExternalSNAT() bool GetExcludeSNATCIDRs() []string GetExternalServiceCIDRs() []string @@ -375,6 +376,44 @@ func (n *linuxNetwork) UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC str return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled) } +func (n *linuxNetwork) CleanUpStaleAWSChains(v4Enabled, v6Enabled bool) error { + ipProtocol := iptables.ProtocolIPv4 + if v6Enabled { + ipProtocol = iptables.ProtocolIPv6 + } + + ipt, err := n.newIptables(ipProtocol) + if err != nil { + return errors.Wrap(err, "stale chain cleanup: failed to create iptables") + } + + existingChains, err := ipt.ListChains("nat") + if err != nil { + return errors.Wrap(err, "stale chain cleanup: failed to list iptables nat chains") + } + + for _, chain := range existingChains { + if !strings.HasPrefix(chain, "AWS-CONNMARK-CHAIN") && !strings.HasPrefix(chain, "AWS-SNAT-CHAIN") { + continue + } + parsedChain := strings.Split(chain, "-") + chainNum, err := strconv.Atoi(parsedChain[len(parsedChain)-1]) + if err != nil { + return errors.Wrap(err, "stale chain cleanup: failed to convert string to int") + } + // Chains 1 --> x (0 indexed) will be stale + if chainNum > 0 { + // No need to clear the chain since computeStaleIptablesRules cleans up all rules already + log.Infof("Deleting stale chain: %s", chain) + err := ipt.DeleteChain("nat", chain) + if err != nil { + return errors.Wrapf(err, "stale chain cleanup: failed to delete chain %s", chain) + } + } + } + return nil +} + func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, v6Enabled bool) error { primaryIntf, err := findPrimaryInterfaceName(primaryMAC) @@ -434,15 +473,13 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne log.Debugf("Total CIDRs to program - %d", len(allCIDRs)) // build IPTABLES chain for SNAT of non-VPC outbound traffic and excluded CIDRs var chains []string - for i := 0; i <= len(allCIDRs); i++ { - chain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", i) - log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) - if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { - log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) - return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") - } - chains = append(chains, chain) + chain := "AWS-SNAT-CHAIN-0" + log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) + if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { + log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) + return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") } + chains = append(chains, chain) // build SNAT rules for outbound non-VPC traffic var iptableRules []iptablesRule @@ -456,23 +493,20 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0", }}) - for i, cidr := range allCIDRs { - curChain := chains[i] - curName := fmt.Sprintf("[%d] AWS-SNAT-CHAIN", i) - nextChain := chains[i+1] + for _, cidr := range allCIDRs { comment := "AWS SNAT CHAIN" if cidr.isExclusion { comment += " EXCLUSION" } - log.Debugf("Setup Host Network: iptables -A %s ! -d %s -t nat -j %s", curChain, cidr, nextChain) + log.Debugf("Setup Host Network: iptables -A %s -d %s -t nat -j %s", chain, cidr, "RETURN") iptableRules = append(iptableRules, iptablesRule{ - name: curName, + name: chain, shouldExist: !n.useExternalSNAT, table: "nat", - chain: curChain, + chain: chain, rule: []string{ - "!", "-d", cidr.cidr, "-m", "comment", "--comment", comment, "-j", nextChain, + "-d", cidr.cidr, "-m", "comment", "--comment", comment, "-j", "RETURN", }}) } @@ -494,15 +528,6 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne } } - lastChain := chains[len(chains)-1] - iptableRules = append(iptableRules, iptablesRule{ - name: "last SNAT rule for non-VPC outbound traffic", - shouldExist: !n.useExternalSNAT, - table: "nat", - chain: lastChain, - rule: snatRule, - }) - snatStaleRules, err := computeStaleIptablesRules(ipt, "nat", "AWS-SNAT-CHAIN", iptableRules, chains) if err != nil { return []iptablesRule{}, err @@ -510,6 +535,14 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne iptableRules = append(iptableRules, snatStaleRules...) + iptableRules = append(iptableRules, iptablesRule{ + name: "last SNAT rule for non-VPC outbound traffic", + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: chain, + rule: snatRule, + }) + iptableRules = append(iptableRules, iptablesRule{ name: "connmark for primary ENI", shouldExist: n.nodePortSupportEnabled, @@ -556,16 +589,15 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable excludeCIDRs := sets.NewString(n.excludeSNATCIDRs...) log.Debugf("Total CIDRs to exempt from connmark rules - %d", len(allCIDRs)) + var chains []string - for i := 0; i <= len(allCIDRs); i++ { - chain := fmt.Sprintf("AWS-CONNMARK-CHAIN-%d", i) - log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) - if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { - log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) - return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") - } - chains = append(chains, chain) + chain := "AWS-CONNMARK-CHAIN-0" + log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) + if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { + log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) + return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") } + chains = append(chains, chain) var iptableRules []iptablesRule log.Debugf("Setup Host Network: iptables -t nat -A PREROUTING -i %s+ -m comment --comment \"AWS, outbound connections\" -j AWS-CONNMARK-CHAIN-0", n.vethPrefix) @@ -590,37 +622,23 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable "-j", "AWS-CONNMARK-CHAIN-0", }}) - for i, cidr := range allCIDRs { - curChain := chains[i] - curName := fmt.Sprintf("[%d] AWS-SNAT-CHAIN", i) - nextChain := chains[i+1] + for _, cidr := range allCIDRs { comment := "AWS CONNMARK CHAIN, VPC CIDR" if excludeCIDRs.Has(cidr) { comment = "AWS CONNMARK CHAIN, EXCLUDED CIDR" } - log.Debugf("Setup Host Network: iptables -A %s ! -d %s -t nat -j %s", curChain, cidr, nextChain) + log.Debugf("Setup Host Network: iptables -A %s -d %s -t nat -j %s", chain, cidr, "RETURN") iptableRules = append(iptableRules, iptablesRule{ - name: curName, + name: chain, shouldExist: !n.useExternalSNAT, table: "nat", - chain: curChain, + chain: chain, rule: []string{ - "!", "-d", cidr, "-m", "comment", "--comment", comment, "-j", nextChain, + "-d", cidr, "-m", "comment", "--comment", comment, "-j", "RETURN", }}) } - iptableRules = append(iptableRules, iptablesRule{ - name: "connmark rule for external outbound traffic", - shouldExist: !n.useExternalSNAT, - table: "nat", - chain: chains[len(chains)-1], - rule: []string{ - "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", - "--set-xmark", fmt.Sprintf("%#x/%#x", n.mainENIMark, n.mainENIMark), - }, - }) - // Force delete existing restore mark rule so that the subsequent rule gets added to the end iptableRules = append(iptableRules, iptablesRule{ name: "connmark to fwmark copy", @@ -652,6 +670,17 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable } iptableRules = append(iptableRules, connmarkStaleRules...) + iptableRules = append(iptableRules, iptablesRule{ + name: "connmark rule for external outbound traffic", + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: chain, + rule: []string{ + "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", + "--set-xmark", fmt.Sprintf("%#x/%#x", n.mainENIMark, n.mainENIMark), + }, + }) + log.Debugf("iptableRules: %v", iptableRules) return iptableRules, nil } @@ -659,7 +688,6 @@ func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptable func (n *linuxNetwork) updateIptablesRules(iptableRules []iptablesRule, ipt iptableswrapper.IPTablesIface) error { for _, rule := range iptableRules { log.Debugf("execute iptable rule : %s", rule.name) - exists, err := ipt.Exists(rule.table, rule.chain, rule.rule...) log.Debugf("rule %v exists %v, err %v", rule, exists, err) if err != nil { @@ -668,10 +696,19 @@ func (n *linuxNetwork) updateIptablesRules(iptableRules []iptablesRule, ipt ipta } if !exists && rule.shouldExist { - err = ipt.Append(rule.table, rule.chain, rule.rule...) - if err != nil { - log.Errorf("host network setup: failed to add %v, %v", rule, err) - return errors.Wrapf(err, "host network setup: failed to add %v", rule) + if rule.name == "AWS-CONNMARK-CHAIN-0" || rule.name == "AWS-SNAT-CHAIN-0" { + // All CIDR rules must go before the SNAT/Mark rule + err = ipt.Insert(rule.table, rule.chain, 1, rule.rule...) + if err != nil { + log.Errorf("host network setup: failed to insert %v, %v", rule, err) + return errors.Wrapf(err, "host network setup: failed to add %v", rule) + } + } else { + err = ipt.Append(rule.table, rule.chain, rule.rule...) + if err != nil { + log.Errorf("host network setup: failed to add %v, %v", rule, err) + return errors.Wrapf(err, "host network setup: failed to add %v", rule) + } } } else if exists && !rule.shouldExist { err = ipt.Delete(rule.table, rule.chain, rule.rule...) @@ -726,7 +763,7 @@ func computeStaleIptablesRules(ipt iptableswrapper.IPTablesIface, table, chainPr return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to list rules from table %s with chain prefix %s", table, chainPrefix) } activeChains := sets.NewString(chains...) - log.Debugf("Setup Host Network: computing stale iptables rules for %s table with chain prefix %s") + log.Debugf("Setup Host Network: computing stale iptables rules for %s table with chain prefix %s", table, chainPrefix) for _, staleRule := range existingRules { if len(staleRule.rule) == 0 && activeChains.Has(staleRule.chain) { log.Debugf("Setup Host Network: active chain found: %s", staleRule.chain) diff --git a/pkg/networkutils/network_test.go b/pkg/networkutils/network_test.go index b16b018ca9..a9f7142efe 100644 --- a/pkg/networkutils/network_test.go +++ b/pkg/networkutils/network_test.go @@ -465,17 +465,21 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, - "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, @@ -512,18 +516,18 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "RETURN") //AWS SNAT CHAN proves backwards compatibility + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") _ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5") _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") @@ -534,17 +538,17 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "AWS-SNAT-CHAIN-3": [][]string{}, - "AWS-SNAT-CHAIN-4": [][]string{}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{}, - "AWS-CONNMARK-CHAIN-4": [][]string{}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, @@ -581,18 +585,18 @@ func TestSetupHostNetworkWithDifferentVethPrefix(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "RETURN") //AWS SNAT CHAN proves backwards compatibility + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") _ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5") _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") @@ -602,17 +606,21 @@ func TestSetupHostNetworkWithDifferentVethPrefix(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, - "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-i", "veth+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, @@ -649,17 +657,17 @@ func TestSetupHostNetworkExternalNATCleanupConnmark(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") @@ -672,16 +680,8 @@ func TestSetupHostNetworkExternalNATCleanupConnmark(t *testing.T) { map[string]map[string][][]string{ "nat": { "AWS-SNAT-CHAIN-0": [][]string{}, - "AWS-SNAT-CHAIN-1": [][]string{}, - "AWS-SNAT-CHAIN-2": [][]string{}, - "AWS-SNAT-CHAIN-3": [][]string{}, - "AWS-SNAT-CHAIN-4": [][]string{}, "POSTROUTING": [][]string{}, "AWS-CONNMARK-CHAIN-0": [][]string{}, - "AWS-CONNMARK-CHAIN-1": [][]string{}, - "AWS-CONNMARK-CHAIN-2": [][]string{}, - "AWS-CONNMARK-CHAIN-3": [][]string{}, - "AWS-CONNMARK-CHAIN-4": [][]string{}, "PREROUTING": [][]string{}, }, "mangle": { @@ -714,17 +714,17 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") @@ -736,17 +736,21 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, - "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, - "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, @@ -782,11 +786,11 @@ func TestUpdateHostIptablesRules(t *testing.T) { } setupNetLinkMocks(ctrl, mockNetLink) - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility - _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "RETURN") //AWS SNAT CHAN proves backwards compatibility + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") - _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0") _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80") @@ -799,13 +803,17 @@ func TestUpdateHostIptablesRules(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, - "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, - "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, - "AWS-CONNMARK-CHAIN-2": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-SNAT-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "RETURN"}, + {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}, + }, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{ + {"-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "RETURN"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}, + }, "PREROUTING": [][]string{ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, {"-i", "veth+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"}, diff --git a/scripts/lib/performance_tests.sh b/scripts/lib/performance_tests.sh index 50233a84a9..3e340468c9 100644 --- a/scripts/lib/performance_tests.sh +++ b/scripts/lib/performance_tests.sh @@ -173,7 +173,7 @@ function install_cw_agent(){ echo "Install Cloudwatch Agent DS" $KUBECTL_PATH apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-serviceaccount.yaml - echo '{ "logs": { "metrics_collected": { "kubernetes": { "metrics_collection_interval": 30, "cluster_name": "eks-net-perf" }},"force_flush_interval": 5 }}' | jq > cwagentconfig.json + echo '{ "logs": { "metrics_collected": { "kubernetes": { "metrics_collection_interval": 30, "cluster_name": "eks-net-perf" }},"force_flush_interval": 5 }}' | jq '.' > cwagentconfig.json $KUBECTL_PATH create cm -n $CW_NAMESPACE cwagentconfig --from-file cwagentconfig.json $KUBECTL_PATH apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml diff --git a/test/agent/cmd/snat-utils/main.go b/test/agent/cmd/snat-utils/main.go index e0096538b4..cf59127013 100644 --- a/test/agent/cmd/snat-utils/main.go +++ b/test/agent/cmd/snat-utils/main.go @@ -76,25 +76,43 @@ func validateIPTableRules(randomizedSNATValue string, numOfCidrs int) error { currChain := "AWS-SNAT-CHAIN-0" lastChain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", numOfCidrs) - i := 0 - for i < numOfCidrs { + + exists, err := iptables.ChainExists("nat", "AWS-SNAT-CHAIN-1") + if err != nil { + return err + } + // If AWS-SNAT-CHAIN-1 exists, we run the old logic + if exists { + i := 0 + for i < numOfCidrs { + rules, err := iptables.List("nat", currChain) + if err != nil { + return err + } + i = i + 1 + nextChain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", i) + foundNextChain := false + for _, rule := range rules { + target := fmt.Sprintf("-j %s", nextChain) + if strings.Contains(rule, target) { + currChain = nextChain + foundNextChain = true + break + } + } + if !foundNextChain { + return fmt.Errorf("failed: AWS-SNAT chain broken for %s", currChain) + } + } + } else { + lastChain = "AWS-SNAT-CHAIN-0" rules, err := iptables.List("nat", currChain) if err != nil { return err } - i = i + 1 - nextChain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", i) - foundNextChain := false - for _, rule := range rules { - target := fmt.Sprintf("-j %s", nextChain) - if strings.Contains(rule, target) { - currChain = nextChain - foundNextChain = true - break - } - } - if foundNextChain == false { - return fmt.Errorf("failed: AWS-SNAT chain broken for %s", currChain) + // One rule per cidr + SNAT rule + chain creation rule + if len(rules) != numOfCidrs+2 { + return fmt.Errorf("failed: AWS-SNAT chain does not contain the correct amount of rules") } } @@ -107,7 +125,6 @@ func validateIPTableRules(randomizedSNATValue string, numOfCidrs int) error { // Check for rule with following pattern match := fmt.Sprintf(".*-j SNAT.*%s", expectedString) r, _ := regexp.Compile(match) - for _, rule := range rules { if r.Match([]byte(rule)) { containsExpectedString = true diff --git a/test/integration/snat/snat_test.go b/test/integration/snat/snat_test.go index f73fe6faf7..f95b6854f6 100644 --- a/test/integration/snat/snat_test.go +++ b/test/integration/snat/snat_test.go @@ -51,7 +51,12 @@ var _ = Describe("SNAT tests", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(vpcOutput.Vpcs)).To(BeNumerically(">", 0)) - numOfCidrs := len(vpcOutput.Vpcs[0].CidrBlockAssociationSet) + numOfCidrs := 0 + for _, vpc := range vpcOutput.Vpcs[0].CidrBlockAssociationSet { + if *vpc.CidrBlockState.State == "associated" { + numOfCidrs = numOfCidrs + 1 + } + } By("Check whether SNAT IP table has random-fully with AWS_VPC_K8S_CNI_RANDOMIZESNAT set to default value of prng") ValidateIPTableRules("prng", numOfCidrs)