From 1ade562e70e05046e95573cd1be99c0169525373 Mon Sep 17 00:00:00 2001 From: Daman Date: Sun, 29 Jan 2023 07:32:54 +0530 Subject: [PATCH] support for ipv6 --- backends/ipvsfullstate/common.go | 59 ++++++++-- backends/ipvsfullstate/flags.go | 5 - .../ipvsfullstate/handle_clusterip_service.go | 111 +++++++++--------- .../handle_loadbalancer_service.go | 73 ++++++------ .../ipvsfullstate/handle_nodeport_service.go | 77 ++++++------ .../ipvsfullstate/internal/ipsets/entry.go | 6 +- .../ipvsfullstate/internal/ipsets/ipset.go | 20 +--- .../internal/ipsets/ipset_util.go | 4 +- .../ipvsfullstate/internal/ipsets/manager.go | 4 +- .../ipvsfullstate/internal/ipsets/runner.go | 2 +- .../ipvsfullstate/internal/ipsets/types.go | 23 +++- .../internal/iptables/manager.go | 108 +++++++++++++---- .../ipvsfullstate/internal/iptables/types.go | 31 ++--- .../ipvsfullstate/internal/ipvs/manager.go | 7 +- backends/ipvsfullstate/internal/ipvs/setup.go | 2 +- .../ipvs/{server.go => virtual_server.go} | 0 backends/ipvsfullstate/ipset_definitions.go | 111 +++++++++++++----- backends/ipvsfullstate/iptable_rules.go | 49 ++++---- backends/ipvsfullstate/setup.go | 42 ++++--- 19 files changed, 455 insertions(+), 279 deletions(-) rename backends/ipvsfullstate/internal/ipvs/{server.go => virtual_server.go} (100%) diff --git a/backends/ipvsfullstate/common.go b/backends/ipvsfullstate/common.go index 83436e1b5..b90c99c06 100644 --- a/backends/ipvsfullstate/common.go +++ b/backends/ipvsfullstate/common.go @@ -1,38 +1,73 @@ package ipvsfullsate import ( + v1 "k8s.io/api/core/v1" + netutils "k8s.io/utils/net" "sigs.k8s.io/kpng/api/localv1" "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipvs" ) -// getClusterIPs safely returns ClusterIPs associated with the service. -func getClusterIPs(service *localv1.Service) []string { +// getClusterIPs returns ClusterIPs for given IPFamily associated with the service. +func getClusterIPs(service *localv1.Service, ipFamily v1.IPFamily) []string { IPs := make([]string, 0) if service.IPs.ClusterIPs != nil { - return service.IPs.ClusterIPs.V4 + if ipFamily == v1.IPv4Protocol { + return service.IPs.ClusterIPs.GetV4() + } else if ipFamily == v1.IPv6Protocol { + return service.IPs.ClusterIPs.GetV6() + } } return IPs } -// getExternalIPs safely returns ExternalIPs associated with the service. -func getExternalIPs(service *localv1.Service) []string { +// getExternalIPs safely returns ExternalIPs for given IPFamily associated with the service. +func getExternalIPs(service *localv1.Service, ipFamily v1.IPFamily) []string { IPs := make([]string, 0) if service.IPs.ExternalIPs != nil { - return service.IPs.ExternalIPs.V4 + if ipFamily == v1.IPv4Protocol { + return service.IPs.ExternalIPs.GetV4() + } else if ipFamily == v1.IPv6Protocol { + return service.IPs.ExternalIPs.GetV6() + } } return IPs } -// getNodeIPs safely returns all Node IPs. -func getNodeIPs() []string { - return *NodeAddresses +// getNodeIPs safely returns all Node IPs for given IPFamily. +func getNodeIPs(ipFamily v1.IPFamily) []string { + IPs := make([]string, 0) + for _, ip := range *NodeAddresses { + if ipFamily == v1.IPv4Protocol && netutils.IsIPv4String(ip) { + IPs = append(IPs, ip) + } else if ipFamily == v1.IPv6Protocol && netutils.IsIPv6String(ip) { + IPs = append(IPs, ip) + } + } + return IPs } -// getLoadBalancerIPs safely returns LoadBalancerIPs associated with the service. -func getLoadBalancerIPs(service *localv1.Service) []string { +// getLoadBalancerIPs safely returns LoadBalancerIPs for given IPFamily associated with the service. +func getLoadBalancerIPs(service *localv1.Service, ipFamily v1.IPFamily) []string { IPs := make([]string, 0) if service.IPs.LoadBalancerIPs != nil { - return service.IPs.LoadBalancerIPs.V4 + if ipFamily == v1.IPv4Protocol { + return service.IPs.LoadBalancerIPs.GetV4() + } else if ipFamily == v1.IPv6Protocol { + return service.IPs.LoadBalancerIPs.GetV6() + } + } + return IPs +} + +// getEndpointIPs returns EndpointIPs for given IPFamily associated with the endpoint. +func getEndpointIPs(endpoint *localv1.Endpoint, ipFamily v1.IPFamily) []string { + IPs := make([]string, 0) + if endpoint.IPs != nil { + if ipFamily == v1.IPv4Protocol { + return endpoint.IPs.GetV4() + } else if ipFamily == v1.IPv6Protocol { + return endpoint.IPs.GetV6() + } } return IPs } diff --git a/backends/ipvsfullstate/flags.go b/backends/ipvsfullstate/flags.go index 37ca1493b..8ccd82184 100644 --- a/backends/ipvsfullstate/flags.go +++ b/backends/ipvsfullstate/flags.go @@ -37,11 +37,6 @@ func interfaceAddresses() []string { panic(err) } - // only IPv4 for now - if ipv4 := ip.To4(); ipv4 == nil { - continue - } - addresses = append(addresses, ip.String()) } return addresses diff --git a/backends/ipvsfullstate/handle_clusterip_service.go b/backends/ipvsfullstate/handle_clusterip_service.go index 9e8ebf5d2..cf049455e 100644 --- a/backends/ipvsfullstate/handle_clusterip_service.go +++ b/backends/ipvsfullstate/handle_clusterip_service.go @@ -1,6 +1,7 @@ package ipvsfullsate import ( + v1 "k8s.io/api/core/v1" "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" "sigs.k8s.io/kpng/client" ) @@ -17,77 +18,81 @@ func (c *Controller) addServiceEndpointsForClusterIP(serviceEndpoints *client.Se // iterate over service ports for _, portMapping := range service.Ports { - // iterate over ClusterIPs - for _, clusterIP := range getClusterIPs(service) { + // iterate over ipFamily + for _, ipFamily := range []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} { - // STEP 1. create virtual server for Cluster IP - server := newVirtualServerForClusterIP(clusterIP, service, portMapping) - c.ipvsManager.ApplyServer(server) + // iterate over ClusterIPs + for _, clusterIP := range getClusterIPs(service, ipFamily) { - // STEP 2. add entry for ClusterIP To kubeClusterIPSet - entry = newEntryForClusterIP(clusterIP, portMapping) - set = c.ipsetsManager.GetSetByName(kubeClusterIPSet) - c.ipsetsManager.AddEntry(entry, set) + // STEP 1. create virtual server for Cluster IP + server := newVirtualServerForClusterIP(clusterIP, service, portMapping) + c.ipvsManager.ApplyServer(server) - // STEP 3. bind the ClusterIP to Host Interface - c.ipvsManager.BindServerToInterface(server) + // STEP 2. add entry for ClusterIP To kubeClusterIPSet + entry = newEntryForClusterIP(clusterIP, portMapping) + set = c.ipsetsManager.GetSetByName(kubeClusterIPSet[ipFamily]) + c.ipsetsManager.AddEntry(entry, set) + + // STEP 3. bind the ClusterIP to Host Interface + c.ipvsManager.BindServerToInterface(server) - // iterate over service endpoint - for _, endpoint := range endpoints { - // iterate over EndpointIPs - for _, endpointIp := range endpoint.IPs.V4 { + // iterate over service endpoint + for _, endpoint := range endpoints { + // iterate over EndpointIPs + for _, endpointIp := range getEndpointIPs(endpoint, ipFamily) { - // STEP 4. add endpoint as a destination to virtual server - destination := newIpvsDestination(endpointIp, endpoint, portMapping) - c.ipvsManager.AddDestination(destination, server) + // STEP 4. add endpoint as a destination to virtual server + destination := newIpvsDestination(endpointIp, endpoint, portMapping) + c.ipvsManager.AddDestination(destination, server) - if endpoint.GetLocal() { - // STEP 5. Add entry for EndpointIP to kubeLoopBackIPSet if endpoint is local - entry = newEntryForLocalEndpoint(endpointIp, endpoint, portMapping) - set = c.ipsetsManager.GetSetByName(kubeLoopBackIPSet) - c.ipsetsManager.AddEntry(entry, set) + if endpoint.GetLocal() { + // STEP 5. Add entry for EndpointIP to kubeLoopBackIPSet if endpoint is local + entry = newEntryForLocalEndpoint(endpointIp, endpoint, portMapping) + set = c.ipsetsManager.GetSetByName(kubeLoopBackIPSet[ipFamily]) + c.ipsetsManager.AddEntry(entry, set) + } } } } - } - - // iterate over ExternalIPs - for _, externalIP := range getExternalIPs(service) { - // STEP 6. create virtual server for ExternalIP - server := newVirtualServerForExternalIP(externalIP, service, portMapping) - c.ipvsManager.ApplyServer(server) + // iterate over ExternalIPs + for _, externalIP := range getExternalIPs(service, ipFamily) { - // create entry for ExternalIP - entry = newEntryForExternalIP(externalIP, portMapping) + // STEP 6. create virtual server for ExternalIP + server := newVirtualServerForExternalIP(externalIP, service, portMapping) + c.ipvsManager.ApplyServer(server) - // STEP 7. add entry for ExternalIP to kubeExternalIPSet - set = c.ipsetsManager.GetSetByName(kubeExternalIPSet) - c.ipsetsManager.AddEntry(entry, set) + // create entry for ExternalIP + entry = newEntryForExternalIP(externalIP, portMapping) - // STEP 8. add entry for ExternalIP to kubeExternalIPLocalSet if external traffic policy is local - if service.GetExternalTrafficToLocal() { - set = c.ipsetsManager.GetSetByName(kubeExternalIPLocalSet) + // STEP 7. add entry for ExternalIP to kubeExternalIPSet + set = c.ipsetsManager.GetSetByName(kubeExternalIPSet[ipFamily]) c.ipsetsManager.AddEntry(entry, set) - } - // STEP 9. bind the ExternalIP to Host Interface - c.ipvsManager.BindServerToInterface(server) + // STEP 8. add entry for ExternalIP to kubeExternalIPLocalSet if external traffic policy is local + if service.GetExternalTrafficToLocal() { + set = c.ipsetsManager.GetSetByName(kubeExternalIPLocalIPSet[ipFamily]) + c.ipsetsManager.AddEntry(entry, set) + } + + // STEP 9. bind the ExternalIP to Host Interface + c.ipvsManager.BindServerToInterface(server) - // iterate over service endpoints - for _, endpoint := range endpoints { - // iterate over EndpointIPs - for _, endpointIp := range endpoint.IPs.V4 { + // iterate over service endpoints + for _, endpoint := range endpoints { + // iterate over EndpointIPs + for _, endpointIp := range getEndpointIPs(endpoint, ipFamily) { - // STEP 10. add endpoint as a destination to virtual server - destination := newIpvsDestination(endpointIp, endpoint, portMapping) - c.ipvsManager.AddDestination(destination, server) + // STEP 10. add endpoint as a destination to virtual server + destination := newIpvsDestination(endpointIp, endpoint, portMapping) + c.ipvsManager.AddDestination(destination, server) - if endpoint.GetLocal() { - // STEP 11. Add entry for EndpointIP to kubeLoopBackIPSet if endpoint is local - entry = newEntryForLocalEndpoint(endpointIp, endpoint, portMapping) - set = c.ipsetsManager.GetSetByName(kubeLoopBackIPSet) - c.ipsetsManager.AddEntry(entry, set) + if endpoint.GetLocal() { + // STEP 11. Add entry for EndpointIP to kubeLoopBackIPSet if endpoint is local + entry = newEntryForLocalEndpoint(endpointIp, endpoint, portMapping) + set = c.ipsetsManager.GetSetByName(kubeLoopBackIPSet[ipFamily]) + c.ipsetsManager.AddEntry(entry, set) + } } } } diff --git a/backends/ipvsfullstate/handle_loadbalancer_service.go b/backends/ipvsfullstate/handle_loadbalancer_service.go index df339e391..8455b1a9c 100644 --- a/backends/ipvsfullstate/handle_loadbalancer_service.go +++ b/backends/ipvsfullstate/handle_loadbalancer_service.go @@ -1,6 +1,7 @@ package ipvsfullsate import ( + v1 "k8s.io/api/core/v1" ipsets2 "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" "sigs.k8s.io/kpng/client" ) @@ -22,50 +23,54 @@ func (c *Controller) addServiceEndpointsForLoadBalancer(serviceEndpoints *client // iterate over service ports for _, portMapping := range service.Ports { - // iterate over LoadBalancerIPs - for _, loadBalancerIP := range getLoadBalancerIPs(service) { + // iterate over ipFamily + for _, ipFamily := range []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} { - // STEP 2. create virtual server for LoadBalancerIP - server := newVirtualServerForLoadBalancer(loadBalancerIP, service, portMapping) - c.ipvsManager.ApplyServer(server) + // iterate over LoadBalancerIPs + for _, loadBalancerIP := range getLoadBalancerIPs(service, ipFamily) { - // STEP 3. add entry for LoadBalancerIP to kubeLoadBalancerSet - set = c.ipsetsManager.GetSetByName(kubeLoadBalancerSet) - entry = newEntryForLoadBalancer(loadBalancerIP, portMapping) - c.ipsetsManager.AddEntry(entry, set) + // STEP 2. create virtual server for LoadBalancerIP + server := newVirtualServerForLoadBalancer(loadBalancerIP, service, portMapping) + c.ipvsManager.ApplyServer(server) - // STEP 4. add entry for LoadBalancerIP to kubeLoadBalancerLocalSet, - // if external traffic policy is local - if service.GetExternalTrafficToLocal() { - set = c.ipsetsManager.GetSetByName(kubeLoadBalancerLocalSet) + // STEP 3. add entry for LoadBalancerIP to kubeLoadBalancerIPSet + set = c.ipsetsManager.GetSetByName(kubeLoadBalancerIPSet[ipFamily]) entry = newEntryForLoadBalancer(loadBalancerIP, portMapping) c.ipsetsManager.AddEntry(entry, set) - } - // STEP 5. add entry for LoadBalancerIP and SourceRanges to kubeLoadBalancerSourceCIDRSet - for _, sourceRange := range getSourceRangesForLoadBalancer(service) { - set = c.ipsetsManager.GetSetByName(kubeLoadBalancerSourceCIDRSet) - entry = newEntryForLoadBalancerSourceRange(loadBalancerIP, sourceRange, portMapping) - c.ipsetsManager.AddEntry(entry, set) - } + // STEP 4. add entry for LoadBalancerIP to kubeLoadBalancerLocalIPSet, + // if external traffic policy is local + if service.GetExternalTrafficToLocal() { + set = c.ipsetsManager.GetSetByName(kubeLoadBalancerLocalIPSet[ipFamily]) + entry = newEntryForLoadBalancer(loadBalancerIP, portMapping) + c.ipsetsManager.AddEntry(entry, set) + } - // STEP 6. add entry for LoadBalancerIP to kubeLoadbalancerFWSet, if source ranges is configured - if len(getSourceRangesForLoadBalancer(service)) > 0 { - set = c.ipsetsManager.GetSetByName(kubeLoadbalancerFWSet) - entry = newEntryForLoadBalancer(loadBalancerIP, portMapping) - c.ipsetsManager.AddEntry(entry, set) - } + // STEP 5. add entry for LoadBalancerIP and SourceRanges to kubeLoadBalancerSourceCIDRIPSet + for _, sourceRange := range getSourceRangesForLoadBalancer(service) { + set = c.ipsetsManager.GetSetByName(kubeLoadBalancerSourceCIDRIPSet[ipFamily]) + entry = newEntryForLoadBalancerSourceRange(loadBalancerIP, sourceRange, portMapping) + c.ipsetsManager.AddEntry(entry, set) + } + + // STEP 6. add entry for LoadBalancerIP to kubeLoadbalancerFWIPSet, if source ranges is configured + if len(getSourceRangesForLoadBalancer(service)) > 0 { + set = c.ipsetsManager.GetSetByName(kubeLoadbalancerFWIPSet[ipFamily]) + entry = newEntryForLoadBalancer(loadBalancerIP, portMapping) + c.ipsetsManager.AddEntry(entry, set) + } - // TODO entries to kubeLoadBalancerSourceIPSet; take reference from upstream ipvs proxier + // TODO entries to kubeLoadBalancerSourceIPSet; take reference from upstream ipvs proxier - // iterate over service endpoints - for _, endpoint := range endpoints { - // iterate over EndpointIPs - for _, endpointIp := range endpoint.IPs.V4 { - // STEP 7. add endpoint as a destination to virtual server - destination := newIpvsDestination(endpointIp, endpoint, portMapping) - c.ipvsManager.AddDestination(destination, server) + // iterate over service endpoints + for _, endpoint := range endpoints { + // iterate over EndpointIPs + for _, endpointIp := range getEndpointIPs(endpoint, ipFamily) { + // STEP 7. add endpoint as a destination to virtual server + destination := newIpvsDestination(endpointIp, endpoint, portMapping) + c.ipvsManager.AddDestination(destination, server) + } } } } diff --git a/backends/ipvsfullstate/handle_nodeport_service.go b/backends/ipvsfullstate/handle_nodeport_service.go index 4eafe78da..2c4997b80 100644 --- a/backends/ipvsfullstate/handle_nodeport_service.go +++ b/backends/ipvsfullstate/handle_nodeport_service.go @@ -1,6 +1,7 @@ package ipvsfullsate import ( + v1 "k8s.io/api/core/v1" "sigs.k8s.io/kpng/api/localv1" "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" "sigs.k8s.io/kpng/client" @@ -22,37 +23,41 @@ func (c *Controller) addServiceEndpointsForNodePort(serviceEndpoints *client.Ser // iterate over service ports for _, portMapping := range service.Ports { - // iterate over NodeIPs - for _, nodeIP := range getNodeIPs() { + // iterate over ipFamily + for _, ipFamily := range []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} { - // STEP 2. create virtual server for NodeIP - server := newVirtualServerForNodePort(nodeIP, service, portMapping) - c.ipvsManager.ApplyServer(server) + // iterate over NodeIPs + for _, nodeIP := range getNodeIPs(ipFamily) { - // STEP 3. add entry for NodeIP to [kubeNodePortSetTCP|kubeNodePortSetUDP|kubeNodePortSetSCTP] - // depending on the protocol - // TODO entry for kubeNodePortSetTCP & kubeNodePortSetUDP should be added once only as they will be same for every node, right now diffstore handles the duplicates. - set = c.ipsetsManager.GetSetByName(getNodePortIpSetNameByProtocol(portMapping.Protocol)) - entry = newEntryForNodePort(nodeIP, portMapping) - c.ipsetsManager.AddEntry(entry, set) + // STEP 2. create virtual server for NodeIP + server := newVirtualServerForNodePort(nodeIP, service, portMapping) + c.ipvsManager.ApplyServer(server) - // STEP 4. add entry for NodeIP to [kubeNodePortLocalSetTCP|kubeNodePortLocalSetUDP|kubeNodePortLocalSetSCTP] - // depending on the protocol if external traffic policy is local - // TODO entry for kubeNodePortLocalSetTCP & kubeNodePortLocalSetUDP should be added once only as they will be same for every node, right now diffstore handles the duplicates. - if service.GetExternalTrafficToLocal() { - set = c.ipsetsManager.GetSetByName(getNodePortLocalIpSetNameByProtocol(portMapping.Protocol)) + // STEP 3. add entry for NodeIP to [kubeNodePortTCPIPSet|kubeNodePortUDPIPSet|kubeNodePortSCTPIPSet] + // depending on the protocol + // TODO entry for kubeNodePortTCPIPSet & kubeNodePortUDPIPSet should be added once only as they will be same for every node, right now diffstore handles the duplicates. + set = c.ipsetsManager.GetSetByName(getNodePortIpSetNameByProtocol(ipFamily, portMapping.Protocol)) entry = newEntryForNodePort(nodeIP, portMapping) c.ipsetsManager.AddEntry(entry, set) - } - // iterate over service endpoints - for _, endpoint := range endpoints { - // iterate over EndpointIPs - for _, endpointIp := range endpoint.IPs.V4 { + // STEP 4. add entry for NodeIP to [kubeNodePortLocalTCPIPSet|kubeNodePortLocalUDPIPSet|kubeNodePortLocalSCTPIPSet] + // depending on the protocol if external traffic policy is local + // TODO entry for kubeNodePortLocalTCPIPSet & kubeNodePortLocalUDPIPSet should be added once only as they will be same for every node, right now diffstore handles the duplicates. + if service.GetExternalTrafficToLocal() { + entry = newEntryForNodePort(nodeIP, portMapping) + set = c.ipsetsManager.GetSetByName(getNodePortLocalIpSetNameByProtocol(ipFamily, portMapping.Protocol)) + c.ipsetsManager.AddEntry(entry, set) + } + + // iterate over service endpoints + for _, endpoint := range endpoints { + // iterate over EndpointIPs + for _, endpointIp := range getEndpointIPs(endpoint, ipFamily) { - // STEP 5. add endpoint as a destination to virtual server - destination := newIpvsDestination(endpointIp, endpoint, portMapping) - c.ipvsManager.AddDestination(destination, server) + // STEP 5. add endpoint as a destination to virtual server + destination := newIpvsDestination(endpointIp, endpoint, portMapping) + c.ipvsManager.AddDestination(destination, server) + } } } } @@ -60,31 +65,31 @@ func (c *Controller) addServiceEndpointsForNodePort(serviceEndpoints *client.Ser } // getNodePortIpSetNameByProtocol returns NodePort IPSet name for -// the given protocol, defaults to TCP. -func getNodePortIpSetNameByProtocol(protocol localv1.Protocol) string { +// the given ipFamily and protocol, defaults to TCP. +func getNodePortIpSetNameByProtocol(ipFamily v1.IPFamily, protocol localv1.Protocol) string { switch protocol { case localv1.Protocol_SCTP: - return kubeNodePortSetSCTP + return kubeNodePortSCTPIPSet[ipFamily] case localv1.Protocol_TCP: - return kubeNodePortSetTCP + return kubeNodePortTCPIPSet[ipFamily] case localv1.Protocol_UDP: - return kubeNodePortSetUDP + return kubeNodePortUDPIPSet[ipFamily] default: - return kubeNodePortSetTCP + return kubeNodePortTCPIPSet[ipFamily] } } // getNodePortLocalIpSetNameByProtocol returns NodePortLocal IPSet name for -// the given protocol, defaults to TCP. -func getNodePortLocalIpSetNameByProtocol(protocol localv1.Protocol) string { +// the given ipFamily and protocol, defaults to TCP. +func getNodePortLocalIpSetNameByProtocol(ipFamily v1.IPFamily, protocol localv1.Protocol) string { switch protocol { case localv1.Protocol_SCTP: - return kubeNodePortLocalSetSCTP + return kubeNodePortLocalSCTPIPSet[ipFamily] case localv1.Protocol_TCP: - return kubeNodePortLocalSetTCP + return kubeNodePortLocalTCPIPSet[ipFamily] case localv1.Protocol_UDP: - return kubeNodePortLocalSetUDP + return kubeNodePortLocalUDPIPSet[ipFamily] default: - return kubeNodePortLocalSetTCP + return kubeNodePortLocalTCPIPSet[ipFamily] } } diff --git a/backends/ipvsfullstate/internal/ipsets/entry.go b/backends/ipvsfullstate/internal/ipsets/entry.go index dff2b3c27..41ea5311b 100644 --- a/backends/ipvsfullstate/internal/ipsets/entry.go +++ b/backends/ipvsfullstate/internal/ipsets/entry.go @@ -149,11 +149,11 @@ func validatePortRange(portRange string) bool { } // checks if given hash family is supported in ipset -func validateHashFamily(family string) bool { - if family == ProtocolFamilyIPV4 || family == ProtocolFamilyIPV6 { +func validateHashFamily(family ProtocolFamily) bool { + if family == ProtocolFamilyIPv4 || family == ProtocolFamilyIPv6 { return true } - klog.Errorf("Currently supported ip set hash families are: [%s, %s], %s is not supported", ProtocolFamilyIPV4, ProtocolFamilyIPV6, family) + klog.Errorf("Currently supported ip set hash families are: [%s, %s], %s is not supported", ProtocolFamilyIPv4, ProtocolFamilyIPv6, family) return false } diff --git a/backends/ipvsfullstate/internal/ipsets/ipset.go b/backends/ipvsfullstate/internal/ipsets/ipset.go index 3a089cdba..e1d76dac3 100644 --- a/backends/ipvsfullstate/internal/ipsets/ipset.go +++ b/backends/ipvsfullstate/internal/ipsets/ipset.go @@ -18,8 +18,6 @@ package ipsets import ( "fmt" - "strings" - "k8s.io/klog/v2" ) @@ -32,23 +30,7 @@ type Set struct { } // newIPSet initialize a new Set struct -func newIPSet(handle Interface, name string, setType SetType, hashFamily string, comment string) *Set { - if hashFamily == ProtocolFamilyIPV6 { - // In dual-stack both ipv4 and ipv6 ipset's can co-exist. To - // ensure unique names the prefix for ipv6 is changed from - // "KUBE-" to "KUBE-6-". The "KUBE-" prefix is kept for - // backward compatibility. The maximum name length of an ipset - // is 31 characters which must be taken into account. The - // ipv4 names are not altered to minimize the risk for - // problems on upgrades. - if strings.HasPrefix(name, "KUBE-") { - name = strings.Replace(name, "KUBE-", "KUBE-6-", 1) - if len(name) > 31 { - klog.Info("Ipset name truncated", "ipSetName", name, "truncatedName", name[:31]) - name = name[:31] - } - } - } +func newIPSet(handle Interface, name string, setType SetType, hashFamily ProtocolFamily, comment string) *Set { set := &Set{ IPSet: IPSet{ Name: name, diff --git a/backends/ipvsfullstate/internal/ipsets/ipset_util.go b/backends/ipvsfullstate/internal/ipsets/ipset_util.go index 899b5d54c..37b8af41e 100644 --- a/backends/ipvsfullstate/internal/ipsets/ipset_util.go +++ b/backends/ipvsfullstate/internal/ipsets/ipset_util.go @@ -72,7 +72,7 @@ type IPSet struct { SetType SetType // HashFamily specifies the protocol family of the IP addresses to be stored in the set. // The default is inet, i.e IPv4. If users want to use IPv6, they should specify inet6. - HashFamily string + HashFamily ProtocolFamily // HashSize specifies the hash table size of ipset. HashSize int // MaxElem specifies the max element number of ipset. @@ -126,7 +126,7 @@ func (set *IPSet) setIPSetDefaults() { } // Default protocol is IPv4 if set.HashFamily == "" { - set.HashFamily = ProtocolFamilyIPV4 + set.HashFamily = ProtocolFamilyIPv4 } // Default ipset type is "hash:ip,port" if len(set.SetType) == 0 { diff --git a/backends/ipvsfullstate/internal/ipsets/manager.go b/backends/ipvsfullstate/internal/ipsets/manager.go index f5333e94d..ab9d5c6c1 100644 --- a/backends/ipvsfullstate/internal/ipsets/manager.go +++ b/backends/ipvsfullstate/internal/ipsets/manager.go @@ -37,8 +37,8 @@ func (m *Manager) Reset() { } // CreateSet doesn't use diffstore, straightaway creates the set and add it to ipsetMap. -func (m *Manager) CreateSet(name string, setType SetType, comment string) (*Set, error) { - set := newIPSet(New(exec.New()), name, setType, ProtocolFamilyIPV4, comment) +func (m *Manager) CreateSet(name string, setType SetType, protocolFamily ProtocolFamily, comment string) (*Set, error) { + set := newIPSet(New(exec.New()), name, setType, protocolFamily, comment) m.ipsetMap[name] = set return set, ensureIPSet(set) } diff --git a/backends/ipvsfullstate/internal/ipsets/runner.go b/backends/ipvsfullstate/internal/ipsets/runner.go index e1a81a80f..b8b30cb48 100644 --- a/backends/ipvsfullstate/internal/ipsets/runner.go +++ b/backends/ipvsfullstate/internal/ipsets/runner.go @@ -39,7 +39,7 @@ func (runner *runner) createSet(set *IPSet, ignoreExistErr bool) error { args := []string{"create", set.Name, string(set.SetType)} if set.SetType == HashIPPortIP || set.SetType == HashIPPort || set.SetType == HashIPPortNet { args = append(args, - "family", set.HashFamily, + "family", set.HashFamily.String(), "hashsize", strconv.Itoa(set.HashSize), "maxelem", strconv.Itoa(set.MaxElem), ) diff --git a/backends/ipvsfullstate/internal/ipsets/types.go b/backends/ipvsfullstate/internal/ipsets/types.go index 787355121..3446d74c1 100644 --- a/backends/ipvsfullstate/internal/ipsets/types.go +++ b/backends/ipvsfullstate/internal/ipsets/types.go @@ -37,11 +37,26 @@ const ( // DefaultPortRange defines the default bitmap:port valid port range. const DefaultPortRange string = "0-65535" +type ProtocolFamily string + +const ( + // ProtocolFamilyIPv4 represents IPv4 protocol. + ProtocolFamilyIPv4 = "inet" + // ProtocolFamilyIPv6 represents IPv6 protocol. + ProtocolFamilyIPv6 = "inet6" +) + +func (p ProtocolFamily) String() string { + return string(p) +} + +type Protocol string + +func (p Protocol) String() string { + return string(p) +} + const ( - // ProtocolFamilyIPV4 represents IPv4 protocol. - ProtocolFamilyIPV4 = "inet" - // ProtocolFamilyIPV6 represents IPv6 protocol. - ProtocolFamilyIPV6 = "inet6" // ProtocolTCP represents TCP protocol. ProtocolTCP = "tcp" // ProtocolUDP represents UDP protocol. diff --git a/backends/ipvsfullstate/internal/iptables/manager.go b/backends/ipvsfullstate/internal/iptables/manager.go index 1a90e969b..f204fb653 100644 --- a/backends/ipvsfullstate/internal/iptables/manager.go +++ b/backends/ipvsfullstate/internal/iptables/manager.go @@ -7,17 +7,39 @@ import ( "text/template" ) -const iptablesRestoreCmd = "iptables-restore" +const ip4tablesRestoreCmd = "iptables-restore" +const ip6tablesRestoreCmd = "ip6tables-restore" var DefaultChains = []Chain{ChainPreRouting, ChainInput, ChainForward, ChainOutput, ChainPostRouting} type Manager struct { - data map[Table]TableData + dataV4 map[Table]TableData + dataV6 map[Table]TableData template *template.Template } +func getIptablesRestoreCmd(protocolFamily ProtocolFamily) string { + if protocolFamily == ProtocolFamilyIPv4 { + return ip4tablesRestoreCmd + } + return ip6tablesRestoreCmd +} + func NewManager() *Manager { - data := map[Table]TableData{ + dataV4 := map[Table]TableData{ + TableNat: { + Table: TableNat, + Chains: []Chain{ChainPreRouting, ChainInput, ChainOutput, ChainPostRouting}, + Rules: []Rule{}, + }, + TableFilter: { + Table: TableFilter, + Chains: []Chain{ChainInput, ChainForward, ChainOutput}, + Rules: []Rule{}, + }, + } + + dataV6 := map[Table]TableData{ TableNat: { Table: TableNat, Chains: []Chain{ChainPreRouting, ChainInput, ChainOutput, ChainPostRouting}, @@ -36,47 +58,86 @@ func NewManager() *Manager { } iptTemplate, err := template.New("Template").Funcs(funcMap).Parse(Template) - klog.V(2).ErrorS(err, "error parsing iptables template") + if err != nil { + klog.Fatalf("error parsing iptables template: Template, error: %e", err) + } iptTemplate, err = iptTemplate.New("TableTemplate").Parse(TableTemplate) - klog.V(2).ErrorS(err, "error parsing iptables template") + if err != nil { + klog.Fatalf("error parsing iptables template: TableTemplate, error: %e", err) + } iptTemplate, err = iptTemplate.New("ChainTemplate").Parse(ChainTemplate) - klog.V(2).ErrorS(err, "error parsing iptables template") + if err != nil { + klog.Fatalf("error parsing iptables template: ChainTemplate, error: %e", err) + } iptTemplate, err = iptTemplate.New("RuleTemplate").Parse(RuleTemplate) - klog.V(2).ErrorS(err, "error parsing iptables template") + if err != nil { + klog.Fatalf("error parsing iptables template: RuleTemplate, error: %e", err) + } iptTemplate, err = iptTemplate.New("MatchTemplate").Parse(MatchTemplate) - klog.V(2).ErrorS(err, "error parsing iptables template") + if err != nil { + klog.Fatalf("error parsing iptables template: MatchTemplate, error: %e", err) + } iptTemplate, err = iptTemplate.New("ProtocolTemplate").Parse(ProtocolTemplate) - klog.V(2).ErrorS(err, "error parsing iptables template") + if err != nil { + klog.Fatalf("error parsing iptables template: ProtocolTemplate, error: %e", err) + } return &Manager{ - data: data, + dataV4: dataV4, + dataV6: dataV6, template: iptTemplate, } } -func (m *Manager) AddChain(chain Chain, table Table) { - tableData, _ := m.data[table] - tableData.Chains = append(tableData.Chains, chain) - m.data[table] = tableData +func (m *Manager) AddChain(chain Chain, table Table, protocolFamily ProtocolFamily) { + if protocolFamily == ProtocolFamilyIPv4 { + tableData, _ := m.dataV4[table] + tableData.Chains = append(tableData.Chains, chain) + m.dataV4[table] = tableData + } else { + tableData, _ := m.dataV6[table] + tableData.Chains = append(tableData.Chains, chain) + m.dataV6[table] = tableData + } } -func (m *Manager) AddRule(rule Rule, table Table) { - tableData, _ := m.data[table] - tableData.Rules = append(tableData.Rules, rule) - m.data[table] = tableData +func (m *Manager) AddRule(rule Rule, table Table, protocolFamily ProtocolFamily) { + if protocolFamily == ProtocolFamilyIPv4 { + tableData, _ := m.dataV4[table] + tableData.Rules = append(tableData.Rules, rule) + m.dataV4[table] = tableData + } else { + tableData, _ := m.dataV6[table] + tableData.Rules = append(tableData.Rules, rule) + m.dataV6[table] = tableData + } + } func (m *Manager) Apply() { - data := make([]TableData, 0) - for _, d := range m.data { + var data []TableData + + // render & restore ipv4 table data + data = make([]TableData, 0) + for _, d := range m.dataV4 { data = append(data, d) } + m.renderAndRestoreTable(data, ProtocolFamilyIPv4) + // render & restore ipv6 table data + data = make([]TableData, 0) + for _, d := range m.dataV6 { + data = append(data, d) + } + m.renderAndRestoreTable(data, ProtocolFamilyIPv6) +} + +func (m *Manager) renderAndRestoreTable(data []TableData, protocolFamily ProtocolFamily) { //######################################################################## reader, writer := io.Pipe() errChan := make(chan error, 1) @@ -88,12 +149,15 @@ func (m *Manager) Apply() { //######################################################################## runner := exec.New() - cmd := runner.Command(iptablesRestoreCmd) + cmd := runner.Command(getIptablesRestoreCmd(protocolFamily)) cmd.SetStdin(reader) output, err := cmd.CombinedOutput() - klog.V(2).ErrorS(err, "unable to write iptable rules", "output", string(output)) + if err != nil { + klog.Fatalf("unable to write iptable rules output: %s error: %e", string(output), err) + } _ = reader.Close() + } func NeedQuotes(option MatchModuleOption) bool { diff --git a/backends/ipvsfullstate/internal/iptables/types.go b/backends/ipvsfullstate/internal/iptables/types.go index bd0818e9a..00d43c305 100644 --- a/backends/ipvsfullstate/internal/iptables/types.go +++ b/backends/ipvsfullstate/internal/iptables/types.go @@ -8,25 +8,19 @@ const ( ) type TableData struct { - Table Table - Chains []Chain - Rules []Rule + Table Table + Chains []Chain + Rules []Rule + ProtocolFamily ProtocolFamily } type Chain string const ( - ChainPreRouting Chain = "PREROUTING" - ChainInput = "INPUT" - ChainForward = "FORWARD" - ChainOutput = "OUTPUT" - ChainPostRouting = "POSTROUTING" - ChainKubeFirewall = "KUBE-FIREWALL" - ChainKubeLoadBalancer = "KUBE-LOAD-BALANCER" - ChainKubeMarkDrop = "KUBE-MARK-DROP" - ChainKubeMarkMasq = "KUBE-MARK-MASQ" - ChainKubeNodePort = "KUBE-NODE-PORT" - ChainKubePostRouting = "KUBE-POSTROUTING" - ChainKubeServices = "KUBE-SERVICES" + ChainPreRouting Chain = "PREROUTING" + ChainInput = "INPUT" + ChainForward = "FORWARD" + ChainOutput = "OUTPUT" + ChainPostRouting = "POSTROUTING" ) type TargetOption string @@ -99,3 +93,10 @@ const ( ProtocolUDP = "udp" ProtocolSCTP = "sctp" ) + +type ProtocolFamily string + +const ( + ProtocolFamilyIPv4 ProtocolFamily = "inet" + ProtocolFamilyIPv6 = "inet6" +) diff --git a/backends/ipvsfullstate/internal/ipvs/manager.go b/backends/ipvsfullstate/internal/ipvs/manager.go index 8047c055f..5d7c284f3 100644 --- a/backends/ipvsfullstate/internal/ipvs/manager.go +++ b/backends/ipvsfullstate/internal/ipvs/manager.go @@ -6,6 +6,7 @@ import ( IPVSLib "github.com/google/seesaw/ipvs" "github.com/vishvananda/netlink" "k8s.io/klog/v2" + netutils "k8s.io/utils/net" "net" "sigs.k8s.io/kpng/client/diffstore" ) @@ -223,7 +224,11 @@ func (m *Manager) Apply() { } func asDummyIP(ip string) string { - return ip + "/32" + if netutils.IsIPv4String(ip) { + return ip + "/32" + } else { + return ip + "/128" + } } // bindIpToInterface adds IP address to the network interface. diff --git a/backends/ipvsfullstate/internal/ipvs/setup.go b/backends/ipvsfullstate/internal/ipvs/setup.go index b0092697c..42e087989 100644 --- a/backends/ipvsfullstate/internal/ipvs/setup.go +++ b/backends/ipvsfullstate/internal/ipvs/setup.go @@ -26,7 +26,7 @@ const ( func (m *Manager) Setup() error { var err error - klog.V(3).Info("stated initializing ipvs manager") + klog.V(3).Info("initializing ipvs manager") err = initializeKernelConfig(NewLinuxKernelHandler()) if err != nil { diff --git a/backends/ipvsfullstate/internal/ipvs/server.go b/backends/ipvsfullstate/internal/ipvs/virtual_server.go similarity index 100% rename from backends/ipvsfullstate/internal/ipvs/server.go rename to backends/ipvsfullstate/internal/ipvs/virtual_server.go diff --git a/backends/ipvsfullstate/ipset_definitions.go b/backends/ipvsfullstate/ipset_definitions.go index 8d5f3b9fc..b982fac6b 100644 --- a/backends/ipvsfullstate/ipset_definitions.go +++ b/backends/ipvsfullstate/ipset_definitions.go @@ -17,79 +17,128 @@ limitations under the License. package ipvsfullsate import ( + v1 "k8s.io/api/core/v1" "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" ) -const ( +var ( kubeLoopBackIPSetComment = "Kubernetes endpoints dst ip:port, source ip for solving hairpin purpose" - kubeLoopBackIPSet = "KUBE-LOOP-BACK" + kubeLoopBackIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-LOOP-BACK", + v1.IPv6Protocol: "KUBE-6-LOOP-BACK", + } kubeClusterIPSetComment = "Kubernetes service cluster ip + port for masquerade purpose" - kubeClusterIPSet = "KUBE-CLUSTER-IP" + kubeClusterIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-CLUSTER-IP", + v1.IPv6Protocol: "KUBE-6-CLUSTER-IP", + } kubeExternalIPSetComment = "Kubernetes service external ip + port for masquerade and filter purpose" - kubeExternalIPSet = "KUBE-EXTERNAL-IP" + kubeExternalIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-EXTERNAL-IP", + v1.IPv6Protocol: "KUBE-6-EXTERNAL-IP", + } kubeExternalIPLocalSetComment = "Kubernetes service external ip + port with externalTrafficPolicy=local" - kubeExternalIPLocalSet = "KUBE-EXTERNAL-IP-LOCAL" + kubeExternalIPLocalIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-EXTERNAL-IP-LOCAL", + v1.IPv6Protocol: "KUBE-6-EXTERNAL-IP-LOCAL", + } kubeLoadBalancerSetComment = "Kubernetes service lb portal" - kubeLoadBalancerSet = "KUBE-LOAD-BALANCER" + kubeLoadBalancerIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-LOAD-BALANCER", + v1.IPv6Protocol: "KUBE-6-LOAD-BALANCER", + } kubeLoadBalancerLocalSetComment = "Kubernetes service load balancer ip + port with externalTrafficPolicy=local" - kubeLoadBalancerLocalSet = "KUBE-LOAD-BALANCER-LOCAL" + kubeLoadBalancerLocalIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-LOAD-BALANCER-LOCAL", + v1.IPv6Protocol: "KUBE-6-LOAD-BALANCER-LOCAL", + } kubeLoadbalancerFWSetComment = "Kubernetes service load balancer ip + port for load balancer with sourceRange" - kubeLoadbalancerFWSet = "KUBE-LOAD-BALANCER-FW" + kubeLoadbalancerFWIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-LOAD-BALANCER-FW", + v1.IPv6Protocol: "KUBE-6-LOAD-BALANCER-FW", + } kubeLoadBalancerSourceIPSetComment = "Kubernetes service load balancer ip + port + source IP for packet filter purpose" - kubeLoadBalancerSourceIPSet = "KUBE-LOAD-BALANCER-SOURCE-IP" + kubeLoadBalancerSourceIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-LOAD-BALANCER-SOURCE-IP", + v1.IPv6Protocol: "KUBE-6-LOAD-BALANCER-SOURCE-IP", + } kubeLoadBalancerSourceCIDRSetComment = "Kubernetes service load balancer ip + port + source cidr for packet filter purpose" - kubeLoadBalancerSourceCIDRSet = "KUBE-LOAD-BALANCER-SOURCE-CIDR" + kubeLoadBalancerSourceCIDRIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-LOAD-BALANCER-SRC-CIDR", + v1.IPv6Protocol: "KUBE-6-LOAD-BALANCER-SRC-CIDR", + } kubeNodePortSetTCPComment = "Kubernetes nodeport TCP port for masquerade purpose" - kubeNodePortSetTCP = "KUBE-NODE-PORT-TCP" + kubeNodePortTCPIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-NODE-PORT-TCP", + v1.IPv6Protocol: "KUBE-6-NODE-PORT-TCP", + } kubeNodePortLocalSetTCPComment = "Kubernetes nodeport TCP port with externalTrafficPolicy=local" - kubeNodePortLocalSetTCP = "KUBE-NODE-PORT-LOCAL-TCP" + kubeNodePortLocalTCPIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-NODE-PORT-LOCAL-TCP", + v1.IPv6Protocol: "KUBE-6-NODE-PORT-LOCAL-TCP", + } kubeNodePortSetUDPComment = "Kubernetes nodeport UDP port for masquerade purpose" - kubeNodePortSetUDP = "KUBE-NODE-PORT-UDP" + kubeNodePortUDPIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-NODE-PORT-UDP", + v1.IPv6Protocol: "KUBE-6-NODE-PORT-UDP", + } kubeNodePortLocalSetUDPComment = "Kubernetes nodeport UDP port with externalTrafficPolicy=local" - kubeNodePortLocalSetUDP = "KUBE-NODE-PORT-LOCAL-UDP" + kubeNodePortLocalUDPIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-NODE-PORT-LOCAL-UDP", + v1.IPv6Protocol: "KUBE-6-NODE-PORT-LOCAL-UDP", + } kubeNodePortSetSCTPComment = "Kubernetes nodeport SCTP port for masquerade purpose with type 'hash ip:port'" - kubeNodePortSetSCTP = "KUBE-NODE-PORT-SCTP-HASH" + kubeNodePortSCTPIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-NODE-PORT-SCTP", + v1.IPv6Protocol: "KUBE-6-NODE-PORT-SCTP", + } kubeNodePortLocalSetSCTPComment = "Kubernetes nodeport SCTP port with externalTrafficPolicy=local with type 'hash ip:port'" - kubeNodePortLocalSetSCTP = "KUBE-NODE-PORT-LOCAL-SCTP-HASH" + kubeNodePortLocalSCTPIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-NODE-PORT-LOCAL-SCTP", + v1.IPv6Protocol: "KUBE-6-NODE-PORT-LOCAL-SCTP", + } kubeHealthCheckNodePortSetComment = "Kubernetes health check node port" - kubeHealthCheckNodePortSet = "KUBE-HEALTH-CHECK-NODE-PORT" + kubeHealthCheckNodePortIPSet = map[v1.IPFamily]string{ + v1.IPv4Protocol: "KUBE-HEALTH-CHECK-NODE-PORT", + v1.IPv6Protocol: "KUBE-6-HEALTH-CHECK-NODE-PORT", + } ) -// ipsetInfo is all ipset we needed in ipvs proxier +// ipsetInfo is all ipset we needed in ipvs var ipsetInfo = []struct { - name string + name map[v1.IPFamily]string setType ipsets.SetType comment string }{ {kubeLoopBackIPSet, ipsets.HashIPPortIP, kubeLoopBackIPSetComment}, {kubeClusterIPSet, ipsets.HashIPPort, kubeClusterIPSetComment}, {kubeExternalIPSet, ipsets.HashIPPort, kubeExternalIPSetComment}, - {kubeExternalIPLocalSet, ipsets.HashIPPort, kubeExternalIPLocalSetComment}, - {kubeLoadBalancerSet, ipsets.HashIPPort, kubeLoadBalancerSetComment}, - {kubeLoadbalancerFWSet, ipsets.HashIPPort, kubeLoadbalancerFWSetComment}, - {kubeLoadBalancerLocalSet, ipsets.HashIPPort, kubeLoadBalancerLocalSetComment}, + {kubeExternalIPLocalIPSet, ipsets.HashIPPort, kubeExternalIPLocalSetComment}, + {kubeLoadBalancerIPSet, ipsets.HashIPPort, kubeLoadBalancerSetComment}, + {kubeLoadbalancerFWIPSet, ipsets.HashIPPort, kubeLoadbalancerFWSetComment}, + {kubeLoadBalancerLocalIPSet, ipsets.HashIPPort, kubeLoadBalancerLocalSetComment}, {kubeLoadBalancerSourceIPSet, ipsets.HashIPPortIP, kubeLoadBalancerSourceIPSetComment}, - {kubeLoadBalancerSourceCIDRSet, ipsets.HashIPPortNet, kubeLoadBalancerSourceCIDRSetComment}, - {kubeNodePortSetTCP, ipsets.BitmapPort, kubeNodePortSetTCPComment}, - {kubeNodePortLocalSetTCP, ipsets.BitmapPort, kubeNodePortLocalSetTCPComment}, - {kubeNodePortSetUDP, ipsets.BitmapPort, kubeNodePortSetUDPComment}, - {kubeNodePortLocalSetUDP, ipsets.BitmapPort, kubeNodePortLocalSetUDPComment}, - {kubeNodePortSetSCTP, ipsets.HashIPPort, kubeNodePortSetSCTPComment}, - {kubeNodePortLocalSetSCTP, ipsets.HashIPPort, kubeNodePortLocalSetSCTPComment}, - {kubeHealthCheckNodePortSet, ipsets.BitmapPort, kubeHealthCheckNodePortSetComment}, + {kubeLoadBalancerSourceCIDRIPSet, ipsets.HashIPPortNet, kubeLoadBalancerSourceCIDRSetComment}, + {kubeNodePortTCPIPSet, ipsets.BitmapPort, kubeNodePortSetTCPComment}, + {kubeNodePortLocalTCPIPSet, ipsets.BitmapPort, kubeNodePortLocalSetTCPComment}, + {kubeNodePortUDPIPSet, ipsets.BitmapPort, kubeNodePortSetUDPComment}, + {kubeNodePortLocalUDPIPSet, ipsets.BitmapPort, kubeNodePortLocalSetUDPComment}, + {kubeNodePortSCTPIPSet, ipsets.HashIPPort, kubeNodePortSetSCTPComment}, + {kubeNodePortLocalSCTPIPSet, ipsets.HashIPPort, kubeNodePortLocalSetSCTPComment}, + {kubeHealthCheckNodePortIPSet, ipsets.BitmapPort, kubeHealthCheckNodePortSetComment}, } diff --git a/backends/ipvsfullstate/iptable_rules.go b/backends/ipvsfullstate/iptable_rules.go index 80ad705e9..77c90f66d 100644 --- a/backends/ipvsfullstate/iptable_rules.go +++ b/backends/ipvsfullstate/iptable_rules.go @@ -2,6 +2,7 @@ package ipvsfullsate import ( "fmt" + v1 "k8s.io/api/core/v1" ipt "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/iptables" ) @@ -35,7 +36,7 @@ const KubeForwardChain = ipt.Chain("KUBE-FORWARD") // KubeLoadBalancerChain is the kubernetes chain for loadbalancer type service. const KubeLoadBalancerChain = ipt.Chain("KUBE-LOAD-BALANCER") -func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { +func GetNatRules(supportsFullyRandomized bool, ipFamily v1.IPFamily) []ipt.Rule { // link default IPTables chains to custom chains where we will program the iptable rules. rules := []ipt.Rule{ @@ -88,7 +89,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeLoopBackIPSet + " dst,dst,src", + Value: kubeLoopBackIPSet[ipFamily] + " dst,dst,src", }, }, }, @@ -105,7 +106,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeLoadBalancerSet + " dst,dst", + Value: kubeLoadBalancerIPSet[ipFamily] + " dst,dst", }, }, }, @@ -121,7 +122,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeLoadbalancerFWSet + " dst,dst", + Value: kubeLoadbalancerFWIPSet[ipFamily] + " dst,dst", }, }, }, @@ -137,7 +138,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeLoadBalancerSourceCIDRSet + " dst,dst,src", + Value: kubeLoadBalancerSourceCIDRIPSet[ipFamily] + " dst,dst,src", }, }, }, @@ -153,7 +154,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeLoadBalancerSourceIPSet + " dst,dst,src", + Value: kubeLoadBalancerSourceIPSet[ipFamily] + " dst,dst,src", }, }, }, @@ -169,7 +170,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeLoadBalancerLocalSet + " dst,dst", + Value: kubeLoadBalancerLocalIPSet[ipFamily] + " dst,dst", }, }, }, @@ -186,7 +187,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeNodePortSetTCP + " dst", + Value: kubeNodePortTCPIPSet[ipFamily] + " dst", }, }, }, @@ -203,7 +204,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeNodePortLocalSetTCP + " dst", + Value: kubeNodePortLocalTCPIPSet[ipFamily] + " dst", }, }, }, @@ -220,7 +221,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeNodePortSetUDP + " dst", + Value: kubeNodePortUDPIPSet[ipFamily] + " dst", }, }, }, @@ -237,7 +238,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeNodePortLocalSetUDP + " dst", + Value: kubeNodePortLocalUDPIPSet[ipFamily] + " dst", }, }, }, @@ -254,7 +255,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeNodePortSetSCTP + " dst,dst", + Value: kubeNodePortSCTPIPSet[ipFamily] + " dst,dst", }, }, }, @@ -271,7 +272,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeNodePortLocalSetSCTP + " dst,dst", + Value: kubeNodePortLocalSCTPIPSet[ipFamily] + " dst,dst", }, }, }) @@ -289,7 +290,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeClusterIPSet + " dst,dst", + Value: kubeClusterIPSet[ipFamily] + " dst,dst", }, }, }) @@ -313,7 +314,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeClusterIPSet + " src,dst", + Value: kubeClusterIPSet[ipFamily] + " src,dst", }, }, }) @@ -333,7 +334,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeExternalIPSet + " dst,dst", + Value: kubeExternalIPSet[ipFamily] + " dst,dst", }, }, }, @@ -349,7 +350,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeExternalIPSet + " dst,dst", + Value: kubeExternalIPSet[ipFamily] + " dst,dst", }, { Module: ipt.MatchModulePhysDev, @@ -376,7 +377,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeExternalIPSet + " dst,dst", + Value: kubeExternalIPSet[ipFamily] + " dst,dst", }, { Module: ipt.MatchModuleAddrType, @@ -401,7 +402,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeExternalIPLocalSet + " dst,dst", + Value: kubeExternalIPLocalIPSet[ipFamily] + " dst,dst", }, { Module: ipt.MatchModulePhysDev, @@ -428,7 +429,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeExternalIPLocalSet + " dst,dst", + Value: kubeExternalIPLocalIPSet[ipFamily] + " dst,dst", }, { Module: ipt.MatchModuleAddrType, @@ -477,7 +478,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeClusterIPSet + " dst,dst", + Value: kubeClusterIPSet[ipFamily] + " dst,dst", }, }, }, @@ -488,7 +489,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeLoadBalancerSet + " dst,dst", + Value: kubeLoadBalancerIPSet[ipFamily] + " dst,dst", }, }, }, @@ -562,7 +563,7 @@ func GetNatRules(supportsFullyRandomized bool) []ipt.Rule { return rules } -func GetFilterRules(supportsFullyRandomized bool) []ipt.Rule { +func GetFilterRules(supportsFullyRandomized bool, ipFamily v1.IPFamily) []ipt.Rule { // link default IPTables chains to custom chains where we will program the iptable rules. rules := []ipt.Rule{ { @@ -640,7 +641,7 @@ func GetFilterRules(supportsFullyRandomized bool) []ipt.Rule { { Module: ipt.MatchModuleSet, ModuleOption: ipt.MatchModuleSetOptionSet, - Value: kubeHealthCheckNodePortSet + " dst", + Value: kubeHealthCheckNodePortIPSet[ipFamily] + " dst", }, }, }) diff --git a/backends/ipvsfullstate/setup.go b/backends/ipvsfullstate/setup.go index 36d57557f..ead3a365f 100644 --- a/backends/ipvsfullstate/setup.go +++ b/backends/ipvsfullstate/setup.go @@ -17,6 +17,7 @@ limitations under the License. package ipvsfullsate import ( + v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/iptables" @@ -25,7 +26,6 @@ import ( // Setup is used for setting up the backend, initialize ipvs, ipsets and iptales. func (b *backend) Setup() { var err error - ipsetList := make(map[string]*ipsets.Set) controller = newController() // setup ipvs manager @@ -43,34 +43,48 @@ func (b *backend) Setup() { klog.Fatal("unable to initialize ipvs manager", "error", err) } - // initialize ipsets + // initialize ip sets for _, is := range ipsetInfo { - set, err := controller.ipsetsManager.CreateSet(is.name, is.setType, is.comment) - ipsetList[set.GetName()] = set + // IPv4 sets + _, err = controller.ipsetsManager.CreateSet(is.name[v1.IPv4Protocol], is.setType, ipsets.ProtocolFamilyIPv4, is.comment) + if err != nil { + klog.Fatal("unable to create ipset", "set", is.name, "error", err) + } + + // IPv6 sets + _, err = controller.ipsetsManager.CreateSet(is.name[v1.IPv6Protocol], is.setType, ipsets.ProtocolFamilyIPv6, is.comment) if err != nil { klog.Fatal("unable to create ipset", "set", is.name, "error", err) } } - // add custom chains to NAT table + // add custom chains to NAT IPv4 and IPv6 table for _, chain := range []iptables.Chain{kubeServicesChain, KubeFireWallChain, kubePostroutingChain, KubeMarkMasqChain, KubeNodePortChain, KubeMarkDropChain, KubeForwardChain, KubeLoadBalancerChain} { - controller.iptManager.AddChain(chain, iptables.TableNat) + controller.iptManager.AddChain(chain, iptables.TableNat, iptables.ProtocolFamilyIPv4) + controller.iptManager.AddChain(chain, iptables.TableNat, iptables.ProtocolFamilyIPv6) } - // add custom chains to FILTER table + // add custom chains to FILTER IPv4 and IPv6 table for _, chain := range []iptables.Chain{KubeForwardChain, KubeNodePortChain} { - controller.iptManager.AddChain(chain, iptables.TableFilter) + controller.iptManager.AddChain(chain, iptables.TableFilter, iptables.ProtocolFamilyIPv4) + controller.iptManager.AddChain(chain, iptables.TableFilter, iptables.ProtocolFamilyIPv6) } - // add rules for NAT table - for _, rule := range GetNatRules(true) { - controller.iptManager.AddRule(rule, iptables.TableNat) + // add rules for NAT IPv4 and IPv6 table + for _, rule := range GetNatRules(true, v1.IPv4Protocol) { + controller.iptManager.AddRule(rule, iptables.TableNat, iptables.ProtocolFamilyIPv4) + } + for _, rule := range GetNatRules(true, v1.IPv6Protocol) { + controller.iptManager.AddRule(rule, iptables.TableNat, iptables.ProtocolFamilyIPv6) } - // add rules for FILTER table - for _, rule := range GetFilterRules(true) { - controller.iptManager.AddRule(rule, iptables.TableFilter) + // add rules for FILTER IPv4 and IPv6 table + for _, rule := range GetFilterRules(true, v1.IPv4Protocol) { + controller.iptManager.AddRule(rule, iptables.TableFilter, iptables.ProtocolFamilyIPv4) + } + for _, rule := range GetFilterRules(true, v1.IPv6Protocol) { + controller.iptManager.AddRule(rule, iptables.TableFilter, iptables.ProtocolFamilyIPv6) } // Apply will write the rules to IPTables.