diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index 3c62aa845b..e905da35f8 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -56,7 +56,10 @@ const ( svcSchedFlagsAnnotation = "kube-router.io/service.schedflags" LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader" - svcIpSetName = "KUBE-SVC-ALL" + localIPsIPSetName = "kube-router-local-ips" + ipvsServicesIPSetName = "kube-router-ipvs-services" + serviceIPsIPSetName = "kube-router-service-ips" + ipvsFirewallChainName = "KUBE-ROUTER-SERVICES" ) var ( @@ -214,6 +217,9 @@ type NetworkServicesController struct { ln LinuxNetworking readyForUpdates bool + // Map of ipsets that we use. + ipsetMap map[string]*utils.Set + svcLister cache.Indexer epLister cache.Indexer podLister cache.Indexer @@ -318,6 +324,12 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control return errors.New(sysctlErr.Error()) } + // https://github.com/cloudnativelabs/kube-router/issues/282 + err = nsc.setupIpvsFirewall() + if err != nil { + return errors.New("Error setting up ipvs firewall: " + err.Error()) + } + // loop forever unitl notified to stop on stopCh for { select { @@ -375,52 +387,110 @@ func (nsc *NetworkServicesController) sync() error { return nil } +func getIpvsFirewallInputChainRule() []string { + // The iptables rule for use in {setup,cleanup}IpvsFirewall. + return []string{ + "-m", "comment", "--comment", "handle traffic to IPVS service IPs in custom chain", + "-m", "set", "--match-set", serviceIPsIPSetName, "dst", + "-j", ipvsFirewallChainName} +} + func (nsc *NetworkServicesController) setupIpvsFirewall() error { - // Add ipset containg all SVCs + /* + - create ipsets + - create firewall rules + */ + + var err error + var ipset *utils.Set + ipSetHandler, err := utils.NewIPSet(false) if err != nil { return err } - svcIpSet, err := ipSetHandler.Create(svcIpSetName, utils.TypeHashIPPort, utils.OptionTimeout, "0") + // Remember ipsets for use in syncIpvsFirewall + nsc.ipsetMap = make(map[string]*utils.Set) + + // Create ipset for local addresses. + ipset, err = ipSetHandler.Create(localIPsIPSetName, utils.TypeHashIP, utils.OptionTimeout, "0") if err != nil { return fmt.Errorf("failed to create ipset: %s", err.Error()) } + nsc.ipsetMap[localIPsIPSetName] = ipset - ipvsSvcs, err := nsc.ln.ipvsGetServices() + // Create 2 ipsets for services. One for 'ip' and one for 'ip,port' + ipset, err = ipSetHandler.Create(serviceIPsIPSetName, utils.TypeHashIP, utils.OptionTimeout, "0") if err != nil { - return errors.New("Failed to list IPVS services: " + err.Error()) + return fmt.Errorf("failed to create ipset: %s", err.Error()) } + nsc.ipsetMap[serviceIPsIPSetName] = ipset - svcSets := make([]string, 0, len(ipvsSvcs)) - for _, ipvsSvc := range ipvsSvcs { - protocol := "udp" - if ipvsSvc.Protocol == syscall.IPPROTO_TCP { - protocol = "tcp" + ipset, err = ipSetHandler.Create(ipvsServicesIPSetName, utils.TypeHashIPPort, utils.OptionTimeout, "0") + if err != nil { + return fmt.Errorf("failed to create ipset: %s", err.Error()) + } + nsc.ipsetMap[ipvsServicesIPSetName] = ipset + + // Setup a custom iptables chain to explicitly allow input traffic to + // ipvs services only. + iptablesCmdHandler, err := iptables.New() + if err != nil { + return errors.New("Failed to initialize iptables executor" + err.Error()) + } + + // ClearChain either clears an existing chain or creates a new one. + err = iptablesCmdHandler.ClearChain("filter", ipvsFirewallChainName) + if err != nil { + return fmt.Errorf("Failed to run iptables command: %s", err.Error()) + } + + var comment string + var args []string + + comment = "allow input traffic to ipvs services" + args = []string{"-m", "comment", "--comment", comment, + "-m", "set", "--match-set", ipvsServicesIPSetName, "dst,dst", + "-j", "ACCEPT"} + exists, err := iptablesCmdHandler.Exists("filter", ipvsFirewallChainName, args...) + if err != nil { + return fmt.Errorf("Failed to run iptables command: %s", err.Error()) + } + if !exists { + err := iptablesCmdHandler.Insert("filter", ipvsFirewallChainName, 1, args...) + if err != nil { + return fmt.Errorf("Failed to run iptables command: %s", err.Error()) } - set := fmt.Sprintf("%s,%s:%d", ipvsSvc.Address.String(), protocol, ipvsSvc.Port) - svcSets = append(svcSets, set) } - err = svcIpSet.Refresh(svcSets, utils.OptionTimeout, "0") + comment = "allow icmp echo requests to service IPs" + args = []string{"-m", "comment", "--comment", comment, + "-p", "icmp", "--icmp-type", "echo-request", + "-j", "ACCEPT"} + err = iptablesCmdHandler.AppendUnique("filter", ipvsFirewallChainName, args...) if err != nil { - return fmt.Errorf("failed to sync ipset: %s", err.Error()) + return fmt.Errorf("Failed to run iptables command: %s", err.Error()) } - // Add iptables rule to allow input traffic to ipvs services - iptablesCmdHandler, err := iptables.New() + // We exclude the local addresses here as that would otherwise block all + // traffic to local addresses if any NodePort service exists. + comment = "reject all unexpected traffic to service IPs" + args = []string{"-m", "comment", "--comment", comment, + "-m", "set", "!", "--match-set", localIPsIPSetName, "dst", + "-j", "REJECT", "--reject-with", "icmp-port-unreachable"} + err = iptablesCmdHandler.AppendUnique("filter", ipvsFirewallChainName, args...) if err != nil { - return errors.New("Failed to initialize iptables executor" + err.Error()) + return fmt.Errorf("Failed to run iptables command: %s", err.Error()) } - comment := "allow input traffic to ipvs services" - args := []string{"-m", "comment", "--comment", comment, "-m", "set", "--match-set", svcIpSetName, "dst,dst", "-j", "ACCEPT"} - exists, err := iptablesCmdHandler.Exists("filter", "INPUT", args...) + // Pass incomming traffic into our custom chain. + ipvsFirewallInputChainRule := getIpvsFirewallInputChainRule() + exists, err = iptablesCmdHandler.Exists("filter", "INPUT", ipvsFirewallInputChainRule...) if err != nil { return fmt.Errorf("Failed to run iptables command: %s", err.Error()) } if !exists { - err := iptablesCmdHandler.Insert("filter", "INPUT", 1, args...) + err = iptablesCmdHandler.Insert("filter", "INPUT", 1, ipvsFirewallInputChainRule...) if err != nil { return fmt.Errorf("Failed to run iptables command: %s", err.Error()) } @@ -429,6 +499,114 @@ func (nsc *NetworkServicesController) setupIpvsFirewall() error { return nil } +func (nsc *NetworkServicesController) cleanupIpvsFirewall() { + /* + - delete firewall rules + - delete ipsets + */ + var err error + + // Clear iptables rules. + iptablesCmdHandler, err := iptables.New() + if err != nil { + glog.Errorf("Failed to initialize iptables executor: %s", err.Error()) + } else { + ipvsFirewallInputChainRule := getIpvsFirewallInputChainRule() + err = iptablesCmdHandler.Delete("filter", "INPUT", ipvsFirewallInputChainRule...) + if err != nil { + glog.Errorf("Failed to run iptables command: %s", err.Error()) + } + + err = iptablesCmdHandler.ClearChain("filter", ipvsFirewallChainName) + if err != nil { + glog.Errorf("Failed to run iptables command: %s", err.Error()) + } + + err = iptablesCmdHandler.DeleteChain("filter", ipvsFirewallChainName) + if err != nil { + glog.Errorf("Failed to run iptables command: %s", err.Error()) + } + } + + // Clear ipsets. + ipSetHandler, err := utils.NewIPSet(false) + if err != nil { + glog.Errorf("Failed to initialize ipset handler: %s", err.Error()) + } else { + err = ipSetHandler.Destroy(localIPsIPSetName) + if err != nil { + glog.Errorf("failed to destroy ipset: %s", err.Error()) + } + + err = ipSetHandler.Destroy(serviceIPsIPSetName) + if err != nil { + glog.Errorf("failed to destroy ipset: %s", err.Error()) + } + + err = ipSetHandler.Destroy(ipvsServicesIPSetName) + if err != nil { + glog.Errorf("failed to destroy ipset: %s", err.Error()) + } + } +} + +func (nsc *NetworkServicesController) syncIpvsFirewall() error { + /* + - update ipsets based on currently active IPVS services + */ + var err error + + localIPsIPSet := nsc.ipsetMap[localIPsIPSetName] + + // Populate local addresses ipset. + addrs, err := getAllLocalIPs() + localIPsSets := make([]string, 0, len(addrs)) + for _, addr := range addrs { + localIPsSets = append(localIPsSets, addr.IP.String()) + } + err = localIPsIPSet.Refresh(localIPsSets, utils.OptionTimeout, "0") + if err != nil { + return fmt.Errorf("failed to sync ipset: %s", err.Error()) + } + + // Populate service ipsets. + ipvsServices, err := nsc.ln.ipvsGetServices() + if err != nil { + return errors.New("Failed to list IPVS services: " + err.Error()) + } + + serviceIPsSets := make([]string, 0, len(ipvsServices)) + ipvsServicesSets := make([]string, 0, len(ipvsServices)) + + for _, ipvsService := range ipvsServices { + protocol := "udp" + if ipvsService.Protocol == syscall.IPPROTO_TCP { + protocol = "tcp" + } + + serviceIPsSet := ipvsService.Address.String() + serviceIPsSets = append(serviceIPsSets, serviceIPsSet) + + ipvsServicesSet := fmt.Sprintf("%s,%s:%d", ipvsService.Address.String(), protocol, ipvsService.Port) + ipvsServicesSets = append(ipvsServicesSets, ipvsServicesSet) + + } + + serviceIPsIPSet := nsc.ipsetMap[serviceIPsIPSetName] + err = serviceIPsIPSet.Refresh(serviceIPsSets, utils.OptionTimeout, "0") + if err != nil { + return fmt.Errorf("failed to sync ipset: %s", err.Error()) + } + + ipvsServicesIPSet := nsc.ipsetMap[ipvsServicesIPSetName] + err = ipvsServicesIPSet.Refresh(ipvsServicesSets, utils.OptionTimeout, "0") + if err != nil { + return fmt.Errorf("failed to sync ipset: %s", err.Error()) + } + + return nil +} + func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoMap) error { start := time.Now() defer func() { @@ -952,7 +1130,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf } } - err = nsc.setupIpvsFirewall() + err = nsc.syncIpvsFirewall() if err != nil { glog.Errorf("Error syncing ipvs svc iptable rules: %s", err.Error()) } @@ -2061,6 +2239,8 @@ func (nsc *NetworkServicesController) Cleanup() { return } + nsc.cleanupIpvsFirewall() + // delete dummy interface used to assign cluster IP's dummyVipInterface, err := netlink.LinkByName(KUBE_DUMMY_IF) if err != nil {