From 866db1549cc401d1ac765a802b5c5194405d8c93 Mon Sep 17 00:00:00 2001 From: MaiWJ <664647065@qq.com> Date: Sun, 2 Sep 2018 23:02:20 +0800 Subject: [PATCH] Add windows host-gw - patch for https://github.com/coreos/flannel/pull/921 --- backend/hostgw/hostgw.go | 1 - backend/hostgw/hostgw_windows.go | 227 +++++++++++++- backend/route_network_windows.go | 199 ++++++++++++ network/netroute/netroute_windows.go | 177 +++++++++++ network/netsh/netsh_windows.go | 440 +++++++++++++++++++++++++++ 5 files changed, 1039 insertions(+), 5 deletions(-) create mode 100644 backend/route_network_windows.go create mode 100644 network/netroute/netroute_windows.go create mode 100644 network/netsh/netsh_windows.go diff --git a/backend/hostgw/hostgw.go b/backend/hostgw/hostgw.go index ebcbfc8eb4..4257ed405d 100644 --- a/backend/hostgw/hostgw.go +++ b/backend/hostgw/hostgw.go @@ -13,7 +13,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// +build !windows package hostgw diff --git a/backend/hostgw/hostgw_windows.go b/backend/hostgw/hostgw_windows.go index f14d12b448..79c1b16504 100644 --- a/backend/hostgw/hostgw_windows.go +++ b/backend/hostgw/hostgw_windows.go @@ -1,5 +1,3 @@ -// +build windows - // Copyright 2015 flannel authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,14 +11,235 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// +build windows package hostgw import ( + "fmt" + "strconv" + "sync" + "time" + + "github.com/Microsoft/hcsshim" + "github.com/coreos/flannel/backend" + "github.com/coreos/flannel/network/netroute" + "github.com/coreos/flannel/network/netsh" + "github.com/coreos/flannel/pkg/ip" + "github.com/coreos/flannel/subnet" log "github.com/golang/glog" + "github.com/juju/errors" + "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/util/json" + utilexec "k8s.io/utils/exec" + "k8s.io/apimachinery/pkg/util/wait" ) func init() { - log.Infof("hostgw is not supported on this platform") + backend.Register("host-gw", New) +} + +type HostgwBackend struct { + sm subnet.Manager + extIface *backend.ExternalInterface +} + +func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) { + if !extIface.ExtAddr.Equal(extIface.IfaceAddr) { + return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend") + } + + be := &HostgwBackend{ + sm: sm, + extIface: extIface, + } + + return be, nil +} + +func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, config *subnet.Config) (backend.Network, error) { + // 1. Parse configuration + cfg := struct { + Name string + DNSServerList string + }{} + if len(config.Backend) > 0 { + if err := json.Unmarshal(config.Backend, &cfg); err != nil { + return nil, errors.Annotate(err, "error decoding windows host-gw backend config") + } + } + if len(cfg.Name) == 0 { + cfg.Name = "cbr0" + } + log.Infof("HOST-GW config: Name=%s DNSServerList=%s", cfg.Name, cfg.DNSServerList) + + n := &backend.RouteNetwork{ + SimpleNetwork: backend.SimpleNetwork{ + ExtIface: be.extIface, + }, + SM: be.sm, + BackendType: "host-gw", + Mtu: be.extIface.Iface.MTU, + LinkIndex: be.extIface.Iface.Index, + } + n.GetRoute = func(lease *subnet.Lease) *netroute.Route { + return &netroute.Route{ + Dst: lease.Subnet.ToIPNet(), + Gw: lease.Attrs.PublicIP.ToIP(), + LinkIndex: n.LinkIndex, + } + } + + // 2. Acquire the lease form subnet manager + attrs := subnet.LeaseAttrs{ + PublicIP: ip.FromIP(be.extIface.ExtAddr), + BackendType: "host-gw", + } + + l, err := be.sm.AcquireLease(ctx, &attrs) + switch err { + case nil: + n.SubnetLease = l + + case context.Canceled, context.DeadlineExceeded: + return nil, err + + default: + return nil, errors.Annotate(err, "failed to acquire lease") + } + + // 3. Check if the network exists and has the expected settings + netshHelper := netsh.NewHelper(utilexec.New()) + createNewNetwork := true + expectedSubnet := n.SubnetLease.Subnet + expectedAddressPrefix := expectedSubnet.String() + expectedGatewayAddress := (expectedSubnet.IP + 1).String() + expectedPodGatewayAddress := expectedSubnet.IP + 2 + networkName := cfg.Name + + existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err == nil { + for _, subnet := range existingNetwork.Subnets { + if subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGatewayAddress { + createNewNetwork = false + log.Infof("Found existing HNSNetwork %s", networkName) + break + } + } + } + + // 4. Create a new HNSNetwork + expectedNetwork := existingNetwork + if createNewNetwork { + if existingNetwork != nil { + if _, err := existingNetwork.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete existing HNSNetwork %s", networkName) + } + log.Infof("Deleted stale HNSNetwork %s", networkName) + } + + expectedNetwork = &hcsshim.HNSNetwork{ + Name: networkName, + Type: "L2Bridge", + DNSServerList: cfg.DNSServerList, + Subnets: []hcsshim.Subnet{ + { + AddressPrefix: expectedAddressPrefix, + GatewayAddress: expectedGatewayAddress, + }, + }, + } + jsonRequest, err := json.Marshal(expectedNetwork) + if err != nil { + return nil, errors.Annotatef(err, "failed to marshal %+v", expectedNetwork) + } + + log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest)) + newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest)) + if err != nil { + return nil, errors.Annotatef(err, "failed to create HNSNetwork %s", networkName) + } + + var waitErr, lastErr error + // Wait for the network to populate Management IP + waitErr = wait.Poll(1*time.Second, 10*time.Second, func() (done bool, err error) { + newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetwork.Id, "") + return len(newNetwork.ManagementIP) == 0, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HNSNetwork by ID: %s", newNetwork.Id) + } + + // Wait for the interface with the management IP + waitErr = wait.Poll(1*time.Second, 10*time.Second, func() (done bool, err error) { + _, lastErr = netshHelper.GetInterfaceByIP(newNetwork.ManagementIP) + return lastErr == nil, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface by IP: %s", newNetwork.ManagementIP) + } + + log.Infof("Created HNSNetwork %s", networkName) + expectedNetwork = newNetwork + } + + // 5. Ensure a 1.2 endpoint on this network in the host compartment + createNewBridgeEndpoint := true + bridgeEndpointName := networkName + "_ep" + existingBridgeEndpoint, err := hcsshim.GetHNSEndpointByName(bridgeEndpointName) + if err == nil && existingBridgeEndpoint.IPAddress.String() == expectedPodGatewayAddress.String() { + log.Infof("Found existing bridge HNSEndpoint %s", bridgeEndpointName) + createNewBridgeEndpoint = false + } + + // 6. Create a bridge HNSEndpoint + expectedBridgeEndpoint := existingBridgeEndpoint + if createNewBridgeEndpoint { + if existingBridgeEndpoint != nil { + if _, err = existingBridgeEndpoint.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete existing bridge HNSEndpoint %s", bridgeEndpointName) + } + log.Infof("Deleted stale bridge HNSEndpoint %s", bridgeEndpointName) + } + + expectedBridgeEndpoint = &hcsshim.HNSEndpoint{ + Name: bridgeEndpointName, + IPAddress: expectedPodGatewayAddress.ToIP(), + VirtualNetwork: expectedNetwork.Id, + } + + log.Infof("Attempting to create bridge HNSEndpoint %+v", expectedBridgeEndpoint) + if existingBridgeEndpoint, err = expectedBridgeEndpoint.Create(); err != nil { + return nil, errors.Annotatef(err, "failed to create bridge HNSEndpoint %s", bridgeEndpointName) + } + + log.Infof("Created bridge HNSEndpoint %s", bridgeEndpointName) + } + + if err = expectedBridgeEndpoint.HostAttach(1); err != nil { + return nil, errors.Annotatef(err, "failed to hot attach bridge HNSEndpoint %s to host compartment", bridgeEndpointName) + } + log.Infof("Attached bridge endpoint %s to host", bridgeEndpointName) + + // 7. Enable forwarding on the host interface and endpoint + for _, interfaceIpAddress := range []string{expectedNetwork.ManagementIP, existingBridgeEndpoint.IPAddress.String()} { + netInterface, err := netshHelper.GetInterfaceByIP(interfaceIpAddress) + if err != nil { + return nil, errors.Annotatef(err, "failed to find interface for IP Address %s", interfaceIpAddress) + } + log.Infof("Found %v NetshInterface with IP %s", netInterface, interfaceIpAddress) + + // When a new hns network is created, the interface is modified, esp the name, index + if expectedNetwork.ManagementIP == netInterface.IpAddress { + n.LinkIndex = netInterface.Idx + n.Name = netInterface.Name + } + + interfaceIdx := strconv.Itoa(netInterface.Idx) + if err := netshHelper.EnableForwarding(interfaceIdx); err != nil { + return nil, errors.Annotatef(err, "failed to enable forwarding on %s index %d", netInterface.Name, interfaceIdx) + } + log.Infof("Enabled forwarding on %s index %d", netInterface.Name, interfaceIdx) + } + + return n, nil } diff --git a/backend/route_network_windows.go b/backend/route_network_windows.go new file mode 100644 index 0000000000..ef79ec1ac6 --- /dev/null +++ b/backend/route_network_windows.go @@ -0,0 +1,199 @@ +// Copyright 2018 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backend + +import ( + "sync" + "time" + + "github.com/coreos/flannel/network/netroute" + log "github.com/golang/glog" + "golang.org/x/net/context" + + "github.com/coreos/flannel/subnet" + "strings" +) + +const ( + routeCheckRetries = 10 +) + +type RouteNetwork struct { + SimpleNetwork + Name string + BackendType string + SM subnet.Manager + GetRoute func(lease *subnet.Lease) *netroute.Route + Mtu int + LinkIndex int + routes []netroute.Route +} + +func (n *RouteNetwork) MTU() int { + return n.Mtu +} + +func (n *RouteNetwork) Run(ctx context.Context) { + wg := sync.WaitGroup{} + + log.Info("Watching for new subnet leases") + evts := make(chan []subnet.Event) + wg.Add(1) + go func() { + subnet.WatchLeases(ctx, n.SM, n.SubnetLease, evts) + wg.Done() + }() + + n.routes = make([]netroute.Route, 0, 10) + wg.Add(1) + go func() { + n.routeCheck(ctx) + wg.Done() + }() + + defer wg.Wait() + + for { + select { + case evtBatch := <-evts: + n.handleSubnetEvents(evtBatch) + + case <-ctx.Done(): + return + } + } +} + +func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) { + netrouteHelper := netroute.NewHelper() + + for _, evt := range batch { + leaseSubnet := evt.Lease.Subnet + leaseAttrs := evt.Lease.Attrs + if !strings.EqualFold(leaseAttrs.BackendType, n.BackendType) { + log.Warningf("Ignoring non-%v subnet(%v): type=%v", n.BackendType, leaseSubnet, leaseAttrs.BackendType) + continue + } + + expectedRoute := n.GetRoute(&evt.Lease) + + switch evt.Type { + case subnet.EventAdded: + log.Infof("Subnet added: %v via %v", leaseSubnet, leaseAttrs.PublicIP) + + existingRoutes, _ := netrouteHelper.GetNetRoutes(expectedRoute.LinkIndex, expectedRoute.Dst) + if len(existingRoutes) > 0 { + existingRoute := existingRoutes[0] + if existingRoute.Equal(expectedRoute) { + continue + } + + log.Warningf("Replacing existing route %v via %v with %v via %v", leaseSubnet, existingRoute.Gw, leaseSubnet, leaseAttrs.PublicIP) + err := netrouteHelper.RemoveNetRoute(&existingRoute) + if err != nil { + log.Errorf("Error removing route: %v", err) + continue + } + } + + err := netrouteHelper.CreateNetRoute(expectedRoute) + if err != nil { + log.Errorf("Error creating route: %v", err) + continue + } + + n.addToRouteList(expectedRoute) + + case subnet.EventRemoved: + log.Infof("Subnet removed: %v", leaseSubnet) + + existingRoutes, _ := netrouteHelper.GetNetRoutes(expectedRoute.LinkIndex, expectedRoute.Dst) + if len(existingRoutes) > 0 { + existingRoute := existingRoutes[0] + if existingRoute.Equal(expectedRoute) { + log.Infof("Removing existing route %v via %v", leaseSubnet, existingRoute.Gw) + + err := netrouteHelper.RemoveNetRoute(&existingRoute) + if err != nil { + log.Warningf("Error removing route: %v", err) + } + } + } + + n.removeFromRouteList(expectedRoute) + + default: + log.Error("Internal error: unknown event type: ", int(evt.Type)) + } + } +} + +func (n *RouteNetwork) addToRouteList(newRoute *netroute.Route) { + for _, route := range n.routes { + if route.Equal(newRoute) { + return + } + } + + n.routes = append(n.routes, *newRoute) +} + +func (n *RouteNetwork) removeFromRouteList(oldRoute *netroute.Route) { + for index, route := range n.routes { + if route.Equal(oldRoute) { + n.routes = append(n.routes[:index], n.routes[index+1:]...) + return + } + } +} + +func (n *RouteNetwork) routeCheck(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(routeCheckRetries * time.Second): + n.checkSubnetExistInRoutes() + } + } +} + +func (n *RouteNetwork) checkSubnetExistInRoutes() { + netrouteHelper := netroute.NewHelper() + + currentRoutes, err := netrouteHelper.GetNetRoutesAll() + if err != nil { + log.Errorf("Error enumerating routes: %v", err) + return + } + for _, route := range n.routes { + exist := false + for _, currentRoute := range currentRoutes { + if route.Equal(¤tRoute) { + exist = true + break + } + } + + if !exist { + err := netrouteHelper.CreateNetRoute(&route) + if err != nil { + log.Warningf("Error recovering route to %v via %v on %v (%v).", route.Dst, route.Gw, route.LinkIndex, err) + continue + } + log.Infof("Recovered route to %v via %v on %v.", route.Dst, route.Gw, route.LinkIndex) + } + } +} diff --git a/network/netroute/netroute_windows.go b/network/netroute/netroute_windows.go new file mode 100644 index 0000000000..ffc116317d --- /dev/null +++ b/network/netroute/netroute_windows.go @@ -0,0 +1,177 @@ +// Copyright 2018 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netroute + +import ( + "bufio" + "bytes" + "fmt" + "math/big" + "net" + "os/exec" + "regexp" + "strconv" + "strings" +) + +// Interface is an injectable interface for running MSFT_NetRoute commands. Implementations must be goroutine-safe. +type Interface interface { + // GetNetRoutesAll returns all net routes on the host + GetNetRoutesAll() ([]Route, error) + // GetNetRoutes returns all net routes by link and destination subnet + GetNetRoutes(linkIndex int, dst *net.IPNet) ([]Route, error) + // Create a new route + CreateNetRoute(route *Route) error + // Remove an existing route + RemoveNetRoute(route *Route) error +} + +type Route struct { + LinkIndex int + Dst *net.IPNet + Gw net.IP + RouteMetric int + IfMetric int +} + +type shell struct { + runnerPath string +} + +func (p *shell) execute(args ...string) (stdOut string, stdErr string, err error) { + args = append([]string{"-NoProfile", "-NonInteractive"}, args...) + cmd := exec.Command(p.runnerPath, args...) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + stdOut, stdErr = stdout.String(), stderr.String() + return +} + +func (p *shell) runScript(cmdLine string) (string, error) { + stdout, _, err := p.execute(cmdLine) + if err != nil { + return "", err + } + + return stdout, nil +} + +func NewHelper() Interface { + psPath, _ := exec.LookPath("powershell.exe") + + runner := &shell{ + runnerPath: psPath, + } + + return runner +} + +func (p *shell) GetNetRoutesAll() ([]Route, error) { + getRouteCmdLine := "Get-NetRoute -ErrorAction Ignore" + + stdout, err := p.runScript(getRouteCmdLine) + if err != nil { + return nil, err + } + + return parseRoutesList(stdout), nil +} +func (p *shell) GetNetRoutes(linkIndex int, dst *net.IPNet) ([]Route, error) { + getRouteCmdLine := fmt.Sprintf("Get-NetRoute -InterfaceIndex %v -DestinationPrefix %v -ErrorAction Ignore", linkIndex, dst.String()) + + stdout, err := p.runScript(getRouteCmdLine) + if err != nil { + return nil, err + } + + return parseRoutesList(stdout), nil +} + +func (p *shell) CreateNetRoute(route *Route) error { + newRouteCmdLine := fmt.Sprintf("New-NetRoute -InterfaceIndex %v -DestinationPrefix %v -NextHop %v -Verbose", route.LinkIndex, route.Dst.String(), route.Gw.String()) + + _, err := p.runScript(newRouteCmdLine) + + return err +} + +func (p *shell) RemoveNetRoute(route *Route) error { + removeRouteCmdLine := fmt.Sprintf("Remove-NetRoute -InterfaceIndex %v -DestinationPrefix %v -NextHop %v -Verbose -Confirm:$false", route.LinkIndex, route.Dst.String(), route.Gw.String()) + + _, err := p.runScript(removeRouteCmdLine) + + return err +} + +func parseRoutesList(stdout string) []Route { + internalWhitespaceRegEx := regexp.MustCompile(`[\s\p{Zs}]{2,}`) + scanner := bufio.NewScanner(strings.NewReader(stdout)) + var routes []Route + for scanner.Scan() { + line := internalWhitespaceRegEx.ReplaceAllString(scanner.Text(), "|") + if strings.HasPrefix(line, "ifIndex") || strings.HasPrefix(line, "----") { + continue + } + + parts := strings.Split(line, "|") + if len(parts) != 5 { + continue + } + + linkIndex, err := strconv.Atoi(parts[0]) + if err != nil { + continue + } + + gatewayAddress := net.ParseIP(parts[2]) + if gatewayAddress == nil { + continue + } + + _, destinationSubnet, err := net.ParseCIDR(parts[1]) + if err != nil { + continue + } + route := Route{ + Dst: destinationSubnet, + Gw: gatewayAddress, + LinkIndex: linkIndex, + } + + routes = append(routes, route) + } + + return routes +} + +func (r *Route) Equal(route *Route) bool { + return r.Dst.IP.Equal(route.Dst.IP) && r.Gw.Equal(route.Gw) && bytes.Equal(r.Dst.Mask, route.Dst.Mask) && r.LinkIndex == route.LinkIndex +} + +func IPToInt(ip net.IP) *big.Int { + if v := ip.To4(); v != nil { + return big.NewInt(0).SetBytes(v) + } + return big.NewInt(0).SetBytes(ip.To16()) +} + +func IntToIP(i *big.Int) net.IP { + return net.IP(i.Bytes()) +} diff --git a/network/netsh/netsh_windows.go b/network/netsh/netsh_windows.go new file mode 100644 index 0000000000..097b12ce01 --- /dev/null +++ b/network/netsh/netsh_windows.go @@ -0,0 +1,440 @@ +// Copyright 2018 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netsh + +import ( + "fmt" + "net" + "os" + "regexp" + "strconv" + "strings" + "sync" + "time" + + log "github.com/golang/glog" + "github.com/juju/errors" + utilexec "k8s.io/utils/exec" +) + +// NetshInterface is an injectable interface for running netsh commands. Implementations must be goroutine-safe. +type NetshInterface interface { + // EnsurePortProxyRule checks if the specified redirect exists, if not creates it + EnsurePortProxyRule(args []string) (bool, error) + // DeletePortProxyRule deletes the specified portproxy rule. If the rule did not exist, return error. + DeletePortProxyRule(args []string) error + // EnsureIPAddress checks if the specified IP Address is added to vEthernet (HNSTransparent) interface, if not, add it. If the address existed, return true. + EnsureIPAddress(args []string, ip net.IP) (bool, error) + // DeleteIPAddress checks if the specified IP address is present and, if so, deletes it. + DeleteIPAddress(args []string) error + // Restore runs `netsh exec` to restore portproxy or addresses using a file. + // TODO Check if this is required, most likely not + Restore(args []string) error + // GetInterfaceToAddIP returns the interface name where Service IP needs to be added + // IP Address needs to be added for netsh portproxy to redirect traffic + // Reads Environment variable INTERFACE_TO_ADD_SERVICE_IP, if it is not defined then "vEthernet (HNSTransparent)" is returned + GetInterfaceToAddIP() string + // GetDefaultGatewayIfaceName returns the interface name that has the default gateway + GetDefaultGatewayIfaceName() (string, error) + // GetInterfaces returns a list of interfaces and addresses + GetInterfaces() ([]Ipv4Interface, error) + // GetInterfaceByName returns an interface by name + GetInterfaceByName(name string) (Ipv4Interface, error) + // GetInterfaceByIP returns an interface by ip address in the format a.b.c.d + GetInterfaceByIP(ipAddr string) (Ipv4Interface, error) + // Enable forwarding on the interface (name or index) + EnableForwarding(iface string) error +} + +const ( + cmdNetsh string = "netsh" +) + +// runner implements NetshInterface in terms of exec("netsh"). +type runner struct { + mu sync.Mutex + exec utilexec.Interface +} + +// Ipv4Interface models IPv4 interface output from: netsh interface ipv4 show addresses +type Ipv4Interface struct { + Idx int + Name string + InterfaceMetric int + DhcpEnabled bool + IpAddress string + SubnetPrefix int + GatewayMetric int + DefaultGatewayAddress string +} + +// NewHelper returns a new NetshInterface which will exec netsh. +func NewHelper(exec utilexec.Interface) NetshInterface { + runner := &runner{ + exec: exec, + } + return runner +} + +// EnsurePortProxyRule checks if the specified redirect exists, if not creates it. +func (runner *runner) EnsurePortProxyRule(args []string) (bool, error) { + log.V(4).Infof("running netsh interface portproxy add v4tov4 %v", args) + out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() + + if err == nil { + return true, nil + } + if ee, ok := err.(utilexec.ExitError); ok { + // netsh uses exit(0) to indicate a success of the operation, + // as compared to a malformed commandline, for example. + if ee.Exited() && ee.ExitStatus() != 0 { + return false, nil + } + } + return false, errors.Annotatef(err, "error checking portproxy rule: %s", out) + +} + +// DeletePortProxyRule deletes the specified portproxy rule. If the rule did not exist, return error. +func (runner *runner) DeletePortProxyRule(args []string) error { + log.V(4).Infof("running netsh interface portproxy delete v4tov4 %v", args) + out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() + + if err == nil { + return nil + } + if ee, ok := err.(utilexec.ExitError); ok { + // netsh uses exit(0) to indicate a success of the operation, + // as compared to a malformed commandline, for example. + if ee.Exited() && ee.ExitStatus() == 0 { + return nil + } + } + return errors.Annotatef(err, "error deleting portproxy rule: %s", out) +} + +// EnsureIPAddress checks if the specified IP Address is added to interface identified by Environment variable INTERFACE_TO_ADD_SERVICE_IP, if not, add it. If the address existed, return true. +func (runner *runner) EnsureIPAddress(args []string, ip net.IP) (bool, error) { + // Check if the ip address exists + intName := runner.GetInterfaceToAddIP() + argsShowAddress := []string{ + "interface", "ipv4", "show", "address", + "name=" + intName, + } + + ipToCheck := ip.String() + + exists, _ := checkIPExists(ipToCheck, argsShowAddress, runner) + if exists == true { + log.V(4).Infof("not adding IP address %q as it already exists", ipToCheck) + return true, nil + } + + // IP Address is not already added, add it now + log.V(4).Infof("running netsh interface ipv4 add address %v", args) + out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() + + if err == nil { + // Once the IP Address is added, it takes a bit to initialize and show up when querying for it + // Query all the IP addresses and see if the one we added is present + // PS: We are using netsh interface ipv4 show address here to query all the IP addresses, instead of + // querying net.InterfaceAddrs() as it returns the IP address as soon as it is added even though it is uninitialized + log.V(3).Infof("Waiting until IP: %v is added to the network adapter", ipToCheck) + for { + if exists, _ := checkIPExists(ipToCheck, argsShowAddress, runner); exists { + return true, nil + } + time.Sleep(500 * time.Millisecond) + } + } + if ee, ok := err.(utilexec.ExitError); ok { + // netsh uses exit(0) to indicate a success of the operation, + // as compared to a malformed commandline, for example. + if ee.Exited() && ee.ExitStatus() != 0 { + return false, nil + } + } + return false, errors.Annotatef(err, "error adding ipv4 address: %s", out) +} + +// DeleteIPAddress checks if the specified IP address is present and, if so, deletes it. +func (runner *runner) DeleteIPAddress(args []string) error { + log.V(4).Infof("running netsh interface ipv4 delete address %v", args) + out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() + + if err == nil { + return nil + } + if ee, ok := err.(utilexec.ExitError); ok { + // netsh uses exit(0) to indicate a success of the operation, + // as compared to a malformed commandline, for example. + if ee.Exited() && ee.ExitStatus() == 0 { + return nil + } + } + return errors.Annotatef(err, "error deleting ipv4 address: %s", out) +} + +// GetInterfaceToAddIP returns the interface name where Service IP needs to be added +// IP Address needs to be added for netsh portproxy to redirect traffic +// Reads Environment variable INTERFACE_TO_ADD_SERVICE_IP, if it is not defined then "vEthernet (HNS Internal NIC)" is returned +func (runner *runner) GetInterfaceToAddIP() string { + if iface := os.Getenv("INTERFACE_TO_ADD_SERVICE_IP"); len(iface) > 0 { + return iface + } + return "vEthernet (HNS Internal NIC)" +} + +// Restore is part of NetshInterface. +func (runner *runner) Restore(args []string) error { + return nil +} + +// checkIPExists checks if an IP address exists in 'netsh interface ipv4 show address' output +func checkIPExists(ipToCheck string, args []string, runner *runner) (bool, error) { + ipAddress, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() + if err != nil { + return false, err + } + ipAddressString := string(ipAddress[:]) + + log.V(3).Infof("Searching for IP: %s in IP dump: %s", ipToCheck, ipAddressString) + showAddressArray := strings.Split(ipAddressString, "\n") + for _, showAddress := range showAddressArray { + if strings.Contains(showAddress, "IP") { + ipFromNetsh := getIP(showAddress) + if ipFromNetsh == ipToCheck { + return true, nil + } + } + } + + return false, nil +} + +// getIP gets ip from showAddress (e.g. "IP Address: 10.96.0.4"). +func getIP(showAddress string) string { + list := strings.SplitN(showAddress, ":", 2) + if len(list) != 2 { + return "" + } + return strings.TrimSpace(list[1]) +} + +func (runner *runner) GetInterfaces() ([]Ipv4Interface, error) { + interfaces, interfaceError := runner.getIpAddressConfigurations() + + if interfaceError != nil { + return nil, interfaceError + } + + indexMap, indexError := runner.getNetworkInterfaceParameters() + + if indexError != nil { + return nil, indexError + } + + // zip them up + for i := 0; i < len(interfaces); i++ { + name := interfaces[i].Name + + if val, ok := indexMap[name]; ok { + interfaces[i].Idx = val + } else { + return nil, errors.New("no index found for interface " + name) + } + } + + return interfaces, nil +} + +// GetInterfaces uses the show addresses command and returns a formatted structure +func (runner *runner) getIpAddressConfigurations() ([]Ipv4Interface, error) { + args := []string{ + "interface", "ipv4", "show", "addresses", + } + + output, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() + if err != nil { + return nil, err + } + interfacesString := string(output[:]) + + outputLines := strings.Split(interfacesString, "\n") + var interfaces []Ipv4Interface + var currentInterface Ipv4Interface + quotedPattern := regexp.MustCompile("\\\"(.*?)\\\"") + cidrPattern := regexp.MustCompile("\\/(.*?)\\ ") + + if err != nil { + return nil, err + } + + for _, outputLine := range outputLines { + if strings.Contains(outputLine, "Configuration for interface") { + if currentInterface != (Ipv4Interface{}) { + interfaces = append(interfaces, currentInterface) + } + match := quotedPattern.FindStringSubmatch(outputLine) + currentInterface = Ipv4Interface{ + Name: match[1], + } + } else { + parts := strings.SplitN(outputLine, ":", 2) + if len(parts) != 2 { + continue + } + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + if strings.HasPrefix(key, "DHCP enabled") { + if value == "Yes" { + currentInterface.DhcpEnabled = true + } + } else if strings.HasPrefix(key, "InterfaceMetric") { + if val, err := strconv.Atoi(value); err == nil { + currentInterface.InterfaceMetric = val + } + } else if strings.HasPrefix(key, "Gateway Metric") { + if val, err := strconv.Atoi(value); err == nil { + currentInterface.GatewayMetric = val + } + } else if strings.HasPrefix(key, "Subnet Prefix") && currentInterface.SubnetPrefix == 0 { + match := cidrPattern.FindStringSubmatch(value) + if val, err := strconv.Atoi(match[1]); err == nil { + currentInterface.SubnetPrefix = val + } + } else if strings.HasPrefix(key, "IP Address") && currentInterface.IpAddress == "" { + currentInterface.IpAddress = value + } else if strings.HasPrefix(key, "Default Gateway") && currentInterface.DefaultGatewayAddress == "" { + currentInterface.DefaultGatewayAddress = value + } + } + } + + // add the last one + if currentInterface != (Ipv4Interface{}) { + interfaces = append(interfaces, currentInterface) + } + + if len(interfaces) == 0 { + return nil, errors.New("no interfaces found in netsh output: " + interfacesString) + } + + return interfaces, nil +} + +func (runner *runner) getNetworkInterfaceParameters() (map[string]int, error) { + args := []string{ + "interface", "ipv4", "show", "interfaces", + } + + output, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() + + if err != nil { + return nil, err + } + + // Split output by line + outputString := string(output[:]) + outputString = strings.TrimSpace(outputString) + var outputLines = strings.Split(outputString, "\n") + + if len(outputLines) < 3 { + return nil, errors.New("unexpected netsh output:\n" + outputString) + } + + // Remove first two lines of header text + outputLines = outputLines[2:] + + indexMap := make(map[string]int) + + reg := regexp.MustCompile("\\s{2,}") + + for _, line := range outputLines { + + line = strings.TrimSpace(line) + + // Split the line by two or more whitespace characters, returning all substrings (n < 0) + splitLine := reg.Split(line, -1) + + name := splitLine[4] + if idx, err := strconv.Atoi(splitLine[0]); err == nil { + indexMap[name] = idx + } + + } + + return indexMap, nil +} + +func (runner *runner) GetDefaultGatewayIfaceName() (string, error) { + interfaces, err := runner.GetInterfaces() + if err != nil { + return "", err + } + + for _, iface := range interfaces { + if iface.DefaultGatewayAddress != "" { + return iface.Name, nil + } + } + + // return "not found" + return "", errors.New("default interface not found") +} + +func (runner *runner) GetInterfaceByName(name string) (Ipv4Interface, error) { + interfaces, err := runner.GetInterfaces() + if err != nil { + return Ipv4Interface{}, err + } + + for _, iface := range interfaces { + if iface.Name == name { + return iface, nil + } + } + + // return "not found" + return Ipv4Interface{}, errors.New("interface not found " + name) +} + +func (runner *runner) GetInterfaceByIP(ipAddr string) (Ipv4Interface, error) { + interfaces, err := runner.GetInterfaces() + if err != nil { + return Ipv4Interface{}, err + } + + for _, iface := range interfaces { + if iface.IpAddress == ipAddr { + return iface, nil + } + } + + // return "not found" + return Ipv4Interface{}, errors.New("interface not found " + ipAddr) +} + +// Enable forwarding on the interface (name or index) +func (runner *runner) EnableForwarding(iface string) error { + args := []string{ + "int", "ipv4", "set", "int", strconv.Quote(iface), "for=en", + } + if stdout, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput(); err != nil { + return fmt.Errorf("failed to enable forwarding on [%v], error: %v. cmd: %v. stdout: %v", iface, err.Error(), strings.Join(args, " "), string(stdout)) + } + + return nil +}