From 3eb41884f8d7eee77e9cbac5647918149896f18d Mon Sep 17 00:00:00 2001 From: Jordi Gil Date: Thu, 6 May 2021 03:14:19 -0400 Subject: [PATCH] MGMT-4718 Capture network latency for L3 (#187) --- src/commands/connectivity_check.go | 356 +++++----- src/commands/connectivity_check_test.go | 834 +++++++++++++++--------- 2 files changed, 735 insertions(+), 455 deletions(-) diff --git a/src/commands/connectivity_check.go b/src/commands/connectivity_check.go index 256502519..ec273a71e 100644 --- a/src/commands/connectivity_check.go +++ b/src/commands/connectivity_check.go @@ -5,18 +5,17 @@ import ( "encoding/xml" "os/exec" "regexp" + "strconv" "strings" + "sync" "github.com/openshift/assisted-installer-agent/src/util" "github.com/openshift/assisted-installer-agent/src/util/nmap" "github.com/openshift/assisted-service/models" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) -type Done struct{} - -type Any interface{} - func getOutgoingNics() []string { ret := make([]string, 0) d := util.NewDependencies() @@ -55,254 +54,285 @@ func getIPAddressFromCIDR(cidr string) string { return "" } -func sendDone(ch chan Any) { - ch <- Done{} +type any interface{} + +type done struct{} + +func sendDone(ch chan any) { + ch <- done{} } -func l3CheckAddressOnNic(address string, outgoingNic string, l3chan chan *models.L3Connectivity) { +const pingCount string = "10" + +func l3CheckAddressOnNic(address string, outgoingNic string, innerChan chan *models.L3Connectivity, conCheck connectivityCmd) { ret := &models.L3Connectivity{ OutgoingNic: outgoingNic, RemoteIPAddress: address, - Successful: false, } - cmd := exec.Command("ping", "-c", "2", "-W", "3", "-I", outgoingNic, address) - _, err := cmd.CombinedOutput() + b, err := conCheck.command("ping", []string{"-c", pingCount, "-W", "3", "-q", "-I", outgoingNic, address}) if err != nil { - log.Infof("Error running ping to %s on interface %s: %s", address, outgoingNic, err.Error()) - ret.Successful = false - } else { - ret.Successful = true + log.Errorf("Error running ping to %s on interface %s: %s", address, outgoingNic, err.Error()) + innerChan <- ret + return + } + err = parsePingCmd(ret, string(b)) + if err != nil { + log.Error(err) + innerChan <- ret + return + } + ret.Successful = true + innerChan <- ret +} + +func regexMatchFor(regex, line string) ([]string, error) { + r := regexp.MustCompile(regex) + p := r.FindStringSubmatch(line) + if len(p) < 2 { + return nil, errors.Errorf("unable to parse %s with regex %s", line, regex) + } + return p, nil +} + +func parsePingCmd(conn *models.L3Connectivity, cmdOutput string) error { + if len(cmdOutput) == 0 { + return errors.Errorf("Missing output for ping or invalid output:\n%s", cmdOutput) + } + parts, err := regexMatchFor(`[\d]+ packets transmitted, [\d]+ received, (([\d]*[.])?[\d]+)% packet loss, time [\d]+ms`, cmdOutput) + if err != nil { + return errors.Errorf("Unable to retrieve packet loss percentage: %s", err) + } + conn.PacketLossPercentage, err = strconv.ParseFloat(parts[1], 64) + if err != nil { + return errors.Errorf("Error while trying to convert value for packet loss '%s': %s", parts[1], err) + } + parts, err = regexMatchFor(`rtt min\/avg\/max\/mdev = .*\/([^\/]+)\/.*\/.* ms`, cmdOutput) + if err != nil { + return errors.Errorf("Unable to retrieve the average RTT for ping: %s", err) + } + conn.AverageRTTMs, err = strconv.ParseFloat(parts[1], 64) + if err != nil { + return errors.Errorf("Error while trying to convert value for packet loss %s: %s", parts[1], err) + } + return nil +} + +func l3CheckConnectivity(addresses []string, dataCh chan any, conCheck connectivityCmd) { + + defer sendDone(dataCh) + wg := sync.WaitGroup{} + wg.Add(len(addresses)) + for _, address := range addresses { + go l3CheckAddress(address, conCheck.getOutgoingNICs(), dataCh, &wg, conCheck) } - l3chan <- ret + wg.Wait() } -func l3CheckAddress(address string, outgoingNics []string, l3chan, doneChan chan Any) { - defer sendDone(doneChan) - innerChan := make(chan *models.L3Connectivity, 1000) +func l3CheckAddress(address string, outgoingNics []string, dataCh chan any, wg *sync.WaitGroup, conCheck connectivityCmd) { + defer wg.Done() + innerChan := make(chan *models.L3Connectivity) for _, nic := range outgoingNics { - go l3CheckAddressOnNic(address, nic, innerChan) + go l3CheckAddressOnNic(address, nic, innerChan, conCheck) } successful := false for i := 0; i != len(outgoingNics); i++ { ret := <-innerChan if ret.Successful { - l3chan <- ret + dataCh <- ret successful = true } } if !successful { ret := &models.L3Connectivity{ RemoteIPAddress: address, - Successful: false, } - l3chan <- ret + dataCh <- ret } } -func l3CheckConnectivity(addresses []string, outgoingNics []string, l3chan chan Any) { - defer sendDone(l3chan) - doneChan := make(chan Any) - for _, address := range addresses { - go l3CheckAddress(address, outgoingNics, l3chan, doneChan) - } - for i := 0; i != len(addresses); i++ { - <-doneChan - } -} - -func macInDstMacs(mac string, allDstMacs []string) bool { - for _, dstMac := range allDstMacs { - if strings.EqualFold(mac, dstMac) { +func macInDstMacs(mac string, allDstMACs []string) bool { + for _, dstMAC := range allDstMACs { + if strings.EqualFold(mac, dstMAC) { return true } } return false } -func l2CheckAddressOnNic(dstAddr string, dstMac string, allDstMacs []string, srcNic string, l2chan chan Any) { - defer sendDone(l2chan) +func l2CheckAddressOnNic(dstAddr string, dstMAC string, allDstMACs []string, srcNIC string, dataCh chan any, conCheck connectivityCmd) { + defer sendDone(dataCh) if util.IsIPv4Addr(dstAddr) { - runArping(dstAddr, dstMac, allDstMacs, srcNic, l2chan) + l2IPv4Cmd(dstAddr, dstMAC, allDstMACs, srcNIC, dataCh, conCheck) } else { - cmd := exec.Command("nmap", "-6", "-sn", "-n", "-oX", "-", "-e", srcNic, dstAddr) - analyzeNmap(dstAddr, dstMac, allDstMacs, srcNic, l2chan, cmd.Output) + analyzeNmap(dstAddr, dstMAC, allDstMACs, srcNIC, dataCh, conCheck) } -} -func runArping(dstAddr string, dstMac string, allDstMacs []string, srcNic string, l2chan chan Any) { - - ret := &models.L2Connectivity{ - OutgoingNic: srcNic, - RemoteIPAddress: dstAddr, - RemoteMac: "", - Successful: false, - } - cmd := exec.Command("arping", "-c", "1", "-w", "2", "-I", srcNic, dstAddr) - bytes, _ := cmd.CombinedOutput() - lines := strings.Split(string(bytes), "\n") - if len(lines) == 0 { - log.Warnf("Missing output for arping") - l2chan <- ret - return - } - - hRgegex := regexp.MustCompile("^ARPING ([^ ]+) from ([^ ]+) ([^ ]+)$") - parts := hRgegex.FindStringSubmatch(lines[0]) - if len(parts) != 4 { - log.Warnf("Wrong format for header line: %s", lines[0]) - l2chan <- ret - return - } - - ret.OutgoingIPAddress = parts[2] - rRegexp := regexp.MustCompile(`^Unicast reply from ([^ ]+) \[([^]]+)\] [^ ]+$`) - for _, line := range lines[1:] { - parts = rRegexp.FindStringSubmatch(line) - if len(parts) != 3 { - continue - } - remoteMac := strings.ToLower(parts[2]) - ret.RemoteMac = remoteMac - ret.Successful = macInDstMacs(remoteMac, allDstMacs) - if !ret.Successful { - log.Warnf("Unexpected mac address for arping %s on nic %s: %s", dstAddr, srcNic, remoteMac) - } else if strings.ToLower(dstMac) != remoteMac { - log.Infof("Received remote mac %s different then expected mac %s", remoteMac, dstMac) - } - l2chan <- ret - } } -func analyzeNmap(dstAddr string, dstMac string, allDstMacs []string, srcNic string, l2chan chan Any, output func() ([]byte, error)) { +func analyzeNmap(dstAddr string, dstMAC string, allDstMACs []string, srcNIC string, dataCh chan any, conCheck connectivityCmd) { ret := &models.L2Connectivity{ - OutgoingNic: srcNic, - OutgoingIPAddress: "", - RemoteIPAddress: dstAddr, - RemoteMac: "", - Successful: false, + OutgoingNic: srcNIC, + RemoteIPAddress: dstAddr, } - out, err := output() + out, err := conCheck.command("nmap", []string{"-6", "-sn", "-n", "-oX", "-", "-e", srcNIC, dstAddr}) if err != nil { log.WithError(err).Warn("nmap command failed") - l2chan <- ret + dataCh <- ret return } var nmaprun nmap.Nmaprun if err := xml.Unmarshal(out, &nmaprun); err != nil { log.WithError(err).Warn("Failed to un-marshal nmap XML") - l2chan <- ret + dataCh <- ret return } for _, h := range nmaprun.Hosts { - if h.Status.State != "up" { continue } - for _, a := range h.Addresses { - if a.AddrType != "mac" { continue } - - remoteMac := strings.ToLower(a.Addr) - ret.RemoteMac = remoteMac - ret.Successful = macInDstMacs(remoteMac, allDstMacs) + remoteMAC := strings.ToLower(a.Addr) + ret.RemoteMac = remoteMAC + ret.Successful = macInDstMacs(remoteMAC, allDstMACs) if !ret.Successful { - log.Warnf("Unexpected MAC address for nmap %s on NIC %s: %s", dstAddr, srcNic, remoteMac) - } else if strings.ToLower(dstMac) != remoteMac { - log.Infof("Received remote MAC %s different then expected MAC %s", remoteMac, dstMac) + log.Warnf("Unexpected MAC address for nmap %s on NIC %s: %s", dstAddr, srcNIC, remoteMAC) + } else if strings.ToLower(dstMAC) != remoteMAC { + log.Infof("Received remote MAC %s different then expected MAC %s", remoteMAC, dstMAC) } - l2chan <- ret + dataCh <- ret return } } - - l2chan <- ret + dataCh <- ret } -func l2CheckAddress(dstAddr string, dstMac string, allDstMacs, sourceNics []string, l2chan chan Any, l2DoneChan chan Any) { - defer sendDone(l2DoneChan) - innerChan := make(chan Any, 1000) - for _, srcNic := range sourceNics { - go l2CheckAddressOnNic(dstAddr, dstMac, allDstMacs, srcNic, innerChan) +func l2CheckAddress(dstAddr string, dstMAC string, allDstMACs []string, dataCh chan any, wg *sync.WaitGroup, conCheck connectivityCmd) { + defer wg.Done() + innerChan := make(chan any) + for _, srcNIC := range conCheck.getOutgoingNICs() { + go l2CheckAddressOnNic(dstAddr, dstMAC, allDstMACs, srcNIC, innerChan, conCheck) } received := false - for numDone := 0; numDone != len(sourceNics); { + for numDone := 0; numDone != len(conCheck.getOutgoingNICs()); { iret := <-innerChan switch ret := iret.(type) { case *models.L2Connectivity: received = true - l2chan <- ret - case Done: + dataCh <- ret + case done: numDone++ } } if !received { - ret := &models.L2Connectivity{ - OutgoingNic: "", - OutgoingIPAddress: "", - RemoteIPAddress: dstAddr, - RemoteMac: "", - Successful: false, + dataCh <- &models.L2Connectivity{ + RemoteIPAddress: dstAddr, } - l2chan <- ret } } -func l2CheckConnectivity(destinationNics []*models.ConnectivityCheckNic, sourceNics []string, l2chan chan Any) { - defer sendDone(l2chan) - doneChan := make(chan Any) - allDstMacs := make([]string, 0) - for _, destNic := range destinationNics { - allDstMacs = append(allDstMacs, destNic.Mac) +func l2CheckConnectivity(dataCh chan any, conCheck connectivityCmd) { + defer sendDone(dataCh) + allDstMACs := make([]string, len(conCheck.getHost().Nics)) + for i, destNic := range conCheck.getHost().Nics { + allDstMACs[i] = destNic.Mac } numAddresses := 0 - for _, destNic := range destinationNics { + for _, destNic := range conCheck.getHost().Nics { + numAddresses += len(destNic.IPAddresses) + } + wg := sync.WaitGroup{} + wg.Add(numAddresses) + for _, destNic := range conCheck.getHost().Nics { for _, address := range destNic.IPAddresses { - numAddresses++ - go l2CheckAddress(address, destNic.Mac, allDstMacs, sourceNics, l2chan, doneChan) + go l2CheckAddress(address, destNic.Mac, allDstMACs, dataCh, &wg, conCheck) } } - for i := 0; i != numAddresses; i++ { - <-doneChan + wg.Wait() +} + +func l2IPv4Cmd(dstAddr string, dstMAC string, allDstMACs []string, srcNIC string, dataCh chan any, conCheck connectivityCmd) { + ret := &models.L2Connectivity{ + OutgoingNic: srcNIC, + RemoteIPAddress: dstAddr, + } + bytes, err := conCheck.command("arping", []string{"-c", "1", "-w", "2", "-I", srcNIC, dstAddr}) + if err != nil { + log.Errorf("Error while processing 'arping' command: %s", err) + dataCh <- ret + return + } + lines := strings.Split(string(bytes), "\n") + if len(lines) == 0 { + log.Warnf("Missing output for arping") + dataCh <- ret + return + } + + hRgegex := regexp.MustCompile("^ARPING ([^ ]+) from ([^ ]+) ([^ ]+)$") + parts := hRgegex.FindStringSubmatch(lines[0]) + if len(parts) != 4 { + log.Warnf("Wrong format for header line: %s", lines[0]) + dataCh <- ret + return + } + + ret.OutgoingIPAddress = parts[2] + rRegexp := regexp.MustCompile(`^Unicast reply from ([^ ]+) \[([^]]+)\] [^ ]+$`) + for _, line := range lines[1:] { + parts = rRegexp.FindStringSubmatch(line) + if len(parts) != 3 { + continue + } + remoteMAC := strings.ToLower(parts[2]) + ret.RemoteMac = remoteMAC + ret.Successful = macInDstMacs(remoteMAC, allDstMACs) + if !ret.Successful { + log.Warnf("Unexpected mac address for arping %s on nic %s: %s", dstAddr, srcNIC, remoteMAC) + } + if strings.ToLower(dstMAC) != remoteMAC { + log.Infof("Received remote mac %s different then expected mac %s", remoteMAC, dstMAC) + } + dataCh <- ret } } -func checkHost(outgoingNics []string, host *models.ConnectivityCheckHost, hostChan chan *models.ConnectivityRemoteHost) { +func checkHost(conCheck connectivityCmd, outCh chan *models.ConnectivityRemoteHost) { ret := &models.ConnectivityRemoteHost{ - HostID: host.HostID, - L2Connectivity: make([]*models.L2Connectivity, 0), - L3Connectivity: make([]*models.L3Connectivity, 0), - } - addresses := getOutgoingAddresses(host.Nics) - ch := make(chan Any, 1000) - go l3CheckConnectivity(addresses, outgoingNics, ch) - go l2CheckConnectivity(host.Nics, outgoingNics, ch) + HostID: conCheck.getHost().HostID, + L2Connectivity: []*models.L2Connectivity{}, + L3Connectivity: []*models.L3Connectivity{}, + } + addresses := getOutgoingAddresses(conCheck.getHost().Nics) + dataCh := make(chan any) + go l3CheckConnectivity(addresses, dataCh, conCheck) + go l2CheckConnectivity(dataCh, conCheck) for numDone := 0; numDone != 2; { - iret := <-ch - switch value := iret.(type) { - case *models.L2Connectivity: - ret.L2Connectivity = append(ret.L2Connectivity, value) + v := <-dataCh + switch value := v.(type) { case *models.L3Connectivity: ret.L3Connectivity = append(ret.L3Connectivity, value) - case Done: + case *models.L2Connectivity: + ret.L2Connectivity = append(ret.L2Connectivity, value) + case done: numDone++ } + } - hostChan <- ret + outCh <- ret } - func ConnectivityCheck(_ string, args ...string) (stdout string, stderr string, exitCode int) { if len(args) != 1 { return "", "Expecting exactly 1 argument for connectivity command", -1 } - params := make(models.ConnectivityCheckParams, 0) + params := models.ConnectivityCheckParams{} err := json.Unmarshal([]byte(args[0]), ¶ms) if err != nil { log.Warnf("Error unmarshalling json %s: %s", args[0], err.Error()) @@ -311,9 +341,10 @@ func ConnectivityCheck(_ string, args ...string) (stdout string, stderr string, nics := getOutgoingNics() hostChan := make(chan *models.ConnectivityRemoteHost) for _, host := range params { - go checkHost(nics, host, hostChan) + h := hostChecker{outgoingNICS: nics, host: host} + go checkHost(h, hostChan) } - ret := models.ConnectivityReport{RemoteHosts: make([]*models.ConnectivityRemoteHost, 0)} + ret := models.ConnectivityReport{RemoteHosts: []*models.ConnectivityRemoteHost{}} for i := 0; i != len(params); i++ { ret.RemoteHosts = append(ret.RemoteHosts, <-hostChan) } @@ -324,3 +355,26 @@ func ConnectivityCheck(_ string, args ...string) (stdout string, stderr string, } return string(bytes), "", 0 } + +type hostChecker struct { + host *models.ConnectivityCheckHost + outgoingNICS []string +} + +type connectivityCmd interface { + command(name string, args []string) ([]byte, error) + getHost() *models.ConnectivityCheckHost + getOutgoingNICs() []string +} + +func (hc hostChecker) getHost() *models.ConnectivityCheckHost { + return hc.host +} + +func (hc hostChecker) getOutgoingNICs() []string { + return hc.outgoingNICS +} + +func (hc hostChecker) command(name string, args []string) ([]byte, error) { + return exec.Command(name, args...).CombinedOutput() +} diff --git a/src/commands/connectivity_check_test.go b/src/commands/connectivity_check_test.go index 9021b4047..20de991e1 100644 --- a/src/commands/connectivity_check_test.go +++ b/src/commands/connectivity_check_test.go @@ -1,11 +1,14 @@ package commands import ( + "errors" "fmt" "testing" - "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" + "github.com/openshift/assisted-service/models" ) @@ -18,326 +21,549 @@ const nmapOut = ` ` -var _ = ginkgo.Describe("nmap analysis test", func() { - - ginkgo.It("Happy flow", func(done ginkgo.Done) { - - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(nmapOut), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:12:00:02", - Successful: true, - } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("Command error", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(nmapOut), fmt.Errorf("nmap command failed") - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "", - Successful: false, - } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("Invalid XML", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte("plain text"), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "", - Successful: false, - } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("Host down", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(` - - - -
-
- - `), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "", - Successful: false, - } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("Lower-case destination MAC address", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(nmapOut), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:12:00:02", - Successful: true, - } - - analyzeNmap("2001:db8::2", "02:42:ac:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("Lower-case discovered MAC address", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(` - - - -
-
- - `), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:12:00:02", - Successful: true, - } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) +var _ = Describe("nmap analysis test", func() { + + tests := []struct { + name string + dstAddr string + dstMAC string + srcNIC string + allDstMACs []string + output func() ([]byte, error) + expected *models.L2Connectivity + }{ + {name: "Happy flow", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(nmapOut), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:12:00:02", + Successful: true, + }, + }, + {name: "Command error", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(nmapOut), fmt.Errorf("nmap command failed") + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + }}, + {name: "Invalid XML", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte("plain text"), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + }}, + + {name: "Host down", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(` + + + +
+
+ + `), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + Successful: false, + }, + }, + {name: "Lower-case destination MAC address", + dstAddr: "2001:db8::2", + dstMAC: "02:42:ac:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(nmapOut), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:12:00:02", + Successful: true, + }, + }, + {name: "Lower-case discovered MAC address", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(` + + + +
+
+ + `), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:12:00:02", + Successful: true, + }, + }, + {name: "No MAC address", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(` + + + +
+ + `), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + }, + }, + {name: "No hosts", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(""), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + }, + }, + {name: "First matching host", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(` + + + +
+
+ + + +
+
+ + `), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:aa:00:02", + Successful: false, + }, + }, + {name: "Multiple hosts, only one up", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(` + + + +
+
+ + + +
+
+ + `), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:12:00:02", + Successful: true, + }, + }, + {name: "Multiple hosts, only one has a MAC address", + dstAddr: "2001:db8::2", + dstMAC: "02:42:AC:12:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(` + + + +
+ + + +
+
+ + `), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:12:00:02", + Successful: true, + }, + }, + {name: "Unexpected MAC address", + dstAddr: "2001:db8::2", + dstMAC: "02:42:CC:14:00:02", + allDstMACs: []string{"02:42:B:14:00:02", "02:42:C:14:00:02"}, + output: func() ([]byte, error) { + return []byte(nmapOut), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:12:00:02", + }, + }, + {name: "MAC different than tried", + dstAddr: "2001:db8::2", + dstMAC: "02:42:CC:10:00:02", + allDstMACs: []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, + output: func() ([]byte, error) { + return []byte(nmapOut), nil + }, + expected: &models.L2Connectivity{ + OutgoingNic: "eth0", + RemoteIPAddress: "2001:db8::2", + RemoteMac: "02:42:ac:12:00:02", + Successful: true, + }, + }, + } + for i := range tests { + t := tests[i] + It(t.name, func() { + out := make(chan any) + go analyzeNmap(t.dstAddr, t.dstMAC, t.allDstMACs, "eth0", out, testNmap{t.output}) + Expect(<-out).To(Equal(t.expected)) + }) + } - ginkgo.It("No MAC address", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(` - - - -
- - `), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "", - Successful: false, - } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("No hosts", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(""), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "", - Successful: false, - } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) +}) - ginkgo.It("First matching host", func(done ginkgo.Done) { - out := make(chan Any, 100) +type testNmap struct { + output func() ([]byte, error) +} - xml := func() ([]byte, error) { - return []byte(` - - - -
-
- - - -
-
- - `), nil - } +func (tn testNmap) command(name string, args []string) ([]byte, error) { + return tn.output() +} - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:aa:00:02", - Successful: false, - } +func (tn testNmap) getHost() *models.ConnectivityCheckHost { + return nil +} +func (tn testNmap) getOutgoingNICs() []string { - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) + return nil +} - ginkgo.It("Multiple hosts, only one up", func(done ginkgo.Done) { - out := make(chan Any, 100) +var _ = Describe("parse ping command", func() { + + tests := []struct { + name string + cmdOutput string + averageRTTMs float64 + packetLoss float64 + errFunc func(string) string + }{ + {name: "Nominal: no packet loss", + cmdOutput: `PING www.acme.com (127.0.0.1) 56(84) bytes of data. + + --- www.acme.com ping statistics --- + 10 packets transmitted, 10 received, 0% packet loss, time 9011ms + rtt min/avg/max/mdev = 14.278/17.099/19.136/1.876 ms`, + averageRTTMs: 17.099, + packetLoss: 0, + }, + {name: "Nominal: with packet loss", + cmdOutput: `PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data. + + --- 192.168.1.1 ping statistics --- + 10 packets transmitted, 4 received, 60% packet loss, time 9164ms + rtt min/avg/max/mdev = 2.616/2.871/3.183/0.255 ms`, + averageRTTMs: 2.871, + packetLoss: 60, + }, + {name: "Nominal: with packet loss with decimals", + cmdOutput: `PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data. + + --- 192.168.1.1 ping statistics --- + 10 packets transmitted, 4 received, 23.33% packet loss, time 9164ms + rtt min/avg/max/mdev = 2.616/2.871/3.183/0.255 ms`, + averageRTTMs: 2.871, + packetLoss: 23.33, + }, + {name: "KO: unable to parse average RTT", + cmdOutput: `PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data. + + --- 192.168.1.1 ping statistics --- + 10 packets transmitted, 4 received, 60% packet loss, time 9164ms + rtt min/average/max/mdev = 2.616/2.871/3.183/0.255 ms`, + averageRTTMs: 0, + packetLoss: 60, + errFunc: func(s string) string { + return fmt.Sprintf(`Unable to retrieve the average RTT for ping: unable to parse %s with regex rtt min\/avg\/max\/mdev = .*\/([^\/]+)\/.*\/.* ms`, s) + }, + }, + {name: "KO: unable to parse packets loss percentage", + cmdOutput: `PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data. + + --- 192.168.1.1 ping statistics --- + 10 packets transmitted, 4 received, 60% packet loss, time 9164ms + rtt min/avg/max/mdev = 2.616/2.871/3.183/0.255 ms`, + errFunc: func(s string) string { + return fmt.Sprintf(`Unable to retrieve packet loss percentage: unable to parse %s with regex [\d]+ packets transmitted, [\d]+ received, (([\d]*[.])?[\d]+)%% packet loss, time [\d]+ms`, s) + }, + }, + } + for i := range tests { + t := tests[i] + It(t.name, func() { + conn := models.L3Connectivity{} + err := parsePingCmd(&conn, t.cmdOutput) + if t.errFunc != nil { + Expect(err.Error()).To(BeEquivalentTo(t.errFunc(t.cmdOutput))) + } else { + Expect(err).To(BeNil()) + } + Expect(conn.AverageRTTMs).Should(Equal(t.averageRTTMs)) + Expect(conn.PacketLossPercentage).Should(Equal(float64(t.packetLoss))) + }) + } - xml := func() ([]byte, error) { - return []byte(` - - - -
-
- - - -
-
- - `), nil - } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:12:00:02", - Successful: true, - } +}) - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) +var _ = Describe("check host parallel validation", func() { + + var ( + hostChan chan *models.ConnectivityRemoteHost + ) + BeforeEach(func() { + hostChan = make(chan *models.ConnectivityRemoteHost) + }) + + AfterEach(func() { + close(hostChan) + }) + + tests := []struct { + name string + nics []string + hosts *models.ConnectivityCheckHost + expected *models.ConnectivityRemoteHost + success bool + l2Conn []*models.L2Connectivity + l3Conn []*models.L3Connectivity + }{ + { + name: "Nominal: IPv4 with 2 addresses", + success: true, + nics: []string{"nic_ipv4"}, + hosts: &models.ConnectivityCheckHost{Nics: []*models.ConnectivityCheckNic{ + {IPAddresses: []string{"192.168.1.1"}, Mac: "74:d0:2b:1c:c6:42"}, + {IPAddresses: []string{"192.168.1.2"}, Mac: "f8:75:a4:4a:33:07"}, + }}, + expected: &models.ConnectivityRemoteHost{ + L2Connectivity: []*models.L2Connectivity{ + {OutgoingNic: "nic_ipv4", + RemoteIPAddress: "192.168.1.1", + OutgoingIPAddress: "192.168.1.133", + Successful: true, + RemoteMac: "74:d0:2b:1c:c6:42"}, + {OutgoingNic: "nic_ipv4", + RemoteIPAddress: "192.168.1.2", + OutgoingIPAddress: "192.168.1.133", + Successful: true, + RemoteMac: "f8:75:a4:4a:33:07"}, + }, + L3Connectivity: []*models.L3Connectivity{ + {AverageRTTMs: 2.871, + OutgoingNic: "nic_ipv4", + PacketLossPercentage: 60, + RemoteIPAddress: "192.168.1.1", + Successful: true, + }, + {AverageRTTMs: 2.871, + OutgoingNic: "nic_ipv4", + PacketLossPercentage: 60, + RemoteIPAddress: "192.168.1.2", + Successful: true, + }, + }, + }, + }, + {name: "Nominal: IPv6", + success: true, + hosts: &models.ConnectivityCheckHost{Nics: []*models.ConnectivityCheckNic{ + {IPAddresses: []string{"fe80::acae:f113:f40:cfe1"}, Mac: "4c:1d:96:af:22:65"}, + }}, + nics: []string{"nic_ipv6"}, + expected: &models.ConnectivityRemoteHost{ + L2Connectivity: []*models.L2Connectivity{ + {OutgoingNic: "nic_ipv6", + RemoteIPAddress: "fe80::acae:f113:f40:cfe1", + RemoteMac: "4c:1d:96:af:22:65", + Successful: true}, + }, + L3Connectivity: []*models.L3Connectivity{ + {AverageRTTMs: 2.871, + OutgoingNic: "nic_ipv6", + PacketLossPercentage: 60, + RemoteIPAddress: "fe80::acae:f113:f40:cfe1", + Successful: true, + }, + }, + }, + }, + {name: "KO: IPv4 unable to connect via ping or arp", + success: false, + nics: []string{"nic_ipv4", "nic_ipv41", "nic_ipv42"}, + hosts: &models.ConnectivityCheckHost{Nics: []*models.ConnectivityCheckNic{ + {IPAddresses: []string{"192.168.1.1"}, Mac: "4c:1d:96:af:22:65"}, + }}, + expected: &models.ConnectivityRemoteHost{ + L2Connectivity: []*models.L2Connectivity{ + {OutgoingNic: "", + RemoteIPAddress: "192.168.1.1", + }, + }, + L3Connectivity: []*models.L3Connectivity{ + {OutgoingNic: "", + RemoteIPAddress: "192.168.1.1", + }, + }, + }, + }, + {name: "KO: IPv6 unable to connect via ping or nmap", + success: false, + nics: []string{"nic_ipv6"}, + hosts: &models.ConnectivityCheckHost{Nics: []*models.ConnectivityCheckNic{ + {IPAddresses: []string{"fe80::acae:f113:f40:cfe1"}, Mac: "4c:1d:96:af:22:65"}, + }}, + expected: &models.ConnectivityRemoteHost{ + L2Connectivity: []*models.L2Connectivity{ + {OutgoingNic: "nic_ipv6", + RemoteIPAddress: "fe80::acae:f113:f40:cfe1", + }}, + L3Connectivity: []*models.L3Connectivity{ + {OutgoingNic: "", + RemoteIPAddress: "fe80::acae:f113:f40:cfe1", + }, + }, + }, + }, + } + + for i := range tests { + t := tests[i] + It(t.name, func() { + h := testHostChecker{outgoingNICS: t.nics, host: t.hosts, success: t.success} + go checkHost(h, hostChan) + r := <-hostChan + Expect(r.L2Connectivity).Should(ContainElements(t.expected.L2Connectivity)) + Expect(r.L3Connectivity).Should(ContainElements(t.expected.L3Connectivity)) + }) + } - ginkgo.It("Multiple hosts, only one has a MAC address", func(done ginkgo.Done) { - out := make(chan Any, 100) +}) - xml := func() ([]byte, error) { - return []byte(` - - - -
- - - -
-
- - `), nil - } +type testHostChecker struct { + success bool + outgoingNICS []string + host *models.ConnectivityCheckHost +} - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:12:00:02", - Successful: true, +func (t testHostChecker) command(name string, args []string) ([]byte, error) { + var mac string + if t.success { + for _, h := range t.host.Nics { + for _, ip := range h.IPAddresses { + if ip == args[len(args)-1] { + mac = h.Mac + break + } + } + if len(mac) > 0 { + break + } } - - analyzeNmap("2001:db8::2", "02:42:AC:12:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("Unexpected MAC address", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(nmapOut), nil + } + switch name { + case "ping": + if t.success { + return []byte(fmt.Sprintf(`PING %[1]s (%[1]s) 56(84) bytes of data. + + --- %[1]s ping statistics --- + 10 packets transmitted, 4 received, 60%% packet loss, time 9164ms + rtt min/avg/max/mdev = 2.616/2.871/3.183/0.255 ms`, args[0])), nil } - - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:12:00:02", - Successful: false, + return nil, errors.New("unable to connect") + case "nmap": + if t.success { + return []byte(fmt.Sprintf(` + + +
+
+ + `, args[7], mac)), nil } - - analyzeNmap("2001:db8::2", "02:42:CC:14:00:02", []string{"02:42:B:14:00:02", "02:42:C:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) - - ginkgo.It("MAC different than tried", func(done ginkgo.Done) { - out := make(chan Any, 100) - - xml := func() ([]byte, error) { - return []byte(nmapOut), nil + return nil, errors.New("unable to connect via nmap") + case "arping": + if t.success { + return []byte(fmt.Sprintf(`ARPING %[1]s from 192.168.1.133 %[2]s +Unicast reply from %[1]s [%[3]s] 3.137ms +Sent 1 probes (1 broadcast(s)) +Received 1 response(s) + `, args[5], args[6], mac)), nil } + return []byte(fmt.Sprintf(`ARPING %[1]s from 192.168.1.133 %[2]s +Sent 1 probes (1 broadcast(s)) +Received 0 response(s)`, args[5], args[6])), nil + default: + log.Errorf("failed to process unknown command %s with arguments %+v", name, args) + return nil, fmt.Errorf("unknown command %s", name) + } +} - expected := &models.L2Connectivity{ - OutgoingNic: "eth0", - OutgoingIPAddress: "", - RemoteIPAddress: "2001:db8::2", - RemoteMac: "02:42:ac:12:00:02", - Successful: true, - } +func (t testHostChecker) getHost() *models.ConnectivityCheckHost { + return t.host +} - analyzeNmap("2001:db8::2", "02:42:CC:10:00:02", []string{"02:42:AC:12:00:02", "02:42:AC:14:00:02"}, "eth0", out, xml) - Expect(<-out).To(Equal(expected)) - close(done) - }, 0.2) -}) +func (t testHostChecker) getOutgoingNICs() []string { + return t.outgoingNICS +} func TestConnectivityCheck(t *testing.T) { - RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "connectivity check tests") + RegisterFailHandler(Fail) + RunSpecs(t, "connectivity check tests") }