Fix in Telemetry plugin and improvements for integration tests (#1414)

* Add DumpInterface to vppcalls in ifplugin Signed-off-by: Ondrej Fabry <[email protected]> * Cleanup vpp integration tests Signed-off-by: Ondrej Fabry <[email protected]> * Fix file name Signed-off-by: Ondrej Fabry <[email protected]> * Hide iptables warning when no config found Signed-off-by: Ondrej Fabry <[email protected]> * Print shm warning directly into stderr Signed-off-by: Ondrej Fabry <[email protected]> * Fix single interface dump for older VPP Signed-off-by: Ondrej Fabry <[email protected]> * Fix retrieving memory stats in telemetry for VPP 19.08 Signed-off-by: Ondrej Fabry <[email protected]> * Add tests for telemetry to VPP integration tests Signed-off-by: Ondrej Fabry <[email protected]> * Fix integration tests - Skip some telemetry integration tests for VPP<=19.04 - Fix GetMemory in telemetry for VPP<=19.04 - Use hard-coded config for VPP - Use DOCKER_ARGS for custom arguments for docker Signed-off-by: Ondrej Fabry <[email protected]> * Update GoVPP Signed-off-by: Ondrej Fabry <[email protected]>
ligato · Jul 19, 2019 · ed775d6 · ed775d6
1 parent 9313c98
commit ed775d6
Show file tree

Hide file tree

Showing 27 changed files with 538 additions and 188 deletions.
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/plugins/govppmux/adapter_puregoclient.go b/plugins/govppmux/adapter_puregoclient.go
@@ -17,29 +17,33 @@
 package govppmux
 
 import (
+	"fmt"
+	"os"
+
 	"git.fd.io/govpp.git/adapter"
 	"git.fd.io/govpp.git/adapter/socketclient"
 	"git.fd.io/govpp.git/adapter/statsclient"
-	"github.com/ligato/cn-infra/logging"
 )
 
+const noShmWarning = `Using shared memory for VPP binary API is not currently supported in pure Go client!
+
+  To use socket client for VPP binary API (recommended):
+   - unset GOVPPMUX_NOSOCK environment variable
+   - remove these settings from govpp.conf config: shm-prefix, connect-via-shm
+
+  If you still want to use shared memory for VPP binary API:
+   - compile your agent with this build tag: vppapiclient
+   - vppapiclient requires CGo and needs VPP to be installed 
+`
+
 // NewVppAdapter returns VPP binary API adapter, implemented as pure Go client.
 func NewVppAdapter(addr string, useShm bool) adapter.VppAPI {
 	if useShm {
-		logging.Warnf(`Using shared memory for VPP binary API is not currently supported in pure Go client!
-
-	To use socket client for VPP binary API:
-	  - unset GOVPPMUX_NOSOCK environment variable
-	  - remove these settings from govpp.conf config: shm-prefix, connect-via-shm
-
-	If you still want to use shared memory for VPP binary API (not recommended):
-	  - compile your agent with this build tag: vppapiclient
-`)
+		fmt.Fprintf(os.Stderr, noShmWarning)
 		panic("No implementation for shared memory in pure Go client!")
 	}
 	// addr is used as socket path
 	return socketclient.NewVppClient(addr)
-
 }
 
 // NewStatsAdapter returns VPP stats API adapter, implemented as pure Go client.

diff --git a/plugins/govppmux/vppcalls/vpp1901/vpe_vppcalls.go b/plugins/govppmux/vppcalls/vpp1901/vpe_vppcalls.go
@@ -31,7 +31,7 @@ func init() {
 	msgs = append(msgs, vpe.AllMessages()...)
 	msgs = append(msgs, memclnt.AllMessages()...)
 
-	vppcalls.Versions["vpp1901"] = vppcalls.HandlerVersion{
+	vppcalls.Versions["19.01"] = vppcalls.HandlerVersion{
 		Msgs: msgs,
 		New: func(ch govppapi.Channel) vppcalls.VpeVppAPI {
 			return NewVpeHandler(ch)

diff --git a/plugins/govppmux/vppcalls/vpp1904/vpe_vppcalls.go b/plugins/govppmux/vppcalls/vpp1904/vpe_vppcalls.go
@@ -31,7 +31,7 @@ func init() {
 	msgs = append(msgs, vpe.AllMessages()...)
 	msgs = append(msgs, memclnt.AllMessages()...)
 
-	vppcalls.Versions["vpp1904"] = vppcalls.HandlerVersion{
+	vppcalls.Versions["19.04"] = vppcalls.HandlerVersion{
 		Msgs: msgs,
 		New: func(ch govppapi.Channel) vppcalls.VpeVppAPI {
 			return NewVpeHandler(ch)

diff --git a/plugins/govppmux/vppcalls/vpp1908/vpe_vppcalls.go b/plugins/govppmux/vppcalls/vpp1908/vpe_vppcalls.go
@@ -31,7 +31,7 @@ func init() {
 	msgs = append(msgs, vpe.AllMessages()...)
 	msgs = append(msgs, memclnt.AllMessages()...)
 
-	vppcalls.Versions["vpp1908"] = vppcalls.HandlerVersion{
+	vppcalls.Versions["19.08"] = vppcalls.HandlerVersion{
 		Msgs: msgs,
 		New: func(ch govppapi.Channel) vppcalls.VpeVppAPI {
 			return NewVpeHandler(ch)

diff --git a/plugins/govppmux/vppcalls/vppcalls_api.go b/plugins/govppmux/vppcalls/vppcalls_api.go
@@ -15,6 +15,14 @@ type VersionInfo struct {
 	BuildDirectory string
 }
 
+// Release returns version in shortened format YY.MM that describes release.
+func (v VersionInfo) Release() string {
+	if len(v.Version) < 5 {
+		return ""
+	}
+	return v.Version[:5]
+}
+
 // VpeInfo contains information about VPP connection and process.
 type VpeInfo struct {
 	PID            uint32

diff --git a/plugins/linux/iptablesplugin/iptablesplugin.go b/plugins/linux/iptablesplugin/iptablesplugin.go
@@ -36,7 +36,8 @@ type IPTablesPlugin struct {
 	Deps
 
 	// From configuration file
-	disabled bool
+	disabled    bool
+	configFound bool
 
 	// system handlers
 	iptHandler linuxcalls.IPTablesAPI
@@ -75,7 +76,7 @@ func (p *IPTablesPlugin) Init() error {
 	// init iptables handler
 	p.iptHandler = linuxcalls.NewIPTablesHandler()
 	err = p.iptHandler.Init()
-	if err != nil {
+	if err != nil && p.configFound {
 		// just warn here, iptables / ip6tables just may not be installed - will return
 		// an error by attempt to configure it
 		p.Log.Warnf("Error by initializing iptables handler: %v", err)
@@ -112,5 +113,6 @@ func (p *IPTablesPlugin) retrieveConfig() (*Config, error) {
 	if err != nil {
 		return nil, err
 	}
+	p.configFound = true
 	return config, err
 }
diff --git a/plugins/telemetry/vppcalls/vpp1901/telemetry_vppcalls.go b/plugins/telemetry/vppcalls/vpp1901/telemetry_vppcalls.go
@@ -63,14 +63,15 @@ var (
 	memoryRe = regexp.MustCompile(
 		`Thread\s+(\d+)\s+(\w+).?\s+` +
 			`virtual memory start 0x[0-9abcdef]+, size ([\dkmg\.]+), ([\dkmg\.]+) pages, page size ([\dkmg\.]+)\s+` +
-			`(?:\s+(?:numa [\d]+|not mapped|unknown): [\dkmg\.]+ pages, [\dkmg\.]+\s+)+\s+` +
+			`(?:page information not available.*\s+)*` +
+			`(?:(?:\s+(?:numa [\d]+|not mapped|unknown): [\dkmg\.]+ pages, [\dkmg\.]+\s+)+\s+)*` +
 			`\s+total: ([\dkmgKMG\.]+), used: ([\dkmgKMG\.]+), free: ([\dkmgKMG\.]+), trimmable: ([\dkmgKMG\.]+)`,
 	)
 )
 
 // GetMemory retrieves `show memory` info.
 func (h *TelemetryHandler) GetMemory(ctx context.Context) (*vppcalls.MemoryInfo, error) {
-	data, err := h.RunCli("show memory")
+	data, err := h.RunCli("show memory main-heap")
 	if err != nil {
 		return nil, err
 	}

diff --git a/plugins/telemetry/vppcalls/vpp1904/telemetry_vppcalls.go b/plugins/telemetry/vppcalls/vpp1904/telemetry_vppcalls.go
@@ -57,8 +57,9 @@ var (
 	// Regular expression to parse output from `show memory`
 	memoryRe = regexp.MustCompile(
 		`Thread\s+(\d+)\s+(\w+).?\s+` +
-			`virtual memory start 0x[0-9abcdef]+, size ([\dkmg\.]+), ([\dkmg\.]+) pages, page size ([\dkmg\.]+)\s+` +
-			`(?:\s+(?:numa [\d]+|not mapped|unknown): [\dkmg\.]+ pages, [\dkmg\.]+\s+)+\s+` +
+			`virtual memory start 0x[0-9a-f]+, size ([\dkmg\.]+), ([\dkmg\.]+) pages, page size ([\dkmg\.]+)\s+` +
+			`(?:page information not available.*\s+)*` +
+			`(?:(?:\s+(?:numa [\d]+|not mapped|unknown): [\dkmg\.]+ pages, [\dkmg\.]+\s+)+\s+)*` +
 			`\s+total: ([\dkmgKMG\.]+), used: ([\dkmgKMG\.]+), free: ([\dkmgKMG\.]+), trimmable: ([\dkmgKMG\.]+)`,
 	)
 )
@@ -69,7 +70,7 @@ func (h *TelemetryHandler) GetMemory(ctx context.Context) (*vppcalls.MemoryInfo,
 }
 
 func (h *TelemetryHandler) getMemoryCLI(ctx context.Context) (*vppcalls.MemoryInfo, error) {
-	data, err := h.vpe.RunCli("show memory")
+	data, err := h.vpe.RunCli("show memory main-heap")
 	if err != nil {
 		return nil, err
 	}

diff --git a/plugins/telemetry/vppcalls/vpp1908/telemetry_vppcalls.go b/plugins/telemetry/vppcalls/vpp1908/telemetry_vppcalls.go
@@ -57,8 +57,9 @@ var (
 	// Regular expression to parse output from `show memory`
 	memoryRe = regexp.MustCompile(
 		`Thread\s+(\d+)\s+(\w+).?\s+` +
-			`virtual memory start 0x[0-9abcdef]+, size ([\dkmg\.]+), ([\dkmg\.]+) pages, page size ([\dkmg\.]+)\s+` +
-			`(?:\s+(?:numa [\d]+|not mapped|unknown): [\dkmg\.]+ pages, [\dkmg\.]+\s+)+\s+` +
+			`virtual memory start 0x[0-9a-f]+, size ([\dkmg\.]+), ([\dkmg\.]+) pages, page size ([\dkmg\.]+)\s+` +
+			`(?:page information not available.*\s+)*` +
+			`(?:(?:\s+(?:numa [\d]+|not mapped|unknown): [\dkmg\.]+ pages, [\dkmg\.]+\s+)*\s+)*` +
 			`\s+total: ([\dkmgKMG\.]+), used: ([\dkmgKMG\.]+), free: ([\dkmgKMG\.]+), trimmable: ([\dkmgKMG\.]+)`,
 	)
 )
@@ -69,7 +70,7 @@ func (h *TelemetryHandler) GetMemory(ctx context.Context) (*vppcalls.MemoryInfo,
 }
 
 func (h *TelemetryHandler) getMemoryCLI(ctx context.Context) (*vppcalls.MemoryInfo, error) {
-	data, err := h.vpe.RunCli("show memory")
+	data, err := h.vpe.RunCli("show memory main-heap")
 	if err != nil {
 		return nil, err
 	}

diff --git a/plugins/telemetry/vppcalls/vpp1908/telemetry_vppcalls_test.go b/plugins/telemetry/vppcalls/vpp1908/telemetry_vppcalls_test.go
@@ -16,13 +16,14 @@ package vpp1908_test
 
 import (
 	"context"
-	"github.com/ligato/vpp-agent/plugins/telemetry/vppcalls/vpp1908"
 	"testing"
 
+	. "github.com/onsi/gomega"
+
 	"github.com/ligato/vpp-agent/plugins/telemetry/vppcalls"
+	"github.com/ligato/vpp-agent/plugins/telemetry/vppcalls/vpp1908"
 	"github.com/ligato/vpp-agent/plugins/vpp/binapi/vpp1908/vpe"
 	"github.com/ligato/vpp-agent/plugins/vpp/vppcallmock"
-	. "github.com/onsi/gomega"
 )
 
 func TestGetBuffers(t *testing.T) {
@@ -410,6 +411,27 @@ no traced allocations
 				Reclaimed: 996.12e6,
 			},
 		},
+		{
+			name: "19.08 update",
+			reply: `Thread 0 vpp_main
+  virtual memory start 0x7ff41b3ca000, size 1048640k, 262160 pages, page size 4k
+    page information not available (errno 1)
+  total: 1.00G, used: 19.81M, free: 1004.25M, trimmable: 1004.24M
+`,
+			threadCount: 1,
+			threadIdx:   0,
+			thread: vppcalls.MemoryThread{
+				ID:        0,
+				Name:      "vpp_main",
+				Size:      1048.64e6,
+				Pages:     262160,
+				PageSize:  4000,
+				Used:      19.81e6,
+				Total:     1e9,
+				Free:      1004.25e6,
+				Reclaimed: 1004.24e6,
+			},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

diff --git a/plugins/telemetry/vppcalls/vppcalls_api.go b/plugins/telemetry/vppcalls/vppcalls_api.go
@@ -19,7 +19,8 @@ import (
 
 	govppapi "git.fd.io/govpp.git/api"
 	log "github.com/ligato/cn-infra/logging"
-	"github.com/ligato/vpp-agent/plugins/govppmux"
+
+	"github.com/ligato/vpp-agent/plugins/govppmux/vppcalls"
 )
 
 var Versions = map[string]HandlerVersion{}
@@ -152,19 +153,20 @@ type BuffersItem struct {
 	NumFree  uint64 `json:"num_free"`
 }
 
-func CompatibleTelemetryHandler(ch govppapi.Channel, vpp govppmux.StatsAPI) TelemetryVppAPI {
-	status, err := vpp.VPPInfo()
+func CompatibleTelemetryHandler(ch govppapi.Channel, vpp govppapi.StatsProvider) TelemetryVppAPI {
+	vpe := vppcalls.CompatibleVpeHandler(ch)
+	info, err := vpe.GetVersionInfo()
 	if err != nil {
-		log.Warnf("retrieving VPP status failed: %v", err)
+		log.Warnf("retrieving VPP info failed: %v", err)
 		return nil
 	}
-	if status.Connected {
-		ver := status.GetReleaseVersion()
+	if ver := info.Release(); ver != "" {
+		log.Debug("telemetry checking release: ", ver)
 		if h, ok := Versions[ver]; ok {
 			if err := ch.CheckCompatiblity(h.Msgs...); err != nil {
-				log.Debugf("version %s not compatible", ver)
+				log.Debugf("telemetry version %s not compatible: %v", ver, err)
 			}
-			log.Debug("found compatible version: ", ver)
+			log.Debug("telemetry found compatible release: ", ver)
 			return h.New(ch, vpp)
 		}
 	}

diff --git a/plugins/vpp/ifplugin/vppcalls/if_vppcalls.go b/plugins/vpp/ifplugin/vppcalls/if_vppcalls.go
@@ -175,7 +175,7 @@ type InterfaceVppAPI interface {
 	AttachInterfaceToBond(ifIdx, bondIfIdx uint32, isPassive, isLongTimeout bool) error
 	// DetachInterfaceFromBond removes interface slave status from any bond interfaces.
 	DetachInterfaceFromBond(ifIdx uint32) error
-    // SetVLanTagRewrite sets VLan tag rewrite rule for given sub-interface
+	// SetVLanTagRewrite sets VLan tag rewrite rule for given sub-interface
 	SetVLanTagRewrite(ifIdx uint32, subIf *interfaces.SubInterface) error
 }
 
@@ -189,6 +189,8 @@ type InterfaceVppRead interface {
 	DumpInterfaces() (map[uint32]*InterfaceDetails, error)
 	// DumpInterfacesByType returns all VPP interfaces of the specified type
 	DumpInterfacesByType(reqType interfaces.Interface_Type) (map[uint32]*InterfaceDetails, error)
+	// DumpInterface dumps specific interface.
+	DumpInterface(ifIdx uint32) (*InterfaceDetails, error)
 	// GetInterfaceVrf reads VRF table to interface
 	GetInterfaceVrf(ifIdx uint32) (vrfID uint32, err error)
 	// GetInterfaceVrfIPv6 reads IPv6 VRF table to interface

diff --git a/plugins/vpp/ifplugin/vppcalls/vpp1901/dump_interface_vppcalls.go b/plugins/vpp/ifplugin/vppcalls/vpp1901/dump_interface_vppcalls.go
@@ -16,6 +16,7 @@ package vpp1901
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"net"
 	"strings"
@@ -47,6 +48,20 @@ func getMtu(vppMtu uint16) uint32 {
 	return uint32(vppMtu)
 }
 
+// DumpInterface dumps specific interface.
+func (h *InterfaceVppHandler) DumpInterface(ifIdx uint32) (*vppcalls.InterfaceDetails, error) {
+	ifaces, err := h.DumpInterfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	iface, ok := ifaces[ifIdx]
+	if !ok {
+		return nil, errors.New("interface index not found in dump")
+	}
+	return iface, nil
+}
+
 // DumpInterfacesByType implements interface handler.
 func (h *InterfaceVppHandler) DumpInterfacesByType(reqType interfaces.Interface_Type) (map[uint32]*vppcalls.InterfaceDetails, error) {
 	// Dump all

diff --git a/plugins/vpp/ifplugin/vppcalls/vpp1904/dump_interface_vppcalls.go b/plugins/vpp/ifplugin/vppcalls/vpp1904/dump_interface_vppcalls.go
@@ -17,6 +17,7 @@ package vpp1904
 import (
 	"bytes"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"net"
 	"strings"
@@ -46,6 +47,20 @@ func getMtu(vppMtu uint16) uint32 {
 	return uint32(vppMtu)
 }
 
+// DumpInterface dumps specific interface.
+func (h *InterfaceVppHandler) DumpInterface(ifIdx uint32) (*vppcalls.InterfaceDetails, error) {
+	ifaces, err := h.DumpInterfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	iface, ok := ifaces[ifIdx]
+	if !ok {
+		return nil, errors.New("interface index not found in dump")
+	}
+	return iface, nil
+}
+
 // DumpInterfacesByType implements interface handler.
 func (h *InterfaceVppHandler) DumpInterfacesByType(reqType interfaces.Interface_Type) (map[uint32]*vppcalls.InterfaceDetails, error) {
 	// Dump all