Performance Metricset on X-Pack MSSQL Metricbeat module (#9826)

Performance Metricset, as it name implies, adds key performance information about a MS SQL instance (no database specific performance info in this Metricset). User connections, page splits, compilations or batch requests are added.

CHANGELOG.next.asciidoc

@@ -111,6 +111,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `key` metricset to the Redis module. {issue}9582[9582] {pull}9657[9657] {pull}9746[9746]
 - Add `socket_summary` metricset to system defaults, removing experimental tag and supporting Windows {pull}9709[9709]
 - Add docker `event` metricset. {pull}9856[9856]
+- Add 'performance' metricset to x-pack mssql module {pull}9826[9826]
 
 *Packetbeat*
 

x-pack/metricbeat/metricbeat.reference.yml

@@ -491,6 +491,7 @@ metricbeat.modules:
 - module: mssql
   metricsets:
     - "db"
+    - "performance"
   hosts: ["sqlserver://sa@localhost"]
   period: 10s
 

x-pack/metricbeat/module/mssql/_meta/config.yml

@@ -1,6 +1,7 @@
 - module: mssql
   metricsets:
     - "db"
+    - "performance"
   hosts: ["sqlserver://sa@localhost"]
   period: 10s
 

x-pack/metricbeat/module/mssql/connection.go

@@ -0,0 +1,27 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package mssql
+
+import (
+	"database/sql"
+
+	"github.com/pkg/errors"
+)
+
+// NewConnection returns a connection already established with MSSQL
+func NewConnection(uri string) (*sql.DB, error) {
+	db, err := sql.Open("sqlserver", uri)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create db instance")
+	}
+
+	// Check the connection before executing all queries to reduce the number
+	// of connection errors that we might encounter.
+	if err = db.Ping(); err != nil {
+		err = errors.Wrap(err, "error doing ping to db")
+	}
+
+	return db, err
+}

x-pack/metricbeat/module/mssql/performance/_meta/data.json

@@ -0,0 +1,45 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "agent": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "event": {
+        "dataset": "mssql.performance",
+        "duration": 115000,
+        "module": "mssql"
+    },
+    "metricset": {
+        "name": "performance"
+    },
+    "mssql": {
+        "performance": {
+            "batch_requests": {
+                "sec": 109695
+            },
+            "buffer_cache_hit": {
+                "pct": 0.54
+            },
+            "compilations": {
+                "sec": 28814
+            },
+            "lock_waits": {
+                "sec": 3
+            },
+            "page_life_expectancy": {
+                "sec": 31783
+            },
+            "page_splits": {
+                "sec": 166
+            },
+            "recompilations": {
+                "sec": 0
+            },
+            "user_connections": 3
+        }
+    },
+    "service": {
+        "address": "172.26.0.2",
+        "type": "mssql"
+    }
+}

x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc

@@ -0,0 +1,12 @@
+`performance` Metricset fetches information from what's commonly known as https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-2017[Performance Counters] in MSSQL.
+
+We fetch the following data:
+
+* Page splits per instance: Cumulative per instance. Show diffs between periodic readings to identify periods of frequent page splits.
+* Page life expectancy in seconds: The expected time in seconds that a data page will remain in the buffer pool.
+*  Lock wait time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high lock contention.
+* Total number of user connections.
+* Cumulative (per instance) recompilations time in seconds: Show diffs between periodic readings to identify periods of high SQL re-compilations.
+* Compilations time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high SQL compilations.
+* Batch requests time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high request activity.
+* Buffer Cache hit: Percentage of data pages found in buffer cache without having to read from disk.

x-pack/metricbeat/module/mssql/performance/_meta/fields.yml

@@ -0,0 +1,63 @@
+- name: performance
+  type: group
+  description: performance metricset fetches information about the Performance Counters
+  fields:
+    - name: page_splits
+      type: group
+      description: Cumulative per instance. Show diffs between periodic readings to identify periods of frequent page splits.
+      fields:
+        - name: sec
+          description: Page splits in seconds
+          type: long
+
+    - name: page_life_expectancy
+      type: group
+      description: The expected time in seconds that a data page will remain in the buffer pool
+      fields:
+        - name: sec
+          description: Page life expectancy in seconds
+          type: long
+
+    - name: lock_waits
+      type: group
+      description: Cumulative per instance. Show diffs between periodic readings to identify periods of high lock contention.
+      fields:
+        - name: sec
+          description: Lock wait time in seconds
+          type: long
+
+    - name: user_connections
+      description: Total number of user connections
+      type: long
+
+    - name: recompilations
+      type: group
+      description: Cumulative per instance. Show diffs between periodic readings to identify periods of high SQL re-compilations.
+      fields:
+        - name: sec
+          description: Recompilations time in seconds
+          type: long
+
+    - name: compilations
+      type: group
+      description: Cumulative per instance. Show diffs between periodic readings to identify periods of high SQL compilations.
+      fields:
+        - name: sec
+          description: Compilations time in seconds
+          type: long
+
+    - name: batch_requests
+      type: group
+      description: Cumulative per instance. Show diffs between periodic readings to identify periods of high request activity.
+      fields:
+        - name: sec
+          description: Batch requests time in seconds
+          type: long
+
+    - name: buffer_cache_hit
+      type: group
+      description: Indicates the percentage of pages found in the buffer cache without having to read from disk
+      fields:
+        - name: pct
+          description: The ratio is the total number of cache hits divided by the total number of cache lookups over the last few thousand page accesses
+          type: double

x-pack/metricbeat/module/mssql/performance/data.go

@@ -0,0 +1,37 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package performance
+
+import (
+	s "github.com/elastic/beats/libbeat/common/schema"
+	c "github.com/elastic/beats/libbeat/common/schema/mapstrstr"
+)
+
+var (
+	schema = s.Schema{
+		"page_splits": s.Object{
+			"sec": c.Int("Page Splits/sec", s.Optional),
+		},
+		"page_life_expectancy": s.Object{
+			"sec": c.Int("Page life expectancy", s.Optional),
+		},
+		"lock_waits": s.Object{
+			"sec": c.Int("Lock Waits/sec", s.Optional),
+		},
+		"user_connections": c.Int("User Connections", s.Optional),
+		"recompilations": s.Object{
+			"sec": c.Int("SQL Re-Compilations/sec", s.Optional),
+		},
+		"compilations": s.Object{
+			"sec": c.Int("SQL Compilations/sec", s.Optional),
+		},
+		"batch_requests": s.Object{
+			"sec": c.Int("Batch Requests/sec", s.Optional),
+		},
+		"buffer_cache_hit": s.Object{
+			"pct": c.Float("Buffer cache hit ratio", s.Optional),
+		},
+	}
+)

x-pack/metricbeat/module/mssql/performance/data_integration_test.go

@@ -0,0 +1,63 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package performance
+
+import (
+	"net/url"
+	"testing"
+
+	_ "github.com/denisenkom/go-mssqldb"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+)
+
+func TestData(t *testing.T) {
+	t.Skip("Skipping `data.json` generation test")
+	_, config, err := getHostURI()
+	if err != nil {
+		t.Fatal("error getting config information", err.Error())
+	}
+
+	f := mbtest.NewReportingMetricSetV2(t, config)
+	events, errs := mbtest.ReportingFetchV2(f)
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+
+	if err = mbtest.WriteEventsReporterV2(f, t, ""); err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func getHostURI() (string, map[string]interface{}, error) {
+	config := mtest.GetConfig("performance")
+
+	host, ok := config["hosts"].([]string)
+	if !ok {
+		return "", nil, errors.New("error getting host name information")
+	}
+
+	username, ok := config["username"].(string)
+	if !ok {
+		return "", nil, errors.New("error getting username information")
+	}
+
+	password, ok := config["password"].(string)
+	if !ok {
+		return "", nil, errors.New("error getting password information")
+	}
+
+	u := &url.URL{
+		Scheme: "sqlserver",
+		User:   url.UserPassword(username, password),
+		Host:   host[0],
+	}
+
+	return u.String(), config, nil
+}