Cherry-pick #21436 to 7.x: Initial work toward synthetic monitors

NOTICE.txt

@@ -8996,6 +8996,37 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
   defined by the Mozilla Public License, v. 2.0.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/go-test/deep
+Version: v1.0.7
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/go-test/[email protected]/LICENSE:
+
+MIT License
+
+Copyright 2015-2017 Daniel Nichter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/gocarina/gocsv
 Version: v0.0.0-20170324095351-ffef3ffc77be

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect
 	github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-test/deep v1.0.7
 	github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be
 	github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e
 	github.com/godror/godror v0.10.4

go.sum

@@ -326,6 +326,8 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
+github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
 github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be h1:zXHeEEJ231bTf/IXqvCfeaqjLpXsq42ybLoT4ROSR6Y=

heartbeat/_meta/fields.common.yml

@@ -122,6 +122,56 @@
               type: long
               description: Duration in microseconds
 
+- key: synthetics
+  title: "Synthetics types"
+  description:
+  fields:
+    - name: synthetics
+      type: group
+      description: >
+        Synthetics related fields.
+      fields:
+        - name: type
+          type: keyword
+        - name: package_version
+          type: keyword
+        - name: index
+          type: integer
+          description: >
+            Indexed used for creating total order of all events
+            in this invocation.
+        - name: payload
+          type: object
+          enabled: false
+        - name: blob
+          type: binary
+          description: binary data payload
+        - name: blob_mime
+          type: keyword
+          description: mime type of blob data
+        - name: step
+          type: group
+          fields:
+            - name: name
+              type: text
+            - name: index
+              type: integer
+        - name: journey
+          type: group
+          fields:
+            - name: name
+              type: text
+            - name: id
+              type: keyword
+        - name: error
+          type: group
+          fields:
+            - name: name
+              type: keyword
+            - name: message
+              type: text
+            - name: stack
+              type: text
 - key: http
   title: "HTTP monitor"
   description:

heartbeat/beater/heartbeat.go

@@ -18,6 +18,7 @@
 package beater
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -99,6 +100,13 @@ func (bt *Heartbeat) Run(b *beat.Beat) error {
 		}
 	}
 
+	if len(bt.config.SyntheticSuites) > 0 {
+		err := bt.RunSyntheticSuiteMonitors(b)
+		if err != nil {
+			return err
+		}
+	}
+
 	if bt.config.Autodiscover != nil {
 		bt.autodiscover, err = bt.makeAutodiscover(b)
 		if err != nil {
@@ -160,6 +168,50 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) {
 	return nil
 }
 
+// Provide hook to define journey list discovery from x-pack
+type JourneyLister func(ctx context.Context, suiteFile string, params common.MapStr) ([]string, error)
+
+var mainJourneyLister JourneyLister
+
+func RegisterJourneyLister(jl JourneyLister) {
+	mainJourneyLister = jl
+}
+
+func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
+	// If we are running without XPack this will be nil
+	if mainJourneyLister == nil {
+		return nil
+	}
+	for _, suite := range bt.config.SyntheticSuites {
+		logp.Info("Listing suite %s", suite.Path)
+		journeyNames, err := mainJourneyLister(context.TODO(), suite.Path, suite.Params)
+		if err != nil {
+			return err
+		}
+		factory := monitors.NewFactory(b.Info, bt.scheduler, false)
+		for _, name := range journeyNames {
+			cfg, err := common.NewConfigFrom(map[string]interface{}{
+				"type":         "browser",
+				"path":         suite.Path,
+				"schedule":     suite.Schedule,
+				"params":       suite.Params,
+				"journey_name": name,
+				"name":         name,
+				"id":           name,
+			})
+			if err != nil {
+				return err
+			}
+			created, err := factory.Create(b.Publisher, cfg)
+			if err != nil {
+				return errors.Wrap(err, "could not create monitor")
+			}
+			created.Start()
+		}
+	}
+	return nil
+}
+
 // makeAutodiscover creates an autodiscover object ready to be started.
 func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, error) {
 	autodiscover, err := autodiscover.NewAutodiscover(

heartbeat/config/config.go

@@ -28,10 +28,11 @@ import (
 // Config defines the structure of heartbeat.yml.
 type Config struct {
 	// Modules is a list of module specific configuration data.
-	Monitors       []*common.Config     `config:"monitors"`
-	ConfigMonitors *common.Config       `config:"config.monitors"`
-	Scheduler      Scheduler            `config:"scheduler"`
-	Autodiscover   *autodiscover.Config `config:"autodiscover"`
+	Monitors        []*common.Config     `config:"monitors"`
+	ConfigMonitors  *common.Config       `config:"config.monitors"`
+	Scheduler       Scheduler            `config:"scheduler"`
+	Autodiscover    *autodiscover.Config `config:"autodiscover"`
+	SyntheticSuites []*SyntheticSuite    `config:"synthetic_suites"`
 }
 
 // Scheduler defines the syntax of a heartbeat.yml scheduler block.
@@ -40,5 +41,12 @@ type Scheduler struct {
 	Location string `config:"location"`
 }
 
+type SyntheticSuite struct {
+	Path     string                 `config:"path"`
+	Name     string                 `config:"id_prefix"`
+	Schedule string                 `config:"schedule"`
+	Params   map[string]interface{} `config:"params"`
+}
+
 // DefaultConfig is the canonical instantiation of Config.
 var DefaultConfig = Config{}

heartbeat/docs/fields.asciidoc

@@ -26,6 +26,7 @@ grouped in the following categories:
 * <<exported-fields-resolve>>
 * <<exported-fields-socks5>>
 * <<exported-fields-summary>>
+* <<exported-fields-synthetics>>
 * <<exported-fields-tcp>>
 * <<exported-fields-tls>>
 
@@ -9299,6 +9300,122 @@ type: integer
 
 --
 
+[[exported-fields-synthetics]]
+== Synthetics types fields
+
+None
+
+
+[float]
+=== synthetics
+
+Synthetics related fields.
+
+
+
+*`synthetics.type`*::
++
+--
+type: keyword
+
+--
+
+*`synthetics.package_version`*::
++
+--
+type: keyword
+
+--
+
+*`synthetics.index`*::
++
+--
+Indexed used for creating total order of all events in this invocation.
+
+
+type: integer
+
+--
+
+*`synthetics.payload`*::
++
+--
+type: object
+
+Object is not enabled.
+
+--
+
+*`synthetics.blob`*::
++
+--
+binary data payload
+
+type: binary
+
+--
+
+*`synthetics.blob_mime`*::
++
+--
+mime type of blob data
+
+type: keyword
+
+--
+
+
+*`synthetics.step.name`*::
++
+--
+type: text
+
+--
+
+*`synthetics.step.index`*::
++
+--
+type: integer
+
+--
+
+
+*`synthetics.journey.name`*::
++
+--
+type: text
+
+--
+
+*`synthetics.journey.id`*::
++
+--
+type: keyword
+
+--
+
+
+*`synthetics.error.name`*::
++
+--
+type: keyword
+
+--
+
+*`synthetics.error.message`*::
++
+--
+type: text
+
+--
+
+*`synthetics.error.stack`*::
++
+--
+type: text
+
+--
+
 [[exported-fields-tcp]]
 == TCP layer fields
 

heartbeat/hbtestllext/isdefs.go

@@ -33,3 +33,11 @@ var IsTime = isdef.Is("time", func(path llpath.Path, v interface{}) *llresult.Re
 	}
 	return llresult.ValidResult(path)
 })
+
+var IsInt64 = isdef.Is("positiveInt64", func(path llpath.Path, v interface{}) *llresult.Results {
+	_, ok := v.(int64)
+	if !ok {
+		return llresult.SimpleResult(path, false, "expected an int64")
+	}
+	return llresult.ValidResult(path)
+})

heartbeat/look/look.go

@@ -29,12 +29,18 @@ import (
 
 // RTT formats a round-trip-time given as time.Duration into an
 // event field. The duration is stored in `{"us": rtt}`.
+// TODO: This returns a time.Duration, which isn't quite right. time.Duration
+// represents nanos, whereas this really returns millis. It should probably
+// return a plain int64 type instead.
 func RTT(rtt time.Duration) common.MapStr {
 	if rtt < 0 {
 		rtt = 0
 	}
 
 	return common.MapStr{
+		// cast to int64 since a go duration is a nano, but we want micros
+		// This makes the types less confusing because other wise the duration
+		// we get back has the wrong unit
 		"us": rtt / (time.Microsecond / time.Nanosecond),
 	}
 }

heartbeat/magefile.go

@@ -80,7 +80,11 @@ func Fields() error {
 
 // Update updates the generated files (aka make update).
 func Update() {
-	mg.SerialDeps(Fields, Config)
+	mg.SerialDeps(Fields, FieldDocs, Config)
+}
+
+func FieldDocs() error {
+	return devtools.Docs.FieldDocs("fields.yml")
 }
 
 // Config generates both the short/reference/docker configs.