Cherry-pick #21436 to 7.x: Initial work toward synthetic monitors

.ci/heartbeat-synthetics.groovy

@@ -0,0 +1,122 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-18 && immutable' }
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    SYNTHETICS = "-synthetics"
+    PIPELINE_LOG_LEVEL = "INFO"
+    BEATS_FOLDER = "x-pack/heartbeat"
+  }
+  options {
+    timeout(time: 3, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  triggers {
+    issueCommentTrigger('(?i)^\\/packag[ing|e] synthetics$')
+  }
+  parameters {
+    booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
+  }
+  stages {
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}")
+        setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+      }
+    }
+    stage('Build and test'){
+      steps {
+        withGithubNotify(context: "Build and test") {
+          withBeatsEnv{
+            dir("${env.BEATS_FOLDER}") {
+              sh(label: 'Build and test', script: 'mage build test')
+            }
+          }
+        }
+      }
+    }
+    stage('Package Linux'){
+      environment {
+        HOME = "${env.WORKSPACE}"
+        PLATFORMS = [
+          '+all',
+          'linux/amd64'
+        ].join(' ')
+      }
+      steps {
+        withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") {
+          release()
+          pushCIDockerImages()
+        }
+      }
+    }
+  }
+}
+
+def pushCIDockerImages(){
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    tagAndPush('heartbeat')
+  }
+}
+
+def tagAndPush(name){
+  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+
+  def tagName = "${libbetaVer}"
+  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}${env.SYNTHETICS}"
+  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
+  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+  retry(3){
+    sh(label:'Change tag and push', script: """
+      docker tag ${oldName} ${newName}
+      docker push ${newName}
+      docker tag ${oldName} ${commitName}
+      docker push ${commitName}
+    """)
+  }
+}
+
+def release(){
+  withBeatsEnv(){
+    dir("${env.BEATS_FOLDER}") {
+      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    }
+  }
+}
+
+/**
+* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
+* therefore the storage bucket in GCP should follow the same folder structure.
+* This is required by https://github.com/elastic/beats-tester
+* e.g.
+* baseDir=name -> return name
+* baseDir=name1/name2/name3-> return name2
+*/
+def getBeatsName(baseDir) {
+  return baseDir.replace('x-pack/', '')
+}
+
+def withBeatsEnv(Closure body) {
+  withMageEnv(){
+    withEnv([
+      "PYTHON_ENV=${WORKSPACE}/python-env"
+    ]) {
+      dir("${env.BASE_DIR}"){
+        body()
+      }
+    }
+  }
+}

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect
 	github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-test/deep v1.0.7
 	github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be
 	github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e
 	github.com/godror/godror v0.10.4

go.sum

@@ -326,6 +326,8 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
+github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
 github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be h1:zXHeEEJ231bTf/IXqvCfeaqjLpXsq42ybLoT4ROSR6Y=

heartbeat/_meta/fields.common.yml

@@ -122,6 +122,56 @@
               type: long
               description: Duration in microseconds
 
+- key: synthetics
+  title: "Synthetics types"
+  description:
+  fields:
+    - name: synthetics
+      type: group
+      description: >
+        Synthetics related fields.
+      fields:
+        - name: type
+          type: keyword
+        - name: package_version
+          type: keyword
+        - name: index
+          type: integer
+          description: >
+            Indexed used for creating total order of all events
+            in this invocation.
+        - name: payload
+          type: object
+          enabled: false
+        - name: blob
+          type: binary
+          description: binary data payload
+        - name: blob_mime
+          type: keyword
+          description: mime type of blob data
+        - name: step
+          type: group
+          fields:
+            - name: name
+              type: text
+            - name: index
+              type: integer
+        - name: journey
+          type: group
+          fields:
+            - name: name
+              type: text
+            - name: id
+              type: keyword
+        - name: error
+          type: group
+          fields:
+            - name: name
+              type: keyword
+            - name: message
+              type: text
+            - name: stack
+              type: text
 - key: http
   title: "HTTP monitor"
   description:

heartbeat/beater/heartbeat.go

@@ -18,6 +18,7 @@
 package beater
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -99,6 +100,13 @@ func (bt *Heartbeat) Run(b *beat.Beat) error {
 		}
 	}
 
+	if len(bt.config.SyntheticSuites) > 0 {
+		err := bt.RunSyntheticSuiteMonitors(b)
+		if err != nil {
+			return err
+		}
+	}
+
 	if bt.config.Autodiscover != nil {
 		bt.autodiscover, err = bt.makeAutodiscover(b)
 		if err != nil {
@@ -160,6 +168,50 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) {
 	return nil
 }
 
+// Provide hook to define journey list discovery from x-pack
+type JourneyLister func(ctx context.Context, suiteFile string, params common.MapStr) ([]string, error)
+
+var mainJourneyLister JourneyLister
+
+func RegisterJourneyLister(jl JourneyLister) {
+	mainJourneyLister = jl
+}
+
+func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
+	// If we are running without XPack this will be nil
+	if mainJourneyLister == nil {
+		return nil
+	}
+	for _, suite := range bt.config.SyntheticSuites {
+		logp.Info("Listing suite %s", suite.Path)
+		journeyNames, err := mainJourneyLister(context.TODO(), suite.Path, suite.Params)
+		if err != nil {
+			return err
+		}
+		factory := monitors.NewFactory(b.Info, bt.scheduler, false)
+		for _, name := range journeyNames {
+			cfg, err := common.NewConfigFrom(map[string]interface{}{
+				"type":         "browser",
+				"path":         suite.Path,
+				"schedule":     suite.Schedule,
+				"params":       suite.Params,
+				"journey_name": name,
+				"name":         name,
+				"id":           name,
+			})
+			if err != nil {
+				return err
+			}
+			created, err := factory.Create(b.Publisher, cfg)
+			if err != nil {
+				return errors.Wrap(err, "could not create monitor")
+			}
+			created.Start()
+		}
+	}
+	return nil
+}
+
 // makeAutodiscover creates an autodiscover object ready to be started.
 func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, error) {
 	autodiscover, err := autodiscover.NewAutodiscover(

heartbeat/config/config.go

@@ -28,10 +28,11 @@ import (
 // Config defines the structure of heartbeat.yml.
 type Config struct {
 	// Modules is a list of module specific configuration data.
-	Monitors       []*common.Config     `config:"monitors"`
-	ConfigMonitors *common.Config       `config:"config.monitors"`
-	Scheduler      Scheduler            `config:"scheduler"`
-	Autodiscover   *autodiscover.Config `config:"autodiscover"`
+	Monitors        []*common.Config     `config:"monitors"`
+	ConfigMonitors  *common.Config       `config:"config.monitors"`
+	Scheduler       Scheduler            `config:"scheduler"`
+	Autodiscover    *autodiscover.Config `config:"autodiscover"`
+	SyntheticSuites []*SyntheticSuite    `config:"synthetic_suites"`
 }
 
 // Scheduler defines the syntax of a heartbeat.yml scheduler block.
@@ -40,5 +41,12 @@ type Scheduler struct {
 	Location string `config:"location"`
 }
 
+type SyntheticSuite struct {
+	Path     string                 `config:"path"`
+	Name     string                 `config:"id_prefix"`
+	Schedule string                 `config:"schedule"`
+	Params   map[string]interface{} `config:"params"`
+}
+
 // DefaultConfig is the canonical instantiation of Config.
 var DefaultConfig = Config{}