feat: Support MySQL/Redis/MongoDB/PostgreSQL/Zookeeper Native Client to check the data

README.md

@@ -139,12 +139,18 @@ On application startup, the configured probes are scheduled for their initial fi
   - **PostgreSQL**. Connect to PostgreSQL server and run `SELECT 1` SQL.
   - **Zookeeper**. Connect to Zookeeper server and run `get /` command.
 
+  Most of them are supported to check the data whether is stored correctly.
+
   ```YAML
   client:
-    - name: Kafka Native Client (local)
-      driver: "kafka"
-      host: "localhost:9093"
-      # mTLS
+    - name: MySQL Native Client (local)
+      driver: "mysql"
+      host: "localhost:3306"
+      user: "root"
+      password: "pass"
+      data: # Optional - Data to check
+        "database:table:column:id:100": "value"
+      # mTLS - Optional
       ca: /path/to/file.ca
       cert: /path/to/file.crt
       key: /path/to/file.key
@@ -760,14 +766,18 @@ host:
 
 ### 3.7 Native Client Probe Configuration
 
+Native Client probe using the SDK to communicate with the remote endpoint. And you can define multiple data, and EaseProbe would help to check the data whether is stored or not.
+
 ```YAML
 # Native Client Probe
 client:
   - name: Redis Native Client (local)
     driver: "redis"  # driver is redis
     host: "localhost:6379"  # server and port
     password: "abc123" # password
-    # mTLS
+    data:         # Optional
+      key: val    # Check that `key` exists and its value is `val`
+    # mTLS - Optional
     ca: /path/to/file.ca
     cert: /path/to/file.crt
     key: /path/to/file.key
@@ -777,13 +787,27 @@ client:
     host: "localhost:3306"
     username: "root"
     password: "pass"
+    data: # Optional, check the specific column value in the table
+      #  Usage: "database:table:column:primary_key:value" : "expected_value"
+      #         transfer to : "SELECT column FROM database.table WHERE primary_key = value"
+      #         the `value` for `primary_key` must be int
+      "test:product:name:id:1" : "EaseProbe" # select name from test.product where id = 1
+      "test:employee:age:id:2" : 45          # select age from test.employee where id = 2
+    # mTLS - Optional
+    ca: /path/to/file.ca
+    cert: /path/to/file.crt
+    key: /path/to/file.key
 
   - name: MongoDB Native Client (local)
     driver: "mongo"
     host: "localhost:27017"
     username: "admin"
     password: "abc123"
     timeout: 5s
+    data: # Optional, find the specific value in the table
+      #  Usage: "database:collection" : "{JSON}"
+      "test:employee" : '{"name":"Hao Chen"}' # find the employee with name "Hao Chen"
+      "test:product" : '{"name":"EaseProbe"}' # find the product with name "EaseProbe"
 
   - name: Memcache Native Client (local)
     driver: "memcache"
@@ -796,7 +820,7 @@ client:
   - name: Kafka Native Client (local)
     driver: "kafka"
     host: "localhost:9093"
-    # mTLS
+    # mTLS - Optional
     ca: /path/to/file.ca
     cert: /path/to/file.crt
     key: /path/to/file.key
@@ -806,12 +830,24 @@ client:
     host: "localhost:5432"
     username: "postgres"
     password: "pass"
+    data: # Optional, check the specific column value in the table
+      #  Usage: "database:table:column:primary_key:value" : "expected_value"
+      #         transfer to : "SELECT column FROM table WHERE primary_key = value"
+      #         the `value` for `primary_key` must be int
+      "test:product:name:id:1" : "EaseProbe" # select name from product where id = 1
+      "test:employee:age:id:2" : 45          # select age from employee where id = 2
+    # mTLS - Optional
+    ca: /path/to/file.ca
+    cert: /path/to/file.crt
+    key: /path/to/file.key
 
   - name: Zookeeper Native Client (local)
     driver: "zookeeper"
     host: "localhost:2181"
     timeout: 5s
-    # mTLS
+    data: # Optional, check the specific value in the path
+      "/path/to/key": "value" # Check that the value of the `/path/to/key` is "value"
+    # mTLS - Optional
     ca: /path/to/file.ca
     cert: /path/to/file.crt
     key: /path/to/file.key

probe/client/client.go

@@ -47,36 +47,37 @@ func (c *Client) Config(gConf global.ProbeSettings) error {
 	tag := c.DriverType.String()
 	name := c.ProbeName
 	c.DefaultProbe.Config(gConf, kind, tag, name, c.Host, c.DoProbe)
-	c.configClientDriver()
-
-	if c.DriverType == conf.Unknown {
-		return fmt.Errorf("[%s / %s ] unknown driver type", kind, name)
+	if err := c.Check(); err != nil {
+		return err
+	}
+	if err := c.configClientDriver(); err != nil {
+		return err
 	}
-
 	log.Debugf("[%s] configuration: %+v, %+v", c.ProbeKind, c, c.Result())
 	return nil
 }
 
-func (c *Client) configClientDriver() {
+func (c *Client) configClientDriver() (err error) {
 	switch c.DriverType {
 	case conf.MySQL:
-		c.client = mysql.New(c.Options)
+		c.client, err = mysql.New(c.Options)
 	case conf.Redis:
-		c.client = redis.New(c.Options)
+		c.client, err = redis.New(c.Options)
 	case conf.Memcache:
-		c.client = memcache.New(c.Options)
+		c.client, err = memcache.New(c.Options)
 	case conf.Mongo:
-		c.client = mongo.New(c.Options)
+		c.client, err = mongo.New(c.Options)
 	case conf.Kafka:
-		c.client = kafka.New(c.Options)
+		c.client, err = kafka.New(c.Options)
 	case conf.PostgreSQL:
-		c.client = postgres.New(c.Options)
+		c.client, err = postgres.New(c.Options)
 	case conf.Zookeeper:
-		c.client = zookeeper.New(c.Options)
+		c.client, err = zookeeper.New(c.Options)
 	default:
 		c.DriverType = conf.Unknown
+		err = fmt.Errorf("Unknown Driver Type")
 	}
-
+	return
 }
 
 // DoProbe return the checking result

probe/client/client_test.go

@@ -23,8 +23,10 @@ import (
 
 	"bou.ke/monkey"
 	"github.com/megaease/easeprobe/global"
+	"github.com/megaease/easeprobe/probe/base"
 	"github.com/megaease/easeprobe/probe/client/conf"
 	"github.com/megaease/easeprobe/probe/client/kafka"
+	"github.com/megaease/easeprobe/probe/client/memcache"
 	"github.com/megaease/easeprobe/probe/client/mongo"
 	"github.com/megaease/easeprobe/probe/client/mysql"
 	"github.com/megaease/easeprobe/probe/client/postgres"
@@ -36,31 +38,36 @@ import (
 func newDummyClient(driver conf.DriverType) Client {
 	return Client{
 		Options: conf.Options{
+			DefaultProbe: base.DefaultProbe{
+				ProbeName: "dummy_" + driver.String(),
+			},
 			Host:       "example.com:1234",
 			DriverType: driver,
 			Username:   "user",
 			Password:   "pass",
+			Data:       map[string]string{},
 			TLS:        global.TLS{},
 		},
 		client: nil,
 	}
 }
 
 func MockProbe[T any](c T) {
-	monkey.PatchInstanceMethod(reflect.TypeOf(c), "Probe", func(_ T) (bool, string) {
+	p := &c
+	monkey.PatchInstanceMethod(reflect.TypeOf(p), "Probe", func(_ *T) (bool, string) {
 		return true, "Successfully"
 	})
 }
 
 func TestClient(t *testing.T) {
-	global.InitEaseProbe("DummyProbe", "icon")
 	clients := []Client{
 		newDummyClient(conf.MySQL),
 		newDummyClient(conf.PostgreSQL),
 		newDummyClient(conf.Redis),
 		newDummyClient(conf.Mongo),
 		newDummyClient(conf.Kafka),
 		newDummyClient(conf.Zookeeper),
+		newDummyClient(conf.Memcache),
 	}
 
 	for _, client := range clients {
@@ -69,6 +76,11 @@ func TestClient(t *testing.T) {
 		assert.Equal(t, "client", client.ProbeKind)
 		assert.Equal(t, client.DriverType.String(), client.ProbeTag)
 
+		client.Host = "wronghost"
+		err = client.Config(global.ProbeSettings{})
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "Invalid Host")
+
 		switch client.DriverType {
 		case conf.MySQL:
 			MockProbe(mysql.MySQL{})
@@ -82,7 +94,13 @@ func TestClient(t *testing.T) {
 			MockProbe(kafka.Kafka{})
 		case conf.Zookeeper:
 			MockProbe(zookeeper.Zookeeper{})
+		case conf.Memcache:
+			MockProbe(memcache.Memcache{})
 		}
+		client.Host = "example.com:1234"
+		err = client.Config(global.ProbeSettings{})
+		assert.Nil(t, err)
+
 		s, m := client.DoProbe()
 		assert.True(t, s)
 		assert.Contains(t, m, "Successfully")
@@ -96,3 +114,15 @@ func TestClient(t *testing.T) {
 
 	monkey.UnpatchAll()
 }
+
+func TestFailed(t *testing.T) {
+
+	c := newDummyClient(conf.Unknown)
+	var cnf *conf.Options
+	monkey.PatchInstanceMethod(reflect.TypeOf(cnf), "Check", func(_ *conf.Options) error {
+		return nil
+	})
+	err := c.Config(global.ProbeSettings{})
+	assert.NotNil(t, err)
+
+}

probe/client/conf/conf.go

@@ -20,6 +20,8 @@ package conf
 import (
 	"encoding/json"
 	"fmt"
+	"net"
+	"strconv"
 	"strings"
 
 	"github.com/megaease/easeprobe/global"
@@ -73,6 +75,21 @@ type Options struct {
 	global.TLS `yaml:",inline"`
 }
 
+// Check do the configuration check
+func (d *Options) Check() error {
+	_, port, err := net.SplitHostPort(d.Host)
+	if err != nil {
+		return fmt.Errorf("Invalid Host: %s. %v", d.Host, err)
+	}
+	if p, err := strconv.Atoi(port); err != nil || p < 1 || p > 65535 {
+		return fmt.Errorf("Invalid Port: %s", port)
+	}
+	if d.DriverType == Unknown {
+		return fmt.Errorf("Unknown driver")
+	}
+	return nil
+}
+
 // DriverTypeMap is the map of driver [name, driver]
 var DriverTypeMap = reverseMap(DriverMap)
 

probe/client/conf/conf_test.go

@@ -67,3 +67,42 @@ func TestDirverType(t *testing.T) {
 	assert.Equal(t, "unknown", d.String())
 
 }
+
+func TestOptionsCheck(t *testing.T) {
+	opts := Options{
+		Host:       "localhost:3306",
+		DriverType: MySQL,
+	}
+	err := opts.Check()
+	assert.Nil(t, err)
+
+	opts.Host = "127.0.0.1:3306"
+	err = opts.Check()
+	assert.Nil(t, err)
+
+	opts.Host = "localhost:3306"
+	opts.DriverType = Unknown
+	err = opts.Check()
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Unknown driver")
+
+	opts.Host = "localhost"
+	err = opts.Check()
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Invalid Host")
+
+	opts.Host = "localhost:3306:1234"
+	err = opts.Check()
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Invalid Host")
+
+	opts.Host = "10.10.10.1:asdf"
+	err = opts.Check()
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Invalid Port")
+
+	opts.Host = "10.10.10.1:123456"
+	err = opts.Check()
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Invalid Port")
+}

probe/client/kafka/kafka.go

@@ -20,6 +20,7 @@ package kafka
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 
 	"github.com/megaease/easeprobe/probe/client/conf"
 	"github.com/segmentio/kafka-go"
@@ -38,25 +39,27 @@ type Kafka struct {
 }
 
 // New create a Kafka client
-func New(opt conf.Options) Kafka {
+func New(opt conf.Options) (*Kafka, error) {
 	tls, err := opt.TLS.Config()
 	if err != nil {
-		log.Errorf("[%s / %s / %s] - TLS Config error - %v", opt.ProbeKind, opt.ProbeName, opt.ProbeTag, err)
+		log.Errorf("[%s / %s / %s] - TLS Config Error - %v", opt.ProbeKind, opt.ProbeName, opt.ProbeTag, err)
+		return nil, fmt.Errorf("TLS Config Error - %v", err)
 	}
-	return Kafka{
+	k := &Kafka{
 		Options: opt,
 		tls:     tls,
 		Context: context.Background(),
 	}
+	return k, nil
 }
 
 // Kind return the name of client
-func (k Kafka) Kind() string {
+func (k *Kafka) Kind() string {
 	return Kind
 }
 
 // Probe do the health check
-func (k Kafka) Probe() (bool, string) {
+func (k *Kafka) Probe() (bool, string) {
 
 	var dialer *kafka.Dialer
 

probe/client/kafka/kafka_test.go

@@ -39,8 +39,9 @@ func TestKaf(t *testing.T) {
 		TLS:        global.TLS{},
 	}
 
-	kaf := New(conf)
+	kaf, err := New(conf)
 	assert.Equal(t, "Kafka", kaf.Kind())
+	assert.Nil(t, err)
 
 	var dialer *kafka.Dialer
 	monkey.PatchInstanceMethod(reflect.TypeOf(dialer), "DialContext", func(_ *kafka.Dialer, _ context.Context, _ string, _ string) (*kafka.Conn, error) {
@@ -78,9 +79,15 @@ func TestKafkaFailed(t *testing.T) {
 		},
 	}
 
-	kaf := New(conf)
+	kaf, err := New(conf)
 	//TLS failed
-	assert.Nil(t, kaf.tls)
+	assert.Nil(t, kaf)
+	assert.NotNil(t, err)
+
+	conf.TLS = global.TLS{}
+	kaf, err = New(conf)
+	assert.NotNil(t, kaf)
+	assert.Nil(t, err)
 	assert.Equal(t, "Kafka", kaf.Kind())
 
 	var dialer *kafka.Dialer