Feature/read metrics from unix socket npipe

CHANGELOG.next.asciidoc

@@ -23,6 +23,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - By default, all Beats-created files and folders will have a umask of 0027 (on POSIX systems). {pull}14119[14119]
 - Adding new `Enterprise` license type to the licenser. {issue}14246[14246]
 - Change wording when we fail to load a CA file to the cert pool. {issue}14309[14309]
+- Allow Metricbeat's beat module to read monitoring information over a named pipe or unix domain socket. {pull}14558[14558]
 
 *Auditbeat*
 

libbeat/api/npipe/listener_windows.go

@@ -65,6 +65,13 @@ func DialContext(npipe string) func(context.Context, string, string) (net.Conn,
 	}
 }
 
+// Dial create a Dial to be use with an http.Client to connect to a pipe.
+func Dial(npipe string) func(string, string) (net.Conn, error) {
+	return func(_, _ string) (net.Conn, error) {
+		return winio.DialPipe(npipe, nil)
+	}
+}
+
 // DefaultSD returns a default SecurityDescriptor which is the minimal required permissions to be
 // able to write to the named pipe. The security descriptor is returned in SDDL format.
 //

libbeat/api/npipe/listerner_posix.go

@@ -31,3 +31,10 @@ func DialContext(npipe string) func(context.Context, string, string) (net.Conn,
 		return nil, errors.New("named pipe doesn't work on linux")
 	}
 }
+
+// Dial create a Dial to be use with an http.Client to connect to a pipe.
+func Dial(npipe string) func(string, string) (net.Conn, error) {
+	return func(_, _ string) (net.Conn, error) {
+		return nil, errors.New("named pipe doesn't work on linux")
+	}
+}

libbeat/docs/http-endpoint.asciidoc

@@ -35,7 +35,7 @@ You can query a unix socket using the `CURL` command and the `--unix-socket` fla
 
 [source,js]
 ----
-curl -XGET --unix-socket '/var/run/{beatname_lc}.sock' 'http://unix/stats/?pretty'
+curl -XGET --unix-socket '/var/run/{beatname_lc}.sock' 'http:/stats/?pretty'
 ----
 
 

libbeat/outputs/transport/tcp.go

@@ -45,7 +45,6 @@ func TestNetDialer(d testing.Driver, timeout time.Duration) Dialer {
 		if err != nil {
 			return nil, err
 		}
-
 		addresses, err := net.LookupHost(host)
 		d.Fatal("dns lookup", err)
 		d.Info("addresses", strings.Join(addresses, ", "))
@@ -59,3 +58,16 @@ func TestNetDialer(d testing.Driver, timeout time.Duration) Dialer {
 		return DialWith(dialer, network, host, addresses, port)
 	})
 }
+
+// UnixDialer creates a Unix Dialer when using unix domain socket.
+func UnixDialer(timeout time.Duration, sockFile string) Dialer {
+	return TestUnixDialer(testing.NullDriver, timeout, sockFile)
+}
+
+// TestUnixDialer creates a Test Unix Dialer when using domain socket.
+func TestUnixDialer(d testing.Driver, timeout time.Duration, sockFile string) Dialer {
+	return DialerFunc(func(network, address string) (net.Conn, error) {
+		d.Info("connecting using unix domain socket", sockFile)
+		return net.DialTimeout("unix", sockFile, timeout)
+	})
+}

metricbeat/helper/dialer_posix.go

@@ -0,0 +1,43 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//+build !windows
+
+package helper
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/elastic/beats/libbeat/outputs/transport"
+	"github.com/elastic/beats/metricbeat/mb"
+)
+
+func makeDialer(t time.Duration, hostData mb.HostData) (transport.Dialer, string, error) {
+	switch hostData.Transport {
+	case mb.TransportNpipe:
+		return nil, "", fmt.Errorf(
+			"cannot use %s as the URI, named pipes are only supported on Windows",
+			hostData.SanitizedURI,
+		)
+	case mb.TransportUnix:
+		return transport.UnixDialer(t, strings.TrimSuffix(hostData.TransportPath, "/")), hostData.SanitizedURI, nil
+	default:
+		return transport.NetDialer(t), hostData.SanitizedURI, nil
+	}
+}

metricbeat/helper/dialer_windows.go

@@ -0,0 +1,46 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//+build windows
+
+package helper
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/elastic/beats/libbeat/api/npipe"
+	"github.com/elastic/beats/libbeat/outputs/transport"
+	"github.com/elastic/beats/metricbeat/mb"
+)
+
+func makeDialer(t time.Duration, hostData mb.HostData) (transport.Dialer, string, error) {
+	switch hostData.Transport {
+	case mb.TransportUnix:
+		return nil, "", fmt.Errorf(
+			"cannot use %s as the URI, unix sockets are not supported on Windows, use npipe instead",
+			hostData.SanitizedURI,
+		)
+	case mb.TransportNpipe:
+		return npipe.DialContext(
+			strings.TrimSuffix(npipe.TransformString(p), "/"),
+		), hostData.SanitizedURI, nil
+	default:
+		return transport.NetDialer(t), hostData.SanitizedURI, nil
+	}
+}

metricbeat/helper/http.go

@@ -33,6 +33,8 @@ import (
 	"github.com/elastic/beats/metricbeat/mb"
 )
 
+// HTTP is a custom HTTP Client that handle the complexity of connection and retrieving information
+// from HTTP endpoint.
 type HTTP struct {
 	hostData mb.HostData
 	client   *http.Client // HTTP client that is reused across requests.
@@ -72,9 +74,12 @@ func newHTTPFromConfig(config Config, name string, hostData mb.HostData) (*HTTP,
 		return nil, err
 	}
 
-	var dialer, tlsDialer transport.Dialer
+	var tlsDialer transport.Dialer
+	dialer, uri, err := makeDialer(config.ConnectTimeout, hostData)
+	if err != nil {
+		return nil, err
+	}
 
-	dialer = transport.NetDialer(config.ConnectTimeout)
 	tlsDialer, err = transport.TLSDialer(dialer, tlsConfig, config.ConnectTimeout)
 	if err != nil {
 		return nil, err
@@ -92,7 +97,7 @@ func newHTTPFromConfig(config Config, name string, hostData mb.HostData) (*HTTP,
 		},
 		headers: config.Headers,
 		method:  "GET",
-		uri:     hostData.SanitizedURI,
+		uri:     uri,
 		body:    nil,
 	}, nil
 }

metricbeat/helper/http_test.go

@@ -18,10 +18,13 @@
 package helper
 
 import (
+	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"runtime"
 	"testing"
 	"time"
 
@@ -158,6 +161,90 @@ func TestAuthentication(t *testing.T) {
 	assert.Equal(t, http.StatusOK, response.StatusCode, "response status code")
 }
 
+func TestOverUnixSocket(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skipf("unix domain socket aren't supported under Windows")
+		return
+	}
+
+	t.Run("at root", func(t *testing.T) {
+		tmpDir, err := ioutil.TempDir("", "testsocket")
+		require.NoError(t, err)
+		defer os.RemoveAll(tmpDir)
+
+		sockFile := tmpDir + "/test.sock"
+
+		l, err := net.Listen("unix", sockFile)
+		require.NoError(t, err)
+
+		defer l.Close()
+
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprintf(w, "ehlo!")
+		})
+
+		go http.Serve(l, mux)
+
+		cfg := defaultConfig()
+		hostData := mb.HostData{
+			Transport:     mb.TransportUnix,
+			TransportPath: sockFile,
+			URI:           "http://unix/",
+			SanitizedURI:  "http://unix",
+		}
+
+		h, err := newHTTPFromConfig(cfg, "test", hostData)
+		require.NoError(t, err)
+
+		r, err := h.FetchResponse()
+		require.NoError(t, err)
+		defer r.Body.Close()
+		content, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, []byte("ehlo!"), content)
+	})
+
+	t.Run("at specific path", func(t *testing.T) {
+		tmpDir, err := ioutil.TempDir("", "testsocket")
+		require.NoError(t, err)
+		defer os.RemoveAll(tmpDir)
+
+		sockFile := tmpDir + "/test.sock"
+		uri := "http://unix/ok"
+
+		l, err := net.Listen("unix", sockFile)
+		require.NoError(t, err)
+
+		defer l.Close()
+
+		mux := http.NewServeMux()
+		mux.HandleFunc("/ok", func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprintf(w, "ehlo!")
+		})
+
+		go http.Serve(l, mux)
+
+		cfg := defaultConfig()
+		hostData := mb.HostData{
+			Transport:     mb.TransportUnix,
+			TransportPath: sockFile,
+			URI:           uri,
+			SanitizedURI:  uri,
+		}
+
+		h, err := newHTTPFromConfig(cfg, "test", hostData)
+		require.NoError(t, err)
+
+		r, err := h.FetchResponse()
+		require.NoError(t, err)
+		defer r.Body.Close()
+		content, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, []byte("ehlo!"), content)
+	})
+}
+
 func checkTimeout(t *testing.T, h *HTTP) {
 	t.Helper()