Skip to content

Commit

Permalink
[Heartbeat] Add Additional ECS tls.* fields (elastic#17687)
Browse files Browse the repository at this point in the history
Work in support of elastic/uptime#161

This patch adds additional ECS [TLS](https://www.elastic.co/guide/en/ecs/current/ecs-tls.html) and [x509](elastic/ecs#762) fields. Note that we are blocked on the x509 fields which are not yet merged into ECS.

Sample output of the `tls.*` fields with this patch is below. Note the somewhat strange nesting of data in `issuer` and `subject`. This is per the ECS spec, but a bit awkward. We may want to break this data out into the more specific ECS `x509` type in the future. For UI work we are likely fine to parse this on the client and display the CN section in most cases. I did break out the CN into its own field in `x509.subject/issuer.common_name`. However, if we do want to aggregate on issuer in the future it's good to have the full distinguished name to do that on.

This PR also refactors some `libbeat` code around parsing TLS versions and adds test coverage there as well.

```json
{
	"tls": {
		"certificate_not_valid_after": "2020-07-16T03:15:39Z",
		"certificate_not_valid_before": "2019-08-16T01:40:25Z",
		"server": {
			"hash": {
				"sha1": "b7b4b89ef0d0caf39d223736f0fdbb03c7b426f1",
				"sha256": "12b00d04db0db8caa302bfde043e88f95baceb91e86ac143e93830b4bbec726d"
			},
			"x509": {
				"issuer": {
					"common_name": "GlobalSign CloudSSL CA - SHA256 - G3",
					"distinguished_name": "CN=GlobalSign CloudSSL CA - SHA256 - G3,O=GlobalSign nv-sa,C=BE"
				},
				"not_after": "2020-07-16T03:15:39Z",
				"not_before": "2019-08-16T01:40:25Z",
				"public_key_algorithm": "RSA",
				"public_key_size": 2048,
				"serial_number": "26610543540289562361990401194",
				"signature_algorithm": "SHA256-RSA",
				"subject": {
					"common_name": "r2.shared.global.fastly.net",
					"distinguished_name": "CN=r2.shared.global.fastly.net,O=Fastly\\, Inc.,L=San Francisco,ST=California,C=US"
				}
			}
		}
	}
}
```

## How to test this PR locally

Run against TLS/Non-TLS endpoints

(cherry picked from commit eb2dc26)
  • Loading branch information
andrewvc committed Apr 27, 2020
1 parent 3d7f736 commit 54b9892
Show file tree
Hide file tree
Showing 22 changed files with 1,155 additions and 242 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Heartbeat*

- Allow a list of status codes for HTTP checks. {pull}15587[15587]

- Add additional ECS compatible fields for TLS information. {pull}17687[17687]

*Heartbeat*

Expand Down
8 changes: 8 additions & 0 deletions heartbeat/_meta/fields.common.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,19 @@
type: keyword
description: >
The monitors configured name
multi_fields:
- name: text
type: text
analyzer: simple

- name: id
type: keyword
description: >
The monitors full job ID as used by heartbeat.
multi_fields:
- name: text
type: text
analyzer: simple

- name: duration
type: group
Expand Down
201 changes: 199 additions & 2 deletions heartbeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,13 @@ type: keyword
--
*`monitor.name.text`*::
+
--
type: text
--
*`monitor.id`*::
+
--
Expand All @@ -225,6 +232,13 @@ type: keyword
--
*`monitor.id.text`*::
+
--
type: text
--
[float]
=== duration
Expand Down Expand Up @@ -7824,7 +7838,10 @@ TLS layer related fields.
*`tls.certificate_not_valid_before`*::
+
--
Earliest time at which the connection's certificates are valid.
deprecated:[7.8.0]
Deprecated in favor of `tls.server.x509.not_before`. Earliest time at which the connection's certificates are valid.
type: date
Expand All @@ -7833,7 +7850,10 @@ type: date
*`tls.certificate_not_valid_after`*::
+
--
Latest time at which the connection's certificates are valid.
deprecated:[7.8.0]
Deprecated in favor of `tls.server.x509.not_after`. Latest time at which the connection's certificates are valid.
type: date
Expand Down Expand Up @@ -7862,3 +7882,180 @@ type: long
--
[float]
=== server
Detailed x509 certificate metadata
*`tls.server.x509.alternative_names`*::
+
--
List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses.
type: keyword
example: *.elastic.co
--
*`tls.server.x509.issuer.common_name`*::
+
--
List of common name (CN) of issuing certificate authority.
type: keyword
example: DigiCert SHA2 High Assurance Server CA
--
*`tls.server.x509.issuer.common_name.text`*::
+
--
type: wildcard
--
*`tls.server.x509.issuer.distinguished_name`*::
+
--
Distinguished name (DN) of issuing certificate authority.
type: keyword
example: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 High Assurance Server CA
--
*`tls.server.x509.not_after`*::
+
--
Time at which the certificate is no longer considered valid.
type: date
example: 2020-07-16 03:15:39
--
*`tls.server.x509.not_before`*::
+
--
Time at which the certificate is first considered valid.
type: date
example: 2019-08-16 01:40:25
--
*`tls.server.x509.public_key_algorithm`*::
+
--
Algorithm used to generate the public key.
type: keyword
example: RSA
--
*`tls.server.x509.public_key_curve`*::
+
--
The curve used by the elliptic curve public key algorithm. This is algorithm specific.
type: keyword
example: nistp521
--
*`tls.server.x509.public_key_exponent`*::
+
--
Exponent used to derive the public key. This is algorithm specific.
type: long
example: 65537
--
*`tls.server.x509.public_key_size`*::
+
--
The size of the public key space in bits.
type: long
example: 2048
--
*`tls.server.x509.serial_number`*::
+
--
Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters.
type: keyword
example: 55FBB9C7DEBF09809D12CCAA
--
*`tls.server.x509.signature_algorithm`*::
+
--
Identifier for certificate signature algorithm. Recommend using names found in Go Lang Crypto library (See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353).
type: keyword
example: SHA256-RSA
--
*`tls.server.x509.subject.subject.common_name`*::
+
--
List of common names (CN) of subject.
type: keyword
example: r2.shared.global.fastly.net
--
*`tls.server.x509.subject.subject.common_name.text`*::
+
--
type: wildcard
--
*`tls.server.x509.subject.subject.distinguished_name`*::
+
--
Distinguished name (DN) of the certificate subject entity.
type: keyword
example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net
--
*`tls.server.x509.version_number`*::
+
--
Version of x509 format.
type: keyword
example: 3
--
60 changes: 53 additions & 7 deletions heartbeat/hbtest/hbtestutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,23 @@
package hbtest

import (
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
"testing"
"time"

"github.com/elastic/beats/v7/heartbeat/monitors/active/dialchain/tlsmeta"
"github.com/elastic/beats/v7/libbeat/common"

"github.com/elastic/beats/v7/heartbeat/hbtestllext"

Expand Down Expand Up @@ -107,13 +113,25 @@ func ServerPort(server *httptest.Server) (uint16, error) {

// TLSChecks validates the given x509 cert at the given position.
func TLSChecks(chainIndex, certIndex int, certificate *x509.Certificate) validator.Validator {
return lookslike.MustCompile(map[string]interface{}{
"tls": map[string]interface{}{
"rtt.handshake.us": isdef.IsDuration,
"certificate_not_valid_before": certificate.NotBefore,
"certificate_not_valid_after": certificate.NotAfter,
},
})
expected := common.MapStr{}
// This function is well tested independently, so we just test that things match up here.
tlsmeta.AddTLSMetadata(expected, tls.ConnectionState{
Version: tls.VersionTLS13,
HandshakeComplete: true,
CipherSuite: tls.TLS_AES_128_GCM_SHA256,
ServerName: certificate.Subject.CommonName,
PeerCertificates: []*x509.Certificate{certificate},
}, time.Duration(1))

expected.Put("tls.rtt.handshake.us", isdef.IsDuration)

return lookslike.MustCompile(expected)
}

func TLSCertChecks(certificate *x509.Certificate) validator.Validator {
expected := common.MapStr{}
tlsmeta.AddCertMetadata(expected, []*x509.Certificate{certificate})
return lookslike.MustCompile(expected)
}

// BaseChecks creates a skima.Validator that represents the "monitor" field present
Expand Down Expand Up @@ -196,6 +214,14 @@ func ErrorChecks(msgSubstr string, errType string) validator.Validator {
})
}

func ExpiredCertChecks(cert *x509.Certificate) validator.Validator {
msg := x509.CertificateInvalidError{Cert: cert, Reason: x509.Expired}.Error()
return lookslike.Compose(
ErrorChecks(msg, "io"),
TLSCertChecks(cert),
)
}

// RespondingTCPChecks creates a skima.Validator that represents the "tcp" field present
// in all heartbeat events that use a Tcp connection as part of their DialChain
func RespondingTCPChecks() validator.Validator {
Expand All @@ -215,3 +241,23 @@ func CertToTempFile(t *testing.T, cert *x509.Certificate) *os.File {
certFile.WriteString(x509util.CertToPEMString(cert))
return certFile
}

func StartHTTPSServer(t *testing.T, tlsCert tls.Certificate) (host string, port string, cert *x509.Certificate, doClose func() error) {
cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
require.NoError(t, err)

// No need to start a real server, since this is invalid, we just
l, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
Certificates: []tls.Certificate{tlsCert},
})
require.NoError(t, err)

srv := &http.Server{Handler: HelloWorldHandler(200)}
go func() {
srv.Serve(l)
}()

host, port, err = net.SplitHostPort(l.Addr().String())
require.NoError(t, err)
return host, port, cert, srv.Close
}
2 changes: 1 addition & 1 deletion heartbeat/include/fields.go

Large diffs are not rendered by default.

Loading

0 comments on commit 54b9892

Please sign in to comment.