From 3ed5e3f54cf80ea9914da38a5eccb50dd13df75c Mon Sep 17 00:00:00 2001 From: urso Date: Wed, 27 Jun 2018 18:26:47 +0200 Subject: [PATCH 1/3] Enable/Disable html escaping in json strings Add support to codecs and outputs to enable/disable escaping of html symbols in JSON strings. By default html escaping is enabled. --- CHANGELOG-developer.asciidoc | 1 + CHANGELOG.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 39 +++++++++++- filebeat/filebeat.reference.yml | 39 +++++++++++- heartbeat/heartbeat.reference.yml | 39 +++++++++++- libbeat/_meta/config.reference.yml | 39 +++++++++++- libbeat/docs/outputconfig.asciidoc | 16 +++++ libbeat/outputs/codec/json/json.go | 25 +++++--- libbeat/outputs/codec/json/json_test.go | 62 ++++++++++++------- libbeat/outputs/console/console.go | 2 +- libbeat/outputs/console/console_test.go | 4 +- libbeat/outputs/elasticsearch/client.go | 5 +- libbeat/outputs/elasticsearch/config.go | 2 + .../outputs/elasticsearch/elasticsearch.go | 1 + libbeat/outputs/elasticsearch/enc.go | 16 +++-- libbeat/outputs/elasticsearch/enc_test.go | 4 +- libbeat/outputs/logstash/async.go | 2 +- libbeat/outputs/logstash/config.go | 2 + libbeat/outputs/logstash/enc.go | 4 +- libbeat/outputs/logstash/sync.go | 2 +- metricbeat/metricbeat.reference.yml | 39 +++++++++++- packetbeat/packetbeat.reference.yml | 39 +++++++++++- winlogbeat/winlogbeat.reference.yml | 39 +++++++++++- 23 files changed, 361 insertions(+), 61 deletions(-) diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index a4c10b360ae..1f0e0f02e28 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -23,6 +23,7 @@ The list below covers the major changes between 6.3.0 and master only. - Moving of TLS helper functions and structs from `output/tls` to `tlscommon`. {pull}7054[7054] - Port fields.yml collector to Golang {pull}6911[6911] - Dashboards under _meta/kibana are expected to be decoded. See https://github.com/elastic/beats/pull/7224 for a conversion script. {pull}7265[7265] +- Constructor `(github.com/elastic/beats/libbeat/output/codec/json).New` expects a new `escapeHTML` parameter. {pull}7445[7445] ==== Bugfixes diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 2579f4fba74..75c7b9659a9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -166,6 +166,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Add support for docker autodiscover to monitor containers on host network {pull}6708[6708] - Add ability to define input configuration as stringified JSON for autodiscover. {pull}7372[7372] - Add processor definition support for hints builder {pull}7386[7386] +- Add support to disable html escaping in outputs. {pull}7445[7445] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index dfc1bbb5ad8..c0e0039a71a 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -335,6 +335,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -440,6 +443,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 3 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -554,6 +560,14 @@ output.elasticsearch: # Kafka version auditbeat is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. #metadata: @@ -656,6 +670,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -757,6 +779,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/auditbeat" @@ -784,8 +814,13 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 029dc15f273..493c09751b6 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -977,6 +977,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -1082,6 +1085,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 3 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -1196,6 +1202,14 @@ output.elasticsearch: # Kafka version filebeat is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. #metadata: @@ -1298,6 +1312,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -1399,6 +1421,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/filebeat" @@ -1426,8 +1456,13 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index d1fab5a0908..c339f68beb3 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -444,6 +444,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -549,6 +552,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 3 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -663,6 +669,14 @@ output.elasticsearch: # Kafka version heartbeat is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. #metadata: @@ -765,6 +779,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -866,6 +888,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/heartbeat" @@ -893,8 +923,13 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 05c46b3e8b6..da9bbf32c14 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -230,6 +230,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -335,6 +338,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 3 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -449,6 +455,14 @@ output.elasticsearch: # Kafka version beatname is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. #metadata: @@ -551,6 +565,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -652,6 +674,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/beatname" @@ -679,8 +709,13 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 2436f4256e0..dad0e071094 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -138,6 +138,13 @@ Increasing the compression level will reduce the network usage but will increase The default value is 0. +===== `escape_html` + +Configure escaping of HTML in strings. Set to `false` to disable escaping. + +The default value is `true`. + + ===== `worker` The number of workers per configured host publishing events to Elasticsearch. This @@ -466,6 +473,12 @@ Increasing the compression level will reduce the network usage but will increase The default value is 3. +===== `escape_html` + +Configure escaping of HTML in strings. Set to `false` to disable escaping. + +The default value is `true`. + ===== `worker` The number of workers per configured host publishing events to Logstash. This @@ -1187,6 +1200,8 @@ codec. By default the `json` codec is used. *`json.pretty`*: If `pretty` is set to true, events will be nicely formatted. The default is false. +*`json.escape_html`*: If `escape_html` is set to false, html symbols will not be escaped in strings. The default is true. + Example configuration that uses the `json` codec with pretty printing enabled to write events to the console: [source,yaml] @@ -1194,6 +1209,7 @@ Example configuration that uses the `json` codec with pretty printing enabled to output.console: codec.json: pretty: true + escape_html: false ------------------------------------------------------------------------------ *`format.string`*: Configurable format string used to create a custom formatted message. diff --git a/libbeat/outputs/codec/json/json.go b/libbeat/outputs/codec/json/json.go index 113252ac950..15913f51423 100644 --- a/libbeat/outputs/codec/json/json.go +++ b/libbeat/outputs/codec/json/json.go @@ -31,18 +31,21 @@ import ( // Encoder for serializing a beat.Event to json. type Encoder struct { - buf bytes.Buffer - folder *gotype.Iterator - pretty bool + buf bytes.Buffer + folder *gotype.Iterator + version string + config config } type config struct { - Pretty bool + Pretty bool + EscapeHTML bool } var defaultConfig = config{ - Pretty: false, + Pretty: false, + EscapeHTML: true, } func init() { @@ -54,19 +57,23 @@ func init() { } } - return New(config.Pretty, info.Version), nil + return New(config.Pretty, config.EscapeHTML, info.Version), nil }) } // New creates a new json Encoder. -func New(pretty bool, version string) *Encoder { - e := &Encoder{pretty: pretty, version: version} +func New(pretty, escapeHTML bool, version string) *Encoder { + e := &Encoder{version: version, config: config{ + Pretty: pretty, + EscapeHTML: escapeHTML, + }} e.reset() return e } func (e *Encoder) reset() { visitor := json.NewVisitor(&e.buf) + visitor.SetEscapeHTML(e.config.EscapeHTML) var err error @@ -93,7 +100,7 @@ func (e *Encoder) Encode(index string, event *beat.Event) ([]byte, error) { } json := e.buf.Bytes() - if !e.pretty { + if !e.config.Pretty { return json, nil } diff --git a/libbeat/outputs/codec/json/json_test.go b/libbeat/outputs/codec/json/json_test.go index e3fef1e18e4..3e2abcfd184 100644 --- a/libbeat/outputs/codec/json/json_test.go +++ b/libbeat/outputs/codec/json/json_test.go @@ -25,22 +25,22 @@ import ( ) func TestJsonCodec(t *testing.T) { - expectedValue := `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"doc","version":"1.2.3"},"msg":"message"}` - - codec := New(false, "1.2.3") - output, err := codec.Encode("test", &beat.Event{Fields: common.MapStr{"msg": "message"}}) - - if err != nil { - t.Errorf("Error during event write %v", err) - } else { - if string(output) != expectedValue { - t.Errorf("Expected value (%s) does not equal with output (%s)", expectedValue, output) - } + type testCase struct { + config config + in common.MapStr + expected string } -} -func TestJsonWriterPrettyPrint(t *testing.T) { - expectedValue := `{ + cases := map[string]testCase{ + "default json": testCase{ + config: defaultConfig, + in: common.MapStr{"msg": "message"}, + expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"doc","version":"1.2.3"},"msg":"message"}`, + }, + "pretty enabled": testCase{ + config: config{Pretty: true}, + in: common.MapStr{"msg": "message"}, + expected: `{ "@timestamp": "0001-01-01T00:00:00.000Z", "@metadata": { "beat": "test", @@ -48,16 +48,32 @@ func TestJsonWriterPrettyPrint(t *testing.T) { "version": "1.2.3" }, "msg": "message" -}` +}`, + }, + "html escaping enabled": testCase{ + config: config{EscapeHTML: true}, + in: common.MapStr{"msg": "world"}, + expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"doc","version":"1.2.3"},"msg":"\u003chello\u003eworld\u003c/hello\u003e"}`, + }, + "html escaping disabled": testCase{ + config: config{EscapeHTML: false}, + in: common.MapStr{"msg": "world"}, + expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"doc","version":"1.2.3"},"msg":"world"}`, + }, + } + + for name, test := range cases { + cfg, fields, expected := test.config, test.in, test.expected - codec := New(true, "1.2.3") - output, err := codec.Encode("test", &beat.Event{Fields: common.MapStr{"msg": "message"}}) + t.Run(name, func(t *testing.T) { + codec := New(cfg.Pretty, cfg.EscapeHTML, "1.2.3") + actual, err := codec.Encode("test", &beat.Event{Fields: fields}) - if err != nil { - t.Errorf("Error during event write %v", err) - } else { - if string(output) != expectedValue { - t.Errorf("Expected value (%s) does not equal with output (%s)", expectedValue, output) - } + if err != nil { + t.Errorf("Error during event write %v", err) + } else if string(actual) != expected { + t.Errorf("Expected value (%s) does not equal with output (%s)", expected, actual) + } + }) } } diff --git a/libbeat/outputs/console/console.go b/libbeat/outputs/console/console.go index e5e7ba632ed..ab02e92aca7 100644 --- a/libbeat/outputs/console/console.go +++ b/libbeat/outputs/console/console.go @@ -70,7 +70,7 @@ func makeConsole( return outputs.Fail(err) } } else { - enc = json.New(config.Pretty, beat.Version) + enc = json.New(config.Pretty, true, beat.Version) } index := beat.Beat diff --git a/libbeat/outputs/console/console_test.go b/libbeat/outputs/console/console_test.go index f2c06a88947..3c165f950e7 100644 --- a/libbeat/outputs/console/console_test.go +++ b/libbeat/outputs/console/console_test.go @@ -78,7 +78,7 @@ func TestConsoleOutput(t *testing.T) { }{ { "single json event (pretty=false)", - json.New(false, "1.2.3"), + json.New(false, true, "1.2.3"), []beat.Event{ {Fields: event("field", "value")}, }, @@ -86,7 +86,7 @@ func TestConsoleOutput(t *testing.T) { }, { "single json event (pretty=true)", - json.New(true, "1.2.3"), + json.New(true, true, "1.2.3"), []beat.Event{ {Fields: event("field", "value")}, }, diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 37091683f58..9e7a3b0a680 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -66,6 +66,7 @@ type ClientSettings struct { Proxy *url.URL TLS *transport.TLSConfig Username, Password string + EscapeHTML bool Parameters map[string]string Headers map[string]string Index outil.Selector @@ -184,9 +185,9 @@ func NewClient( var encoder bodyEncoder compression := s.CompressionLevel if compression == 0 { - encoder = newJSONEncoder(nil) + encoder = newJSONEncoder(nil, s.EscapeHTML) } else { - encoder, err = newGzipEncoder(compression, nil) + encoder, err = newGzipEncoder(compression, nil, s.EscapeHTML) if err != nil { return nil, err } diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index e4ea7de1364..c398f8196cf 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -33,6 +33,7 @@ type elasticsearchConfig struct { ProxyURL string `config:"proxy_url"` LoadBalance bool `config:"loadbalance"` CompressionLevel int `config:"compression_level" validate:"min=0, max=9"` + EscapeHTML bool `config:"escape_html"` TLS *tlscommon.Config `config:"ssl"` BulkMaxSize int `config:"bulk_max_size"` MaxRetries int `config:"max_retries"` @@ -60,6 +61,7 @@ var ( Timeout: 90 * time.Second, MaxRetries: 3, CompressionLevel: 0, + EscapeHTML: true, TLS: nil, LoadBalance: true, Backoff: Backoff{ diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 673acecb014..dd50990a9ad 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -154,6 +154,7 @@ func makeES( Timeout: config.Timeout, CompressionLevel: config.CompressionLevel, Observer: observer, + EscapeHTML: config.EscapeHTML, }, &connectCallbackRegistry) if err != nil { return outputs.Fail(err) diff --git a/libbeat/outputs/elasticsearch/enc.go b/libbeat/outputs/elasticsearch/enc.go index 03288c4e9aa..c8e91a31f6f 100644 --- a/libbeat/outputs/elasticsearch/enc.go +++ b/libbeat/outputs/elasticsearch/enc.go @@ -53,12 +53,16 @@ type bulkWriter interface { type jsonEncoder struct { buf *bytes.Buffer folder *gotype.Iterator + + escapeHTML bool } type gzipEncoder struct { buf *bytes.Buffer gzip *gzip.Writer folder *gotype.Iterator + + escapeHTML bool } type event struct { @@ -66,11 +70,11 @@ type event struct { Fields common.MapStr `struct:",inline"` } -func newJSONEncoder(buf *bytes.Buffer) *jsonEncoder { +func newJSONEncoder(buf *bytes.Buffer, escapeHTML bool) *jsonEncoder { if buf == nil { buf = bytes.NewBuffer(nil) } - e := &jsonEncoder{buf: buf} + e := &jsonEncoder{buf: buf, escapeHTML: escapeHTML} e.resetState() return e } @@ -82,6 +86,8 @@ func (b *jsonEncoder) Reset() { func (b *jsonEncoder) resetState() { var err error visitor := json.NewVisitor(b.buf) + visitor.SetEscapeHTML(b.escapeHTML) + b.folder, err = gotype.NewIterator(visitor, gotype.Folders( codec.MakeTimestampEncoder(), @@ -137,7 +143,7 @@ func (b *jsonEncoder) Add(meta, obj interface{}) error { return nil } -func newGzipEncoder(level int, buf *bytes.Buffer) (*gzipEncoder, error) { +func newGzipEncoder(level int, buf *bytes.Buffer, escapeHTML bool) (*gzipEncoder, error) { if buf == nil { buf = bytes.NewBuffer(nil) } @@ -146,7 +152,7 @@ func newGzipEncoder(level int, buf *bytes.Buffer) (*gzipEncoder, error) { return nil, err } - g := &gzipEncoder{buf: buf, gzip: w} + g := &gzipEncoder{buf: buf, gzip: w, escapeHTML: escapeHTML} g.resetState() return g, nil } @@ -154,6 +160,8 @@ func newGzipEncoder(level int, buf *bytes.Buffer) (*gzipEncoder, error) { func (g *gzipEncoder) resetState() { var err error visitor := json.NewVisitor(g.gzip) + visitor.SetEscapeHTML(g.escapeHTML) + g.folder, err = gotype.NewIterator(visitor, gotype.Folders( codec.MakeTimestampEncoder(), diff --git a/libbeat/outputs/elasticsearch/enc_test.go b/libbeat/outputs/elasticsearch/enc_test.go index d0c66902131..135bdf0d5f5 100644 --- a/libbeat/outputs/elasticsearch/enc_test.go +++ b/libbeat/outputs/elasticsearch/enc_test.go @@ -29,7 +29,7 @@ import ( ) func TestJSONEncoderMarshalBeatEvent(t *testing.T) { - encoder := newJSONEncoder(nil) + encoder := newJSONEncoder(nil, true) event := beat.Event{ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC), Fields: common.MapStr{ @@ -46,7 +46,7 @@ func TestJSONEncoderMarshalBeatEvent(t *testing.T) { } func TestJSONEncoderMarshalMonitoringEvent(t *testing.T) { - encoder := newJSONEncoder(nil) + encoder := newJSONEncoder(nil, true) event := report.Event{ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC), Fields: common.MapStr{ diff --git a/libbeat/outputs/logstash/async.go b/libbeat/outputs/logstash/async.go index 1e2ad5ba1a1..2d080a50dd1 100644 --- a/libbeat/outputs/logstash/async.go +++ b/libbeat/outputs/logstash/async.go @@ -68,7 +68,7 @@ func newAsyncClient( logp.Warn(`The async Logstash client does not support the "ttl" option`) } - enc := makeLogstashEventEncoder(beat, config.Index) + enc := makeLogstashEventEncoder(beat, config.EscapeHTML, config.Index) queueSize := config.Pipelining - 1 timeout := config.Timeout diff --git a/libbeat/outputs/logstash/config.go b/libbeat/outputs/logstash/config.go index acbf51a7ccc..36e2a618716 100644 --- a/libbeat/outputs/logstash/config.go +++ b/libbeat/outputs/logstash/config.go @@ -38,6 +38,7 @@ type Config struct { TLS *tlscommon.Config `config:"ssl"` Proxy transport.ProxyConfig `config:",inline"` Backoff Backoff `config:"backoff"` + EscapeHTML bool `config:"escape_html"` } type Backoff struct { @@ -59,6 +60,7 @@ var defaultConfig = Config{ Init: 1 * time.Second, Max: 60 * time.Second, }, + EscapeHTML: true, } func newConfig() *Config { diff --git a/libbeat/outputs/logstash/enc.go b/libbeat/outputs/logstash/enc.go index 60004f1904a..00cb304c646 100644 --- a/libbeat/outputs/logstash/enc.go +++ b/libbeat/outputs/logstash/enc.go @@ -22,8 +22,8 @@ import ( "github.com/elastic/beats/libbeat/outputs/codec/json" ) -func makeLogstashEventEncoder(info beat.Info, index string) func(interface{}) ([]byte, error) { - enc := json.New(false, info.Version) +func makeLogstashEventEncoder(info beat.Info, escapeHTML bool, index string) func(interface{}) ([]byte, error) { + enc := json.New(false, escapeHTML, info.Version) return func(event interface{}) ([]byte, error) { return enc.Encode(index, event.(*beat.Event)) } diff --git a/libbeat/outputs/logstash/sync.go b/libbeat/outputs/logstash/sync.go index 53fad295e21..534aebf60e8 100644 --- a/libbeat/outputs/logstash/sync.go +++ b/libbeat/outputs/logstash/sync.go @@ -57,7 +57,7 @@ func newSyncClient( } var err error - enc := makeLogstashEventEncoder(beat, config.Index) + enc := makeLogstashEventEncoder(beat, config.EscapeHTML, config.Index) c.client, err = v2.NewSyncClientWithConn(conn, v2.JSONEncoder(enc), v2.Timeout(config.Timeout), diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 44524a94faf..c95dde25c1b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -850,6 +850,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -955,6 +958,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 3 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -1069,6 +1075,14 @@ output.elasticsearch: # Kafka version metricbeat is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. #metadata: @@ -1171,6 +1185,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -1272,6 +1294,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/metricbeat" @@ -1299,8 +1329,13 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index e5552f3a4d8..7ba589055a6 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -707,6 +707,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -812,6 +815,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 3 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -926,6 +932,14 @@ output.elasticsearch: # Kafka version packetbeat is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. #metadata: @@ -1028,6 +1042,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -1129,6 +1151,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/packetbeat" @@ -1156,8 +1186,13 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 81c7f26e3ef..250f462c170 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -259,6 +259,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -364,6 +367,9 @@ output.elasticsearch: # Set gzip compression level. #compression_level: 3 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -478,6 +484,14 @@ output.elasticsearch: # Kafka version winlogbeat is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. #metadata: @@ -580,6 +594,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -681,6 +703,14 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/winlogbeat" @@ -708,8 +738,13 @@ output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== From 6a2cac2e33f0b13341ffc80539c8f9ff5c169284 Mon Sep 17 00:00:00 2001 From: urso Date: Wed, 27 Jun 2018 19:16:32 +0200 Subject: [PATCH 2/3] Disable html escaping in pipeline debug processor --- libbeat/publisher/pipeline/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index 2f0001700c6..e2dcb7c73c0 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -295,7 +295,7 @@ func debugPrintProcessor(info beat.Info) *processorFn { // beat.Client is shared between multiple go-routines by accident) var mux sync.Mutex - encoder := json.New(true, info.Version) + encoder := json.New(true, false, info.Version) return newProcessor("debugPrint", func(event *beat.Event) (*beat.Event, error) { mux.Lock() defer mux.Unlock() From e318c8254bcb3bf9cbe3cd0330799dc39076335b Mon Sep 17 00:00:00 2001 From: urso Date: Wed, 27 Jun 2018 19:27:43 +0200 Subject: [PATCH 3/3] Fix metricbeat test build --- metricbeat/mb/module/example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/mb/module/example_test.go b/metricbeat/mb/module/example_test.go index e80b5a74b93..0f84ba8bdc8 100644 --- a/metricbeat/mb/module/example_test.go +++ b/metricbeat/mb/module/example_test.go @@ -147,7 +147,7 @@ func ExampleRunner() { } func encodeEvent(event beat.Event) (string, error) { - output, err := json.New(false, "1.2.3").Encode("noindex", &event) + output, err := json.New(false, true, "1.2.3").Encode("noindex", &event) if err != nil { return "", nil }