From c0dbd6a2301b5c1725ff7200f54d5d4f35672419 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 26 Jun 2019 15:20:06 -0400 Subject: [PATCH 01/50] Add kafka input to filebeat import list --- filebeat/include/list.go | 1 + 1 file changed, 1 insertion(+) diff --git a/filebeat/include/list.go b/filebeat/include/list.go index 6876c7a1a62..44996dead8f 100644 --- a/filebeat/include/list.go +++ b/filebeat/include/list.go @@ -23,6 +23,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/filebeat/input/container" _ "github.com/elastic/beats/filebeat/input/docker" + _ "github.com/elastic/beats/filebeat/input/kafka" _ "github.com/elastic/beats/filebeat/input/log" _ "github.com/elastic/beats/filebeat/input/redis" _ "github.com/elastic/beats/filebeat/input/stdin" From 878a9e7e2d98db54eaafb5a0e7ea1a8b13f4446c Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 8 Jul 2019 15:54:27 -0400 Subject: [PATCH 02/50] Initial skeleton of filebeat kafka input --- filebeat/input/kafka/config.go | 73 ++++ filebeat/input/kafka/input.go | 183 +++++++++ .../input/kafka/kafka_integration_test.go | 348 ++++++++++++++++++ 3 files changed, 604 insertions(+) create mode 100644 filebeat/input/kafka/config.go create mode 100644 filebeat/input/kafka/input.go create mode 100644 filebeat/input/kafka/kafka_integration_test.go diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go new file mode 100644 index 00000000000..467a0b23ac3 --- /dev/null +++ b/filebeat/input/kafka/config.go @@ -0,0 +1,73 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kafka + +import ( + "fmt" + + "github.com/Shopify/sarama" + "github.com/elastic/beats/libbeat/common/kafka" + "github.com/elastic/beats/libbeat/logp" +) + +var defaultConfig = kafkaInputConfig{ + Version: kafka.Version("1.0.0"), + GroupID: "FilebeatGroup", +} + +type kafkaInputConfig struct { + // Kafka hosts with port, e.g. "localhost:9092" + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + Version kafka.Version `config:"version"` + GroupID string `config:"group_id"` +} + +// Validate validates the config. +func (c *kafkaInputConfig) Validate() error { + + return nil +} + +func stringInSlice(str string, list []string) bool { + for _, v := range list { + if v == str { + return true + } + } + return false +} + +func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { + k := sarama.NewConfig() + + version, ok := config.Version.Get() + if !ok { + return nil, fmt.Errorf("Unknown/unsupported kafka version: %v", config.Version) + } + k.Version = version + + k.Consumer.Return.Errors = true + k.Consumer.Offsets.Initial = sarama.OffsetOldest + + if err := k.Validate(); err != nil { + logp.Err("Invalid kafka configuration: %v", err) + return nil, err + } + return k, nil +} diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go new file mode 100644 index 00000000000..7fa2677633b --- /dev/null +++ b/filebeat/input/kafka/input.go @@ -0,0 +1,183 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kafka + +import ( + "context" + "fmt" + "time" + + "github.com/Shopify/sarama" + "github.com/elastic/beats/filebeat/channel" + "github.com/elastic/beats/filebeat/input" + "github.com/elastic/beats/filebeat/util" + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + + "github.com/pkg/errors" +) + +func init() { + err := input.Register("kafka", NewInput) + if err != nil { + panic(err) + } +} + +// Input contains the input and its config +type Input struct { + config kafkaInputConfig + rawConfig *common.Config // The Config given to NewInput + started bool + outlet channel.Outleter + consumerGroup sarama.ConsumerGroup + goContext context.Context +} + +// NewInput creates a new kafka input +func NewInput( + cfg *common.Config, + outletFactory channel.Connector, + inputContext input.Context, +) (input.Input, error) { + + out, err := outletFactory(cfg, inputContext.DynamicFields) + if err != nil { + return nil, err + } + + //forwarder := harvester.NewForwarder(out) + + config := defaultConfig + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrap(err, "reading kafka input config") + } + + saramaConfig, err := newSaramaConfig(config) + if err != nil { + return nil, errors.Wrap(err, "initializing Sarama config") + } + consumerGroup, err := + sarama.NewConsumerGroup(config.Hosts, config.GroupID, saramaConfig) + if err != nil { + return nil, errors.Wrap(err, "initializing kafka consumer group") + } + + // Sarama uses standard go contexts to control cancellation, so we need to + // wrap our input context channel in that interface. + goContext, cancel := context.WithCancel(context.Background()) + go func() { + select { + case <-inputContext.Done: + logp.Info("Closing kafka context because input stopped.") + cancel() + return + } + }() + + input := &Input{ + config: config, + rawConfig: cfg, + started: false, + outlet: out, + consumerGroup: consumerGroup, + goContext: goContext, + } + + return input, nil +} + +func (p *Input) newConsumerGroup() (sarama.ConsumerGroup, error) { + consumerGroup, err := + sarama.NewConsumerGroup(p.config.Hosts, p.config.GroupID, nil) + return consumerGroup, err +} + +// Run starts the input by scanning for incoming messages and errors. +func (p *Input) Run() { + if !p.started { + // Track errors + go func() { + for err := range p.consumerGroup.Errors() { + // TODO: handle + fmt.Println("ERROR", err) + } + }() + + go func() { + for { + handler := groupHandler{input: p} + + err := p.consumerGroup.Consume(p.goContext, p.config.Topics, handler) + if err != nil { + fmt.Printf("Consume error: %v\n", err) + //panic(err) + // TODO: report error + } + } + }() + p.started = true + } +} + +func (p *Input) Wait() { +} + +func (p *Input) Stop() { +} + +type groupHandler struct { + input *Input +} + +func createEvent( + sess sarama.ConsumerGroupSession, + claim sarama.ConsumerGroupClaim, + message *sarama.ConsumerMessage, +) *util.Data { + data := util.NewData() + data.Event = beat.Event{ + Timestamp: time.Now(), + Fields: common.MapStr{ + "message": string(message.Value), + "kafka": common.MapStr{ + "topic": claim.Topic(), + "partition": claim.Partition(), + }, + // TODO: add more metadata + }, + } + return data +} + +func (groupHandler) Setup(session sarama.ConsumerGroupSession) error { + return nil +} +func (groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { + return nil +} +func (h groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + event := createEvent(sess, claim, msg) + fmt.Printf("event: %v\n", event) + h.input.outlet.OnEvent(event) + sess.MarkMessage(msg, "") + } + return nil +} diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go new file mode 100644 index 00000000000..c73e8778ae2 --- /dev/null +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -0,0 +1,348 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kafka + +import ( + "encoding/json" + "fmt" + "math/rand" + "os" + "strconv" + "sync" + "testing" + "time" + + "github.com/Shopify/sarama" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/filebeat/channel" + "github.com/elastic/beats/filebeat/input" + "github.com/elastic/beats/filebeat/util" + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/fmtstr" + _ "github.com/elastic/beats/libbeat/outputs/codec/format" + _ "github.com/elastic/beats/libbeat/outputs/codec/json" +) + +const ( + kafkaDefaultHost = "kafka" + kafkaDefaultPort = "9092" +) + +type eventInfo struct { + events []beat.Event +} + +type eventCapturer struct { + closed bool + c chan struct{} + closeOnce sync.Once + events chan *util.Data +} + +func NewEventCapturer(events chan *util.Data) channel.Outleter { + return &eventCapturer{ + c: make(chan struct{}), + events: events, + } +} + +func (o *eventCapturer) OnEvent(event *util.Data) bool { + o.events <- event + return true +} + +func (o *eventCapturer) Close() error { + o.closeOnce.Do(func() { + o.closed = true + close(o.c) + }) + return nil +} + +func (o *eventCapturer) Done() <-chan struct{} { + return o.c +} + +func TestInput(t *testing.T) { + id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int()) + testTopic := fmt.Sprintf("Filebeat-TestInput-%s", id) + context := input.Context{ + Done: make(chan struct{}), + BeatDone: make(chan struct{}), + } + + // Send test messages to the topic for the input to read. + messageStrs := []string{"testing", "stuff", "blah"} + for _, s := range messageStrs { + writeToKafkaTopic(t, testTopic, s, time.Second*20) + } + + // Setup the input config + config, _ := common.NewConfigFrom(common.MapStr{ + "hosts": "kafka:9092", + "topics": []string{testTopic}, + }) + + // Route input events through our capturer instead of sending through ES. + events := make(chan *util.Data, 100) + defer close(events) + capturer := NewEventCapturer(events) + defer capturer.Close() + connector := func(*common.Config, *common.MapStrPointer) (channel.Outleter, error) { + return channel.SubOutlet(capturer), nil + } + + input, err := NewInput(config, connector, context) + if err != nil { + t.Fatal(err) + } + + // Run the input and wait for finalization + input.Run() + + timeout := time.After(30 * time.Second) + done := make(chan struct{}) + for _, m := range messageStrs { + select { + case event := <-events: + result, err := event.GetEvent().Fields.GetValue("message") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, result, m) + if state := event.GetState(); state.Finished { + //assert.Equal(t, len(logs), int(state.Offset), "file has not been fully read") + go func() { + //closer(context, input.(*Input)) + close(done) + }() + } + case <-done: + return + case <-timeout: + t.Fatal("timeout waiting for closed state") + } + } +} + +func validateJSON(t *testing.T, value []byte, event beat.Event) { + var decoded map[string]interface{} + err := json.Unmarshal(value, &decoded) + if err != nil { + t.Errorf("can not json decode event value: %v", value) + return + } + assert.Equal(t, decoded["type"], event.Fields["type"]) + assert.Equal(t, decoded["message"], event.Fields["message"]) +} + +func makeValidateFmtStr(fmt string) func(*testing.T, []byte, beat.Event) { + fmtString := fmtstr.MustCompileEvent(fmt) + return func(t *testing.T, value []byte, event beat.Event) { + expectedMessage, err := fmtString.Run(&event) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, string(expectedMessage), string(value)) + } +} + +func strDefault(a, defaults string) string { + if len(a) == 0 { + return defaults + } + return a +} + +func getenv(name, defaultValue string) string { + return strDefault(os.Getenv(name), defaultValue) +} + +func getTestKafkaHost() string { + return fmt.Sprintf("%v:%v", + getenv("KAFKA_HOST", kafkaDefaultHost), + getenv("KAFKA_PORT", kafkaDefaultPort), + ) +} + +func writeToKafkaTopic( + t *testing.T, topic string, message string, timeout time.Duration, +) { + config := sarama.NewConfig() + config.Producer.RequiredAcks = sarama.WaitForAll + config.Producer.Return.Successes = true + config.Producer.Partitioner = sarama.NewHashPartitioner + + hosts := []string{getTestKafkaHost()} + producer, err := sarama.NewSyncProducer(hosts, config) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := producer.Close(); err != nil { + t.Fatal(err) + } + }() + + msg := &sarama.ProducerMessage{ + Topic: topic, + Value: sarama.StringEncoder(message), + } + + _, _, err = producer.SendMessage(msg) + if err != nil { + t.Fatal(err) + } +} + +func makeConfig(t *testing.T, in map[string]interface{}) *common.Config { + cfg, err := common.NewConfigFrom(in) + if err != nil { + t.Fatal(err) + } + return cfg +} + +func newTestConsumer(t *testing.T) sarama.Consumer { + hosts := []string{getTestKafkaHost()} + consumer, err := sarama.NewConsumer(hosts, nil) + if err != nil { + t.Fatal(err) + } + return consumer +} + +var testTopicOffsets = map[string]int64{} + +func testReadFromKafkaTopic( + t *testing.T, topic string, nMessages int, + timeout time.Duration, +) []*sarama.ConsumerMessage { + consumer := newTestConsumer(t) + defer func() { + consumer.Close() + }() + + offset, found := testTopicOffsets[topic] + if !found { + offset = sarama.OffsetOldest + } + + partitionConsumer, err := consumer.ConsumePartition(topic, 0, offset) + if err != nil { + t.Fatal(err) + } + defer func() { + partitionConsumer.Close() + }() + + timer := time.After(timeout) + var messages []*sarama.ConsumerMessage + for i := 0; i < nMessages; i++ { + select { + case msg := <-partitionConsumer.Messages(): + messages = append(messages, msg) + testTopicOffsets[topic] = msg.Offset + 1 + case <-timer: + break + } + } + + return messages +} + +func flatten(infos []eventInfo) []beat.Event { + var out []beat.Event + for _, info := range infos { + out = append(out, info.events...) + } + return out +} + +func single(fields common.MapStr) []eventInfo { + return []eventInfo{ + { + events: []beat.Event{ + {Timestamp: time.Now(), Fields: fields}, + }, + }, + } +} + +func randMulti(batches, n int, event common.MapStr) []eventInfo { + var out []eventInfo + for i := 0; i < batches; i++ { + var data []beat.Event + for j := 0; j < n; j++ { + tmp := common.MapStr{} + for k, v := range event { + tmp[k] = v + } + //tmp["message"] = randString(100) + data = append(data, beat.Event{Timestamp: time.Now(), Fields: tmp}) + } + + out = append(out, eventInfo{data}) + } + return out +} + +func setupInput(t *testing.T, context input.Context, closer func(input.Context, *Input)) { + // Setup the input + config, _ := common.NewConfigFrom(common.MapStr{ + "host": "localhost:9092", + }) + + events := make(chan *util.Data, 100) + defer close(events) + capturer := NewEventCapturer(events) + defer capturer.Close() + connector := func(*common.Config, *common.MapStrPointer) (channel.Outleter, error) { + return channel.SubOutlet(capturer), nil + } + + input, err := NewInput(config, connector, context) + if err != nil { + t.Error(err) + return + } + + // Run the input and wait for finalization + input.Run() + + timeout := time.After(30 * time.Second) + done := make(chan struct{}) + for { + select { + case event := <-events: + if state := event.GetState(); state.Finished { + //assert.Equal(t, len(logs), int(state.Offset), "file has not been fully read") + go func() { + closer(context, input.(*Input)) + close(done) + }() + } + case <-done: + return + case <-timeout: + t.Fatal("timeout waiting for closed state") + } + } +} From 5f6c0d95fb4b8b6ff52e000db740193130fe73ab Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 10 Jul 2019 11:08:35 -0400 Subject: [PATCH 03/50] Cleanup --- filebeat/input/kafka/input.go | 2 + .../input/kafka/kafka_integration_test.go | 147 +----------------- 2 files changed, 3 insertions(+), 146 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 7fa2677633b..9230e7d8f91 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -159,6 +159,8 @@ func createEvent( "kafka": common.MapStr{ "topic": claim.Topic(), "partition": claim.Partition(), + "offset": message.Offset, + //message.Timestamp }, // TODO: add more metadata }, diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index c73e8778ae2..1e4299ae68e 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -118,7 +118,6 @@ func TestInput(t *testing.T) { input.Run() timeout := time.After(30 * time.Second) - done := make(chan struct{}) for _, m := range messageStrs { select { case event := <-events: @@ -127,17 +126,8 @@ func TestInput(t *testing.T) { t.Fatal(err) } assert.Equal(t, result, m) - if state := event.GetState(); state.Finished { - //assert.Equal(t, len(logs), int(state.Offset), "file has not been fully read") - go func() { - //closer(context, input.(*Input)) - close(done) - }() - } - case <-done: - return case <-timeout: - t.Fatal("timeout waiting for closed state") + t.Fatal("timeout waiting for incoming events") } } } @@ -211,138 +201,3 @@ func writeToKafkaTopic( t.Fatal(err) } } - -func makeConfig(t *testing.T, in map[string]interface{}) *common.Config { - cfg, err := common.NewConfigFrom(in) - if err != nil { - t.Fatal(err) - } - return cfg -} - -func newTestConsumer(t *testing.T) sarama.Consumer { - hosts := []string{getTestKafkaHost()} - consumer, err := sarama.NewConsumer(hosts, nil) - if err != nil { - t.Fatal(err) - } - return consumer -} - -var testTopicOffsets = map[string]int64{} - -func testReadFromKafkaTopic( - t *testing.T, topic string, nMessages int, - timeout time.Duration, -) []*sarama.ConsumerMessage { - consumer := newTestConsumer(t) - defer func() { - consumer.Close() - }() - - offset, found := testTopicOffsets[topic] - if !found { - offset = sarama.OffsetOldest - } - - partitionConsumer, err := consumer.ConsumePartition(topic, 0, offset) - if err != nil { - t.Fatal(err) - } - defer func() { - partitionConsumer.Close() - }() - - timer := time.After(timeout) - var messages []*sarama.ConsumerMessage - for i := 0; i < nMessages; i++ { - select { - case msg := <-partitionConsumer.Messages(): - messages = append(messages, msg) - testTopicOffsets[topic] = msg.Offset + 1 - case <-timer: - break - } - } - - return messages -} - -func flatten(infos []eventInfo) []beat.Event { - var out []beat.Event - for _, info := range infos { - out = append(out, info.events...) - } - return out -} - -func single(fields common.MapStr) []eventInfo { - return []eventInfo{ - { - events: []beat.Event{ - {Timestamp: time.Now(), Fields: fields}, - }, - }, - } -} - -func randMulti(batches, n int, event common.MapStr) []eventInfo { - var out []eventInfo - for i := 0; i < batches; i++ { - var data []beat.Event - for j := 0; j < n; j++ { - tmp := common.MapStr{} - for k, v := range event { - tmp[k] = v - } - //tmp["message"] = randString(100) - data = append(data, beat.Event{Timestamp: time.Now(), Fields: tmp}) - } - - out = append(out, eventInfo{data}) - } - return out -} - -func setupInput(t *testing.T, context input.Context, closer func(input.Context, *Input)) { - // Setup the input - config, _ := common.NewConfigFrom(common.MapStr{ - "host": "localhost:9092", - }) - - events := make(chan *util.Data, 100) - defer close(events) - capturer := NewEventCapturer(events) - defer capturer.Close() - connector := func(*common.Config, *common.MapStrPointer) (channel.Outleter, error) { - return channel.SubOutlet(capturer), nil - } - - input, err := NewInput(config, connector, context) - if err != nil { - t.Error(err) - return - } - - // Run the input and wait for finalization - input.Run() - - timeout := time.After(30 * time.Second) - done := make(chan struct{}) - for { - select { - case event := <-events: - if state := event.GetState(); state.Finished { - //assert.Equal(t, len(logs), int(state.Offset), "file has not been fully read") - go func() { - closer(context, input.(*Input)) - close(done) - }() - } - case <-done: - return - case <-timeout: - t.Fatal("timeout waiting for closed state") - } - } -} From 1c83d553fa02c5c8ead2ce4625c8b49ec5dcb6ff Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 10 Jul 2019 11:54:54 -0400 Subject: [PATCH 04/50] Turn on Wait() and Stop() --- filebeat/input/kafka/input.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 9230e7d8f91..b988e2268da 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -47,7 +47,8 @@ type Input struct { started bool outlet channel.Outleter consumerGroup sarama.ConsumerGroup - goContext context.Context + kafkaContext context.Context + kafkaCancel context.CancelFunc // The CancelFunc for kafkaContext } // NewInput creates a new kafka input @@ -81,12 +82,12 @@ func NewInput( // Sarama uses standard go contexts to control cancellation, so we need to // wrap our input context channel in that interface. - goContext, cancel := context.WithCancel(context.Background()) + kafkaContext, kafkaCancel := context.WithCancel(context.Background()) go func() { select { case <-inputContext.Done: logp.Info("Closing kafka context because input stopped.") - cancel() + kafkaCancel() return } }() @@ -97,7 +98,8 @@ func NewInput( started: false, outlet: out, consumerGroup: consumerGroup, - goContext: goContext, + kafkaContext: kafkaContext, + kafkaCancel: kafkaCancel, } return input, nil @@ -124,7 +126,7 @@ func (p *Input) Run() { for { handler := groupHandler{input: p} - err := p.consumerGroup.Consume(p.goContext, p.config.Topics, handler) + err := p.consumerGroup.Consume(p.kafkaContext, p.config.Topics, handler) if err != nil { fmt.Printf("Consume error: %v\n", err) //panic(err) @@ -136,10 +138,14 @@ func (p *Input) Run() { } } +// Wait shuts down the Input by cancelling the internal context. func (p *Input) Wait() { + p.Stop() } +// Stop shuts down the Input by cancelling the internal context. func (p *Input) Stop() { + p.kafkaCancel() } type groupHandler struct { @@ -171,9 +177,11 @@ func createEvent( func (groupHandler) Setup(session sarama.ConsumerGroupSession) error { return nil } + func (groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } + func (h groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { for msg := range claim.Messages() { event := createEvent(sess, claim, msg) From 7298e9cc2b74d42e68772e4d984ecde1358f4583 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 10 Jul 2019 12:47:04 -0400 Subject: [PATCH 05/50] add InitialOffset configuration parameter --- filebeat/input/kafka/config.go | 53 +++++++++++++++---- .../input/kafka/kafka_integration_test.go | 5 +- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 467a0b23ac3..0ef176a9201 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -25,22 +25,43 @@ import ( "github.com/elastic/beats/libbeat/logp" ) -var defaultConfig = kafkaInputConfig{ - Version: kafka.Version("1.0.0"), - GroupID: "FilebeatGroup", -} +type initialOffset int + +const ( + initialOffsetOldest initialOffset = iota + initialOffsetNewest +) + +var ( + defaultConfig = kafkaInputConfig{ + Version: kafka.Version("1.0.0"), + InitialOffset: initialOffsetOldest, + } + + initialOffsets = map[string]initialOffset{ + "oldest": initialOffsetOldest, + "newest": initialOffsetNewest, + } +) type kafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - Version kafka.Version `config:"version"` - GroupID string `config:"group_id"` + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` +} + +func (off initialOffset) asSaramaOffset() int64 { + return map[initialOffset]int64{ + initialOffsetOldest: sarama.OffsetOldest, + initialOffsetNewest: sarama.OffsetNewest, + }[off] } // Validate validates the config. func (c *kafkaInputConfig) Validate() error { - return nil } @@ -63,7 +84,7 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k.Version = version k.Consumer.Return.Errors = true - k.Consumer.Offsets.Initial = sarama.OffsetOldest + k.Consumer.Offsets.Initial = config.InitialOffset.asSaramaOffset() if err := k.Validate(); err != nil { logp.Err("Invalid kafka configuration: %v", err) @@ -71,3 +92,15 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { } return k, nil } + +// Unpack validates and unpack the "initial_offset" config option +func (off *initialOffset) Unpack(value string) error { + initialOffset, ok := initialOffsets[value] + if !ok { + return fmt.Errorf("invalid initialOffset '%s'", value) + } + + *off = initialOffset + + return nil +} diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 1e4299ae68e..33337c36f26 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -96,8 +96,9 @@ func TestInput(t *testing.T) { // Setup the input config config, _ := common.NewConfigFrom(common.MapStr{ - "hosts": "kafka:9092", - "topics": []string{testTopic}, + "hosts": "kafka:9092", + "topics": []string{testTopic}, + "group_id": "filebeat", }) // Route input events through our capturer instead of sending through ES. From 7345c187890ddf120fe4c87219d298e2f6189a99 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 10 Jul 2019 15:11:30 -0400 Subject: [PATCH 06/50] Document new kafka output fields --- filebeat/docs/fields.asciidoc | 38 +++++++++++++++++++++++++++ filebeat/input/kafka/_meta/fields.yml | 21 +++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 filebeat/input/kafka/_meta/fields.yml diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 8bb7b5b1ad6..5c19a95c8df 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -29,6 +29,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -6984,6 +6985,43 @@ type: text Message part of the trace. +-- + +[[exported-fields-kafka-input]] +== Kafka Input fields + +Kafka metadata added by the kafka input + + + +*`kafka.topic`*:: ++ +-- +type: keyword + +Kafka topic + + +-- + +*`kafka.partition`*:: ++ +-- +type: long + +Kafka partition number + + +-- + +*`kafka.offset`*:: ++ +-- +type: long + +Kafka offset of this message + + -- [[exported-fields-kibana]] diff --git a/filebeat/input/kafka/_meta/fields.yml b/filebeat/input/kafka/_meta/fields.yml new file mode 100644 index 00000000000..d877776bbad --- /dev/null +++ b/filebeat/input/kafka/_meta/fields.yml @@ -0,0 +1,21 @@ +- key: kafka-input + title: Kafka Input + description: > + Kafka metadata added by the kafka input + short_config: false + anchor: kafka-input + fields: + - name: kafka.topic + type: keyword + description: > + Kafka topic + + - name: kafka.partition + type: long + description: > + Kafka partition number + + - name: kafka.offset + type: long + description: > + Kafka offset of this message From b925014a389cadb03d13b234c0069af5e5e289b7 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 11 Jul 2019 10:57:20 -0400 Subject: [PATCH 07/50] Add username / password and ssl config --- filebeat/input/kafka/config.go | 42 ++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 0ef176a9201..ae23606ab85 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -18,11 +18,14 @@ package kafka import ( + "errors" "fmt" "github.com/Shopify/sarama" "github.com/elastic/beats/libbeat/common/kafka" + "github.com/elastic/beats/libbeat/common/transport/tlscommon" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/outputs" ) type initialOffset int @@ -46,11 +49,14 @@ var ( type kafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - GroupID string `config:"group_id" validate:"required"` - Version kafka.Version `config:"version"` - InitialOffset initialOffset `config:"initial_offset"` + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` } func (off initialOffset) asSaramaOffset() int64 { @@ -62,6 +68,17 @@ func (off initialOffset) asSaramaOffset() int64 { // Validate validates the config. func (c *kafkaInputConfig) Validate() error { + if len(c.Hosts) == 0 { + return errors.New("no hosts configured") + } + + if err := c.Version.Validate(); err != nil { + return err + } + + if c.Username != "" && c.Password == "" { + return fmt.Errorf("password must be set when username is configured") + } return nil } @@ -86,6 +103,21 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k.Consumer.Return.Errors = true k.Consumer.Offsets.Initial = config.InitialOffset.asSaramaOffset() + tls, err := outputs.LoadTLSConfig(config.TLS) + if err != nil { + return nil, err + } + if tls != nil { + k.Net.TLS.Enable = true + k.Net.TLS.Config = tls.BuildModuleConfig("") + } + + if config.Username != "" { + k.Net.SASL.Enable = true + k.Net.SASL.User = config.Username + k.Net.SASL.Password = config.Password + } + if err := k.Validate(); err != nil { logp.Err("Invalid kafka configuration: %v", err) return nil, err From 1f6ecc3fde6e57f5b8597b5aa526a7183455ff1c Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 11 Jul 2019 11:04:32 -0400 Subject: [PATCH 08/50] Add parameter for client id --- filebeat/input/kafka/config.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index ae23606ab85..983d319b9c7 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -39,6 +39,7 @@ var ( defaultConfig = kafkaInputConfig{ Version: kafka.Version("1.0.0"), InitialOffset: initialOffsetOldest, + ClientID: "filebeat", } initialOffsets = map[string]initialOffset{ @@ -52,6 +53,7 @@ type kafkaInputConfig struct { Hosts []string `config:"hosts" validate:"required"` Topics []string `config:"topics" validate:"required"` GroupID string `config:"group_id" validate:"required"` + ClientID string `config:"client_id"` Version kafka.Version `config:"version"` InitialOffset initialOffset `config:"initial_offset"` TLS *tlscommon.Config `config:"ssl"` @@ -59,13 +61,6 @@ type kafkaInputConfig struct { Password string `config:"password"` } -func (off initialOffset) asSaramaOffset() int64 { - return map[initialOffset]int64{ - initialOffsetOldest: sarama.OffsetOldest, - initialOffsetNewest: sarama.OffsetNewest, - }[off] -} - // Validate validates the config. func (c *kafkaInputConfig) Validate() error { if len(c.Hosts) == 0 { @@ -82,15 +77,6 @@ func (c *kafkaInputConfig) Validate() error { return nil } -func stringInSlice(str string, list []string) bool { - for _, v := range list { - if v == str { - return true - } - } - return false -} - func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k := sarama.NewConfig() @@ -118,6 +104,9 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k.Net.SASL.Password = config.Password } + // configure client ID + k.ClientID = config.ClientID + if err := k.Validate(); err != nil { logp.Err("Invalid kafka configuration: %v", err) return nil, err @@ -125,6 +114,15 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { return k, nil } +// asSaramaOffset converts an initialOffset enum to the corresponding +// sarama offset value. +func (off initialOffset) asSaramaOffset() int64 { + return map[initialOffset]int64{ + initialOffsetOldest: sarama.OffsetOldest, + initialOffsetNewest: sarama.OffsetNewest, + }[off] +} + // Unpack validates and unpack the "initial_offset" config option func (off *initialOffset) Unpack(value string) error { initialOffset, ok := initialOffsets[value] From 207bebec622406fc4e4ce9dc0cef36ec55b5b3d8 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 11 Jul 2019 11:28:01 -0400 Subject: [PATCH 09/50] Add metric registry to sarama reader --- filebeat/input/kafka/config.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 983d319b9c7..958988a8728 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -25,6 +25,8 @@ import ( "github.com/elastic/beats/libbeat/common/kafka" "github.com/elastic/beats/libbeat/common/transport/tlscommon" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/monitoring" + "github.com/elastic/beats/libbeat/monitoring/adapter" "github.com/elastic/beats/libbeat/outputs" ) @@ -107,6 +109,14 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { // configure client ID k.ClientID = config.ClientID + k.MetricRegistry = adapter.GetGoMetrics( + monitoring.Default, + "filebeat.inputs.kafka", + adapter.Rename("incoming-byte-rate", "bytes_read"), + adapter.Rename("outgoing-byte-rate", "bytes_write"), + adapter.GoMetricsNilify, + ) + if err := k.Validate(); err != nil { logp.Err("Invalid kafka configuration: %v", err) return nil, err From ec386a01500c99161a1c17df142d7bfd41ec33ca Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 11 Jul 2019 14:49:07 -0400 Subject: [PATCH 10/50] Log kafka errors --- filebeat/input/kafka/input.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index b988e2268da..a3ecca77267 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -49,6 +49,7 @@ type Input struct { consumerGroup sarama.ConsumerGroup kafkaContext context.Context kafkaCancel context.CancelFunc // The CancelFunc for kafkaContext + log *logp.Logger } // NewInput creates a new kafka input @@ -100,6 +101,7 @@ func NewInput( consumerGroup: consumerGroup, kafkaContext: kafkaContext, kafkaCancel: kafkaCancel, + log: logp.NewLogger("kafka input").With("hosts", config.Hosts), } return input, nil @@ -117,8 +119,7 @@ func (p *Input) Run() { // Track errors go func() { for err := range p.consumerGroup.Errors() { - // TODO: handle - fmt.Println("ERROR", err) + p.log.Errorw("Error reading from kafka", "error", err) } }() @@ -128,9 +129,7 @@ func (p *Input) Run() { err := p.consumerGroup.Consume(p.kafkaContext, p.config.Topics, handler) if err != nil { - fmt.Printf("Consume error: %v\n", err) - //panic(err) - // TODO: report error + p.log.Errorw("Kafka consume error", "error", err) } } }() From 0aaa71003e929e6ddfb5b778eb12c677883b40b9 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 16 Jul 2019 13:42:19 -0400 Subject: [PATCH 11/50] Add remaining kafka metadata fields --- filebeat/input/kafka/input.go | 46 ++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index a3ecca77267..837b4e17006 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -64,8 +64,6 @@ func NewInput( return nil, err } - //forwarder := harvester.NewForwarder(out) - config := defaultConfig if err := cfg.Unpack(&config); err != nil { return nil, errors.Wrap(err, "reading kafka input config") @@ -147,11 +145,22 @@ func (p *Input) Stop() { p.kafkaCancel() } +func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { + array := []interface{}{} + for _, header := range headers { + array = append(array, common.MapStr{ + "key": header.Key, + "value": header.Value, + }) + } + return array +} + type groupHandler struct { input *Input } -func createEvent( +func (h groupHandler) createEvent( sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim, message *sarama.ConsumerMessage, @@ -159,17 +168,26 @@ func createEvent( data := util.NewData() data.Event = beat.Event{ Timestamp: time.Now(), - Fields: common.MapStr{ - "message": string(message.Value), - "kafka": common.MapStr{ - "topic": claim.Topic(), - "partition": claim.Partition(), - "offset": message.Offset, - //message.Timestamp - }, - // TODO: add more metadata - }, } + eventFields := common.MapStr{ + "message": string(message.Value), + } + kafkaMetadata := common.MapStr{ + "topic": claim.Topic(), + "partition": claim.Partition(), + "offset": message.Offset, + "key": message.Key, + } + version, ok := h.input.config.Version.Get() + if ok && version.IsAtLeast(sarama.V0_10_0_0) { + data.Event.Timestamp = message.Timestamp + kafkaMetadata["block_timestamp"] = message.BlockTimestamp + } + if ok && version.IsAtLeast(sarama.V0_11_0_0) { + kafkaMetadata["headers"] = arrayForKafkaHeaders(message.Headers) + } + eventFields["kafka"] = kafkaMetadata + data.Event.Fields = eventFields return data } @@ -183,7 +201,7 @@ func (groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { func (h groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { for msg := range claim.Messages() { - event := createEvent(sess, claim, msg) + event := h.createEvent(sess, claim, msg) fmt.Printf("event: %v\n", event) h.input.outlet.OnEvent(event) sess.MarkMessage(msg, "") From 95e6bff3b92da847b9ac76e77acb2fb9fa7a772e Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 16 Jul 2019 13:47:32 -0400 Subject: [PATCH 12/50] document new metadata fields --- filebeat/input/kafka/_meta/fields.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/filebeat/input/kafka/_meta/fields.yml b/filebeat/input/kafka/_meta/fields.yml index d877776bbad..7d10108d558 100644 --- a/filebeat/input/kafka/_meta/fields.yml +++ b/filebeat/input/kafka/_meta/fields.yml @@ -19,3 +19,19 @@ type: long description: > Kafka offset of this message + + - name: kafka.key + type: keyword + description: > + Kafka key, corresponding to the Kafka value stored in the message + + - name: kafka.block_timestamp + type: date + description: > + Kafka outer (compressed) block timestamp + + - name: kafka.headers + type: array + description: > + The array of kafka headers, each an object containing subfields + "key" and "value". From 5bb711cdfb61d09af68b556e17a4f41dcad9a89d Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 16 Jul 2019 13:59:20 -0400 Subject: [PATCH 13/50] Adjust kafka producer version / test message in tests --- filebeat/input/kafka/kafka_integration_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 33337c36f26..86892e7c48c 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -180,6 +180,7 @@ func writeToKafkaTopic( config.Producer.RequiredAcks = sarama.WaitForAll config.Producer.Return.Successes = true config.Producer.Partitioner = sarama.NewHashPartitioner + config.Version = sarama.V1_0_0_0 hosts := []string{getTestKafkaHost()} producer, err := sarama.NewSyncProducer(hosts, config) @@ -195,6 +196,12 @@ func writeToKafkaTopic( msg := &sarama.ProducerMessage{ Topic: topic, Value: sarama.StringEncoder(message), + Headers: []sarama.RecordHeader{ + sarama.RecordHeader{ + Key: []byte("testkey"), + Value: []byte("testvalue"), + }, + }, } _, _, err = producer.SendMessage(msg) From 9601e0cbe6a29d92beb71e784ab4c5c07a29e182 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 16 Jul 2019 14:06:21 -0400 Subject: [PATCH 14/50] Don't record BlockTimestamp if it's zero --- filebeat/input/kafka/input.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 837b4e17006..7c3bae728e6 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -178,12 +178,14 @@ func (h groupHandler) createEvent( "offset": message.Offset, "key": message.Key, } - version, ok := h.input.config.Version.Get() - if ok && version.IsAtLeast(sarama.V0_10_0_0) { + version, versionOk := h.input.config.Version.Get() + if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { data.Event.Timestamp = message.Timestamp - kafkaMetadata["block_timestamp"] = message.BlockTimestamp + if !message.BlockTimestamp.IsZero() { + kafkaMetadata["block_timestamp"] = message.BlockTimestamp + } } - if ok && version.IsAtLeast(sarama.V0_11_0_0) { + if versionOk && version.IsAtLeast(sarama.V0_11_0_0) { kafkaMetadata["headers"] = arrayForKafkaHeaders(message.Headers) } eventFields["kafka"] = kafkaMetadata From 2d8d37ca52f2e8783ec67e27743054db56f3ced9 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 16 Jul 2019 14:41:06 -0400 Subject: [PATCH 15/50] Remove debug printf --- filebeat/input/kafka/input.go | 2 -- filebeat/input/kafka/kafka_integration_test.go | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 7c3bae728e6..c03bc2cf2f1 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -19,7 +19,6 @@ package kafka import ( "context" - "fmt" "time" "github.com/Shopify/sarama" @@ -204,7 +203,6 @@ func (groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { func (h groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { for msg := range claim.Messages() { event := h.createEvent(sess, claim, msg) - fmt.Printf("event: %v\n", event) h.input.outlet.OnEvent(event) sess.MarkMessage(msg, "") } diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 86892e7c48c..e231d8c62b0 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -64,6 +64,7 @@ func NewEventCapturer(events chan *util.Data) channel.Outleter { } func (o *eventCapturer) OnEvent(event *util.Data) bool { + fmt.Printf("event: %v\n", event) o.events <- event return true } From e6b8b53b807cc3718cfbeaa368f2d5378dff1a0f Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 17 Jul 2019 11:35:59 -0400 Subject: [PATCH 16/50] make fmt --- filebeat/docs/fields.asciidoc | 30 +++++++++++++++++++ filebeat/input/kafka/config.go | 1 + filebeat/input/kafka/input.go | 1 + .../input/kafka/kafka_integration_test.go | 2 ++ 4 files changed, 34 insertions(+) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 5c19a95c8df..f97fbfe8e6e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -7022,6 +7022,36 @@ type: long Kafka offset of this message +-- + +*`kafka.key`*:: ++ +-- +type: keyword + +Kafka key, corresponding to the Kafka value stored in the message + + +-- + +*`kafka.block_timestamp`*:: ++ +-- +type: date + +Kafka outer (compressed) block timestamp + + +-- + +*`kafka.headers`*:: ++ +-- +type: array + +The array of kafka headers, each an object containing subfields "key" and "value". + + -- [[exported-fields-kibana]] diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 958988a8728..1cc63e51116 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/Shopify/sarama" + "github.com/elastic/beats/libbeat/common/kafka" "github.com/elastic/beats/libbeat/common/transport/tlscommon" "github.com/elastic/beats/libbeat/logp" diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index c03bc2cf2f1..c637fd9e5b2 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -22,6 +22,7 @@ import ( "time" "github.com/Shopify/sarama" + "github.com/elastic/beats/filebeat/channel" "github.com/elastic/beats/filebeat/input" "github.com/elastic/beats/filebeat/util" diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index e231d8c62b0..d8ea62dd2f1 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build integration + package kafka import ( From 426c98f99d065bb9343ecfc78ace792630621718 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 18 Jul 2019 11:44:27 -0400 Subject: [PATCH 17/50] Add kafka container to filebeat integration tests --- filebeat/docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml index 746e7bc313d..52437a78a8d 100644 --- a/filebeat/docker-compose.yml +++ b/filebeat/docker-compose.yml @@ -12,6 +12,8 @@ services: - ES_PORT=9200 - ES_USER=beats - ES_PASS=testing + - KAFKA_HOST=kafka + - KAFKA_PORT=9092 - KIBANA_HOST=kibana - KIBANA_PORT=5601 working_dir: /go/src/github.com/elastic/beats/filebeat @@ -27,6 +29,7 @@ services: image: busybox depends_on: elasticsearch: { condition: service_healthy } + kafka: { condition: service_healthy } kibana: { condition: service_healthy } redis: { condition: service_healthy } @@ -35,6 +38,14 @@ services: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + kafka: + build: ${ES_BEATS}/testing/environments/docker/kafka + expose: + - 9092 + - 2181 + environment: + - ADVERTISED_HOST=kafka + kibana: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml From 43f1cde0d374d2f70a50d7e7b85f62d0d8a15b70 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 18 Jul 2019 14:22:39 -0400 Subject: [PATCH 18/50] regenerate docs --- filebeat/docs/fields.asciidoc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 51ff1c4c8a6..9486bd4356e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -7232,61 +7232,61 @@ Kafka metadata added by the kafka input *`kafka.topic`*:: + -- -type: keyword - Kafka topic +type: keyword + -- *`kafka.partition`*:: + -- -type: long - Kafka partition number +type: long + -- *`kafka.offset`*:: + -- -type: long - Kafka offset of this message +type: long + -- *`kafka.key`*:: + -- -type: keyword - Kafka key, corresponding to the Kafka value stored in the message +type: keyword + -- *`kafka.block_timestamp`*:: + -- -type: date - Kafka outer (compressed) block timestamp +type: date + -- *`kafka.headers`*:: + -- -type: array - The array of kafka headers, each an object containing subfields "key" and "value". +type: array + -- [[exported-fields-kibana]] From da3eb9988b3402c6944c5b67790343dcd7f39bca Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 18 Jul 2019 15:55:29 -0400 Subject: [PATCH 19/50] Remove unused test helpers --- .../input/kafka/kafka_integration_test.go | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index d8ea62dd2f1..335eba2e027 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -20,7 +20,6 @@ package kafka import ( - "encoding/json" "fmt" "math/rand" "os" @@ -37,7 +36,6 @@ import ( "github.com/elastic/beats/filebeat/util" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/fmtstr" _ "github.com/elastic/beats/libbeat/outputs/codec/format" _ "github.com/elastic/beats/libbeat/outputs/codec/json" ) @@ -99,7 +97,7 @@ func TestInput(t *testing.T) { // Setup the input config config, _ := common.NewConfigFrom(common.MapStr{ - "hosts": "kafka:9092", + "hosts": getTestKafkaHost(), "topics": []string{testTopic}, "group_id": "filebeat", }) @@ -136,28 +134,6 @@ func TestInput(t *testing.T) { } } -func validateJSON(t *testing.T, value []byte, event beat.Event) { - var decoded map[string]interface{} - err := json.Unmarshal(value, &decoded) - if err != nil { - t.Errorf("can not json decode event value: %v", value) - return - } - assert.Equal(t, decoded["type"], event.Fields["type"]) - assert.Equal(t, decoded["message"], event.Fields["message"]) -} - -func makeValidateFmtStr(fmt string) func(*testing.T, []byte, beat.Event) { - fmtString := fmtstr.MustCompileEvent(fmt) - return func(t *testing.T, value []byte, event beat.Event) { - expectedMessage, err := fmtString.Run(&event) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, string(expectedMessage), string(value)) - } -} - func strDefault(a, defaults string) string { if len(a) == 0 { return defaults From 3da5f9940a210651077cf6d25772e4082f38dc11 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 18 Jul 2019 16:41:46 -0400 Subject: [PATCH 20/50] Add header verification to kafka integration test --- .../input/kafka/kafka_integration_test.go | 93 +++++++++++++++---- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 335eba2e027..7091b3d4421 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -64,7 +64,6 @@ func NewEventCapturer(events chan *util.Data) channel.Outleter { } func (o *eventCapturer) OnEvent(event *util.Data) bool { - fmt.Printf("event: %v\n", event) o.events <- event return true } @@ -81,6 +80,18 @@ func (o *eventCapturer) Done() <-chan struct{} { return o.c } +type testMessage struct { + message string + headers []sarama.RecordHeader +} + +func recordHeader(key, value string) sarama.RecordHeader { + return sarama.RecordHeader{ + Key: []byte(key), + Value: []byte(value), + } +} + func TestInput(t *testing.T) { id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int()) testTopic := fmt.Sprintf("Filebeat-TestInput-%s", id) @@ -90,9 +101,24 @@ func TestInput(t *testing.T) { } // Send test messages to the topic for the input to read. - messageStrs := []string{"testing", "stuff", "blah"} - for _, s := range messageStrs { - writeToKafkaTopic(t, testTopic, s, time.Second*20) + messages := []testMessage{ + testMessage{message: "testing"}, + testMessage{ + message: "stuff", + headers: []sarama.RecordHeader{ + recordHeader("X-Test-Header", "test header value"), + }, + }, + testMessage{ + message: "things", + headers: []sarama.RecordHeader{ + recordHeader("keys and things", "3^3 = 27"), + recordHeader("kafka yay", "3^3 - 2^4 = 11"), + }, + }, + } + for _, m := range messages { + writeToKafkaTopic(t, testTopic, m.message, m.headers, time.Second*20) } // Setup the input config @@ -120,20 +146,59 @@ func TestInput(t *testing.T) { input.Run() timeout := time.After(30 * time.Second) - for _, m := range messageStrs { + for _, m := range messages { select { case event := <-events: - result, err := event.GetEvent().Fields.GetValue("message") + text, err := event.GetEvent().Fields.GetValue("message") if err != nil { t.Fatal(err) } - assert.Equal(t, result, m) + assert.Equal(t, text, m.message) + + checkMatchingHeaders(t, event.GetEvent(), m.headers) case <-timeout: t.Fatal("timeout waiting for incoming events") } } } +func checkMatchingHeaders( + t *testing.T, event beat.Event, expected []sarama.RecordHeader, +) { + kafka, err := event.Fields.GetValue("kafka") + if err != nil { + t.Error(err) + return + } + kafkaMap, ok := kafka.(common.MapStr) + if !ok { + t.Error("event.Fields.kafka isn't MapStr") + return + } + headers, err := kafkaMap.GetValue("headers") + if err != nil { + t.Error(err) + return + } + headerArray, ok := headers.([]interface{}) + if !ok { + t.Error("event.Fields.kafka.headers isn't a []interface{}") + return + } + assert.Equal(t, len(expected), len(headerArray)) + for i := 0; i < len(expected); i++ { + headerMap, ok := headerArray[i].(common.MapStr) + if !ok { + t.Errorf("event.Fields.kafka.headers[%v] isn't a MapStr", i) + continue + } + key, _ := headerMap.GetValue("key") + value, _ := headerMap.GetValue("value") + assert.Equal(t, expected[i].Key, key) + assert.Equal(t, expected[i].Value, value) + } +} + func strDefault(a, defaults string) string { if len(a) == 0 { return defaults @@ -153,7 +218,8 @@ func getTestKafkaHost() string { } func writeToKafkaTopic( - t *testing.T, topic string, message string, timeout time.Duration, + t *testing.T, topic string, message string, + headers []sarama.RecordHeader, timeout time.Duration, ) { config := sarama.NewConfig() config.Producer.RequiredAcks = sarama.WaitForAll @@ -173,14 +239,9 @@ func writeToKafkaTopic( }() msg := &sarama.ProducerMessage{ - Topic: topic, - Value: sarama.StringEncoder(message), - Headers: []sarama.RecordHeader{ - sarama.RecordHeader{ - Key: []byte("testkey"), - Value: []byte("testvalue"), - }, - }, + Topic: topic, + Value: sarama.StringEncoder(message), + Headers: headers, } _, _, err = producer.SendMessage(msg) From 6bdb13fa836b1d78ab6e99969ae14fa48f5c4919 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 22 Jul 2019 15:49:36 -0400 Subject: [PATCH 21/50] Addressing review comments --- filebeat/input/kafka/config.go | 36 ++++++++++++++++--------------- filebeat/input/kafka/input.go | 39 +++++++++++++++------------------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 1cc63e51116..6d17c746602 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -31,6 +31,19 @@ import ( "github.com/elastic/beats/libbeat/outputs" ) +type kafkaInputConfig struct { + // Kafka hosts with port, e.g. "localhost:9092" + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + ClientID string `config:"client_id"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` +} + type initialOffset int const ( @@ -39,29 +52,18 @@ const ( ) var ( - defaultConfig = kafkaInputConfig{ - Version: kafka.Version("1.0.0"), - InitialOffset: initialOffsetOldest, - ClientID: "filebeat", - } - initialOffsets = map[string]initialOffset{ "oldest": initialOffsetOldest, "newest": initialOffsetNewest, } ) -type kafkaInputConfig struct { - // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - GroupID string `config:"group_id" validate:"required"` - ClientID string `config:"client_id"` - Version kafka.Version `config:"version"` - InitialOffset initialOffset `config:"initial_offset"` - TLS *tlscommon.Config `config:"ssl"` - Username string `config:"username"` - Password string `config:"password"` +func defaultConfig() kafkaInputConfig { + return kafkaInputConfig{ + Version: kafka.Version("1.0.0"), + InitialOffset: initialOffsetOldest, + ClientID: "filebeat", + } } // Validate validates the config. diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index c637fd9e5b2..e626c98990b 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -41,7 +41,7 @@ func init() { } // Input contains the input and its config -type Input struct { +type kafkaInput struct { config kafkaInputConfig rawConfig *common.Config // The Config given to NewInput started bool @@ -64,7 +64,7 @@ func NewInput( return nil, err } - config := defaultConfig + config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, errors.Wrap(err, "reading kafka input config") } @@ -91,7 +91,7 @@ func NewInput( } }() - input := &Input{ + input := &kafkaInput{ config: config, rawConfig: cfg, started: false, @@ -105,44 +105,39 @@ func NewInput( return input, nil } -func (p *Input) newConsumerGroup() (sarama.ConsumerGroup, error) { - consumerGroup, err := - sarama.NewConsumerGroup(p.config.Hosts, p.config.GroupID, nil) - return consumerGroup, err -} - // Run starts the input by scanning for incoming messages and errors. -func (p *Input) Run() { - if !p.started { +func (input *kafkaInput) Run() { + if !input.started { // Track errors go func() { - for err := range p.consumerGroup.Errors() { - p.log.Errorw("Error reading from kafka", "error", err) + for err := range input.consumerGroup.Errors() { + input.log.Errorw("Error reading from kafka", "error", err) } }() go func() { for { - handler := groupHandler{input: p} + handler := groupHandler{input: input} - err := p.consumerGroup.Consume(p.kafkaContext, p.config.Topics, handler) + err := input.consumerGroup.Consume( + input.kafkaContext, input.config.Topics, handler) if err != nil { - p.log.Errorw("Kafka consume error", "error", err) + input.log.Errorw("Kafka consume error", "error", err) } } }() - p.started = true + input.started = true } } // Wait shuts down the Input by cancelling the internal context. -func (p *Input) Wait() { - p.Stop() +func (input *kafkaInput) Wait() { + input.Stop() } // Stop shuts down the Input by cancelling the internal context. -func (p *Input) Stop() { - p.kafkaCancel() +func (input *kafkaInput) Stop() { + input.kafkaCancel() } func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { @@ -157,7 +152,7 @@ func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { } type groupHandler struct { - input *Input + input *kafkaInput } func (h groupHandler) createEvent( From bfeaeb038a96c7714cb2595bae3bcca20a808b06 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 23 Jul 2019 16:11:24 -0400 Subject: [PATCH 22/50] Review comments --- filebeat/input/kafka/input.go | 38 +++++++++---------- .../input/kafka/kafka_integration_test.go | 2 +- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index e626c98990b..4cbd0315d8c 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -44,7 +44,6 @@ func init() { type kafkaInput struct { config kafkaInputConfig rawConfig *common.Config // The Config given to NewInput - started bool outlet channel.Outleter consumerGroup sarama.ConsumerGroup kafkaContext context.Context @@ -94,7 +93,6 @@ func NewInput( input := &kafkaInput{ config: config, rawConfig: cfg, - started: false, outlet: out, consumerGroup: consumerGroup, kafkaContext: kafkaContext, @@ -107,32 +105,32 @@ func NewInput( // Run starts the input by scanning for incoming messages and errors. func (input *kafkaInput) Run() { - if !input.started { - // Track errors - go func() { - for err := range input.consumerGroup.Errors() { - input.log.Errorw("Error reading from kafka", "error", err) - } - }() + // Track errors + go func() { + for err := range input.consumerGroup.Errors() { + input.log.Errorw("Error reading from kafka", "error", err) + } + }() - go func() { - for { - handler := groupHandler{input: input} + go func() { + for { + handler := groupHandler{input: input} - err := input.consumerGroup.Consume( - input.kafkaContext, input.config.Topics, handler) - if err != nil { - input.log.Errorw("Kafka consume error", "error", err) - } + err := input.consumerGroup.Consume( + input.kafkaContext, input.config.Topics, handler) + if err != nil { + input.log.Errorw("Kafka consume error", "error", err) } - }() - input.started = true - } + } + }() } // Wait shuts down the Input by cancelling the internal context. func (input *kafkaInput) Wait() { input.Stop() + // TODO: wait on any messages still pending internal delivery + // Wait for the consumer group to shut down + input.consumerGroup.Close() } // Stop shuts down the Input by cancelling the internal context. diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 7091b3d4421..87d62f2ee2f 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -200,7 +200,7 @@ func checkMatchingHeaders( } func strDefault(a, defaults string) string { - if len(a) == 0 { + if a == "" { return defaults } return a From 8061d8691161cc85781792896785c86e07598734 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 23 Jul 2019 17:28:37 -0400 Subject: [PATCH 23/50] Add several more kafka configuration settings --- filebeat/input/kafka/config.go | 73 +++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 6d17c746602..13a10cd027c 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -20,6 +20,7 @@ package kafka import ( "errors" "fmt" + "time" "github.com/Shopify/sarama" @@ -39,11 +40,28 @@ type kafkaInputConfig struct { ClientID string `config:"client_id"` Version kafka.Version `config:"version"` InitialOffset initialOffset `config:"initial_offset"` + RetryBackoff time.Duration `config:"retry_backoff" validate:"min=0"` + MaxWaitTime time.Duration `config:"max_wait_time"` + Fetch *kafkaFetch `config:"fetch"` + Rebalance *kafkaRebalance `config:"rebalance"` TLS *tlscommon.Config `config:"ssl"` Username string `config:"username"` Password string `config:"password"` } +type kafkaFetch struct { + Min int32 `config:"min" validate:"min=1"` + Default int32 `config:"default" validate:"min=1"` + Max int32 `config:"max" validate:"min=0"` +} + +type kafkaRebalance struct { + Strategy rebalanceStrategy `config:"strategy"` + Timeout time.Duration `config:"timeout"` + MaxRetries int `config:"max_retries"` + RetryBackoff time.Duration `config:"retry_backoff" validate:"min=0"` +} + type initialOffset int const ( @@ -51,18 +69,44 @@ const ( initialOffsetNewest ) +type rebalanceStrategy int + +const ( + rebalanceStrategyRange rebalanceStrategy = iota + rebalanceStrategyRoundRobin +) + var ( initialOffsets = map[string]initialOffset{ "oldest": initialOffsetOldest, "newest": initialOffsetNewest, } + rebalanceStrategies = map[string]rebalanceStrategy{ + "range": rebalanceStrategyRange, + "roundrobin": rebalanceStrategyRoundRobin, + } ) +// The default config for the kafka input. When in doubt, default values +// were chosen to match sarama's defaults. func defaultConfig() kafkaInputConfig { return kafkaInputConfig{ Version: kafka.Version("1.0.0"), InitialOffset: initialOffsetOldest, ClientID: "filebeat", + RetryBackoff: 2 * time.Second, + MaxWaitTime: 250 * time.Millisecond, + Fetch: &kafkaFetch{ + Min: 1, + Default: (1 << 20), // 1 MB + Max: 0, + }, + Rebalance: &kafkaRebalance{ + Strategy: rebalanceStrategyRange, + Timeout: 60 * time.Second, + MaxRetries: 4, + RetryBackoff: 2 * time.Second, + }, } } @@ -93,6 +137,18 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k.Consumer.Return.Errors = true k.Consumer.Offsets.Initial = config.InitialOffset.asSaramaOffset() + k.Consumer.Retry.Backoff = config.RetryBackoff + k.Consumer.MaxWaitTime = config.MaxWaitTime + + k.Consumer.Fetch.Min = config.Fetch.Min + k.Consumer.Fetch.Default = config.Fetch.Default + k.Consumer.Fetch.Max = config.Fetch.Max + + k.Consumer.Group.Rebalance.Strategy = + config.Rebalance.Strategy.asSaramaStrategy() + k.Consumer.Group.Rebalance.Timeout = config.Rebalance.Timeout + k.Consumer.Group.Rebalance.Retry.Backoff = config.Rebalance.RetryBackoff + k.Consumer.Group.Rebalance.Retry.Max = config.Rebalance.MaxRetries tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { @@ -142,8 +198,23 @@ func (off *initialOffset) Unpack(value string) error { if !ok { return fmt.Errorf("invalid initialOffset '%s'", value) } - *off = initialOffset + return nil +} +func (st rebalanceStrategy) asSaramaStrategy() sarama.BalanceStrategy { + return map[rebalanceStrategy]sarama.BalanceStrategy{ + rebalanceStrategyRange: sarama.BalanceStrategyRange, + rebalanceStrategyRoundRobin: sarama.BalanceStrategyRoundRobin, + }[st] +} + +// Unpack validates and unpack the "rebalance.strategy" config option +func (st *rebalanceStrategy) Unpack(value string) error { + strategy, ok := rebalanceStrategies[value] + if !ok { + return fmt.Errorf("invalid rebalance strategy '%s'", value) + } + *st = strategy return nil } From 678d71b1d9f2c54a91c4efdd492de15955a504e0 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 25 Jul 2019 17:01:26 -0400 Subject: [PATCH 24/50] Document kafka input configuration --- filebeat/docs/inputs/input-kafka.asciidoc | 107 ++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 filebeat/docs/inputs/input-kafka.asciidoc diff --git a/filebeat/docs/inputs/input-kafka.asciidoc b/filebeat/docs/inputs/input-kafka.asciidoc new file mode 100644 index 00000000000..91bc36fe559 --- /dev/null +++ b/filebeat/docs/inputs/input-kafka.asciidoc @@ -0,0 +1,107 @@ +:type: kafka + +[id="{beatname_lc}-input-{type}"] +=== Kafka input + +++++ +Kafka +++++ + +Use the `kafka` input to read from topics in a Kafka cluster. + +To configure this input, specify a list of <> to use for this +cluster, a list of <> to track, and a <> +to connect with. + +Example configuration: + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: kafka + hosts: + - kafka-broker-1:9092 + - kafka-broker-2:9092 + topics: ["my-topic"] + group_id: "filebeat" + +---- + + +[id="{beatname_lc}-input-{type}-options"] +==== Configuration options + +The `kafka` input supports the following configuration options plus the +<<{beatname_lc}-input-{type}-common-options>> described later. + +[float] +[[hosts]] +===== `hosts` + +A list of Kafka hosts (brokers) for this cluster. + +[float] +[[topics]] +===== `topics` + +A list of topics to read from. + +[float] +[[groupid]] +===== `group_id` + +The Kafka consumer group id. + +[float] +===== `client_id` + +The Kafka client id (optional). + +[float] +===== `version` + +The version of the Kafka protocol to use (defaults to `"1.0.0"`). + +[float] +===== `initial_offset` + +The initial offset to start reading, either "oldest" or "newest". Defaults to +"oldest". + +===== `retry_backoff` + +How long to wait before retrying a failed read. Default is 2s. + +===== `max_wait_time` + +How long to wait for the minimum number of input bytes while reading. Default +is 250ms. + +===== `fetch` + +Kafka fetch settings: + +*`min`*:: The minimum number of bytes to wait for. Defaults to 1. + +*`default`*:: The default number of bytes to read per request. Defaults to 1MB. + +*`max`*:: The maximum number of bytes to read per request. Defaults to 0 +(no limit). + +===== `rebalance` + +Kafka rebalance settings: + +*`strategy`*:: Either `"range"` or `"roundrobin"`. Defaults to `"range"`. + +*`timeout`*:: How long to wait for an attempted rebalance. Defaults to 60s. + +*`max_retries`*:: How many times to retry if rebalancing fails. Defaults to 4. + +*`retry_backoff`*:: How long to wait after an unsuccessful rebalance attempt. +Defaults to 2s. + +[id="{beatname_lc}-input-{type}-common-options"] +include::../inputs/input-common-options.asciidoc[] + +:type!: From 6a45e0509928ed8b479a9bc858703aa212e14099 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Fri, 26 Jul 2019 15:40:44 -0400 Subject: [PATCH 25/50] Update for new outlet api --- filebeat/input/kafka/input.go | 54 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 4cbd0315d8c..1132bdc0b5e 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -19,13 +19,13 @@ package kafka import ( "context" + "sync" "time" "github.com/Shopify/sarama" "github.com/elastic/beats/filebeat/channel" "github.com/elastic/beats/filebeat/input" - "github.com/elastic/beats/filebeat/util" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -49,16 +49,21 @@ type kafkaInput struct { kafkaContext context.Context kafkaCancel context.CancelFunc // The CancelFunc for kafkaContext log *logp.Logger + runOnce sync.Once } // NewInput creates a new kafka input func NewInput( cfg *common.Config, - outletFactory channel.Connector, + connector channel.Connector, inputContext input.Context, ) (input.Input, error) { - out, err := outletFactory(cfg, inputContext.DynamicFields) + out, err := connector.ConnectWith(cfg, beat.ClientConfig{ + Processing: beat.ProcessingConfig{ + DynamicFields: inputContext.DynamicFields, + }, + }) if err != nil { return nil, err } @@ -105,24 +110,26 @@ func NewInput( // Run starts the input by scanning for incoming messages and errors. func (input *kafkaInput) Run() { - // Track errors - go func() { - for err := range input.consumerGroup.Errors() { - input.log.Errorw("Error reading from kafka", "error", err) - } - }() + input.runOnce.Do(func() { + // Track errors + go func() { + for err := range input.consumerGroup.Errors() { + input.log.Errorw("Error reading from kafka", "error", err) + } + }() - go func() { - for { - handler := groupHandler{input: input} + go func() { + for { + handler := groupHandler{input: input} - err := input.consumerGroup.Consume( - input.kafkaContext, input.config.Topics, handler) - if err != nil { - input.log.Errorw("Kafka consume error", "error", err) + err := input.consumerGroup.Consume( + input.kafkaContext, input.config.Topics, handler) + if err != nil { + input.log.Errorw("Kafka consume error", "error", err) + } } - } - }() + }() + }) } // Wait shuts down the Input by cancelling the internal context. @@ -157,9 +164,8 @@ func (h groupHandler) createEvent( sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim, message *sarama.ConsumerMessage, -) *util.Data { - data := util.NewData() - data.Event = beat.Event{ +) beat.Event { + event := beat.Event{ Timestamp: time.Now(), } eventFields := common.MapStr{ @@ -173,7 +179,7 @@ func (h groupHandler) createEvent( } version, versionOk := h.input.config.Version.Get() if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { - data.Event.Timestamp = message.Timestamp + event.Timestamp = message.Timestamp if !message.BlockTimestamp.IsZero() { kafkaMetadata["block_timestamp"] = message.BlockTimestamp } @@ -182,8 +188,8 @@ func (h groupHandler) createEvent( kafkaMetadata["headers"] = arrayForKafkaHeaders(message.Headers) } eventFields["kafka"] = kafkaMetadata - data.Event.Fields = eventFields - return data + event.Fields = eventFields + return event } func (groupHandler) Setup(session sarama.ConsumerGroupSession) error { From 5cee0820cb0ffdcc262aa48043a098b027484a37 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 29 Jul 2019 15:33:45 -0400 Subject: [PATCH 26/50] Add end-to-end ACK to the kafka input / synchronize access to the session --- filebeat/input/kafka/input.go | 61 +++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 1132bdc0b5e..9379cce736d 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -46,12 +46,21 @@ type kafkaInput struct { rawConfig *common.Config // The Config given to NewInput outlet channel.Outleter consumerGroup sarama.ConsumerGroup + sessionState *kafkaSessionState kafkaContext context.Context kafkaCancel context.CancelFunc // The CancelFunc for kafkaContext log *logp.Logger runOnce sync.Once } +// A synchronized wrapper to read and write the kafka session, since it may +// change while ACKs are still pending. +type kafkaSessionState struct { + session sarama.ConsumerGroupSession + mutex sync.Mutex // Hold to access the session field + waitGroup sync.WaitGroup // Hold while using the session field +} + // NewInput creates a new kafka input func NewInput( cfg *common.Config, @@ -59,10 +68,27 @@ func NewInput( inputContext input.Context, ) (input.Input, error) { + // We create the empty session state first because it must be referenced by + // the ACK callback in the connector configuration. + sessionState := &kafkaSessionState{} + out, err := connector.ConnectWith(cfg, beat.ClientConfig{ Processing: beat.ProcessingConfig{ DynamicFields: inputContext.DynamicFields, }, + ACKEvents: func(events []interface{}) { + sessionState.accessSession(func(session sarama.ConsumerGroupSession) { + if session == nil { + // The kafka connection is closed and / or is being rebalanced. + return + } + for _, event := range events { + if cm, ok := event.(*sarama.ConsumerMessage); ok { + session.MarkMessage(cm, "") + } + } + }) + }, }) if err != nil { return nil, err @@ -100,6 +126,7 @@ func NewInput( rawConfig: cfg, outlet: out, consumerGroup: consumerGroup, + sessionState: sessionState, kafkaContext: kafkaContext, kafkaCancel: kafkaCancel, log: logp.NewLogger("kafka input").With("hosts", config.Hosts), @@ -108,6 +135,33 @@ func NewInput( return input, nil } +// A helper to safely use the current sarama session for the duration of the +// given callback. Used when ACKing messages outside the body of the main +// sarama callbacks. The session parameter may be nil if there is no active +// session. +func (state *kafkaSessionState) accessSession( + fn func(session sarama.ConsumerGroupSession), +) { + state.mutex.Lock() + state.waitGroup.Add(1) + session := state.session + state.mutex.Unlock() + defer state.waitGroup.Done() + fn(session) +} + +// A helper to safely set the session field after waiting on any pending +// operations. +func (state *kafkaSessionState) setSession(sess sarama.ConsumerGroupSession) { + state.mutex.Lock() + // Once we claim the mutex we still wait for any pending ACKs to be + // sent. (These may well fail if the session is ending, but that's better + // than calling a stale pointer.) + state.waitGroup.Wait() + state.session = sess + state.mutex.Unlock() +} + // Run starts the input by scanning for incoming messages and errors. func (input *kafkaInput) Run() { input.runOnce.Do(func() { @@ -135,7 +189,6 @@ func (input *kafkaInput) Run() { // Wait shuts down the Input by cancelling the internal context. func (input *kafkaInput) Wait() { input.Stop() - // TODO: wait on any messages still pending internal delivery // Wait for the consumer group to shut down input.consumerGroup.Close() } @@ -192,11 +245,13 @@ func (h groupHandler) createEvent( return event } -func (groupHandler) Setup(session sarama.ConsumerGroupSession) error { +func (h groupHandler) Setup(session sarama.ConsumerGroupSession) error { + h.input.sessionState.setSession(session) return nil } -func (groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { +func (h groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { + h.input.sessionState.setSession(nil) return nil } From a16e862e524c411a7bc349d1939a08ecb9558468 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 29 Jul 2019 16:02:16 -0400 Subject: [PATCH 27/50] Update integration test --- .../input/kafka/kafka_integration_test.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 87d62f2ee2f..b175e43876c 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build integration +// + build integration package kafka @@ -33,7 +33,6 @@ import ( "github.com/elastic/beats/filebeat/channel" "github.com/elastic/beats/filebeat/input" - "github.com/elastic/beats/filebeat/util" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" _ "github.com/elastic/beats/libbeat/outputs/codec/format" @@ -53,17 +52,17 @@ type eventCapturer struct { closed bool c chan struct{} closeOnce sync.Once - events chan *util.Data + events chan beat.Event } -func NewEventCapturer(events chan *util.Data) channel.Outleter { +func NewEventCapturer(events chan beat.Event) channel.Outleter { return &eventCapturer{ c: make(chan struct{}), events: events, } } -func (o *eventCapturer) OnEvent(event *util.Data) bool { +func (o *eventCapturer) OnEvent(event beat.Event) bool { o.events <- event return true } @@ -129,13 +128,13 @@ func TestInput(t *testing.T) { }) // Route input events through our capturer instead of sending through ES. - events := make(chan *util.Data, 100) + events := make(chan beat.Event, 100) defer close(events) capturer := NewEventCapturer(events) defer capturer.Close() - connector := func(*common.Config, *common.MapStrPointer) (channel.Outleter, error) { + connector := channel.ConnectorFunc(func(_ *common.Config, _ beat.ClientConfig) (channel.Outleter, error) { return channel.SubOutlet(capturer), nil - } + }) input, err := NewInput(config, connector, context) if err != nil { @@ -149,13 +148,13 @@ func TestInput(t *testing.T) { for _, m := range messages { select { case event := <-events: - text, err := event.GetEvent().Fields.GetValue("message") + text, err := event.Fields.GetValue("message") if err != nil { t.Fatal(err) } assert.Equal(t, text, m.message) - checkMatchingHeaders(t, event.GetEvent(), m.headers) + checkMatchingHeaders(t, event, m.headers) case <-timeout: t.Fatal("timeout waiting for incoming events") } From e2137cb42e222f50b5488692d0fce4bc4b216d60 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 29 Jul 2019 16:04:27 -0400 Subject: [PATCH 28/50] make integration test an integration test againk not unit --- filebeat/input/kafka/kafka_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index b175e43876c..e7ac3a39ef7 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// + build integration +// +build integration package kafka From cd8a3d9340e018d229fb8e22966c4aeb5993d874 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 29 Jul 2019 16:54:54 -0400 Subject: [PATCH 29/50] Replace sarama context with a minimal wrapper suggested by @urso --- filebeat/input/kafka/input.go | 56 ++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 9379cce736d..b91541fd5c4 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -44,11 +44,10 @@ func init() { type kafkaInput struct { config kafkaInputConfig rawConfig *common.Config // The Config given to NewInput + context input.Context outlet channel.Outleter consumerGroup sarama.ConsumerGroup sessionState *kafkaSessionState - kafkaContext context.Context - kafkaCancel context.CancelFunc // The CancelFunc for kafkaContext log *logp.Logger runOnce sync.Once } @@ -109,26 +108,13 @@ func NewInput( return nil, errors.Wrap(err, "initializing kafka consumer group") } - // Sarama uses standard go contexts to control cancellation, so we need to - // wrap our input context channel in that interface. - kafkaContext, kafkaCancel := context.WithCancel(context.Background()) - go func() { - select { - case <-inputContext.Done: - logp.Info("Closing kafka context because input stopped.") - kafkaCancel() - return - } - }() - input := &kafkaInput{ config: config, rawConfig: cfg, + context: inputContext, outlet: out, consumerGroup: consumerGroup, sessionState: sessionState, - kafkaContext: kafkaContext, - kafkaCancel: kafkaCancel, log: logp.NewLogger("kafka input").With("hosts", config.Hosts), } @@ -176,11 +162,20 @@ func (input *kafkaInput) Run() { for { handler := groupHandler{input: input} + // Sarama uses standard go contexts to control cancellation, so we need + // to wrap our input context channel in that interface. err := input.consumerGroup.Consume( - input.kafkaContext, input.config.Topics, handler) + doneChannelContext(input.context.Done), input.config.Topics, handler) if err != nil { input.log.Errorw("Kafka consume error", "error", err) } + + // If Consume returned because the context was cancelled, don't resume. + select { + case <-input.context.Done: + return + default: + } } }() }) @@ -195,7 +190,7 @@ func (input *kafkaInput) Wait() { // Stop shuts down the Input by cancelling the internal context. func (input *kafkaInput) Stop() { - input.kafkaCancel() + close(input.context.Done) } func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { @@ -209,6 +204,31 @@ func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { return array } +// A barebones implementation of context.Context wrapped around the done +// channels that are more common in the beats codebase. This could be added +// as a utility in a shared part of the code, but right now it's a special +// case for sarama which requires a context.Context, so it's private until +// there's at least one other use case. +type channelCtx <-chan struct{} + +func doneChannelContext(ch <-chan struct{}) context.Context { + return channelCtx(ch) +} + +func (c channelCtx) Deadline() (deadline time.Time, ok bool) { return } +func (c channelCtx) Done() <-chan struct{} { + return (<-chan struct{})(c) +} +func (c channelCtx) Err() error { + select { + case <-c: + return context.Canceled + default: + return nil + } +} +func (c channelCtx) Value(key interface{}) interface{} { return nil } + type groupHandler struct { input *kafkaInput } From e54b9d87f7306ec8468e0e1c02ae0389e2e13ed1 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 31 Jul 2019 11:58:59 -0400 Subject: [PATCH 30/50] Clarify docs --- filebeat/docs/inputs/input-kafka.asciidoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/filebeat/docs/inputs/input-kafka.asciidoc b/filebeat/docs/inputs/input-kafka.asciidoc index 91bc36fe559..7af2b2c9ce0 100644 --- a/filebeat/docs/inputs/input-kafka.asciidoc +++ b/filebeat/docs/inputs/input-kafka.asciidoc @@ -9,9 +9,9 @@ Use the `kafka` input to read from topics in a Kafka cluster. -To configure this input, specify a list of <> to use for this -cluster, a list of <> to track, and a <> -to connect with. +To configure this input, specify a list of one or more <> in the +cluster to bootstrap the connection with, a list of <> to +track, and a <> for the connection. Example configuration: @@ -38,7 +38,7 @@ The `kafka` input supports the following configuration options plus the [[hosts]] ===== `hosts` -A list of Kafka hosts (brokers) for this cluster. +A list of Kafka bootstrapping hosts (brokers) for this cluster. [float] [[topics]] From 418364ac1d0a3e08bc05d4b518e079240fbe8c48 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 31 Jul 2019 12:21:40 -0400 Subject: [PATCH 31/50] Addressing review comments --- filebeat/input/kafka/config.go | 8 ++++---- filebeat/input/kafka/input.go | 10 +++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 13a10cd027c..7f810012790 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -42,8 +42,8 @@ type kafkaInputConfig struct { InitialOffset initialOffset `config:"initial_offset"` RetryBackoff time.Duration `config:"retry_backoff" validate:"min=0"` MaxWaitTime time.Duration `config:"max_wait_time"` - Fetch *kafkaFetch `config:"fetch"` - Rebalance *kafkaRebalance `config:"rebalance"` + Fetch kafkaFetch `config:"fetch"` + Rebalance kafkaRebalance `config:"rebalance"` TLS *tlscommon.Config `config:"ssl"` Username string `config:"username"` Password string `config:"password"` @@ -96,12 +96,12 @@ func defaultConfig() kafkaInputConfig { ClientID: "filebeat", RetryBackoff: 2 * time.Second, MaxWaitTime: 250 * time.Millisecond, - Fetch: &kafkaFetch{ + Fetch: kafkaFetch{ Min: 1, Default: (1 << 20), // 1 MB Max: 0, }, - Rebalance: &kafkaRebalance{ + Rebalance: kafkaRebalance{ Strategy: rebalanceStrategyRange, Timeout: 60 * time.Second, MaxRetries: 4, diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index b91541fd5c4..be1fa7ff913 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -43,7 +43,6 @@ func init() { // Input contains the input and its config type kafkaInput struct { config kafkaInputConfig - rawConfig *common.Config // The Config given to NewInput context input.Context outlet channel.Outleter consumerGroup sarama.ConsumerGroup @@ -110,7 +109,6 @@ func NewInput( input := &kafkaInput{ config: config, - rawConfig: cfg, context: inputContext, outlet: out, consumerGroup: consumerGroup, @@ -205,10 +203,9 @@ func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { } // A barebones implementation of context.Context wrapped around the done -// channels that are more common in the beats codebase. This could be added -// as a utility in a shared part of the code, but right now it's a special -// case for sarama which requires a context.Context, so it's private until -// there's at least one other use case. +// channels that are more common in the beats codebase. +// TODO(faec): Generalize this to a common utility in a shared library +// (https://github.com/elastic/beats/issues/13125). type channelCtx <-chan struct{} func doneChannelContext(ch <-chan struct{}) context.Context { @@ -279,7 +276,6 @@ func (h groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim saram for msg := range claim.Messages() { event := h.createEvent(sess, claim, msg) h.input.outlet.OnEvent(event) - sess.MarkMessage(msg, "") } return nil } From 73aa0bc1c25739b6265833eb66eee23a257d44fe Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 31 Jul 2019 15:34:41 -0400 Subject: [PATCH 32/50] Fix kafka input Stop() --- filebeat/input/kafka/input.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index be1fa7ff913..3503b4b6e21 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -186,9 +186,12 @@ func (input *kafkaInput) Wait() { input.consumerGroup.Close() } -// Stop shuts down the Input by cancelling the internal context. +// Stop closes the input's outlet on close. We don't need to shutdown the +// kafka consumer group explicitly, because it listens to the original input +// done channel passed in by input.Runner, and that channel is already closed +// as part of the shutdown process in Runner.Stop(). func (input *kafkaInput) Stop() { - close(input.context.Done) + input.outlet.Close() } func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { From 76f77922a7254453bf1990b191efb1b274f644b5 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 10:30:00 -0400 Subject: [PATCH 33/50] revised run loop in progress --- filebeat/input/kafka/config.go | 40 +++++---- filebeat/input/kafka/input.go | 89 +++++++++++-------- .../input/kafka/kafka_integration_test.go | 5 +- 3 files changed, 77 insertions(+), 57 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 7f810012790..cbc5b98071c 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -34,19 +34,20 @@ import ( type kafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - GroupID string `config:"group_id" validate:"required"` - ClientID string `config:"client_id"` - Version kafka.Version `config:"version"` - InitialOffset initialOffset `config:"initial_offset"` - RetryBackoff time.Duration `config:"retry_backoff" validate:"min=0"` - MaxWaitTime time.Duration `config:"max_wait_time"` - Fetch kafkaFetch `config:"fetch"` - Rebalance kafkaRebalance `config:"rebalance"` - TLS *tlscommon.Config `config:"ssl"` - Username string `config:"username"` - Password string `config:"password"` + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + ClientID string `config:"client_id"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` + InitRetryBackoff time.Duration `config:"init_retry_backoff" validate:"min=0"` + ConsumeRetryBackoff time.Duration `config:"consume_retry_backoff" validate:"min=0"` + MaxWaitTime time.Duration `config:"max_wait_time"` + Fetch kafkaFetch `config:"fetch"` + Rebalance kafkaRebalance `config:"rebalance"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` } type kafkaFetch struct { @@ -91,11 +92,12 @@ var ( // were chosen to match sarama's defaults. func defaultConfig() kafkaInputConfig { return kafkaInputConfig{ - Version: kafka.Version("1.0.0"), - InitialOffset: initialOffsetOldest, - ClientID: "filebeat", - RetryBackoff: 2 * time.Second, - MaxWaitTime: 250 * time.Millisecond, + Version: kafka.Version("1.0.0"), + InitialOffset: initialOffsetOldest, + ClientID: "filebeat", + InitRetryBackoff: 30 * time.Second, + ConsumeRetryBackoff: 2 * time.Second, + MaxWaitTime: 250 * time.Millisecond, Fetch: kafkaFetch{ Min: 1, Default: (1 << 20), // 1 MB @@ -137,7 +139,7 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k.Consumer.Return.Errors = true k.Consumer.Offsets.Initial = config.InitialOffset.asSaramaOffset() - k.Consumer.Retry.Backoff = config.RetryBackoff + k.Consumer.Retry.Backoff = config.ConsumeRetryBackoff k.Consumer.MaxWaitTime = config.MaxWaitTime k.Consumer.Fetch.Min = config.Fetch.Min diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 3503b4b6e21..3c0e4fed261 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/filebeat/input" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/kafka" "github.com/elastic/beats/libbeat/logp" "github.com/pkg/errors" @@ -43,10 +44,10 @@ func init() { // Input contains the input and its config type kafkaInput struct { config kafkaInputConfig + saramaConfig *sarama.Config context input.Context outlet channel.Outleter consumerGroup sarama.ConsumerGroup - sessionState *kafkaSessionState log *logp.Logger runOnce sync.Once } @@ -101,19 +102,13 @@ func NewInput( if err != nil { return nil, errors.Wrap(err, "initializing Sarama config") } - consumerGroup, err := - sarama.NewConsumerGroup(config.Hosts, config.GroupID, saramaConfig) - if err != nil { - return nil, errors.Wrap(err, "initializing kafka consumer group") - } input := &kafkaInput{ - config: config, - context: inputContext, - outlet: out, - consumerGroup: consumerGroup, - sessionState: sessionState, - log: logp.NewLogger("kafka input").With("hosts", config.Hosts), + config: config, + saramaConfig: saramaConfig, + context: inputContext, + outlet: out, + log: logp.NewLogger("kafka input").With("hosts", config.Hosts), } return input, nil @@ -146,33 +141,52 @@ func (state *kafkaSessionState) setSession(sess sarama.ConsumerGroupSession) { state.mutex.Unlock() } +func (input *kafkaInput) runConsumerGroup() { + // Sarama uses standard go contexts to control cancellation, so we need + // to wrap our input context channel in that interface. + context := doneChannelContext(input.context.Done) + handler := &groupHandler{ + version: input.config.Version, + outlet: input.outlet, + } + + // Create a consumer group and listen to its error channel. + consumerGroup, err := + sarama.NewConsumerGroup( + input.config.Hosts, input.config.GroupID, input.saramaConfig) + if err != nil { + input.log.Errorw( + "Error initializing kafka consumer group", "error", err) + return + } + go func() { + for err := range consumerGroup.Errors() { + input.log.Errorw("Error reading from kafka", "error", err) + } + }() + + err = consumerGroup.Consume(context, input.config.Topics, handler) + if err != nil { + input.log.Errorw("Kafka consume error", "error", err) + } +} + // Run starts the input by scanning for incoming messages and errors. func (input *kafkaInput) Run() { input.runOnce.Do(func() { - // Track errors - go func() { - for err := range input.consumerGroup.Errors() { - input.log.Errorw("Error reading from kafka", "error", err) - } - }() - go func() { for { - handler := groupHandler{input: input} - - // Sarama uses standard go contexts to control cancellation, so we need - // to wrap our input context channel in that interface. - err := input.consumerGroup.Consume( - doneChannelContext(input.context.Done), input.config.Topics, handler) - if err != nil { - input.log.Errorw("Kafka consume error", "error", err) - } + // Try to start the consumer group event loop: create a consumer + // group client (wbich connects to the kafka cluster) and call + // Consume (which starts an asynchronous consumer). + input.runConsumerGroup() - // If Consume returned because the context was cancelled, don't resume. + // If runConsumerGroup returns, then either input.context.Done has + // been closed (in which case we should shut down) select { case <-input.context.Done: return - default: + case <-time.After(input.config.InitRetryBackoff): } } }() @@ -230,10 +244,13 @@ func (c channelCtx) Err() error { func (c channelCtx) Value(key interface{}) interface{} { return nil } type groupHandler struct { - input *kafkaInput + sync.Mutex + version kafka.Version + state kafkaSessionState + outlet channel.Outleter } -func (h groupHandler) createEvent( +func (h *groupHandler) createEvent( sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim, message *sarama.ConsumerMessage, @@ -250,7 +267,7 @@ func (h groupHandler) createEvent( "offset": message.Offset, "key": message.Key, } - version, versionOk := h.input.config.Version.Get() + version, versionOk := h.version.Get() if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { event.Timestamp = message.Timestamp if !message.BlockTimestamp.IsZero() { @@ -266,19 +283,19 @@ func (h groupHandler) createEvent( } func (h groupHandler) Setup(session sarama.ConsumerGroupSession) error { - h.input.sessionState.setSession(session) + h.state.setSession(session) return nil } func (h groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { - h.input.sessionState.setSession(nil) + h.state.setSession(nil) return nil } func (h groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { for msg := range claim.Messages() { event := h.createEvent(sess, claim, msg) - h.input.outlet.OnEvent(event) + h.outlet.OnEvent(event) } return nil } diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index e7ac3a39ef7..e022197813a 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build integration +// + build integration package kafka @@ -117,7 +117,8 @@ func TestInput(t *testing.T) { }, } for _, m := range messages { - writeToKafkaTopic(t, testTopic, m.message, m.headers, time.Second*20) + fmt.Printf("Would have sent %v\n", m) + //(t, testTopic, m.message, m.headers, time.Second*20) } // Setup the input config From 1a29d7b4ac0ab038bf86fdfab3747702aec6fe9f Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 12:14:37 -0400 Subject: [PATCH 34/50] refactor groupHandler / fix end-to-end ACK --- filebeat/input/kafka/input.go | 121 ++++++++---------- .../input/kafka/kafka_integration_test.go | 20 ++- 2 files changed, 73 insertions(+), 68 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 3c0e4fed261..6a5b7dd7f85 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -43,21 +43,13 @@ func init() { // Input contains the input and its config type kafkaInput struct { - config kafkaInputConfig - saramaConfig *sarama.Config - context input.Context - outlet channel.Outleter - consumerGroup sarama.ConsumerGroup - log *logp.Logger - runOnce sync.Once -} - -// A synchronized wrapper to read and write the kafka session, since it may -// change while ACKs are still pending. -type kafkaSessionState struct { - session sarama.ConsumerGroupSession - mutex sync.Mutex // Hold to access the session field - waitGroup sync.WaitGroup // Hold while using the session field + config kafkaInputConfig + saramaConfig *sarama.Config + context input.Context + outlet channel.Outleter + saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active + log *logp.Logger + runOnce sync.Once } // NewInput creates a new kafka input @@ -67,26 +59,16 @@ func NewInput( inputContext input.Context, ) (input.Input, error) { - // We create the empty session state first because it must be referenced by - // the ACK callback in the connector configuration. - sessionState := &kafkaSessionState{} - out, err := connector.ConnectWith(cfg, beat.ClientConfig{ Processing: beat.ProcessingConfig{ DynamicFields: inputContext.DynamicFields, }, ACKEvents: func(events []interface{}) { - sessionState.accessSession(func(session sarama.ConsumerGroupSession) { - if session == nil { - // The kafka connection is closed and / or is being rebalanced. - return - } - for _, event := range events { - if cm, ok := event.(*sarama.ConsumerMessage); ok { - session.MarkMessage(cm, "") - } + for _, event := range events { + if meta, ok := event.(eventMeta); ok { + meta.handler.ack(meta.message) } - }) + } }, }) if err != nil { @@ -114,33 +96,6 @@ func NewInput( return input, nil } -// A helper to safely use the current sarama session for the duration of the -// given callback. Used when ACKing messages outside the body of the main -// sarama callbacks. The session parameter may be nil if there is no active -// session. -func (state *kafkaSessionState) accessSession( - fn func(session sarama.ConsumerGroupSession), -) { - state.mutex.Lock() - state.waitGroup.Add(1) - session := state.session - state.mutex.Unlock() - defer state.waitGroup.Done() - fn(session) -} - -// A helper to safely set the session field after waiting on any pending -// operations. -func (state *kafkaSessionState) setSession(sess sarama.ConsumerGroupSession) { - state.mutex.Lock() - // Once we claim the mutex we still wait for any pending ACKs to be - // sent. (These may well fail if the session is ending, but that's better - // than calling a stale pointer.) - state.waitGroup.Wait() - state.session = sess - state.mutex.Unlock() -} - func (input *kafkaInput) runConsumerGroup() { // Sarama uses standard go contexts to control cancellation, so we need // to wrap our input context channel in that interface. @@ -150,7 +105,7 @@ func (input *kafkaInput) runConsumerGroup() { outlet: input.outlet, } - // Create a consumer group and listen to its error channel. + // Create a consumer group and make sure it's closed before we return. consumerGroup, err := sarama.NewConsumerGroup( input.config.Hosts, input.config.GroupID, input.saramaConfig) @@ -159,6 +114,13 @@ func (input *kafkaInput) runConsumerGroup() { "Error initializing kafka consumer group", "error", err) return } + input.saramaWaitGroup.Add(1) + defer func() { + consumerGroup.Close() + input.saramaWaitGroup.Done() + }() + + // Listen asynchronously to any errors during the consume process go func() { for err := range consumerGroup.Errors() { input.log.Errorw("Error reading from kafka", "error", err) @@ -196,8 +158,8 @@ func (input *kafkaInput) Run() { // Wait shuts down the Input by cancelling the internal context. func (input *kafkaInput) Wait() { input.Stop() - // Wait for the consumer group to shut down - input.consumerGroup.Close() + // Wait for sarama to shut down + input.saramaWaitGroup.Wait() } // Stop closes the input's outlet on close. We don't need to shutdown the @@ -243,13 +205,24 @@ func (c channelCtx) Err() error { } func (c channelCtx) Value(key interface{}) interface{} { return nil } +// The group handler for the sarama consumer group interface. In addition to +// providing the basic consumption callbacks needed by sarama, groupHandler is +// also currently responsible for marshalling kafka messages into beat.Event, +// and passing ACKs from the output channel back to the kafka cluster. type groupHandler struct { sync.Mutex version kafka.Version - state kafkaSessionState + session sarama.ConsumerGroupSession outlet channel.Outleter } +// The metadata attached to incoming events so they can be ACKed once they've +// been successfully sent. +type eventMeta struct { + handler *groupHandler + message *sarama.ConsumerMessage +} + func (h *groupHandler) createEvent( sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim, @@ -257,6 +230,10 @@ func (h *groupHandler) createEvent( ) beat.Event { event := beat.Event{ Timestamp: time.Now(), + Private: eventMeta{ + handler: h, + message: message, + }, } eventFields := common.MapStr{ "message": string(message.Value), @@ -282,17 +259,31 @@ func (h *groupHandler) createEvent( return event } -func (h groupHandler) Setup(session sarama.ConsumerGroupSession) error { - h.state.setSession(session) +func (h *groupHandler) Setup(session sarama.ConsumerGroupSession) error { + h.Lock() + h.session = session + h.Unlock() return nil } -func (h groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { - h.state.setSession(nil) +func (h *groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { + h.Lock() + h.session = nil + h.Unlock() return nil } -func (h groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { +// ack informs the kafka cluster that this message has been consumed. Called +// from the input's ACKEvents handler. +func (h *groupHandler) ack(message *sarama.ConsumerMessage) { + h.Lock() + if h.session != nil { + h.session.MarkMessage(message, "") + } + h.Unlock() +} + +func (h *groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { for msg := range claim.Messages() { event := h.createEvent(sess, claim, msg) h.outlet.OnEvent(event) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index e022197813a..52dd6cd3ad6 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// + build integration +// +build integration package kafka @@ -117,8 +117,7 @@ func TestInput(t *testing.T) { }, } for _, m := range messages { - fmt.Printf("Would have sent %v\n", m) - //(t, testTopic, m.message, m.headers, time.Second*20) + writeToKafkaTopic(t, testTopic, m.message, m.headers, time.Second*20) } // Setup the input config @@ -160,6 +159,21 @@ func TestInput(t *testing.T) { t.Fatal("timeout waiting for incoming events") } } + + // Close the done channel and make sure the beat shuts down in a reasonable + // amount of time. + close(context.Done) + didClose := make(chan struct{}) + go func() { + input.Wait() + close(didClose) + }() + + select { + case <-time.After(30 * time.Second): + t.Fatal("timeout waiting for beat to shut down") + case <-didClose: + } } func checkMatchingHeaders( From c9031774939cc68890854c90e163df62e7d880a2 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 12:34:27 -0400 Subject: [PATCH 35/50] Use strings for kafka headers --- filebeat/input/kafka/input.go | 4 ++-- filebeat/input/kafka/kafka_integration_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 6a5b7dd7f85..cf24311b521 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -174,8 +174,8 @@ func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { array := []interface{}{} for _, header := range headers { array = append(array, common.MapStr{ - "key": header.Key, - "value": header.Value, + "key": string(header.Key), + "value": string(header.Value), }) } return array diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 52dd6cd3ad6..4de11a69378 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build integration +// + build integration package kafka @@ -208,8 +208,8 @@ func checkMatchingHeaders( } key, _ := headerMap.GetValue("key") value, _ := headerMap.GetValue("value") - assert.Equal(t, expected[i].Key, key) - assert.Equal(t, expected[i].Value, value) + assert.Equal(t, string(expected[i].Key), key) + assert.Equal(t, string(expected[i].Value), value) } } From f5dd36060062067e7d57d9b12781237d478c0df5 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 12:43:30 -0400 Subject: [PATCH 36/50] Make kafka message keys strings on indexing --- filebeat/input/kafka/input.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index cf24311b521..e8f6b611e41 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -242,7 +242,7 @@ func (h *groupHandler) createEvent( "topic": claim.Topic(), "partition": claim.Partition(), "offset": message.Offset, - "key": message.Key, + "key": string(message.Key), } version, versionOk := h.version.Get() if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { From 4e73ac82ba5d4642cfc24a85affe2c180840835a Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 12:46:43 -0400 Subject: [PATCH 37/50] Adjust config parameter names --- filebeat/input/kafka/config.go | 42 +++++++++++++++++----------------- filebeat/input/kafka/input.go | 5 ++-- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index cbc5b98071c..091d0f86b90 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -34,20 +34,20 @@ import ( type kafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - GroupID string `config:"group_id" validate:"required"` - ClientID string `config:"client_id"` - Version kafka.Version `config:"version"` - InitialOffset initialOffset `config:"initial_offset"` - InitRetryBackoff time.Duration `config:"init_retry_backoff" validate:"min=0"` - ConsumeRetryBackoff time.Duration `config:"consume_retry_backoff" validate:"min=0"` - MaxWaitTime time.Duration `config:"max_wait_time"` - Fetch kafkaFetch `config:"fetch"` - Rebalance kafkaRebalance `config:"rebalance"` - TLS *tlscommon.Config `config:"ssl"` - Username string `config:"username"` - Password string `config:"password"` + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + ClientID string `config:"client_id"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` + ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` + ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` + MaxWaitTime time.Duration `config:"max_wait_time"` + Fetch kafkaFetch `config:"fetch"` + Rebalance kafkaRebalance `config:"rebalance"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` } type kafkaFetch struct { @@ -92,12 +92,12 @@ var ( // were chosen to match sarama's defaults. func defaultConfig() kafkaInputConfig { return kafkaInputConfig{ - Version: kafka.Version("1.0.0"), - InitialOffset: initialOffsetOldest, - ClientID: "filebeat", - InitRetryBackoff: 30 * time.Second, - ConsumeRetryBackoff: 2 * time.Second, - MaxWaitTime: 250 * time.Millisecond, + Version: kafka.Version("1.0.0"), + InitialOffset: initialOffsetOldest, + ClientID: "filebeat", + ConnectBackoff: 30 * time.Second, + ConsumeBackoff: 2 * time.Second, + MaxWaitTime: 250 * time.Millisecond, Fetch: kafkaFetch{ Min: 1, Default: (1 << 20), // 1 MB @@ -139,7 +139,7 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k.Consumer.Return.Errors = true k.Consumer.Offsets.Initial = config.InitialOffset.asSaramaOffset() - k.Consumer.Retry.Backoff = config.ConsumeRetryBackoff + k.Consumer.Retry.Backoff = config.ConsumeBackoff k.Consumer.MaxWaitTime = config.MaxWaitTime k.Consumer.Fetch.Min = config.Fetch.Min diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index e8f6b611e41..32db99aea92 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -144,11 +144,12 @@ func (input *kafkaInput) Run() { input.runConsumerGroup() // If runConsumerGroup returns, then either input.context.Done has - // been closed (in which case we should shut down) + // been closed (in which case we should shut down) or there was an + // error, and we should try running it again after the backoff interval. select { case <-input.context.Done: return - case <-time.After(input.config.InitRetryBackoff): + case <-time.After(input.config.ConnectBackoff): } } }() From f2441d83c31dcd58704af789f1241e198e198f0a Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 15:02:41 -0400 Subject: [PATCH 38/50] Update changed config fields in docs --- filebeat/docs/inputs/input-kafka.asciidoc | 7 ++++++- filebeat/input/kafka/_meta/fields.yml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/filebeat/docs/inputs/input-kafka.asciidoc b/filebeat/docs/inputs/input-kafka.asciidoc index 7af2b2c9ce0..a8e71095d11 100644 --- a/filebeat/docs/inputs/input-kafka.asciidoc +++ b/filebeat/docs/inputs/input-kafka.asciidoc @@ -68,7 +68,12 @@ The version of the Kafka protocol to use (defaults to `"1.0.0"`). The initial offset to start reading, either "oldest" or "newest". Defaults to "oldest". -===== `retry_backoff` +===== `connect_backoff` + +How long to wait before trying to reconnect to the kafka cluster after a +fatal error. Default is 30s. + +===== `consume_backoff` How long to wait before retrying a failed read. Default is 2s. diff --git a/filebeat/input/kafka/_meta/fields.yml b/filebeat/input/kafka/_meta/fields.yml index 7d10108d558..3803f635939 100644 --- a/filebeat/input/kafka/_meta/fields.yml +++ b/filebeat/input/kafka/_meta/fields.yml @@ -33,5 +33,5 @@ - name: kafka.headers type: array description: > - The array of kafka headers, each an object containing subfields + The array of kafka headers, each an object containing string subfields "key" and "value". From 6ad363ecb252587844e9853eae7f6679e9bc4ead Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 15:24:19 -0400 Subject: [PATCH 39/50] Add IsolationLevel config option --- filebeat/input/kafka/config.go | 30 +++++++++++++++++++ .../input/kafka/kafka_integration_test.go | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 091d0f86b90..7e2ea6acb0a 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -43,6 +43,7 @@ type kafkaInputConfig struct { ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` MaxWaitTime time.Duration `config:"max_wait_time"` + IsolationLevel isolationLevel `config:"isolation_level"` Fetch kafkaFetch `config:"fetch"` Rebalance kafkaRebalance `config:"rebalance"` TLS *tlscommon.Config `config:"ssl"` @@ -77,6 +78,13 @@ const ( rebalanceStrategyRoundRobin ) +type isolationLevel int + +const ( + isolationLevelReadUncommitted = iota + isolationLevelReadCommitted +) + var ( initialOffsets = map[string]initialOffset{ "oldest": initialOffsetOldest, @@ -86,6 +94,10 @@ var ( "range": rebalanceStrategyRange, "roundrobin": rebalanceStrategyRoundRobin, } + isolationLevels = map[string]isolationLevel{ + "read_uncommitted": isolationLevelReadUncommitted, + "read_committed": isolationLevelReadCommitted, + } ) // The default config for the kafka input. When in doubt, default values @@ -98,6 +110,7 @@ func defaultConfig() kafkaInputConfig { ConnectBackoff: 30 * time.Second, ConsumeBackoff: 2 * time.Second, MaxWaitTime: 250 * time.Millisecond, + IsolationLevel: isolationLevelReadUncommitted, Fetch: kafkaFetch{ Min: 1, Default: (1 << 20), // 1 MB @@ -141,6 +154,7 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k.Consumer.Offsets.Initial = config.InitialOffset.asSaramaOffset() k.Consumer.Retry.Backoff = config.ConsumeBackoff k.Consumer.MaxWaitTime = config.MaxWaitTime + k.Consumer.IsolationLevel = config.IsolationLevel.asSaramaIsolationLevel() k.Consumer.Fetch.Min = config.Fetch.Min k.Consumer.Fetch.Default = config.Fetch.Default @@ -220,3 +234,19 @@ func (st *rebalanceStrategy) Unpack(value string) error { *st = strategy return nil } + +func (is isolationLevel) asSaramaIsolationLevel() sarama.IsolationLevel { + return map[isolationLevel]sarama.IsolationLevel{ + isolationLevelReadUncommitted: sarama.ReadUncommitted, + isolationLevelReadCommitted: sarama.ReadCommitted, + }[is] +} + +func (is *isolationLevel) Unpack(value string) error { + isolationLevel, ok := isolationLevels[value] + if !ok { + return fmt.Errorf("invalid isolation level '%s'", value) + } + *is = isolationLevel + return nil +} diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 4de11a69378..08212651496 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// + build integration +// +build integration package kafka From df98c90e3c3a1e46d2748a8e7cc2bd9e6d0dbf5e Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 1 Aug 2019 15:35:29 -0400 Subject: [PATCH 40/50] Document IsolationLevel --- filebeat/docs/fields.asciidoc | 2 +- filebeat/docs/inputs/input-kafka.asciidoc | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 0b2a8373341..4ef4af8db9f 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -7379,7 +7379,7 @@ type: date *`kafka.headers`*:: + -- -The array of kafka headers, each an object containing subfields "key" and "value". +The array of kafka headers, each an object containing string subfields "key" and "value". type: array diff --git a/filebeat/docs/inputs/input-kafka.asciidoc b/filebeat/docs/inputs/input-kafka.asciidoc index a8e71095d11..0c0b7a7afc5 100644 --- a/filebeat/docs/inputs/input-kafka.asciidoc +++ b/filebeat/docs/inputs/input-kafka.asciidoc @@ -82,6 +82,15 @@ How long to wait before retrying a failed read. Default is 2s. How long to wait for the minimum number of input bytes while reading. Default is 250ms. +===== `isolation_level` + +This configures the Kafka group isolation level: + +- `"read_uncommitted"` returns _all_ messages in the message channel. +- `"read_committed"` hides messages that are part of an aborted transaction. + +The default is `"read_uncommitted"`. + ===== `fetch` Kafka fetch settings: From a90d7c5bbae5bfcec8210298086de77a39324037 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 8 Aug 2019 15:32:34 -0400 Subject: [PATCH 41/50] working on corrected index template --- filebeat/_meta/fields.common.yml | 42 +++++++++ filebeat/docs/fields.asciidoc | 129 ++++++++++++-------------- filebeat/include/fields.go | 2 +- filebeat/input/kafka/_meta/fields.yml | 37 -------- 4 files changed, 104 insertions(+), 106 deletions(-) delete mode 100644 filebeat/input/kafka/_meta/fields.yml diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml index 5674e7f9930..555438689c9 100644 --- a/filebeat/_meta/fields.common.yml +++ b/filebeat/_meta/fields.common.yml @@ -150,3 +150,45 @@ type: keyword description: > Name of organization associated with the autonomous system. + + - name: kafka + type: group + fields: + - name: topic + type: keyword + description: > + Kafka topic + + - name: partition + type: long + description: > + Kafka partition number + + - name: offset + type: long + description: > + Kafka offset of this message + + - name: key + type: keyword + description: > + Kafka key, corresponding to the Kafka value stored in the message + + - name: block_timestamp + type: date + description: > + Kafka outer (compressed) block timestamp + + - name: headers + type: nested + description: > + Kafka headers for this message. + fields: + - name: key + type: keyword + description: > + The key for a kafka header + - name: value + type: keyword + description: > + The value for a kafka header diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4ef4af8db9f..a3c3042baba 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -31,7 +31,6 @@ grouped in the following categories: * <> * <> * <> -* <> * <> * <> * <> @@ -7319,73 +7318,6 @@ type: text -- -[[exported-fields-kafka-input]] -== Kafka Input fields - -Kafka metadata added by the kafka input - - - -*`kafka.topic`*:: -+ --- -Kafka topic - - -type: keyword - --- - -*`kafka.partition`*:: -+ --- -Kafka partition number - - -type: long - --- - -*`kafka.offset`*:: -+ --- -Kafka offset of this message - - -type: long - --- - -*`kafka.key`*:: -+ --- -Kafka key, corresponding to the Kafka value stored in the message - - -type: keyword - --- - -*`kafka.block_timestamp`*:: -+ --- -Kafka outer (compressed) block timestamp - - -type: date - --- - -*`kafka.headers`*:: -+ --- -The array of kafka headers, each an object containing string subfields "key" and "value". - - -type: array - --- - [[exported-fields-kibana]] == kibana fields @@ -7895,6 +7827,67 @@ type: keyword -- + +*`kafka.topic`*:: ++ +-- +Kafka topic + + +type: keyword + +-- + +*`kafka.partition`*:: ++ +-- +Kafka partition number + + +type: long + +-- + +*`kafka.offset`*:: ++ +-- +Kafka offset of this message + + +type: long + +-- + +*`kafka.key`*:: ++ +-- +Kafka key, corresponding to the Kafka value stored in the message + + +type: keyword + +-- + +*`kafka.block_timestamp`*:: ++ +-- +Kafka outer (compressed) block timestamp + + +type: date + +-- + +*`kafka.headers`*:: ++ +-- +Kafka headers for this message. + + +type: nested + +-- + [[exported-fields-logstash]] == logstash fields diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go index e4db3741d4e..4e1fd26a98d 100644 --- a/filebeat/include/fields.go +++ b/filebeat/include/fields.go @@ -32,5 +32,5 @@ func init() { // AssetFieldsYml returns asset data. // This is the base64 encoded gzipped contents of fields.yml. func AssetFieldsYml() string { - return "eJzsfWtzG7ly6Pf9Fbjaqis7oaiH5ccqdZKrI3t3lfVDseRsTrIpEZwBSaxmgFkAQ5p76/73W+jGax6kKFv0sSs6p2otkjPoRqPRaPTze/Lr6fu3529/+l/kpSRCGsJyboiZcU0mvGAk54plplgOCDdkQTWZMsEUNSwn4yUxM0ZenV2SSsnfWWYG331PxlSznEgB38+Z0lwKcjg8GB4Ov/ueXBSMakbmXHNDZsZU+mR/f8rNrB4PM1nus4Jqw7N9lmliJNH1dMq0IdmMiimDr+ywE86KXA+/+26P3LDlCWGZ/o4Qw03BTuwD3xGSM50pXhkuBXxFfnTvEPf2yXeE7BFBS3ZCdv+P4SXThpbV7neEEFKwOStOSCYVg8+K/VFzxfITYlSNX5llxU5ITg1+bMDbfUkN27djksWMCSATmzNhiFR8yoUl3/A7eI+QK0trruGhPLzHPhpFM0vmiZJlHGFgAfOMFsWSKFYpppkwXEwBkBsxgutdMC1rlbEA/3ySvIC/kRnVREiPbUECeQbIGnNa1AyQDshUsqoLC8YN64BNuNIG3m+hpVjG+DxiVfGKFVxEvN47muN6kYlUhBYFjqCHuE7sIy0ru+i7RweHz/YOnu4dPbk6eHFy8PTkyfHwxdMn/7mbLHNBx6zQvQuMqynHlovhC/zzGr+/YcuFVHnPQp/V2sjSPrCPNKkoVzrM4YwKMmaktlvCSELznJTMUMLFRKqS2kHs925O5HIm6yKHbZhJYSgXRDBtlw7RAfa1/zstClwDTahiRBtpCUW1xzQg8MoTaJTL7IapEaEiJ6ObF3rkyNGipHuPVlXBM4qznEi5N6bK/cTE/MRu+LzO7M8JfUumNZ2yNQQ27KPpoeKPUpFCTh0dgB3cWG7xHTXwJ/uk+3lAZGV4yf8MbGfZZM7Zwm4JLgiFp+0XTAWiWHDaqDoztSVbIaeaLLiZydoQKiLXN3AYEGlmTDnpQTJc2UyKjBomEsY30iJREkpmdUnFnmI0p+OCEV2XJVVLIpMNl+7Csi4Mr4owd03YR67tjp+xZQRYjrlgOeHCSCJFeLq9I35mRSHJr1IVebJEhk7XbYCU0flUSMWu6VjO2Qk5PDg67q7ca66NnY97TwdON3RKGM1mfpbNzfpfO5F/dgZkh4n50c5/p1uVTplATnFS/TR8MVWyrk7IUQ8fXc0YvhlWye0iJ1spoWO7yCgFJ2ZhN4+Vn8aebxPP+2JpaU7tJiwKu+0GJGcG/5CKyLFmam6XB9lVWjabSbtSUhFDb5gmJaO6Vqy0D7hhw2PtzakJF1lR54z8lVErBmCumpR0SWihJVG1sG87uEoP4UCDiQ7/wU3VDalnVkaOWRTHwNkWf8oL7XkPiaRqIew+kUggi1syP7/fFzOmUuE9o1XFLAfaycJODVMFwW4JIBw3TqQ0Qhq75n6yJ+QcwWVWEZATnDTsW7sRBxG/oWUF4hSRMaNmmOzf04s3oJK4g7M5IbfitKr27VR4xoYk8kYqfHPJPOlA6oKeQfgEuYVrYo9XYmZK1tMZ+aNmtR1fL7VhpSYFv2HkFzq5oQPynuUc+aNSMmNaczH1i+Ie13U2s0L6tZxqQ/WM4DzIJZDbkQw3IjA5kjBoK3F3sGrGSqZocc291HH7mX00TORRFnV29cp93d5LrzwMwnO7RSacKWQfrh0hH/EJSCAQU/px4Guv09iTTJWgHXgFjmZKanv4a0OV3U/j2pARLjfPR7AediUcMRKh8YIeT54eHEwahGhPP4izz5r6B8H/sOrN3ecdjlvLosjY8N4CzvUxI8DGPF85vbwxPfvfbUzQaS2wv1KJ0FlBTSg+heIQj6ApnzNQW6hwr+HT7ucZK6pJXdhNZDe1m2EY2Cwk+dFtaMKFNlRkTo1pySNtAYNQskzijlMSj1NWUUWdCuKmr4lgLMf7x2LGs1kXVNjZmSwtMKteJ/M+n1jF10semCqKJP+VnBgmSMEmhrCyMsvuUk6kbKyiXahtrOLVslqzfF7aWQBEG7rUhBYL+0+grVUF9cyzJi6r08bxXXuaDyNpRJDZgarxWWRxB2LM4iNwhPFJY+HjirUZoLH4Jc1m9krQJXE6jqezu2xugdT/7q6xTWK3cHo2PBge7KnsKFFjsoK39Jiz+M0aRebUvWkZLmcTUPgorhwX3HBqJAglSgQzC6lurKYjGChUdtd53FBBUWxKVQ4Hlz2XpNCD5Hk8tMYcb/pcWs13UsiFvaFZna6hNl+dXbhRcVdENDu42S/s4wlmIEU0E0Fdsc9c/u0tqWh2w8wj/XgIUFDTrpQ0MpNFBxTeaO2x0gDq9SwF13VmL0VeE/BUMooKTQGZIbmUJQtnc61RxzFMlWTHX9Ol2olavWITphqoiNYENaoZ7meng+LKjlnQwUAHTQiAKBCLlpj6ZY4gUvxRm3ZM5AHYnVPr2hLEjRqVPy4ser/XAhcAdEHU7rwRpWewSF8hTWdIK9RxvfZgj/nba7jz4nj7Hk6wUoCsxmPCXoQ1K6kwPAMlnX007kRhH1FXGKAA/y5Idn+uGEnm3E6X/8miYm8nyhQo+5qbmrrlOJ+QpaxVgDGhReGZjwt/rBk2lWo5sI96gagNLwrChFVtHd+iacQKzZxpY9nDktQSbMKLIuhctKqUrBSnhhXLOyh1NM8V03pb+hxwO2rwjrccQCd7g5gpx3xay1oXS+RmeCcI7IUli5YlA5MQKewFkApyfjEglOSytAsgFaGkFvwj0dLyyZCQv0XKuiMCbBZRK5gxoujC4+T5fjR0X4yQZM0TTtgLQDzA8hptFngDHQ15NbKojIaI1sje4iomcqdioH4gRUQCrhNuxfyqjJeG6VuOlEIGVR9vFs3XGuvwV/sD3iqCYc+th702W3GAt4H28XL44riBGE5qC4ed2784/rABc8rkMONmeb0lxfSMmyWA6sz+jRRGMVp00ZHCcMGE2RZObxMlOQDr4PdWKjMjpyVTPKM9SNbCqOU11/I6k/lWSIcgyPnlO2JBdDA8O12J1rZW06HUu6BnVNC8S6lCZqlKvwqdKZPXleRBLjWNUlJMualzlNUFNfChg8Hu/yU7hRQ7J2Tv+ZPhs8PjF08OBmSnoGbnhBw/HT49ePrD4Qvy/3Y7SHbpdX9i+oNmas/L4uQn1PY8eQbE6d54AssJmSoq6oIqbpapUF2SzAp3UDkS4XnmZWa42SCHc4WnacaEYcopXpNCSkVEXY6ZGoAmP+NRrdFhUESvINVsqbn9w1vWMr+tdYLCW2kS7wHYDbkgtDayBBE+ZdLPtqv/j6U2UuzlWWdtFJtyKba5094DhHUbbe/fzlbhtaWt5nDq3Wn/VrMxaxKKV7fgEB5oMuf5RTigvUSEwyLlLDQCSMHs2RtM2ucX82P7xfnF/FlUPFpnbUmzLdDmzenZKqxT4KjS3uGobwC5wLc/6WA/auIhlbm7vqGN4iswk8qsm3etmRqykvJiSyLNSjQCAPwy9CAwqYuiZ3PcKxK7mlgwABbkGJ1TXtBx0d0zp8WYKUNecaENc1pWA19Q5Ydbs752LZATZ20HwMFIAjfH/aqgxjJCD10Rzy0SNlWPEFgXiRnVs62dl0gpC4dYOHazZVIpZi+rDVP/BK8l9kF70AgplqnjEPdSIsk+aObMmCOYBc/xOgEf7OxGwb2USTHBtaJFA6ZVQDIq4jWaeHdwS/Q5CFsQf+9akrhus1aQioBDF6stHVmXMyuYUPcA1w8XXUSSLUlhSzZsa7JGkMG05r9YbVnDKBCC7JF7yQxDETAXTRQNruHo9MIrMlqMveQFuzFZ6eSakDfMKJ6h8Vmnxm0qyKuzIzRtWw6ZMJPNmAbVKxmdcKOdXzEiabmr6Q5v+DW5DkbTJgpuXFUL57BUrJQmmFiJrI3mOUsgtTFDnChxHjU/Ib/oIr7q1Mam5x4HjQOB69AB96ejHZbriKoj2F2MKBlcarYnmXevIoEQFrhM1ZQK/iduep4HN7jbZUuS88mEqdSQAsoxB+cvobg99wwTVBjCxJwrKcqmZhV56/TXywCc5wPyk5TTgiH/k3fvfyLnOTqqwYza2fBddfrZs2fPnz9/8eLFDz/80CQnnpC8sJf+P6Ot5L6peprAIRaOpQoaaICnYavETdQRDrXeY1SbvcOWnuu8C9tjh3PvVTp/6aUX4Oo3YRtRvnd49OT46bPnL344oOMsZ5ODfoy3eGQHnFP/XxfrRCuHL7turHvD6I2XA4lHay0ZzdGwZDmvy6bqrOSc5yFwYZuqDkoAD3DoN2calEUXekDon7ViAzLNqkHYyFKRnE+5oYXMGBXdk26hG9PCq+OWJuVujp+43dLjGAW9o74/khtfrnF4hQebTg3nbujEzCVhPBXL+IT7i2PAAm32zi/lTPdykg6SBGAyzTzcGSuqRIGE8wpDWsPQ2p2EYmkJZHjJ7nBAbUXHc0pwnDzPm3uYl3S6VZmS7g0AFuyliNCCajKueWHscd6DmqHTLWEWOcvhRadNBJKo0PXQk+jQNfGhbWELQF2oZQPuFlcjzjlahII0QZbdljjB0UlJBZ1a7Q3kSeCDjiTBqNREjCSutVSQvGx9vUaUJI+ud8Gi9pw8DSZWtAPtN6Mze8ZMvK63+VtR+jh/69foEGz4MzfyCkY1FgO678krGIYF7+D/bK9guijegugi91ub6Iu5BtNt8OAffPAP3g9KD/7BzWn24B988A9+S/7B5BD71pyEDdTJlj2Fdzjsv5y7cCUFHnyGDz7DB58hefAZfms+Q0wUb6WKr7MmvGGG7qWr4+2NLhUdQW5ym78tO6Enxfzz8reS9HtQyFzsr4TJaGLkkIxYpofuoRFm+3g0IoeDG88yZVlrgzlPsBmKTuQ3Ib/a6/cfNVNLCGXHZK/ARlzkPGOa7O25a3ZJlx4hyPYv+HRmij5vWTIbeN8VKLCoFfY05cKwqXIR5jT/3aLqz9Fsxkraoj9pZOHqrgZ5ODwYHqSco5RsmLZfhS/WJ6RG03IG2UsuGB4HhH1ExZLccBHNGB8wF6HE/Cl8DszZmHppiVcw9M1aMvs0VJBRGdVMx5xNPy1Ye240KybRJUsFjn4Hm9SWdGYgJgzu7w1oO2QOwaZ2ukUTes/p2YNBmui+Go2Q7N47WZ+2nfLYvJUs9Gq+YdIzrm+f68QnPvR7TwrplUD0siieNXglsOQp5NE3s5Es+3iZYhnKLlmSZwzmwBmuI41pw15Iv475/iBYfA40JOHwktkbrHdJ2W/tQGGMmDotJ8kk3Hh+KOpTcQlkm/roCxdTEXOnUKEnY4YpUk4vd2NSb781ktBUJR6gRbMnAWvMzIIxC8lnWojcBU4E5yQCc7lLmEydFdIe8uTUr8Tt5MYblBuylIrZazjYmAoYETNb4GOakQ4I9RM6ecwNG3O6G1RPuSWSvGSlVEtihRxkzrjh8oTwkeHmdSGYQrc/j0nz7mFtlSCWY8r8XSJANrAPfXLkB45OMlph7QiXLtn0Frjs2WABcWlqcQPypCTMkJyDnxJWL2oXMyrICB/w+UmjmIoZFsLu9REQZI/m+WhARo7l94DlGXw14QXbyxSzjDbCpB5fwCWMGDK1Pce5mXELpwRzT/eQtErXXkW1tsTcw7yt5nHhUN/GcrzCzeAgtIkfDrkZn85colq/DAQJCQfopLMqYUxYHciLay0OMsRo4NdUM6Fdwli0XtGAZsArjuy1I+pTCH+lym5uKJQwqSEQLag+cmJVoQFZMFIVFGwFLgiB0DBk4apy0CxjlYFkaReXgGeaV50GpMJyTLVm6KrKaN1vUIOVBqdeFA1hkZGzblnjUCmpvY6OyXGQTmhbfxklK5OgslCYs2IUeNbnpGNS6xKz/zq1hRyToAJptyq3Yj1zBplYDSrkCCZfxWV1uIYxg0TtKd4Uisq0RcW5IKXUJslaBKuqZaKFjIWXNPrYxqxHS8Yt7T9m0XWVNcsPZbTIwE/prDsFXYazCujkTjpXMQpUeHfoxOiVxtEBywKv+rIrSht/6rKc8FZtAI9JKQWPGbskGWJ3FzRZv2L2o48LM5LcMFaRukJmhZfSslVNqkKuOmDapKMVmajmZbQYpCsbnYY9t+2cGqrZbba2T5JkqT3EgWml8mdS2K2MRv6Re2ZEHlnJrpkh++441sw8tvzszeVYgsIqD0TX44g+XH9KmdcF0yDqGtsulZOoGdgVrJXltWLpq01xEYGmF35kkfgTgrGL6rCFh7siRhtqmoFPea02cfb02Ddbb3JR1eba/yiokJplMqahy9qkD1D9hhcF732mUizjGtbtsHcxXzrQjePEEisB26w3gRIBzmsgHX5mVmdUjNwIuRBp1bXIpaZ/1/stDdAF3t1x9CRWKdw5xCb2yFXCO6LakdttkQ2DWi4I39sDb576o6xUL6g9u7ACUSuIaYsmwZ+pnpFHFVMzWmmoQwT1eSZcTJmqFBfmsV1PRRfuzDDSLgAcrUaGCeSslEIbZacP9yWwSnCz7LHi+yjQvr9O/3r28otdec9f2tmEEJlEnW3h3Fui5oZvxECfrHDb8fsrprkzfMrnEETdVu0WTgVrh/0lLOl5Nh5uvgqcuwomtr41mmJLG4dvR3HMkRVszOrhtKCqHH2dCh4g2TRygNze9nnnTgd0Ga+tzIMVidJbVOPJZLT2+SdVKLnVnXi51H80w0a8qraNqb+nC7ALhdqCcgJucBW46YNTkdbIkhVKrJD2nMnZR4YyP5fZdRKPnHNtOSXH8x4cDKBOMqqyGcsjw45rQ3io9qTsQc7mXpcdXaOuNepS8pJV5PAHcvDi5OjZyeEBRhGfvfrx5OB/f394dPxPlyyr7QTwEzEzq/LjnULhd4dD9+jhgfsj7kypSqLrzCqWk7pANaSqWO5fwH+1yv5yeDC0/z8kuTZ/ORoeDo+GR7oyfzk8etL0ncraZHJ7oRpWfDkQqyRYo/ZqtBfYS0yGNqa4mXXzjG2MnFRU8tVtoq0GH3TSyZHQ1QGdUF7UivXKpDDiRrJpc5kUxt1cNiHOjbVTXN9c62RTrtqmk0LSXjPse65vCIyARfu4tMzZVNseseF0SLRjXKJlASjqx9EU80Ezd3kCxypcX9xVD/W1GVPtENyA+7WQqtyA/1ZOYvct2G34nyyHYW+Z0CCY1qxGPgmTOLBreXhw0FMArqRcYACO82wuZQ1rVmKEJhVghXRFjOCyTLXmU6EThHTz/miHWFDMjNbMco+I00CqOd8RLQpfoqmluGo2Z0k0070EP1y6MVumu7CgHmZLAfh1htFWUQ/0N/P4htsLJaMCJOucqeQGH3R2S1hw4VgpvRutRHXllZDEIAc3aXrDCJhaHSjOfLKi0FwbMD8jLb23rrW7dp+3CGuvCp99J8ALx623AmelTO8FDUlm7wfR2rPiYmCvNVtMTttNjtl4+UoKrDamtLuro7UhrS9K3AHt3BwO56bmWihG86UTOzmb0Low5HKprQIQTRiJ9DlHgwlgSgvM+FtwnZpCTqNADkARJDDKCVgnhRTgJTh/6YDvvKqVrNj+aakNUzktdx4ne3g8VmyOjgv/+OXVzmPwiAjy888nZRmZm9PCP7V38PTk4GDncWsvb6tC4nuG7AJHkNO0a/S6hbm4ivR0LiFvM+QsxKrjEP5hddNhWqF4wp1y7Hx1P/rPa8v6QU39ll+HaGa6lxRwmWkytlKhaWF1rif7K3jjvcMEzCsgK2PJPgvO1Q73Ch3VWmY8lgYGNc3X9GsUmtMDK633neXGyw10+MCCWvVEauaqgaPTAECee2WVvEFLnyXrf/14/ua/feVwHf1WLvMXiv+BYxu1Ha9adHM26GTC0LpqH2/Nx3NNUnLfGaPu4ubeMEVmlQx8TX3Re0CxZIZi3Cy4SFriK2d2+lsSXi9h8BXZcJimXbTUE4DdjVW5P3kKqxygtHWOkBBSyAVhVC8tioYBC42XSNDwck/kRuXO9hBdu7WIuwvFoaA7xtdZ0fnT+cvHqwkbeW7buKSZvV08uOhEcdxjcrHMWbMzhUfCu8hSOUWaBoetJRhbpBJ6WFRkZmjRqk7ZUY6OD581cbxfweAsSqDhlDLnE94WDnIhtpbQjKeDBbALJhPVzRasqNmWzfWCmplXars8qvmfm9B5VZQ1TM2OYVca0q7Io2AokfZCQ/Pc624jOxbEv4GrfPS4pV5SNWXmeoukuAIIQGzQOPSyLLi4aQU9bzEBH8gFxlJwKQ1IzhUoGQ6TFkXqrYnUKxfKCdL0A0hTFe/fSXTWo8uWqEVGTsOppkymCtpP7uMa/ewnJtNgvYwqe0mL9VVoNAn73JO0lAwVqY7UbPCTpKs0FD2nlOVM8WBjMyybgW0+tgywmJ1fJLEz6KRUe7quqoIHb+VGys3Xk6H31WfnfYWZeV9ZVt5Xn5H3kI33dWbjfY2ZeF9BFl73suDPr/DF6hPsKmT7JLHAJXOm1hh8Ds+4oHJovMAKNqdhczqtLHEDf0ppk68qs+lLpzOFoAWpGyHdP/vPa81EvgBPw0zkyvKTTJZVbTB82FWLCh2lzi4xXta3heo3WKYdoaJZBfs/xUJAzeQBH3sNaiGoKb1Bw2m4sJ0r0DXEB7sRZ1TlC6rYgMy5MjUtfKEnPSAvoSJIUm0HjFDkl3rMlGAG2gPl7E51NFQ244ZliVPrXpOlKh8s5xs5JPA6+/zji2fXz5rlGh6qJjxUTbg7Sg9VEzan2YOe9lA1YftVE+z5uSVMdn92Y6fVEdM4EpO02vM+14VzS5ORx2xkdYfS7l/FTK2wFGyn2OLuWq3uXlvsoZ6TFnA61YGOPqbJNYzBJOQBuMidNz3or1bF5WIKEQouIH1tEVXUlF1IM7oELWVH0J4PKNWmwqdVxAANiFf9RQy2U8niZ7eU/TC3xZ9v1/ImGNNc3jtwZcKRCSd+gOJgGO3hhCREev1R0wJM42FMV1IMqzJgGp5FwFnnYvYSZIXDWmt7kiiSs4znkCBrdVdgoyjYpX2+tfBSDye05MVyS0fTu0uC45NH3tanWD6jZkByNuZUDMhEMTbW+YAsuMjlIrr/YxU9eLKDd11sqz5HR+d19TFAy/c+H5997jN7+1VQmlkavJG/0zlrz+DGqvxfbA4ILaANdy5FFy5eqOsaGh4PD/YOD4/2XF5YG/stKjQr6O/DlxPqryL4f7Sx9dfmL4Wxh+f43upGUg9IPa6FqdfxOlUL3uH13uoK20N+Ux45PBgeHg8PG9huK9jFtwNtid8fpXKVwX21YteT1nkeGnXY7RDQ1HgUKiyPoJD8vBwkCjBEXie6brisD9KWr0kN8tTjEc/qMGLfmd1T6+Sh4lCTux4qDj1UHHqoOPR1VxyaGdOw4v98dXUBn+/So8S+FMJhh74+DBnVqhj5wFSG0dRJV01AUhUeX9cUd3N7vn9hLPPlsKfi7W0BGbdWvb1sxGc00SQAtU3eFy+er0bRBdNsaQ9fuesILsZaLH9mRSHJQqoi78d2C7S8koYWrYiXFkUfWWRhs88YtXpAV7k6PH7ST+CSmZncWqJfg6QIqpUAjUyOqQFQLmbM0pwBI0khF0xBzrcVob4G1ZBcMpcoK7O69HFeYWztSrbsnPuweqvlvTq73Omax6bMDEgFtWOq2vSSCVpEq60FbL13w8eUmpRyndW0skef7O+PCzkdum+HmSz3W7jrSgrNvvg+R7CbbvQUyS+709fhuXqre3y/9F532H7aZndIa0NNrXtMvZuivjrFpklTBNRv8T0+aLrJtnvFA7xW3ZkPh2mnE19vyp3or93HWw90tDnRRpkfCbmdaWbOJiczTH4bd8h3PtPJYhW8IK5SWCd7ETsINJKfF1SJ0YCMoGia/YP3JIoypRrT2WbCrU9ja+Rx2cn4BFzaLl4AWz95ItGJJ1ijqeAG3e+G1FAiJqitFVWNeojnaPdUNJYjHLlhveKGXJFaSKEJvi8gY0dMM/X8WrhR0gTRVn6om+ygMyGfABzGnNE5C7lH2i4qxiJnvp4ihhiiZYCJTGKzBEUEW5CCC6ahm9w8uaXY+03BqIDEtSbKn5u/TLR06cm7u6AH2LM+NQ6PvQUMtIXPTmMG9xs4Kt4s3d4P1nTMlkmlwdvkq1uK9vlcm2acB9pTyrIWjv4YFiznTHkJEoNKCK5CkrPj4jR02t3IP/FJUSF+9Fa1jnYWkS8UdJe4jAo7c2wx0+QUr25TPmcCI3RTqE7CVUoamcmiWaqIqjE3iqpo+icusdXlk0FJQo2bouSZkj6PaQAcSAstAdgSd358WN8sKxbNaTz7Y0AmNGNjKW8GxCy4Mei14Jos0opEVtTEMlGxyCeZM5En1ZQgZBq7KYbwYnvE5iGcOBRMwF2wn1vF+/wCY6j1AKqK6wFJxlxw5dMGv0LVnPJmJ7j77s+yiyoXqlpGUaFBEYcVGUu7b7hirn5bI7t/5CpTwZsu6T4tq+6/94V+BmTkN6v7Cc8uHldC12WXAE+evWgQwEkQs7zeXifMUzRlQalPyCgDoZ0Usj+/wEqTjpuoJgtWFE7Ihfn47RejFZrybxhS0SkxUhZ7dCqkNjyz2qPIqWp02gzDTgq5SBfjNaNKYNI6NeFqNOVmVo/hUmQZBEqr7Qfi7fF8z+pqPeWBT2bv/lG/Pf75H9/89PTN3/ZfzM7Vf1z8kR3/57/9efCXxlIE1tiCerPz0g/u9TQvro2ikwnPhr+J98zOB8svxeP05DdBfgvE+Y38A+FiLGuR/yYI+Qcia5N84sIwJWiBnywHxU+1AMb9Tfwmfp0xkY5Z0qpKChS7/rH28NrDlnplTA51dWoH4UBKFJt0zCC57DC7mkC8kp38nLPFEHFYAdiTRipSMcVLZphCRBpIb4ZTRKSBgf0XXBkOWDpyADrcabOTo32DbyZSLajKWX79OcEHSUuOkKfutmvyk1OQKyU/9tSq+uFoeDg8HDaLp3Aq6DWGL21JwJyfvj0lF146vAVQ5JHfuYvFYmhxGEo13ceDGWrb7nt5sofIdb8YfpyZskiS6C+dHIHzytcx8W9pJ39oATUtQIKBxvOWmR8LucDyavCXs9iGcQs59be+2pls++bUIXgz5XDbbhFUjsZLIsHLCcXGpT99dQxh8+dSG9ufwGr3K5/wBtqf1yXFHbhukE86ct27PYdu/KXn2PU/Rv3MHcD9B+9R00jhuWYbV9nXz/3tIp6ZEFNB2MchnGgDUgBH/U4zq0laotmzN2q4X5/mFvwjwT3usd4GCS8tw1MdeDkRYqi1gyuVxkIQjPyCcNJtGJoHRAoXdGmFU51XA2KyakB4NX+2x7OyGhBmsuHjr4/yJmsRfktxCed46Ly7PIc07AIP0UUaP+DZ+rWl4tDS7hgpmNySKs2yAal4CQT9+shpkU5MA65STaNlxLv0u3X5HyK83q0VUrGM08Jz8CAkx2IcXOdKjcUlQuHdnBmWmYEfH17C6iK3j7jXPN+ccpUUe21mvIYIEUqyWhtZhrQPHBRakIO32021VfNEigmf1rEViZFE1WJzAhAtJ8aCS2qhNdNQJlyxBS0KPbAarqohpAcpxKXYrxRMEYbyQYleh0y0RM2ElipUuFqwcQOLBAgEgRdSa9I3tCXk6cUbRw2dtln13JAacChWg15hv3ECCgfHMBKxHKSV4nCeOrCC9rVekB10VJjXkNhXWHFjujor5I2zrf5RsxoHJq+uXkPikhTANf6u50pFN9uYOHbylibFwDQIBa1yBv0BHD2gI+yrs8s7GJ0ekm0ekm3ujtJDss3mNHtItnlItvmmk23auTbh9G3aPz7NKNNtkdo//Bdrc9pQVB+yHh6yHh6yHh6yHu4/60EzxWmxXYOxv187YO68v62I1v01B/PdBlKxGpq6rCtsz5RLdrQXQ685eUN0HGlZMT3si7rxrgKVth3wF0+Iwsk1/FNp1yLs4xL+kEXBIEwHL7H2r3gF7YmN8GM2SNrwPt8nUcPMEUIasz5sYbC+t+o9sFQiWGLY0pQK/mdU9r2Zp/39LXEg6Tj+fs+E4tkMGQcu9qt6l5UVFf6Ulsrpqw2ma0VqpIEhsTfpjBUVlOWmSlEx9e16jKt8m/T8oQKDdMBj0IzaD2jE+dylTsffIU8lRfWL1YtJ+SOoB1GqN1gpiOBLEMG3sNMV2Flb7QJWsI5sSffNow+/Sc3wG1cLv2Gd8BtSCL9hbfCrVwUTD2lo5uGk3EXy1cbNtFcKt9D1t/+ky6iIp13MwXM252bvOwhsDE2Eeb6f8LILKmnE1YIA9h1YhxXk4k0ME0QbutS+/rHv7ovduGnonwUKYsXRUQOZioUc0yKpRO/RjQalzepfTTfJQPi0GDCl6NKFSwCRqJqCIy21k72BPpNOn8DpVUoalhlwnnDD540kyI7e6T7uER1SNPfIXhH+rHW4U+wR3/6nGUXBPrKshi4IWyLF6Ri6wzAM13Ur6KkSoXd2yH6t1f6Yi30/ty9Rt9LtOHcKhYWyVwtoM0EyWhQMUsanipYhAVLzkhe0pxNwG/nq1izRO2WNXIQt2D18jo6bgUlVB/bnZ61cUCgU45Zz106vD5HWlfczG6lc+S6rKSe5hildV8DRweGzvYOne0dPrg5enBw8PXlyPHzx9Ml/tjptzBSj+WYp4Xei0BUMTM5f3r5AIPW3zdkApBXvYmkI3w8wywFZHfykLi6kSvcFOaMCw7jHsc+mOQlDJqUOCCVjJRcabA8+OcQh4WXBgo1JRacs6aQqsZt9c4kWUt1wMb3G+KZO8+x7TXNzsEiA5c0X4QhtS6uZLNk+LbBhRUwci4EB7kx/n3y19kyPrXUY9kH31UonNOMFN/ZwrvhcYjtiJWvopV9xliUdrKA7i19sMJDAA7rdVsWFw2vGoAl7ScXSKmEZhAbYq+2rs0vf1ekqRcENjc3ywIaDN8hygFdjyCzwZyE0rbIgfJkq6RxTcH7rSoo87iKX/iLIyFFxOAozOYXGv4qZYPCxFIouBKYHSf7QmJEaihxBm/1gPRm4eM9BZAIfCTcgWcGhLZh/lIo8BEelAahQBATsA1UFPWWLgpxfeLXCyIg9r0YD1K0oqDvCEc1VNsBow/MLYhSfc1oUywERkpTUGEhwYeGY4AaAUcXyARkvQ9BOCuqEDsfDbJiP7mJm2KQFR7/z5rQI+XDnFxrXWIqkEXV6k+/G/1xuFv3jnuvJC3LM42pDhGCUTArhIpUmwRDnwikUm1KVY5yK1thePD6vsU06D7GUVt3EUNZMqqRR8Y9Skauzi9AXCIRmQBNxyxi3nx2BuOBQaOLyb29dGOcj7Qv2e7387CLBZQhAsF5MCL5tQ3I1cItlhx5++Zox8EL7foggFVywDaGZqb3TFiP5mCrJThhvB8slT4JamWIhWohrX2EMfnbXDO9b7mZUeVHiisVmKNh0C0Q6DyeQLhsAKPSyglm4EWMoEBb7+L0WWbzH4E53b/cNFkkbC4HEIe3uxWXcQ4e9z1l1T57h8Pt+Cs2+KnjtormV8iUVhmc+uN5lZbGP2BrJybN4I7JXtUld2Mfm3E6X/8kS86YgGVNwEYyJUV5WqQBjQovCyyrf0T+jhk2lWqKwcglx2vCiIExAQz14bEVqiyXYhFsd2Q1Lq0rJSnFqWLG8y+UMJfm21CF0FmCrPVyYcHRgUqUXMOWYT2tZ62KJ3AzvBFVnYcmiw+0AXBPUivEBob4YHxaugRJ+0vLJkJC/Rcq6Io5pfRLcVYouYhoC8v1o6L5wObJNNU7YkyEmMOY1hqPhvXJkzx8ogDNEtEYDkjN7ZEHKqi9uHZsFwjnD280l7zt/7K+QOAal12PqnfPquN7SsH+69pMXzfhynNQtmH1SoRvEBsdvta16CJl7CJl7CJl7CJl7CJn7pkPmPjFibbcbsuYD1iJn4fWz5Q8m5xfzY/vF+cX8WVQ8WmftF4t06wuz+7wstQuXnvYpB3vLaHl7wtPdDJYSyoasnPdDPc2HepoP9TTJQz3Nb62epitsAs8lZjX/1S2hVr4sSttIY9LfpOppcWQVJIfcgmqSyaKAHtS3hFNNuMhdiSnPnZAVjmwZ6oB52PZJH7GwuQ2BVTNWMkWLLRb7eOVhpOJJOq3Qo/+IT0AHgLbk+nG70hPPky4VYO7RhGZKak0UA8eWq50zcgPC7ssl9HwyXX3wBT2ePD04mDS1nG1sp92uaPYF92oh0LqKGHen7EwVuAOL0MR02SCdKzJQ0humCTekklrzMTqPAuuEoYGFksRL5FnBOgzV1/nCG/KVXaeKKc5EBg4rrWum0Vhox1IstxNwLcaiTR/d+GFc36ye51g2IIZSwD3MMzsa07iYQvNl17ass6L5k+fsKRtP2AFlz7LjH54f5WP2w+Tg8PkxPXz25Pl4/OLo+PnktgIJ99/TwnN4jOR1+78nmDe9WoUXIbzX8T6cRuAICbUlCrnQcMlayECeeMfyY0GPCy8qVGQ+rxjY30Mtd7wGiobzkjfqU7gmGWG3wfGW9mIpsNSaQ88uY86tzjmu7cx9vStcW1WDLyScODOpje5nXzTde1O1myzBkjBuKq3ABJdDDgncckJeFVQbnjnHUkJmmILLPPbHNCrhtTZMNa5K6NT4K6NGd4fg2lInZxNaFwYqElXBNxroZaBtNEjkMCafECGJHyM0JOkpgpjOYS9NeU3iB8xWLDSu7Q2M3+LTv0+w/J12F7zo/Z0urR31455ztiEk7YkOUjJRGPxMVkhKGCSmJMOua2LXZMZBizvCoKHewaix8H3VMdPfG8uxvTD33X/34anNBQmOlobO012VKMOg1oK8IdTuGgwdZwY7rrd0nnkESQP7dQubDY+GaV0F9Mc01L/4zRrtD5+63TvnHT6AFVoH9pt1T5sjJW64WxxwqfvIeeG+SjeRc3g9uIm+EjcRroezJqVljDompS/mK0KUHnxFD76i+0HpwVe0Oc0efEUPvqJvyleE1fi+NV+Rw5ps21e0+en+BR1GPZN/cBg9OIweHEbkwWH0rTmMaoUSy1kLPrx/DR9Xmwo+vH/tL/euYybRdQVVPjEHzwIygE5FFazlh/evXQE/92QIjJ8xMlaMYpKFXAjChZFEZzNmhQveoAaQMubel8TL/k3MAn1XvPvbNC/djd2RWxWD0EBgZ7FYDJ2lapjJnaatFrJrMgrWA6BnSZcYTu3Cfa2agNUGga4Yfl4sY+oubU6NuIwcsANDjwbNBi4OP9a3BpV1KkOnFXe1d9aBjorYnEKDrhNFp+X2Okzt2tM2MbfVqiB0Yly1kNH3o4TQRlY7LQvo6PuR75fi2sOgFu6QbsmMLWa+n0/wqLT8D3YiXtr1dAk8EIJdaxZXa5kYZLCiRJgXF9DOEE740YAsZgwSAUyjQ4ximRTaqBqskJZ7MMbcW4Sa1qhUjenpitZc/pPj4yf7aHP9lz/+0rDBfm9ks1Juf7+i+zyssP8OzNG1LAIW0SFzKcy2q1+/lcbFrnPRU690kJanycPuhDqtfjEHmIhDdbo8NIPUuEJO3a3Pvsq1y3D+vdYmBv37arVWsK3s9xMyvcJrYVgKTtAF1QHRQUPw9rqDP2lh7Wgrfm4p/1onK3nfa37hhu9t1hlxMNtSkC6gx1ADdiKDHIF2hrdcQe4h0Ta5hnTwOD5+0s0uPX7SQAqyxLa1Ma3wBQCOiYOFA/DFX3BuvXMI+8DStMVsHRn/LyDj2UcoWJy0m0ihQKYLnrCh95eQ9l3YoYkJHatLJbjDq8ZXnqIAb1yb8NQgAYaTxaCOMGLo+lRWJuIDqOOTI/d2y1XX8EWTMTMLxuIxD7lYC4nKQ+sgQ61pW2t7CaOv3gMgXXZachazaEcnvecx4rtCTnUU6C3fatOYhES4pBg01GR9e6LildPBO061/oJD8CieS9DcmM1pOKydxtZ0tP2YFOygc7QYMbAXpxcV+w1n2m0Ff8HDRj9mRgW8xnOf/epV+pCv605K2GbgxXRUKu8SgPV3tIt8QyaRb8Aa8vc2hDzYQG61gXx15o+v1vKhmbqmU38lSiQ7id9uIN9xDC/lYwSnveS7Kki++EU4WRxyV/bO50ogzeTCtUtdsHGIMIEAm6QuJlafoMpqC3VA1esXm4tk7HvxpXayg9ZeEn4x8yEEX6qbU8IhSLoOUpd0QhX/khfaD8It6LwZZRSZq8eb/ycvCrr/dHhAHiEZ/4mcXXxwJCXvLsnh0fUhNtT0tdwek9OqKtivbPwLN/vPDp4OD4eHT4M4efTLz1dvXg/wnZ9YdiMfExf3tH94NDwgb+SYF2z/8Omrw+MXjk77zw7apWwfimP3Yv1QHPuhOPbnYfw/tjj2dlH9967UXXE0WCn43Z4FckLGDFoFUZHNpMKPe5ksS0DT6RJ/xWca0P4ZBj3z5gh8BV4PIZP+8gDKZeFKibjy1t+tiH8EfFtNH/pIsraTg5t1Y2SL2dDwkv0Zo/1wYFrwYAGtqJmduPtp6+GSTxVFeEbVrDk6zqUxrBz/zrLQvhs+XN86k38Op1igLKyj75IF5HRRpU0MoBN/A4GoOK0E8sq+1Cq1CWVq8py7MkFWd4c4VxeTD3BCwbB0DUl/RPmqFVyDVkQtCdluLGSHO7qLaJkofW7t+sGgvWzXHbiXR9eODmGyDMwXPg9iU9a+4pgLwlnM0bFXI7d7s0LWedyoZ/ajt31ANDt1CW09lH7jfkV9PGu8qi0LsNynjtA8v4YHrv2QvnKcVOlWbswaXhhWSlrWj+aAIIXcL3sf1/Noqu66Vyw//iTltGA4Y+TGHuC8pFPWA5qWfI+Os/zw6EmvKI3Qz+0I5PxlsDEgnUJqE075e3Jq2QTzs4o8FQchpIkZOgwkASLfwme9D6/lswSGRzCmCq4HEyYUnr8zpA22TgvWpvsngebSnq4TAbMemHthmLywKSx3gPGCm+X1BsfG+rc2hep4fNOF6+yvTeFgHOJGMBqP9o7v5VEusxvgVSeQXvrPPdsLf4P0pHbSifvN7ms9k8pc4/l3Qia00CxRVxDeXhBGK9SKgBbpPR1XnWLuRExjcfqJlRCs/5Veoq0AZSXO3aGBpBNp89o7QW29uRnQTwdX0DErtBWcV+9evrMa3IIYSUpaWSGr2b90cGmoU2S9SkXWqxYo0xGFoedce55Hvv0ZP/UMcm71oYRb3bFgX/c5mcOEQaELfh97unPj1dllmmLEQ84Qy/RwWRZD9xymnVPlArWl2ItvtkzLiPp6Tl+9NA37rx9iLGXBqNiQvJNIEfA5xmXvwpV6OK550QXZXdFweu8cvnh5ePDDzmbovLskAKHZVqYPkUzmrHcfrMNFG8VMNtscGQ/Fd2sNHHhTj5kSDHOGHB/+kn7XM278PSh7Tc0tDkpSLlwvVeNLt0rWBtLrea5N8Urm/WLnTps5oUAlXfvuXlB1jwz/VEgXMicfzl92AUFuQ0Wz+5tUHLELTOYdkf+ZwLyxrgsMxeXtYnkzQE7+l7TqQgLnEJb4vC9wyZD9MBWDtEHNzP0SNI67gqw5qwq5hHC+ewUcx10BGLLCJ3Vx71NOBl4B+hat41MBh2FvBduvYn0+XBzXifPYAKXT/qRnXF/PPkjxcIXsk7ppc5W7iFz2cVMlzxeG7/TTID2Knpvx77KQN5zu0drInOtMztOrwL/ir+Sl+2VJ0udIcs+91VbRM1R65jk8wpCrjI3uuSEadJrG2TtY6rzdFfPiiJwEBBLraz9Mvs7qu8pmR7OZ85bOwAgdfNjNGvGM+xLblgg5yWtsY2+oMnXVMJWC2ilViamFwdYI/vqKKloyw6BY0piBddCuG7SVZxhfhl/YjxhOxnNATbM5VBKqqDIaQ6jOLwYkbXPB8wHEKICXqIESFTn2VgALYB8JXb27Ssm8zszdCXnl8nhx77phrFIW5rYO7CezSwPsrg4OhUcJ5Me3gE4aMd4RsmuxmKQx4/QTXtCh3kw769vj4XMt7gz9w/vXZGavelBJAsA5bgVM1hE9q1XLR9K8lKyA+msIMPfzwxIXyOLuAkdrM2PCcEwy9YHHXqwVchql2Gs5JRNeIMIYXLHOLVL4xwsuml6PxjQLOR3ax4ZJ6G8faRX7o+aK5VFlv4XgALtV4syiAlSAFjFpsHYILgVq9TU6AiQDhB9jZ5QTMtqfU7VfyOm+azRYyOlo2J2nC2RvVrr47MleNspZdKYsp8775OcNFexdhmYPknIy0awpU5L45k9bBhzTRWZWUhlo/ioYimRNaNtlZa+WtLwvClnOxRHJYsYEUMFu8igDMOTfbchdbXJZm127G+zfTKndJnpcVLVJ7aoRHdAKbqUKDIB1flrrFdcKuxXAQdPMKgBSIlNC+5VYgynA8LaZERZwkpg+77IqELh2/TScPPyRFwxcmCggHLs3Zo2+Qc3+qFnbjfM5HOIHTLQIEJJpqGzgi6UGgYFda5b3x6VuQMI+GkWjMRYPbC4VN8t+VPyv94aKHzDEWAOcddQAZYOb5TVcLe9Ths7qkuJ2Ae+rB7R+UbaOhgfUQiM0JcSObveJgGg64uzwPZJzUtBpb9GbdLD+Ewde9RD6lnpmTDXE5i2aDd0JfF0wMW2dmj3e4MarY5kvh2l5nrVOk1aA5e0XrmhTDHPuvrL6ltYO9e6uYAM9K6Rat/g7qmVB7LmhMFg+iN7VksiDLmVeF5tFVTQeXUt2y+rX4Do3tKw2GjxTLKl8uHZ0dAkNqTHqXuM20nEjezt7l73UMDHnSgowzcyp4nY3a7JQ3BgmoBohjLCryb9evnsLa2M31hRSixWfx5DgVjFaqhgWYIuBN3DQQ0ImFrQLWq87BJvjuhOyE1mCpI01VO582clZOIvTPqotdSIrq0+Ecn725sKVYOkO2fEUbz5kT/QJn376kD/1D+kVYj3Es3/VIbpi2NPaSCFLWesQEQjDtKCkNUe2DCpOqNF3/dPkkw//aHTPb/W6x/tKG7n1BPg7o/b/AwAA//9M7Ht5" + return "eJzsfWtzG7ly6Pf9Fbjeqis7oUYPy16vUie5OrZ3Vzl+KJadzUlOSgRnQBKrGWAWwJDm3rr//Ra68ZoHKcoWfeyKkqqz1nAG3Wg0Go1+fk9+PXv35vzNz/+LvJBESENYwQ0xc67JlJeMFFyx3JSrEeGGLKkmMyaYooYVZLIiZs7Iy+eXpFbyN5ab0XffkwnVrCBSwPMFU5pLQY6yw+wo++57clEyqhlZcM0NmRtT69ODgxk382aS5bI6YCXVhucHLNfESKKb2YxpQ/I5FTMGj+ywU87KQmfffbdPrtnqlLBcf0eI4aZkp/aF7wgpmM4Vrw2XAh6Rn9w3xH19+h0h+0TQip2Svf9jeMW0oVW99x0hhJRswcpTkkvF4G/Ffm+4YsUpMarBR2ZVs1NSUIN/tuDtvaCGHdgxyXLOBJCJLZgwRCo+48KSL/sOviPkvaU11/BSEb5jH42iuSXzVMkqjjCygHlOy3JFFKsV00wYLmYAyI0YwQ0umJaNylmAfz5NPsDfyJxqIqTHtiSBPCNkjQUtGwZIB2RqWTelBeOGdcCmXGkD33fQUixnfBGxqnnNSi4iXu8czXG9yFQqQssSR9AZrhP7SKvaLvre8eHR0/3DJ/vHj98fPjs9fHL6+CR79uTxf+4ly1zSCSv14ALjasqJ5WJ4gP+8wufXbLWUqhhY6OeNNrKyLxwgTWrKlQ5zeE4FmTDS2C1hJKFFQSpmKOFiKlVF7SD2uZsTuZzLpixgG+ZSGMoFEUzbpUN0gH3t/52VJa6BJlQxoo20hKLaYxoQeOkJNC5kfs3UmFBRkPH1Mz125OhQ0n1H67rkOcVZTqXcn1DlfmJicWo3fNHk9ueEvhXTms7YBgIb9tEMUPEnqUgpZ44OwA5uLLf4jhr4k33T/Twisja84n8EtrNssuBsabcEF4TC2/YBU4EoFpw2qslNY8lWypkmS27msjGEisj1LRxGRJo5U056kBxXNpcip4aJhPGNtEhUhJJ5U1Gxrxgt6KRkRDdVRdWKyGTDpbuwakrD6zLMXRP2kWu74+dsFQFWEy5YQbgwkkgR3u7uiF9YWUryq1RlkSyRobNNGyBldD4TUrErOpELdkqODo9P+iv3imtj5+O+04HTDZ0RRvO5n2V7s/7Xg8g/D0bkAROL4wf/nW5VOmMCOcVJ9bPwYKZkU5+S4wE+ej9n+GVYJbeLnGylhE7sIqMUnJql3TxWfhp7vk0974uVpTm1m7As7bYbkYIZ/IdURE40Uwu7PMiu0rLZXNqVkooYes00qRjVjWKVfcENG17rbk5NuMjLpmDkz4xaMQBz1aSiK0JLLYlqhP3awVU6gwMNJpr9g5uqG1LPrYycsCiOgbMt/pSX2vMeEkk1Qth9IpFAFrdkfn6/L+dMpcJ7TuuaWQ60k4WdGqYKgt0SQDhunEpphDR2zf1kT8k5gsutIiCnOGnYt3YjjiJ+mWUF4hSRCaMmS/bv2cVrUEncwdmekFtxWtcHdio8ZxmJvJEK30IyTzqQuqBnED5FbuGa2OOVmLmSzWxOfm9YY8fXK21YpUnJrxn5C51e0xF5xwqO/FErmTOtuZj5RXGv6yafWyH9Ss60oXpOcB7kEsjtSIYbEZgcSRi0lbg7WD1nFVO0vOJe6rj9zD4aJoooi3q7eu2+7u6llx4G4YXdIlPOFLIP146QD/kUJBCIKf0o8LXXaexJpirQDrwCR3MltT38taHK7qdJY8gYl5sXY1gPuxKOGInQeEZPpk8OD6ctQnSnH8TZZ039g+C/W/Xm9vMOx61lUWRs+G4J5/qEEWBjXqydXtGanv3fXUzQaS2wv1KJ0FtBTSi+heIQj6AZXzBQW6hwn+Hb7uc5K+tpU9pNZDe1m2EY2Cwl+cltaMKFNlTkTo3pyCNtAYNQskzijlMSj1NWU0WdCuKmr4lgrMD7x3LO83kfVNjZuawsMKteJ/M+n1rF10semCqKJP9ITg0TpGRTQ1hVm1V/KadStlbRLtQuVvH9qt6wfF7aWQBEG7rShJZL+59AW6sK6rlnTVxWp43jt/Y0zyJpRJDZgarxXWRxB2LC4itwhPFpa+HjinUZoLX4Fc3n9krQJ3E6jqezu2zugNT/7q6xbWJ3cHqaHWaH+yo/TtSYvOQdPeZ5fLJBkTlzX1qGK9gUFD6KK8cFN5waCUKJEsHMUqprq+kIBgqV3XUeN1RQFJtRVcDBZc8lKfQoeR8PrQnHmz6XVvOdlnJpb2hWp2upze+fX7hRcVdENHu42Qf29QQzkCKaiaCu2Hcu//qG1DS/ZuahfpQBFNS0ayWNzGXZA4U3WnustIB6PUvBdZ3ZS5HXBDyVjKJCU0AmI5eyYuFsbjTqOIapijzw13SpHkStXrEpUy1URGeCGtUM97PTQXFlJyzoYKCDJgRAFIhFS8z8MkcQKf6oTTsm8gDszml0YwniRo3KHxcWvd8agQsAuiBqd96IMjBYpK+QpjekFeq4Xvuwx/ztNdx5cbwDDydYKUBW4zFhL8KaVVQYnoOSzj4ad6Kwj6grjFCAfxckuz9XjCQLbqfL/2BRsbcTZQqUfc1NQ91ynE/JSjYqwJjSsvTMx4U/1gybSbUa2Ve9QNSGlyVhwqq2jm/RNGKFZsG0sexhSWoJNuVlGXQuWtdK1opTw8rVLZQ6WhSKab0rfQ64HTV4x1sOoJO9QcxUEz5rZKPLFXIzfBME9tKSRcuKgUmIlPYCSAU5vxgRSgpZ2QWQilDSCP6RaGn5JCPkr5Gy7ogAm0XUCuaMKLr0OHm+H2fuwRhJ1j7hhL0AxAOsaNBmgTfQccbrsUVlnCFaY3uLq5konIqB+oEUEQm4TrgV86syWRmmbzhSShlUfbxZtD9rrcOf7Q94qwiGPbce9tpsxQHeBrrHy9GzkxZiOKkdHHZu/+L4WQvmjMks52Z1tSPF9Dk3KwDVm/1rKYxitOyjI4XhggmzK5zeJEpyANbD741UZk7OKqZ4TgeQbIRRqyuu5VUui52QDkGQ88u3xILoYfj8bC1au1pNh9Lggj6nghZ9SpUyT1X6dejMmLyqJQ9yqW2UkmLGTVOgrC6pgT96GOz9X/KglOLBKdn/4XH29Ojk2ePDEXlQUvPglJw8yZ4cPvnx6Bn5f3s9JPv0ujsx/UEzte9lcfITanuePCPidG88geWUzBQVTUkVN6tUqK5IboU7qByJ8HzuZWa42SCHc4Wnac6EYcopXtNSSkVEU02YGoEmP+dRrdFhUESvJPV8pbn9h7es5X5b6wSFN9Ik3gOwG3JBaGNkBSJ8xqSfbV//n0htpNgv8t7aKDbjUuxyp70DCJs22v6/PV+H1462msNpcKf9W8MmrE0oXt+AQ3ihzZznF+GA9hIRDouUs9AIIAWzZ28waZ9fLE7sg/OLxdOoeHTO2ormO6DN67Pn67BOgaNKe4ujvgXkAr/+pIP9uI2HVOb2+oY2iq/BTCqzad6NZipjFeXljkSalWgEAPhlGEBg2pTlwOa4UyT2NLFgACzIMbqgvKSTsr9nzsoJU4a85EIb5rSsFr6gymc7s772LZBTZ20HwMFIAjfHg7qkxjLCAF0Rzx0SNlWPEFgfiTnV852dl0gpC4dYOHaz5VIpZi+rLVP/FK8l9kV70AgpVqnjEPdSIsk+aObMmGOYBS/wOgF/2NmNg3spl2KKa0XLFkyrgORUxGs08e7gjuhzEHYg/t52JHHTZa0gFQGHPlY7OrIu51Ywoe4Brh8u+ogkW5LClmzZ1mSDIINpzT9Yb1nDKBCC7FF4yQxDETAXTRUNruHo9MIrMlqMveQFuzFZ6+SaktfMKJ6j8Vmnxm0qyMvnx2jathwyZSafMw2qVzI64UY7v2JE0nJX2x3e8mtyHYymbRTcuKoRzmGpWCVNMLES2RjNC5ZA6mKGOFHiPGp+Qn7RRfzUqY1tzz0OGgcC16ED7k9HOyzXEVVHsNsYUXK41OxOMu+9jwRCWOAyVTMq+B+46XkR3OBul61IwadTplJDCijHHJy/hOL23DdMUGEIEwuupKjamlXkrbNfLwNwXozIz1LOSob8T96++5mcF+ioBjNqb8P31emnT5/+8MMPz549+/HHH9vkxBOSl/bS/0e0ldw1Vc8SOMTCsVRBAw3wNGyVuIl6wqHR+4xqs3/U0XOdd2F37HDuvUrnL7z0Alz9JuwiyvePjh+fPHn6w7MfD+kkL9j0cBjjHR7ZAefU/9fHOtHK4WHfjXVnGL32ciDxaG0koznOKlbwpmqrzkoueBECF3ap6qAE8AAzvznToCy61CNC/2gUG5FZXo/CRpaKFHzGDS1lzqjon3RL3ZoWXh13NCl3c/zE7ZYexyjoHfX9kdx6uMHhFV5sOzWcu6EXM5eE8dQs51PuL44BC7TZO7+UM93LaTpIEoDJNPNw56ysEwUSzisMaQ1Da3cSipUlkOEVu8UBtRMdzynBcfK8aO9hXtHZTmVKujcAWLCXIkJLqsmk4aWxx/kAaobOdoRZ5CyHF521EUiiQjdDT6JDN8SHdoUtAHWhli24O1yNOOdoEQrSBFl2V+IERycVFXRmtTeQJ4EPepIEo1ITMZK41lJB8qLzeIMoSV7d7IJF7Tl5G0ysaAc6aEdnDoyZeF1v8rei9HH+1q/RIdjyZ27lFYxqLAZ035FXMAwL3sH/2V7BdFG8BdFF7nc20RdzDabb4N4/eO8fvBuU7v2D29Ps3j947x/8lvyDySH2rTkJW6iTHXsKb3HYfzl34VoK3PsM732G9z5Dcu8z/NZ8hpgo3kkV32RNeM0M3U9Xx9sbXSo6gtzmNn9TdsJAivnn5W8l6fegkLnYXwmT0cTIjIxZrjP30hizfTwakcPBjWeZsmq0wZwn2AxlL/KbkF/t9fv3hqkVhLJjsldgIy4KnjNN9vfdNbuiK48QZPuXfDY35ZC3LJkNfO8KFFjUSnuacmHYTLkIc1r8ZlH152g+ZxXt0J+0snB1X4M8yg6zw5RzlJIt0/bL8GBzQmo0LeeQveSC4XFA2EdUrMg1F9GM8QFzESrMn8L3wJyNqZeWeCVD36wls09DBRmVU810zNn004K150azchpdslTg6LewSe1IZwZiwuD+3oC2Q+YQbGunOzShD5yeAxikie7r0QjJ7oOT9WnbKY8tOslCLxdbJj3j+g65Tnziw7D3pJReCUQvi+J5i1cCS55BHn07G8myj5cplqHskiV5xmAOnOM60pg27IX0q5jvD4LF50BDEg6vmL3BepeUfWoHCmPE1Gk5TSbhxvNDUZ+KSyDb1EdfuJiKmDuFCj2ZMEyRcnq5G5N6+62RhKYq8QgtmgMJWBNmloxZSD7TQhQucCI4JxGYy13CZOq8lPaQJ2d+JW4mN96g3JCVVMxew8HGVMKImNkCf6YZ6YDQMKGT19ywMae7RfWUWyLJK1ZJtSJWyEHmjBuuSAgfGW7RlIIpdPvzmDTvXtZWCWIFpszfJgJkC/vQJ0d+4OgkpzXWjnDpkm1vgcueDRYQl6YWNyBPSsJk5Bz8lLB6UbuYU0HG+ILPTxrHVMywEHavj4Eg+7QoxiMydiy/DyzP4NGUl2w/V8wy2hiTenwBlzBiyNT2HOdmxi2cCsw9/UPSKl37NdXaEnMf87bax4VDfRfL8RI3g4PQJX445OZ8NneJasMyECQkHKDT3qqEMWF1IC+uszjIEOORX1PNhHYJY9F6RQOaAa84steOqE8h/JUqu7mhUMK0gUC0oPrIqVWFRmTJSF1SsBW4IARCw5Clq8pB85zVBpKlXVwCnmledRqRGssxNZqhqyqnzbBBDVYanHpRNIRFRs66YY1DpaTuOjomx0F6oW3DZZSsTILKQmHOilHgWZ+TjkmtK8z+69UWckyCCqTdqtyK9dwZZGI1qJAjmDyKy+pwDWMGiTpQvCkUlemKinNBKqlNkrUIVlXLREsZCy9p9LFN2ICWjFva/5lH11XeLj+U0zIHP6Wz7pR0Fc4qoJM76VzFKFDh3aETo1daRwcsC3zqy64obfypywrCO7UBPCaVFDxm7JJkiL090GT9itk/fVyYkeSasZo0NTIrfJSWrWpTFXLVAdM2Ha3IRDUvp+UoXdnoNBy4bRfUUM1usrV9kiRL7SEOTCeVP5fCbmU08o/dO2Py0Ep2zQw5cMexZuaR5WdvLscSFFZ5ILqZRPTh+lPJoimZBlHX2napnETNwK5goyyvlStfbYqLCDS98COLxJ8QjF1Uhy283Bcx2lDTDnwqGrWNs2fAvtn5kou6MVf+R0GF1CyXMQ1dNiZ9gerXvCz54Du1YjnXsG5Hg4v5woFuHSeWWAnYdr0JlAhwXgPp8G9mdUbFyLWQS5FWXYtcaoZ3vd/SAF3g3R1HT2KVwp1DbGOPXCe8I6o9ud0V2TCo5YLw3B54i9QfZaV6Se3ZhRWIOkFMOzQJ/kL1nDysmZrTWkMdIqjPM+VixlStuDCP7HoqunRnhpF2AeBoNTJMoGCVFNooO324L4FVgpvVgBXfR4EO/evsz89ffLEr7/kLO5sQIpOosx2cB0vUXPOtGOiTFW47/nDFNHeGz/gCgqi7qt3SqWDdsL+EJT3PxsPNV4FzV8HE1rdBU+xo4/B0HMccW8HGrB5OS6qq8dep4AGSbSMHyO1dn3fudECX8cbKPFiRKL1Ftd5MRuuef1KFklv9iVcr/Xs7bMSraruY+ju6BLtQqC0op+AGV4GbPjgVaYMsWaPECmnPmYJ9ZCjzC5lfJfHIBdeWUwo878HBAOokoyqfsyIy7KQxhIdqT8oe5GzhddnxFepa4z4lL1lNjn4kh89Oj5+eHh1iFPHzlz+dHv7v74+OT/7pkuWNnQD+Rczcqvx4p1D47Chzrx4dun/EnSlVRXSTW8Vy2pSohtQ1K/wH+F+t8j8dHWb2/49Ioc2fjrOj7Dg71rX509Hx47bvVDYml7sL1bDiy4FYJ8FatVejvcBeYnK0McXNrNtnbGvkpKKSr24TbTX4opNOjoSuDuiU8rJRbFAmhRG3kk3by6Qw7vayCXFurZ3i+vpKJ5ty3TadlpIOmmHfcX1NYAQs2selZc622vaQZbOMaMe4RMsSUNSPoinmg2bu8gSOVbi+uKse6mtzprohuAH3KyFVtQX/rZ3E3huw2/A/WAHD3jChUTCtWY18GiZxaNfy6PBwoABcRbnAABzn2VzJBtaswghNKsAK6YoYwWWZas1nQicI6fb90Q6xpJgZrZnlHhGngVRzviNalr5EU0dx1WzBkmimOwl+uHRjdkx3YUE9zI4C8Osco62iHuhv5vELtxcqRgVI1gVTyQ0+6OyWsODCsVJ6L1qJmtorIYlBDm7S9JoRMLU6UJz5ZEWhuTZgfkZaem9dZ3ft/dAhrL0qfPadAC8cN94KnJUyvRe0JJm9H0Rrz5qLgb3W7DA5bS85ZuPlKymw2prS3p6O1oa0vihxB7Rzczic25prqRgtVk7sFGxKm9KQy5W2CkA0YSTS5xwNJoApLTHjb8l1ago5iwI5AEWQwCinYJ0UUoCX4PyFA/7gZaNkzQ7OKm2YKmj14FGyhycTxRbouPCvX75/8Ag8IoL88stpVUXm5rT0b+0fPjk9PHzwqLOXd1Uh8R1DdoEjyGnaDXrdwlxcRXq6kJC3GXIWYtVxCP+wummWViiecqccO1/dT/7vjWX9oKZ+x69DNDP9Swq4zDSZWKnQtrA615P9Fbzx3mEC5hWQlbFknwXnaod7hY5qLXMeSwODmuZr+rUKzemRldYHznLj5QY6fGBBrXoiNXPVwNFpACDPvbJKXqOlz5L1v346f/3fvnK4jn4rl/kLxf/AsY3ajlct+jkbdDplaF21r3fm47kmKbnvjFG3cXNvmSKzTga+or7oPaBYMUMxbhZcJB3xVTA7/R0Jrxcw+JpsOEzTLjvqCcDux6rcnTyFVQ5QujpHSAgp5ZIwqlcWRcOAhSYrJGj4eCByo3Zne4iu3VnE3YXiUNAd4+us6Pz5/MWj9YSNPLdrXNLM3j4eXPSiOO4wuVgWrN2ZwiPhXWSpnCJtg8POEowtUgk9LCoyN7TsVKfsKUcnR0/bON6tYHAWJdBwKlnwKe8KB7kUO0toxtPBAtgDk4nqZwvW1OzK5npBzdwrtX0e1fyPbei8LsoapmbHsCsNaVfkYTCUSHuhoUXhdbexHQvi38BVPn7UUS+pmjFztUNSvAcIQGzQOPSqKrm47gQ97zABH8gFxlJwKY1IwRUoGQ6TDkWanYnU9y6UE6TpB5CmKt6/k+ish5cdUYuMnIZTzZhMFbSf3Z8b9LOfmUyD9XKq7CUt1leh0STsc0/SUjJUpDpSu8FPkq7SUvScUlYwxYONzbB8Drb52DLAYnZ+kcTOoJNS7eumrksevJVbKTdfT4beV5+d9xVm5n1lWXlffUbefTbe15mN9zVm4n0FWXj9y4I/v8KD9SfY+5Dtk8QCV8yZWmPwObzjgsqh8QIr2YKGzem0ssQN/CmlTb6qzKYvnc4UghakboV0/+L/3mgm8gV4WmYiV5af5LKqG4Phw65aVOgo9fwS42V9W6hhg2XaESqaVbD/UywE1E4e8LHXoBaCmjIYNJyGC9u5Al1DfLAbcU5VsaSKjciCK9PQ0hd60iPyAiqCJNV2wAhF/tJMmBLMQHuggt2qjobK59ywPHFq3WmyVO2D5XwjhwReb59/fPb06mm7XMN91YT7qgm3R+m+asL2NLvX0+6rJuy+aoI9P3eEyd4vbuy0OmIaR2KSVnve57p0bmky9piNre5Q2f2rmGkUloLtFVvc26jV3WmLPdRz0gJOZzrQ0cc0uYYxmIQ8Ahe586YH/dWquFzMIELBBaRvLKKKmrILaUaXoKXsGNrzAaW6VPi0ihigAfF6uIjBbipZ/OKWchjmrvjzzUbeBGOay3sHrkw4MuHED1AcDKM9nJCESK/fG1qCaTyM6UqKYVUGTMOzCDjrXMxegqxwWGttTxJFCpbzAhJkre4KbBQFu7TvdxZe6mxKK16udnQ0vb0kOD556G19ihVzakakYBNOxYhMFWMTXYzIkotCLqP7P1bRgzd7eDflrupz9HReVx8DtHzv8/HZ5z6zd1gFpbmlwWv5G12w7gyurcr/xeaA0ALacOdSdOnihfquoewkO9w/Ojred3lhXex3qNCsob8PX06ov47g/9HF1l+bvxTGHp7je6sbST0izaQRptnE61QteY/XB6sr7A75bXnk6DA7OsmOWtjuKtjFtwPtiN+fpHKVwX21YteT1nkeWnXY7RDQ1HgcKiyPoZD8oholCjBEXie6brisj9KWr0kN8tTjEc/qMOLQmT1Q6+S+4lCbu+4rDt1XHLqvOPR1VxyaG9Oy4v/y/v0F/H2bHiX2oxAOm/n6MGTcqHLsA1MZRlMnXTUBSVV6fF1T3O3t+f6DiSxW2UDF25sCMm6senvZis9oo0kAape8z579sB5FF0yzoz383l1HcDE2YvkLK0tJllKVxTC2O6Dle2lo2Yl46VD0oUUWNvucUasH9JWro5PHwwSumJnLnSX6tUiKoDoJ0MjkmBoA5WImLM0ZMJKUcskU5HxbEeprUGXkkrlEWZk3lY/zCmNrV7LlwbkPq7da3svnlw/65rEZMyNSQ+2YujGDZIIW0WpnAVvv3PAxpSalXG81rezRpwcHk1LOMvc0y2V10MFd11Jo9sX3OYLddqOnSH7Znb4Jz/Vb3eP7pfe6w/bTNrtDWhtqGj1g6t0W9fUpNm2aIqBhi+/JYdtNttsrHuC17s58lKWdTny9KXeiv3J/3nigo82Jtsr8SMjtTDNztjmZYfK7uEO+9ZlOFqvgBXGVwnrZi9hBoJX8vKRKjEdkDEXT7D/4QKIoU6o1nV0m3Po0tlYel52MT8Cl3eIFsPWTNxKdeIo1mkpu0P1uSAMlYoLaWlPVqod4jnZPRWM5wrEb1ituyBWphRSa4PsCMnbENFPPr4UbJU0Q7eSHusmOehPyCcBhzDldsJB7pO2iYixy7uspYoghWgaYyCU2S1BEsCUpuWAausktkluKvd+UjApIXGuj/Ln5y0RLl568twd6gD3rU+PwxFvAQFv47DRmcL+Bo+L1yu39YE3HbJlUGrxJHt1QtM/n2rTjPNCeUlWNcPTHsGC5YMpLkBhUQnAVkpwdF6eh0+5G/o1Pigrxo3eqdXSziHyhoNvEZdTYmWOHmSZneHWb8QUTGKGbQnUSrlbSyFyW7VJFVE24UVRF0z9xia0unwxKEmrcFBXPlfR5TCPgQFpqCcBWuPPjy/p6VbNoTuP57yMypTmbSHk9ImbJjUGvBddkmVYksqImlomKRT7JgokiqaYEIdPYTTGEF9sjtgjhxKFgAu6Cg8Iq3ucXGEOtR1BVXI9IMuaSK582+BWq5pS3O8HddX+WPVS5UNUyigoNijisyETafcMVc/XbWtn9Y1eZCr50SfdpWXX/3Bf6GZGx36zuJzy7eFwJ3VR9Ajx++qxFACdBzOpqd50wz9CUBaU+IaMMhHZSyP78AitNOm6imixZWTohF+bjt1+MVmjLvyykolNipCz36UxIbXhutUdRUNXqtBmGnZZymS7GK0aVwKR1asLVaMbNvJnApcgyCJRWOwjE2+fFvtXVBsoDn87f/qN+c/LLP77++cnrvx48m5+r/7j4PT/5z3/74/BPraUIrLED9ebBCz+419O8uDaKTqc8z/4m3jE7Hyy/FI/T078J8rdAnL+RfyBcTGQjir8JQv6ByMYkf3FhmBK0xL8sB8W/GgGM+zfxN/HrnIl0zIrWdVKg2PWPtYfXPrbUq2JyqKtTOwoHUqLYpGMGyWWH2dME4pXs5BecLTPEYQ1gTxqpSM0Ur5hhChFpIb0dThGRFgb2v+DKcMDSkQPQ7EGXnRztW3wzlWpJVcGKq88JPkhacoQ8dbddk5+cglwr+XGgVtWPx9lRdpS1i6dwKugVhi/tSMCcn705IxdeOrwBUOSh37nL5TKzOGRSzQ7wYIbatgdenuwjcv0H2ce5qcokif7SyRE4r3wdE/+VdvKHllDTAiQYaDxvmPmplEssrwb/chbbMG4pZ/7W1ziT7dCcegRvpxzu2i2CytFkRSR4OaHYuPSnr44hbP5c6mL7M1jtfuVT3kL787qkuAPXDfJJR677duDQjb8MHLv+x6ifuQN4+OA9bhspPNfs4ir76gd/u4hnJsRUEPYxgxNtRErgqN9objVJSzR79kYN9+vT3IJ/JLjHPda7IOGlZXiqAy8nQgy1dnCl0lgIgpG/IJx0G4bmAZHCJV1Z4dQU9YiYvB4RXi+e7vO8qkeEmTx79PVR3uQdwu8oLuEcD523l+eQhl3iIbpM4wc8W7+yVMws7U6QgsktqdYsH5GaV0DQr4+cFunENOAq1bRaRrxNn23K/xDh836tkJrlnJaeg0chORbj4HpXaiwuEQrvFsyw3Iz8+PARVhe5ecT99vnmlKuk2Gs74zVEiFCSN9rIKqR94KDQghy83W6qnZonUkz5rImtSIwkqhHbE4BoOTUWXFILrZ2GMuWKLWlZ6pHVcFUDIT1IIS7FQa1gijCUD0r0OmSiJWomtFShwtWSTVpYJEAgCLyUWpOhoS0hzy5eO2rotM2q54bUgEOxGvQa+40TUDg4hpGI1SitFIfz1IEVtK/1guygo8K8gcS+woob09VZIa+dbfX3hjU4MHn5/hUkLkkBXOPveq5UdLuNiWMnb2lSDEyDUNCqYNAfwNEDOsK+fH55C6PTfbLNfbLN7VG6T7bZnmb3yTb3yTbfdLJNN9cmnL5t+8enGWX6LVKHh/9ibU5biup91sN91sN91sN91sPdZz1opjgtd2sw9vdrB8yd9zcV0bq75mC+20AqVkNTl02F7ZlyyY72Yug1J2+IjiOtaqazoagb7ypQadsBf/GEKJxCw39q7VqEfVzBP2RZMgjTwUus/Ve8gg7ERvgxWyRteZ/vkqhh5gghjVnPOhhs7q16ByyVCJYYtjSjgv8RlX1v5uk+vyEOJB3H3++ZUDyfI+PAxX5d77KqpsKf0lI5fbXFdJ1IjTQwJPYmnbOyhrLcVCkqZr5dj3GVb5OeP1RgkA54DNpR+wGNOJ/b1On4O+SppKh+sXoxKX8E9SBK9RYrBRF8CSL4BnZ6D3bWTruANawjO9J9++jDb1Iz/MbVwm9YJ/yGFMJvWBv86lXBxEMamnk4KXeRPNq6mfZa4Ra6/g6fdDkV8bSLOXjO5tzufQeBjaGJMC8OEl52QSWtuFoQwL4Da1ZDLt7UMEG0oSvt6x/77r7YjZuG/lmgINYcHTWQqVjKCS2TSvQe3WhQ2q7+1WybDIRPiwFTiq5cuAQQiaoZONJSO9lr6DPp9AmcXq2kYbkB5wk3fNFKguzpne7PfaJDiuY+2S/DPxsd7hT7xLf/aUdRsI8sb6ALwo5IcTaB7jAMw3XdCnqqROi9HXLQaHUw4eLAz+1L1K10O86dQmGh7NUC2kyQnJYlg5TxmaJVSIDUvOIlHegE3EW+vjFL9FZZIxdhC/YPn+OTdmBS3YP9+VkrFxQKxbjl3LPTG0Kkc+X9zEYq732X1ZSTXMOUvivg+PDo6f7hk/3jx+8Pn50ePjl9fJI9e/L4PzudNuaK0WK7lPBbUeg9DEzOX9y8QCD1d83ZAKQT72JpCM9HmOWArA5+UhcXUqf7gjynAsO4J7HPpjkNQyalDgglEyWXGmwPPjnEIeFlwZJNSE1nLOmkKrGbfXuJllJdczG7wvimXvPsO01zc7BIgOXNF+EI7UqruazYAS2xYUVMHIuBAe5Mf5c82nimx9Y6DPug+2qlU5rzkht7ONd8IbEdsZIN9NKvOcuTDlbQncUvNhhI4AXdbaviwuE1Y9CEvaJiZZWwHEID7NX25fNL39XpfYqCGxqb5YENB2+Q1QivxpBZ4M9CaFplQfgyVdI5puD81rUURdxFLv1FkLGjYjYOMzmDxr+KmWDwsRSKLgSmR0n+0ISRBoocQZv9YD0ZuXjPUWQCHwk3InnJoS2Yf5WKIgRHpQGoUAQE7AN1DT1ly5KcX3i1wsiIPa/HI9StKKg7whHNVTbAaMPzC2IUX3BalqsREZJU1BhIcGHhmOAGgFHFihGZrELQTgrqlGaTLM+K8W3MDNu04Bh23pyVIR/u/ELjGkuRNKJOb/L9+J/L7aJ/3HsDeUGOeVxtiBCMkkshXKTSNBjiXDiFYjOqCoxT0Rrbi8f3NbZJ5yGW0qqbGMqaS5U0Kv5JKvL++UXoCwRCM6CJuOWM278dgbjgUGji8q9vXBjnQ+0L9nu9/PlFgksGQLBeTAi+7UJyNXDLVY8efvnaMfBC+36IIBVcsA2huWm80xYj+ZiqyIMw3gMslzwNamWKheggrn2FMfjZXTO8b7mfUeVFiSsWm6Ng0x0Q6TycQLpsAaDQywpm4UaMoUBY7OO3RuTxHoM73X09NFgkbSwEEoe0uxeXcR8d9j5n1b35HIc/8FNo91XBaxctrJSvqDA898H1LiuLfcTWSE6exRuRvapNm9K+tuB2uvwPlpg3BcmZgotgTIzyskoFGFNall5W+Y7+OTVsJtUKhZVLiNOGlyVhAhrqwWtrUlsswabc6shuWFrXStaKU8PK1W0uZyjJd6UOobMAW+3hwoSjA5MqvYCpJnzWyEaXK+Rm+CaoOktLFh1uB+CaoFaMjwj1xfiwcA2U8JOWTzJC/hop64o4pvVJcFcpuoxpCMj348w9cDmybTVO2JMhJjAWDYaj4b1ybM8fKICTIVrjESmYPbIgZdUXt47NAuGc4d3mknedP/ZnSByD0usx9c55dVxvadg/ffvJs3Z8OU7qBsw+qdANYoPjd9pW3YfM3YfM3YfM3YfM3YfMfdMhc58YsbbXD1nzAWuRs/D62fEHk/OLxYl9cH6xeBoVj85Z+8Ui3YbC7D4vS+3Cpad9ysHeMVrenPB0O4OlhLIha+d9X0/zvp7mfT1Ncl9P81urp+kKm8B7iVnNP7oh1MqXRekaaUz6m1QDLY6sguSQW1JNclmW0IP6hnCqKReFKzHluROywpEtQx0wD9u+6SMWtrchsHrOKqZoucNiHy89jFQ8SacVevQf8inoANCWXD/qVnriRdKlAsw9mtBcSa2JYuDYcrVzxm5A2H2FhJ5Ppq8PPqMn0yeHh9O2lrOL7bTXF82+4F4jBFpXEeP+lJ2pAndgGZqYrlqkc0UGKnrNNOGG1FJrPkHnUWCdMDSwUJJ4iTwrWI+hhjpfeEO+sutUM8WZyMFhpXXDNBoL7ViKFXYCrsVYtOmjGz+M65vV8wLLBsRQCriHeWZHYxoXM2i+7NqW9Va0ePwDe8ImU3ZI2dP85McfjosJ+3F6ePTDCT16+viHyeTZ8ckP05sKJNx9TwvP4TGS1+3/gWDe9GoVPoTwXsf7cBqBIyTUlijlUsMlaykDeeIdy48FPS68qFCR+bxiYH8PtdzxGihazkveqk/hmmSE3QbHW9qLpcRSaw49u4wFtzrnpLEz9/WucG1VA76QcOLMpTZ6mH3RdO9N1W6yBEvCuKl0AhNcDjkkcMspeVlSbXjuHEsJmWEKLvPYH9OohDfaMNW6KqFT48+MGt0fgmtLnYJNaVMaqEhUB99ooJeBttEgkcOYfEqEJH6M0JBkoAhiOof9NOU1iR8wO7HQuLY3MH6HT/8+wfK32l3wofd3urR21I8HztmWkLQnOkjJRGHwM1kjKWGQmJIMu66NXZsZRx3uCIOGegfj1sIPVcdMf28tx+7C3Pf+3YenthckOFpaOk9/VaIMg1oL8ppQu2swdJwZ7Lje0XkWESQN7NcvbJYdZ2ldBfTHtNS/+GSD9odv3eyd8w4fwAqtAwftuqftkRI33A0OuNR95LxwX6WbyDm87t1EX4mbCNfDWZPSMkY9k9IX8xUhSve+ontf0d2gdO8r2p5m976ie1/RN+Urwmp835qvyGFNdu0r2v50/4IOo4HJ3zuM7h1G9w4jcu8w+tYcRo1CieWsBR/evYI/15sKPrx75S/3rmMm0U0NVT4xB88CMoBOTRWs5Yd3r1wBP/dmCIyfMzJRjGKShVwKwoWRROdzZoUL3qBGkDLmvpfEy/5tzAJDV7y72zQv3I3dkVuVo9BA4MFyucycpSrL5YO2rRaya3IK1gOgZ0VXGE7twn2tmoDVBoGuGH5ermLqLm1PjbiMHLADQ48GzUYuDj/WtwaVdSZDpxV3tXfWgZ6K2J5Ci65TRWfV7jpM7dnTNjG3NaokdGpctZDx9+OE0EbWDzoW0PH3Y98vxbWHQS3cId2RGTvMfD+f4lFp+R/sRLyy6+kSeCAEu9EsrtYqMchgRYkwLy6gnSGc8OMRWc4ZJAKYVocYxXIptFENWCEt92CMubcIta1RqRoz0BWtvfynJyePD9Dm+i+//6llg/3eyHal3OF+RXd5WGH/HZija1kELKJD5lKYbV+/fiONi13nYqBe6SgtT1OE3Ql1Wv1ijjARh+p0eWgOqXGlnLlbn/2Ua5fh/FujTQz699VqrWBb2+8nZHqFz8KwFJygS6oDoqOW4B10B3/SwtrR1vzcUf61Tlbyrtf8wg0/2Kwz4mB2pSBdQI+hFuxEBjkCPchuuILcQaJtcg3p4XFy8rifXXryuIUUZIntamNa4QsAHBMHCwfgi7/g3AbnEPaBpWmH2Xoy/l9AxrOPULA4aTeRQoFMFzxhQ+8vIe23sEMTEzpWl0pwh0+NrzxFAd6kMeGtUQIMJ4tBHWHE0PWpqk3EB1DHN8fu646rruWLJhNmlozFYx5ysZYSlYfOQYZa067W9hJGX78HQLo86MhZzKIdnw6ex4jvGjnVU6B3fKtNYxIS4ZJi0FKT9c2Jiu+dDt5zqg0XHIJX8VyC5sZsQcNh7TS2tqPtp6RgB12gxYiBvTi9qNgnnGm3FfwFDxv9mDkV8BkvfParV+lDvq47KWGbgRfTUam6TQDW39Eu8g2ZRL4Ba8jf2xBybwO50Qby1Zk/vlrLh2bqis78lSiR7CQ+3UK+4xheyscITnvJd1WQfPGLcLI45N7bO58rgTSXS9cudckmIcIEAmySuphYfYIqqy00AVWvX2wvkrHvxZfayQ5ad0n4xdyHEHypbk4JhyDpekhd0ilV/EteaD8It6CLdpRRZK4Bb/4fvCzpwZPskDxEMv4TeX7xwZGUvL0kR8dXR9hQ09dye0TO6rpkv7LJX7g5eHr4JDvKjp4EcfLwL7+8f/1qhN/8zPJr+Yi4uKeDo+PskLyWE16yg6MnL49Onjk6HTw97JayvS+OPYj1fXHs++LYn4fx/9ji2LtF9d/7UnfN0WCl4Hf7FsgpmTBoFURFPpcK/9zPZVUBmk6X+DO+04L2zzDoc2+OwE/g8xAy6S8PoFyWrpSIK2/93Zr4R8C30/RhiCQbOzm4WbdGtphlhlfsjxjthwPTkgcLaE3N/NTdTzsvV3ymKMIzqmHt0XEurWHl5DeWh/bd8MfVjTP553CKBcrCOvouWUBOF1XaxgA68bcQiIrTWiAv7UedUptQpqYouCsTZHV3iHN1MfkAJxQMS9eQDEeUr1vBDWhF1JKQ7dZC9rijv4iWidL3Nq4fDDrIdv2BB3l04+gQJsvAfOHzILZl7fccc0E4izk69mrkdm9eyqaIG/W5/dPbPiCanbqEtgFKv3a/oj6etz7VlgVY4VNHaFFcwQtXfkhfOU6qdCu3Zg0fZLWSlvWjOSBIIffL/sfNPJqqu+4Ty48/SzkrGc4YuXEAOK/ojA2AphXfp5O8ODp+PChKI/RzOwI5fxFsDEinkNqEU/6enFk2wfysskjFQQhpYoZmgSRA5Bv4bPDljXyWwPAIxlTBzWDChML7t4a0xdbpwNp2/yTQXNrTVSJgNgNzH2TJB9vCcgcYL7lZXW1xbGz+aluojse3Xbje/toWDsYhbgWj9erg+F4eFTK/Bl51AumF/3tge+FvkJ7UTTpxv9l9redSmSs8/07JlJaaJeoKwtsPwmiNWhHQIoOn47pTzJ2IaSzOMLESgg1/Mki0NaCsxLk9NJB0Im1eeyuonS+3A/rp4Eo6YaW2gvP92xdvrQa3JEaSitZWyGr2Lz1cWuoU2axSkc2qBcp0RCHznGvP88i3v+BfA4OcW30o4VZ3LNjPfU5mljAodMEfYk93brx8fpmmGPGQM8Ryna2qMnPvYdo5VS5QW4r9+GXHtIyob+b09UvTsv/6ISZSloyKLck7jRQBn2Nc9j5cqbNJw8s+yP6KhtP7wdGzF0eHPz7YDp23lwQgtNvKDCGSy4IN7oNNuGijmMnn2yPjofhurYEDr5sJU4JhzpDjw7+kzwbGjb8HZa+tucVBScqFm6Vq/OhGydpCejPPdSley2JY7NxqMycUqKVr3z0IqhmQ4Z8K6UIW5MP5iz4gyG2oaX53k4oj9oHJoifyPxOYN9b1gaG4vFksbwfIyf+K1n1I4BzCEp93BS4ZchimYpA2qJm5W4LGcdeQtWB1KVcQznengOO4awBDVvi0Ke98ysnAa0DfoHV8KuAw7I1gh1Wsz4eL4zpxHhug9NqfDIzr69kHKR6ukENSN22uchuRyz5uq+T5wvC9fhpkQNFzM/5NlvKa033aGFlwnctFehX4V/yVvHC/rEj6HknuuTfaKgaGSs88h0cYcp2x0b2XoUGnbZy9haXO210xL47IaUAgsb4Ow+SbrL7rbHY0nztv6RyM0MGH3a4Rz7gvsW2JUJCiwTb2hirT1C1TKaidUlWYWhhsjeCvr6miFTMMiiVNGFgH7bpBW3mG8WX4wP6J4WS8ANQ0W0AloZoqozGE6vxiRNI2F7wYQYwCeIlaKFFRYG8FsAAOkdDVu6uVLJrc3J6Q710eL+5dN4xVysLcNoH9ZHZpgd3TwaHwMIH86AbQSSPGW0J2LRaTNGacfsILOtSb6WZ9ezx8rsWtoX9494rM7VUPKkkAOMetgMkmoueN6vhI2peSNVB/DQHmfn5Y4gJZ3F3gaGPmTBiOSaY+8NiLtVLOohR7JWdkyktEGIMrNrlFSv96yUXb69GaZilnmX0tS0J/h0ir2O8NV6yIKvsNBAfYnRJnFhWgArSISYO1Q3ApUGuo0REgGSD8FDujnJLxwYKqg1LODlyjwVLOxll/ni6QvV3p4rMne9kqZ9Gbspw575OfN1SwdxmaA0jK6VSztkxJ4ps/bRlwTBeZWUtloPmrYCiSNaFdl5W9WtLqrihkORdHJMs5E0AFu8mjDMCQf7ch97QpZGP27G6w/2ZK7bXR46JuTGpXjeiAVnAjVWAArPPTWa+4VtitAA6adlYBkBKZEtqvxBpMAYa3zYyxgJPE9HmXVYHAteun4eThT7xk4MJEAeHYvTVr9A1q9nvDum6cz+EQP2CiRYCQTENlA1+sNAgM7FqzujsudQMS9tEoGo2xeGBzqbhZDaPif70zVPyAIcYa4GyiBigb3Kyu4Gp5lzJ03lQUtwt4Xz2gzYuyczQ8oA4aoSkhdnS7SwRE2xFnhx+QnNOSzgaL3qSDDZ848KmHMLTUc2PqDJu3aJa5E/iqZGLWOTUHvMGtTyeyWGVpeZ6NTpNOgOXNF65oUwxz7n+y/pbWDfXur2ALPSukOrf4W6plQey5oTBYPoje9ZLIg65k0ZTbRVW0Xt1IdsvqV+A6N7Sqtxo8VyypfLhxdHQJZdQYdadxG+m4kb2dvcteaphYcCUFmGYWVHG7mzVZKm4ME1CNEEbY0+RfL9++gbWxG2sGqcWKL2JIcKcYLVUMC7DFwBs46CEhEwvaBa3XHYLtcd0J2YssQdLGGiq3vuwULJzFaR/VjjqRV/UnQjl//vrClWDpD9nzFG8/5ED0CZ99+pA/Dw/pFWKd4dm/7hBdM+xZY6SQlWx0iAiEYTpQ0pojOwYVJ9Tqu/5p8smHf7S653d63eN9pYvcZgJ8Tahd0+k13fJsSaKKZM3ztUdEfL7JimkBu4F640PVAxNtHIOcshWAMJLjlwHvW3rf+nRA7orllWcnaPrgrtnqLgh3zVajdrNHfyXB393FBDLgY1foNThNSplf9867iF+RltbdhhaNYYo8zGVVQ42l4hGCIBFED4c5owVTfbeLYNqwW5HGjeTkflyLLHm3y9jrV2jTKt2ADnGn0DXzlUmvEwQHYcOy3S105IQB+P8/AAD//3nMjTE=" } diff --git a/filebeat/input/kafka/_meta/fields.yml b/filebeat/input/kafka/_meta/fields.yml deleted file mode 100644 index 3803f635939..00000000000 --- a/filebeat/input/kafka/_meta/fields.yml +++ /dev/null @@ -1,37 +0,0 @@ -- key: kafka-input - title: Kafka Input - description: > - Kafka metadata added by the kafka input - short_config: false - anchor: kafka-input - fields: - - name: kafka.topic - type: keyword - description: > - Kafka topic - - - name: kafka.partition - type: long - description: > - Kafka partition number - - - name: kafka.offset - type: long - description: > - Kafka offset of this message - - - name: kafka.key - type: keyword - description: > - Kafka key, corresponding to the Kafka value stored in the message - - - name: kafka.block_timestamp - type: date - description: > - Kafka outer (compressed) block timestamp - - - name: kafka.headers - type: array - description: > - The array of kafka headers, each an object containing string subfields - "key" and "value". From c30212593dfa1341296943b4cd817772c722a8d0 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Fri, 9 Aug 2019 16:20:28 -0400 Subject: [PATCH 42/50] add compromise data layout for kafka headers --- filebeat/_meta/fields.common.yml | 14 +++----------- filebeat/input/kafka/input.go | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml index 555438689c9..da88144ca30 100644 --- a/filebeat/_meta/fields.common.yml +++ b/filebeat/_meta/fields.common.yml @@ -180,15 +180,7 @@ Kafka outer (compressed) block timestamp - name: headers - type: nested + type: array description: > - Kafka headers for this message. - fields: - - name: key - type: keyword - description: > - The key for a kafka header - - name: value - type: keyword - description: > - The value for a kafka header + An array of Kafka header strings for this message, in the form + ": ". diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 32db99aea92..2ffc1873772 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -19,6 +19,8 @@ package kafka import ( "context" + "fmt" + "strings" "sync" "time" @@ -171,13 +173,19 @@ func (input *kafkaInput) Stop() { input.outlet.Close() } -func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []interface{} { - array := []interface{}{} +func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []string { + array := []string{} for _, header := range headers { - array = append(array, common.MapStr{ - "key": string(header.Key), - "value": string(header.Value), - }) + // Rather than indexing headers in the same object structure Kafka does + // (which would give maximal fidelity, but would be effectively unsearchable + // in elasticsearch and kibana) we compromise by serializing them all as + // strings in the form ": ". For this we need to mask + // occurrences of ":" in the original key, which we expect to be uncommon. + // We may consider another approach in the future when it's more clear what + // the most common use cases are. + key := strings.ReplaceAll(string(header.Key), ":", "_") + value := string(header.Value) + array = append(array, fmt.Sprintf("%s: %s", key, value)) } return array } From 6ae3210149e94187f88ed14dd3fc5ec06f638e2b Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Fri, 9 Aug 2019 16:34:47 -0400 Subject: [PATCH 43/50] addressing review comments --- filebeat/input/kafka/input.go | 54 ++++++++++--------- .../input/kafka/kafka_integration_test.go | 2 +- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 2ffc1873772..5156c73d2da 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -45,13 +45,13 @@ func init() { // Input contains the input and its config type kafkaInput struct { - config kafkaInputConfig - saramaConfig *sarama.Config - context input.Context - outlet channel.Outleter - saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active - log *logp.Logger - runOnce sync.Once + config kafkaInputConfig + saramaConfig *sarama.Config + context input.Context + outlet channel.Outleter + saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active + log *logp.Logger + runOnce, stopOnce sync.Once } // NewInput creates a new kafka input @@ -170,7 +170,9 @@ func (input *kafkaInput) Wait() { // done channel passed in by input.Runner, and that channel is already closed // as part of the shutdown process in Runner.Stop(). func (input *kafkaInput) Stop() { - input.outlet.Close() + input.stopOnce.Do(func() { + input.outlet.Close() + }) } func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []string { @@ -237,34 +239,36 @@ func (h *groupHandler) createEvent( claim sarama.ConsumerGroupClaim, message *sarama.ConsumerMessage, ) beat.Event { - event := beat.Event{ - Timestamp: time.Now(), - Private: eventMeta{ - handler: h, - message: message, - }, - } - eventFields := common.MapStr{ - "message": string(message.Value), - } - kafkaMetadata := common.MapStr{ + timestamp := time.Now() + kafkaFields := common.MapStr{ "topic": claim.Topic(), "partition": claim.Partition(), "offset": message.Offset, "key": string(message.Key), } + version, versionOk := h.version.Get() if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { - event.Timestamp = message.Timestamp + timestamp = message.Timestamp if !message.BlockTimestamp.IsZero() { - kafkaMetadata["block_timestamp"] = message.BlockTimestamp + kafkaFields["block_timestamp"] = message.BlockTimestamp } } if versionOk && version.IsAtLeast(sarama.V0_11_0_0) { - kafkaMetadata["headers"] = arrayForKafkaHeaders(message.Headers) + kafkaFields["headers"] = arrayForKafkaHeaders(message.Headers) } - eventFields["kafka"] = kafkaMetadata - event.Fields = eventFields + event := beat.Event{ + Timestamp: timestamp, + Fields: common.MapStr{ + "message": string(message.Value), + "kafka": kafkaFields, + }, + Private: eventMeta{ + handler: h, + message: message, + }, + } + return event } @@ -286,10 +290,10 @@ func (h *groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { // from the input's ACKEvents handler. func (h *groupHandler) ack(message *sarama.ConsumerMessage) { h.Lock() + defer h.Unlock() if h.session != nil { h.session.MarkMessage(message, "") } - h.Unlock() } func (h *groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 08212651496..f303bab626b 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -121,7 +121,7 @@ func TestInput(t *testing.T) { } // Setup the input config - config, _ := common.NewConfigFrom(common.MapStr{ + config, _ := common.MustNewConfigFrom(common.MapStr{ "hosts": getTestKafkaHost(), "topics": []string{testTopic}, "group_id": "filebeat", From 43b5bec2a96b70ad10e2dc302a998836f920b387 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Fri, 9 Aug 2019 16:56:12 -0400 Subject: [PATCH 44/50] use exponential backoff for connecting --- filebeat/input/kafka/input.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 5156c73d2da..cb11f167061 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/beats/filebeat/input" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/backoff" "github.com/elastic/beats/libbeat/common/kafka" "github.com/elastic/beats/libbeat/logp" @@ -139,6 +140,13 @@ func (input *kafkaInput) runConsumerGroup() { func (input *kafkaInput) Run() { input.runOnce.Do(func() { go func() { + + // If the consumer fails to connect, we use exponential backoff with + // jitter up to 8 * the initial backoff interval. + backoff := backoff.NewEqualJitterBackoff( + input.context.Done, + input.config.ConnectBackoff, + 8*input.config.ConnectBackoff) for { // Try to start the consumer group event loop: create a consumer // group client (wbich connects to the kafka cluster) and call @@ -148,10 +156,18 @@ func (input *kafkaInput) Run() { // If runConsumerGroup returns, then either input.context.Done has // been closed (in which case we should shut down) or there was an // error, and we should try running it again after the backoff interval. + waitChan := make(chan struct{}) + go func() { + defer close(waitChan) + backoff.Wait() + }() + select { case <-input.context.Done: + // We are shutting down, return return - case <-time.After(input.config.ConnectBackoff): + case <-waitChan: + // We are still running after the backoff delay, try again } } }() From 62488ef5667e9ccd329fc73593e11fcc28dc69c2 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Fri, 9 Aug 2019 17:11:33 -0400 Subject: [PATCH 45/50] shutdown outlet via the CloseRef config field --- filebeat/input/kafka/input.go | 37 ++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index cb11f167061..c5217297c4b 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -46,13 +46,13 @@ func init() { // Input contains the input and its config type kafkaInput struct { - config kafkaInputConfig - saramaConfig *sarama.Config - context input.Context - outlet channel.Outleter - saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active - log *logp.Logger - runOnce, stopOnce sync.Once + config kafkaInputConfig + saramaConfig *sarama.Config + context input.Context + outlet channel.Outleter + saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active + log *logp.Logger + runOnce sync.Once } // NewInput creates a new kafka input @@ -73,6 +73,7 @@ func NewInput( } } }, + CloseRef: doneChannelContext(inputContext.Done), }) if err != nil { return nil, err @@ -174,23 +175,23 @@ func (input *kafkaInput) Run() { }) } -// Wait shuts down the Input by cancelling the internal context. +// Stop doesn't need to do anything because the kafka consumer group and the +// input's outlet both have a context based on input.context.Done and will +// shut themselves down, since the done channel is already closed as part of +// the shutdown process in Runner.Stop(). +func (input *kafkaInput) Stop() { +} + +// Wait should shut down the input and wait for it to complete, however (see +// Stop above) we don't need to take actions to shut down as long as the +// input.config.Done channel is closed, so we just make a (currently no-op) +// call to Stop() and then wait for sarama to signal completion. func (input *kafkaInput) Wait() { input.Stop() // Wait for sarama to shut down input.saramaWaitGroup.Wait() } -// Stop closes the input's outlet on close. We don't need to shutdown the -// kafka consumer group explicitly, because it listens to the original input -// done channel passed in by input.Runner, and that channel is already closed -// as part of the shutdown process in Runner.Stop(). -func (input *kafkaInput) Stop() { - input.stopOnce.Do(func() { - input.outlet.Close() - }) -} - func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []string { array := []string{} for _, header := range headers { From 60f436c9ce93d9b5f6f8ef213c5a01880bd3d4f7 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 12 Aug 2019 15:07:25 -0400 Subject: [PATCH 46/50] Fixing backoff wait call --- filebeat/input/kafka/input.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index c5217297c4b..66ac66345ee 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -154,21 +154,13 @@ func (input *kafkaInput) Run() { // Consume (which starts an asynchronous consumer). input.runConsumerGroup() - // If runConsumerGroup returns, then either input.context.Done has - // been closed (in which case we should shut down) or there was an - // error, and we should try running it again after the backoff interval. - waitChan := make(chan struct{}) - go func() { - defer close(waitChan) - backoff.Wait() - }() + // If runConsumerGroup returns, we wait for the backoff interval. + backoff.Wait() + // Check the Done channel before we try again. select { case <-input.context.Done: - // We are shutting down, return return - case <-waitChan: - // We are still running after the backoff delay, try again } } }() From e223a64a43d54e836a34138c648e422468482ad1 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 12 Aug 2019 15:27:14 -0400 Subject: [PATCH 47/50] Add wait_close parameter --- filebeat/docs/fields.asciidoc | 4 ++-- filebeat/docs/inputs/input-kafka.asciidoc | 5 +++++ filebeat/include/fields.go | 2 +- filebeat/input/kafka/config.go | 2 ++ filebeat/input/kafka/input.go | 13 +++++++------ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index a3c3042baba..620839c5779 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -7881,10 +7881,10 @@ type: date *`kafka.headers`*:: + -- -Kafka headers for this message. +An array of Kafka header strings for this message, in the form ": ". -type: nested +type: array -- diff --git a/filebeat/docs/inputs/input-kafka.asciidoc b/filebeat/docs/inputs/input-kafka.asciidoc index 0c0b7a7afc5..ced85bc705d 100644 --- a/filebeat/docs/inputs/input-kafka.asciidoc +++ b/filebeat/docs/inputs/input-kafka.asciidoc @@ -82,6 +82,11 @@ How long to wait before retrying a failed read. Default is 2s. How long to wait for the minimum number of input bytes while reading. Default is 250ms. +===== `wait_close` + +When shutting down, how long to wait for in-flight messages to be delivered +and acknowledged. + ===== `isolation_level` This configures the Kafka group isolation level: diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go index 4e1fd26a98d..b3b64ef47bc 100644 --- a/filebeat/include/fields.go +++ b/filebeat/include/fields.go @@ -32,5 +32,5 @@ func init() { // AssetFieldsYml returns asset data. // This is the base64 encoded gzipped contents of fields.yml. func AssetFieldsYml() string { - return "eJzsfWtzG7ly6Pf9Fbjeqis7oUYPy16vUie5OrZ3Vzl+KJadzUlOSgRnQBKrGWAWwJDm3rr//Ra68ZoHKcoWfeyKkqqz1nAG3Wg0Go1+fk9+PXv35vzNz/+LvJBESENYwQ0xc67JlJeMFFyx3JSrEeGGLKkmMyaYooYVZLIiZs7Iy+eXpFbyN5ab0XffkwnVrCBSwPMFU5pLQY6yw+wo++57clEyqhlZcM0NmRtT69ODgxk382aS5bI6YCXVhucHLNfESKKb2YxpQ/I5FTMGj+ywU87KQmfffbdPrtnqlLBcf0eI4aZkp/aF7wgpmM4Vrw2XAh6Rn9w3xH19+h0h+0TQip2Svf9jeMW0oVW99x0hhJRswcpTkkvF4G/Ffm+4YsUpMarBR2ZVs1NSUIN/tuDtvaCGHdgxyXLOBJCJLZgwRCo+48KSL/sOviPkvaU11/BSEb5jH42iuSXzVMkqjjCygHlOy3JFFKsV00wYLmYAyI0YwQ0umJaNylmAfz5NPsDfyJxqIqTHtiSBPCNkjQUtGwZIB2RqWTelBeOGdcCmXGkD33fQUixnfBGxqnnNSi4iXu8czXG9yFQqQssSR9AZrhP7SKvaLvre8eHR0/3DJ/vHj98fPjs9fHL6+CR79uTxf+4ly1zSCSv14ALjasqJ5WJ4gP+8wufXbLWUqhhY6OeNNrKyLxwgTWrKlQ5zeE4FmTDS2C1hJKFFQSpmKOFiKlVF7SD2uZsTuZzLpixgG+ZSGMoFEUzbpUN0gH3t/52VJa6BJlQxoo20hKLaYxoQeOkJNC5kfs3UmFBRkPH1Mz125OhQ0n1H67rkOcVZTqXcn1DlfmJicWo3fNHk9ueEvhXTms7YBgIb9tEMUPEnqUgpZ44OwA5uLLf4jhr4k33T/Twisja84n8EtrNssuBsabcEF4TC2/YBU4EoFpw2qslNY8lWypkmS27msjGEisj1LRxGRJo5U056kBxXNpcip4aJhPGNtEhUhJJ5U1Gxrxgt6KRkRDdVRdWKyGTDpbuwakrD6zLMXRP2kWu74+dsFQFWEy5YQbgwkkgR3u7uiF9YWUryq1RlkSyRobNNGyBldD4TUrErOpELdkqODo9P+iv3imtj5+O+04HTDZ0RRvO5n2V7s/7Xg8g/D0bkAROL4wf/nW5VOmMCOcVJ9bPwYKZkU5+S4wE+ej9n+GVYJbeLnGylhE7sIqMUnJql3TxWfhp7vk0974uVpTm1m7As7bYbkYIZ/IdURE40Uwu7PMiu0rLZXNqVkooYes00qRjVjWKVfcENG17rbk5NuMjLpmDkz4xaMQBz1aSiK0JLLYlqhP3awVU6gwMNJpr9g5uqG1LPrYycsCiOgbMt/pSX2vMeEkk1Qth9IpFAFrdkfn6/L+dMpcJ7TuuaWQ60k4WdGqYKgt0SQDhunEpphDR2zf1kT8k5gsutIiCnOGnYt3YjjiJ+mWUF4hSRCaMmS/bv2cVrUEncwdmekFtxWtcHdio8ZxmJvJEK30IyTzqQuqBnED5FbuGa2OOVmLmSzWxOfm9YY8fXK21YpUnJrxn5C51e0xF5xwqO/FErmTOtuZj5RXGv6yafWyH9Ss60oXpOcB7kEsjtSIYbEZgcSRi0lbg7WD1nFVO0vOJe6rj9zD4aJoooi3q7eu2+7u6llx4G4YXdIlPOFLIP146QD/kUJBCIKf0o8LXXaexJpirQDrwCR3MltT38taHK7qdJY8gYl5sXY1gPuxKOGInQeEZPpk8OD6ctQnSnH8TZZ039g+C/W/Xm9vMOx61lUWRs+G4J5/qEEWBjXqydXtGanv3fXUzQaS2wv1KJ0FtBTSi+heIQj6AZXzBQW6hwn+Hb7uc5K+tpU9pNZDe1m2EY2Cwl+cltaMKFNlTkTo3pyCNtAYNQskzijlMSj1NWU0WdCuKmr4lgrMD7x3LO83kfVNjZuawsMKteJ/M+n1rF10semCqKJP9ITg0TpGRTQ1hVm1V/KadStlbRLtQuVvH9qt6wfF7aWQBEG7rShJZL+59AW6sK6rlnTVxWp43jt/Y0zyJpRJDZgarxXWRxB2LC4itwhPFpa+HjinUZoLX4Fc3n9krQJ3E6jqezu2zugNT/7q6xbWJ3cHqaHWaH+yo/TtSYvOQdPeZ5fLJBkTlzX1qGK9gUFD6KK8cFN5waCUKJEsHMUqprq+kIBgqV3XUeN1RQFJtRVcDBZc8lKfQoeR8PrQnHmz6XVvOdlnJpb2hWp2upze+fX7hRcVdENHu42Qf29QQzkCKaiaCu2Hcu//qG1DS/ZuahfpQBFNS0ayWNzGXZA4U3WnustIB6PUvBdZ3ZS5HXBDyVjKJCU0AmI5eyYuFsbjTqOIapijzw13SpHkStXrEpUy1URGeCGtUM97PTQXFlJyzoYKCDJgRAFIhFS8z8MkcQKf6oTTsm8gDszml0YwniRo3KHxcWvd8agQsAuiBqd96IMjBYpK+QpjekFeq4Xvuwx/ztNdx5cbwDDydYKUBW4zFhL8KaVVQYnoOSzj4ad6Kwj6grjFCAfxckuz9XjCQLbqfL/2BRsbcTZQqUfc1NQ91ynE/JSjYqwJjSsvTMx4U/1gybSbUa2Ve9QNSGlyVhwqq2jm/RNGKFZsG0sexhSWoJNuVlGXQuWtdK1opTw8rVLZQ6WhSKab0rfQ64HTV4x1sOoJO9QcxUEz5rZKPLFXIzfBME9tKSRcuKgUmIlPYCSAU5vxgRSgpZ2QWQilDSCP6RaGn5JCPkr5Gy7ogAm0XUCuaMKLr0OHm+H2fuwRhJ1j7hhL0AxAOsaNBmgTfQccbrsUVlnCFaY3uLq5konIqB+oEUEQm4TrgV86syWRmmbzhSShlUfbxZtD9rrcOf7Q94qwiGPbce9tpsxQHeBrrHy9GzkxZiOKkdHHZu/+L4WQvmjMks52Z1tSPF9Dk3KwDVm/1rKYxitOyjI4XhggmzK5zeJEpyANbD741UZk7OKqZ4TgeQbIRRqyuu5VUui52QDkGQ88u3xILoYfj8bC1au1pNh9Lggj6nghZ9SpUyT1X6dejMmLyqJQ9yqW2UkmLGTVOgrC6pgT96GOz9X/KglOLBKdn/4XH29Ojk2ePDEXlQUvPglJw8yZ4cPvnx6Bn5f3s9JPv0ujsx/UEzte9lcfITanuePCPidG88geWUzBQVTUkVN6tUqK5IboU7qByJ8HzuZWa42SCHc4Wnac6EYcopXtNSSkVEU02YGoEmP+dRrdFhUESvJPV8pbn9h7es5X5b6wSFN9Ik3gOwG3JBaGNkBSJ8xqSfbV//n0htpNgv8t7aKDbjUuxyp70DCJs22v6/PV+H1462msNpcKf9W8MmrE0oXt+AQ3ihzZznF+GA9hIRDouUs9AIIAWzZ28waZ9fLE7sg/OLxdOoeHTO2ormO6DN67Pn67BOgaNKe4ujvgXkAr/+pIP9uI2HVOb2+oY2iq/BTCqzad6NZipjFeXljkSalWgEAPhlGEBg2pTlwOa4UyT2NLFgACzIMbqgvKSTsr9nzsoJU4a85EIb5rSsFr6gymc7s772LZBTZ20HwMFIAjfHg7qkxjLCAF0Rzx0SNlWPEFgfiTnV852dl0gpC4dYOHaz5VIpZi+rLVP/FK8l9kV70AgpVqnjEPdSIsk+aObMmGOYBS/wOgF/2NmNg3spl2KKa0XLFkyrgORUxGs08e7gjuhzEHYg/t52JHHTZa0gFQGHPlY7OrIu51Ywoe4Brh8u+ogkW5LClmzZ1mSDIINpzT9Yb1nDKBCC7FF4yQxDETAXTRUNruHo9MIrMlqMveQFuzFZ6+SaktfMKJ6j8Vmnxm0qyMvnx2jathwyZSafMw2qVzI64UY7v2JE0nJX2x3e8mtyHYymbRTcuKoRzmGpWCVNMLES2RjNC5ZA6mKGOFHiPGp+Qn7RRfzUqY1tzz0OGgcC16ED7k9HOyzXEVVHsNsYUXK41OxOMu+9jwRCWOAyVTMq+B+46XkR3OBul61IwadTplJDCijHHJy/hOL23DdMUGEIEwuupKjamlXkrbNfLwNwXozIz1LOSob8T96++5mcF+ioBjNqb8P31emnT5/+8MMPz549+/HHH9vkxBOSl/bS/0e0ldw1Vc8SOMTCsVRBAw3wNGyVuIl6wqHR+4xqs3/U0XOdd2F37HDuvUrnL7z0Alz9JuwiyvePjh+fPHn6w7MfD+kkL9j0cBjjHR7ZAefU/9fHOtHK4WHfjXVnGL32ciDxaG0koznOKlbwpmqrzkoueBECF3ap6qAE8AAzvznToCy61CNC/2gUG5FZXo/CRpaKFHzGDS1lzqjon3RL3ZoWXh13NCl3c/zE7ZYexyjoHfX9kdx6uMHhFV5sOzWcu6EXM5eE8dQs51PuL44BC7TZO7+UM93LaTpIEoDJNPNw56ysEwUSzisMaQ1Da3cSipUlkOEVu8UBtRMdzynBcfK8aO9hXtHZTmVKujcAWLCXIkJLqsmk4aWxx/kAaobOdoRZ5CyHF521EUiiQjdDT6JDN8SHdoUtAHWhli24O1yNOOdoEQrSBFl2V+IERycVFXRmtTeQJ4EPepIEo1ITMZK41lJB8qLzeIMoSV7d7IJF7Tl5G0ysaAc6aEdnDoyZeF1v8rei9HH+1q/RIdjyZ27lFYxqLAZ035FXMAwL3sH/2V7BdFG8BdFF7nc20RdzDabb4N4/eO8fvBuU7v2D29Ps3j947x/8lvyDySH2rTkJW6iTHXsKb3HYfzl34VoK3PsM732G9z5Dcu8z/NZ8hpgo3kkV32RNeM0M3U9Xx9sbXSo6gtzmNn9TdsJAivnn5W8l6fegkLnYXwmT0cTIjIxZrjP30hizfTwakcPBjWeZsmq0wZwn2AxlL/KbkF/t9fv3hqkVhLJjsldgIy4KnjNN9vfdNbuiK48QZPuXfDY35ZC3LJkNfO8KFFjUSnuacmHYTLkIc1r8ZlH152g+ZxXt0J+0snB1X4M8yg6zw5RzlJIt0/bL8GBzQmo0LeeQveSC4XFA2EdUrMg1F9GM8QFzESrMn8L3wJyNqZeWeCVD36wls09DBRmVU810zNn004K150azchpdslTg6LewSe1IZwZiwuD+3oC2Q+YQbGunOzShD5yeAxikie7r0QjJ7oOT9WnbKY8tOslCLxdbJj3j+g65Tnziw7D3pJReCUQvi+J5i1cCS55BHn07G8myj5cplqHskiV5xmAOnOM60pg27IX0q5jvD4LF50BDEg6vmL3BepeUfWoHCmPE1Gk5TSbhxvNDUZ+KSyDb1EdfuJiKmDuFCj2ZMEyRcnq5G5N6+62RhKYq8QgtmgMJWBNmloxZSD7TQhQucCI4JxGYy13CZOq8lPaQJ2d+JW4mN96g3JCVVMxew8HGVMKImNkCf6YZ6YDQMKGT19ywMae7RfWUWyLJK1ZJtSJWyEHmjBuuSAgfGW7RlIIpdPvzmDTvXtZWCWIFpszfJgJkC/vQJ0d+4OgkpzXWjnDpkm1vgcueDRYQl6YWNyBPSsJk5Bz8lLB6UbuYU0HG+ILPTxrHVMywEHavj4Eg+7QoxiMydiy/DyzP4NGUl2w/V8wy2hiTenwBlzBiyNT2HOdmxi2cCsw9/UPSKl37NdXaEnMf87bax4VDfRfL8RI3g4PQJX445OZ8NneJasMyECQkHKDT3qqEMWF1IC+uszjIEOORX1PNhHYJY9F6RQOaAa84steOqE8h/JUqu7mhUMK0gUC0oPrIqVWFRmTJSF1SsBW4IARCw5Clq8pB85zVBpKlXVwCnmledRqRGssxNZqhqyqnzbBBDVYanHpRNIRFRs66YY1DpaTuOjomx0F6oW3DZZSsTILKQmHOilHgWZ+TjkmtK8z+69UWckyCCqTdqtyK9dwZZGI1qJAjmDyKy+pwDWMGiTpQvCkUlemKinNBKqlNkrUIVlXLREsZCy9p9LFN2ICWjFva/5lH11XeLj+U0zIHP6Wz7pR0Fc4qoJM76VzFKFDh3aETo1daRwcsC3zqy64obfypywrCO7UBPCaVFDxm7JJkiL090GT9itk/fVyYkeSasZo0NTIrfJSWrWpTFXLVAdM2Ha3IRDUvp+UoXdnoNBy4bRfUUM1usrV9kiRL7SEOTCeVP5fCbmU08o/dO2Py0Ep2zQw5cMexZuaR5WdvLscSFFZ5ILqZRPTh+lPJoimZBlHX2napnETNwK5goyyvlStfbYqLCDS98COLxJ8QjF1Uhy283Bcx2lDTDnwqGrWNs2fAvtn5kou6MVf+R0GF1CyXMQ1dNiZ9gerXvCz54Du1YjnXsG5Hg4v5woFuHSeWWAnYdr0JlAhwXgPp8G9mdUbFyLWQS5FWXYtcaoZ3vd/SAF3g3R1HT2KVwp1DbGOPXCe8I6o9ud0V2TCo5YLw3B54i9QfZaV6Se3ZhRWIOkFMOzQJ/kL1nDysmZrTWkMdIqjPM+VixlStuDCP7HoqunRnhpF2AeBoNTJMoGCVFNooO324L4FVgpvVgBXfR4EO/evsz89ffLEr7/kLO5sQIpOosx2cB0vUXPOtGOiTFW47/nDFNHeGz/gCgqi7qt3SqWDdsL+EJT3PxsPNV4FzV8HE1rdBU+xo4/B0HMccW8HGrB5OS6qq8dep4AGSbSMHyO1dn3fudECX8cbKPFiRKL1Ftd5MRuuef1KFklv9iVcr/Xs7bMSraruY+ju6BLtQqC0op+AGV4GbPjgVaYMsWaPECmnPmYJ9ZCjzC5lfJfHIBdeWUwo878HBAOokoyqfsyIy7KQxhIdqT8oe5GzhddnxFepa4z4lL1lNjn4kh89Oj5+eHh1iFPHzlz+dHv7v74+OT/7pkuWNnQD+Rczcqvx4p1D47Chzrx4dun/EnSlVRXSTW8Vy2pSohtQ1K/wH+F+t8j8dHWb2/49Ioc2fjrOj7Dg71rX509Hx47bvVDYml7sL1bDiy4FYJ8FatVejvcBeYnK0McXNrNtnbGvkpKKSr24TbTX4opNOjoSuDuiU8rJRbFAmhRG3kk3by6Qw7vayCXFurZ3i+vpKJ5ty3TadlpIOmmHfcX1NYAQs2selZc622vaQZbOMaMe4RMsSUNSPoinmg2bu8gSOVbi+uKse6mtzprohuAH3KyFVtQX/rZ3E3huw2/A/WAHD3jChUTCtWY18GiZxaNfy6PBwoABcRbnAABzn2VzJBtaswghNKsAK6YoYwWWZas1nQicI6fb90Q6xpJgZrZnlHhGngVRzviNalr5EU0dx1WzBkmimOwl+uHRjdkx3YUE9zI4C8Osco62iHuhv5vELtxcqRgVI1gVTyQ0+6OyWsODCsVJ6L1qJmtorIYlBDm7S9JoRMLU6UJz5ZEWhuTZgfkZaem9dZ3ft/dAhrL0qfPadAC8cN94KnJUyvRe0JJm9H0Rrz5qLgb3W7DA5bS85ZuPlKymw2prS3p6O1oa0vihxB7Rzczic25prqRgtVk7sFGxKm9KQy5W2CkA0YSTS5xwNJoApLTHjb8l1ago5iwI5AEWQwCinYJ0UUoCX4PyFA/7gZaNkzQ7OKm2YKmj14FGyhycTxRbouPCvX75/8Ag8IoL88stpVUXm5rT0b+0fPjk9PHzwqLOXd1Uh8R1DdoEjyGnaDXrdwlxcRXq6kJC3GXIWYtVxCP+wummWViiecqccO1/dT/7vjWX9oKZ+x69DNDP9Swq4zDSZWKnQtrA615P9Fbzx3mEC5hWQlbFknwXnaod7hY5qLXMeSwODmuZr+rUKzemRldYHznLj5QY6fGBBrXoiNXPVwNFpACDPvbJKXqOlz5L1v346f/3fvnK4jn4rl/kLxf/AsY3ajlct+jkbdDplaF21r3fm47kmKbnvjFG3cXNvmSKzTga+or7oPaBYMUMxbhZcJB3xVTA7/R0Jrxcw+JpsOEzTLjvqCcDux6rcnTyFVQ5QujpHSAgp5ZIwqlcWRcOAhSYrJGj4eCByo3Zne4iu3VnE3YXiUNAd4+us6Pz5/MWj9YSNPLdrXNLM3j4eXPSiOO4wuVgWrN2ZwiPhXWSpnCJtg8POEowtUgk9LCoyN7TsVKfsKUcnR0/bON6tYHAWJdBwKlnwKe8KB7kUO0toxtPBAtgDk4nqZwvW1OzK5npBzdwrtX0e1fyPbei8LsoapmbHsCsNaVfkYTCUSHuhoUXhdbexHQvi38BVPn7UUS+pmjFztUNSvAcIQGzQOPSqKrm47gQ97zABH8gFxlJwKY1IwRUoGQ6TDkWanYnU9y6UE6TpB5CmKt6/k+ish5cdUYuMnIZTzZhMFbSf3Z8b9LOfmUyD9XKq7CUt1leh0STsc0/SUjJUpDpSu8FPkq7SUvScUlYwxYONzbB8Drb52DLAYnZ+kcTOoJNS7eumrksevJVbKTdfT4beV5+d9xVm5n1lWXlffUbefTbe15mN9zVm4n0FWXj9y4I/v8KD9SfY+5Dtk8QCV8yZWmPwObzjgsqh8QIr2YKGzem0ssQN/CmlTb6qzKYvnc4UghakboV0/+L/3mgm8gV4WmYiV5af5LKqG4Phw65aVOgo9fwS42V9W6hhg2XaESqaVbD/UywE1E4e8LHXoBaCmjIYNJyGC9u5Al1DfLAbcU5VsaSKjciCK9PQ0hd60iPyAiqCJNV2wAhF/tJMmBLMQHuggt2qjobK59ywPHFq3WmyVO2D5XwjhwReb59/fPb06mm7XMN91YT7qgm3R+m+asL2NLvX0+6rJuy+aoI9P3eEyd4vbuy0OmIaR2KSVnve57p0bmky9piNre5Q2f2rmGkUloLtFVvc26jV3WmLPdRz0gJOZzrQ0cc0uYYxmIQ8Ahe586YH/dWquFzMIELBBaRvLKKKmrILaUaXoKXsGNrzAaW6VPi0ihigAfF6uIjBbipZ/OKWchjmrvjzzUbeBGOay3sHrkw4MuHED1AcDKM9nJCESK/fG1qCaTyM6UqKYVUGTMOzCDjrXMxegqxwWGttTxJFCpbzAhJkre4KbBQFu7TvdxZe6mxKK16udnQ0vb0kOD556G19ihVzakakYBNOxYhMFWMTXYzIkotCLqP7P1bRgzd7eDflrupz9HReVx8DtHzv8/HZ5z6zd1gFpbmlwWv5G12w7gyurcr/xeaA0ALacOdSdOnihfquoewkO9w/Ojred3lhXex3qNCsob8PX06ov47g/9HF1l+bvxTGHp7je6sbST0izaQRptnE61QteY/XB6sr7A75bXnk6DA7OsmOWtjuKtjFtwPtiN+fpHKVwX21YteT1nkeWnXY7RDQ1HgcKiyPoZD8oholCjBEXie6brisj9KWr0kN8tTjEc/qMOLQmT1Q6+S+4lCbu+4rDt1XHLqvOPR1VxyaG9Oy4v/y/v0F/H2bHiX2oxAOm/n6MGTcqHLsA1MZRlMnXTUBSVV6fF1T3O3t+f6DiSxW2UDF25sCMm6senvZis9oo0kAape8z579sB5FF0yzoz383l1HcDE2YvkLK0tJllKVxTC2O6Dle2lo2Yl46VD0oUUWNvucUasH9JWro5PHwwSumJnLnSX6tUiKoDoJ0MjkmBoA5WImLM0ZMJKUcskU5HxbEeprUGXkkrlEWZk3lY/zCmNrV7LlwbkPq7da3svnlw/65rEZMyNSQ+2YujGDZIIW0WpnAVvv3PAxpSalXG81rezRpwcHk1LOMvc0y2V10MFd11Jo9sX3OYLddqOnSH7Znb4Jz/Vb3eP7pfe6w/bTNrtDWhtqGj1g6t0W9fUpNm2aIqBhi+/JYdtNttsrHuC17s58lKWdTny9KXeiv3J/3nigo82Jtsr8SMjtTDNztjmZYfK7uEO+9ZlOFqvgBXGVwnrZi9hBoJX8vKRKjEdkDEXT7D/4QKIoU6o1nV0m3Po0tlYel52MT8Cl3eIFsPWTNxKdeIo1mkpu0P1uSAMlYoLaWlPVqod4jnZPRWM5wrEb1ituyBWphRSa4PsCMnbENFPPr4UbJU0Q7eSHusmOehPyCcBhzDldsJB7pO2iYixy7uspYoghWgaYyCU2S1BEsCUpuWAausktkluKvd+UjApIXGuj/Ln5y0RLl568twd6gD3rU+PwxFvAQFv47DRmcL+Bo+L1yu39YE3HbJlUGrxJHt1QtM/n2rTjPNCeUlWNcPTHsGC5YMpLkBhUQnAVkpwdF6eh0+5G/o1Pigrxo3eqdXSziHyhoNvEZdTYmWOHmSZneHWb8QUTGKGbQnUSrlbSyFyW7VJFVE24UVRF0z9xia0unwxKEmrcFBXPlfR5TCPgQFpqCcBWuPPjy/p6VbNoTuP57yMypTmbSHk9ImbJjUGvBddkmVYksqImlomKRT7JgokiqaYEIdPYTTGEF9sjtgjhxKFgAu6Cg8Iq3ucXGEOtR1BVXI9IMuaSK582+BWq5pS3O8HddX+WPVS5UNUyigoNijisyETafcMVc/XbWtn9Y1eZCr50SfdpWXX/3Bf6GZGx36zuJzy7eFwJ3VR9Ajx++qxFACdBzOpqd50wz9CUBaU+IaMMhHZSyP78AitNOm6imixZWTohF+bjt1+MVmjLvyykolNipCz36UxIbXhutUdRUNXqtBmGnZZymS7GK0aVwKR1asLVaMbNvJnApcgyCJRWOwjE2+fFvtXVBsoDn87f/qN+c/LLP77++cnrvx48m5+r/7j4PT/5z3/74/BPraUIrLED9ebBCz+419O8uDaKTqc8z/4m3jE7Hyy/FI/T078J8rdAnL+RfyBcTGQjir8JQv6ByMYkf3FhmBK0xL8sB8W/GgGM+zfxN/HrnIl0zIrWdVKg2PWPtYfXPrbUq2JyqKtTOwoHUqLYpGMGyWWH2dME4pXs5BecLTPEYQ1gTxqpSM0Ur5hhChFpIb0dThGRFgb2v+DKcMDSkQPQ7EGXnRztW3wzlWpJVcGKq88JPkhacoQ8dbddk5+cglwr+XGgVtWPx9lRdpS1i6dwKugVhi/tSMCcn705IxdeOrwBUOSh37nL5TKzOGRSzQ7wYIbatgdenuwjcv0H2ce5qcokif7SyRE4r3wdE/+VdvKHllDTAiQYaDxvmPmplEssrwb/chbbMG4pZ/7W1ziT7dCcegRvpxzu2i2CytFkRSR4OaHYuPSnr44hbP5c6mL7M1jtfuVT3kL787qkuAPXDfJJR677duDQjb8MHLv+x6ifuQN4+OA9bhspPNfs4ir76gd/u4hnJsRUEPYxgxNtRErgqN9objVJSzR79kYN9+vT3IJ/JLjHPda7IOGlZXiqAy8nQgy1dnCl0lgIgpG/IJx0G4bmAZHCJV1Z4dQU9YiYvB4RXi+e7vO8qkeEmTx79PVR3uQdwu8oLuEcD523l+eQhl3iIbpM4wc8W7+yVMws7U6QgsktqdYsH5GaV0DQr4+cFunENOAq1bRaRrxNn23K/xDh836tkJrlnJaeg0chORbj4HpXaiwuEQrvFsyw3Iz8+PARVhe5ecT99vnmlKuk2Gs74zVEiFCSN9rIKqR94KDQghy83W6qnZonUkz5rImtSIwkqhHbE4BoOTUWXFILrZ2GMuWKLWlZ6pHVcFUDIT1IIS7FQa1gijCUD0r0OmSiJWomtFShwtWSTVpYJEAgCLyUWpOhoS0hzy5eO2rotM2q54bUgEOxGvQa+40TUDg4hpGI1SitFIfz1IEVtK/1guygo8K8gcS+woob09VZIa+dbfX3hjU4MHn5/hUkLkkBXOPveq5UdLuNiWMnb2lSDEyDUNCqYNAfwNEDOsK+fH55C6PTfbLNfbLN7VG6T7bZnmb3yTb3yTbfdLJNN9cmnL5t+8enGWX6LVKHh/9ibU5biup91sN91sN91sN91sPdZz1opjgtd2sw9vdrB8yd9zcV0bq75mC+20AqVkNTl02F7ZlyyY72Yug1J2+IjiOtaqazoagb7ypQadsBf/GEKJxCw39q7VqEfVzBP2RZMgjTwUus/Ve8gg7ERvgxWyRteZ/vkqhh5gghjVnPOhhs7q16ByyVCJYYtjSjgv8RlX1v5uk+vyEOJB3H3++ZUDyfI+PAxX5d77KqpsKf0lI5fbXFdJ1IjTQwJPYmnbOyhrLcVCkqZr5dj3GVb5OeP1RgkA54DNpR+wGNOJ/b1On4O+SppKh+sXoxKX8E9SBK9RYrBRF8CSL4BnZ6D3bWTruANawjO9J9++jDb1Iz/MbVwm9YJ/yGFMJvWBv86lXBxEMamnk4KXeRPNq6mfZa4Ra6/g6fdDkV8bSLOXjO5tzufQeBjaGJMC8OEl52QSWtuFoQwL4Da1ZDLt7UMEG0oSvt6x/77r7YjZuG/lmgINYcHTWQqVjKCS2TSvQe3WhQ2q7+1WybDIRPiwFTiq5cuAQQiaoZONJSO9lr6DPp9AmcXq2kYbkB5wk3fNFKguzpne7PfaJDiuY+2S/DPxsd7hT7xLf/aUdRsI8sb6ALwo5IcTaB7jAMw3XdCnqqROi9HXLQaHUw4eLAz+1L1K10O86dQmGh7NUC2kyQnJYlg5TxmaJVSIDUvOIlHegE3EW+vjFL9FZZIxdhC/YPn+OTdmBS3YP9+VkrFxQKxbjl3LPTG0Kkc+X9zEYq732X1ZSTXMOUvivg+PDo6f7hk/3jx+8Pn50ePjl9fJI9e/L4PzudNuaK0WK7lPBbUeg9DEzOX9y8QCD1d83ZAKQT72JpCM9HmOWArA5+UhcXUqf7gjynAsO4J7HPpjkNQyalDgglEyWXGmwPPjnEIeFlwZJNSE1nLOmkKrGbfXuJllJdczG7wvimXvPsO01zc7BIgOXNF+EI7UqruazYAS2xYUVMHIuBAe5Mf5c82nimx9Y6DPug+2qlU5rzkht7ONd8IbEdsZIN9NKvOcuTDlbQncUvNhhI4AXdbaviwuE1Y9CEvaJiZZWwHEID7NX25fNL39XpfYqCGxqb5YENB2+Q1QivxpBZ4M9CaFplQfgyVdI5puD81rUURdxFLv1FkLGjYjYOMzmDxr+KmWDwsRSKLgSmR0n+0ISRBoocQZv9YD0ZuXjPUWQCHwk3InnJoS2Yf5WKIgRHpQGoUAQE7AN1DT1ly5KcX3i1wsiIPa/HI9StKKg7whHNVTbAaMPzC2IUX3BalqsREZJU1BhIcGHhmOAGgFHFihGZrELQTgrqlGaTLM+K8W3MDNu04Bh23pyVIR/u/ELjGkuRNKJOb/L9+J/L7aJ/3HsDeUGOeVxtiBCMkkshXKTSNBjiXDiFYjOqCoxT0Rrbi8f3NbZJ5yGW0qqbGMqaS5U0Kv5JKvL++UXoCwRCM6CJuOWM278dgbjgUGji8q9vXBjnQ+0L9nu9/PlFgksGQLBeTAi+7UJyNXDLVY8efvnaMfBC+36IIBVcsA2huWm80xYj+ZiqyIMw3gMslzwNamWKheggrn2FMfjZXTO8b7mfUeVFiSsWm6Ng0x0Q6TycQLpsAaDQywpm4UaMoUBY7OO3RuTxHoM73X09NFgkbSwEEoe0uxeXcR8d9j5n1b35HIc/8FNo91XBaxctrJSvqDA898H1LiuLfcTWSE6exRuRvapNm9K+tuB2uvwPlpg3BcmZgotgTIzyskoFGFNall5W+Y7+OTVsJtUKhZVLiNOGlyVhAhrqwWtrUlsswabc6shuWFrXStaKU8PK1W0uZyjJd6UOobMAW+3hwoSjA5MqvYCpJnzWyEaXK+Rm+CaoOktLFh1uB+CaoFaMjwj1xfiwcA2U8JOWTzJC/hop64o4pvVJcFcpuoxpCMj348w9cDmybTVO2JMhJjAWDYaj4b1ybM8fKICTIVrjESmYPbIgZdUXt47NAuGc4d3mknedP/ZnSByD0usx9c55dVxvadg/ffvJs3Z8OU7qBsw+qdANYoPjd9pW3YfM3YfM3YfM3YfM3YfMfdMhc58YsbbXD1nzAWuRs/D62fEHk/OLxYl9cH6xeBoVj85Z+8Ui3YbC7D4vS+3Cpad9ysHeMVrenPB0O4OlhLIha+d9X0/zvp7mfT1Ncl9P81urp+kKm8B7iVnNP7oh1MqXRekaaUz6m1QDLY6sguSQW1JNclmW0IP6hnCqKReFKzHluROywpEtQx0wD9u+6SMWtrchsHrOKqZoucNiHy89jFQ8SacVevQf8inoANCWXD/qVnriRdKlAsw9mtBcSa2JYuDYcrVzxm5A2H2FhJ5Ppq8PPqMn0yeHh9O2lrOL7bTXF82+4F4jBFpXEeP+lJ2pAndgGZqYrlqkc0UGKnrNNOGG1FJrPkHnUWCdMDSwUJJ4iTwrWI+hhjpfeEO+sutUM8WZyMFhpXXDNBoL7ViKFXYCrsVYtOmjGz+M65vV8wLLBsRQCriHeWZHYxoXM2i+7NqW9Va0ePwDe8ImU3ZI2dP85McfjosJ+3F6ePTDCT16+viHyeTZ8ckP05sKJNx9TwvP4TGS1+3/gWDe9GoVPoTwXsf7cBqBIyTUlijlUsMlaykDeeIdy48FPS68qFCR+bxiYH8PtdzxGihazkveqk/hmmSE3QbHW9qLpcRSaw49u4wFtzrnpLEz9/WucG1VA76QcOLMpTZ6mH3RdO9N1W6yBEvCuKl0AhNcDjkkcMspeVlSbXjuHEsJmWEKLvPYH9OohDfaMNW6KqFT48+MGt0fgmtLnYJNaVMaqEhUB99ooJeBttEgkcOYfEqEJH6M0JBkoAhiOof9NOU1iR8wO7HQuLY3MH6HT/8+wfK32l3wofd3urR21I8HztmWkLQnOkjJRGHwM1kjKWGQmJIMu66NXZsZRx3uCIOGegfj1sIPVcdMf28tx+7C3Pf+3YenthckOFpaOk9/VaIMg1oL8ppQu2swdJwZ7Lje0XkWESQN7NcvbJYdZ2ldBfTHtNS/+GSD9odv3eyd8w4fwAqtAwftuqftkRI33A0OuNR95LxwX6WbyDm87t1EX4mbCNfDWZPSMkY9k9IX8xUhSve+ontf0d2gdO8r2p5m976ie1/RN+Urwmp835qvyGFNdu0r2v50/4IOo4HJ3zuM7h1G9w4jcu8w+tYcRo1CieWsBR/evYI/15sKPrx75S/3rmMm0U0NVT4xB88CMoBOTRWs5Yd3r1wBP/dmCIyfMzJRjGKShVwKwoWRROdzZoUL3qBGkDLmvpfEy/5tzAJDV7y72zQv3I3dkVuVo9BA4MFyucycpSrL5YO2rRaya3IK1gOgZ0VXGE7twn2tmoDVBoGuGH5ermLqLm1PjbiMHLADQ48GzUYuDj/WtwaVdSZDpxV3tXfWgZ6K2J5Ci65TRWfV7jpM7dnTNjG3NaokdGpctZDx9+OE0EbWDzoW0PH3Y98vxbWHQS3cId2RGTvMfD+f4lFp+R/sRLyy6+kSeCAEu9EsrtYqMchgRYkwLy6gnSGc8OMRWc4ZJAKYVocYxXIptFENWCEt92CMubcIta1RqRoz0BWtvfynJyePD9Dm+i+//6llg/3eyHal3OF+RXd5WGH/HZija1kELKJD5lKYbV+/fiONi13nYqBe6SgtT1OE3Ql1Wv1ijjARh+p0eWgOqXGlnLlbn/2Ua5fh/FujTQz699VqrWBb2+8nZHqFz8KwFJygS6oDoqOW4B10B3/SwtrR1vzcUf61Tlbyrtf8wg0/2Kwz4mB2pSBdQI+hFuxEBjkCPchuuILcQaJtcg3p4XFy8rifXXryuIUUZIntamNa4QsAHBMHCwfgi7/g3AbnEPaBpWmH2Xoy/l9AxrOPULA4aTeRQoFMFzxhQ+8vIe23sEMTEzpWl0pwh0+NrzxFAd6kMeGtUQIMJ4tBHWHE0PWpqk3EB1DHN8fu646rruWLJhNmlozFYx5ysZYSlYfOQYZa067W9hJGX78HQLo86MhZzKIdnw6ex4jvGjnVU6B3fKtNYxIS4ZJi0FKT9c2Jiu+dDt5zqg0XHIJX8VyC5sZsQcNh7TS2tqPtp6RgB12gxYiBvTi9qNgnnGm3FfwFDxv9mDkV8BkvfParV+lDvq47KWGbgRfTUam6TQDW39Eu8g2ZRL4Ba8jf2xBybwO50Qby1Zk/vlrLh2bqis78lSiR7CQ+3UK+4xheyscITnvJd1WQfPGLcLI45N7bO58rgTSXS9cudckmIcIEAmySuphYfYIqqy00AVWvX2wvkrHvxZfayQ5ad0n4xdyHEHypbk4JhyDpekhd0ilV/EteaD8It6CLdpRRZK4Bb/4fvCzpwZPskDxEMv4TeX7xwZGUvL0kR8dXR9hQ09dye0TO6rpkv7LJX7g5eHr4JDvKjp4EcfLwL7+8f/1qhN/8zPJr+Yi4uKeDo+PskLyWE16yg6MnL49Onjk6HTw97JayvS+OPYj1fXHs++LYn4fx/9ji2LtF9d/7UnfN0WCl4Hf7FsgpmTBoFURFPpcK/9zPZVUBmk6X+DO+04L2zzDoc2+OwE/g8xAy6S8PoFyWrpSIK2/93Zr4R8C30/RhiCQbOzm4WbdGtphlhlfsjxjthwPTkgcLaE3N/NTdTzsvV3ymKMIzqmHt0XEurWHl5DeWh/bd8MfVjTP553CKBcrCOvouWUBOF1XaxgA68bcQiIrTWiAv7UedUptQpqYouCsTZHV3iHN1MfkAJxQMS9eQDEeUr1vBDWhF1JKQ7dZC9rijv4iWidL3Nq4fDDrIdv2BB3l04+gQJsvAfOHzILZl7fccc0E4izk69mrkdm9eyqaIG/W5/dPbPiCanbqEtgFKv3a/oj6etz7VlgVY4VNHaFFcwQtXfkhfOU6qdCu3Zg0fZLWSlvWjOSBIIffL/sfNPJqqu+4Ty48/SzkrGc4YuXEAOK/ojA2AphXfp5O8ODp+PChKI/RzOwI5fxFsDEinkNqEU/6enFk2wfysskjFQQhpYoZmgSRA5Bv4bPDljXyWwPAIxlTBzWDChML7t4a0xdbpwNp2/yTQXNrTVSJgNgNzH2TJB9vCcgcYL7lZXW1xbGz+aluojse3Xbje/toWDsYhbgWj9erg+F4eFTK/Bl51AumF/3tge+FvkJ7UTTpxv9l9redSmSs8/07JlJaaJeoKwtsPwmiNWhHQIoOn47pTzJ2IaSzOMLESgg1/Mki0NaCsxLk9NJB0Im1eeyuonS+3A/rp4Eo6YaW2gvP92xdvrQa3JEaSitZWyGr2Lz1cWuoU2axSkc2qBcp0RCHznGvP88i3v+BfA4OcW30o4VZ3LNjPfU5mljAodMEfYk93brx8fpmmGPGQM8Ryna2qMnPvYdo5VS5QW4r9+GXHtIyob+b09UvTsv/6ISZSloyKLck7jRQBn2Nc9j5cqbNJw8s+yP6KhtP7wdGzF0eHPz7YDp23lwQgtNvKDCGSy4IN7oNNuGijmMnn2yPjofhurYEDr5sJU4JhzpDjw7+kzwbGjb8HZa+tucVBScqFm6Vq/OhGydpCejPPdSley2JY7NxqMycUqKVr3z0IqhmQ4Z8K6UIW5MP5iz4gyG2oaX53k4oj9oHJoifyPxOYN9b1gaG4vFksbwfIyf+K1n1I4BzCEp93BS4ZchimYpA2qJm5W4LGcdeQtWB1KVcQznengOO4awBDVvi0Ke98ysnAa0DfoHV8KuAw7I1gh1Wsz4eL4zpxHhug9NqfDIzr69kHKR6ukENSN22uchuRyz5uq+T5wvC9fhpkQNFzM/5NlvKa033aGFlwnctFehX4V/yVvHC/rEj6HknuuTfaKgaGSs88h0cYcp2x0b2XoUGnbZy9haXO210xL47IaUAgsb4Ow+SbrL7rbHY0nztv6RyM0MGH3a4Rz7gvsW2JUJCiwTb2hirT1C1TKaidUlWYWhhsjeCvr6miFTMMiiVNGFgH7bpBW3mG8WX4wP6J4WS8ANQ0W0AloZoqozGE6vxiRNI2F7wYQYwCeIlaKFFRYG8FsAAOkdDVu6uVLJrc3J6Q710eL+5dN4xVysLcNoH9ZHZpgd3TwaHwMIH86AbQSSPGW0J2LRaTNGacfsILOtSb6WZ9ezx8rsWtoX9494rM7VUPKkkAOMetgMkmoueN6vhI2peSNVB/DQHmfn5Y4gJZ3F3gaGPmTBiOSaY+8NiLtVLOohR7JWdkyktEGIMrNrlFSv96yUXb69GaZilnmX0tS0J/h0ir2O8NV6yIKvsNBAfYnRJnFhWgArSISYO1Q3ApUGuo0REgGSD8FDujnJLxwYKqg1LODlyjwVLOxll/ni6QvV3p4rMne9kqZ9Gbspw575OfN1SwdxmaA0jK6VSztkxJ4ps/bRlwTBeZWUtloPmrYCiSNaFdl5W9WtLqrihkORdHJMs5E0AFu8mjDMCQf7ch97QpZGP27G6w/2ZK7bXR46JuTGpXjeiAVnAjVWAArPPTWa+4VtitAA6adlYBkBKZEtqvxBpMAYa3zYyxgJPE9HmXVYHAteun4eThT7xk4MJEAeHYvTVr9A1q9nvDum6cz+EQP2CiRYCQTENlA1+sNAgM7FqzujsudQMS9tEoGo2xeGBzqbhZDaPif70zVPyAIcYa4GyiBigb3Kyu4Gp5lzJ03lQUtwt4Xz2gzYuyczQ8oA4aoSkhdnS7SwRE2xFnhx+QnNOSzgaL3qSDDZ848KmHMLTUc2PqDJu3aJa5E/iqZGLWOTUHvMGtTyeyWGVpeZ6NTpNOgOXNF65oUwxz7n+y/pbWDfXur2ALPSukOrf4W6plQey5oTBYPoje9ZLIg65k0ZTbRVW0Xt1IdsvqV+A6N7Sqtxo8VyypfLhxdHQJZdQYdadxG+m4kb2dvcteaphYcCUFmGYWVHG7mzVZKm4ME1CNEEbY0+RfL9++gbWxG2sGqcWKL2JIcKcYLVUMC7DFwBs46CEhEwvaBa3XHYLtcd0J2YssQdLGGiq3vuwULJzFaR/VjjqRV/UnQjl//vrClWDpD9nzFG8/5ED0CZ99+pA/Dw/pFWKd4dm/7hBdM+xZY6SQlWx0iAiEYTpQ0pojOwYVJ9Tqu/5p8smHf7S653d63eN9pYvcZgJ8Tahd0+k13fJsSaKKZM3ztUdEfL7JimkBu4F640PVAxNtHIOcshWAMJLjlwHvW3rf+nRA7orllWcnaPrgrtnqLgh3zVajdrNHfyXB393FBDLgY1foNThNSplf9867iF+RltbdhhaNYYo8zGVVQ42l4hGCIBFED4c5owVTfbeLYNqwW5HGjeTkflyLLHm3y9jrV2jTKt2ADnGn0DXzlUmvEwQHYcOy3S105IQB+P8/AAD//3nMjTE=" + return "eJzsfWlzGzmS6Pf+FXjqiCd7l6IOy0drd2afRnZ3a8eH1pK3d3Z7QwSrQBKtKqAaQJFmv3j//QUycdVBirJFjx2rnoixSFYhE4lEIpHn9+SX0/dvz9/+9L/IS0mENITl3BAz45pMeMFIzhXLTLEcEG7IgmoyZYIpalhOxktiZoy8OrsklZK/scwMvvuejKlmOZECvp8zpbkU5HB4MDwcfvc9uSgY1YzMueaGzIyp9Mn+/pSbWT0eZrLcZwXVhmf7LNPESKLr6ZRpQ7IZFVMGX9lhJ5wVuR5+990euWHLE8Iy/R0hhpuCndgHviMkZzpTvDJcCviK/OjeIe7tk+8I2SOCluyE7P4fw0umDS2r3e8IIaRgc1ackEwqBp8V+73miuUnxKgavzLLip2QnBr82IC3+5Iatm/HJIsZE0AmNmfCEKn4lAtLvuF38B4hV5bWXMNDeXiPfTSKZpbMEyXLOMLAAuYZLYolUaxSTDNhuJgCIDdiBNe7YFrWKmMB/vkkeQF/IzOqiZAe24IE8gyQNea0qBkgHZCpZFUXFowb1gGbcKUNvN9CS7GM8XnEquIVK7iIeL13NMf1IhOpCC0KHEEPcZ3YR1pWdtF3jw4On+0dPN07enJ18OLk4OnJk+Phi6dP/nM3WeaCjlmhexcYV1OOLRfDF/jnNX5/w5YLqfKehT6rtZGlfWAfaVJRrnSYwxkVZMxIbbeEkYTmOSmZoYSLiVQltYPY792cyOVM1kUO2zCTwlAuiGDaLh2iA+xr/zstClwDTahiRBtpCUW1xzQg8MoTaJTL7IapEaEiJ6ObF3rkyNGipHuPVlXBM4qznEi5N6bK/cTE/MRu+LzO7M8JfUumNZ2yNQQ27KPpoeKPUpFCTh0dgB3cWG7xHTXwJ/uk+3lAZGV4yf8IbGfZZM7Zwm4JLgiFp+0XTAWiWHDaqDoztSVbIaeaLLiZydoQKiLXN3AYEGlmTDnpQTJc2UyKjBomEsY30iJREkpmdUnFnmI0p+OCEV2XJVVLIpMNl+7Csi4Mr4owd03YR67tjp+xZQRYjrlgOeHCSCJFeLq9I35mRSHJL1IVebJEhk7XbYCU0flUSMWu6VjO2Qk5PDg67q7ca66NnY97TwdON3RKGM1mfpbNzfpfO5F/dgZkh4n50c5/p1uVTplATnFS/TR8MVWyrk7IUQ8fXc0YvhlWye0iJ1spoWO7yCgFJ2ZhN4+Vn8aebxPP+2JpaU7tJiwKu+0GJGcG/5CKyLFmam6XB9lVWjabSbtSUhFDb5gmJaO6Vqy0D7hhw2PtzakJF1lR54z8hVErBmCumpR0SWihJVG1sG87uEoP4UCDiQ7/wU3VDalnVkaOWRTHwNkWf8oL7XkPiaRqIew+kUggi1syP7/fFzOmUuE9o1XFLAfaycJODVMFwW4JIBw3TqQ0Qhq75n6yJ+QcwWVWEZATnDTsW7sRBxG/oWUF4hSRMaNmmOzf04s3oJK4g7M5IbfitKr27VR4xoYk8kYqfHPJPOlA6oKeQfgEuYVrYo9XYmZK1tMZ+b1mtR1fL7VhpSYFv2Hkr3RyQwfkPcs58kelZMa05mLqF8U9rutsZoX0aznVhuoZwXmQSyC3IxluRGByJGHQVuLuYNWMlUzR4pp7qeP2M/tomMijLOrs6pX7ur2XXnkYhOd2i0w4U8g+XDtCPuITkEAgpvTjwNdep7EnmSpBO/AKHM2U1Pbw14Yqu5/GtSEjXG6ej2A97Eo4YiRC4wU9njw9OJg0CNGefhBnnzX1D4L/btWbu887HLeWRZGx4b0FnOtjRoCNeb5yenljevb/tzFBp7XA/kolQmcFNaH4FIpDPIKmfM5AbaHCvYZPu59nrKgmdWE3kd3UboZhYLOQ5Ee3oQkX2lCROTWmJY+0BQxCyTKJO05JPE5ZRRV1KoibviaCsRzvH4sZz2ZdUGFnZ7K0wKx6ncz7fGIVXy95YKookvxXcmKYIAWbGMLKyiy7SzmRsrGKdqG2sYpXy2rN8nlpZwEQbehSE1os7D+BtlYV1DPPmrisThvHd+1pPoykEUFmB6rGZ5HFHYgxi4/AEcYnjYWPK9ZmgMbilzSb2StBl8TpOJ7O7rK5BVL/u7vGNondwunZ8GB4sKeyo0SNyQre0mPO4jdrFJlT96ZluJxNQOGjuHJccMOpkSCUKBHMLKS6sZqOYKBQ2V3ncUMFRbEpVTkcXPZckkIPkufx0BpzvOlzaTXfSSEX9oZmdbqG2nx1duFGxV0R0ezgZr+wjyeYgRTRTAR1xT5z+be3pKLZDTOP9OMhQEFNu1LSyEwWHVB4o7XHSgOo17MUXNeZvRR5TcBTySgqNAVkhuRSliyczbVGHccwVZIdf02Xaidq9YpNmGqgIloT1KhmuJ+dDoorO2ZBBwMdNCEAokAsWmLqlzmCSPFHbdoxkQdgd06ta0sQN2pU/riw6P1WC1wA0AVRu/NGlJ7BIn2FNJ0hrVDH9dqDPeZvr+HOi+PtezjBSgGyGo8JexHWrKTC8AyUdPbRuBOFfURdYYAC/Lsg2f25YiSZcztd/geLir2dKFOg7GtuauqW43xClrJWAcaEFoVnPi78sWbYVKrlwD7qBaI2vCgIE1a1dXyLphErNHOmjWUPS1JLsAkviqBz0apSslKcGlYs76DU0TxXTOtt6XPA7ajBO95yAJ3sDWKmHPNpLWtdLJGb4Z0gsBeWLFqWDExCpLAXQCrI+cWAUJLL0i6AVISSWvCPREvLJ0NC/hYp644IsFlErWDGiKILj5Pn+9HQfTFCkjVPOGEvAPEAy2u0WeANdDTk1ciiMhoiWiN7i6uYyJ2KgfqBFBEJuE64FfOrMl4apm85UgoZVH28WTRfa6zDX+wPeKsIhj23HvbabMUB3gbax8vhi+MGYjipLRx2bv/i+MMGzCmTw4yb5fWWFNMzbpYAqjP7N1IYxWjRRUcKwwUTZls4vU2U5ACsg99bqcyMnJZM8Yz2IFkLo5bXXMvrTOZbIR2CIOeX74gF0cHw7HQlWttaTYdS74KeUUHzLqUKmaUq/Sp0pkxeV5IHudQ0Skkx5abOUVYX1MCHDga7/5fsFFLsnJC950+Gzw6PXzw5GJCdgpqdE3L8dPj04OkPhy/I/9vtINml1/2J6Q+aqT0vi5OfUNvz5BkQp3vjCSwnZKqoqAuquFmmQnVJMivcQeVIhOeZl5nhZoMczhWephkThimneE0KKRURdTlmagCa/IxHtUaHQRG9glSzpeb2D29Zy/y21gkKb6VJvAdgN+SC0NrIEkT4lEk/267+P5baSLGXZ521UWzKpdjmTnsPENZttL1/O1uF15a2msOpd6f9W83GrEkoXt2CQ3igyZznF+GA9hIRDouUs9AIIAWzZ28waZ9fzI/tF+cX82dR8WidtSXNtkCbN6dnq7BOgaNKe4ejvgHkAt/+pIP9qImHVObu+oY2iq/ATCqzbt61ZmrISsqLLYk0K9EIAPDL0IPApC6Kns1xr0jsamLBAFiQY3ROeUHHRXfPnBZjpgx5xYU2zGlZDXxBlR9uzfratUBOnLUdAAcjCdwc96uCGssIPXRFPLdI2FQ9QmBdJGZUz7Z2XiKlLBxi4djNlkmlmL2sNkz9E7yW2AftQSOkWKaOQ9xLiST7oJkzY45gFjzH6wR8sLMbBfdSJsUE14oWDZhWAcmoiNdo4t3BLdHnIGxB/L1rSeK6zVpBKgIOXay2dGRdzqxgQt0DXD9cdBFJtiSFLdmwrckaQQbTmv9itWUNo0AIskfuJTMMRcBcNFE0uIaj0wuvyGgx9pIX7MZkpZNrQt4wo3iGxmedGrepIK/OjtC0bTlkwkw2YxpUr2R0wo12fsWIpOWupju84dfkOhhNmyi4cVUtnMNSsVKaYGIlsjaa5yyB1MYMcaLEedT8hPyii/iqUxubnnscNA4ErkMH3J+OdliuI6qOYHcxomRwqdmeZN69igRCWOAyVVMq+B+46Xke3OBuly1JzicTplJDCijHHJy/hOL23DNMUGEIE3OupCibmlXkrdNfLgNwng/IT1JOC4b8T969/4mc5+ioBjNqZ8N31elnz549f/78xYsXP/zwQ5OceELywl76/4i2kvum6mkCh1g4lipooAGehq0SN1FHONR6j1Ft9g5beq7zLmyPHc69V+n8pZdegKvfhG1E+d7h0ZPjp8+ev/jhgI6znE0O+jHe4pEdcE79f12sE60cvuy6se4NozdeDiQerbVkNEfDkuW8Lpuqs5JznofAhW2qOigBPMCh35xpUBZd6AGhf9SKDcg0qwZhI0tFcj7lhhYyY1R0T7qFbkwLr45bmpS7OX7idkuPYxT0jvr+SG58ucbhFR5sOjWcu6ETM5eE8VQs4xPuL44BC7TZO7+UM93LSTpIEoDJNPNwZ6yoEgUSzisMaQ1Da3cSiqUlkOElu8MBtRUdzynBcfI8b+5hXtLpVmVKujcAWLCXIkILqsm45oWxx3kPaoZOt4RZ5CyHF502EUiiQtdDT6JD18SHtoUtAHWhlg24W1yNOOdoEQrSBFl2W+IERyclFXRqtTeQJ4EPOpIEo1ITMZK41lJB8rL19RpRkjy63gWL2nPyNJhY0Q6034zO7Bkz8bre5m9F6eP8rV+jQ7Dhz9zIKxjVWAzovievYBgWvIP/s72C6aJ4C6KL3G9toi/mGky3wYN/8ME/eD8oPfgHN6fZg3/wwT/4LfkHk0PsW3MSNlAnW/YU3uGw/3LuwpUUePAZPvgMH3yG5MFn+K35DDFRvJUqvs6a8IYZupeujrc3ulR0BLnJbf627ISeFPPPy99K0u9BIXOxvxImo4mRQzJimR66h0aY7ePRiBwObjzLlGWtDeY8wWYoOpHfhPxir9+/10wtIZQdk70CG3GR84xpsrfnrtklXXqEINu/4NOZKfq8Zcls4H1XoMCiVtjTlAvDpspFmNP8N4uqP0ezGStpi/6kkYWruxrk4fBgeJByjlKyYdp+Fb5Yn5AaTcsZZC+5YHgcEPYRFUtyw0U0Y3zAXIQS86fwOTBnY+qlJV7B0DdryezTUEFGZVQzHXM2/bRg7bnRrJhElywVOPodbFJb0pmBmDC4vzeg7ZA5BJva6RZN6D2nZw8GaaL7ajRCsnvvZH3adspj81ay0Kv5hknPuL59rhOf+NDvPSmkVwLRy6J41uCVwJKnkEffzEay7ONlimUou2RJnjGYA2e4jjSmDXsh/Trm+4Ng8TnQkITDS2ZvsN4lZb+1A4UxYuq0nCSTcOP5oahPxSWQbeqjL1xMRcydQoWejBmmSDm93I1Jvf3WSEJTlXiAFs2eBKwxMwvGLCSfaSFyFzgRnJMIzOUuYTJ1Vkh7yJNTvxK3kxtvUG7IUipmr+FgYypgRMxsgY9pRjog1E/o5DE3bMzpblA95ZZI8pKVUi2JFXKQOeOGyxPCR4ab14VgCt3+PCbNu4e1VYJYjinzd4kA2cA+9MmRHzg6yWiFtSNcumTTW+CyZ4MFxKWpxQ3Ik5IwQ3IOfkpYvahdzKggI3zA5yeNYipmWAi710dAkD2a56MBGTmW3wOWZ/DVhBdsL1PMMtoIk3p8AZcwYsjU9hznZsYtnBLMPd1D0ipdexXV2hJzD/O2mseFQ30by/EKN4OD0CZ+OORmfDpziWr9MhAkJBygk86qhDFhdSAvrrU4yBCjgV9TzYR2CWPRekUDmgGvOLLXjqhPIfyFKru5oVDCpIZAtKD6yIlVhQZkwUhVULAVuCAEQsOQhavKQbOMVQaSpV1cAp5pXnUakArLMdWaoasqo3W/QQ1WGpx6UTSERUbOumWNQ6Wk9jo6JsdBOqFt/WWUrEyCykJhzopR4Fmfk45JrUvM/uvUFnJMggqk3arcivXMGWRiNaiQI5h8FZfV4RrGDBK1p3hTKCrTFhXngpRSmyRrEayqlokWMhZe0uhjG7MeLRm3tP+YRddV1iw/lNEiAz+ls+4UdBnOKqCTO+lcxShQ4d2hE6NXGkcHLAu86suuKG38qctywlu1ATwmpRQ8ZuySZIjdXdBk/YrZjz4uzEhyw1hF6gqZFV5Ky1Y1qQq56oBpk45WZKKal9FikK5sdBr23LZzaqhmt9naPkmSpfYQB6aVyp9JYbcyGvlH7pkReWQlu2aG7LvjWDPz2PKzN5djCQqrPBBdjyP6cP0pZV4XTIOoa2y7VE6iZmBXsFaW14qlrzbFRQSaXviRReJPCMYuqsMWHu6KGG2oaQY+5bXaxNnTY99svclFVZtr/6OgQmqWyZiGLmuTPkD1G14UvPeZSrGMa1i3w97FfOlAN44TS6wEbLPeBEoEOK+BdPiZWZ1RMXIj5EKkVdcil5r+Xe+3NEAXeHfH0ZNYpXDnEJvYI1cJ74hqR263RTYMarkgfG8PvHnqj7JSvaD27MIKRK0gpi2aBH+mekYeVUzNaKWhDhHU55lwMWWqUlyYx3Y9FV24M8NIuwBwtBoZJpCzUgptlJ0+3JfAKsHNsseK76NA+/46/cvZyy925T1/aWcTQmQSdbaFc2+Jmhu+EQN9ssJtx++vmObO8CmfQxB1W7VbOBWsHfaXsKTn2Xi4+Spw7iqY2PrWaIotbRy+HcUxR1awMauH04KqcvR1KniAZNPIAXJ72+edOx3QZby2Mg9WJEpvUY0nk9Ha559UoeRWd+LlUv/eDBvxqto2pv6eLsAuFGoLygm4wVXgpg9ORVojS1YosULacyZnHxnK/Fxm10k8cs615ZQcz3twMIA6yajKZiyPDDuuDeGh2pOyBzmbe112dI261qhLyUtWkcMfyMGLk6NnJ4cHGEV89urHk4P//f3h0fE/XbKsthPAT8TMrMqPdwqF3x0O3aOHB+6PuDOlKomuM6tYTuoC1ZCqYrl/Af/VKvvT4cHQ/u+Q5Nr86Wh4ODwaHunK/Onw6EnTdyprk8nthWpY8eVArJJgjdqr0V5gLzEZ2pjiZtbNM7YxclJRyVe3ibYafNBJJ0dCVwd0QnlRK9Yrk8KIG8mmzWVSGHdz2YQ4N9ZOcX1zrZNNuWqbTgpJe82w77m+ITACFu3j0jJnU217xIbTIdGOcYmWBaCoH0dTzAfN3OUJHKtwfXFXPdTXZky1Q3AD7tdCqnID/ls5id23YLfhf7Achr1lQoNgWrMa+SRM4sCu5eHBQU8BuJJygQE4zrO5lDWsWYkRmlSAFdIVMYLLMtWaT4VOENLN+6MdYkExM1ozyz0iTgOp5nxHtCh8iaaW4qrZnCXRTPcS/HDpxmyZ7sKCepgtBeCXGUZbRT3Q38zjG24vlIwKkKxzppIbfNDZLWHBhWOl9G60EtWVV0ISgxzcpOkNI2BqdaA488mKQnNtwPyMtPTeutbu2n3eIqy9Knz2nQAvHLfeCpyVMr0XNCSZvR9Ea8+Ki4G91mwxOW03OWbj5SspsNqY0u6ujtaGtL4ocQe0c3M4nJuaa6EYzZdO7ORsQuvCkMultgpANGEk0uccDSaAKS0w42/BdWoKOY0COQBFkMAoJ2CdFFKAl+D8pQO+86pWsmL7p6U2TOW03Hmc7OHxWLE5Oi7845dXO4/BIyLIzz+flGVkbk4L/9TewdOTg4Odx629vK0Kie8ZsgscQU7TrtHrFubiKtLTuYS8zZCzEKuOQ/iH1U2HaYXiCXfKsfPV/eg/ry3rBzX1W34dopnpXlLAZabJ2EqFpoXVuZ7sr+CN9w4TMK+ArIwl+yw4VzvcK3RUa5nxWBoY1DRf069RaE4PrLTed5YbLzfQ4QMLatUTqZmrBo5OAwB57pVV8gYtfZas//Xj+Zv/9pXDdfRbucxfKP4Hjm3Udrxq0c3ZoJMJQ+uqfbw1H881Scl9Z4y6i5t7wxSZVTLwNfVF7wHFkhmKcbPgImmJr5zZ6W9JeL2EwVdkw2GadtFSTwB2N1bl/uQprHKA0tY5QkJIIReEUb20KBoGLDReIkHDyz2RG5U720N07dYi7i4Uh4LuGF9nRedP5y8fryZs5Llt45Jm9nbx4KITxXGPycUyZ83OFB4J7yJL5RRpGhy2lmBskUroYVGRmaFFqzplRzk6PnzWxPF+BYOzKIGGU8qcT3hbOMiF2FpCM54OFsAumExUN1uwomZbNtcLamZeqe3yqOZ/bELnVVHWMDU7hl1pSLsij4KhRNoLDc1zr7uN7FgQ/wau8tHjlnpJ1ZSZ6y2S4gogALFB49DLsuDiphX0vMUEfCAXGEvBpTQgOVegZDhMWhSptyZSr1woJ0jTDyBNVbx/J9FZjy5bohYZOQ2nmjKZKmg/uY9r9LOfmEyD9TKq7CUt1leh0STsc0/SUjJUpDpSs8FPkq7SUPScUpYzxYONzbBsBrb52DLAYnZ+kcTOoJNS7em6qgoevJUbKTdfT4beV5+d9xVm5n1lWXlffUbeQzbe15mN9zVm4n0FWXjdy4I/v8IXq0+wq5Dtk8QCl8yZWmPwOTzjgsqh8QIr2JyGzem0ssQN/CmlTb6qzKYvnc4UghakboR0/+w/rzUT+QI8DTORK8tPMllWtcHwYVctKnSUOrvEeFnfFqrfYJl2hIpmFez/FAsBNZMHfOw1qIWgpvQGDafhwnauQNcQH+xGnFGVL6hiAzLnytS08IWe9IC8hIogSbUdMEKRv9ZjpgQz0B4oZ3eqo6GyGTcsS5xa95osVflgOd/IIYHX2ecfXzy7ftYs1/BQNeGhasLdUXqomrA5zR70tIeqCduvmmDPzy1hsvuzGzutjpjGkZik1Z73uS6cW5qMPGYjqzuUdv8qZmqFpWA7xRZ312p199piD/WctIDTqQ509DFNrmEMJiEPwEXuvOlBf7UqLhdTiFBwAelri6iipuxCmtElaCk7gvZ8QKk2FT6tIgZoQLzqL2KwnUoWP7ul7Ie5Lf58u5Y3wZjm8t6BKxOOTDjxAxQHw2gPJyQh0uv3mhZgGg9jupJiWJUB0/AsAs46F7OXICsc1lrbk0SRnGU8hwRZq7sCG0XBLu3zrYWXejihJS+WWzqa3l0SHJ888rY+xfIZNQOSszGnYkAmirGxzgdkwUUuF9H9H6vowZMdvOtiW/U5Ojqvq48BWr73+fjsc5/Z26+C0szS4I38jc5ZewY3VuX/YnNAaAFtuHMpunDxQl3X0PB4eLB3eHi05/LC2thvUaFZQX8fvpxQfxXB/6ONrb82fymMPTzH91Y3knpA6nEtTL2O16la8A6v91ZX2B7ym/LI4cHw8Hh42MB2W8Euvh1oS/z+KJWrDO6rFbuetM7z0KjDboeApsajUGF5BIXk5+UgUYAh8jrRdcNlfZC2fE1qkKcej3hWhxH7zuyeWicPFYea3PVQceih4tBDxaGvu+LQzJiGFf/nq6sL+HyXHiX2pRAOO/T1YcioVsXIB6YyjKZOumoCkqrw+LqmuJvb8/0LY5kvhz0Vb28LyLi16u1lIz6jiSYBqG3yvnjxfDWKLphmS3v4yl1HcDHWYvkzKwpJFlIVeT+2W6DllTS0aEW8tCj6yCILm33GqNUDusrV4fGTfgKXzMzk1hL9GiRFUK0EaGRyTA2AcjFjluYMGEkKuWAKcr6tCPU1qIbkkrlEWZnVpY/zCmNrV7Jl59yH1Vst79XZ5U7XPDZlZkAqqB1T1aaXTNAiWm0tYOu9Gz6m1KSU66ymlT36ZH9/XMjp0H07zGS538JdV1Jo9sX3OYLddKOnSH7Znb4Oz9Vb3eP7pfe6w/bTNrtDWhtqat1j6t0U9dUpNk2aIqB+i+/xQdNNtt0rHuC16s58OEw7nfh6U+5Ef+0+3nqgo82JNsr8SMjtTDNzNjmZYfLbuEO+85lOFqvgBXGVwjrZi9hBoJH8vKBKjAZkBEXT7B+8J1GUKdWYzjYTbn0aWyOPy07GJ+DSdvEC2PrJE4lOPMEaTQU36H43pIYSMUFtrahq1EM8R7unorEc4cgN6xU35IrUQgpN8H0BGTtimqnn18KNkiaItvJD3WQHnQn5BOAw5ozOWcg90nZRMRY58/UUMcQQLQNMZBKbJSgi2IIUXDAN3eTmyS3F3m8KRgUkrjVR/tz8ZaKlS0/e3QU9wJ71qXF47C1goC18dhozuN/AUfFm6fZ+sKZjtkwqDd4mX91StM/n2jTjPNCeUpa1cPTHsGA5Z8pLkBhUQnAVkpwdF6eh0+5G/olPigrxo7eqdbSziHyhoLvEZVTYmWOLmSaneHWb8jkTGKGbQnUSrlLSyEwWzVJFVI25UVRF0z9xia0unwxKEmrcFCXPlPR5TAPgQFpoCcCWuPPjw/pmWbFoTuPZ7wMyoRkbS3kzIGbBjUGvBddkkVYksqImlomKRT7JnIk8qaYEIdPYTTGEF9sjNg/hxKFgAu6C/dwq3ucXGEOtB1BVXA9IMuaCK582+BWq5pQ3O8Hdd3+WXVS5UNUyigoNijisyFjafcMVc/XbGtn9I1eZCt50SfdpWXX/vS/0MyAjv1ndT3h28bgSui67BHjy7EWDAE6CmOX19jphnqIpC0p9QkYZCO2kkP35BVaadNxENVmwonBCLszHb78YrdCUf8OQik6JkbLYo1MhteGZ1R5FTlWj02YYdlLIRboYrxlVApPWqQlXoyk3s3oMlyLLIFBabT8Qb4/ne1ZX6ykPfDJ794/67fHP//jmp6dv/rb/Ynau/uPi9+z4P//tj4M/NZYisMYW1Judl35wr6d5cW0UnUx4NvxVvGd2Plh+KR6nJ78K8msgzq/kHwgXY1mL/FdByD8QWZvkExeGKUEL/GQ5KH6qBTDur+JX8cuMiXTMklZVUqDY9Y+1h9cettQrY3Koq1M7CAdSotikYwbJZYfZ1QTilezk55wthojDCsCeNFKRiileMsMUItJAejOcIiINDOy/4MpwwNKRA9DhTpudHO0bfDORakFVzvLrzwk+SFpyhDx1t12Tn5yCXCn5sadW1Q9Hw8Ph4bBZPIVTQa8xfGlLAub89O0pufDS4S2AIo/8zl0sFkOLw1Cq6T4ezFDbdt/Lkz1ErvvF8OPMlEWSRH/p5AicV76OiX9LO/lDC6hpARIMNJ63zPxYyAWWV4O/nMU2jFvIqb/11c5k2zenDsGbKYfbdougcjReEgleTig2Lv3pq2MImz+X2tj+BFa7X/iEN9D+vC4p7sB1g3zSkeve7Tl04y89x67/Mepn7gDuP3iPmkYKzzXbuMq+fu5vF/HMhJgKwj4O4UQbkAI46jeaWU3SEs2evVHD/fo0t+AfCe5xj/U2SHhpGZ7qwMuJEEOtHVypNBaCYOSvCCfdhqF5QKRwQZdWONV5NSAmqwaEV/NnezwrqwFhJhs+/voob7IW4bcUl3COh867y3NIwy7wEF2k8QOerV9bKg4t7Y6RgsktqdIsG5CKl0DQr4+cFunENOAq1TRaRrxLv1uX/yHC691aIRXLOC08Bw9CcizGwXWu1FhcIhTezZlhmRn48eElrC5y+4h7zfPNKVdJsddmxmuIEKEkq7WRZUj7wEGhBTl4u91UWzVPpJjwaR1bkRhJVC02JwDRcmIsuKQWWjMNZcIVW9Ci0AOr4aoaQnqQQlyK/UrBFGEoH5TodchES9RMaKlChasFGzewSIBAEHghtSZ9Q1tCnl68cdTQaZtVzw2pAYdiNegV9hsnoHBwDCMRy0FaKQ7nqQMraF/rBdlBR4V5DYl9hRU3pquzQt442+rvNatxYPLq6jUkLkkBXOPveq5UdLONiWMnb2lSDEyDUNAqZ9AfwNEDOsK+Oru8g9HpIdnmIdnm7ig9JNtsTrOHZJuHZJtvOtmmnWsTTt+m/ePTjDLdFqn9w3+xNqcNRfUh6+Eh6+Eh6+Eh6+H+sx40U5wW2zUY+/u1A+bO+9uKaN1fczDfbSAVq6Gpy7rC9ky5ZEd7MfSakzdEx5GWFdPDvqgb7ypQadsBf/GEKJxcwz+Vdi3CPi7hD1kUDMJ08BJr/4pX0J7YCD9mg6QN7/N9EjXMHCGkMevDFgbre6veA0slgiWGLU2p4H9EZd+bedrf3xIHko7j7/dMKJ7NkHHgYr+qd1lZUeFPaamcvtpgulakRhoYEnuTzlhRQVluqhQVU9+ux7jKt0nPHyowSAc8Bs2o/YBGnM9d6nT8HfJUUlS/WL2YlD+CehCleoOVggi+BBF8CztdgZ211S5gBevIlnTfPPrwm9QMv3G18BvWCb8hhfAb1ga/elUw8ZCGZh5Oyl0kX23cTHulcAtdf/tPuoyKeNrFHDxnc272voPAxtBEmOf7CS+7oJJGXC0IYN+BdVhBLt7EMEG0oUvt6x/77r7YjZuG/lmgIFYcHTWQqVjIMS2SSvQe3WhQ2qz+1XSTDIRPiwFTii5duAQQiaopONJSO9kb6DPp9AmcXqWkYZkB5wk3fN5Iguzone7jHtEhRXOP7BXhz1qHO8Ue8e1/mlEU7CPLauiCsCVSnI6hOwzDcF23gp4qEXpnh+zXWu2Pudj3c/sSdSvdjnOnUFgoe7WANhMko0XBIGV8qmgZEiA1L3lBezoBt5Gvbs0SvVPWyEXYgt3D5+i4GZhUdWB/ftbKBYVCMW45d+30+hBpXXk/s5HKle+ymnKSa5jSdQUcHRw+2zt4unf05OrgxcnB05Mnx8MXT5/8Z6vTxkwxmm+WEn4nCl3BwOT85e0LBFJ/25wNQFrxLpaG8P0AsxyQ1cFP6uJCqnRfkDMqMIx7HPtsmpMwZFLqgFAyVnKhwfbgk0McEl4WLNiYVHTKkk6qErvZN5doIdUNF9NrjG/qNM++1zQ3B4sEWN58EY7QtrSayZLt0wIbVsTEsRgY4M7098lXa8/02FqHYR90X610QjNecGMP54rPJbYjVrKGXvoVZ1nSwQq6s/jFBgMJPKDbbVVcOLxmDJqwl1QsrRKWQWiAvdq+Orv0XZ2uUhTc0NgsD2w4eIMsB3g1hswCfxZC0yoLwpepks4xBee3rqTI4y5y6S+CjBwVh6Mwk1No/KuYCQYfS6HoQmB6kOQPjRmpocgRtNkP1pOBi/ccRCbwkXADkhUc2oL5R6nIQ3BUGoAKRUDAPlBV0FO2KMj5hVcrjIzY82o0QN2KgrojHNFcZQOMNjy/IEbxOadFsRwQIUlJjYEEFxaOCW4AGFUsH5DxMgTtpKBO6HA8zIb56C5mhk1acPQ7b06LkA93fqFxjaVIGlGnN/lu/M/lZtE/7rmevCDHPK42RAhGyaQQLlJpEgxxLpxCsSlVOcapaI3txePzGtuk8xBLadVNDGXNpEoaFf8oFbk6uwh9gUBoBjQRt4xx+9kRiAsOhSYu//bWhXE+0r5gv9fLzy4SXIYABOvFhODbNiRXA7dYdujhl68ZAy+074cIUsEF2xCamdo7bTGSj6mS7ITxdrBc8iSolSkWooW49hXG4Gd3zfC+5W5GlRclrlhshoJNt0Ck83AC6bIBgEIvK5iFGzGGAmGxj99qkcV7DO5093bfYJG0sRBIHNLuXlzGPXTY+5xV9+QZDr/vp9Dsq4LXLppbKV9SYXjmg+tdVhb7iK2RnDyLNyJ7VZvUhX1szu10+R8sMW8KkjEFF8GYGOVllQowJrQovKzyHf0zathUqiUKK5cQpw0vCsIENNSDx1aktliCTbjVkd2wtKqUrBSnhhXLu1zOUJJvSx1CZwG22sOFCUcHJlV6AVOO+bSWtS6WyM3wTlB1FpYsOtwOwDVBrRgfEOqL8WHhGijhJy2fDAn5W6SsK+KY1ifBXaXoIqYhIN+Phu4LlyPbVOOEPRliAmNeYzga3itH9vyBAjhDRGs0IDmzRxakrPri1rFZIJwzvN1c8r7zx/4CiWNQej2m3jmvjustDfunaz950Ywvx0ndgtknFbpBbHD8Vtuqh5C5h5C5h5C5h5C5h5C5bzpk7hMj1na7IWs+YC1yFl4/W/5gcn4xP7ZfnF/Mn0XFo3XWfrFIt74wu8/LUrtw6WmfcrC3jJa3JzzdzWApoWzIynk/1NN8qKf5UE+TPNTT/NbqabrCJvBcYlbzX90SauXLorSNNCb9TaqeFkdWQXLILagmmSwK6EF9SzjVhIvclZjy3AlZ4ciWoQ6Yh22f9BELm9sQWDVjJVO02GKxj1ceRiqepNMKPfqP+AR0AGhLrh+3Kz3xPOlSAeYeTWimpNZEMXBsudo5Izcg7L5cQs8n09UHX9DjydODg0lTy9nGdtrtimZfcK8WAq2riHF3ys5UgTuwCE1Mlw3SuSIDJb1hmnBDKqk1H6PzKLBOGBpYKEm8RJ4VrMNQfZ0vvCFf2XWqmOJMZOCw0rpmGo2FdizFcjsB12Is2vTRjR/G9c3qeY5lA2IoBdzDPLOjMY2LKTRfdm3LOiuaP3nOnrLxhB1Q9iw7/uH5UT5mP0wODp8f08NnT56Pxy+Ojp9PbiuQcP89LTyHx0het/97gnnTq1V4EcJ7He/DaQSOkFBbopALDZeshQzkiXcsPxb0uPCiQkXm84qB/T3UcsdroGg4L3mjPoVrkhF2GxxvaS+WAkutOfTsMubc6pzj2s7c17vCtVU1+ELCiTOT2uh+9kXTvTdVu8kSLAnjptIKTHA55JDALSfkVUG14ZlzLCVkhim4zGN/TKMSXmvDVOOqhE6NvzBqdHcIri11cjahdWGgIlEVfKOBXgbaRoNEDmPyCRGS+DFCQ5KeIojpHPbSlNckfsBsxULj2t7A+C0+/fsEy99pd8GL3t/p0tpRP+45ZxtC0p7oICUThcHPZIWkhEFiSjLsuiZ2TWYctLgjDBrqHYwaC99XHTP9vbEc2wtz3/13H57aXJDgaGnoPN1ViTIMai3IG0LtrsHQcWaw43pL55lHkDSwX7ew2fBomNZVQH9MQ/2L36zR/vCp271z3uEDWKF1YL9Z97Q5UuKGu8UBl7qPnBfuq3QTOYfXg5voK3ET4Xo4a1JaxqhjUvpiviJE6cFX9OAruh+UHnxFm9PswVf04Cv6pnxFWI3vW/MVOazJtn1Fm5/uX9Bh1DP5B4fRg8PowWFEHhxG35rDqFYosZy14MP71/Bxtangw/vX/nLvOmYSXVdQ5RNz8CwgA+hUVMFafnj/2hXwc0+GwPgZI2PFKCZZyIUgXBhJdDZjVrjgDWoAKWPufUm87N/ELNB3xbu/TfPS3dgduVUxCA0EdhaLxdBZqoaZ3GnaaiG7JqNgPQB6lnSJ4dQu3NeqCVhtEOiK4efFMqbu0ubUiMvIATsw9GjQbODi8GN9a1BZpzJ0WnFXe2cd6KiIzSk06DpRdFpur8PUrj1tE3NbrQpCJ8ZVCxl9P0oIbWS107KAjr4f+X4prj0MauEO6ZbM2GLm+/kEj0rL/2An4qVdT5fAAyHYtWZxtZaJQQYrSoR5cQHtDOGEHw3IYsYgEcA0OsQolkmhjarBCmm5B2PMvUWoaY1K1ZiermjN5T85Pn6yjzbXf/n9Tw0b7PdGNivl9vcrus/DCvvvwBxdyyJgER0yl8Jsu/r1W2lc7DoXPfVKB2l5mjzsTqjT6hdzgIk4VKfLQzNIjSvk1N367Ktcuwzn32ptYtC/r1ZrBdvKfj8h0yu8Foal4ARdUB0QHTQEb687+JMW1o624ueW8q91spL3veYXbvjeZp0RB7MtBekCegw1YCcyyBFoZ3jLFeQeEm2Ta0gHj+PjJ93s0uMnDaQgS2xbG9MKXwDgmDhYOABf/AXn1juHsA8sTVvM1pHx/wIynn2EgsVJu4kUCmS64Akben8Jad+FHZqY0LG6VII7vGp85SkK8Ma1CU8NEmA4WQzqCCOGrk9lZSI+gDo+OXJvt1x1DV80GTOzYCwe85CLtZCoPLQOMtSatrW2lzD66j0A0mWnJWcxi3Z00nseI74r5FRHgd7yrTaNSUiES4pBQ03WtycqXjkdvONU6y84BI/iuQTNjdmchsPaaWxNR9uPScEOOkeLEQN7cXpRsd9wpt1W8Bc8bPRjZlTAazz32a9epQ/5uu6khG0GXkxHpfIuAVh/R7vIN2QS+QasIX9vQ8iDDeRWG8hXZ/74ai0fmqlrOvVXokSyk/jtBvIdx/BSPkZw2ku+q4Lki1+Ek8Uhd2XvfK4E0kwuXLvUBRuHCBMIsEnqYmL1CaqstlAHVL1+sblIxr4XX2onO2jtJeEXMx9C8KW6OSUcgqTrIHVJJ1TxL3mh/SDcgs6bUUaRuXq8+X/woqD7T4cH5BGS8Z/I2cUHR1Ly7pIcHl0fYkNNX8vtMTmtqoL9wsZ/5Wb/2cHT4eHw8GkQJ4/++vPVm9cDfOcnlt3Ix8TFPe0fHg0PyBs55gXbP3z66vD4haPT/rODdinbh+LYvVg/FMd+KI79eRj/jy2OvV1U/70rdVccDVYKfrdngZyQMYNWQVRkM6nw414myxLQdLrEX/CZBrQ/w6Bn3hyBr8DrIWTSXx5AuSxcKRFX3vq7FfGPgG+r6UMfSdZ2cnCzboxsMRsaXrI/YrQfDkwLHiygFTWzE3c/bT1c8qmiCM+omjVHx7k0hpXj31gW2nfDh+tbZ/LncIoFysI6+i5ZQE4XVdrEADrxNxCIitNKIK/sS61Sm1CmJs+5KxNkdXeIc3Ux+QAnFAxL15D0R5SvWsE1aEXUkpDtxkJ2uKO7iJaJ0ufWrh8M2st23YF7eXTt6BAmy8B84fMgNmXtK465IJzFHB17NXK7NytknceNemY/etsHRLNTl9DWQ+k37lfUx7PGq9qyAMt96gjN82t44NoP6SvHSZVu5cas4YVhpaRl/WgOCFLI/bL3cT2Ppuque8Xy409STguGM0Zu7AHOSzplPaBpyffoOMsPj570itII/dyOQM5fBhsD0imkNuGUvyenlk0wP6vIU3EQQpqYocNAEiDyLXzW+/BaPktgeARjquB6MGFC4fk7Q9pg67Rgbbp/Emgu7ek6ETDrgbkXhskLm8JyBxgvuFleb3BsrH9rU6iOxzdduM7+2hQOxiFuBKPxaO/4Xh7lMrsBXnUC6aX/3LO98DdIT2onnbjf7L7WM6nMNZ5/J2RCC80SdQXh7QVhtEKtCGiR3tNx1SnmTsQ0FqefWAnB+l/pJdoKUFbi3B0aSDqRNq+9E9TWm5sB/XRwBR2zQlvBefXu5TurwS2IkaSklRWymv1LB5eGOkXWq1RkvWqBMh1RGHrOted55Nuf8VPPIOdWH0q41R0L9nWfkzlMGBS64Pexpzs3Xp1dpilGPOQMsUwPl2UxdM9h2jlVLlBbir34Zsu0jKiv5/TVS9Ow//ohxlIWjIoNyTuJFAGfY1z2Llyph+OaF12Q3RUNp/fO4YuXhwc/7GyGzrtLAhCabWX6EMlkznr3wTpctFHMZLPNkfFQfLfWwIE39ZgpwTBnyPHhX9PvesaNvwdlr6m5xUFJyoXrpWp86VbJ2kB6Pc+1KV7JvF/s3GkzJxSopGvf3Quq7pHhnwrpQubkw/nLLiDIbahodn+TiiN2gcm8I/I/E5g31nWBobi8XSxvBsjJ/5JWXUjgHMISn/cFLhmyH6ZikDaomblfgsZxV5A1Z1UhlxDOd6+A47grAENW+KQu7n3KycArQN+idXwq4DDsrWD7VazPh4vjOnEeG6B02p/0jOvr2QcpHq6QfVI3ba5yF5HLPm6q5PnC8J1+GqRH0XMz/k0W8obTPVobmXOdyXl6FfhX/JW8dL8sSfocSe65t9oqeoZKzzyHRxhylbHRPTdEg07TOHsHS523u2JeHJGTgEBife2HyddZfVfZ7Gg2c97SGRihgw+7WSOecV9i2xIhJ3mNbewNVaauGqZSUDulKjG1MNgawV9fUUVLZhgUSxozsA7adYO28gzjy/AL+xHDyXgOqGk2h0pCFVVGYwjV+cWApG0ueD6AGAXwEjVQoiLH3gpgAewjoat3VymZ15m5OyGvXB4v7l03jFXKwtzWgf1kdmmA3dXBofAogfz4FtBJI8Y7QnYtFpM0Zpx+wgs61JtpZ317PHyuxZ2hf3j/mszsVQ8qSQA4x62AyTqiZ7Vq+Uial5IVUH8JAeZ+fljiAlncXeBobWZMGI5Jpj7w2Iu1Qk6jFHstp2TCC0QYgyvWuUUK/3jBRdPr0ZhmIadD+9gwCf3tI61iv9dcsTyq7LcQHGC3SpxZVIAK0CImDdYOwaVArb5GR4BkgPBj7IxyQkb7c6r2Czndd40GCzkdDbvzdIHszUoXnz3Zy0Y5i86U5dR5n/y8oYK9y9DsQVJOJpo1ZUoS3/xpy4BjusjMSioDzV8FQ5GsCW27rOzVkpb3RSHLuTgiWcyYACrYTR5lAIb8uw25q00ua7Nrd4P9mym120SPi6o2qV01ogNawa1UgQGwzk9rveJaYbcCOGiaWQVASmRKaL8SazAFGN42M8ICThLT511WBQLXrp+Gk4c/8oKBCxMFhGP3xqzRN6jZ7zVru3E+h0P8gIkWAUIyDZUNfLHUIDCwa83y/rjUDUjYR6NoNMbigc2l4mbZj4r/9d5Q8QOGGGuAs44aoGxws7yGq+V9ytBZXVLcLuB99YDWL8rW0fCAWmiEpoTY0e0+ERBNR5wdvkdyTgo67S16kw7Wf+LAqx5C31LPjKmG2LxFs6E7ga8LJqatU7PHG9x4dSzz5TAtz7PWadIKsLz9whVtimHO3VdW39Laod7dFWygZ4VU6xZ/R7UsiD03FAbLB9G7WhJ50KXM62KzqIrGo2vJbln9GlznhpbVRoNniiWVD9eOji6hITVG3WvcRjpuZG9n77KXGibmXEkBppk5VdzuZk0WihvDBFQjhBF2NfnXy3dvYW3sxppCarHi8xgS3CpGSxXDAmwx8AYOekjIxIJ2Qet1h2BzXHdCdiJLkLSxhsqdLzs5C2dx2ke1pU5kZfWJUM7P3ly4EizdITue4s2H7Ik+4dNPH/Kn/iG9QqyHePavOkRXDHtaGylkKWsdIgJhmBaUtObIlkHFCTX6rn+afPLhH43u+a1e93hfaSO3ngBfE2o3dHJDNzxbkqgiWfFs5RERv19nxbSA3UCd8aHqgYk2jl5O2QhAGMnxS4/3Lb1vfTogd8XyyrMTNF1wN2x5H4S7YctBs9mjv5Lg7+5iAhnwsSv0CpzGhcxuOuddxC9PS+tuQovaMEUeZbKsoMZS/hhBkAiig8OM0ZyprtsFUug2A36aNMhERHBQFyfrq0TH1Rl4yiQhyPjfzj/fsOWfT8g/Ax3/vDP87v8HAAD//7D/XGA=" } diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 7e2ea6acb0a..ddc505bf4c7 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -42,6 +42,7 @@ type kafkaInputConfig struct { InitialOffset initialOffset `config:"initial_offset"` ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` + WaitClose time.Duration `config:"wait_close" validate:"min=0"` MaxWaitTime time.Duration `config:"max_wait_time"` IsolationLevel isolationLevel `config:"isolation_level"` Fetch kafkaFetch `config:"fetch"` @@ -109,6 +110,7 @@ func defaultConfig() kafkaInputConfig { ClientID: "filebeat", ConnectBackoff: 30 * time.Second, ConsumeBackoff: 2 * time.Second, + WaitClose: 2 * time.Second, MaxWaitTime: 250 * time.Millisecond, IsolationLevel: isolationLevelReadUncommitted, Fetch: kafkaFetch{ diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 66ac66345ee..1cea74d32db 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -62,6 +62,11 @@ func NewInput( inputContext input.Context, ) (input.Input, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrap(err, "reading kafka input config") + } + out, err := connector.ConnectWith(cfg, beat.ClientConfig{ Processing: beat.ProcessingConfig{ DynamicFields: inputContext.DynamicFields, @@ -73,17 +78,13 @@ func NewInput( } } }, - CloseRef: doneChannelContext(inputContext.Done), + CloseRef: doneChannelContext(inputContext.Done), + WaitClose: config.WaitClose, }) if err != nil { return nil, err } - config := defaultConfig() - if err := cfg.Unpack(&config); err != nil { - return nil, errors.Wrap(err, "reading kafka input config") - } - saramaConfig, err := newSaramaConfig(config) if err != nil { return nil, errors.Wrap(err, "initializing Sarama config") From 135da3145097b0bc3b4bbf3993422fc6f807c890 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 12 Aug 2019 15:39:17 -0400 Subject: [PATCH 48/50] Update header handling in integration test --- .../input/kafka/kafka_integration_test.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index f303bab626b..4d4687d963a 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build integration +// + build integration package kafka @@ -24,6 +24,7 @@ import ( "math/rand" "os" "strconv" + "strings" "sync" "testing" "time" @@ -121,7 +122,7 @@ func TestInput(t *testing.T) { } // Setup the input config - config, _ := common.MustNewConfigFrom(common.MapStr{ + config := common.MustNewConfigFrom(common.MapStr{ "hosts": getTestKafkaHost(), "topics": []string{testTopic}, "group_id": "filebeat", @@ -194,20 +195,21 @@ func checkMatchingHeaders( t.Error(err) return } - headerArray, ok := headers.([]interface{}) + headerArray, ok := headers.([]string) if !ok { - t.Error("event.Fields.kafka.headers isn't a []interface{}") + t.Error("event.Fields.kafka.headers isn't a []string") return } assert.Equal(t, len(expected), len(headerArray)) for i := 0; i < len(expected); i++ { - headerMap, ok := headerArray[i].(common.MapStr) - if !ok { - t.Errorf("event.Fields.kafka.headers[%v] isn't a MapStr", i) + splitIndex := strings.Index(headerArray[i], ": ") + if splitIndex == -1 { + t.Errorf( + "event.Fields.kafka.headers[%v] doesn't have form 'key: value'", i) continue } - key, _ := headerMap.GetValue("key") - value, _ := headerMap.GetValue("value") + key := headerArray[i][:splitIndex] + value := headerArray[i][splitIndex+2:] assert.Equal(t, string(expected[i].Key), key) assert.Equal(t, string(expected[i].Value), value) } From 5f64da9c78d5f1854a34d9ad93b9307439cd8e41 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 15 Aug 2019 11:22:34 -0400 Subject: [PATCH 49/50] Fix backoff handling again... --- filebeat/input/kafka/input.go | 19 +++++++------------ .../input/kafka/kafka_integration_test.go | 9 +++++---- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 1cea74d32db..3598241ec53 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -101,10 +101,7 @@ func NewInput( return input, nil } -func (input *kafkaInput) runConsumerGroup() { - // Sarama uses standard go contexts to control cancellation, so we need - // to wrap our input context channel in that interface. - context := doneChannelContext(input.context.Done) +func (input *kafkaInput) runConsumerGroup(context context.Context) { handler := &groupHandler{ version: input.config.Version, outlet: input.outlet, @@ -142,6 +139,9 @@ func (input *kafkaInput) runConsumerGroup() { func (input *kafkaInput) Run() { input.runOnce.Do(func() { go func() { + // Sarama uses standard go contexts to control cancellation, so we need + // to wrap our input context channel in that interface. + context := doneChannelContext(input.context.Done) // If the consumer fails to connect, we use exponential backoff with // jitter up to 8 * the initial backoff interval. @@ -149,20 +149,15 @@ func (input *kafkaInput) Run() { input.context.Done, input.config.ConnectBackoff, 8*input.config.ConnectBackoff) - for { + + for context.Err() == nil { // Try to start the consumer group event loop: create a consumer // group client (wbich connects to the kafka cluster) and call // Consume (which starts an asynchronous consumer). - input.runConsumerGroup() + input.runConsumerGroup(context) // If runConsumerGroup returns, we wait for the backoff interval. backoff.Wait() - - // Check the Done channel before we try again. - select { - case <-input.context.Done: - return - } } }() }) diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index 4d4687d963a..71e44cf6219 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// + build integration +// +build integration package kafka @@ -123,9 +123,10 @@ func TestInput(t *testing.T) { // Setup the input config config := common.MustNewConfigFrom(common.MapStr{ - "hosts": getTestKafkaHost(), - "topics": []string{testTopic}, - "group_id": "filebeat", + "hosts": getTestKafkaHost(), + "topics": []string{testTopic}, + "group_id": "filebeat", + "wait_close": 0, }) // Route input events through our capturer instead of sending through ES. From 419ddab0ca95685c157d809a53294c8c408d3eb2 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 15 Aug 2019 11:38:18 -0400 Subject: [PATCH 50/50] Adjust what the connection backoff responds to --- filebeat/input/kafka/input.go | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 3598241ec53..98b7f15c1f0 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -101,21 +101,14 @@ func NewInput( return input, nil } -func (input *kafkaInput) runConsumerGroup(context context.Context) { +func (input *kafkaInput) runConsumerGroup( + context context.Context, consumerGroup sarama.ConsumerGroup, +) { handler := &groupHandler{ version: input.config.Version, outlet: input.outlet, } - // Create a consumer group and make sure it's closed before we return. - consumerGroup, err := - sarama.NewConsumerGroup( - input.config.Hosts, input.config.GroupID, input.saramaConfig) - if err != nil { - input.log.Errorw( - "Error initializing kafka consumer group", "error", err) - return - } input.saramaWaitGroup.Add(1) defer func() { consumerGroup.Close() @@ -129,7 +122,7 @@ func (input *kafkaInput) runConsumerGroup(context context.Context) { } }() - err = consumerGroup.Consume(context, input.config.Topics, handler) + err := consumerGroup.Consume(context, input.config.Topics, handler) if err != nil { input.log.Errorw("Kafka consume error", "error", err) } @@ -151,13 +144,24 @@ func (input *kafkaInput) Run() { 8*input.config.ConnectBackoff) for context.Err() == nil { - // Try to start the consumer group event loop: create a consumer - // group client (wbich connects to the kafka cluster) and call - // Consume (which starts an asynchronous consumer). - input.runConsumerGroup(context) - - // If runConsumerGroup returns, we wait for the backoff interval. - backoff.Wait() + // Connect to Kafka with a new consumer group. + consumerGroup, err := sarama.NewConsumerGroup( + input.config.Hosts, input.config.GroupID, input.saramaConfig) + if err != nil { + input.log.Errorw( + "Error initializing kafka consumer group", "error", err) + backoff.Wait() + continue + } + // We've successfully connected, reset the backoff timer. + backoff.Reset() + + // We have a connected consumer group now, try to start the main event + // loop by calling Consume (which starts an asynchronous consumer). + // In an ideal run, this function never returns until shutdown; if it + // does, it means the errors have been logged and the consumer group + // has been closed, so we try creating a new one in the next iteration. + input.runConsumerGroup(context, consumerGroup) } }() })