Fix kafka plugin and rename to kafka_consumer

fixes #371

plugins/kafka_consumer/kafka_consumer.go

@@ -1,36 +1,32 @@
 package kafka_consumer
 
 import (
-	"os"
-	"os/signal"
+	"fmt"
+	"log"
 	"time"
 
-	"github.com/Shopify/sarama"
 	"github.com/influxdb/influxdb/models"
 	"github.com/influxdb/telegraf/plugins"
 	"github.com/wvanbergen/kafka/consumergroup"
 )
 
 type Kafka struct {
-	ConsumerGroupName string
-	Topic             string
-	ZookeeperPeers    []string
-	Consumer          *consumergroup.ConsumerGroup
-	BatchSize         int
+	ConsumerGroup            string
+	Topics                   []string
+	ZookeeperPeers           []string
+	Consumer                 *consumergroup.ConsumerGroup
+	MaxMessagesPerCollection int
 }
 
 var sampleConfig = `
   # topic to consume
-  topic = "topic_with_metrics"
-
-  # the name of the consumer group
-  consumerGroupName = "telegraf_metrics_consumers"
-
+  topics = ["telegraf"]
   # an array of Zookeeper connection strings
-  zookeeperPeers = ["localhost:2181"]
-
-  # Batch size of points sent to InfluxDB
-  batchSize = 1000
+  zookeeper_peers = ["localhost:2181"]
+  # the name of the consumer group
+  consumer_group = "telegraf_metrics_consumers"
+  # Maximum number of messages to consume per collection interval
+  max_messages_per_collection = 100000
 `
 
 func (k *Kafka) SampleConfig() string {
@@ -50,115 +46,55 @@ type Metric struct {
 
 func (k *Kafka) Gather(acc plugins.Accumulator) error {
 	var consumerErr error
-	metricQueue := make(chan []byte, 200)
 
-	if k.Consumer == nil {
+	if k.Consumer == nil || k.Consumer.Closed() {
 		k.Consumer, consumerErr = consumergroup.JoinConsumerGroup(
-			k.ConsumerGroupName,
-			[]string{k.Topic},
+			k.ConsumerGroup,
+			k.Topics,
 			k.ZookeeperPeers,
 			nil,
 		)
 
 		if consumerErr != nil {
 			return consumerErr
 		}
-
-		c := make(chan os.Signal, 1)
-		halt := make(chan bool, 1)
-		signal.Notify(c, os.Interrupt)
-		go func() {
-			<-c
-			halt <- true
-			emitMetrics(k, acc, metricQueue)
-			k.Consumer.Close()
-		}()
-
-		go readFromKafka(k.Consumer.Messages(),
-			metricQueue,
-			k.BatchSize,
-			k.Consumer.CommitUpto,
-			halt)
 	}
 
-	return emitMetrics(k, acc, metricQueue)
-}
-
-func emitMetrics(k *Kafka, acc plugins.Accumulator, metricConsumer <-chan []byte) error {
-	timeout := time.After(1 * time.Second)
-
+	kafkaMsgChan := k.Consumer.Messages()
+	msgCount := 0
 	for {
+		// If we don't receive a message for 100ms, we assume we can exit
+		// unread messages will be gotten on the next call to Gather()
+		msgTimeout := time.NewTimer(100 * time.Millisecond)
 		select {
-		case batch := <-metricConsumer:
-			var points []models.Point
-			var err error
-			if points, err = models.ParsePoints(batch); err != nil {
+		case msg := <-kafkaMsgChan:
+			msgCount++
+
+			fmt.Println(string(msg.Value))
+			points, err := models.ParsePoints(msg.Value)
+			if err != nil {
 				return err
 			}
-
 			for _, point := range points {
 				acc.AddFields(point.Name(), point.Fields(), point.Tags(), point.Time())
 			}
-		case <-timeout:
-			return nil
-		}
-	}
-}
-
-const millisecond = 1000000 * time.Nanosecond
-
-type ack func(*sarama.ConsumerMessage) error
-
-func readFromKafka(
-	kafkaMsgs <-chan *sarama.ConsumerMessage,
-	metricProducer chan<- []byte,
-	maxBatchSize int,
-	ackMsg ack,
-	halt <-chan bool,
-) {
-	batch := make([]byte, 0)
-	currentBatchSize := 0
-	timeout := time.After(500 * millisecond)
-	var msg *sarama.ConsumerMessage
-
-	for {
-		select {
-		case msg = <-kafkaMsgs:
-			if currentBatchSize != 0 {
-				batch = append(batch, '\n')
-			}
-
-			batch = append(batch, msg.Value...)
-			currentBatchSize++
 
-			if currentBatchSize == maxBatchSize {
-				metricProducer <- batch
-				currentBatchSize = 0
-				batch = make([]byte, 0)
-				ackMsg(msg)
+			k.Consumer.CommitUpto(msg)
+			if msgCount >= k.MaxMessagesPerCollection {
+				log.Printf("kafka_consumer plugin hit the max_messages_per_collection"+
+					" limit (%v), you may want to increase this number.",
+					k.MaxMessagesPerCollection)
+				return nil
 			}
-		case <-timeout:
-			if currentBatchSize != 0 {
-				metricProducer <- batch
-				currentBatchSize = 0
-				batch = make([]byte, 0)
-				ackMsg(msg)
-			}
-
-			timeout = time.After(500 * millisecond)
-		case <-halt:
-			if currentBatchSize != 0 {
-				metricProducer <- batch
-				ackMsg(msg)
-			}
-
-			return
+		case <-msgTimeout.C:
+			return nil
 		}
 	}
+	return nil
 }
 
 func init() {
-	plugins.Add("kafka", func() plugins.Plugin {
+	plugins.Add("kafka_consumer", func() plugins.Plugin {
 		return &Kafka{}
 	})
 }