Merge branch 'develop' into stable

Makefile

@@ -1,20 +1,26 @@
-.PHONY: go-check all log-courier gem test doc clean
+.PHONY: prepare all log-courier gem test doc profile benchmark jrprofile jrbenchmark clean
 
 MAKEFILE := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
 GOPATH := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE))))
 export GOPATH := $(GOPATH)
 
 TAGS :=
 BINS := bin/log-courier bin/lc-tlscert bin/lc-admin
+GOTESTS := log-courier lc-tlscert lc-admin lc-lib/...
 TESTS := spec/courier_spec.rb spec/tcp_spec.rb spec/gem_spec.rb spec/multiline_spec.rb
 
+ifneq (,$(findstring curvekey,$(MAKECMDGOALS)))
+with := zmq4
+endif
+
 ifeq ($(with),zmq3)
 TAGS := $(TAGS) zmq zmq_3_x
 TESTS := $(TESTS) spec/plainzmq_spec.rb
 endif
 ifeq ($(with),zmq4)
 TAGS := $(TAGS) zmq zmq_4_x
 BINS := $(BINS) bin/lc-curvekey
+GOTESTS := $(GOTESTS) lc-curvekey
 TESTS := $(TESTS) spec/plainzmq_spec.rb spec/zmq_spec.rb
 endif
 
@@ -34,8 +40,16 @@ gem:
 	gem build log-courier.gemspec
 
 test: all vendor/bundle/.GemfileModT
+	go get -d -tags "$(TAGS)" $(GOTESTS)
+	go test -tags "$(TAGS)" $(GOTESTS)
 	bundle exec rspec $(TESTS)
 
+selfsigned: | bin/lc-tlscert
+	bin/lc-tlscert
+
+curvekey: | bin/lc-curvekey
+	bin/lc-curvekey
+
 doc:
 	@npm --version >/dev/null || (echo "'npm' not found. You need to install node.js.")
 	@npm install doctoc >/dev/null || (echo "Failed to perform local install of doctoc.")
@@ -70,12 +84,13 @@ ifneq ($(implyclean),yes)
 	rm -f log-courier-*.gem
 endif
 
-go-check:
-	@go version >/dev/null || (echo "Go not found. You need to install Go: http://golang.org/doc/install"; false)
-	@go version | grep -q 'go version go1.[123]' || (echo "Go version 1.2 or 1.3 required, you have a version of Go that is not supported."; false)
+prepare:
+	@go version >/dev/null || (echo "Go not found. You need to install Go version 1.2 or 1.3: http://golang.org/doc/install"; false)
+	@go version | grep -q 'go version go1.[23]' || (echo "Go version 1.2 or 1.3 required, you have a version of Go that is not supported."; false)
 	@echo "GOPATH: $${GOPATH}"
+	build/fix_version
 
-bin/%: FORCE | go-check
+bin/%: FORCE | prepare
 	go get -d -tags "$(TAGS)" $*
 	go install -tags "$(TAGS)" $*
 

README.md

@@ -1,10 +1,10 @@
 # Log Courier [![Build Status](https://travis-ci.org/driskell/log-courier.svg?branch=develop)](https://travis-ci.org/driskell/log-courier)
 
-Log Courier is a tool created to transmit log files speedily and securely to
+Log Courier is a tool created to ship log files speedily and securely to
 remote [Logstash](http://logstash.net) instances for processing whilst using
 small amounts of local resources. The project is an enhanced fork of
 [Logstash Forwarder](https://github.com/elasticsearch/logstash-forwarder) 0.3.1
-with many enhancements and behavioural improvements.
+with many fixes and behavioural improvements.
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
@@ -16,7 +16,6 @@ with many enhancements and behavioural improvements.
   - [Building](#building)
   - [Logstash Integration](#logstash-integration)
   - [Generating Certificates and Keys](#generating-certificates-and-keys)
-- [Command Line Options](#command-line-options)
 - [Documentation](#documentation)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -25,19 +24,23 @@ with many enhancements and behavioural improvements.
 
 Log Courier implements the following features:
 
-* Tail log files, following rotations and resuming at the last offset on
-restart
-* Read from standard input for lightweight shipping of a program's output
-* Extra event fields, arrays and hashes on a per file basis
-* Fast and secure transmission of logs using TLS with both server and client
-certificate verification
-* Secure transmission of logs via CurveZMQ to multiple receivers simultaneously
-(optional, requires ZeroMQ 4+)
-* Plaintext transmission over plain ZMQ and TCP when security is not required
-* Multiline codec to combine multiple lines into single events prior to shipping
-* Load multiple configuration files from a directory for ease of use with
-configuration management
-* Reload the configuration without restarting
+* Follow active log files
+* Follow rotations
+* Follow standard input stream
+* Suspend tailing after periods of inactivity
+* Set [extra fields](docs/Configuration.md#fields), supporting hashes and arrays
+(`tags: ['one','two']`)
+* [Reload configuration](docs/Configuration.md#reloading) without restarting
+* Secure TLS shipping transport with server certificate verification
+* TLS client certificate verification
+* Secure CurveZMQ shipping transport to load balance across multiple Logstash
+instances (optional, requires ZeroMQ 4+)
+* Plaintext TCP shipping transport for configuration simplicity in local networks
+* Plaintext ZMQ shipping transport
+* [Administration utility](docs/AdministrationUtility.md) to monitor the
+shipping speed and status
+* [Multiline](docs/codecs/Multiline.md) codec
+* [Filter](docs/codecs/Filter.md) codec
 
 Log Courier integrates with Logstash using an event receiver ruby gem. An event
 sender ruby gem is also available to allow fast and secure transmission between
@@ -47,10 +50,23 @@ two Logstash instances.
 
 ### Build Requirements
 
-1. The [go](http://golang.org/doc/install) compiler tools (1.2 or 1.3)
-1. [Logstash](http://logstash.net) 1.4.x
-1. (Optional) [ZeroMQ](http://zeromq.org/intro:get-the-software) (3.2 or 4.0 for
-CurveZMQ)
+1. \*nix, OS X or Windows
+1. The [golang](http://golang.org/doc/install) compiler tools (1.2 or 1.3)
+1. [git](http://git-scm.com)
+1. GNU make
+1. (Optional) [ZeroMQ](http://zeromq.org/intro:get-the-software) (>=3.2 for
+plaintext ZMQ, >=4.0 for secure CurveZMQ)
+
+*\*nix: Most requirements can usually be installed by your favourite package
+manager. The optional ZeroMQ >=3.2 is usually also available via the package
+manager. ZeroMQ >=4.0 may need to be build and installed manually.*
+
+*OS X: Git and GNU make are provided automatically by XCode. The optional ZeroMQ
+can be installed via [Homebrew](http://brew.sh).*
+
+*Windows: GNU make for Windows can be found
+[here](http://gnuwin32.sourceforge.net/packages/make.htm). The optional ZeroMQ
+may need to be built and installed manually.*
 
 ### Building
 
@@ -85,28 +101,19 @@ found on the [Logstash Integration](docs/LogstashIntegration.md) page.
 
 ### Generating Certificates and Keys
 
-After Log Courier is built you will find a utility named lc-tlscert inside the
-'bin' folder alongside the main log-courier program. This will generate a
-self-signed certificate to get you started quickly with the TLS transport, and
-the necessary Log Courier and Logstash configuration snippets to make it work.
-
-Likewise, a utility called lc-curvekey is produced when ZMQ support is enabled.
-This utility will generate CurveZMQ key pairs as well as the necessary
-configuration snippets.
-
-## Command Line Options
+Running `make selfsigned` will automatically build and run the `lc-tlscert`
+utility that can quickly and easily generate a self-signed certificate for the
+TLS shipping transport.
 
-The `log-courier` command accepts the following command line options.
+Likewise, running `make curvekey` will automatically build and run the
+`lc-curvekey` utility that can quickly and easily generate CurveZMQ key pairs
+for the CurveZMQ shipping transport.
 
-	-config="": The config file to load
-	-config-test=false: Test the configuration specified by -config and exit
-	-cpuprofile="": write cpu profile to file
-	-from-beginning=false: On first run, read new files from the beginning instead of the end
-	-list-supported=false: List supported transports and codecs
-	-version=false: show version information
+Both tools also generate the required configuration file snippets.
 
 ## Documentation
 
 * [Administration Utility](docs/AdministrationUtility.md)
+* [Command Line Arguments](docs/CommandLineArguments.md)
 * [Configuration](docs/Configuration.md)
 * [Change Log](docs/ChangeLog.md)

build/fix_version

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# If this is not a git repository, use the existing version
+if [ ! -d '.git' ]; then
+	exit
+fi
+
+if [ -n "$1" ]; then
+	VERSION="$1"
+else
+	VERSION="$(git describe)"
+	if [ $(git status -s 2>/dev/null | wc -l) -ne 0 ]; then
+		VERSION="${VERSION}-dirty"
+	fi
+fi
+
+sed "s/<VERSION>/${VERSION}/g" src/lc-lib/core/version.go.template > src/lc-lib/core/version.go
+sed "s/<VERSION>/${VERSION#v}/g" log-courier.gemspec.template > log-courier.gemspec
+
+echo "Setting Log Courier Version ${VERSION}"

docs/ChangeLog.md

@@ -4,7 +4,8 @@
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 **Table of Contents**  *generated with [DocToc](http://doctoc.herokuapp.com/)*
 
-- [Latest Development](#latest-development)
+- [1.0](#10)
+- [0.15](#015)
 - [0.14](#014)
 - [0.13](#013)
 - [0.12](#012)
@@ -15,6 +16,22 @@
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
+## 1.0
+
+*23rd October 2014*
+
+* Remove `ping` command from `lc-admin` (#49)
+* Empty lines in a log file are incorrectly merged with the following line (#51)
+* Don't require a connection to Log Courier when running `lc-admin help` (#50)
+* Bring back `make selfsigned` to quickly generate self-signed TLS certificates
+(#25)
+* Implement `make curvekey` to quickly generate curve key pairs (#25)
+* Fix hanging ZMQ transport on transport error
+* Fix timeout on log-courier side when Logstash busy due to non-thread safe
+timeout timer in the log-courier gem
+* Gracefully handle multiline events greater than 10 MiB in size by splitting
+events
+
 ## 0.15
 
 *23rd September 2014*

docs/CommandLineArguments.md

@@ -0,0 +1,65 @@
+# Command Line Arguments
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](http://doctoc.herokuapp.com/)*
+
+- [Overview](#overview)
+- [`-config=<path>`](#-config=path)
+- [`-config-test`](#-config-test)
+- [`-cpuprofile=<path>`](#-cpuprofile=path)
+- [`-from-beginning`](#-from-beginning)
+- [`-list-supported`](#-list-supported)
+- [`-version`](#-version)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Overview
+
+The `log-courier` command accepts various command line arguments.
+
+## `-config=<path>`
+
+The path to the JSON configuration file to load.
+
+```
+log-courier -config=/etc/log-courier/log-courier.json
+```
+
+## `-config-test`
+
+Load the configuration and test it for validity, then exit.
+
+Will exit with code 0 if the configuration is valid and would not prevent
+log-courier from starting up. Will exit with code 1 if an error occurred,
+printing the error to standard output.
+
+## `-cpuprofile=<path>`
+
+The path to file to write CPU profiling information to, when investigating
+performance problems. Log Courier will run for a small period of time and then
+quit, writing the profiling information to this file.
+
+This flag should generally only be used when requested by a developer.
+
+## `-from-beginning`
+
+The `.log-courier` file stores the current shipping status as logs are shipped
+so that in the event of a service restart, not a single log entry is missed.
+
+In the event that the `.log-courier` file does not exist, Log Courier will by
+default start the initial shipping of log files from the end of the file.
+Setting this flag in the initial startup of Log Courier will trigger files to
+start shipping from the beginning of the file instead of the end.
+
+After the first `.log-courier` status file is written, all subsequent newly
+discovered log files will start from the begining, regardless of this flag.
+
+## `-list-supported`
+
+Print a list of available transports and codecs provided by this build of Log
+Courier, then exit.
+
+## `-version`
+
+Print the version of this build of Log Courier, then exit.

docs/Configuration.md

@@ -211,8 +211,8 @@ give around a 25% boost of events per second, at the expense of more memory
 usage.
 
 *For most installations you should leave this at the default as it can
-easily cope with over 10,000 events a second on most machines and uses little
-memory. It is useful only in very specific circumstances.*
+easily cope with over 10,000 events a second and uses little memory. It is
+useful only in very specific circumstances.*
 
 ### `"spool max bytes"`
 
@@ -224,6 +224,8 @@ does not have enough room for the next event, it will be flushed immediately.
 If this value is modified, the receiving end should also be configured with the
 new limit. For the Logstash plugin, this is the `max_packet_size` setting.
 
+The maximum value for this setting is 2147483648 (2 GiB).
+
 ### `"spool timeout"`
 
 *Duration. Optional. Default: 5*
@@ -243,6 +245,8 @@ field added.
 If the `fields` configuration already contained a "tags" entry, and it is an
 array, it will be appended to. Otherwise, the "tag" field will be left as is.
 
+This setting can not be greater than the `spool max bytes` setting.
+
 ### `"log level"`
 
 *String. Optional. Default: "info".  

docs/codecs/Multiline.md

@@ -16,6 +16,7 @@ option.
   - [`"negate"`](#negate)
   - [`"what"`](#what)
   - [`"previous timeout"`](#previous-timeout)
+  - [`"max multiline bytes"`](#max-multiline-bytes)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -72,3 +73,12 @@ offers a solution to this.
 When using `"previous"`, if `"previous timeout"` is not 0 any buffered lines
 will be flushed as a single event if no more lines are received within the
 specified time period.
+
+### `"max multiline bytes"`
+
+*Number. Optional. Default: `spool max bytes`*
+
+The maximum multiline length to process. If a multiline block exeeds this
+length, it will be split across multiple events.
+
+This setting can not be greater than the `spool max bytes` setting.

lib/log-courier/server.rb

@@ -106,7 +106,7 @@ def process_ping(message, comm)
 
     def process_jdat(message, comm, event_queue)
       # Now we have the data, aim to respond within 5 seconds
-      reset_ack_timeout
+      ack_timeout = Time.now.to_i + 5
 
       # OK - first is a nonce - we send this back with sequence acks
       # This allows the client to know what is being acknowledged
@@ -165,12 +165,12 @@ def process_jdat(message, comm, event_queue)
 
         # Queue the event
         begin
-          event_queue.push event, @ack_timeout - Time.now.to_i
+          event_queue.push event, [0, ack_timeout - Time.now.to_i].max
         rescue TimeoutError
           # Full pipeline, partial ack
           # NOTE: comm.send can raise a Timeout::Error of its own
           comm.send 'ACKN', [nonce, sequence].pack('A*N')
-          reset_ack_timeout
+          ack_timeout = Time.now.to_i + 5
           retry
         end
 
@@ -181,10 +181,5 @@ def process_jdat(message, comm, event_queue)
       # NOTE: comm.send can raise a Timeout::Error
       comm.send 'ACKN', [nonce, sequence].pack('A*N')
     end
-
-    def reset_ack_timeout
-      # TODO: Make a constant or configurable
-      @ack_timeout = Time.now.to_i + 5
-    end
   end
 end

lib/log-courier/server_tcp.rb

@@ -100,7 +100,7 @@ def initialize(options = {})
         end
 
         if @options[:port] == 0
-          @logger.warn '[LogCourierServer] Transport is listening on ephemeral port ' + @port.to_s
+          @logger.warn '[LogCourierServer] Transport ' + @options[:transport] + ' is listening on ephemeral port ' + @port.to_s
         end
       rescue => e
         raise "[LogCourierServer] Failed to initialise: #{e}"