Merge branch 'develop' into stable

.gitignore

@@ -1,12 +1,13 @@
 Gemfile.lock
 bin
-log-courier-*.gem
 node_modules
 pkg
 spec/tmp
 src/github.com
 vendor
+version.txt
 .Makefile.tags
 .bundle
 .log-courier
 .vagrant
+*.gem

.travis.yml

@@ -5,8 +5,13 @@ go:
   - 1.2
   - 1.3
 
+# Until below issue is fixed, we need to install keys for rvm get stable
+# https://github.com/travis-ci/travis-ci/issues/2919
+before_install:
+  - gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 BF04FF17
+
+# Tests require ruby - ensure latest rvm list available and then switch to 2.0
 # https://github.com/travis-ci/travis-ci/issues/2220
-# Tests require ruby
 install:
   - rvm get stable
   - rvm use 2.0 --install --binary --fuzzy

Gemfile

@@ -1,9 +1,15 @@
 source 'https://rubygems.org'
+
+# Log Courier gem dependencies
 gem 'ffi-rzmq'
 gem 'multi_json'
-gem 'rspec'
 
+# Log Courier gem JSON parsers
 gem 'oj',        :platforms => :mri
+gem 'jrjackson', :platforms => :jruby
+
+# Profiler for MRI
 gem 'ruby-prof', :platforms => :mri
 
-gem 'jrjackson', :platforms => :jruby
+# Tests
+gem 'rspec'

Makefile

@@ -1,4 +1,4 @@
-.PHONY: prepare all log-courier gem test doc profile benchmark jrprofile jrbenchmark clean
+.PHONY: prepare fix_version all log-courier gem gem_plugins push_gems test doc profile benchmark jrprofile jrbenchmark clean
 
 MAKEFILE := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
 GOPATH := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE))))
@@ -7,7 +7,7 @@ export GOPATH := $(GOPATH)
 TAGS :=
 BINS := bin/log-courier bin/lc-tlscert bin/lc-admin
 GOTESTS := log-courier lc-tlscert lc-admin lc-lib/...
-TESTS := spec/courier_spec.rb spec/tcp_spec.rb spec/gem_spec.rb spec/multiline_spec.rb
+TESTS := spec/courier_spec.rb spec/tcp_spec.rb spec/gem_spec.rb
 
 ifneq (,$(findstring curvekey,$(MAKECMDGOALS)))
 with := zmq4
@@ -32,14 +32,21 @@ SAVETAGS := $(shell echo "$(TAGS)" >.Makefile.tags)
 endif
 endif
 
-all: log-courier
+all: | log-courier
 
-log-courier: $(BINS)
+log-courier: | $(BINS)
 
-gem:
+gem: | fix_version
 	gem build log-courier.gemspec
 
-test: all vendor/bundle/.GemfileModT
+gem_plugins: | fix_version
+	gem build logstash-input-log-courier.gemspec
+	gem build logstash-output-log-courier.gemspec
+
+push_gems: | gem gem_plugins fix_version vendor/bundle/.GemfileModT
+	build/push_gems
+
+test: | all vendor/bundle/.GemfileModT
 	go get -d -tags "$(TAGS)" $(GOTESTS)
 	go test -tags "$(TAGS)" $(GOTESTS)
 	bundle exec rspec $(TESTS)
@@ -56,20 +63,20 @@ doc:
 	@node_modules/.bin/doctoc README.md
 	@for F in docs/*.md docs/codecs/*.md; do node_modules/.bin/doctoc $$F; done
 
-profile: all vendor/bundle/.GemfileModT
+profile: | all vendor/bundle/.GemfileModT
 	bundle exec rspec spec/profile_spec.rb
 
-benchmark: all vendor/bundle/.GemfileModT
+benchmark: | all vendor/bundle/.GemfileModT
 	bundle exec rspec spec/benchmark_spec.rb
 
 vendor/bundle/.GemfileModT: Gemfile
 	bundle install --path vendor/bundle
 	@touch $@
 
-jrprofile: all vendor/bundle/.GemfileModT
+jrprofile: | all vendor/bundle/.GemfileJRubyModT
 	jruby --profile -G vendor/bundle/jruby/1.9/bin/rspec spec/benchmark_spec.rb
 
-jrbenchmark: all vendor/bundle/.GemfileJRubyModT
+jrbenchmark: | all vendor/bundle/.GemfileJRubyModT
 	jruby -G vendor/bundle/jruby/1.9/bin/rspec spec/benchmark_spec.rb
 
 vendor/bundle/.GemfileJRubyModT: Gemfile
@@ -79,19 +86,20 @@ vendor/bundle/.GemfileJRubyModT: Gemfile
 clean:
 	go clean -i ./...
 ifneq ($(implyclean),yes)
+	rm -rf src/github.com
 	rm -rf vendor/bundle
 	rm -f Gemfile.lock
-	rm -f log-courier-*.gem
+	rm -f *.gem
 endif
 
-prepare:
+fix_version:
+	build/fix_version
+
+prepare: | fix_version
 	@go version >/dev/null || (echo "Go not found. You need to install Go version 1.2 or 1.3: http://golang.org/doc/install"; false)
 	@go version | grep -q 'go version go1.[23]' || (echo "Go version 1.2 or 1.3 required, you have a version of Go that is not supported."; false)
 	@echo "GOPATH: $${GOPATH}"
-	build/fix_version
 
-bin/%: FORCE | prepare
+bin/%: prepare
 	go get -d -tags "$(TAGS)" $*
 	go install -tags "$(TAGS)" $*
-
-FORCE:

README.md

@@ -12,9 +12,10 @@ with many fixes and behavioural improvements.
 
 - [Features](#features)
 - [Installation](#installation)
-  - [Build Requirements](#build-requirements)
+  - [Requirements](#requirements)
   - [Building](#building)
   - [Logstash Integration](#logstash-integration)
+  - [Building with ZMQ support](#building-with-zmq-support)
   - [Generating Certificates and Keys](#generating-certificates-and-keys)
 - [Documentation](#documentation)
 
@@ -35,69 +36,82 @@ Log Courier implements the following features:
 * TLS client certificate verification
 * Secure CurveZMQ shipping transport to load balance across multiple Logstash
 instances (optional, requires ZeroMQ 4+)
-* Plaintext TCP shipping transport for configuration simplicity in local networks
+* Plaintext TCP shipping transport for configuration simplicity in local
+networks
 * Plaintext ZMQ shipping transport
 * [Administration utility](docs/AdministrationUtility.md) to monitor the
 shipping speed and status
 * [Multiline](docs/codecs/Multiline.md) codec
 * [Filter](docs/codecs/Filter.md) codec
-
-Log Courier integrates with Logstash using an event receiver ruby gem. An event
-sender ruby gem is also available to allow fast and secure transmission between
-two Logstash instances.
+* [Logstash Integration](docs/LogstashIntegration.md) with an input and output
+plugin
 
 ## Installation
 
-### Build Requirements
+### Requirements
 
 1. \*nix, OS X or Windows
 1. The [golang](http://golang.org/doc/install) compiler tools (1.2 or 1.3)
 1. [git](http://git-scm.com)
 1. GNU make
-1. (Optional) [ZeroMQ](http://zeromq.org/intro:get-the-software) (>=3.2 for
-plaintext ZMQ, >=4.0 for secure CurveZMQ)
-
-*\*nix: Most requirements can usually be installed by your favourite package
-manager. The optional ZeroMQ >=3.2 is usually also available via the package
-manager. ZeroMQ >=4.0 may need to be build and installed manually.*
-
-*OS X: Git and GNU make are provided automatically by XCode. The optional ZeroMQ
-can be installed via [Homebrew](http://brew.sh).*
 
-*Windows: GNU make for Windows can be found
-[here](http://gnuwin32.sourceforge.net/packages/make.htm). The optional ZeroMQ
-may need to be built and installed manually.*
+***\*nix:*** *Most requirements can usually be installed by your favourite package
+manager.*  
+***OS X:*** *Git and GNU make are provided automatically by XCode.*  
+***Windows:*** *GNU make for Windows can be found
+[here](http://gnuwin32.sourceforge.net/packages/make.htm).*
 
 ### Building
 
-To build without the optional ZMQ support, simply run `make` as
-follows.
+To build, simply run `make` as follows.
 
 	git clone https://github.com/driskell/log-courier
 	cd log-courier
 	make
 
 The log-courier program can then be found in the 'bin' folder.
 
-To build with the optional ZMQ support use the following.
-
-	git clone https://github.com/driskell/log-courier
-	cd log-courier
-	make with=zmq3
-
-For CurveZMQ support (ZMQ with public key encryption) replace `zmq3` with
-`zmq4`.
-
-*If you receive errors whilst running `make` try `gmake` instead.*
+*Note: If you receive errors whilst running `make`, try `gmake` instead.*
 
 ### Logstash Integration
 
 Log Courier does not utilise the lumberjack Logstash plugin and instead uses its
 own custom plugin. This allows significant enhancements to the integration far
 beyond the lumberjack protocol allows.
 
-Details instructions on the plugin and how to install it into Logstash can be
-found on the [Logstash Integration](docs/LogstashIntegration.md) page.
+Install using the Logstash 1.5+ Plugin manager.
+
+	cd /path/to/logstash
+	bin/logstash plugin install logstash-input-log-courier
+
+Detailed instructions, including integration with Logstash 1.4.x, can be found
+on the [Logstash Integration](docs/LogstashIntegration.md) page.
+
+### Building with ZMQ support
+
+To use the 'plainzmq' and 'zmq' transports, you will need to install
+[ZeroMQ](http://zeromq.org/intro:get-the-software) (>=3.2 for cleartext
+'plainzmq', >=4.0 for encrypted 'zmq').
+
+***\*nix:*** *ZeroMQ >=3.2 is usually available via the package manager. ZeroMQ >=4.0
+may need to be built and installed manually.*  
+***OS X:*** *ZeroMQ can be installed via [Homebrew](http://brew.sh).*  
+***Windows:*** *ZeroMQ will need to be built and installed manually.*
+
+Once the required version of ZeroMQ is installed, run the corresponding `make`
+command to build Log Courier with the ZMQ transports.
+
+	# ZeroMQ >=3.2 - cleartext 'plainzmq' transport
+	make with=zmq3
+	# ZeroMQ >=4.0 - both cleartext 'plainzmq' and encrypted 'zmq' transport
+	make with=zmq4
+
+*Note: If you receive errors whilst running `make`, try `gmake` instead.*
+
+**Please ensure that the versions of ZeroMQ installed on the Logstash hosts and
+the Log Courier hosts are of the same major version. A Log Courier host that has
+ZeroMQ 4.0.5 will not work with a Logstash host using ZeroMQ 3.2.4 (but will
+work with a Logstash host using ZeroMQ 4.0.4.)**
 
 ### Generating Certificates and Keys
 
@@ -107,7 +121,8 @@ TLS shipping transport.
 
 Likewise, running `make curvekey` will automatically build and run the
 `lc-curvekey` utility that can quickly and easily generate CurveZMQ key pairs
-for the CurveZMQ shipping transport.
+for the CurveZMQ shipping transport. This tool is only available when Log
+Courier is built with ZeroMQ >=4.0.
 
 Both tools also generate the required configuration file snippets.
 

build/fix_version

@@ -8,13 +8,24 @@ fi
 if [ -n "$1" ]; then
 	VERSION="$1"
 else
-	VERSION="$(git describe)"
-	if [ $(git status -s 2>/dev/null | wc -l) -ne 0 ]; then
-		VERSION="${VERSION}-dirty"
-	fi
+	# Describe version from Git, and ensure the only "-xxx" is the git revision
+	# This ensures that gem builds only add one ".pre" tag automatically
+	VERSION="$(git describe | sed 's/-\([0-9][0-9]*\)-\([0-9a-z][0-9a-z]*\)$/.\1.\2/g')"
 fi
 
-sed "s/<VERSION>/${VERSION}/g" src/lc-lib/core/version.go.template > src/lc-lib/core/version.go
-sed "s/<VERSION>/${VERSION#v}/g" log-courier.gemspec.template > log-courier.gemspec
+# Patch version.go
+sed "s/\\(const  *Log_Courier_Version  *string  *=  *\"\\)[^\"]*\\(\"\\)/\\1${VERSION}\\2/g" src/lc-lib/core/version.go > src/lc-lib/core/version.go.tmp
+\mv -f src/lc-lib/core/version.go.tmp src/lc-lib/core/version.go
 
-echo "Setting Log Courier Version ${VERSION}"
+# Patch the gemspecs
+for GEM in log-courier logstash-input-log-courier logstash-output-log-courier; do
+	sed "s/\\(gem.version *= *'\\)[^']*\\('\\)/\\1${VERSION#v}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp
+	\mv -f ${GEM}.gemspec.tmp ${GEM}.gemspec
+	[ ${GEM#logstash-} != $GEM ] && {
+		sed "s/\\(gem.add_runtime_dependency  *'log-courier' *,  *'= *\\)[^']*\\('\\)/\\1${VERSION#v}\\2/g" ${GEM}.gemspec > ${GEM}.gemspec.tmp
+		\mv -f ${GEM}.gemspec.tmp ${GEM}.gemspec
+	}
+done
+
+echo "${VERSION#v}" > version.txt
+echo "Set Log Courier Version ${VERSION}"

build/push_gems

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+for GEM in *-$(cat version.txt).gem; do
+	echo "- ${GEM}"
+	gem push $GEM
+done
+
+:

docs/ChangeLog.md

@@ -4,6 +4,7 @@
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 **Table of Contents**  *generated with [DocToc](http://doctoc.herokuapp.com/)*
 
+- [1.1](#11)
 - [1.0](#10)
 - [0.15](#015)
 - [0.14](#014)
@@ -16,6 +17,25 @@
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
+## 1.1
+
+*30th October 2014*
+
+* Implement gems for the new Logstash plugin system (#60)
+* Fix gem build failing on develop branch with old rubygems versions due to a
+malformed version string (#62)
+* Fix ZeroMQ transports in the ruby gem with Logstash 1.4.x (#63)
+* Fix build issue with ZeroMQ 3.2 and `make with=zmq3`
+* Fix partial acknowledgements not being passed to registrar and persisted to
+disk
+* Fix a race condition when the spooler flushes to prevent a timeout occurring
+one or more times after a flush due to size
+* Print informational messages containing ZMQ library version information during
+gem and log-courier startup to aid in diagnostics
+* Raise a friendly error when trying to use the zmq transport in the Log Courier
+gem with incompatible versions of libzmq
+* Various fixes and improvements to log-courier, gem, build and tests
+
 ## 1.0
 
 *23rd October 2014*