YALP

Distributed log parsing and collection.

Install and Configure

Brief install guide:

$ sudo apt-get install rabbitmq-server
$ virtualenv /srv/yalp_env
$ source /srv/yalp_env/bin/activate
(yalp_env) $ pip install yalp

/srv/yalp.yml:

# Celery configuration
broker_url: amqp://guest:guest@localhost:5672//
inputs:
  - file:
      path: '/var/log/nginx/access.log'
parsers:
  - grok:
      pattern: '%{COMBINEDAPACHELOG}'
  - timestamp:
      field: timestamp
  - goip:
      field: clientip
      geoip_dat: /usr/share/GeoLiteCity.dat
  - user_agent:
      field: agent
  - url:
      field: request
outputs:
  - elasticsearch:
      uri: http://localhost:9200

(yalp_env) $ yalp-inputs -c /srv/yalp.yml
(yalp_env) $ yalp-parsers -c /srv/yalp.yml
(yalp_env) $ yalp-outputs -c /srv/yalp.yml