elk-logger/logstash_tail

#!/usr/bin/python

import pdb
import argparse
import sys
from pyparsing import *
import json
import os
import patterns
import json
import logging
import logstash
import time

def follow(thefile):
    while True:
        line = thefile.readline()
        if not line:
            time.sleep(0.1)
            continue
        yield line


parse = argparse.ArgumentParser(description='Convert log file to json.')
parse.add_argument('--format', dest='format', action='store',default=None, help='Format e.g. Ansible')
parse.add_argument('--hostport', dest='hostport', action='store',default=None, help='Logstash UDP host:port')
parse.add_argument('--file', dest='filename', action='store',default=None, help='Filename to follow')

args = parse.parse_args()

host,port = args.hostport.split(':')

format = args.format

logger = logging.getLogger('python-logstash-logger')
logger.setLevel(logging.INFO)
logger.addHandler(logstash.LogstashHandler(host, int(port), version=1))

def send_log(log):
    
    try:
       msg = log['desc']
    except KeyError:
       msg = 'Metadata'

    time.sleep(0.1)
    try:
        logger.info(msg, extra=log)
    except Exception,e:
        logger.info('Logstash log failed', extra={'xos_fatal':1,'exception':str(e)})
        pass

### Read parser from patterns.py

def extract_global(l):
    g = None

    try:
        global_tag = "==>" + Word(alphanums) + ":" + SkipTo(LineEnd(),include=True)
        s = global_tag.parseString(l)
        l = s[3]+'\n'
        g = s[1]
    except:
        pass

    return g,l

def serialize_raw(s):
    for l in s.splitlines():
        g,l = extract_global(l)
        report(l, g)

def report(payload,g):
    if (not payload):
        return

    if (type(payload)!=dict):
        payload = {'desc':payload, 'global_tag': g}

    send_log(payload)
    #print payload

def run_logger():
    lst = []
    for n in dir(patterns.Parser):
        sym = getattr(patterns.Parser, n)
        if isinstance(sym,ParserElement):
            lst.append(sym)

    default = SkipTo(LineEnd(),include=True)

    xos_logger = Or(lst)

    inp = ''
    lc = 0

    if (args.filename):
        source = follow(open(args.filename))

    while True:
        if (not args.filename):
            l = sys.stdin.readline()
        else:
            l = source.next()

        if (l==""):
            break
        elif len(l)>200:
            continue

        inp += l
        lc = len(inp.split('\n'))

        if (lc>4):
            top = inp.splitlines()[0]
            try:
                line_idx = inp.index('\n')
                inp=inp[line_idx+1:]

                g,top = extract_global(top)
                report(top,g)
                lc = len(inp.split('\n'))
            except ValueError:
                pass

        try:
            s = xos_logger.scanString(inp)

            last = None
            first = True
            for i in s:

                if (first):
                    pending = inp[:i[1]]
                    serialize_raw(pending)

                first = False

                try:
                    payload = i[0][0]
                    g,_ = extract_global(inp)
                    report(payload,g)
                except IndexError:
                    pass

                last = i

            if (last):
                inp = inp[last[2]:]
                lines = inp.splitlines()
                lc=len(lines)
            
        except Exception,e:
            # We don't want logging to hold up execution
            #print str(e)
            pass

def main():
    run_logger()
    
        
if __name__ == "__main__":
    main()