blob: 3543011fa23cbda0ca181affec7dc5e1cea7d2ca [file] [log] [blame]
#!/usr/bin/python
import pdb
import argparse
import sys
from pyparsing import *
import json
import os
import patterns
import json
import logging
import logstash
import time
def follow(thefile):
while True:
line = thefile.readline()
if not line:
time.sleep(0.1)
continue
yield line
parse = argparse.ArgumentParser(description='Convert log file to json.')
parse.add_argument('--format', dest='format', action='store',default=None, help='Format e.g. Ansible')
parse.add_argument('--hostport', dest='hostport', action='store',default=None, help='Logstash UDP host:port')
parse.add_argument('--file', dest='filename', action='store',default=None, help='Filename to follow')
args = parse.parse_args()
host,port = args.hostport.split(':')
format = args.format
logger = logging.getLogger('python-logstash-logger')
logger.setLevel(logging.INFO)
logger.addHandler(logstash.LogstashHandler(host, int(port), version=1))
def send_log(log):
try:
msg = log['desc']
except KeyError:
msg = 'Metadata'
time.sleep(0.1)
try:
logger.info(msg, extra=log)
except Exception,e:
logger.info('Logstash log failed', extra={'xos_fatal':1,'exception':str(e)})
pass
### Read parser from patterns.py
def extract_global(l):
g = None
try:
global_tag = "==>" + Word(alphanums) + ":" + SkipTo(LineEnd(),include=True)
s = global_tag.parseString(l)
l = s[3]+'\n'
g = s[1]
except:
pass
return g,l
def serialize_raw(s):
for l in s.splitlines():
g,l = extract_global(l)
report(l, g)
def report(payload,g):
if (not payload):
return
if (type(payload)!=dict):
payload = {'desc':payload, 'global_tag': g}
send_log(payload)
def run_logger():
lst = []
for n in dir(patterns.Parser):
sym = getattr(patterns.Parser, n)
if isinstance(sym,ParserElement):
lst.append(sym)
default = SkipTo(LineEnd(),include=True)
xos_logger = Or(lst)
inp = ''
lc = 0
if (args.filename):
source = follow(open(args.filename))
while True:
if (not args.filename):
l = sys.stdin.readline()
else:
l = source.next()
if (l==""):
break
inp += l
lc += 1
if (lc>4):
top = inp.splitlines()[0]
line_idx = inp.index('\n')
inp=inp[line_idx+1:]
g,top = extract_global(top)
report(top,g)
lc-=1
try:
s = xos_logger.scanString(inp)
last = None
first = True
for i in s:
if (first):
pending = inp[:i[1]]
serialize_raw(pending)
first = False
try:
payload = i[0][0]
g,_ = extract_global(inp)
report(payload,g)
except IndexError:
pass
last = i
if (last):
inp = inp[last[2]:]
lines = inp.splitlines()
lc=len(lines)
except Exception,e:
# We don't want logging to hold up execution
#print str(e)
pass
def main():
run_logger()
if __name__ == "__main__":
main()