CORD-705: ELK logging and debugger for CORD
Change-Id: I3406d95161e71976ee3b3bf689f89069b4d9de73
diff --git a/elk-logger/logstash_tail b/elk-logger/logstash_tail
new file mode 100755
index 0000000..3543011
--- /dev/null
+++ b/elk-logger/logstash_tail
@@ -0,0 +1,161 @@
+#!/usr/bin/python
+
+import pdb
+import argparse
+import sys
+from pyparsing import *
+import json
+import os
+import patterns
+import json
+import logging
+import logstash
+import time
+
+def follow(thefile):
+ while True:
+ line = thefile.readline()
+ if not line:
+ time.sleep(0.1)
+ continue
+ yield line
+
+
+parse = argparse.ArgumentParser(description='Convert log file to json.')
+parse.add_argument('--format', dest='format', action='store',default=None, help='Format e.g. Ansible')
+parse.add_argument('--hostport', dest='hostport', action='store',default=None, help='Logstash UDP host:port')
+parse.add_argument('--file', dest='filename', action='store',default=None, help='Filename to follow')
+
+
+args = parse.parse_args()
+
+host,port = args.hostport.split(':')
+
+format = args.format
+
+logger = logging.getLogger('python-logstash-logger')
+logger.setLevel(logging.INFO)
+logger.addHandler(logstash.LogstashHandler(host, int(port), version=1))
+
+def send_log(log):
+
+ try:
+ msg = log['desc']
+ except KeyError:
+ msg = 'Metadata'
+
+ time.sleep(0.1)
+ try:
+ logger.info(msg, extra=log)
+ except Exception,e:
+ logger.info('Logstash log failed', extra={'xos_fatal':1,'exception':str(e)})
+ pass
+
+### Read parser from patterns.py
+
+def extract_global(l):
+ g = None
+
+ try:
+ global_tag = "==>" + Word(alphanums) + ":" + SkipTo(LineEnd(),include=True)
+ s = global_tag.parseString(l)
+ l = s[3]+'\n'
+ g = s[1]
+ except:
+ pass
+
+ return g,l
+
+def serialize_raw(s):
+ for l in s.splitlines():
+ g,l = extract_global(l)
+ report(l, g)
+
+def report(payload,g):
+ if (not payload):
+ return
+
+ if (type(payload)!=dict):
+ payload = {'desc':payload, 'global_tag': g}
+
+ send_log(payload)
+
+def run_logger():
+ lst = []
+ for n in dir(patterns.Parser):
+ sym = getattr(patterns.Parser, n)
+ if isinstance(sym,ParserElement):
+ lst.append(sym)
+
+ default = SkipTo(LineEnd(),include=True)
+
+ xos_logger = Or(lst)
+
+ inp = ''
+ lc = 0
+
+
+
+ if (args.filename):
+ source = follow(open(args.filename))
+
+ while True:
+ if (not args.filename):
+ l = sys.stdin.readline()
+ else:
+ l = source.next()
+
+ if (l==""):
+ break
+
+ inp += l
+ lc += 1
+
+ if (lc>4):
+ top = inp.splitlines()[0]
+ line_idx = inp.index('\n')
+ inp=inp[line_idx+1:]
+
+ g,top = extract_global(top)
+ report(top,g)
+
+ lc-=1
+
+ try:
+ s = xos_logger.scanString(inp)
+
+ last = None
+ first = True
+ for i in s:
+
+ if (first):
+ pending = inp[:i[1]]
+ serialize_raw(pending)
+
+ first = False
+
+ try:
+ payload = i[0][0]
+ g,_ = extract_global(inp)
+ report(payload,g)
+ except IndexError:
+ pass
+
+ last = i
+
+ if (last):
+ inp = inp[last[2]:]
+ lines = inp.splitlines()
+ lc=len(lines)
+
+ except Exception,e:
+ # We don't want logging to hold up execution
+ #print str(e)
+ pass
+
+def main():
+ run_logger()
+
+
+if __name__ == "__main__":
+ main()