Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | import pdb |
| 4 | import argparse |
| 5 | import sys |
| 6 | from pyparsing import * |
| 7 | import json |
| 8 | import os |
| 9 | import patterns |
| 10 | import json |
| 11 | import logging |
| 12 | import logstash |
| 13 | import time |
| 14 | |
| 15 | def follow(thefile): |
| 16 | while True: |
| 17 | line = thefile.readline() |
| 18 | if not line: |
| 19 | time.sleep(0.1) |
| 20 | continue |
| 21 | yield line |
| 22 | |
| 23 | |
| 24 | parse = argparse.ArgumentParser(description='Convert log file to json.') |
| 25 | parse.add_argument('--format', dest='format', action='store',default=None, help='Format e.g. Ansible') |
| 26 | parse.add_argument('--hostport', dest='hostport', action='store',default=None, help='Logstash UDP host:port') |
| 27 | parse.add_argument('--file', dest='filename', action='store',default=None, help='Filename to follow') |
Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 28 | |
| 29 | args = parse.parse_args() |
| 30 | |
| 31 | host,port = args.hostport.split(':') |
| 32 | |
| 33 | format = args.format |
| 34 | |
| 35 | logger = logging.getLogger('python-logstash-logger') |
| 36 | logger.setLevel(logging.INFO) |
| 37 | logger.addHandler(logstash.LogstashHandler(host, int(port), version=1)) |
| 38 | |
| 39 | def send_log(log): |
| 40 | |
| 41 | try: |
| 42 | msg = log['desc'] |
| 43 | except KeyError: |
| 44 | msg = 'Metadata' |
| 45 | |
| 46 | time.sleep(0.1) |
| 47 | try: |
| 48 | logger.info(msg, extra=log) |
| 49 | except Exception,e: |
| 50 | logger.info('Logstash log failed', extra={'xos_fatal':1,'exception':str(e)}) |
| 51 | pass |
| 52 | |
| 53 | ### Read parser from patterns.py |
| 54 | |
| 55 | def extract_global(l): |
| 56 | g = None |
| 57 | |
| 58 | try: |
| 59 | global_tag = "==>" + Word(alphanums) + ":" + SkipTo(LineEnd(),include=True) |
| 60 | s = global_tag.parseString(l) |
| 61 | l = s[3]+'\n' |
| 62 | g = s[1] |
| 63 | except: |
| 64 | pass |
| 65 | |
| 66 | return g,l |
| 67 | |
| 68 | def serialize_raw(s): |
| 69 | for l in s.splitlines(): |
| 70 | g,l = extract_global(l) |
| 71 | report(l, g) |
| 72 | |
| 73 | def report(payload,g): |
| 74 | if (not payload): |
| 75 | return |
| 76 | |
| 77 | if (type(payload)!=dict): |
| 78 | payload = {'desc':payload, 'global_tag': g} |
| 79 | |
| 80 | send_log(payload) |
Sapan Bhatia | 1064f7e | 2017-02-15 17:51:30 -0800 | [diff] [blame] | 81 | #print payload |
Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 82 | |
| 83 | def run_logger(): |
| 84 | lst = [] |
| 85 | for n in dir(patterns.Parser): |
| 86 | sym = getattr(patterns.Parser, n) |
| 87 | if isinstance(sym,ParserElement): |
| 88 | lst.append(sym) |
| 89 | |
| 90 | default = SkipTo(LineEnd(),include=True) |
| 91 | |
| 92 | xos_logger = Or(lst) |
| 93 | |
| 94 | inp = '' |
| 95 | lc = 0 |
| 96 | |
Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 97 | if (args.filename): |
| 98 | source = follow(open(args.filename)) |
| 99 | |
| 100 | while True: |
| 101 | if (not args.filename): |
| 102 | l = sys.stdin.readline() |
| 103 | else: |
| 104 | l = source.next() |
| 105 | |
| 106 | if (l==""): |
| 107 | break |
Sapan Bhatia | a573079 | 2017-03-02 11:02:13 +0100 | [diff] [blame] | 108 | elif len(l)>200: |
| 109 | continue |
Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 110 | |
| 111 | inp += l |
Sapan Bhatia | 1064f7e | 2017-02-15 17:51:30 -0800 | [diff] [blame] | 112 | lc = len(inp.split('\n')) |
Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 113 | |
| 114 | if (lc>4): |
| 115 | top = inp.splitlines()[0] |
Sapan Bhatia | 1064f7e | 2017-02-15 17:51:30 -0800 | [diff] [blame] | 116 | try: |
| 117 | line_idx = inp.index('\n') |
| 118 | inp=inp[line_idx+1:] |
Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 119 | |
Sapan Bhatia | 1064f7e | 2017-02-15 17:51:30 -0800 | [diff] [blame] | 120 | g,top = extract_global(top) |
| 121 | report(top,g) |
| 122 | lc = len(inp.split('\n')) |
| 123 | except ValueError: |
| 124 | pass |
Sapan Bhatia | 01ddea6 | 2017-02-10 11:28:48 -0800 | [diff] [blame] | 125 | |
| 126 | try: |
| 127 | s = xos_logger.scanString(inp) |
| 128 | |
| 129 | last = None |
| 130 | first = True |
| 131 | for i in s: |
| 132 | |
| 133 | if (first): |
| 134 | pending = inp[:i[1]] |
| 135 | serialize_raw(pending) |
| 136 | |
| 137 | first = False |
| 138 | |
| 139 | try: |
| 140 | payload = i[0][0] |
| 141 | g,_ = extract_global(inp) |
| 142 | report(payload,g) |
| 143 | except IndexError: |
| 144 | pass |
| 145 | |
| 146 | last = i |
| 147 | |
| 148 | if (last): |
| 149 | inp = inp[last[2]:] |
| 150 | lines = inp.splitlines() |
| 151 | lc=len(lines) |
| 152 | |
| 153 | except Exception,e: |
| 154 | # We don't want logging to hold up execution |
| 155 | #print str(e) |
| 156 | pass |
| 157 | |
| 158 | def main(): |
| 159 | run_logger() |
| 160 | |
| 161 | |
| 162 | if __name__ == "__main__": |
| 163 | main() |