blob: 6b29718bc99f34f1392eedd63236ef2e639c3eef [file] [log] [blame]
Sapan Bhatia01ddea62017-02-10 11:28:48 -08001#!/usr/bin/python
2
3import pdb
4import argparse
5import sys
6from pyparsing import *
7import json
8import os
9import patterns
10import json
11import logging
12import logstash
13import time
14
15def follow(thefile):
16 while True:
17 line = thefile.readline()
18 if not line:
19 time.sleep(0.1)
20 continue
21 yield line
22
23
24parse = argparse.ArgumentParser(description='Convert log file to json.')
25parse.add_argument('--format', dest='format', action='store',default=None, help='Format e.g. Ansible')
26parse.add_argument('--hostport', dest='hostport', action='store',default=None, help='Logstash UDP host:port')
27parse.add_argument('--file', dest='filename', action='store',default=None, help='Filename to follow')
Sapan Bhatia01ddea62017-02-10 11:28:48 -080028
29args = parse.parse_args()
30
31host,port = args.hostport.split(':')
32
33format = args.format
34
35logger = logging.getLogger('python-logstash-logger')
36logger.setLevel(logging.INFO)
37logger.addHandler(logstash.LogstashHandler(host, int(port), version=1))
38
39def send_log(log):
40
41 try:
42 msg = log['desc']
43 except KeyError:
44 msg = 'Metadata'
45
46 time.sleep(0.1)
47 try:
48 logger.info(msg, extra=log)
49 except Exception,e:
50 logger.info('Logstash log failed', extra={'xos_fatal':1,'exception':str(e)})
51 pass
52
53### Read parser from patterns.py
54
55def extract_global(l):
56 g = None
57
58 try:
59 global_tag = "==>" + Word(alphanums) + ":" + SkipTo(LineEnd(),include=True)
60 s = global_tag.parseString(l)
61 l = s[3]+'\n'
62 g = s[1]
63 except:
64 pass
65
66 return g,l
67
68def serialize_raw(s):
69 for l in s.splitlines():
70 g,l = extract_global(l)
71 report(l, g)
72
73def report(payload,g):
74 if (not payload):
75 return
76
77 if (type(payload)!=dict):
78 payload = {'desc':payload, 'global_tag': g}
79
80 send_log(payload)
Sapan Bhatia1064f7e2017-02-15 17:51:30 -080081 #print payload
Sapan Bhatia01ddea62017-02-10 11:28:48 -080082
83def run_logger():
84 lst = []
85 for n in dir(patterns.Parser):
86 sym = getattr(patterns.Parser, n)
87 if isinstance(sym,ParserElement):
88 lst.append(sym)
89
90 default = SkipTo(LineEnd(),include=True)
91
92 xos_logger = Or(lst)
93
94 inp = ''
95 lc = 0
96
Sapan Bhatia01ddea62017-02-10 11:28:48 -080097 if (args.filename):
98 source = follow(open(args.filename))
99
100 while True:
101 if (not args.filename):
102 l = sys.stdin.readline()
103 else:
104 l = source.next()
105
106 if (l==""):
107 break
Sapan Bhatiaa5730792017-03-02 11:02:13 +0100108 elif len(l)>200:
109 continue
Sapan Bhatia01ddea62017-02-10 11:28:48 -0800110
111 inp += l
Sapan Bhatia1064f7e2017-02-15 17:51:30 -0800112 lc = len(inp.split('\n'))
Sapan Bhatia01ddea62017-02-10 11:28:48 -0800113
114 if (lc>4):
115 top = inp.splitlines()[0]
Sapan Bhatia1064f7e2017-02-15 17:51:30 -0800116 try:
117 line_idx = inp.index('\n')
118 inp=inp[line_idx+1:]
Sapan Bhatia01ddea62017-02-10 11:28:48 -0800119
Sapan Bhatia1064f7e2017-02-15 17:51:30 -0800120 g,top = extract_global(top)
121 report(top,g)
122 lc = len(inp.split('\n'))
123 except ValueError:
124 pass
Sapan Bhatia01ddea62017-02-10 11:28:48 -0800125
126 try:
127 s = xos_logger.scanString(inp)
128
129 last = None
130 first = True
131 for i in s:
132
133 if (first):
134 pending = inp[:i[1]]
135 serialize_raw(pending)
136
137 first = False
138
139 try:
140 payload = i[0][0]
141 g,_ = extract_global(inp)
142 report(payload,g)
143 except IndexError:
144 pass
145
146 last = i
147
148 if (last):
149 inp = inp[last[2]:]
150 lines = inp.splitlines()
151 lc=len(lines)
152
153 except Exception,e:
154 # We don't want logging to hold up execution
155 #print str(e)
156 pass
157
158def main():
159 run_logger()
160
161
162if __name__ == "__main__":
163 main()