CORD-705: ELK logging and debugger for CORD
Change-Id: I3406d95161e71976ee3b3bf689f89069b4d9de73
diff --git a/elk-logger/elk-ciab.diff b/elk-logger/elk-ciab.diff
new file mode 100644
index 0000000..68f8260
--- /dev/null
+++ b/elk-logger/elk-ciab.diff
@@ -0,0 +1,296 @@
+
+project onos-apps/apps/vtn/
+diff --git a/xos/synchronizer/vtn_synchronizer_config b/xos/synchronizer/vtn_synchronizer_config
+index dad5cb6..f317dee 100644
+--- a/xos/synchronizer/vtn_synchronizer_config
++++ b/xos/synchronizer/vtn_synchronizer_config
+@@ -33,6 +33,7 @@ driver=openstack
+ pretend=False
+ backoff_disabled=True
+ enable_watchers=True
++logstash_hostport=10.100.198.222:5617
+
+ [nova]
+ ca_ssl_cert=/etc/ssl/certs/ca-certificates.crt
+
+project orchestration/service-profile/
+diff --git a/cord-pod-ansible/files/monitoring_synchronizer_config b/cord-pod-ansible/files/monitoring_synchronizer_config
+index 1ee96f0..932a32b 100644
+--- a/cord-pod-ansible/files/monitoring_synchronizer_config
++++ b/cord-pod-ansible/files/monitoring_synchronizer_config
+@@ -37,6 +37,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/cord-pod-ansible/files/vcpe_synchronizer_config b/cord-pod-ansible/files/vcpe_synchronizer_config
+index 9da6ede..9ef259b 100644
+--- a/cord-pod-ansible/files/vcpe_synchronizer_config
++++ b/cord-pod-ansible/files/vcpe_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/cord-pod-ansible/files/vtr_synchronizer_config b/cord-pod-ansible/files/vtr_synchronizer_config
+index 223ab00..4e3008c 100644
+--- a/cord-pod-ansible/files/vtr_synchronizer_config
++++ b/cord-pod-ansible/files/vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/cord-pod/files/monitoring_synchronizer_config b/cord-pod/files/monitoring_synchronizer_config
+index cfd73a0..485f993 100644
+--- a/cord-pod/files/monitoring_synchronizer_config
++++ b/cord-pod/files/monitoring_synchronizer_config
+@@ -37,6 +37,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/cord-pod/files/openstack_synchronizer_config b/cord-pod/files/openstack_synchronizer_config
+index 3b7f8e9..29077e2 100644
+--- a/cord-pod/files/openstack_synchronizer_config
++++ b/cord-pod/files/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/cord-pod/files/vcpe_synchronizer_config b/cord-pod/files/vcpe_synchronizer_config
+index babe94f..34e0f2b 100644
+--- a/cord-pod/files/vcpe_synchronizer_config
++++ b/cord-pod/files/vcpe_synchronizer_config
+@@ -39,6 +39,7 @@ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
+ enable_watchers=True
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/cord-pod/files/vtr_synchronizer_config b/cord-pod/files/vtr_synchronizer_config
+index 3d3209c..56af7fc 100644
+--- a/cord-pod/files/vtr_synchronizer_config
++++ b/cord-pod/files/vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/mcord/files/monitoring_synchronizer_config b/mcord/files/monitoring_synchronizer_config
+index cfd73a0..485f993 100644
+--- a/mcord/files/monitoring_synchronizer_config
++++ b/mcord/files/monitoring_synchronizer_config
+@@ -37,6 +37,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/mcord/files/openstack_synchronizer_config b/mcord/files/openstack_synchronizer_config
+index 82c2f79..58dece3 100644
+--- a/mcord/files/openstack_synchronizer_config
++++ b/mcord/files/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=False
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/opencloud/files/openstack_synchronizer_config b/opencloud/files/openstack_synchronizer_config
+index 74d68e3..8b0f8f9 100644
+--- a/opencloud/files/openstack_synchronizer_config
++++ b/opencloud/files/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos/
+diff --git a/xos/synchronizers/onboarding/onboarding_synchronizer_config b/xos/synchronizers/onboarding/onboarding_synchronizer_config
+index fb33044..6877378 100644
+--- a/xos/synchronizers/onboarding/onboarding_synchronizer_config
++++ b/xos/synchronizers/onboarding/onboarding_synchronizer_config
+@@ -32,4 +32,4 @@ backoff_disabled=True
+ pretend=False
+ save_ansible_output=True
+ node_key=/opt/cord_profile/node_key
+-
++logstash_hostport=10.100.198.222:5617
+diff --git a/xos/synchronizers/syndicate/syndicate_synchronizer_config b/xos/synchronizers/syndicate/syndicate_synchronizer_config
+index 7c9d2d2..c9f3cf5 100644
+--- a/xos/synchronizers/syndicate/syndicate_synchronizer_config
++++ b/xos/synchronizers/syndicate/syndicate_synchronizer_config
+@@ -29,6 +29,7 @@ steps_dir=/opt/xos/synchronizers/syndicate/steps
+ deleters_dir=/opt/xos/synchronizers/syndicate/deleters
+ log_file=console
+ driver=None
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/fabric/
+diff --git a/xos/synchronizer/fabric_synchronizer_config b/xos/synchronizer/fabric_synchronizer_config
+index e98abe5..b15fcc3 100644
+--- a/xos/synchronizer/fabric_synchronizer_config
++++ b/xos/synchronizer/fabric_synchronizer_config
+@@ -21,3 +21,4 @@ pretend=False
+ backoff_disabled=True
+ save_ansible_output=True
+ proxy_ssh=False
++logstash_hostport=10.100.198.222:5617
+
+project orchestration/xos_services/globalxos/
+diff --git a/xos/synchronizer/globalxos_synchronizer_config b/xos/synchronizer/globalxos_synchronizer_config
+index f06031e..86ae9cc 100644
+--- a/xos/synchronizer/globalxos_synchronizer_config
++++ b/xos/synchronizer/globalxos_synchronizer_config
+@@ -22,7 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
+-
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/hypercache/
+diff --git a/xos/synchronizer/hpc_synchronizer_config b/xos/synchronizer/hpc_synchronizer_config
+index 9d4e70a..015cd77 100644
+--- a/xos/synchronizer/hpc_synchronizer_config
++++ b/xos/synchronizer/hpc_synchronizer_config
+@@ -30,6 +30,7 @@ log_file=console
+ #/var/log/hpc.log
+ driver=None
+ #cmi_hostname=openclouddev0.internet2.edu
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/metro-net/
+diff --git a/xos/synchronizer/metronetwork_synchronizer_config b/xos/synchronizer/metronetwork_synchronizer_config
+index 86847b9..99de62a 100644
+--- a/xos/synchronizer/metronetwork_synchronizer_config
++++ b/xos/synchronizer/metronetwork_synchronizer_config
+@@ -32,6 +32,7 @@ driver=None
+ pretend=False
+ backoff_disabled=True
+ fofum_disabled=True
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/monitoring/
+diff --git a/xos/synchronizer/monitoring_synchronizer_config b/xos/synchronizer/monitoring_synchronizer_config
+index 5cfe5e3..c30361e 100644
+--- a/xos/synchronizer/monitoring_synchronizer_config
++++ b/xos/synchronizer/monitoring_synchronizer_config
+@@ -35,6 +35,7 @@ save_ansible_output=True
+ # set proxy_ssh to false on cloudlab
+ proxy_ssh=False
+ full_setup=True
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/onos-service/
+diff --git a/xos/synchronizer/onos_synchronizer_config b/xos/synchronizer/onos_synchronizer_config
+index c7a2f2c..a7f48ba 100644
+--- a/xos/synchronizer/onos_synchronizer_config
++++ b/xos/synchronizer/onos_synchronizer_config
+@@ -35,6 +35,7 @@ save_ansible_output=True
+ # set proxy_ssh to false on cloudlab
+ proxy_ssh=False
+ full_setup=True
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/openstack/
+diff --git a/xos/synchronizer/openstack_synchronizer_config b/xos/synchronizer/openstack_synchronizer_config
+index 3b7f8e9..29077e2 100644
+--- a/xos/synchronizer/openstack_synchronizer_config
++++ b/xos/synchronizer/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/vsg/
+diff --git a/xos/synchronizer/vsg_synchronizer_config b/xos/synchronizer/vsg_synchronizer_config
+index fd254a2..2a12003 100644
+--- a/xos/synchronizer/vsg_synchronizer_config
++++ b/xos/synchronizer/vsg_synchronizer_config
+@@ -39,6 +39,7 @@ proxy_ssh=True
+ proxy_ssh_key=/opt/cord_profile/node_key
+ proxy_ssh_user=root
+ enable_watchers=True
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+
+project orchestration/xos_services/vtr/
+diff --git a/xos/synchronizer/vtn_vtr_synchronizer_config b/xos/synchronizer/vtn_vtr_synchronizer_config
+index 2c9140a..a10e28b 100644
+--- a/xos/synchronizer/vtn_vtr_synchronizer_config
++++ b/xos/synchronizer/vtn_vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/opt/xos/synchronizers/vtr/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/xos/synchronizer/vtr_synchronizer_config b/xos/synchronizer/vtr_synchronizer_config
+index a8db7ed..4743d65 100644
+--- a/xos/synchronizer/vtr_synchronizer_config
++++ b/xos/synchronizer/vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/opt/cord_profile/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
diff --git a/elk-logger/logstash_tail b/elk-logger/logstash_tail
new file mode 100755
index 0000000..3543011
--- /dev/null
+++ b/elk-logger/logstash_tail
@@ -0,0 +1,161 @@
+#!/usr/bin/python
+
+import pdb
+import argparse
+import sys
+from pyparsing import *
+import json
+import os
+import patterns
+import json
+import logging
+import logstash
+import time
+
+def follow(thefile):
+ while True:
+ line = thefile.readline()
+ if not line:
+ time.sleep(0.1)
+ continue
+ yield line
+
+
+parse = argparse.ArgumentParser(description='Convert log file to json.')
+parse.add_argument('--format', dest='format', action='store',default=None, help='Format e.g. Ansible')
+parse.add_argument('--hostport', dest='hostport', action='store',default=None, help='Logstash UDP host:port')
+parse.add_argument('--file', dest='filename', action='store',default=None, help='Filename to follow')
+
+
+args = parse.parse_args()
+
+host,port = args.hostport.split(':')
+
+format = args.format
+
+logger = logging.getLogger('python-logstash-logger')
+logger.setLevel(logging.INFO)
+logger.addHandler(logstash.LogstashHandler(host, int(port), version=1))
+
+def send_log(log):
+
+ try:
+ msg = log['desc']
+ except KeyError:
+ msg = 'Metadata'
+
+ time.sleep(0.1)
+ try:
+ logger.info(msg, extra=log)
+ except Exception,e:
+ logger.info('Logstash log failed', extra={'xos_fatal':1,'exception':str(e)})
+ pass
+
+### Read parser from patterns.py
+
+def extract_global(l):
+ g = None
+
+ try:
+ global_tag = "==>" + Word(alphanums) + ":" + SkipTo(LineEnd(),include=True)
+ s = global_tag.parseString(l)
+ l = s[3]+'\n'
+ g = s[1]
+ except:
+ pass
+
+ return g,l
+
+def serialize_raw(s):
+ for l in s.splitlines():
+ g,l = extract_global(l)
+ report(l, g)
+
+def report(payload,g):
+ if (not payload):
+ return
+
+ if (type(payload)!=dict):
+ payload = {'desc':payload, 'global_tag': g}
+
+ send_log(payload)
+
+def run_logger():
+ lst = []
+ for n in dir(patterns.Parser):
+ sym = getattr(patterns.Parser, n)
+ if isinstance(sym,ParserElement):
+ lst.append(sym)
+
+ default = SkipTo(LineEnd(),include=True)
+
+ xos_logger = Or(lst)
+
+ inp = ''
+ lc = 0
+
+
+
+ if (args.filename):
+ source = follow(open(args.filename))
+
+ while True:
+ if (not args.filename):
+ l = sys.stdin.readline()
+ else:
+ l = source.next()
+
+ if (l==""):
+ break
+
+ inp += l
+ lc += 1
+
+ if (lc>4):
+ top = inp.splitlines()[0]
+ line_idx = inp.index('\n')
+ inp=inp[line_idx+1:]
+
+ g,top = extract_global(top)
+ report(top,g)
+
+ lc-=1
+
+ try:
+ s = xos_logger.scanString(inp)
+
+ last = None
+ first = True
+ for i in s:
+
+ if (first):
+ pending = inp[:i[1]]
+ serialize_raw(pending)
+
+ first = False
+
+ try:
+ payload = i[0][0]
+ g,_ = extract_global(inp)
+ report(payload,g)
+ except IndexError:
+ pass
+
+ last = i
+
+ if (last):
+ inp = inp[last[2]:]
+ lines = inp.splitlines()
+ lc=len(lines)
+
+ except Exception,e:
+ # We don't want logging to hold up execution
+ #print str(e)
+ pass
+
+def main():
+ run_logger()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/elk-logger/patterns.py b/elk-logger/patterns.py
new file mode 100644
index 0000000..5aff6f2
--- /dev/null
+++ b/elk-logger/patterns.py
@@ -0,0 +1,31 @@
+from pyparsing import *
+
+class Parser:
+ test_pattern = "{{{" + SkipTo("}}}",include=True)
+ test_pattern.setParseAction(lambda x:{"test":x[1][0]})
+
+ test_pattern2 = "1" + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + "2"
+ test_pattern2.setParseAction(lambda x:{"test":x[1]})
+
+ ansible = "TASK [" + SkipTo(']',include=True) + SkipTo(LineEnd(),include=True) + oneOf('changed ok failed error unreachable') + ': ' + SkipTo(LineEnd(),include=True)
+ ansible.setParseAction(lambda x:{"task":x[1][0],"status":x[3],"subject": x[5][0],"ansible":1})
+
+ ansible2 = "TASK [" + SkipTo(']',include=True) + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + oneOf('changed ok failed error unreachable') + ': ' + SkipTo(LineEnd(),include=True)
+ ansible2.setParseAction(lambda x:{"task":x[1][0],"status":x[4],"subject": x[6][0],"ansible":1})
+
+ ansible3 = "TASK [" + SkipTo(']',include=True) + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + oneOf('changed ok failed error unreachable') + ': ' + SkipTo(LineEnd(),include=True)
+ ansible3.setParseAction(lambda x:{"task":x[1][0],"status":x[5],"subject": x[7][0],"ansible":1})
+
+ # Vagrant
+ vagrant_start = Literal("[") + SkipTo("]",include=True) + "Importing base box '" + SkipTo("'",include=True) + SkipTo(LineEnd(),include=True)
+ vagrant_start.setParseAction(lambda x:{"vm":x[1][0],"image":x[3][0],"vagrant":1,"global":"Booting VM"})
+
+ vagrant_start = Literal("[") + SkipTo("]",include=True) + "Machine booted and ready!" + SkipTo(LineEnd(), include=True)
+ vagrant_start.setParseAction(lambda x:{"vm":x[1][0], "vagrant":1, "global":"VM Booted up"})
+
+ gradle_start = "Downloading https://services.gradle.org" + SkipTo(LineEnd(), include=True)
+ gradle_start.setParseAction(lambda x:{"gradle":1, "global":"Gradle started"})
+
+ debian_preparing = "Preparing to unpack" + Word(printables) + SkipTo(LineEnd(), include=True)
+ debian_preparing.setParseAction(lambda x:{"debian":1, "package":x[1]})
+