CORD-705: ELK logging and debugger for CORD
Change-Id: I3406d95161e71976ee3b3bf689f89069b4d9de73
diff --git a/Vagrantfile b/Vagrantfile
index 70fce06..1ce337e 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -10,6 +10,29 @@
end
config.vm.synced_folder '.', '/vagrant', disabled: true
+ config.vm.define "elastic" do |d|
+ d.ssh.forward_agent = true
+ d.vm.box = "ubuntu/trusty64"
+ d.vm.synced_folder '.', '/vagrant', disabled: true
+
+ d.vm.network :forwarded_port, guest: 5601, host: 5601, host_ip: '*'
+ d.vm.network :forwarded_port, guest: 80, host: 9080, host_ip: '*'
+ d.vm.network :forwarded_port, guest: 9200, host: 9200, host_ip: '*'
+ d.vm.network :forwarded_port, guest: 5617, host: 5617, host_ip: '*', protocol: "udp"
+
+ d.vm.hostname = "elastic"
+ d.vm.network "private_network", ip: "10.100.198.222"
+ d.vm.provision :shell, path: "scripts/bootstrap_ansible.sh"
+ d.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /cord/build/ansible/elastic.yml -c local"
+ d.vm.provider "virtualbox" do |v|
+ v.memory = 4096
+ end
+ d.vm.provider :libvirt do |v, override|
+ v.memory = 4096
+ override.vm.synced_folder "..", "/cord", type: "nfs"
+ end
+ end
+
config.vm.define "corddev" do |d|
d.ssh.forward_agent = true
d.vm.box = "ubuntu/trusty64"
diff --git a/ansible/elastic.yml b/ansible/elastic.yml
new file mode 100644
index 0000000..990bc5b
--- /dev/null
+++ b/ansible/elastic.yml
@@ -0,0 +1,7 @@
+- hosts: localhost
+ remote_user: vagrant
+ serial: 1
+ roles:
+ - common
+ - java8-oracle
+ - install_elk
diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml
index 4bc74b9..add8842 100644
--- a/ansible/roles/common/defaults/main.yml
+++ b/ansible/roles/common/defaults/main.yml
@@ -1,6 +1,7 @@
hosts: [
{ host_ip: "10.100.198.200", host_name: "corddev"},
{ host_ip: "10.100.198.201", host_name: "prod"},
+ { host_ip: "10.100.198.222", host_name: "elastic"},
]
use_latest_for:
diff --git a/ansible/roles/install_elk/files/10-udp-input.conf b/ansible/roles/install_elk/files/10-udp-input.conf
new file mode 100644
index 0000000..d144f5b
--- /dev/null
+++ b/ansible/roles/install_elk/files/10-udp-input.conf
@@ -0,0 +1,6 @@
+input {
+ udp {
+ port => 5617
+ codec => json
+ }
+}
diff --git a/ansible/roles/install_elk/files/30-elasticsearch-output.conf b/ansible/roles/install_elk/files/30-elasticsearch-output.conf
new file mode 100644
index 0000000..ab51625
--- /dev/null
+++ b/ansible/roles/install_elk/files/30-elasticsearch-output.conf
@@ -0,0 +1,4 @@
+output {
+ elasticsearch { hosts => ["localhost:9200"] }
+ stdout { codec => rubydebug }
+}
diff --git a/ansible/roles/install_elk/tasks/main.yml b/ansible/roles/install_elk/tasks/main.yml
new file mode 100644
index 0000000..c807425
--- /dev/null
+++ b/ansible/roles/install_elk/tasks/main.yml
@@ -0,0 +1,54 @@
+- name: Add ELK repositories
+ apt_repository: repo={{ item }} state="present"
+ with_items:
+ - deb http://packages.elastic.co/elasticsearch/2.x/debian stable main
+ - deb http://packages.elastic.co/kibana/4.4/debian stable main
+ - deb http://packages.elastic.co/logstash/2.2/debian stable main
+
+- name: Install ELK
+ apt: name={{ item }} state=installed update_cache=yes force=yes
+ with_items:
+ - elasticsearch
+ - kibana
+ - logstash
+
+- name: Configure Logstash
+ copy: src={{ item }} dest=/etc/logstash/conf.d/{{ item }} owner=root group=root mode=0644
+ with_items:
+ - 10-udp-input.conf
+ - 30-elasticsearch-output.conf
+
+- name: Start ELK
+ service: name={{ item }} state=started enabled=yes
+ with_items:
+ - logstash
+ - elasticsearch
+ - kibana
+
+- name: Add ELK repositories
+ apt_repository: repo={{ item }} state="present"
+ with_items:
+ - deb http://packages.elastic.co/elasticsearch/2.x/debian stable main
+ - deb http://packages.elastic.co/kibana/4.4/debian stable main
+ - deb http://packages.elastic.co/logstash/2.2/debian stable main
+
+- name: Install ELK
+ apt: name={{ item }} state=installed update_cache=yes force=yes
+ with_items:
+ - elasticsearch
+ - kibana
+ - logstash
+
+- name: Configure Logstash
+ copy: src={{ item }} dest=/etc/logstash/conf.d/{{ item }} owner=root group=root mode=0644
+ with_items:
+ - 10-udp-input.conf
+ - 30-elasticsearch-output.conf
+
+- name: Start ELK
+ service: name={{ item }} state=started enabled=yes
+ with_items:
+ - logstash
+ - elasticsearch
+ - kibana
+
diff --git a/elk-logger/elk-ciab.diff b/elk-logger/elk-ciab.diff
new file mode 100644
index 0000000..68f8260
--- /dev/null
+++ b/elk-logger/elk-ciab.diff
@@ -0,0 +1,296 @@
+
+project onos-apps/apps/vtn/
+diff --git a/xos/synchronizer/vtn_synchronizer_config b/xos/synchronizer/vtn_synchronizer_config
+index dad5cb6..f317dee 100644
+--- a/xos/synchronizer/vtn_synchronizer_config
++++ b/xos/synchronizer/vtn_synchronizer_config
+@@ -33,6 +33,7 @@ driver=openstack
+ pretend=False
+ backoff_disabled=True
+ enable_watchers=True
++logstash_hostport=10.100.198.222:5617
+
+ [nova]
+ ca_ssl_cert=/etc/ssl/certs/ca-certificates.crt
+
+project orchestration/service-profile/
+diff --git a/cord-pod-ansible/files/monitoring_synchronizer_config b/cord-pod-ansible/files/monitoring_synchronizer_config
+index 1ee96f0..932a32b 100644
+--- a/cord-pod-ansible/files/monitoring_synchronizer_config
++++ b/cord-pod-ansible/files/monitoring_synchronizer_config
+@@ -37,6 +37,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/cord-pod-ansible/files/vcpe_synchronizer_config b/cord-pod-ansible/files/vcpe_synchronizer_config
+index 9da6ede..9ef259b 100644
+--- a/cord-pod-ansible/files/vcpe_synchronizer_config
++++ b/cord-pod-ansible/files/vcpe_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/cord-pod-ansible/files/vtr_synchronizer_config b/cord-pod-ansible/files/vtr_synchronizer_config
+index 223ab00..4e3008c 100644
+--- a/cord-pod-ansible/files/vtr_synchronizer_config
++++ b/cord-pod-ansible/files/vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/cord-pod/files/monitoring_synchronizer_config b/cord-pod/files/monitoring_synchronizer_config
+index cfd73a0..485f993 100644
+--- a/cord-pod/files/monitoring_synchronizer_config
++++ b/cord-pod/files/monitoring_synchronizer_config
+@@ -37,6 +37,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/cord-pod/files/openstack_synchronizer_config b/cord-pod/files/openstack_synchronizer_config
+index 3b7f8e9..29077e2 100644
+--- a/cord-pod/files/openstack_synchronizer_config
++++ b/cord-pod/files/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/cord-pod/files/vcpe_synchronizer_config b/cord-pod/files/vcpe_synchronizer_config
+index babe94f..34e0f2b 100644
+--- a/cord-pod/files/vcpe_synchronizer_config
++++ b/cord-pod/files/vcpe_synchronizer_config
+@@ -39,6 +39,7 @@ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
+ enable_watchers=True
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/cord-pod/files/vtr_synchronizer_config b/cord-pod/files/vtr_synchronizer_config
+index 3d3209c..56af7fc 100644
+--- a/cord-pod/files/vtr_synchronizer_config
++++ b/cord-pod/files/vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/mcord/files/monitoring_synchronizer_config b/mcord/files/monitoring_synchronizer_config
+index cfd73a0..485f993 100644
+--- a/mcord/files/monitoring_synchronizer_config
++++ b/mcord/files/monitoring_synchronizer_config
+@@ -37,6 +37,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/root/setup/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/mcord/files/openstack_synchronizer_config b/mcord/files/openstack_synchronizer_config
+index 82c2f79..58dece3 100644
+--- a/mcord/files/openstack_synchronizer_config
++++ b/mcord/files/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=False
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+diff --git a/opencloud/files/openstack_synchronizer_config b/opencloud/files/openstack_synchronizer_config
+index 74d68e3..8b0f8f9 100644
+--- a/opencloud/files/openstack_synchronizer_config
++++ b/opencloud/files/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos/
+diff --git a/xos/synchronizers/onboarding/onboarding_synchronizer_config b/xos/synchronizers/onboarding/onboarding_synchronizer_config
+index fb33044..6877378 100644
+--- a/xos/synchronizers/onboarding/onboarding_synchronizer_config
++++ b/xos/synchronizers/onboarding/onboarding_synchronizer_config
+@@ -32,4 +32,4 @@ backoff_disabled=True
+ pretend=False
+ save_ansible_output=True
+ node_key=/opt/cord_profile/node_key
+-
++logstash_hostport=10.100.198.222:5617
+diff --git a/xos/synchronizers/syndicate/syndicate_synchronizer_config b/xos/synchronizers/syndicate/syndicate_synchronizer_config
+index 7c9d2d2..c9f3cf5 100644
+--- a/xos/synchronizers/syndicate/syndicate_synchronizer_config
++++ b/xos/synchronizers/syndicate/syndicate_synchronizer_config
+@@ -29,6 +29,7 @@ steps_dir=/opt/xos/synchronizers/syndicate/steps
+ deleters_dir=/opt/xos/synchronizers/syndicate/deleters
+ log_file=console
+ driver=None
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/fabric/
+diff --git a/xos/synchronizer/fabric_synchronizer_config b/xos/synchronizer/fabric_synchronizer_config
+index e98abe5..b15fcc3 100644
+--- a/xos/synchronizer/fabric_synchronizer_config
++++ b/xos/synchronizer/fabric_synchronizer_config
+@@ -21,3 +21,4 @@ pretend=False
+ backoff_disabled=True
+ save_ansible_output=True
+ proxy_ssh=False
++logstash_hostport=10.100.198.222:5617
+
+project orchestration/xos_services/globalxos/
+diff --git a/xos/synchronizer/globalxos_synchronizer_config b/xos/synchronizer/globalxos_synchronizer_config
+index f06031e..86ae9cc 100644
+--- a/xos/synchronizer/globalxos_synchronizer_config
++++ b/xos/synchronizer/globalxos_synchronizer_config
+@@ -22,7 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
+-
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/hypercache/
+diff --git a/xos/synchronizer/hpc_synchronizer_config b/xos/synchronizer/hpc_synchronizer_config
+index 9d4e70a..015cd77 100644
+--- a/xos/synchronizer/hpc_synchronizer_config
++++ b/xos/synchronizer/hpc_synchronizer_config
+@@ -30,6 +30,7 @@ log_file=console
+ #/var/log/hpc.log
+ driver=None
+ #cmi_hostname=openclouddev0.internet2.edu
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/metro-net/
+diff --git a/xos/synchronizer/metronetwork_synchronizer_config b/xos/synchronizer/metronetwork_synchronizer_config
+index 86847b9..99de62a 100644
+--- a/xos/synchronizer/metronetwork_synchronizer_config
++++ b/xos/synchronizer/metronetwork_synchronizer_config
+@@ -32,6 +32,7 @@ driver=None
+ pretend=False
+ backoff_disabled=True
+ fofum_disabled=True
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/monitoring/
+diff --git a/xos/synchronizer/monitoring_synchronizer_config b/xos/synchronizer/monitoring_synchronizer_config
+index 5cfe5e3..c30361e 100644
+--- a/xos/synchronizer/monitoring_synchronizer_config
++++ b/xos/synchronizer/monitoring_synchronizer_config
+@@ -35,6 +35,7 @@ save_ansible_output=True
+ # set proxy_ssh to false on cloudlab
+ proxy_ssh=False
+ full_setup=True
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/onos-service/
+diff --git a/xos/synchronizer/onos_synchronizer_config b/xos/synchronizer/onos_synchronizer_config
+index c7a2f2c..a7f48ba 100644
+--- a/xos/synchronizer/onos_synchronizer_config
++++ b/xos/synchronizer/onos_synchronizer_config
+@@ -35,6 +35,7 @@ save_ansible_output=True
+ # set proxy_ssh to false on cloudlab
+ proxy_ssh=False
+ full_setup=True
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/openstack/
+diff --git a/xos/synchronizer/openstack_synchronizer_config b/xos/synchronizer/openstack_synchronizer_config
+index 3b7f8e9..29077e2 100644
+--- a/xos/synchronizer/openstack_synchronizer_config
++++ b/xos/synchronizer/openstack_synchronizer_config
+@@ -22,6 +22,7 @@ save_ansible_output=True
+ backoff_disabled=True
+ pretend=False
+ images_directory=/opt/xos/images
++logstash_hostport=10.100.198.222:5617
+
+ [feefie]
+ client_id='vicci_dev_central'
+
+project orchestration/xos_services/vsg/
+diff --git a/xos/synchronizer/vsg_synchronizer_config b/xos/synchronizer/vsg_synchronizer_config
+index fd254a2..2a12003 100644
+--- a/xos/synchronizer/vsg_synchronizer_config
++++ b/xos/synchronizer/vsg_synchronizer_config
+@@ -39,6 +39,7 @@ proxy_ssh=True
+ proxy_ssh_key=/opt/cord_profile/node_key
+ proxy_ssh_user=root
+ enable_watchers=True
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+
+project orchestration/xos_services/vtr/
+diff --git a/xos/synchronizer/vtn_vtr_synchronizer_config b/xos/synchronizer/vtn_vtr_synchronizer_config
+index 2c9140a..a10e28b 100644
+--- a/xos/synchronizer/vtn_vtr_synchronizer_config
++++ b/xos/synchronizer/vtn_vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/opt/xos/synchronizers/vtr/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
+diff --git a/xos/synchronizer/vtr_synchronizer_config b/xos/synchronizer/vtr_synchronizer_config
+index a8db7ed..4743d65 100644
+--- a/xos/synchronizer/vtr_synchronizer_config
++++ b/xos/synchronizer/vtr_synchronizer_config
+@@ -38,6 +38,7 @@ full_setup=True
+ proxy_ssh=True
+ proxy_ssh_key=/opt/cord_profile/node_key
+ proxy_ssh_user=root
++logstash_hostport=10.100.198.222:5617
+
+ [networking]
+ use_vtn=True
diff --git a/elk-logger/logstash_tail b/elk-logger/logstash_tail
new file mode 100755
index 0000000..3543011
--- /dev/null
+++ b/elk-logger/logstash_tail
@@ -0,0 +1,161 @@
+#!/usr/bin/python
+
+import pdb
+import argparse
+import sys
+from pyparsing import *
+import json
+import os
+import patterns
+import json
+import logging
+import logstash
+import time
+
+def follow(thefile):
+ while True:
+ line = thefile.readline()
+ if not line:
+ time.sleep(0.1)
+ continue
+ yield line
+
+
+parse = argparse.ArgumentParser(description='Convert log file to json.')
+parse.add_argument('--format', dest='format', action='store',default=None, help='Format e.g. Ansible')
+parse.add_argument('--hostport', dest='hostport', action='store',default=None, help='Logstash UDP host:port')
+parse.add_argument('--file', dest='filename', action='store',default=None, help='Filename to follow')
+
+
+args = parse.parse_args()
+
+host,port = args.hostport.split(':')
+
+format = args.format
+
+logger = logging.getLogger('python-logstash-logger')
+logger.setLevel(logging.INFO)
+logger.addHandler(logstash.LogstashHandler(host, int(port), version=1))
+
+def send_log(log):
+
+ try:
+ msg = log['desc']
+ except KeyError:
+ msg = 'Metadata'
+
+ time.sleep(0.1)
+ try:
+ logger.info(msg, extra=log)
+ except Exception,e:
+ logger.info('Logstash log failed', extra={'xos_fatal':1,'exception':str(e)})
+ pass
+
+### Read parser from patterns.py
+
+def extract_global(l):
+ g = None
+
+ try:
+ global_tag = "==>" + Word(alphanums) + ":" + SkipTo(LineEnd(),include=True)
+ s = global_tag.parseString(l)
+ l = s[3]+'\n'
+ g = s[1]
+ except:
+ pass
+
+ return g,l
+
+def serialize_raw(s):
+ for l in s.splitlines():
+ g,l = extract_global(l)
+ report(l, g)
+
+def report(payload,g):
+ if (not payload):
+ return
+
+ if (type(payload)!=dict):
+ payload = {'desc':payload, 'global_tag': g}
+
+ send_log(payload)
+
+def run_logger():
+ lst = []
+ for n in dir(patterns.Parser):
+ sym = getattr(patterns.Parser, n)
+ if isinstance(sym,ParserElement):
+ lst.append(sym)
+
+ default = SkipTo(LineEnd(),include=True)
+
+ xos_logger = Or(lst)
+
+ inp = ''
+ lc = 0
+
+
+
+ if (args.filename):
+ source = follow(open(args.filename))
+
+ while True:
+ if (not args.filename):
+ l = sys.stdin.readline()
+ else:
+ l = source.next()
+
+ if (l==""):
+ break
+
+ inp += l
+ lc += 1
+
+ if (lc>4):
+ top = inp.splitlines()[0]
+ line_idx = inp.index('\n')
+ inp=inp[line_idx+1:]
+
+ g,top = extract_global(top)
+ report(top,g)
+
+ lc-=1
+
+ try:
+ s = xos_logger.scanString(inp)
+
+ last = None
+ first = True
+ for i in s:
+
+ if (first):
+ pending = inp[:i[1]]
+ serialize_raw(pending)
+
+ first = False
+
+ try:
+ payload = i[0][0]
+ g,_ = extract_global(inp)
+ report(payload,g)
+ except IndexError:
+ pass
+
+ last = i
+
+ if (last):
+ inp = inp[last[2]:]
+ lines = inp.splitlines()
+ lc=len(lines)
+
+ except Exception,e:
+ # We don't want logging to hold up execution
+ #print str(e)
+ pass
+
+def main():
+ run_logger()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/elk-logger/patterns.py b/elk-logger/patterns.py
new file mode 100644
index 0000000..5aff6f2
--- /dev/null
+++ b/elk-logger/patterns.py
@@ -0,0 +1,31 @@
+from pyparsing import *
+
+class Parser:
+ test_pattern = "{{{" + SkipTo("}}}",include=True)
+ test_pattern.setParseAction(lambda x:{"test":x[1][0]})
+
+ test_pattern2 = "1" + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + "2"
+ test_pattern2.setParseAction(lambda x:{"test":x[1]})
+
+ ansible = "TASK [" + SkipTo(']',include=True) + SkipTo(LineEnd(),include=True) + oneOf('changed ok failed error unreachable') + ': ' + SkipTo(LineEnd(),include=True)
+ ansible.setParseAction(lambda x:{"task":x[1][0],"status":x[3],"subject": x[5][0],"ansible":1})
+
+ ansible2 = "TASK [" + SkipTo(']',include=True) + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + oneOf('changed ok failed error unreachable') + ': ' + SkipTo(LineEnd(),include=True)
+ ansible2.setParseAction(lambda x:{"task":x[1][0],"status":x[4],"subject": x[6][0],"ansible":1})
+
+ ansible3 = "TASK [" + SkipTo(']',include=True) + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + SkipTo(LineEnd(),include=True) + oneOf('changed ok failed error unreachable') + ': ' + SkipTo(LineEnd(),include=True)
+ ansible3.setParseAction(lambda x:{"task":x[1][0],"status":x[5],"subject": x[7][0],"ansible":1})
+
+ # Vagrant
+ vagrant_start = Literal("[") + SkipTo("]",include=True) + "Importing base box '" + SkipTo("'",include=True) + SkipTo(LineEnd(),include=True)
+ vagrant_start.setParseAction(lambda x:{"vm":x[1][0],"image":x[3][0],"vagrant":1,"global":"Booting VM"})
+
+ vagrant_start = Literal("[") + SkipTo("]",include=True) + "Machine booted and ready!" + SkipTo(LineEnd(), include=True)
+ vagrant_start.setParseAction(lambda x:{"vm":x[1][0], "vagrant":1, "global":"VM Booted up"})
+
+ gradle_start = "Downloading https://services.gradle.org" + SkipTo(LineEnd(), include=True)
+ gradle_start.setParseAction(lambda x:{"gradle":1, "global":"Gradle started"})
+
+ debian_preparing = "Preparing to unpack" + Word(printables) + SkipTo(LineEnd(), include=True)
+ debian_preparing.setParseAction(lambda x:{"debian":1, "package":x[1]})
+
diff --git a/scripts/cord-in-a-box.sh b/scripts/cord-in-a-box.sh
index b018f22..e984f47 100755
--- a/scripts/cord-in-a-box.sh
+++ b/scripts/cord-in-a-box.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-set -e
+set -e -x
CORDDIR=~/cord
VMDIR=/cord/build/
@@ -11,11 +11,20 @@
REPO_BRANCH="master"
VERSION_STRING="CiaB development version"
+exec > >(tee -i $CORDDIR/install.out)
+exec 2>&1
+
function add_box() {
vagrant box list | grep $1 | grep virtualbox || vagrant box add $1
vagrant box list | grep $1 | grep libvirt || vagrant mutate $1 libvirt --input-provider virtualbox
}
+function run_stage {
+ echo "==> "$1": Starting"
+ $1
+ echo "==> "$1": Complete"
+}
+
function cleanup_from_previous_test() {
echo "## Cleanup ##"
@@ -36,7 +45,8 @@
sudo apt-get update
[ -e vagrant_1.8.5_x86_64.deb ] || wget https://releases.hashicorp.com/vagrant/1.8.5/vagrant_1.8.5_x86_64.deb
dpkg -l vagrant || sudo dpkg -i vagrant_1.8.5_x86_64.deb
- sudo apt-get -y install qemu-kvm libvirt-bin libvirt-dev curl nfs-kernel-server git build-essential
+ sudo apt-get -y install qemu-kvm libvirt-bin libvirt-dev curl nfs-kernel-server git build-essential python-pip
+ sudo pip install pyparsing python-logstash
[ -e ~/.ssh/id_rsa ] || ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
@@ -98,6 +108,25 @@
fi
}
+function elk_up() {
+ cd $CORDDIR/build
+
+ sudo su $USER -c 'vagrant up elastic --provider libvirt'
+
+ # This is a workaround for a weird issue with ARP cache timeout breaking 'vagrant ssh'
+ # It allows SSH'ing to the machine via 'ssh corddev'
+ sudo su $USER -c "vagrant ssh-config elastic > $SSHCONFIG"
+
+ cd $CORDDIR
+ if [ ! -f /tmp/elk-patch-applied ]; then
+ touch /tmp/elk-patch-applied
+ sh build/scripts/repo-apply.sh build/elk-logger/elk-ciab.diff
+ fi
+
+ sudo chmod +x build/elk-logger/logstash_tail
+ build/elk-logger/logstash_tail --file install.out --hostport 10.100.198.222:5617 &
+}
+
function vagrant_vms_up() {
cd $CORDDIR/build
@@ -256,16 +285,17 @@
# What to do
if [[ $CLEANUP -eq 1 ]]
then
- cleanup_from_previous_test
+ run_stage cleanup_from_previous_test
fi
echo ""
echo "Preparing to install $VERSION_STRING ($REPO_BRANCH branch)"
echo ""
-bootstrap
-cloudlab_setup
-vagrant_vms_up
+run_stage bootstrap
+run_stage cloudlab_setup
+run_stage elk_up
+run_stage vagrant_vms_up
if [[ $SETUP_ONLY -ne 0 ]]
then
@@ -273,9 +303,9 @@
exit 0
fi
-install_head_node
-set_up_maas_user
-leaf_spine_up
+run_stage install_head_node
+run_stage set_up_maas_user
+run_stage leaf_spine_up
if [[ $NUM_COMPUTE_NODES -gt 3 ]]
then
@@ -283,11 +313,13 @@
NUM_COMPUTE_NODES=3
fi
+echo "==> Adding compute nodes: Starting"
for i in `seq 1 $NUM_COMPUTE_NODES`;
do
echo adding the compute node: compute-node-$i
add_compute_node compute-node-$i build_compute-node-$i
done
+echo "==> Adding compute nodes: Complete"
# run diagnostics both before/after the fabric/e2e tests
if [[ $DIAGNOSTICS -eq 1 ]]
@@ -298,17 +330,17 @@
if [[ $FABRIC -ne 0 ]]
then
- initialize_fabric
+ run_stage initialize_fabric
fi
if [[ $RUN_TEST -eq 1 ]]
then
- run_e2e_test
+ run_stage run_e2e_test
fi
if [[ $DIAGNOSTICS -eq 1 ]]
then
- run_diagnostics
+ run_stage run_diagnostics
fi
exit 0
diff --git a/scripts/repo-apply.sh b/scripts/repo-apply.sh
new file mode 100755
index 0000000..f3a5662
--- /dev/null
+++ b/scripts/repo-apply.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+## Script to patch up diff reated by `repo diff`
+
+# from https://groups.google.com/d/msg/repo-discuss/43juvD1qGIQ/7maptZVcEjsJ
+if [ -z "$1" ] || [ ! -e "$1" ]; then
+ echo "Usages: $0 <repo_diff_file>";
+ exit 0;
+fi
+
+rm -fr _tmp_splits*
+cat $1 | csplit -qf '' -b "_tmp_splits.%d.diff" - '/^project.*\/$/' '{*}'
+
+working_dir=`pwd`
+
+for proj_diff in `ls _tmp_splits.*.diff`
+do
+ chg_dir=`cat $proj_diff | grep '^project.*\/$' | cut -d " " -f 2`
+ echo "FILE: $proj_diff $chg_dir"
+ if [ -e $chg_dir ]; then
+ ( cd $chg_dir; \
+ cat $working_dir/$proj_diff | grep -v '^project.*\/$' | patch -Np1;);
+ else
+ echo "$0: Project directory $chg_dir don't exists.";
+ fi
+done