Test:Provide a setup phase for cord-tester which just starts/provisions required containers for testing.
It also starts a container management rpc server for remote container restarts with configuration from tests.
Use simple xmlrpc server for container management over our own in cord-test server.
Add dependencies pertaining to the new change in tester for the dockerfile.
This way, we now can setup the cord-tester in one node and launch the test from a test container in another node in the podd.

Change-Id: Ie99540e5455f46ee515c7c5341af7ec94892e438
diff --git a/Dockerfile.tester b/Dockerfile.tester
index e36faad..6cc827b 100644
--- a/Dockerfile.tester
+++ b/Dockerfile.tester
@@ -15,7 +15,7 @@
  cd openvswitch-2.5.0 && \
  ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --disable-ssl && make && make install)
 RUN service openvswitch-switch restart || /bin/true
-RUN pip install -U scapy scapy-ssl_tls monotonic configObj docker-py pyyaml nsenter pyroute2 netaddr
+RUN pip install -U scapy scapy-ssl_tls monotonic configObj docker-py pyyaml nsenter pyroute2 netaddr python-daemon
 RUN mv /usr/sbin/tcpdump /sbin/
 RUN ln -sf /sbin/tcpdump /usr/sbin/tcpdump
 WORKDIR /root
diff --git a/src/test/setup/cord-test.py b/src/test/setup/cord-test.py
index 5b44999..29f4ee1 100755
--- a/src/test/setup/cord-test.py
+++ b/src/test/setup/cord-test.py
@@ -22,7 +22,7 @@
 from OltConfig import OltConfig
 from threadPool import ThreadPool
 from CordContainer import *
-from CordTestServer import cord_test_server_start, cord_test_server_stop
+from CordTestServer import cord_test_server_start, cord_test_server_stop, CORD_TEST_HOST, CORD_TEST_PORT
 
 class CordTester(Container):
     sandbox = '/root/test'
@@ -164,7 +164,7 @@
  cd openvswitch-{} && \
  ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --disable-ssl && make && make install)
 RUN service openvswitch-switch restart || /bin/true
-RUN pip install -U scapy scapy-ssl_tls monotonic configObj docker-py pyyaml nsenter pyroute2 netaddr
+RUN pip install -U scapy scapy-ssl_tls monotonic configObj docker-py pyyaml nsenter pyroute2 netaddr python-daemon
 RUN mv /usr/sbin/tcpdump /sbin/
 RUN ln -sf /sbin/tcpdump /usr/sbin/tcpdump
 WORKDIR /root
@@ -221,10 +221,21 @@
 onos_app_version = '2.0-SNAPSHOT'
 cord_tester_base = os.path.dirname(os.path.realpath(__file__))
 onos_app_file = os.path.abspath('{0}/../apps/ciena-cordigmp-'.format(cord_tester_base) + onos_app_version + '.oar')
+cord_test_server_address = '{}:{}'.format(CORD_TEST_HOST, CORD_TEST_PORT)
 
 def runTest(args):
     #Start the cord test tcp server
-    test_server = cord_test_server_start()
+    test_server_params = args.server.split(':')
+    test_host = test_server_params[0]
+    test_port = CORD_TEST_PORT
+    if len(test_server_params) > 1:
+        test_port = int(test_server_params[1])
+    try:
+        test_server = cord_test_server_start(daemonize = False, cord_test_host = test_host, cord_test_port = test_port)
+    except:
+        ##Most likely a server instance is already running (daemonized earlier)
+        test_server = None
+
     test_containers = []
     #These tests end up restarting ONOS/quagga/radius
     tests_exempt = ('vrouter',)
@@ -278,9 +289,12 @@
         quagga = Quagga(update = update_map['quagga'])
         quagga_ip = quagga.ip()
 
+
     test_cnt_env = { 'ONOS_CONTROLLER_IP' : onos_ip,
                      'ONOS_AAA_IP' : radius_ip if radius_ip is not None else '',
                      'QUAGGA_IP': quagga_ip if quagga_ip is not None else '',
+                     'CORD_TEST_HOST' : test_host,
+                     'CORD_TEST_PORT' : test_port,
                    }
     if args.olt:
         olt_conf_test_loc = os.path.join(CordTester.sandbox_setup, 'olt_config.json')
@@ -328,7 +342,64 @@
             test_cnt.setup_intfs(port_num = port_num)
         test_cnt.run_tests()
 
-    cord_test_server_stop(test_server)
+    if test_server:
+        cord_test_server_stop(test_server)
+
+##Starts onos/radius/quagga containers as appropriate
+def setupCordTester(args):
+    onos_cnt = {'tag':'latest'}
+    update_map = { 'quagga' : False, 'radius' : False }
+    update_map[args.update.lower()] = True
+
+    if args.update.lower() == 'all':
+       for c in update_map.keys():
+           update_map[c] = True
+
+    onos_ip = None
+    radius_ip = None
+    quagga_ip = None
+
+    ##If onos/radius was already started
+    if args.test_controller:
+        ips = args.test_controller.split('/')
+        onos_ip = ips[0]
+        if len(ips) > 1:
+            radius_ip = ips[1]
+        else:
+            radius_ip = None
+
+    #don't spawn onos if the user had started it externally
+    onos_cnt['image'] = args.onos.split(':')[0]
+    if args.onos.find(':') >= 0:
+        onos_cnt['tag'] = args.onos.split(':')[1]
+
+    if onos_ip is None:
+        onos = Onos(image = onos_cnt['image'], tag = onos_cnt['tag'], boot_delay = 60)
+        onos_ip = onos.ip()
+
+    ##Start Radius container if not started
+    if radius_ip is None:
+        radius = Radius( update = update_map['radius'])
+        radius_ip = radius.ip()
+
+    print('Radius server running with IP %s' %radius_ip)
+    print('Onos IP %s' %onos_ip)
+    print('Installing cord tester ONOS app %s' %onos_app_file)
+    OnosCtrl.install_app(args.app, onos_ip = onos_ip)
+
+    if args.quagga == True:
+        #Start quagga. Builds container if required
+        quagga = Quagga(update = update_map['quagga'])
+        quagga_ip = quagga.ip()
+        print('Quagga running with IP %s' %quagga_ip)
+
+    #Finally start the test server and daemonize
+    params = args.server.split(':')
+    ip = params[0]
+    port = CORD_TEST_PORT
+    if len(params) > 1:
+        port = int(params[1])
+    cord_test_server_start(daemonize = True, cord_test_host = ip, cord_test_port = port)
 
 def cleanupTests(args):
     test_container = '{}:latest'.format(CordTester.IMAGE)
@@ -363,6 +434,8 @@
     parser_run.add_argument('-p', '--olt', action='store_true', help='Use OLT config')
     parser_run.add_argument('-e', '--test-controller', default='', type=str, help='External test controller ip for Onos and/or radius server. '
                         'Eg: 10.0.0.2/10.0.0.3 to specify ONOS and Radius ip to connect')
+    parser_run.add_argument('-r', '--server', default=cord_test_server_address, type=str,
+                            help='ip:port address to connect for cord test server for container requests')
     parser_run.add_argument('-k', '--keep', action='store_true', help='Keep test container after tests')
     parser_run.add_argument('-s', '--start-switch', action='store_true', help='Start OVS when running under OLT config')
     parser_run.add_argument('-u', '--update', default='none', choices=['test','quagga','radius', 'all'], type=str, help='Update cord tester container images. '
@@ -374,6 +447,21 @@
                             help='Specify number of test containers to spawn for tests')
     parser_run.set_defaults(func=runTest)
 
+
+    parser_setup = subparser.add_parser('setup', help='Setup cord tester environment')
+    parser_setup.add_argument('-o', '--onos', default=onos_image_default, type=str, help='ONOS container image')
+    parser_setup.add_argument('-r', '--server', default=cord_test_server_address, type=str,
+                              help='ip:port address for cord test server to listen for container restart requests')
+    parser_setup.add_argument('-q', '--quagga',action='store_true',help='Provision quagga container for vrouter')
+    parser_setup.add_argument('-a', '--app', default=onos_app_file, type=str, help='Cord ONOS app filename')
+    parser_setup.add_argument('-e', '--test-controller', default='', type=str, help='External test controller ip for Onos and/or radius server. '
+                        'Eg: 10.0.0.2/10.0.0.3 to specify ONOS and Radius ip to connect')
+    parser_setup.add_argument('-u', '--update', default='none', choices=['quagga','radius', 'all'], type=str, help='Update cord tester container images. '
+                        'Eg: --update=quagga to rebuild quagga image.'
+                        '    --update=radius to rebuild radius server image.'
+                        '    --update=all to rebuild all cord tester images.')
+    parser_setup.set_defaults(func=setupCordTester)
+
     parser_list = subparser.add_parser('list', help='List test cases')
     parser_list.add_argument('-t', '--test', default='all', help='Specify test type to list test cases. '
                              'Eg: -t tls to list tls test cases.'
diff --git a/src/test/utils/CordContainer.py b/src/test/utils/CordContainer.py
index dcc5de4..f22dbfd 100644
--- a/src/test/utils/CordContainer.py
+++ b/src/test/utils/CordContainer.py
@@ -231,7 +231,7 @@
             for _,g in self.host_guest_map:
                 volumes.append(g)
             if network_cfg is not None:
-                json_data = json.dumps(network_cfg)
+                json_data = json.dumps(network_cfg, indent=4)
                 with open('{}/network-cfg.json'.format(self.host_config_dir), 'w') as f:
                     f.write(json_data)
             print('Starting ONOS container %s' %self.name)
@@ -358,3 +358,7 @@
 '''.format(onos_quagga_ip)
         super(Quagga, cls).build_image(dockerfile, image)
         print('Done building image %s' %image)
+
+def reinitContainerClients():
+    docker_netns.dckr = Client()
+    Container.dckr = Client()
diff --git a/src/test/utils/CordTestServer.py b/src/test/utils/CordTestServer.py
index 2037d71..1889536 100644
--- a/src/test/utils/CordTestServer.py
+++ b/src/test/utils/CordTestServer.py
@@ -1,80 +1,99 @@
-# 
+#
 # Copyright 2016-present Ciena Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import SocketServer as socketserver
-import threading
-import socket
-from CordContainer import Onos, Quagga
+from CordContainer import Container, Onos, Quagga, Radius, reinitContainerClients
 from nose.tools import nottest
+from SimpleXMLRPCServer import SimpleXMLRPCServer
+import daemon
+import xmlrpclib
+import os
+import json
+import time
+import threading
 
-##Server to handle container restart requests from test container.
+##Server to handle container restart/stop requests from test container.
 ##Used now to restart ONOS from vrouter test container
 
 CORD_TEST_HOST = '172.17.0.1'
 CORD_TEST_PORT = 25000
 
-class CordTestServer(socketserver.BaseRequestHandler):
+class QuaggaStopWrapper(Container):
+    def __init__(self, name = Quagga.NAME, image = Quagga.IMAGE, tag = 'latest'):
+        super(QuaggaStopWrapper, self).__init__(name, image, tag = tag)
+        if self.exists():
+            self.kill()
 
-    def restart_onos(self, *args):
+class CordTestServer(object):
+
+    def __restart_onos(self, config = None):
+        onos_config = '{}/network-cfg.json'.format(Onos.host_config_dir)
+        if config is None:
+            try:
+                os.unlink(onos_config)
+            except:
+                pass
         print('Restarting ONOS')
-        onos = Onos(restart = True)
-        self.request.sendall('DONE')
+        Onos(restart = True, network_cfg = config)
+        return 'DONE'
 
-    def restart_quagga(self, *args):
+    def restart_onos(self, kwargs):
+        return self.__restart_onos(**kwargs)
+
+    def __restart_quagga(self, config = None, boot_delay = 30 ):
         config_file = Quagga.quagga_config_file
-        boot_delay = 15
-        if args:
-            config_file = args[0]
-            if len(args) > 1:
-                boot_delay = int(args[1])
-        print('Restarting QUAGGA with config file %s, delay %d secs'%(config_file, boot_delay))
-        quagga = Quagga(restart = True, config_file = config_file, boot_delay = boot_delay)
-        self.request.sendall('DONE')
+        if config is not None:
+            quagga_config = '{}/testrib_gen.conf'.format(Quagga.host_quagga_config)
+            config_file = '{}/testrib_gen.conf'.format(Quagga.guest_quagga_config)
+            with open(quagga_config, 'w+') as fd:
+                fd.write(str(config))
+        print('Restarting QUAGGA with config file %s, delay %d' %(config_file, boot_delay))
+        Quagga(restart = True, config_file = config_file, boot_delay = boot_delay)
+        return 'DONE'
 
-    def restart_radius(self, *args):
-        print('Restarting RADIUS Server')
-        radius = Radius(restart = True)
-        self.request.sendall('DONE')
+    def restart_quagga(self, kwargs):
+        return self.__restart_quagga(**kwargs)
 
-    callback_table = { 'RESTART_ONOS' : restart_onos,
-                       'RESTART_QUAGGA' : restart_quagga,
-                       'RESTART_RADIUS' : restart_radius,
-                     }
-
-    def handle(self):
-        data = self.request.recv(1024).strip()
-        cmd = data.split()[0]
+    def stop_quagga(self):
+        quaggaStop = QuaggaStopWrapper()
+        time.sleep(2)
         try:
-            #args = ' '.join(data.split()[1:])
-            args = data.split()[1:]
-        except:
-            args = None
+            quagga_config_gen = '{}/testrib_gen.conf'.format(Quagga.host_quagga_config)
+            os.unlink(quagga_config_gen)
+        except: pass
+        return 'DONE'
 
-        if self.callback_table.has_key(cmd):
-            self.callback_table[cmd](self, *args)
-
-class ThreadedTestServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
-    allow_reuse_address = True
+    def restart_radius(self):
+        print('Restarting RADIUS Server')
+        Radius(restart = True)
+        return 'DONE'
 
 @nottest
-def cord_test_server_start():
-    server = ThreadedTestServer( (CORD_TEST_HOST, CORD_TEST_PORT), CordTestServer)
-    task = threading.Thread(target = server.serve_forever)
-    ##terminate when main thread exits
-    task.daemon = True
-    task.start()
+def cord_test_server_start(daemonize = True, cord_test_host = CORD_TEST_HOST, cord_test_port = CORD_TEST_PORT):
+    server = SimpleXMLRPCServer( (cord_test_host, cord_test_port) )
+    server.register_instance(CordTestServer())
+    if daemonize is True:
+        d = daemon.DaemonContext(files_preserve = [server],
+                                 detach_process = True)
+        with d:
+            reinitContainerClients()
+            server.serve_forever()
+    else:
+        task = threading.Thread(target = server.serve_forever)
+        ##terminate when main thread exits
+        task.daemon = True
+        task.start()
     return server
 
 @nottest
@@ -83,27 +102,44 @@
     server.server_close()
 
 @nottest
-def cord_test_onos_restart():
+def get_cord_test_loc():
+    host = os.getenv('CORD_TEST_HOST', CORD_TEST_HOST)
+    port = int(os.getenv('CORD_TEST_PORT', CORD_TEST_PORT))
+    return host, port
+
+def rpc_server_instance():
+    '''Stateless'''
+    host, port = get_cord_test_loc()
+    rpc_server = 'http://{}:{}'.format(host, port)
+    return xmlrpclib.Server(rpc_server, allow_none = True)
+
+@nottest
+def __cord_test_onos_restart(**kwargs):
+    return rpc_server_instance().restart_onos(kwargs)
+
+@nottest
+def cord_test_onos_restart(config = None):
     '''Send ONOS restart to server'''
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.connect( (CORD_TEST_HOST, CORD_TEST_PORT) )
-    s.sendall('RESTART_ONOS\n')
-    data = s.recv(1024).strip()
-    s.close()
+    data = __cord_test_onos_restart(config = config)
     if data == 'DONE':
         return True
     return False
 
 @nottest
-def cord_test_quagga_restart(config_file = None, boot_delay = 30):
+def __cord_test_quagga_restart(**kwargs):
+    return rpc_server_instance().restart_quagga(kwargs)
+
+@nottest
+def cord_test_quagga_restart(config = None, boot_delay = 30):
     '''Send QUAGGA restart to server'''
-    if config_file is None:
-        config_file = Quagga.quagga_config_file
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.connect( (CORD_TEST_HOST, CORD_TEST_PORT) )
-    s.sendall('RESTART_QUAGGA {0} {1}\n'.format(config_file, boot_delay))
-    data = s.recv(1024).strip()
-    s.close()
+    data = __cord_test_quagga_restart(config = config, boot_delay = boot_delay)
+    if data == 'DONE':
+        return True
+    return False
+
+@nottest
+def cord_test_quagga_stop():
+    data = rpc_server_instance().stop_quagga()
     if data == 'DONE':
         return True
     return False
@@ -111,11 +147,7 @@
 @nottest
 def cord_test_radius_restart():
     '''Send Radius server restart to server'''
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.connect( (CORD_TEST_HOST, CORD_TEST_PORT) )
-    s.sendall('RESTART_RADIUS\n')
-    data = s.recv(1024).strip()
-    s.close()
+    data = rpc_server_instance().restart_radius()
     if data == 'DONE':
         return True
     return False
diff --git a/src/test/vrouter/vrouterTest.py b/src/test/vrouter/vrouterTest.py
index c81eb8d..61fe80c 100644
--- a/src/test/vrouter/vrouterTest.py
+++ b/src/test/vrouter/vrouterTest.py
@@ -1,12 +1,12 @@
-# 
+#
 # Copyright 2016-present Ciena Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -81,7 +81,7 @@
             cls.port_map = g_subscriber_port_map
         #cls.vrouter_host_load(host = cls.GATEWAY)
         time.sleep(3)
-        
+
     @classmethod
     def tearDownClass(cls):
         '''Deactivate the vrouter apps'''
@@ -148,27 +148,19 @@
         else:
             config = network_cfg
         log.info('Restarting ONOS with new network configuration')
-        cfg = json.dumps(config)
-        with open('{}/network-cfg.json'.format(cls.onos_config_path), 'w') as f:
-            f.write(cfg)
-
-        return cord_test_onos_restart()
+        return cord_test_onos_restart(config = config)
 
     @classmethod
     def start_quagga(cls, networks = 4):
         log.info('Restarting Quagga container with configuration for %d networks' %(networks))
         config = cls.generate_conf(networks = networks)
-        host_config_file = '{}/testrib_gen.conf'.format(Quagga.host_quagga_config)
-        guest_config_file = os.path.join(Quagga.guest_quagga_config, 'testrib_gen.conf')
-        with open(host_config_file, 'w') as f:
-            f.write(config)
         if networks <= 10000:
             boot_delay = 25
         else:
             delay_map = [60, 100, 150, 200, 300, 450, 600, 800, 1000, 1200]
             n = min(networks/100000, len(delay_map)-1)
             boot_delay = delay_map[n]
-        cord_test_quagga_restart(config_file = guest_config_file, boot_delay = boot_delay)
+        cord_test_quagga_restart(config = config, boot_delay = boot_delay)
 
     @classmethod
     def zgenerate_vrouter_conf(cls, networks = 4):
@@ -187,7 +179,7 @@
                 except:
                     port_map[device_port_key] = { 'interfaces' : [] }
                     interfaces = port_map[device_port_key]['interfaces']
-                    
+
                 ips = '%d.%d.%d.2/24'%( (n >> 24) & 0xff, ( ( n >> 16) & 0xff ), ( (n >> 8 ) & 0xff ) )
                 if num < cls.MAX_PORTS - 1:
                     interface_dict = { 'name' : 'b1-{}'.format(port), 'ips': [ips], 'mac' : '00:00:00:00:00:01' }
@@ -202,7 +194,7 @@
         quagga_router_dict = quagga_dict['apps']['org.onosproject.router']['router']
         quagga_router_dict['ospfEnabled'] = True
         quagga_router_dict['interfaces'] = interface_list
-        quagga_router_dict['controlPlaneConnectPoint'] = '{0}/{1}'.format(cls.device_id, 
+        quagga_router_dict['controlPlaneConnectPoint'] = '{0}/{1}'.format(cls.device_id,
                                                                           networks + 1 if networks < cls.MAX_PORTS else cls.MAX_PORTS )
         return (cls.vrouter_device_dict, ports_dict, quagga_dict)
 
@@ -276,7 +268,7 @@
         zebra_routes = '\n'.join(net_list)
         #log.info('Zebra routes: \n:%s\n' %cls.zebra_conf + zebra_routes)
         return cls.zebra_conf + zebra_routes
-    
+
     @classmethod
     def vrouter_activate(cls, deactivate = False):
         app = 'org.onosproject.vrouter'
@@ -296,7 +288,7 @@
         ##Start quagga
         cls.start_quagga(networks = networks)
         return vrouter_configs
-    
+
     def vrouter_port_send_recv(self, ingress, egress, dst_mac, dst_ip, positive_test = True):
         src_mac = '00:00:00:00:00:02'
         src_ip = '1.1.1.1'
@@ -343,7 +335,7 @@
                 ##Verify if flows are setup by sending traffic across
                 self.vrouter_port_send_recv(ingress, egress, dst_mac, dst_ip, positive_test = positive_test)
             num += 1
-    
+
     def __vrouter_network_verify(self, networks, peers = 1, positive_test = True):
         _, ports_map, egress_map = self.vrouter_configure(networks = networks, peers = peers)
         self.cliEnter()
@@ -430,12 +422,12 @@
         '''Test vrouter with 1000 routes'''
         res = self.__vrouter_network_verify(1000, peers = 1)
         assert_equal(res, True)
-    
+
     def test_vrouter_10(self):
         '''Test vrouter with 10000 routes'''
         res = self.__vrouter_network_verify(10000, peers = 1)
         assert_equal(res, True)
-    
+
     @nottest
     def test_vrouter_11(self):
         '''Test vrouter with 100000 routes'''