Complete overhaul of the implementation of ONOS cord instance wrapper for cord-tester.
It now patches the onos-cord compose instance in place and restores it back on cleanup.
A sample manifest-cord.json illustrates the usage if one wants to wrap the existing onos-cord running instance
in CiaB head-node (called prod)
This is required because of the manner in which the onos-cord instance networks are created automatically matching the docker networks that already exist for the onos instances.
The service profile spec allows one to synchronize the configuration/apps for the onos-cord running instance.
Change-Id: Icd066d8b953eccb7dcd7330775b548d36a7fb33e
diff --git a/src/test/setup/cord-test.py b/src/test/setup/cord-test.py
index b4e4f6a..3e66fd8 100755
--- a/src/test/setup/cord-test.py
+++ b/src/test/setup/cord-test.py
@@ -444,25 +444,29 @@
head_node = test_manifest.head_node
iterations = test_manifest.iterations
onos_cord_loc = test_manifest.onos_cord
-
+ service_profile = test_manifest.service_profile
+ synchronizer = test_manifest.synchronizer
+ onos_cord = None
if onos_cord_loc:
if onos_cord_loc.find(os.path.sep) < 0:
onos_cord_loc = os.path.join(os.getenv('HOME'), onos_cord_loc)
- #check if the wrapper is already active. If yes, we back out
- if os.access(OnosCord.onos_cord_dir, os.F_OK):
- onos_cord_loc = None
- else:
- if not os.access(onos_cord_loc, os.F_OK):
- print('ONOS cord config location %s is not accessible' %onos_cord_loc)
- sys.exit(1)
-
- onos_cord = None
- if onos_cord_loc:
+ if not os.access(onos_cord_loc, os.F_OK):
+ print('ONOS cord config location %s is not accessible' %onos_cord_loc)
+ sys.exit(1)
if not onos_ip:
##Unexpected case. Specify the external controller ip when running on cord node
print('Specify ONOS ip using \"-e\" option when running the cord-tester on cord node')
sys.exit(1)
- onos_cord = OnosCord(onos_ip, onos_cord_loc)
+ if not service_profile:
+ print('Specify service profile location for the ONOS cord instance. Eg: $HOME/service-profile/cord-pod')
+ sys.exit(1)
+ if not synchronizer:
+ print('Specify synchronizer to use for the ONOS cord instance. Eg: vtn, fabric, cord')
+ sys.exit(1)
+ if not os.access(service_profile, os.F_OK):
+ print('Service profile location for ONOS cord instance does not exist')
+ sys.exit(1)
+ onos_cord = OnosCord(onos_ip, onos_cord_loc, service_profile, synchronizer)
try:
test_server = cord_test_server_start(daemonize = False, cord_test_host = test_host, cord_test_port = test_port,
@@ -665,9 +669,8 @@
test_cnt.run_tests()
if test_server:
- if onos_cord_loc:
- if OnosCord.restore_onos_cord(onos_cord_loc, onos_ip) is False:
- OnosCord.cleanup()
+ if onos_cord:
+ onos_cord.restore()
cord_test_server_stop(test_server)
return status
@@ -696,6 +699,13 @@
test_manifest = TestManifest(manifest = dest)
use_manifest = True
+ onos_ip = test_manifest.onos_ip
+ radius_ip = test_manifest.radius_ip
+ head_node = test_manifest.head_node
+ iterations = test_manifest.iterations
+ service_profile = test_manifest.service_profile
+ synchronizer = test_manifest.synchronizer
+ onos_cord = None
onos_cord_loc = test_manifest.onos_cord
if onos_cord_loc:
if onos_cord_loc.find(os.path.sep) < 0:
@@ -703,18 +713,20 @@
if not os.access(onos_cord_loc, os.F_OK):
print('ONOS cord config location %s is not accessible' %onos_cord_loc)
sys.exit(1)
-
- onos_ip = test_manifest.onos_ip
- radius_ip = test_manifest.radius_ip
- head_node = test_manifest.head_node
- iterations = test_manifest.iterations
- onos_cord = None
- if onos_cord_loc:
if not onos_ip:
##Unexpected case. Specify the external controller ip when running on cord node
print('Specify ONOS ip using \"-e\" option when running the cord-tester on cord node')
sys.exit(1)
- onos_cord = OnosCord(onos_ip, onos_cord_loc)
+ if not service_profile:
+ print('Specify service profile location for the ONOS cord instance. Eg: $HOME/service-profile/cord-pod')
+ sys.exit(1)
+ if not synchronizer:
+ print('Specify synchronizer to use for the ONOS cord instance. Eg: vtn, fabric, cord')
+ sys.exit(1)
+ if not os.access(service_profile, os.F_OK):
+ print('Service profile location for ONOS cord instance does not exist')
+ sys.exit(1)
+ onos_cord = OnosCord(onos_ip, onos_cord_loc, service_profile, synchronizer)
Container.IMAGE_PREFIX = test_manifest.image_prefix
#don't spawn onos if the user had started it externally
@@ -875,6 +887,8 @@
args.onos_ip = manifest.onos_ip
args.radius_ip = manifest.radius_ip
args.onos_cord = manifest.onos_cord
+ args.service_profile = manifest.service_profile
+ args.synchronizer = manifest.synchronizer
else:
args.onos_ip = None
args.radius_ip = None
@@ -906,9 +920,12 @@
Onos.cleanup_runtime()
if args.onos_cord:
- #restore the ONOS cord instance
- if OnosCord.restore_onos_cord(args.onos_cord, args.onos_ip) is False:
- OnosCord.cleanup()
+ #try restoring the onos cord instance
+ try:
+ onos_cord = OnosCord(args.onos_ip, args.onos_cord, args.service_profile, args.synchronizer, start = False)
+ onos_cord.restore(force = True)
+ except Exception as e:
+ print(e)
if args.xos:
##cleanup XOS images
@@ -1117,6 +1134,12 @@
parser_run.add_argument('-network', '--network', default='', type=str, help='Docker network to attach')
parser_run.add_argument('-onos-cord', '--onos-cord', default='', type=str,
help='Specify config location for ONOS cord when running on podd')
+ parser_run.add_argument('-service-profile', '--service-profile', default='', type=str,
+ help='Specify config location for ONOS cord service profile when running on podd.'
+ 'Eg: $HOME/service-profile/cord-pod')
+ parser_run.add_argument('-synchronizer', '--synchronizer', default='', type=str,
+ help='Specify the synchronizer to use for ONOS cord instance when running on podd.'
+ 'Eg: vtn,fabric,cord')
parser_run.set_defaults(func=runTest)
parser_setup = subparser.add_parser('setup', help='Setup cord tester environment')
@@ -1139,6 +1162,12 @@
parser_setup.add_argument('-s', '--start-switch', action='store_true', help='Start OVS when running under OLT config')
parser_setup.add_argument('-onos-cord', '--onos-cord', default='', type=str,
help='Specify config location for ONOS cord when running on podd')
+ parser_setup.add_argument('-service-profile', '--service-profile', default='', type=str,
+ help='Specify config location for ONOS cord service profile when running on podd.'
+ 'Eg: $HOME/service-profile/cord-pod')
+ parser_setup.add_argument('-synchronizer', '--synchronizer', default='', type=str,
+ help='Specify the synchronizer to use for ONOS cord instance when running on podd.'
+ 'Eg: vtn,fabric,cord')
parser_setup.add_argument('-m', '--manifest', default='', type=str, help='Provide test configuration manifest')
parser_setup.add_argument('-p', '--prefix', default='', type=str, help='Provide container image prefix')
parser_setup.add_argument('-i', '--identity-file', default=identity_file_default,
@@ -1195,6 +1224,12 @@
'Eg: 10.0.0.2/10.0.0.3 to specify ONOS and Radius ip')
parser_cleanup.add_argument('-onos-cord', '--onos-cord', default='', type=str,
help='Specify config location for ONOS cord instance when running on podd to restore')
+ parser_cleanup.add_argument('-service-profile', '--service-profile', default='', type=str,
+ help='Specify config location for ONOS cord service profile when running on podd.'
+ 'Eg: $HOME/service-profile/cord-pod')
+ parser_cleanup.add_argument('-synchronizer', '--synchronizer', default='', type=str,
+ help='Specify the synchronizer to use for ONOS cord instance when running on podd.'
+ 'Eg: vtn,fabric,cord')
parser_cleanup.add_argument('-m', '--manifest', default='', type=str, help='Provide test manifest')
parser_cleanup.set_defaults(func=cleanupTests)
diff --git a/src/test/setup/manifest-cord.json b/src/test/setup/manifest-cord.json
new file mode 100644
index 0000000..a83be39
--- /dev/null
+++ b/src/test/setup/manifest-cord.json
@@ -0,0 +1,13 @@
+{
+ "onos_instances": 1,
+ "olt": true,
+ "start_switch": true,
+ "onos_image": "onosproject/onos:latest",
+ "onos" : "172.19.0.2",
+ "onos_cord" : "/home/vagrant/onos-cord",
+ "service_profile" : "/home/vagrant/service-profile/cord-pod",
+ "synchronizer" : "vtn",
+ "docker_network" : "onoscord_default",
+ "log_level" : "INFO",
+ "jvm_heap_size" : "1G"
+}
diff --git a/src/test/utils/CordContainer.py b/src/test/utils/CordContainer.py
index 1a7656b..5f1ea2c 100644
--- a/src/test/utils/CordContainer.py
+++ b/src/test/utils/CordContainer.py
@@ -25,7 +25,7 @@
from nsenter import Namespace
from docker import Client
from docker import utils as dockerutils
-from shutil import rmtree
+import shutil
from OnosCtrl import OnosCtrl
from OnosLog import OnosLog
from threadPool import ThreadPool
@@ -292,132 +292,122 @@
class OnosCord(Container):
"""Use this when running the cord tester agent on the onos compute node"""
- onos_cord_dir = os.path.join(os.getenv('HOME'), 'cord-tester-cord')
onos_config_dir_guest = '/root/onos/config'
- onos_config_dir = os.path.join(onos_cord_dir, 'config')
- docker_yaml = os.path.join(onos_cord_dir, 'docker-compose.yml')
- onos_cfg_save_loc = os.path.join(onos_cord_dir, 'network-cfg.json.saved')
- def __init__(self, onos_ip, conf, boot_delay = 60):
+ def __init__(self, onos_ip, conf, service_profile, synchronizer, start = True, boot_delay = 60):
+ if not os.access(conf, os.F_OK):
+ raise Exception('ONOS cord configuration location %s is invalid' %conf)
+ if not os.access(service_profile, os.F_OK):
+ raise Exception('ONOS cord service profile location is not accessible' %service_profile)
self.onos_ip = onos_ip
- self.cord_conf_dir = conf
+ self.onos_cord_dir = conf
self.boot_delay = boot_delay
- if os.access(self.cord_conf_dir, os.F_OK) and not os.access(self.onos_cord_dir, os.F_OK):
- os.mkdir(self.onos_cord_dir)
- os.mkdir(self.onos_config_dir)
- ##copy the config file from cord-tester-config
- cmd = 'cp {}/* {}'.format(self.cord_conf_dir, self.onos_cord_dir)
- os.system(cmd)
+ self.synchronizer = synchronizer
+ self.service_profile = service_profile
+ self.docker_yaml = os.path.join(conf, 'docker-compose.yml')
+ self.docker_yaml_saved = os.path.join(conf, 'docker-compose.yml.saved')
+ self.onos_config_dir = os.path.join(conf, 'config')
+ self.onos_cfg_save_loc = os.path.join(conf, 'network-cfg.json.saved')
+ instance_active = False
+ #if we have a wrapper onos instance already active, back out
+ if os.access(self.onos_config_dir, os.F_OK) or os.access(self.docker_yaml_saved, os.F_OK):
+ instance_active = True
+ else:
+ if start is True:
+ os.mkdir(self.onos_config_dir)
+ shutil.copy(self.docker_yaml, self.docker_yaml_saved)
+ self.start_wrapper = instance_active is False and start is True
##update the docker yaml with the config volume
with open(self.docker_yaml, 'r') as f:
yaml_config = yaml.load(f)
image = yaml_config['services'].keys()[0]
- name = 'cordtestercord_{}_1'.format(image)
+ cord_conf_dir_basename = os.path.basename(self.onos_cord_dir.replace('-', ''))
+ xos_onos_name = '{}_{}_1'.format(cord_conf_dir_basename, image)
volumes = yaml_config['services'][image]['volumes']
config_volumes = filter(lambda e: e.find(self.onos_config_dir_guest) >= 0, volumes)
if not config_volumes:
config_volume = '{}:{}'.format(self.onos_config_dir, self.onos_config_dir_guest)
volumes.append(config_volume)
- docker_yaml_changed = '{}-changed'.format(self.docker_yaml)
- with open(docker_yaml_changed, 'w') as wf:
- yaml.dump(yaml_config, wf)
-
- os.rename(docker_yaml_changed, self.docker_yaml)
+ if self.start_wrapper:
+ docker_yaml_changed = '{}-changed'.format(self.docker_yaml)
+ with open(docker_yaml_changed, 'w') as wf:
+ yaml.dump(yaml_config, wf)
+ os.rename(docker_yaml_changed, self.docker_yaml)
self.volumes = volumes
- super(OnosCord, self).__init__(name, image, tag = '')
- cord_conf_dir_basename = os.path.basename(self.cord_conf_dir.replace('-', ''))
- self.xos_onos_name = '{}_{}_1'.format(cord_conf_dir_basename, image)
##Create an container instance of xos onos
- self.xos_onos = Container(self.xos_onos_name, image, tag = '')
- #fetch the current config of onos cord instance
- try:
- self.last_cfg = OnosCtrl.get_config(controller = onos_ip)
- except:
- self.last_cfg = None
+ self.xos_onos = Container(xos_onos_name, image, tag = '')
+ self.last_cfg = None
+ if self.start_wrapper:
+ #fetch the current config of onos cord instance and save it
+ try:
+ self.last_cfg = OnosCtrl.get_config(controller = onos_ip)
+ json_data = json.dumps(self.last_cfg, indent=4)
+ with open(self.onos_cfg_save_loc, 'w') as f:
+ f.write(json_data)
+ except:
+ pass
+ #start the container back with the shared onos config volume
+ self.start()
def start(self, restart = False, network_cfg = None):
- if restart is True:
- if self.exists():
- ##Kill the existing instance
- print('Killing container %s' %self.name)
- self.kill()
- if self.xos_onos.exists():
- if self.last_cfg is not None:
- #save the current network config of onos cord instance
- json_data = json.dumps(self.last_cfg, indent=4)
- with open(self.onos_cfg_save_loc, 'w') as f:
- f.write(json_data)
- print('Killing container %s' %self.xos_onos.name)
- self.xos_onos.kill()
-
- if network_cfg is not None:
+ if network_cfg:
json_data = json.dumps(network_cfg, indent=4)
with open('{}/network-cfg.json'.format(self.onos_config_dir), 'w') as f:
f.write(json_data)
-
- #start the container using docker-compose
- cmd = 'cd {} && docker-compose up -d'.format(self.onos_cord_dir)
- os.system(cmd)
- #Delay to make sure ONOS fully boots
+ if restart is False:
+ #stop and start and synchronize the services before installing tester cord apps
+ cmds = [ 'cd {} && docker-compose down'.format(self.onos_cord_dir),
+ 'cd {} && docker-compose up -d'.format(self.onos_cord_dir),
+ 'sleep 45',
+ 'cd {} && make {}'.format(self.service_profile, self.synchronizer)
+ ]
+ for cmd in cmds:
+ try:
+ print(cmd)
+ os.system(cmd)
+ except:pass
+ Onos.install_cord_apps(onos_ip = self.onos_ip)
+ else:
+ cmd = 'cd {} && docker-compose restart'.format(self.onos_cord_dir)
+ try:
+ os.system(cmd)
+ except: pass
+ print('Waiting %d seconds for ONOS instance to start' %self.boot_delay)
time.sleep(self.boot_delay)
- Onos.install_cord_apps(onos_ip = self.onos_ip)
def build_image(self):
build_cmd = 'cd {} && docker-compose build'.format(self.onos_cord_dir)
os.system(build_cmd)
- @classmethod
- def cleanup(cls):
- if not os.access(cls.onos_cord_dir, os.F_OK):
+ def restore(self, force = False):
+ restore = self.start_wrapper is True or force is True
+ if not restore:
return
- cmd = 'cd {} && docker-compose down'.format(cls.onos_cord_dir)
- try:
- os.system(cmd)
- except: pass
-
- print('Cleaning up the ONOS cord wrapper directory at %s' %(cls.onos_cord_dir))
- try:
- os.system('rm -rf {}'.format(cls.onos_cord_dir))
- except:
- pass
-
- @classmethod
- def restore_onos_cord(cls, onos_cord, onos_ip):
- #bring down the onos cord wrapper container
- #if there is no saved config, there is nothing to restore as it was never restarted
- if not os.access(cls.onos_cfg_save_loc, os.F_OK):
- return False
- if not onos_cord or not os.access(onos_cord, os.F_OK):
- return False
-
- print('Stopping the existing ONOS cord wrapper instance at %s' %(cls.onos_cord_dir))
- cmd = 'cd {} && docker-compose down'.format(cls.onos_cord_dir)
- try:
- os.system(cmd)
- except:pass
-
- print('Starting the ONOS cord instance at %s' %(onos_cord))
- #bring back up the onos cord container
- cmd = 'cd {} && docker-compose up -d'.format(onos_cord)
- try:
- os.system(cmd)
- time.sleep(30)
- except:
- pass
-
- #now restore back the old config
- print('Restoring back the saved ONOS cord config at %s for ONOS cord instance' %(cls.onos_cfg_save_loc))
- with open(cls.onos_cfg_save_loc, 'r') as f:
- config = json.load(f)
+ #restore the config files back. The synchronizer restore should bring the last config back
+ cmds = ['cd {} && docker-compose down'.format(self.onos_cord_dir),
+ 'rm -rf {}'.format(self.onos_config_dir),
+ 'mv {} {}'.format(self.docker_yaml_saved, self.docker_yaml),
+ 'cd {} && docker-compose up -d'.format(self.onos_cord_dir),
+ 'sleep 45',
+ 'cd {} && make {}'.format(self.service_profile, self.synchronizer)
+ ]
+ for cmd in cmds:
try:
- OnosCtrl.config(config, controller = onos_ip)
- os.unlink(cls.onos_cfg_save_loc)
+ print(cmd)
+ os.system(cmd)
except: pass
- cls.cleanup()
- return True
+ #We may not have to restore the config but still it should match synchronizer last config
+ if os.access(self.onos_cfg_save_loc, os.F_OK):
+ with open(self.onos_cfg_save_loc, 'r') as f:
+ cfg = json.load(f)
+ try:
+ OnosCtrl.config(cfg, controller = self.onos_ip)
+ os.unlink(self.onos_cfg_save_loc)
+ except:
+ pass
class OnosCordStopWrapper(Container):
onos_cord_dir = os.path.join(os.getenv('HOME'), 'cord-tester-cord')
@@ -508,7 +498,7 @@
def remove_data_map(cls, host_volume, guest_volume_dir):
host_volume_dir = os.path.join(cls.setup_dir, os.path.basename(host_volume))
if os.path.exists(host_volume_dir):
- rmtree(host_volume_dir)
+ shutil.rmtree(host_volume_dir)
def remove_data_volume(self):
if self.data_map is not None:
diff --git a/src/test/utils/TestManifest.py b/src/test/utils/TestManifest.py
index 792926a..b5ec2d4 100644
--- a/src/test/utils/TestManifest.py
+++ b/src/test/utils/TestManifest.py
@@ -42,6 +42,8 @@
if len(ips) > 1:
self.radius_ip = ips[1]
self.onos_cord = args.onos_cord if args.onos_cord else None
+ self.service_profile = args.service_profile if args.service_profile else None
+ self.synchronizer = args.synchronizer if args.synchronizer else None
self.docker_network = args.network if args.network else None
self.iterations = None
self.server = args.server
@@ -55,6 +57,8 @@
if self.onos_ip and self.radius_ip:
self.test_controller = '{}/{}'.format(self.onos_ip, self.radius_ip)
self.onos_cord = data.get('onos_cord', None)
+ self.service_profile = data.get('service_profile', None)
+ self.synchronizer = data.get('synchronizer', None)
self.head_node = data.get('head_node', platform.node())
self.log_level = data.get('log_level', 'INFO').upper()
self.onos_instances = data.get('onos_instances', 1)