VOL-281: OfAgent integration in swarm cluster
- Auto connect/reconnect of agent with a voltha instance
- Survives and cleans up after a voltha disconnect
- Added new compose files to start ofagent and onos in swarm cluster
Amendments:
- Fixed logs to have uniform format
- Removed instructions to start/stop ONOS service in cluster scripts
- Added missing change in local handler to exit streaming rpc calls
after ofagent termination
- Renamed references from voltha to vcore where necessary
Change-Id: Icb4611d92be35b48e557e6b12f7d2074282175ea
diff --git a/tests/itests/ofagent/test_ofagent_recovery.py b/tests/itests/ofagent/test_ofagent_recovery.py
new file mode 100644
index 0000000..38ea471
--- /dev/null
+++ b/tests/itests/ofagent/test_ofagent_recovery.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+import os
+import time
+import json
+
+from tests.itests.voltha.rest_base import RestBase
+
+this_dir = os.path.abspath(os.path.dirname(__file__))
+
+from tests.itests.docutests.test_utils import run_command_to_completion_with_raw_stdout
+
+log = logging.getLogger(__name__)
+
+DOCKER_COMPOSE_FILE = "compose/docker-compose-ofagent-test.yml"
+
+command_defs = dict(
+ docker_stop="docker stop {}",
+ docker_start="docker start {}",
+ docker_compose_start_all="docker-compose -f {} up -d "
+ .format(DOCKER_COMPOSE_FILE),
+ docker_compose_stop="docker-compose -f {} stop"
+ .format(DOCKER_COMPOSE_FILE),
+ docker_compose_rm_f="docker-compose -f {} rm -f"
+ .format(DOCKER_COMPOSE_FILE),
+ onos_form_cluster="./tests/itests/ofagent/onos-form-cluster",
+ onos1_ip="docker inspect --format '{{ .NetworkSettings.Networks.compose_default.IPAddress }}' onos1",
+ onos2_ip="docker inspect --format '{{ .NetworkSettings.Networks.compose_default.IPAddress }}' onos2",
+ onos3_ip="docker inspect --format '{{ .NetworkSettings.Networks.compose_default.IPAddress }}' onos3",
+ add_olt='''curl -k -s -X POST -d '{"type": "simulated_olt"}' \
+ https://localhost:8881/api/v1/local/devices''',
+ enable_olt="curl -k -s -X POST https://localhost:8881/api/v1/local/devices/{}/enable",
+ get_onos_devices="curl -u karaf:karaf http://localhost:8181/onos/v1/devices")
+
+
+class OfagentRecoveryTest(RestBase):
+ def setUp(self):
+ # Run Voltha,OFAgent,3 ONOS and form ONOS cluster.
+ print "Starting all containers ..."
+ cmd = command_defs['docker_compose_start_all']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+ print "Waiting for all containers to be ready ..."
+ time.sleep(60)
+ cmd = command_defs['onos1_ip']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+ onos1_ip = out
+ print "ONOS1 IP is {}".format(onos1_ip)
+ cmd = command_defs['onos2_ip']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+ onos2_ip = out
+ print "ONOS2 IP is {}".format(onos2_ip)
+ cmd = command_defs['onos3_ip']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+ onos3_ip = out
+ print "ONOS3 IP is {}".format(onos3_ip)
+ cmd = command_defs['onos_form_cluster'] + ' {} {} {}'.format(onos1_ip.strip(),
+ onos2_ip.strip(),
+ onos3_ip.strip())
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+ print "Cluster Output :{} ".format(out)
+
+ def tearDown(self):
+ # Stopping and Removing Voltha,OFAgent,3 ONOS.
+ print "Stopping and removing all containers ..."
+ cmd = command_defs['docker_compose_stop']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+ print "Waiting for all containers to be stopped ..."
+ time.sleep(1)
+ cmd = command_defs['docker_compose_rm_f']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+
+ def add_device(self):
+ print "Adding device"
+
+ cmd = command_defs['add_olt']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+ device = json.loads(out)
+
+ print "Added device - id:{}, type:{}".format(device['id'], device['type'])
+ time.sleep(5)
+
+ return device
+
+ def enable_device(self, device_id):
+ print "Enabling device - id:{}".format(device_id)
+
+ cmd = command_defs['enable_olt'].format(device_id)
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+
+ time.sleep(30)
+ print "Enabled device - id:{}".format(device_id)
+
+ def get_device(self, device_id, expected_code=200):
+ print "Getting device - id:{}".format(device_id)
+
+ device = self.get('/api/v1/local/devices/{}'.format(device_id),
+ expected_code=expected_code)
+
+ if device is not None:
+ print "Got device - id:{}, type:{}".format(device['id'], device['type'])
+ else:
+ print "Unable to get device - id:{}".format(device_id)
+
+ return device
+
+ def get_onos_devices(self):
+ print "Getting ONOS devices ..."
+ cmd = command_defs['get_onos_devices']
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+
+ if out is not None:
+ onos_devices = json.loads(out)
+ print "Got ONOS devices"
+ else:
+ onos_devices = None
+ print "Unable to get ONOS devices"
+
+ return onos_devices
+
+ def stop_container(self, container):
+ print "Stopping {} ...".format(container)
+
+ cmd = command_defs['docker_stop'].format(container)
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+
+ time.sleep(10)
+ print "Stopped {}".format(container)
+
+ def start_container(self, container):
+ print "Starting {} ...".format(container)
+
+ cmd = command_defs['docker_start'].format(container)
+ out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
+ self.assertEqual(rc, 0)
+
+ time.sleep(10)
+ print "Started {}".format(container)
+
+ def test_01_recovery_after_voltha_restart(self):
+ # Add and enable a new OLT device
+ device_1 = self.add_device()
+ self.enable_device(device_1['id'])
+
+ # Verify that the device was propagated in ONOS
+ onos_devices = self.get_onos_devices()
+
+ self.assertEqual(len(onos_devices['devices']), 1)
+
+ # Restart voltha
+ self.stop_container('compose_voltha_1')
+ self.assertEqual(self.get_device(device_1['id'], 503), None)
+ self.start_container('compose_voltha_1')
+
+ # Get the device from VOLTHA after restart
+ device_1_after = self.get_device(device_1['id'])
+ self.assertEqual(device_1_after['id'], device_1['id'])
+
+ # Get the device from ONOS after restart
+ onos_devices = self.get_onos_devices()
+
+ self.assertEqual(len(onos_devices['devices']), 1)
+
+ # Add a new device
+ device_2 = self.add_device()
+ self.enable_device(device_2['id'])
+
+ # Ensure that ONOS has picked up the new device
+ onos_devices = self.get_onos_devices()
+
+ self.assertEqual(len(onos_devices['devices']), 2)
+
+ def test_02_recovery_after_ofagent_restart(self):
+ # Add and enable a new OLT device
+ device_1 = self.add_device()
+ self.enable_device(device_1['id'])
+
+ # Verify that the device was propagated in ONOS
+ onos_devices = self.get_onos_devices()
+
+ self.assertEqual(len(onos_devices['devices']), 1)
+
+ # Restart ofagent
+ self.stop_container('compose_ofagent_1')
+
+ # Try to create a device while ofagent is down
+ # this will succeed from a voltha point of view
+ # but it will not be propagated to ONOS until ofagent is back up
+ device_fail = self.add_device()
+ self.enable_device(device_fail['id'])
+ onos_devices = self.get_onos_devices()
+
+ # Onos should only have 1 device
+ self.assertNotEqual(len(onos_devices['devices']), 2)
+
+ self.start_container('compose_ofagent_1')
+
+ # Get the device from VOLTHA after restart
+ device_1_after = self.get_device(device_1['id'])
+ self.assertEqual(device_1_after['id'], device_1['id'])
+
+ # Get the device from ONOS after restart
+ onos_devices = self.get_onos_devices()
+ self.assertEqual(len(onos_devices['devices']), 2)
+
+ # Add a new device
+ device_2 = self.add_device()
+ self.enable_device(device_2['id'])
+
+ # Ensure that ONOS has picked up the new device
+ onos_devices = self.get_onos_devices()
+
+ self.assertEqual(len(onos_devices['devices']), 3)