blob: dba3563c2a46708665966f41ddd3d9b619554a0f [file] [log] [blame]
Stephane Barbarie2940dac2017-08-18 14:15:17 -04001#!/usr/bin/env python
2#
3# Copyright 2017 the original author or authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17import logging
18import os
Stephane Barbariecd51f992017-09-07 16:37:02 -040019from time import time, sleep
Stephane Barbarie2940dac2017-08-18 14:15:17 -040020import json
21
Stephane Barbariecd51f992017-09-07 16:37:02 -040022from google.protobuf.json_format import MessageToDict
Stephane Barbarie2940dac2017-08-18 14:15:17 -040023from tests.itests.voltha.rest_base import RestBase
Stephane Barbariecd51f992017-09-07 16:37:02 -040024from common.utils.consulhelpers import get_endpoint_from_consul
25from voltha.protos.device_pb2 import Device
26from unittest import skip
Stephane Barbarie2940dac2017-08-18 14:15:17 -040027
28this_dir = os.path.abspath(os.path.dirname(__file__))
29
30from tests.itests.docutests.test_utils import run_command_to_completion_with_raw_stdout
31
32log = logging.getLogger(__name__)
33
34DOCKER_COMPOSE_FILE = "compose/docker-compose-ofagent-test.yml"
Stephane Barbariecd51f992017-09-07 16:37:02 -040035LOCAL_CONSUL = "localhost:8500"
Stephane Barbarie2940dac2017-08-18 14:15:17 -040036
37command_defs = dict(
38 docker_stop="docker stop {}",
39 docker_start="docker start {}",
40 docker_compose_start_all="docker-compose -f {} up -d "
41 .format(DOCKER_COMPOSE_FILE),
42 docker_compose_stop="docker-compose -f {} stop"
43 .format(DOCKER_COMPOSE_FILE),
44 docker_compose_rm_f="docker-compose -f {} rm -f"
45 .format(DOCKER_COMPOSE_FILE),
46 onos_form_cluster="./tests/itests/ofagent/onos-form-cluster",
47 onos1_ip="docker inspect --format '{{ .NetworkSettings.Networks.compose_default.IPAddress }}' onos1",
48 onos2_ip="docker inspect --format '{{ .NetworkSettings.Networks.compose_default.IPAddress }}' onos2",
49 onos3_ip="docker inspect --format '{{ .NetworkSettings.Networks.compose_default.IPAddress }}' onos3",
Stephane Barbarie2940dac2017-08-18 14:15:17 -040050 get_onos_devices="curl -u karaf:karaf http://localhost:8181/onos/v1/devices")
51
Stephane Barbariecd51f992017-09-07 16:37:02 -040052@skip('Test case hangs at REST calls during execution. Refer to VOL-425 and VOL-427')
Stephane Barbarie2940dac2017-08-18 14:15:17 -040053class OfagentRecoveryTest(RestBase):
54 def setUp(self):
55 # Run Voltha,OFAgent,3 ONOS and form ONOS cluster.
56 print "Starting all containers ..."
57 cmd = command_defs['docker_compose_start_all']
58 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
59 self.assertEqual(rc, 0)
60 print "Waiting for all containers to be ready ..."
Stephane Barbariecd51f992017-09-07 16:37:02 -040061 sleep(60)
Stephane Barbarie2940dac2017-08-18 14:15:17 -040062 cmd = command_defs['onos1_ip']
63 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
64 self.assertEqual(rc, 0)
65 onos1_ip = out
66 print "ONOS1 IP is {}".format(onos1_ip)
67 cmd = command_defs['onos2_ip']
68 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
69 self.assertEqual(rc, 0)
70 onos2_ip = out
71 print "ONOS2 IP is {}".format(onos2_ip)
72 cmd = command_defs['onos3_ip']
73 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
74 self.assertEqual(rc, 0)
75 onos3_ip = out
76 print "ONOS3 IP is {}".format(onos3_ip)
77 cmd = command_defs['onos_form_cluster'] + ' {} {} {}'.format(onos1_ip.strip(),
78 onos2_ip.strip(),
79 onos3_ip.strip())
80 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
81 self.assertEqual(rc, 0)
82 print "Cluster Output :{} ".format(out)
83
Stephane Barbariecd51f992017-09-07 16:37:02 -040084 self.get_rest_endpoint()
85
Stephane Barbarie2940dac2017-08-18 14:15:17 -040086 def tearDown(self):
87 # Stopping and Removing Voltha,OFAgent,3 ONOS.
88 print "Stopping and removing all containers ..."
89 cmd = command_defs['docker_compose_stop']
90 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
91 self.assertEqual(rc, 0)
92 print "Waiting for all containers to be stopped ..."
Stephane Barbariecd51f992017-09-07 16:37:02 -040093 sleep(1)
Stephane Barbarie2940dac2017-08-18 14:15:17 -040094 cmd = command_defs['docker_compose_rm_f']
95 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
96 self.assertEqual(rc, 0)
97
Stephane Barbariecd51f992017-09-07 16:37:02 -040098 def wait_till(self, msg, predicate, interval=0.1, timeout=5.0):
99 deadline = time() + timeout
100 while time() < deadline:
101 if predicate():
102 return
103 sleep(interval)
104 self.fail('Timed out while waiting for condition: {}'.format(msg))
105
106 def get_rest_endpoint(self):
107 # Retrieve details on the REST entry point
108 rest_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'envoy-8443')
109
110 # Construct the base_url
111 self.base_url = 'https://' + rest_endpoint
112
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400113 def add_device(self):
114 print "Adding device"
115
Stephane Barbariecd51f992017-09-07 16:37:02 -0400116 device = Device(
117 type='simulated_olt'
118 )
119 device = self.post('/api/v1/devices', MessageToDict(device),
120 expected_http_code=200)
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400121
122 print "Added device - id:{}, type:{}".format(device['id'], device['type'])
Stephane Barbariecd51f992017-09-07 16:37:02 -0400123 sleep(5)
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400124
125 return device
126
127 def enable_device(self, device_id):
128 print "Enabling device - id:{}".format(device_id)
129
Stephane Barbariecd51f992017-09-07 16:37:02 -0400130 path = '/api/v1/devices/{}'.format(device_id)
131 self.post(path + '/enable', expected_http_code=200)
132 device = self.get(path)
133 self.assertEqual(device['admin_state'], 'ENABLED')
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400134
Stephane Barbariecd51f992017-09-07 16:37:02 -0400135 self.wait_till(
136 'admin state moves to ACTIVATING or ACTIVE',
137 lambda: self.get(path)['oper_status'] in ('ACTIVATING', 'ACTIVE'),
138 timeout=0.5)
139
140 # eventually, it shall move to active state and by then we shall have
141 # device details filled, connect_state set, and device ports created
142 self.wait_till(
143 'admin state ACTIVE',
144 lambda: self.get(path)['oper_status'] == 'ACTIVE',
145 timeout=0.5)
146 device = self.get(path)
147 images = device['images']
148 image = images['image']
149 image_1 = image[0]
150 version = image_1['version']
151 self.assertNotEqual(version, '')
152 self.assertEqual(device['connect_status'], 'REACHABLE')
153
154 ports = self.get(path + '/ports')['items']
155 self.assertEqual(len(ports), 2)
156
157 sleep(30)
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400158 print "Enabled device - id:{}".format(device_id)
159
160 def get_device(self, device_id, expected_code=200):
161 print "Getting device - id:{}".format(device_id)
162
Stephane Barbariecd51f992017-09-07 16:37:02 -0400163 device = self.get('/api/v1/devices/{}'.format(device_id),
164 expected_http_code=expected_code)
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400165
166 if device is not None:
167 print "Got device - id:{}, type:{}".format(device['id'], device['type'])
168 else:
169 print "Unable to get device - id:{}".format(device_id)
170
171 return device
172
173 def get_onos_devices(self):
174 print "Getting ONOS devices ..."
175 cmd = command_defs['get_onos_devices']
176 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
177 self.assertEqual(rc, 0)
178
179 if out is not None:
180 onos_devices = json.loads(out)
181 print "Got ONOS devices"
182 else:
183 onos_devices = None
184 print "Unable to get ONOS devices"
185
186 return onos_devices
187
188 def stop_container(self, container):
189 print "Stopping {} ...".format(container)
190
191 cmd = command_defs['docker_stop'].format(container)
192 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
193 self.assertEqual(rc, 0)
194
Stephane Barbariecd51f992017-09-07 16:37:02 -0400195 sleep(10)
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400196 print "Stopped {}".format(container)
197
198 def start_container(self, container):
199 print "Starting {} ...".format(container)
200
201 cmd = command_defs['docker_start'].format(container)
202 out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
203 self.assertEqual(rc, 0)
204
Stephane Barbariecd51f992017-09-07 16:37:02 -0400205 sleep(10)
Stephane Barbarie2940dac2017-08-18 14:15:17 -0400206 print "Started {}".format(container)
207
208 def test_01_recovery_after_voltha_restart(self):
209 # Add and enable a new OLT device
210 device_1 = self.add_device()
211 self.enable_device(device_1['id'])
212
213 # Verify that the device was propagated in ONOS
214 onos_devices = self.get_onos_devices()
215
216 self.assertEqual(len(onos_devices['devices']), 1)
217
218 # Restart voltha
219 self.stop_container('compose_voltha_1')
220 self.assertEqual(self.get_device(device_1['id'], 503), None)
221 self.start_container('compose_voltha_1')
222
223 # Get the device from VOLTHA after restart
224 device_1_after = self.get_device(device_1['id'])
225 self.assertEqual(device_1_after['id'], device_1['id'])
226
227 # Get the device from ONOS after restart
228 onos_devices = self.get_onos_devices()
229
230 self.assertEqual(len(onos_devices['devices']), 1)
231
232 # Add a new device
233 device_2 = self.add_device()
234 self.enable_device(device_2['id'])
235
236 # Ensure that ONOS has picked up the new device
237 onos_devices = self.get_onos_devices()
238
239 self.assertEqual(len(onos_devices['devices']), 2)
240
241 def test_02_recovery_after_ofagent_restart(self):
242 # Add and enable a new OLT device
243 device_1 = self.add_device()
244 self.enable_device(device_1['id'])
245
246 # Verify that the device was propagated in ONOS
247 onos_devices = self.get_onos_devices()
248
249 self.assertEqual(len(onos_devices['devices']), 1)
250
251 # Restart ofagent
252 self.stop_container('compose_ofagent_1')
253
254 # Try to create a device while ofagent is down
255 # this will succeed from a voltha point of view
256 # but it will not be propagated to ONOS until ofagent is back up
257 device_fail = self.add_device()
258 self.enable_device(device_fail['id'])
259 onos_devices = self.get_onos_devices()
260
261 # Onos should only have 1 device
262 self.assertNotEqual(len(onos_devices['devices']), 2)
263
264 self.start_container('compose_ofagent_1')
265
266 # Get the device from VOLTHA after restart
267 device_1_after = self.get_device(device_1['id'])
268 self.assertEqual(device_1_after['id'], device_1['id'])
269
270 # Get the device from ONOS after restart
271 onos_devices = self.get_onos_devices()
272 self.assertEqual(len(onos_devices['devices']), 2)
273
274 # Add a new device
275 device_2 = self.add_device()
276 self.enable_device(device_2['id'])
277
278 # Ensure that ONOS has picked up the new device
279 onos_devices = self.get_onos_devices()
280
281 self.assertEqual(len(onos_devices['devices']), 3)