[CORD-3165] Updating backend_status and catching OLT Activation errors

Change-Id: I353bca1d462b725126e755dcff32528366f72eb1
diff --git a/xos/synchronizer/steps/sync_olt_device.py b/xos/synchronizer/steps/sync_olt_device.py
index 97e90a2..5eeeafd 100644
--- a/xos/synchronizer/steps/sync_olt_device.py
+++ b/xos/synchronizer/steps/sync_olt_device.py
@@ -32,6 +32,8 @@
     provides = [OLTDevice]
     observes = OLTDevice
 
+    max_attempt = 120  # we give 10 minutes to the OLT to activate
+
     @staticmethod
     def get_ids_from_logical_device(o):
         voltha = Helpers.get_voltha_info(o.volt_service)
@@ -88,6 +90,8 @@
 
     def activate_olt(self, model):
 
+        attempted = 0
+
         voltha = Helpers.get_voltha_info(model.volt_service)
 
         # Enable device
@@ -96,16 +100,24 @@
         if request.status_code != 200:
             raise Exception("Failed to enable OLT device: %s" % request.text)
 
+        model.backend_status = "Waiting for device to be activated"
+        model.save(always_update_timestamp=False) # we don't want to kickoff a new loop
+
         # Read state
         request = requests.get("%s:%d/api/v1/devices/%s" % (voltha['url'], voltha['port'], model.device_id)).json()
-        while request['oper_status'] == "ACTIVATING":
+        while request['oper_status'] == "ACTIVATING" and attempted < self.max_attempt:
             log.info("Waiting for OLT device %s (%s) to activate" % (model.name, model.device_id))
             sleep(5)
             request = requests.get("%s:%d/api/v1/devices/%s" % (voltha['url'], voltha['port'], model.device_id)).json()
+            attempted = attempted + 1
+
 
         model.admin_state = request['admin_state']
         model.oper_status = request['oper_status']
 
+        if model.oper_status != "ACTIVE":
+            raise Exception("It was not possible to activate OLTDevice with id %s" % model.id)
+
         # Find the of_id of the device
         model = self.get_ids_from_logical_device(model)
         model.save()
@@ -162,11 +174,17 @@
             log.info("Pushing OLT device to VOLTHA", object=str(model), **model.tologdict())
             model = self.pre_provision_olt_device(model)
             self.activate_olt(model)
+        elif model.oper_status != "ACTIVE":
+            raise Exception("It was not possible to activate OLTDevice with id %s" % model.id)
         else:
             log.info("OLT device already exists in VOLTHA", object=str(model), **model.tologdict())
 
         # TODO configure onos only if we have: Switch datapath id, Switch port, Uplink
-        self.configure_onos(model)
+        if model.switch_datapath_id and model.switch_port and model.uplink:
+            log.info("Pushing OLT device to ONOS-VOLTHA", object=str(model), **model.tologdict())
+            self.configure_onos(model)
+        else:
+            raise DeferredException("Not pushing OLTDevice (%s) to ONOS-VOLTHA as parameters are missing" %  model.id)
 
     def delete_record(self, model):
         log.info("Deleting OLT device", object=str(model), **model.tologdict())
diff --git a/xos/synchronizer/steps/test_sync_olt_device.py b/xos/synchronizer/steps/test_sync_olt_device.py
index 2ee20ff..4700b05 100644
--- a/xos/synchronizer/steps/test_sync_olt_device.py
+++ b/xos/synchronizer/steps/test_sync_olt_device.py
@@ -111,6 +111,7 @@
         o.admin_state = None
         o.oper_status = None
         o.of_id = None
+        o.id = 1
 
         o.tologdict.return_value = {'name': "Mock VOLTServiceInstance"}
 
@@ -191,7 +192,7 @@
 
         m.post("http://voltha_url:1234/api/v1/devices", status_code=200, json={"id": "123"}, additional_matcher=functools.partial(match_json, expected_conf))
         m.post("http://voltha_url:1234/api/v1/devices/123/enable", status_code=200)
-        m.get("http://voltha_url:1234/api/v1/devices/123", json={"oper_status": "ENABLED", "admin_state": "ACTIVE"})
+        m.get("http://voltha_url:1234/api/v1/devices/123", json={"oper_status": "ACTIVE", "admin_state": "ENABLED"})
         logical_devices = {
             "items": [
                 {"root_device_id": "123", "id": "0001000ce2314000", "datapath_id": "55334486016"},
@@ -203,10 +204,10 @@
         m.post("http://onos_voltha_url:4321/onos/v1/network/configuration/", status_code = 200, additional_matcher=match_onos_req, json={})
 
         self.sync_step().sync_record(self.o)
-        self.assertEqual(self.o.admin_state, "ACTIVE")
-        self.assertEqual(self.o.oper_status, "ENABLED")
+        self.assertEqual(self.o.admin_state, "ENABLED")
+        self.assertEqual(self.o.oper_status, "ACTIVE")
         self.assertEqual(self.o.of_id, "0001000ce2314000")
-        self.o.save.assert_called_once()
+        self.assertEqual(self.o.save.call_count, 2) # we're updating the backend_status when activating and then adding logical device ids
 
     @requests_mock.Mocker()
     def test_sync_record_success_mac_address(self, m):
@@ -226,7 +227,7 @@
         m.post("http://voltha_url:1234/api/v1/devices", status_code=200, json={"id": "123"},
                additional_matcher=functools.partial(match_json, expected_conf))
         m.post("http://voltha_url:1234/api/v1/devices/123/enable", status_code=200)
-        m.get("http://voltha_url:1234/api/v1/devices/123", json={"oper_status": "ENABLED", "admin_state": "ACTIVE"})
+        m.get("http://voltha_url:1234/api/v1/devices/123", json={"oper_status": "ACTIVE", "admin_state": "ENABLED"})
         logical_devices = {
             "items": [
                 {"root_device_id": "123", "id": "0001000ce2314000", "datapath_id": "55334486016"},
@@ -239,17 +240,55 @@
                additional_matcher=match_onos_req, json={})
 
         self.sync_step().sync_record(self.o)
-        self.assertEqual(self.o.admin_state, "ACTIVE")
-        self.assertEqual(self.o.oper_status, "ENABLED")
+        self.assertEqual(self.o.admin_state, "ENABLED")
+        self.assertEqual(self.o.oper_status, "ACTIVE")
         self.assertEqual(self.o.of_id, "0001000ce2314000")
-        self.o.save.assert_called_once()
+        self.assertEqual(self.o.save.call_count, 2)
+
+    @requests_mock.Mocker()
+    def test_sync_record_enable_timeout(self, m):
+        """
+        If device.enable fails we need to tell the suer
+        """
+
+        expected_conf = {
+            "type": self.o.device_type,
+            "host_and_port": "%s:%s" % (self.o.host, self.o.port)
+        }
+
+        m.post("http://voltha_url:1234/api/v1/devices", status_code=200, json={"id": "123"},
+               additional_matcher=functools.partial(match_json, expected_conf))
+        m.post("http://voltha_url:1234/api/v1/devices/123/enable", status_code=200)
+        m.get("http://voltha_url:1234/api/v1/devices/123", [
+                  {"json": {"oper_status": "ACTIVATING", "admin_state": "ENABLED"}, "status_code": 200},
+                  {"json": {"oper_status": "ERROR", "admin_state": "FAILED"}, "status_code": 200}
+              ])
+
+        logical_devices = {
+            "items": [
+                {"root_device_id": "123", "id": "0001000ce2314000", "datapath_id": "55334486016"},
+                {"root_device_id": "0001cc4974a62b87", "id": "0001000000000001"}
+            ]
+        }
+        m.get("http://voltha_url:1234/api/v1/logical_devices", status_code=200, json=logical_devices)
+
+        m.post("http://onos_voltha_url:4321/onos/v1/network/configuration/", status_code=200,
+               additional_matcher=match_onos_req, json={})
+
+        with self.assertRaises(Exception) as e:
+            self.sync_step().sync_record(self.o)
+
+        self.assertEqual(e.exception.message, "It was not possible to activate OLTDevice with id 1")
+        self.assertEqual(self.o.oper_status, "ERROR")
+        self.assertEqual(self.o.admin_state, "FAILED")
+        self.assertEqual(self.o.save.call_count, 1)
 
     @requests_mock.Mocker()
     def test_sync_record_already_existing_in_voltha(self, m):
         # mock device feedback state
         self.o.device_id = "123"
-        self.o.admin_state = "ACTIVE"
-        self.o.oper_status = "ENABLED"
+        self.o.admin_state = "ENABLED"
+        self.o.oper_status = "ACTIVE"
         self.o.dp_id = "of:0000000ce2314000"
         self.o.of_id = "0001000ce2314000"