Adding retry mechanism for ONU are not yet discovered by XOS

Change-Id: Ibd7bb08f48a89b9ac3951cb86ec02f8b216078c8
diff --git a/samples/onu_activate_event.py b/samples/onu_activate_event.py
index 56f4c76..8786ed5 100644
--- a/samples/onu_activate_event.py
+++ b/samples/onu_activate_event.py
@@ -24,6 +24,6 @@
     'uni_port_id': 16,
     'of_dpid': 'of:109299321'
 })
-producer = KafkaProducer(bootstrap_servers="cord-kafka-kafka")
+producer = KafkaProducer(bootstrap_servers="cord-kafka")
 producer.send("onu.events", event)
 producer.flush()
\ No newline at end of file
diff --git a/xos/synchronizer/event_steps/onu_event.py b/xos/synchronizer/event_steps/onu_event.py
index a7658f2..8dc4698 100644
--- a/xos/synchronizer/event_steps/onu_event.py
+++ b/xos/synchronizer/event_steps/onu_event.py
@@ -15,6 +15,7 @@
 
 
 import json
+import time
 import os
 import sys
 from synchronizers.new_base.eventstep import EventStep
@@ -26,22 +27,29 @@
     topics = ["onu.events"]
     technology = "kafka"
 
+    max_onu_retry = 50
+
     def __init__(self, *args, **kwargs):
         super(ONUEventStep, self).__init__(*args, **kwargs)
 
-    def get_oss_service(self, onu_serial_number):
+    def get_oss_service(self, onu_serial_number, retry=0):
         try:
             onu = ONUDevice.objects.get(serial_number=onu_serial_number)
         except IndexError as e:
-            # TODO create ONU if it does not exists
-            raise Exception("No ONUDevice with serial_number %s is present in XOS" % onu_serial_number)
+            self.log.info("onu.events: get_oss_service", retry=retry, max_onu_retry=self.max_onu_retry)
+            if retry < self.max_onu_retry:
+                self.log.info("onu.events: No ONUDevice with serial_number %s is present in XOS, keep trying" % onu_serial_number)
+                time.sleep(10)
+                return self.get_oss_service(onu_serial_number, retry + 1)
+            else:
+                raise Exception("onu.events: No ONUDevice with serial_number %s is present in XOS" % onu_serial_number)
 
         volt_service = onu.pon_port.olt_device.volt_service
         service = Service.objects.get(id=volt_service.id)
         osses = [s for s in service.subscriber_services if s.kind.lower() == "oss"]
 
         if len(osses) > 1:
-            self.log.warn("More than one OSS found for %s" % volt_service.name)
+            self.log.warn("onu.events: More than one OSS found for %s" % volt_service.name)
         try:
             return osses[0].leaf_model
         except IndexError as e:
@@ -51,12 +59,13 @@
         oss = self.get_oss_service(event["serial_number"])
 
         if not oss:
-            self.log.info("Not processing events as no OSS service is present (is it a provider of vOLT?")
+            self.log.info("onu.events: Not processing events as no OSS service is present (is it a provider of vOLT?")
         else:
             try:
+                self.log.info("onu.events: Calling OSS for ONUDevice with serial_number %s" % event["serial_number"])
                 oss.validate_onu(event)
             except Exception, e:
-                self.log.exception("Failing to validate ONU in OSS Service %s" % oss.name)
+                self.log.exception("onu.events: Failing to validate ONU in OSS Service %s" % oss.name)
                 raise e
 
     def process_event(self, event):
diff --git a/xos/synchronizer/event_steps/test_onu_events.py b/xos/synchronizer/event_steps/test_onu_events.py
index 3de65c1..805b39d 100644
--- a/xos/synchronizer/event_steps/test_onu_events.py
+++ b/xos/synchronizer/event_steps/test_onu_events.py
@@ -98,13 +98,14 @@
         self.service.subscriber_services = []
 
     def test_missing_onu(self):
+        self.event_step.max_onu_retry = 0
         with patch.object(ONUDevice.objects, "get_items") as onu_device_mock:
             onu_device_mock.side_effect = IndexError("No ONU")
 
         with self.assertRaises(Exception) as e:
             self.event_step.process_event(self.event)
 
-        self.assertEqual(e.exception.message, "No ONUDevice with serial_number %s is present in XOS" % self.onu.serial_number)
+        self.assertEqual(e.exception.message, "onu.events: No ONUDevice with serial_number %s is present in XOS" % self.onu.serial_number)
 
     def test_do_nothing(self):
         with patch.object(ONUDevice.objects, "get_items") as onu_device_mock , \
@@ -116,7 +117,7 @@
 
             self.event_step.process_event(self.event)
 
-            logInfo.assert_called_with("Not processing events as no OSS service is present (is it a provider of vOLT?")
+            logInfo.assert_called_with("onu.events: Not processing events as no OSS service is present (is it a provider of vOLT?")
 
     def test_call_oss(self):
         self.service.subscriber_services = [self.oss]