VOL-2555:
- Wait for ONU Deactivation Completed Indication before clearing the ONU
- Add unit tests for delete onu timeout and failure cases
- Fix UT of DeleteScheduler

Change-Id: Ie287f733934ca5f6eb06b365e426074622724bb7
diff --git a/agent/src/core_api_handler.cc b/agent/src/core_api_handler.cc
index 053c82c..7a90107 100644
--- a/agent/src/core_api_handler.cc
+++ b/agent/src/core_api_handler.cc
@@ -251,6 +251,7 @@
 
         bcmos_fastlock_init(&data_lock, 0);
         bcmos_fastlock_init(&alloc_cfg_wait_lock, 0);
+        bcmos_fastlock_init(&onu_deactivate_wait_lock, 0);
         OPENOLT_LOG(INFO, openolt_log_id, "Enable OLT - %s-%s\n", VENDOR_ID, MODEL_ID);
 
         //check BCM daemon is connected or not
@@ -1046,10 +1047,10 @@
     // of onu_cfg is passed. This is one-of case where we need to add test specific
     // code in production code.
     err = bcmolt_cfg_get__onu_state_stub(dev_id, &onu_cfg);
-    onu_state = onu_cfg.data.onu_state;
     #else
     err = bcmolt_cfg_get(dev_id, &onu_cfg.hdr);
     #endif
+    onu_state = onu_cfg.data.onu_state;
     if (err == BCM_ERR_OK) {
         switch (onu_state) {
             case BCMOLT_ONU_STATE_ACTIVE:
@@ -1070,25 +1071,23 @@
 
 Status DeleteOnu_(uint32_t intf_id, uint32_t onu_id,
     const char *vendor_id, const char *vendor_specific) {
+    bcmos_errno err = BCM_ERR_OK;
 
     OPENOLT_LOG(INFO, openolt_log_id,  "DeleteOnu ONU %d on PON %d : vendor id %s, vendor specific %s\n",
         onu_id, intf_id, vendor_id, vendor_specific_to_str(vendor_specific).c_str());
 
     // Need to deactivate before removing it (BAL rules)
-
     DeactivateOnu_(intf_id, onu_id, vendor_id, vendor_specific);
-    // Sleep to allow the state to propagate
-    // We need the subscriber terminal object to be admin down before removal
-    // Without sleep the race condition is lost by ~ 20 ms
-    std::this_thread::sleep_for(std::chrono::milliseconds(100));
-
-    // TODO: Delete the schedulers and queues.
+    err = wait_for_onu_deactivate_complete(intf_id, onu_id);
+    if (err) {
+        OPENOLT_LOG(ERROR, openolt_log_id, "failed to delete onu intf_id %d, onu_id %d\n",
+                intf_id, onu_id);
+        return bcm_to_grpc_err(err, "Failed to delete ONU");
+    }
 
     bcmolt_onu_cfg cfg_obj;
     bcmolt_onu_key key;
 
-    //OPENOLT_LOG(INFO, openolt_log_id, "Processing subscriber terminal cfg clear for sub_term_id %d  and intf_id %d\n",
-    //    onu_id, intf_id);
     OPENOLT_LOG(INFO, openolt_log_id, "Processing onu cfg clear for onu_id %d  and intf_id %d\n",
         onu_id, intf_id);
 
@@ -1096,10 +1095,9 @@
     key.pon_ni = intf_id;
     BCMOLT_CFG_INIT(&cfg_obj, onu, key);
 
-    bcmos_errno err = bcmolt_cfg_clear(dev_id, &cfg_obj.hdr);
+    err = bcmolt_cfg_clear(dev_id, &cfg_obj.hdr);
     if (err != BCM_ERR_OK)
     {
-       //OPENOLT_LOG(ERROR, openolt_log_id, "Failed to clear information for BAL subscriber_terminal_id %d, Interface ID %d, err = %s\n", onu_id, intf_id, bcmos_strerror(err));
        OPENOLT_LOG(ERROR, openolt_log_id, "Failed to clear information for BAL onu_id %d, Interface ID %d, err = %s\n", onu_id, intf_id, bcmos_strerror(err));
         return Status(grpc::StatusCode::INTERNAL, "Failed to delete ONU");
     }
diff --git a/agent/src/core_data.cc b/agent/src/core_data.cc
index dc98400..8cb51a8 100644
--- a/agent/src/core_data.cc
+++ b/agent/src/core_data.cc
@@ -122,11 +122,6 @@
 /* 'qmp_id_to_qmp_map' maps TM Queue Mapping Profile ID to TM Queue Mapping Profile */
 std::map<int, std::vector < uint32_t > > qmp_id_to_qmp_map;
 
-// Flag used to watch whether mocked alloc_cfg_compltd_key is added to alloc_cfg_compltd_map
-#ifdef TEST_MODE
-bool ALLOC_CFG_FLAG = false;
-#endif
-
 // Map used to track response from BAL for ITU PON Alloc Configuration.
 // The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
 // used for pushing (from BAL) and popping (at application) the results.
@@ -134,6 +129,12 @@
 // Lock to protect critical section data structure used for handling AllocObject configuration response.
 bcmos_fastlock alloc_cfg_wait_lock;
 
+// Map used to track response from BAL for Onu Deactivation Completed Indication
+// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// used for pushing (from BAL) and popping (at application) the results.
+std::map<onu_deact_compltd_key,  Queue<onu_deactivate_complete_result> *> onu_deact_compltd_map;
+// Lock to protect critical section data structure used for handling Onu Deactivation Completed Indication
+bcmos_fastlock onu_deactivate_wait_lock;
 
 /*** ACL Handling related data start ***/
 
diff --git a/agent/src/core_data.h b/agent/src/core_data.h
index 5c900f5..4de8114 100644
--- a/agent/src/core_data.h
+++ b/agent/src/core_data.h
@@ -41,6 +41,8 @@
 
 #define ALLOC_CFG_COMPLETE_WAIT_TIMEOUT 1000 // in milli-seconds
 
+#define ONU_DEACTIVATE_COMPLETE_WAIT_TIMEOUT 1000 // in milli-seconds
+
 #define MIN_ALLOC_ID_GPON 256
 #define MIN_ALLOC_ID_XGSPON 1024
 
@@ -104,9 +106,19 @@
     AllocCfgStatus status;
 } alloc_cfg_complete_result;
 
+typedef struct {
+    uint32_t pon_intf_id;
+    uint32_t onu_id;
+    bcmolt_result result;
+    bcmolt_deactivation_fail_reason reason;
+} onu_deactivate_complete_result;
+
 // key for map used for tracking ITU PON Alloc Configuration results from BAL
 typedef std::tuple<uint32_t, uint32_t> alloc_cfg_compltd_key;
 
+// key for map used for tracking Onu Deactivation Completed Indication
+typedef std::tuple<uint32_t, uint32_t> onu_deact_compltd_key;
+
 // The elements in this acl_classifier_key structure constitute key to
 // acl_classifier_to_acl_id_map.
 // Fill invalid values in the acl_classifier_key structure to -1.
@@ -183,17 +195,19 @@
 /* 'qmp_id_to_qmp_map' maps TM Queue Mapping Profile ID to TM Queue Mapping Profile */
 extern std::map<int, std::vector < uint32_t > > qmp_id_to_qmp_map;
 
-// Flag used to watch whether mocked alloc_cfg_compltd_key is added to alloc_cfg_compltd_map
-#ifdef TEST_MODE
-extern bool ALLOC_CFG_FLAG;
-#endif
-
 // Map used to track response from BAL for ITU PON Alloc Configuration.
 // The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
 // used for pushing (from BAL) and popping (at application) the results.
 extern std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *> alloc_cfg_compltd_map;
+// Map used to track response from BAL for Onu Deactivation Completed Indication
+// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// used for pushing (from BAL) and popping (at application) the results.
+extern std::map<onu_deact_compltd_key,  Queue<onu_deactivate_complete_result> *> onu_deact_compltd_map;
+
 // Lock to protect critical section data structure used for handling AllocObject configuration response.
 extern bcmos_fastlock alloc_cfg_wait_lock;
+// Lock to protect critical section data structure used for handling Onu deactivation completed Indication
+extern bcmos_fastlock onu_deactivate_wait_lock;
 
 
 /*** ACL Handling related data start ***/
diff --git a/agent/src/core_utils.cc b/agent/src/core_utils.cc
index f63ca52..b96c0d5 100644
--- a/agent/src/core_utils.cc
+++ b/agent/src/core_utils.cc
@@ -485,9 +485,6 @@
     alloc_cfg_compltd_key k(intf_id, alloc_id);
     alloc_cfg_compltd_map[k] =  &cfg_result;
     bcmos_errno err = BCM_ERR_OK;
-    #ifdef TEST_MODE
-    ALLOC_CFG_FLAG = true;
-    #endif
 
     // Try to pop the result from BAL with a timeout of ALLOC_CFG_COMPLETE_WAIT_TIMEOUT ms
     std::pair<alloc_cfg_complete_result, bool> result = cfg_result.pop(ALLOC_CFG_COMPLETE_WAIT_TIMEOUT);
@@ -499,9 +496,6 @@
         alloc_cfg_compltd_map[k] = NULL;
         bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
         err = BCM_ERR_INTERNAL;
-        #ifdef TEST_MODE
-        ALLOC_CFG_FLAG = false;
-        #endif
     }
     else if (result.first.status == ALLOC_CFG_STATUS_FAIL) {
         OPENOLT_LOG(ERROR, openolt_log_id, "error processing alloc cfg request intf_id %d, alloc_id %d\n",
@@ -535,9 +529,41 @@
     bcmos_fastlock_lock(&alloc_cfg_wait_lock);
     alloc_cfg_compltd_map.erase(k);
     bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
-    #ifdef TEST_MODE
-    ALLOC_CFG_FLAG = false;
-    #endif
+    return err;
+}
+
+// This method handles waiting for OnuDeactivate Completed Indication
+bcmos_errno wait_for_onu_deactivate_complete(uint32_t intf_id, uint32_t onu_id) {
+    Queue<onu_deactivate_complete_result> deact_result;
+    onu_deact_compltd_key k(intf_id, onu_id);
+    onu_deact_compltd_map[k] =  &deact_result;
+    bcmos_errno err = BCM_ERR_OK;
+
+    // Try to pop the result from BAL with a timeout of ONU_DEACTIVATE_COMPLETE_WAIT_TIMEOUT ms
+    std::pair<onu_deactivate_complete_result, bool> result = deact_result.pop(ONU_DEACTIVATE_COMPLETE_WAIT_TIMEOUT);
+    if (result.second == false) {
+        OPENOLT_LOG(ERROR, openolt_log_id, "timeout waiting for onu deactivate complete indication intf_id %d, onu_id %d\n",
+                    intf_id, onu_id);
+        // Invalidate the queue pointer.
+        bcmos_fastlock_lock(&onu_deactivate_wait_lock);
+        onu_deact_compltd_map[k] = NULL;
+        bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
+        err = BCM_ERR_INTERNAL;
+    }
+    else if (result.first.result == BCMOLT_RESULT_FAIL) {
+        OPENOLT_LOG(ERROR, openolt_log_id, "error processing onu deactivate request intf_id %d, onu_id %d, fail_reason %d\n",
+                    intf_id, onu_id, result.first.reason);
+        err = BCM_ERR_INTERNAL;
+    } else if (result.first.result == BCMOLT_RESULT_SUCCESS) {
+        OPENOLT_LOG(INFO, openolt_log_id, "success processing onu deactivate request intf_id %d, onu_id %d\n",
+                    intf_id, onu_id);
+    }
+
+    // Remove entry from map
+    bcmos_fastlock_lock(&onu_deactivate_wait_lock);
+    onu_deact_compltd_map.erase(k);
+    bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
+
     return err;
 }
 
diff --git a/agent/src/core_utils.h b/agent/src/core_utils.h
index 5a7c6bd..4ed72d2 100644
--- a/agent/src/core_utils.h
+++ b/agent/src/core_utils.h
@@ -68,6 +68,7 @@
 void clear_qos_type(uint32_t pon_intf_id, uint32_t onu_id, uint32_t uni_id);
 std::string GetDirection(int direction);
 bcmos_errno wait_for_alloc_action(uint32_t intf_id, uint32_t alloc_id, AllocCfgAction action);
+bcmos_errno wait_for_onu_deactivate_complete(uint32_t intf_id, uint32_t onu_id);
 char* openolt_read_sysinfo(const char* field_name, char* field_val);
 Status pushOltOperInd(uint32_t intf_id, const char *type, const char *state);
 void openolt_cli_get_prompt_cb(bcmcli_session *session, char *buf, uint32_t max_len);
diff --git a/agent/src/indications.cc b/agent/src/indications.cc
index 078d673..7e16476 100644
--- a/agent/src/indications.cc
+++ b/agent/src/indications.cc
@@ -869,6 +869,33 @@
                     onu_ind_data->set_oper_state("down");
                     onu_ind_data->set_admin_state("down");
                     onu_ind.set_allocated_onu_ind(onu_ind_data);
+
+                    onu_deact_compltd_key onu_key((uint32_t)key->pon_ni, (uint32_t) key->onu_id);
+                    onu_deactivate_complete_result res;
+                    res.pon_intf_id = (uint32_t)key->pon_ni;
+                    res.onu_id = (uint32_t) key->onu_id;
+                    res.result = data->status;
+                    res.reason = data->fail_reason;
+
+                    OPENOLT_LOG(INFO, openolt_log_id, "received onu deactivate result, pon intf %u, onu_id %u, status %u, reason %u\n",
+                            key->pon_ni, key->onu_id, data->status, data->fail_reason);
+
+                    bcmos_fastlock_lock(&onu_deactivate_wait_lock);
+                    // Push the result from BAL to queue
+                    std::map<onu_deact_compltd_key,  Queue<onu_deactivate_complete_result> *>::iterator it = onu_deact_compltd_map.find(onu_key);
+                    if (it == onu_deact_compltd_map.end()) {
+                        // could be case of spurious aysnc response, OR, the application timed-out waiting for response and cleared the key.
+                        bcmolt_msg_free(msg);
+                        OPENOLT_LOG(ERROR, openolt_log_id, "onu deactivate completed key not found for pon intf %u, onu_id %u\n",
+                            key->pon_ni, key->onu_id);
+                        bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
+                        return;
+                    }
+                    if (it->second) {
+                        // Push the result
+                        it->second->push(res);
+                    }
+                    bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
                 }
             }
     }
diff --git a/agent/test/src/test_core.cc b/agent/test/src/test_core.cc
index bb7baff..3d9ec41 100644
--- a/agent/test/src/test_core.cc
+++ b/agent/test/src/test_core.cc
@@ -25,7 +25,6 @@
 extern std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *> alloc_cfg_compltd_map;
 extern dev_log_id openolt_log_id;
 extern bcmos_fastlock alloc_cfg_wait_lock;
-extern bool ALLOC_CFG_FLAG;
 
 class TestOltEnable : public Test {
  protected:
@@ -968,6 +967,28 @@
 
         virtual void TearDown() {
         }
+    public:
+       static int PushOnuDeactCompltResult(bcmolt_result result, bcmolt_deactivation_fail_reason reason) {
+                onu_deactivate_complete_result res;
+                res.pon_intf_id = 0;
+                res.onu_id = 1;
+                res.result = result;
+                res.reason = reason;
+                // We need to wait for some time to allow the Onu Deactivation Reqeuest to be triggered
+                // before we push the result.
+                std::this_thread::sleep_for(std::chrono::milliseconds(100));
+                bcmos_fastlock_lock(&onu_deactivate_wait_lock);
+                onu_deact_compltd_key k(0, 1);
+                std::map<onu_deact_compltd_key,  Queue<onu_deactivate_complete_result> *>::iterator it = onu_deact_compltd_map.find(k);
+                if (it == onu_deact_compltd_map.end()) {
+                    OPENOLT_LOG(ERROR, openolt_log_id, "onu deact key not found for pon_intf=%d, onu_id=%d\n", 0, 1);
+                } else {
+                    it->second->push(res);
+                    OPENOLT_LOG(INFO, openolt_log_id, "Pushed ONU deact completed result\n");
+                }
+                bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
+                return 0;
+       }
 };
 
 // Test 1 - DeleteOnu success case
@@ -986,12 +1007,16 @@
     ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
     ON_CALL(balMock, bcmolt_cfg_clear(_, _)).WillByDefault(Return(onu_cfg_clear_res));
 
-    Status status = DeleteOnu_(pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+    future<int> push_onu_deact_complt_res = \
+             async(launch::async,TestDeleteOnu::PushOnuDeactCompltResult, BCMOLT_RESULT_SUCCESS, BCMOLT_DEACTIVATION_FAIL_REASON_NONE);
+    future<Status> future_res = async(launch::async, DeleteOnu_, pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+    Status status = future_res.get();
+    int res = push_onu_deact_complt_res.get();
     ASSERT_TRUE( status.error_message() == Status::OK.error_message() );
 }
 
-// Test 2 - DeleteOnu failure case
-TEST_F(TestDeleteOnu, DeleteOnuFailure) {
+// Test 2 - DeleteOnu failure case - BAL Clear ONU fails
+TEST_F(TestDeleteOnu, DeleteOnuFailureClearOnuFail) {
     bcmos_errno onu_cfg_get_stub_res = BCM_ERR_OK;
     bcmos_errno onu_oper_sub_res = BCM_ERR_OK;
     bcmos_errno onu_cfg_clear_res = BCM_ERR_INTERNAL;
@@ -1006,10 +1031,59 @@
     ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
     ON_CALL(balMock, bcmolt_cfg_clear(_, _)).WillByDefault(Return(onu_cfg_clear_res));
 
-    Status status = DeleteOnu_(pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+    future<int> push_onu_deact_complt_res = \
+                async(launch::async,TestDeleteOnu::PushOnuDeactCompltResult, BCMOLT_RESULT_SUCCESS, BCMOLT_DEACTIVATION_FAIL_REASON_NONE);
+    future<Status> future_res = async(launch::async, DeleteOnu_, pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+
+    Status status = future_res.get();
+    int res = push_onu_deact_complt_res.get();
     ASSERT_FALSE( status.error_message() == Status::OK.error_message() );
 }
 
+// Test 3 - DeleteOnu failure case - onu deactivation fails
+TEST_F(TestDeleteOnu, DeleteOnuFailureDeactivationFail) {
+    bcmos_errno onu_cfg_get_stub_res = BCM_ERR_OK;
+    bcmos_errno onu_oper_sub_res = BCM_ERR_OK;
+
+    bcmolt_onu_cfg onu_cfg;
+    bcmolt_onu_key onu_key;
+    BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
+    onu_cfg.data.onu_state = BCMOLT_ONU_STATE_ACTIVE;
+    EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
+                     .WillOnce(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+
+    ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
+
+    future<int> push_onu_deact_complt_res = \
+                async(launch::async,TestDeleteOnu::PushOnuDeactCompltResult, BCMOLT_RESULT_FAIL, BCMOLT_DEACTIVATION_FAIL_REASON_FAIL);
+    future<Status> future_res = async(launch::async, DeleteOnu_, pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+
+    Status status = future_res.get();
+    int res = push_onu_deact_complt_res.get();
+    ASSERT_FALSE( status.error_message() == Status::OK.error_message() );
+}
+
+// Test 4 - DeleteOnu failure case - onu deactivation timesout
+TEST_F(TestDeleteOnu, DeleteOnuFailureDeactivationTimeout) {
+    bcmos_errno onu_cfg_get_stub_res = BCM_ERR_OK;
+    bcmos_errno onu_oper_sub_res = BCM_ERR_OK;
+
+    bcmolt_onu_cfg onu_cfg;
+    bcmolt_onu_key onu_key;
+    BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
+    onu_cfg.data.onu_state = BCMOLT_ONU_STATE_ACTIVE;
+    EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
+                     .WillOnce(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+
+    ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
+
+    future<Status> future_res = async(launch::async, DeleteOnu_, pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+
+    Status status = future_res.get();
+    ASSERT_FALSE( status.error_message() == Status::OK.error_message() );
+}
+
+
 ////////////////////////////////////////////////////////////////////////////
 // For testing OmciMsgOut functionality
 ////////////////////////////////////////////////////////////////////////////
@@ -1574,27 +1648,27 @@
         }
 
     public:
-        static void PushAllocCfgResult(AllocObjectState state, AllocCfgStatus status) {
-            if(ALLOC_CFG_FLAG) {
-                alloc_cfg_compltd_key k(0, 1024);
-                alloc_cfg_complete_result res;
-                res.pon_intf_id = 0;
-                res.alloc_id = 1024;
-                res.state = state;
-                res.status = status;
+        static int PushAllocCfgResult(AllocObjectState state, AllocCfgStatus status) {
+            alloc_cfg_compltd_key k(0, 1024);
+            alloc_cfg_complete_result res;
+            res.pon_intf_id = 0;
+            res.alloc_id = 1024;
+            res.state = state;
+            res.status = status;
 
-                bcmos_fastlock_lock(&alloc_cfg_wait_lock);
-                std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *>::iterator it = alloc_cfg_compltd_map.find(k);
-                if (it == alloc_cfg_compltd_map.end()) {
-                    OPENOLT_LOG(ERROR, openolt_log_id, "alloc config key not found for alloc_id = %u, pon_intf = %u\n", 1024, 0);
-                } else {
-                    it->second->push(res);
-                    OPENOLT_LOG(INFO, openolt_log_id, "Pushed mocked alloc cfg result\n");
-                }
-                bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
+            // We need to wait for some time to allow the Alloc Cfg Request to be triggered
+            // before we push the result.
+            std::this_thread::sleep_for(std::chrono::milliseconds(100));
+            bcmos_fastlock_lock(&alloc_cfg_wait_lock);
+            std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *>::iterator it = alloc_cfg_compltd_map.find(k);
+            if (it == alloc_cfg_compltd_map.end()) {
+                OPENOLT_LOG(ERROR, openolt_log_id, "alloc config key not found for alloc_id = %u, pon_intf = %u\n", 1024, 0);
             } else {
-                PushAllocCfgResult(state, status); 
+                it->second->push(res);
+                OPENOLT_LOG(INFO, openolt_log_id, "Pushed mocked alloc cfg result\n");
             }
+            bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
+            return 0;
         }
 };
 
@@ -1612,9 +1686,11 @@
     ON_CALL(balMock, bcmolt_cfg_set(_, _)).WillByDefault(Return(olt_cfg_set_res));
 
     future<Status> future_res = async(launch::async, CreateTrafficSchedulers_, traffic_scheds);
-    async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_ACTIVE, ALLOC_CFG_STATUS_SUCCESS);
+    future<int> push_alloc_cfg_complt = \
+                async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_ACTIVE, ALLOC_CFG_STATUS_SUCCESS);
 
     Status status = future_res.get();
+    int res = push_alloc_cfg_complt.get();
     ASSERT_TRUE( status.error_message() == Status::OK.error_message() );
 }
 
@@ -1649,9 +1725,12 @@
     ON_CALL(balMock, bcmolt_cfg_set(_, _)).WillByDefault(Return(olt_cfg_set_res));
 
     future<Status> future_res = async(launch::async, CreateTrafficSchedulers_, traffic_scheds);
-    async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_ACTIVE, ALLOC_CFG_STATUS_FAIL);
+    future<int> push_alloc_cfg_complt = \
+                async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_ACTIVE, ALLOC_CFG_STATUS_FAIL);
 
     Status status = future_res.get();
+    int res = push_alloc_cfg_complt.get();
+
     ASSERT_TRUE( status.error_message() != Status::OK.error_message() );
 }
 
@@ -1669,9 +1748,11 @@
     ON_CALL(balMock, bcmolt_cfg_set(_, _)).WillByDefault(Return(olt_cfg_set_res));
 
     future<Status> future_res = async(launch::async, CreateTrafficSchedulers_, traffic_scheds);
-    async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_INACTIVE, ALLOC_CFG_STATUS_SUCCESS);
+    future<int> push_alloc_cfg_complt = \
+                async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_INACTIVE, ALLOC_CFG_STATUS_SUCCESS);
 
     Status status = future_res.get();
+    int res = push_alloc_cfg_complt.get();
     ASSERT_TRUE( status.error_message() != Status::OK.error_message() );
 }
 
@@ -1832,9 +1913,11 @@
     ON_CALL(balMock, bcmolt_cfg_set(_, _)).WillByDefault(Return(olt_cfg_set_res));
 
     future<Status> future_res = async(launch::async, CreateTrafficSchedulers_, traffic_scheds);
+    future<int> push_alloc_cfg_complt = \
     async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_ACTIVE, ALLOC_CFG_STATUS_SUCCESS);
 
     Status status = future_res.get();
+    int res = push_alloc_cfg_complt.get();
     ASSERT_TRUE( status.error_message() == Status::OK.error_message() );
 }
 
@@ -1852,9 +1935,11 @@
     ON_CALL(balMock, bcmolt_cfg_set(_, _)).WillByDefault(Return(olt_cfg_set_res));
 
     future<Status> future_res = async(launch::async, CreateTrafficSchedulers_, traffic_scheds);
+    future<int> push_alloc_cfg_complt = \
     async(launch::async, TestCreateTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_ACTIVE, ALLOC_CFG_STATUS_SUCCESS);
 
     Status status = future_res.get();
+    int res = push_alloc_cfg_complt.get();
     ASSERT_TRUE( status.error_message() == Status::OK.error_message() );
 }
 
@@ -1948,27 +2033,24 @@
         }
 
     public:
-        static void PushAllocCfgResult(AllocObjectState state, AllocCfgStatus status) {
-            if(ALLOC_CFG_FLAG) {
-                alloc_cfg_compltd_key k(0, 1025);
-                alloc_cfg_complete_result res;
-                res.pon_intf_id = 0;
-                res.alloc_id = 1025;
-                res.state = state;
-                res.status = status;
+        static int PushAllocCfgResult(AllocObjectState state, AllocCfgStatus status) {
+            alloc_cfg_compltd_key k(0, 1025);
+            alloc_cfg_complete_result res;
+            res.pon_intf_id = 0;
+            res.alloc_id = 1025;
+            res.state = state;
+            res.status = status;
 
-                bcmos_fastlock_lock(&alloc_cfg_wait_lock);
-                std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *>::iterator it = alloc_cfg_compltd_map.find(k);
-                if (it == alloc_cfg_compltd_map.end()) {
-                    OPENOLT_LOG(ERROR, openolt_log_id, "alloc config key not found for alloc_id = %u, pon_intf = %u\n", 1025, 0);
-                } else {
-                    it->second->push(res);
-                    OPENOLT_LOG(INFO, openolt_log_id, "Pushed mocked alloc cfg result\n");
-                }
-                bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
+            bcmos_fastlock_lock(&alloc_cfg_wait_lock);
+            std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *>::iterator it = alloc_cfg_compltd_map.find(k);
+            if (it == alloc_cfg_compltd_map.end()) {
+                OPENOLT_LOG(ERROR, openolt_log_id, "alloc config key not found for alloc_id = %u, pon_intf = %u\n", 1025, 0);
             } else {
-                PushAllocCfgResult(state, status); 
+                it->second->push(res);
+                OPENOLT_LOG(INFO, openolt_log_id, "Pushed mocked alloc cfg result\n");
             }
+            bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
+            return 0;
         }
 };
 
@@ -1988,9 +2070,11 @@
                      .WillOnce(DoAll(SetArg1ToBcmOltPonCfg(pon_cfg), Return(olt_cfg_get_pon_stub_res)));
 
     future<Status> future_res = async(launch::async, RemoveTrafficSchedulers_, traffic_scheds);
+    future<int> push_alloc_cfg_complt = \
     async(launch::async, TestRemoveTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_NOT_CONFIGURED, ALLOC_CFG_STATUS_SUCCESS);
 
     Status status = future_res.get();
+    int res = push_alloc_cfg_complt.get();
     ASSERT_TRUE( status.error_message() == Status::OK.error_message() );
 }
 
@@ -2010,9 +2094,11 @@
                      .WillOnce(DoAll(SetArg1ToBcmOltPonCfg(pon_cfg), Return(olt_cfg_get_pon_stub_res)));
 
     future<Status> future_res = async(launch::async, RemoveTrafficSchedulers_, traffic_scheds);
+    future<int> push_alloc_cfg_complt = \
     async(launch::async, TestRemoveTrafficSchedulers::PushAllocCfgResult, ALLOC_OBJECT_STATE_INACTIVE, ALLOC_CFG_STATUS_SUCCESS);
 
     Status status = future_res.get();
+    int res = push_alloc_cfg_complt.get();
     ASSERT_TRUE( status.error_message() != Status::OK.error_message() );
 }