VOL-2682: On onu los and onu dgi, upstream schedulers removal, deactivate
          and delete onu should work
          - UT for Deactivate and Delete Onu
          - Bumped version to 2.1.4

Change-Id: I115d0965860516282729483833fdc71a84cef0ee
diff --git a/VERSION b/VERSION
index ac2cdeb..7d2ed7c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1.3
+2.1.4
diff --git a/agent/src/core_api_handler.cc b/agent/src/core_api_handler.cc
index 006e6fb..e602779 100644
--- a/agent/src/core_api_handler.cc
+++ b/agent/src/core_api_handler.cc
@@ -756,8 +756,9 @@
     bcmolt_pon_interface_key intf_key = {.pon_ni = (bcmolt_interface)intf_id};
     bcmolt_pon_interface_set_pon_interface_state pon_interface_set_state;
     bcmolt_interface_state state;
+    bcmolt_status los_status;
 
-    err = get_pon_interface_status((bcmolt_interface)intf_id, &state);
+    err = get_pon_interface_status((bcmolt_interface)intf_id, &state, &los_status);
     if (err == BCM_ERR_OK) {
         if (state == BCMOLT_INTERFACE_STATE_ACTIVE_WORKING) {
             OPENOLT_LOG(WARNING, openolt_log_id, "PON interface: %d already enabled\n", intf_id);
@@ -1062,6 +1063,7 @@
                     OPENOLT_LOG(ERROR, openolt_log_id, "Failed to deactivate ONU %d on PON %d, err = %s\n", onu_id, intf_id, bcmos_strerror(err));
                     return bcm_to_grpc_err(err, "Failed to deactivate ONU");
                 }
+                OPENOLT_LOG(INFO, openolt_log_id, "Deactivated ONU, onu_id %d on PON %d\n", onu_id, intf_id);
                 break;
         }
     }
@@ -1072,16 +1074,34 @@
 Status DeleteOnu_(uint32_t intf_id, uint32_t onu_id,
     const char *vendor_id, const char *vendor_specific) {
     bcmos_errno err = BCM_ERR_OK;
+    bcmolt_onu_state onu_state;
 
     OPENOLT_LOG(INFO, openolt_log_id,  "DeleteOnu ONU %d on PON %d : vendor id %s, vendor specific %s\n",
         onu_id, intf_id, vendor_id, vendor_specific_to_str(vendor_specific).c_str());
 
     // Need to deactivate before removing it (BAL rules)
     DeactivateOnu_(intf_id, onu_id, vendor_id, vendor_specific);
-    err = wait_for_onu_deactivate_complete(intf_id, onu_id);
-    if (err) {
-        OPENOLT_LOG(ERROR, openolt_log_id, "failed to delete onu intf_id %d, onu_id %d\n",
-                intf_id, onu_id);
+
+    err = get_onu_status((bcmolt_interface)intf_id, onu_id, &onu_state);
+    if (err == BCM_ERR_OK) {
+        if (onu_state == BCMOLT_ONU_STATE_ACTIVE) {
+            OPENOLT_LOG(INFO, openolt_log_id, "Onu is Active, onu_id: %d, waiting for onu deactivate complete response\n",
+                intf_id);
+            err = wait_for_onu_deactivate_complete(intf_id, onu_id);
+            if (err) {
+                OPENOLT_LOG(ERROR, openolt_log_id, "failed to delete onu intf_id %d, onu_id %d\n",
+                        intf_id, onu_id);
+                return bcm_to_grpc_err(err, "Failed to delete ONU");
+            }
+        }
+        else if (onu_state == BCMOLT_ONU_STATE_INACTIVE) {
+            OPENOLT_LOG(INFO, openolt_log_id, "Onu is Inactive, onu_id: %d, not waiting for onu deactivate complete response\n",
+                intf_id);
+        }
+    }
+    else {
+        OPENOLT_LOG(ERROR, openolt_log_id, "Failed to fetch Onu status, onu_id = %d, intf_id = %d, err = %s\n",
+            onu_id, intf_id, bcmos_strerror(err));
         return bcm_to_grpc_err(err, "Failed to delete ONU");
     }
 
@@ -1102,6 +1122,7 @@
         return Status(grpc::StatusCode::INTERNAL, "Failed to delete ONU");
     }
 
+    OPENOLT_LOG(INFO, openolt_log_id, "Deleted ONU, onu_id %d on PON %d\n", onu_id, intf_id);
     return Status::OK;
 }
 
@@ -2037,6 +2058,7 @@
 
     bcmos_errno err;
     bcmolt_interface_state state;
+    bcmolt_status los_status;
     uint16_t sched_id;
 
     if (direction == upstream) {
@@ -2054,10 +2076,10 @@
             return err;
         }
 
-        err = get_pon_interface_status((bcmolt_interface)intf_id, &state);
+        err = get_pon_interface_status((bcmolt_interface)intf_id, &state, &los_status);
         if (err == BCM_ERR_OK) {
-            if (state == BCMOLT_INTERFACE_STATE_ACTIVE_WORKING) {
-                OPENOLT_LOG(INFO, openolt_log_id, "PON interface: %d is enabled, waiting for alloc cfg clear response\n",
+            if (state == BCMOLT_INTERFACE_STATE_ACTIVE_WORKING && los_status == BCMOLT_STATUS_OFF) {
+                OPENOLT_LOG(INFO, openolt_log_id, "PON interface: %d is enabled and LoS status is OFF, waiting for alloc cfg clear response\n",
                     intf_id);
                 err = wait_for_alloc_action(intf_id, alloc_id, ALLOC_OBJECT_DELETE);
                 if (err) {
@@ -2066,13 +2088,16 @@
                     return err;
                 }
             }
+            else if (state == BCMOLT_INTERFACE_STATE_ACTIVE_WORKING && los_status == BCMOLT_STATUS_ON) {
+                OPENOLT_LOG(INFO, openolt_log_id, "PON interface: %d is enabled but LoS status is ON, not waiting for alloc cfg clear response\n",
+                    intf_id);
+            }
             else if (state == BCMOLT_INTERFACE_STATE_INACTIVE) {
                 OPENOLT_LOG(INFO, openolt_log_id, "PON interface: %d is disabled, not waiting for alloc cfg clear response\n",
                     intf_id);
             }
-        }
-        else {
-            OPENOLT_LOG(ERROR, openolt_log_id, "Failed to fetch PON interface state, intf_id = %d, err = %s\n",
+        } else {
+            OPENOLT_LOG(ERROR, openolt_log_id, "Failed to fetch PON interface status, intf_id = %d, err = %s\n",
                 intf_id, bcmos_strerror(err));
             return err;
         }
diff --git a/agent/src/core_utils.cc b/agent/src/core_utils.cc
index b96c0d5..9bcee03 100644
--- a/agent/src/core_utils.cc
+++ b/agent/src/core_utils.cc
@@ -691,7 +691,31 @@
     }
 }
 
-bcmos_errno get_pon_interface_status(bcmolt_interface pon_ni, bcmolt_interface_state *state) {
+bcmos_errno get_onu_status(bcmolt_interface pon_ni, int onu_id, bcmolt_onu_state *onu_state) {
+    bcmos_errno err;
+    bcmolt_onu_cfg onu_cfg;
+    bcmolt_onu_key onu_key;
+    onu_key.pon_ni = pon_ni;
+    onu_key.onu_id = onu_id;
+
+    BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
+    BCMOLT_FIELD_SET_PRESENT(&onu_cfg.data, onu_cfg_data, onu_state);
+    BCMOLT_FIELD_SET_PRESENT(&onu_cfg.data, onu_cfg_data, itu);
+    #ifdef TEST_MODE
+    // It is impossible to mock the setting of onu_cfg.data.onu_state because
+    // the actual bcmolt_cfg_get passes the address of onu_cfg.hdr and we cannot
+    // set the onu_cfg.data.onu_state. So a new stub function is created and address
+    // of onu_cfg is passed. This is one-of case where we need to add test specific
+    // code in production code.
+    err = bcmolt_cfg_get__onu_state_stub(dev_id, &onu_cfg);
+    #else
+    err = bcmolt_cfg_get(dev_id, &onu_cfg.hdr);
+    #endif
+    *onu_state = onu_cfg.data.onu_state;
+    return err;
+}
+
+bcmos_errno get_pon_interface_status(bcmolt_interface pon_ni, bcmolt_interface_state *state, bcmolt_status *los_status) {
     bcmos_errno err;
     bcmolt_pon_interface_key pon_key;
     bcmolt_pon_interface_cfg pon_cfg;
@@ -699,6 +723,7 @@
 
     BCMOLT_CFG_INIT(&pon_cfg, pon_interface, pon_key);
     BCMOLT_FIELD_SET_PRESENT(&pon_cfg.data, pon_interface_cfg_data, state);
+    BCMOLT_FIELD_SET_PRESENT(&pon_cfg.data, pon_interface_cfg_data, los_status);
     BCMOLT_FIELD_SET_PRESENT(&pon_cfg.data, pon_interface_cfg_data, itu);
     #ifdef TEST_MODE
     // It is impossible to mock the setting of pon_cfg.data.state because
@@ -711,6 +736,7 @@
     err = bcmolt_cfg_get(dev_id, &pon_cfg.hdr);
     #endif
     *state = pon_cfg.data.state;
+    *los_status = pon_cfg.data.los_status;
     return err;
 }
 
diff --git a/agent/src/core_utils.h b/agent/src/core_utils.h
index 4ed72d2..ad48474 100644
--- a/agent/src/core_utils.h
+++ b/agent/src/core_utils.h
@@ -77,7 +77,8 @@
 bcmos_errno bcm_cli_quit(bcmcli_session *session, const bcmcli_cmd_parm parm[], uint16_t n_parms);
 int get_status_bcm_cli_quit(void);
 bcmos_errno bcmolt_apiend_cli_init();
-bcmos_errno get_pon_interface_status(bcmolt_interface pon_ni, bcmolt_interface_state *state);
+bcmos_errno get_pon_interface_status(bcmolt_interface pon_ni, bcmolt_interface_state *state, bcmolt_status *los_status);
+bcmos_errno get_onu_status(bcmolt_interface pon_ni, int onu_id, bcmolt_onu_state *onu_state);
 bcmos_errno bcmolt_cfg_get_mult_retry(bcmolt_oltid olt, bcmolt_cfg *cfg);
 unsigned NumNniIf_();
 unsigned NumPonIf_();
diff --git a/agent/src/indications.cc b/agent/src/indications.cc
index 7e16476..0d643e0 100644
--- a/agent/src/indications.cc
+++ b/agent/src/indications.cc
@@ -233,6 +233,10 @@
 
                     intf_ind->set_intf_id(key->pon_ni);
                     SET_OPER_STATE(intf_ind, data->new_state);
+
+                    OPENOLT_LOG(INFO, openolt_log_id, "intf indication, intf_type %s, intf_id %d, oper_state %s\n",
+                        bcmolt_to_grpc_intf_type(BCMOLT_INTERFACE_TYPE_PON).c_str(), key->pon_ni, intf_ind->oper_state().c_str());
+
                     ind.set_allocated_intf_ind(intf_ind);
                     break;
                 }
@@ -316,6 +320,11 @@
                 {
                     bcmolt_onu_xgpon_alarm_data *data = &((bcmolt_onu_xgpon_alarm *)msg)->data;
                     bcmolt_onu_key *key = &((bcmolt_onu_xgpon_alarm *)msg)->key;
+
+                    int port_no = interface_key_to_port_no(key->pon_ni, BCMOLT_INTERFACE_TYPE_PON);
+
+                    OPENOLT_LOG(INFO, openolt_log_id, "onu alarm indication, pon_ni %d, onu_id %d, port_no %d, los_status %s, lob_status %s, lopc_miss_status %s, lopc_mic_error_status %s\n", key->pon_ni, key->onu_id, port_no, alarm_status_to_string(data->xgpon_onu_alarm.losi).c_str(), alarm_status_to_string(data->xgpon_onu_alarm.lobi).c_str(), alarm_status_to_string(data->xgpon_onu_alarm.lopci_miss).c_str(), alarm_status_to_string(data->xgpon_onu_alarm.lopci_mic_error).c_str());
+
                     onu_alarm_ind->set_los_status(alarm_status_to_string(data->xgpon_onu_alarm.losi));
                     onu_alarm_ind->set_lob_status(alarm_status_to_string(data->xgpon_onu_alarm.lobi));
                     onu_alarm_ind->set_lopc_miss_status(alarm_status_to_string(data->xgpon_onu_alarm.lopci_miss));
@@ -359,6 +368,11 @@
                     bcmolt_onu_dgi* dgi_data = (bcmolt_onu_dgi *)msg;
                     bcmolt_onu_key *key = &((bcmolt_onu_dgi *)msg)->key;
 
+                    int port_no = interface_key_to_port_no(key->pon_ni, BCMOLT_INTERFACE_TYPE_PON);
+
+                    OPENOLT_LOG(INFO, openolt_log_id, "onu dyinggasp indication, pon_ni %d, onu_id %d, port_no %d, status %s\n",
+                        key->pon_ni, key->onu_id, port_no, alarm_status_to_string(dgi_data->data.alarm_status).c_str());
+
                     dgi_ind->set_status(alarm_status_to_string(dgi_data->data.alarm_status));
                     dgi_ind->set_intf_id(key->pon_ni);
                     dgi_ind->set_onu_id(key->onu_id);
@@ -886,7 +900,7 @@
                     if (it == onu_deact_compltd_map.end()) {
                         // could be case of spurious aysnc response, OR, the application timed-out waiting for response and cleared the key.
                         bcmolt_msg_free(msg);
-                        OPENOLT_LOG(ERROR, openolt_log_id, "onu deactivate completed key not found for pon intf %u, onu_id %u\n",
+                        OPENOLT_LOG(WARNING, openolt_log_id, "onu deactivate completed key not found for pon intf %u, onu_id %u\n",
                             key->pon_ni, key->onu_id);
                         bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
                         return;
diff --git a/agent/test/src/test_core.cc b/agent/test/src/test_core.cc
index d82a697..5082fac 100644
--- a/agent/test/src/test_core.cc
+++ b/agent/test/src/test_core.cc
@@ -1002,7 +1002,7 @@
     BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
     onu_cfg.data.onu_state = BCMOLT_ONU_STATE_ACTIVE;
     EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
-                     .WillOnce(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+                     .WillRepeatedly(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
 
     ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
     ON_CALL(balMock, bcmolt_cfg_clear(_, _)).WillByDefault(Return(onu_cfg_clear_res));
@@ -1026,7 +1026,7 @@
     BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
     onu_cfg.data.onu_state = BCMOLT_ONU_STATE_ACTIVE;
     EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
-                     .WillOnce(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+                     .WillRepeatedly(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
 
     ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
     ON_CALL(balMock, bcmolt_cfg_clear(_, _)).WillByDefault(Return(onu_cfg_clear_res));
@@ -1050,7 +1050,7 @@
     BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
     onu_cfg.data.onu_state = BCMOLT_ONU_STATE_ACTIVE;
     EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
-                     .WillOnce(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+                     .WillRepeatedly(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
 
     ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
 
@@ -1073,7 +1073,7 @@
     BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
     onu_cfg.data.onu_state = BCMOLT_ONU_STATE_ACTIVE;
     EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
-                     .WillOnce(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+                     .WillRepeatedly(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
 
     ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
 
@@ -1083,6 +1083,44 @@
     ASSERT_FALSE( status.error_message() == Status::OK.error_message() );
 }
 
+// Test 5 - DeleteOnu success case - Onu is Inactive so won't wait for onu deactivation response
+TEST_F(TestDeleteOnu, DeleteOnuSuccessDontwaitforDeactivationResp) {
+    bcmos_errno onu_cfg_get_stub_res = BCM_ERR_OK;
+    bcmos_errno onu_oper_sub_res = BCM_ERR_OK;
+    bcmos_errno onu_cfg_clear_res = BCM_ERR_OK;
+
+    bcmolt_onu_cfg onu_cfg;
+    bcmolt_onu_key onu_key;
+    BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
+    onu_cfg.data.onu_state = BCMOLT_ONU_STATE_INACTIVE;
+    EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
+                     .WillRepeatedly(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+
+    ON_CALL(balMock, bcmolt_oper_submit(_, _)).WillByDefault(Return(onu_oper_sub_res));
+    ON_CALL(balMock, bcmolt_cfg_clear(_, _)).WillByDefault(Return(onu_cfg_clear_res));
+
+    future<Status> future_res = async(launch::async, DeleteOnu_, pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+
+    Status status = future_res.get();
+    ASSERT_TRUE( status.error_message() == Status::OK.error_message() );
+}
+
+// Test 6 - DeleteOnu failure case - Failed to fetch Onu status
+TEST_F(TestDeleteOnu, DeleteOnuFailureFetchOnuStatusFailed) {
+    bcmos_errno onu_cfg_get_stub_res = BCM_ERR_INTERNAL;
+
+    bcmolt_onu_cfg onu_cfg;
+    bcmolt_onu_key onu_key;
+    BCMOLT_CFG_INIT(&onu_cfg, onu, onu_key);
+    onu_cfg.data.onu_state = BCMOLT_ONU_STATE_INACTIVE;
+    EXPECT_GLOBAL_CALL(bcmolt_cfg_get__onu_state_stub, bcmolt_cfg_get__onu_state_stub(_, _))
+                     .WillRepeatedly(DoAll(SetArg1ToBcmOltOnuCfg(onu_cfg), Return(onu_cfg_get_stub_res)));
+
+    future<Status> future_res = async(launch::async, DeleteOnu_, pon_id, onu_id, vendor_id.c_str(), vendor_specific.c_str());
+
+    Status status = future_res.get();
+    ASSERT_FALSE( status.error_message() == Status::OK.error_message() );
+}
 
 ////////////////////////////////////////////////////////////////////////////
 // For testing OmciMsgOut functionality