VOL-2555:
- Wait for ONU Deactivation Completed Indication before clearing the ONU
- Add unit tests for delete onu timeout and failure cases
- Fix UT of DeleteScheduler
Change-Id: Ie287f733934ca5f6eb06b365e426074622724bb7
diff --git a/agent/src/core_api_handler.cc b/agent/src/core_api_handler.cc
index 053c82c..7a90107 100644
--- a/agent/src/core_api_handler.cc
+++ b/agent/src/core_api_handler.cc
@@ -251,6 +251,7 @@
bcmos_fastlock_init(&data_lock, 0);
bcmos_fastlock_init(&alloc_cfg_wait_lock, 0);
+ bcmos_fastlock_init(&onu_deactivate_wait_lock, 0);
OPENOLT_LOG(INFO, openolt_log_id, "Enable OLT - %s-%s\n", VENDOR_ID, MODEL_ID);
//check BCM daemon is connected or not
@@ -1046,10 +1047,10 @@
// of onu_cfg is passed. This is one-of case where we need to add test specific
// code in production code.
err = bcmolt_cfg_get__onu_state_stub(dev_id, &onu_cfg);
- onu_state = onu_cfg.data.onu_state;
#else
err = bcmolt_cfg_get(dev_id, &onu_cfg.hdr);
#endif
+ onu_state = onu_cfg.data.onu_state;
if (err == BCM_ERR_OK) {
switch (onu_state) {
case BCMOLT_ONU_STATE_ACTIVE:
@@ -1070,25 +1071,23 @@
Status DeleteOnu_(uint32_t intf_id, uint32_t onu_id,
const char *vendor_id, const char *vendor_specific) {
+ bcmos_errno err = BCM_ERR_OK;
OPENOLT_LOG(INFO, openolt_log_id, "DeleteOnu ONU %d on PON %d : vendor id %s, vendor specific %s\n",
onu_id, intf_id, vendor_id, vendor_specific_to_str(vendor_specific).c_str());
// Need to deactivate before removing it (BAL rules)
-
DeactivateOnu_(intf_id, onu_id, vendor_id, vendor_specific);
- // Sleep to allow the state to propagate
- // We need the subscriber terminal object to be admin down before removal
- // Without sleep the race condition is lost by ~ 20 ms
- std::this_thread::sleep_for(std::chrono::milliseconds(100));
-
- // TODO: Delete the schedulers and queues.
+ err = wait_for_onu_deactivate_complete(intf_id, onu_id);
+ if (err) {
+ OPENOLT_LOG(ERROR, openolt_log_id, "failed to delete onu intf_id %d, onu_id %d\n",
+ intf_id, onu_id);
+ return bcm_to_grpc_err(err, "Failed to delete ONU");
+ }
bcmolt_onu_cfg cfg_obj;
bcmolt_onu_key key;
- //OPENOLT_LOG(INFO, openolt_log_id, "Processing subscriber terminal cfg clear for sub_term_id %d and intf_id %d\n",
- // onu_id, intf_id);
OPENOLT_LOG(INFO, openolt_log_id, "Processing onu cfg clear for onu_id %d and intf_id %d\n",
onu_id, intf_id);
@@ -1096,10 +1095,9 @@
key.pon_ni = intf_id;
BCMOLT_CFG_INIT(&cfg_obj, onu, key);
- bcmos_errno err = bcmolt_cfg_clear(dev_id, &cfg_obj.hdr);
+ err = bcmolt_cfg_clear(dev_id, &cfg_obj.hdr);
if (err != BCM_ERR_OK)
{
- //OPENOLT_LOG(ERROR, openolt_log_id, "Failed to clear information for BAL subscriber_terminal_id %d, Interface ID %d, err = %s\n", onu_id, intf_id, bcmos_strerror(err));
OPENOLT_LOG(ERROR, openolt_log_id, "Failed to clear information for BAL onu_id %d, Interface ID %d, err = %s\n", onu_id, intf_id, bcmos_strerror(err));
return Status(grpc::StatusCode::INTERNAL, "Failed to delete ONU");
}
diff --git a/agent/src/core_data.cc b/agent/src/core_data.cc
index dc98400..8cb51a8 100644
--- a/agent/src/core_data.cc
+++ b/agent/src/core_data.cc
@@ -122,11 +122,6 @@
/* 'qmp_id_to_qmp_map' maps TM Queue Mapping Profile ID to TM Queue Mapping Profile */
std::map<int, std::vector < uint32_t > > qmp_id_to_qmp_map;
-// Flag used to watch whether mocked alloc_cfg_compltd_key is added to alloc_cfg_compltd_map
-#ifdef TEST_MODE
-bool ALLOC_CFG_FLAG = false;
-#endif
-
// Map used to track response from BAL for ITU PON Alloc Configuration.
// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
// used for pushing (from BAL) and popping (at application) the results.
@@ -134,6 +129,12 @@
// Lock to protect critical section data structure used for handling AllocObject configuration response.
bcmos_fastlock alloc_cfg_wait_lock;
+// Map used to track response from BAL for Onu Deactivation Completed Indication
+// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// used for pushing (from BAL) and popping (at application) the results.
+std::map<onu_deact_compltd_key, Queue<onu_deactivate_complete_result> *> onu_deact_compltd_map;
+// Lock to protect critical section data structure used for handling Onu Deactivation Completed Indication
+bcmos_fastlock onu_deactivate_wait_lock;
/*** ACL Handling related data start ***/
diff --git a/agent/src/core_data.h b/agent/src/core_data.h
index 5c900f5..4de8114 100644
--- a/agent/src/core_data.h
+++ b/agent/src/core_data.h
@@ -41,6 +41,8 @@
#define ALLOC_CFG_COMPLETE_WAIT_TIMEOUT 1000 // in milli-seconds
+#define ONU_DEACTIVATE_COMPLETE_WAIT_TIMEOUT 1000 // in milli-seconds
+
#define MIN_ALLOC_ID_GPON 256
#define MIN_ALLOC_ID_XGSPON 1024
@@ -104,9 +106,19 @@
AllocCfgStatus status;
} alloc_cfg_complete_result;
+typedef struct {
+ uint32_t pon_intf_id;
+ uint32_t onu_id;
+ bcmolt_result result;
+ bcmolt_deactivation_fail_reason reason;
+} onu_deactivate_complete_result;
+
// key for map used for tracking ITU PON Alloc Configuration results from BAL
typedef std::tuple<uint32_t, uint32_t> alloc_cfg_compltd_key;
+// key for map used for tracking Onu Deactivation Completed Indication
+typedef std::tuple<uint32_t, uint32_t> onu_deact_compltd_key;
+
// The elements in this acl_classifier_key structure constitute key to
// acl_classifier_to_acl_id_map.
// Fill invalid values in the acl_classifier_key structure to -1.
@@ -183,17 +195,19 @@
/* 'qmp_id_to_qmp_map' maps TM Queue Mapping Profile ID to TM Queue Mapping Profile */
extern std::map<int, std::vector < uint32_t > > qmp_id_to_qmp_map;
-// Flag used to watch whether mocked alloc_cfg_compltd_key is added to alloc_cfg_compltd_map
-#ifdef TEST_MODE
-extern bool ALLOC_CFG_FLAG;
-#endif
-
// Map used to track response from BAL for ITU PON Alloc Configuration.
// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
// used for pushing (from BAL) and popping (at application) the results.
extern std::map<alloc_cfg_compltd_key, Queue<alloc_cfg_complete_result> *> alloc_cfg_compltd_map;
+// Map used to track response from BAL for Onu Deactivation Completed Indication
+// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// used for pushing (from BAL) and popping (at application) the results.
+extern std::map<onu_deact_compltd_key, Queue<onu_deactivate_complete_result> *> onu_deact_compltd_map;
+
// Lock to protect critical section data structure used for handling AllocObject configuration response.
extern bcmos_fastlock alloc_cfg_wait_lock;
+// Lock to protect critical section data structure used for handling Onu deactivation completed Indication
+extern bcmos_fastlock onu_deactivate_wait_lock;
/*** ACL Handling related data start ***/
diff --git a/agent/src/core_utils.cc b/agent/src/core_utils.cc
index f63ca52..b96c0d5 100644
--- a/agent/src/core_utils.cc
+++ b/agent/src/core_utils.cc
@@ -485,9 +485,6 @@
alloc_cfg_compltd_key k(intf_id, alloc_id);
alloc_cfg_compltd_map[k] = &cfg_result;
bcmos_errno err = BCM_ERR_OK;
- #ifdef TEST_MODE
- ALLOC_CFG_FLAG = true;
- #endif
// Try to pop the result from BAL with a timeout of ALLOC_CFG_COMPLETE_WAIT_TIMEOUT ms
std::pair<alloc_cfg_complete_result, bool> result = cfg_result.pop(ALLOC_CFG_COMPLETE_WAIT_TIMEOUT);
@@ -499,9 +496,6 @@
alloc_cfg_compltd_map[k] = NULL;
bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
err = BCM_ERR_INTERNAL;
- #ifdef TEST_MODE
- ALLOC_CFG_FLAG = false;
- #endif
}
else if (result.first.status == ALLOC_CFG_STATUS_FAIL) {
OPENOLT_LOG(ERROR, openolt_log_id, "error processing alloc cfg request intf_id %d, alloc_id %d\n",
@@ -535,9 +529,41 @@
bcmos_fastlock_lock(&alloc_cfg_wait_lock);
alloc_cfg_compltd_map.erase(k);
bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
- #ifdef TEST_MODE
- ALLOC_CFG_FLAG = false;
- #endif
+ return err;
+}
+
+// This method handles waiting for OnuDeactivate Completed Indication
+bcmos_errno wait_for_onu_deactivate_complete(uint32_t intf_id, uint32_t onu_id) {
+ Queue<onu_deactivate_complete_result> deact_result;
+ onu_deact_compltd_key k(intf_id, onu_id);
+ onu_deact_compltd_map[k] = &deact_result;
+ bcmos_errno err = BCM_ERR_OK;
+
+ // Try to pop the result from BAL with a timeout of ONU_DEACTIVATE_COMPLETE_WAIT_TIMEOUT ms
+ std::pair<onu_deactivate_complete_result, bool> result = deact_result.pop(ONU_DEACTIVATE_COMPLETE_WAIT_TIMEOUT);
+ if (result.second == false) {
+ OPENOLT_LOG(ERROR, openolt_log_id, "timeout waiting for onu deactivate complete indication intf_id %d, onu_id %d\n",
+ intf_id, onu_id);
+ // Invalidate the queue pointer.
+ bcmos_fastlock_lock(&onu_deactivate_wait_lock);
+ onu_deact_compltd_map[k] = NULL;
+ bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
+ err = BCM_ERR_INTERNAL;
+ }
+ else if (result.first.result == BCMOLT_RESULT_FAIL) {
+ OPENOLT_LOG(ERROR, openolt_log_id, "error processing onu deactivate request intf_id %d, onu_id %d, fail_reason %d\n",
+ intf_id, onu_id, result.first.reason);
+ err = BCM_ERR_INTERNAL;
+ } else if (result.first.result == BCMOLT_RESULT_SUCCESS) {
+ OPENOLT_LOG(INFO, openolt_log_id, "success processing onu deactivate request intf_id %d, onu_id %d\n",
+ intf_id, onu_id);
+ }
+
+ // Remove entry from map
+ bcmos_fastlock_lock(&onu_deactivate_wait_lock);
+ onu_deact_compltd_map.erase(k);
+ bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
+
return err;
}
diff --git a/agent/src/core_utils.h b/agent/src/core_utils.h
index 5a7c6bd..4ed72d2 100644
--- a/agent/src/core_utils.h
+++ b/agent/src/core_utils.h
@@ -68,6 +68,7 @@
void clear_qos_type(uint32_t pon_intf_id, uint32_t onu_id, uint32_t uni_id);
std::string GetDirection(int direction);
bcmos_errno wait_for_alloc_action(uint32_t intf_id, uint32_t alloc_id, AllocCfgAction action);
+bcmos_errno wait_for_onu_deactivate_complete(uint32_t intf_id, uint32_t onu_id);
char* openolt_read_sysinfo(const char* field_name, char* field_val);
Status pushOltOperInd(uint32_t intf_id, const char *type, const char *state);
void openolt_cli_get_prompt_cb(bcmcli_session *session, char *buf, uint32_t max_len);
diff --git a/agent/src/indications.cc b/agent/src/indications.cc
index 078d673..7e16476 100644
--- a/agent/src/indications.cc
+++ b/agent/src/indications.cc
@@ -869,6 +869,33 @@
onu_ind_data->set_oper_state("down");
onu_ind_data->set_admin_state("down");
onu_ind.set_allocated_onu_ind(onu_ind_data);
+
+ onu_deact_compltd_key onu_key((uint32_t)key->pon_ni, (uint32_t) key->onu_id);
+ onu_deactivate_complete_result res;
+ res.pon_intf_id = (uint32_t)key->pon_ni;
+ res.onu_id = (uint32_t) key->onu_id;
+ res.result = data->status;
+ res.reason = data->fail_reason;
+
+ OPENOLT_LOG(INFO, openolt_log_id, "received onu deactivate result, pon intf %u, onu_id %u, status %u, reason %u\n",
+ key->pon_ni, key->onu_id, data->status, data->fail_reason);
+
+ bcmos_fastlock_lock(&onu_deactivate_wait_lock);
+ // Push the result from BAL to queue
+ std::map<onu_deact_compltd_key, Queue<onu_deactivate_complete_result> *>::iterator it = onu_deact_compltd_map.find(onu_key);
+ if (it == onu_deact_compltd_map.end()) {
+ // could be case of spurious aysnc response, OR, the application timed-out waiting for response and cleared the key.
+ bcmolt_msg_free(msg);
+ OPENOLT_LOG(ERROR, openolt_log_id, "onu deactivate completed key not found for pon intf %u, onu_id %u\n",
+ key->pon_ni, key->onu_id);
+ bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
+ return;
+ }
+ if (it->second) {
+ // Push the result
+ it->second->push(res);
+ }
+ bcmos_fastlock_unlock(&onu_deactivate_wait_lock, 0);
}
}
}