VOL-4063: Downstream packets dropped when AES encryption enabled with RSYS3200G

Change-Id: I16d2965466bb4427ca06afcb1d530c03c9b17d61
diff --git a/agent/src/core_api_handler.cc b/agent/src/core_api_handler.cc
index babd9ce..600b57f 100644
--- a/agent/src/core_api_handler.cc
+++ b/agent/src/core_api_handler.cc
@@ -362,6 +362,7 @@
         bcmos_fastlock_init(&flow_id_bitset_lock, 0);
         bcmos_fastlock_init(&voltha_flow_to_device_flow_lock, 0);
         bcmos_fastlock_init(&alloc_cfg_wait_lock, 0);
+        bcmos_fastlock_init(&gem_cfg_wait_lock, 0);
         bcmos_fastlock_init(&onu_deactivate_wait_lock, 0);
         bcmos_fastlock_init(&acl_packet_trap_handler_lock, 0);
 
@@ -2762,8 +2763,8 @@
         return err;
     }
 
-    if (direction.compare(upstream) == 0) {
-        Status st = install_gem_port(access_intf_id, onu_id, gemport_id);
+    if (direction == upstream || direction == downstream) {
+        Status st = install_gem_port(access_intf_id, onu_id, uni_id, gemport_id);
         if (st.error_code() != grpc::StatusCode::ALREADY_EXISTS && st.error_code() != grpc::StatusCode::OK) {
             OPENOLT_LOG(ERROR, openolt_log_id, "failed to created gemport=%d, access_intf=%d, onu_id=%d\n", gemport_id, access_intf_id, onu_id);
             return BCM_ERR_INTERNAL;
@@ -2842,6 +2843,17 @@
     bcmolt_tm_queue_key key = { };
     bcmos_errno err;
 
+    // Gemports are bi-directional (except in multicast case). We create the gem port when we create the
+    // upstream/downstream queue (see CreateQueue function) and it makes sense to delete them when remove the queues.
+    // For multicast case we do not manage the install/remove of gem port in agent application. It is managed by BAL.
+    if (direction == upstream || direction == downstream) {
+        Status st = remove_gem_port(access_intf_id, onu_id, uni_id, gemport_id);
+        if (st.error_code() != grpc::StatusCode::OK) {
+            OPENOLT_LOG(ERROR, openolt_log_id, "failed to remove gemport=%d, access_intf=%d, onu_id=%d\n", gemport_id, access_intf_id, onu_id);
+            return BCM_ERR_INTERNAL;
+        }
+    }
+
     if (direction == downstream) {
         if (is_tm_sched_id_present(access_intf_id, onu_id, uni_id, direction, tech_profile_id)) {
             key.sched_id = get_tm_sched_id(access_intf_id, onu_id, uni_id, direction, tech_profile_id);
@@ -2851,15 +2863,6 @@
             return BCM_ERR_OK;
         }
     } else {
-        // Gemports are bi-directional (except in multicast case). We create the gem port when we create the
-        // upstream queue (see CreateQueue function) and it makes sense to delete them when remove the upstream queues.
-        // For multicast case we do not manage the install/remove of gem port in agent application. It is managed by BAL.
-        // Moreover it also makes sense to remove when upstream queue is getting removed because the upstream queue MUST exist always.
-        // It is possible that the downstream queues are not created for a subscriber (for ex: upstream EAPoL trap flow only exists
-        // but no other flow, and in this case only upstream scheduler and queues exist. We do not have a scenario where only downstream
-        // subscriber flows exist but no upstream )
-        remove_gem_port(access_intf_id, gemport_id);
-
         /* In the upstream we use pre-created queues on the NNI scheduler that are used by all subscribers.
         They should not be removed. So, lets return OK. */
         return BCM_ERR_OK;
diff --git a/agent/src/core_data.cc b/agent/src/core_data.cc
index a53ca0e..65fc62c 100644
--- a/agent/src/core_data.cc
+++ b/agent/src/core_data.cc
@@ -129,8 +129,21 @@
 // Lock to protect critical section data structure used for handling AllocObject configuration response.
 bcmos_fastlock alloc_cfg_wait_lock;
 
+// Map used to track response from BAL for ITU PON Gem Configuration.
+// The key is gem_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// used for pushing (from BAL) and popping (at application) the results.
+std::map<gem_cfg_compltd_key,  Queue<gem_cfg_complete_result> *> gem_cfg_compltd_map;
+// Lock to protect critical section data structure used for handling GemObject configuration response.
+bcmos_fastlock gem_cfg_wait_lock;
+
+/* This represents the Key to 'gemport_status_map' map.
+ Represents (pon_intf_id, onu_id, uni_id, gemport_id) */
+typedef std::tuple<uint32_t, uint32_t, uint32_t, uint32_t> gemport_status_map_key_tuple;
+/* 'gemport_status_map' maps gemport_status_map_key_tuple to boolean value */
+std::map<gemport_status_map_key_tuple, bool> gemport_status_map;
+
 // Map used to track response from BAL for Onu Deactivation Completed Indication
-// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// The key is onu_deact_compltd_key and value is a concurrent thread-safe queue which is
 // used for pushing (from BAL) and popping (at application) the results.
 std::map<onu_deact_compltd_key,  Queue<onu_deactivate_complete_result> *> onu_deact_compltd_map;
 // Lock to protect critical section data structure used for handling Onu Deactivation Completed Indication
diff --git a/agent/src/core_data.h b/agent/src/core_data.h
index 12edb9e..57b2fe2 100644
--- a/agent/src/core_data.h
+++ b/agent/src/core_data.h
@@ -51,6 +51,10 @@
 }
 
 #define ALLOC_CFG_COMPLETE_WAIT_TIMEOUT 5000 // in milli-seconds
+#define GEM_CFG_COMPLETE_WAIT_TIMEOUT 5000 // in milli-seconds
+
+// max retry count to find gem config key in gem_cfg_compltd_map
+#define MAX_GEM_CFG_KEY_CHECK 5
 
 #define ONU_DEACTIVATE_COMPLETE_WAIT_TIMEOUT 5000 // in milli-seconds
 
@@ -118,6 +122,31 @@
     AllocCfgStatus status;
 } alloc_cfg_complete_result;
 
+enum GemCfgAction {
+    GEM_OBJECT_CREATE,
+    GEM_OBJECT_DELETE,
+    GEM_OBJECT_ENCRYPT
+};
+
+enum GemObjectState {
+    GEM_OBJECT_STATE_NOT_CONFIGURED,
+    GEM_OBJECT_STATE_INACTIVE,
+    GEM_OBJECT_STATE_PROCESSING,
+    GEM_OBJECT_STATE_ACTIVE
+};
+
+enum GemCfgStatus {
+    GEM_CFG_STATUS_SUCCESS,
+    GEM_CFG_STATUS_FAIL
+};
+
+typedef struct {
+    uint32_t pon_intf_id;
+    uint32_t gem_port_id;
+    GemObjectState state;
+    GemCfgStatus status;
+} gem_cfg_complete_result;
+
 typedef struct {
     uint32_t pon_intf_id;
     uint32_t onu_id;
@@ -128,6 +157,9 @@
 // key for map used for tracking ITU PON Alloc Configuration results from BAL
 typedef std::tuple<uint32_t, uint32_t> alloc_cfg_compltd_key;
 
+// key for map used for tracking ITU PON Gem Configuration results from BAL
+typedef std::tuple<uint32_t, uint32_t> gem_cfg_compltd_key;
+
 // key for map used for tracking Onu Deactivation Completed Indication
 typedef std::tuple<uint32_t, uint32_t> onu_deact_compltd_key;
 
@@ -240,13 +272,27 @@
 // The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
 // used for pushing (from BAL) and popping (at application) the results.
 extern std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *> alloc_cfg_compltd_map;
+
+// Map used to track response from BAL for ITU PON Gem Configuration.
+// The key is gem_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// used for pushing (from BAL) and popping (at application) the results.
+extern std::map<gem_cfg_compltd_key,  Queue<gem_cfg_complete_result> *> gem_cfg_compltd_map;
+
+/* This represents the Key to 'gemport_status_map' map.
+ Represents (pon_intf_id, onu_id, uni_id, gemport_id) */
+typedef std::tuple<uint32_t, uint32_t, uint32_t, uint32_t> gemport_status_map_key_tuple;
+/* 'gemport_status_map' maps gemport_status_map_key_tuple to boolean value */
+extern std::map<gemport_status_map_key_tuple, bool> gemport_status_map;
+
 // Map used to track response from BAL for Onu Deactivation Completed Indication
-// The key is alloc_cfg_compltd_key and value is a concurrent thread-safe queue which is
+// The key is onu_deact_compltd_key and value is a concurrent thread-safe queue which is
 // used for pushing (from BAL) and popping (at application) the results.
 extern std::map<onu_deact_compltd_key,  Queue<onu_deactivate_complete_result> *> onu_deact_compltd_map;
 
 // Lock to protect critical section data structure used for handling AllocObject configuration response.
 extern bcmos_fastlock alloc_cfg_wait_lock;
+// Lock to protect critical section data structure used for handling GemObject configuration response.
+extern bcmos_fastlock gem_cfg_wait_lock;
 // Lock to protect critical section data structure used for handling Onu deactivation completed Indication
 extern bcmos_fastlock onu_deactivate_wait_lock;
 
diff --git a/agent/src/core_utils.cc b/agent/src/core_utils.cc
index e8060aa..6c027d5 100644
--- a/agent/src/core_utils.cc
+++ b/agent/src/core_utils.cc
@@ -573,7 +573,9 @@
 bcmos_errno wait_for_alloc_action(uint32_t intf_id, uint32_t alloc_id, AllocCfgAction action) {
     Queue<alloc_cfg_complete_result> cfg_result;
     alloc_cfg_compltd_key k(intf_id, alloc_id);
+    bcmos_fastlock_lock(&alloc_cfg_wait_lock);
     alloc_cfg_compltd_map[k] =  &cfg_result;
+    bcmos_fastlock_unlock(&alloc_cfg_wait_lock, 0);
     bcmos_errno err = BCM_ERR_OK;
 
     // Try to pop the result from BAL with a timeout of ALLOC_CFG_COMPLETE_WAIT_TIMEOUT ms
@@ -622,6 +624,71 @@
     return err;
 }
 
+// This method handles waiting for GemObject configuration.
+// Returns error if the GemObject is not in the appropriate state based on action requested.
+bcmos_errno wait_for_gem_action(uint32_t intf_id, uint32_t gem_port_id, GemCfgAction action) {
+    Queue<gem_cfg_complete_result> cfg_result;
+    gem_cfg_compltd_key k(intf_id, gem_port_id);
+    bcmos_fastlock_lock(&gem_cfg_wait_lock);
+    gem_cfg_compltd_map[k] =  &cfg_result;
+    bcmos_fastlock_unlock(&gem_cfg_wait_lock, 0);
+    bcmos_errno err = BCM_ERR_OK;
+
+    // Try to pop the result from BAL with a timeout of GEM_CFG_COMPLETE_WAIT_TIMEOUT ms
+    std::pair<gem_cfg_complete_result, bool> result = cfg_result.pop(GEM_CFG_COMPLETE_WAIT_TIMEOUT);
+    if (result.second == false) {
+        OPENOLT_LOG(ERROR, openolt_log_id, "timeout waiting for gem cfg complete indication intf_id %d, gem_port_id %d\n",
+                    intf_id, gem_port_id);
+        // Invalidate the queue pointer.
+        bcmos_fastlock_lock(&gem_cfg_wait_lock);
+        gem_cfg_compltd_map[k] = NULL;
+        bcmos_fastlock_unlock(&gem_cfg_wait_lock, 0);
+        err = BCM_ERR_INTERNAL;
+    }
+    else if (result.first.status == GEM_CFG_STATUS_FAIL) {
+        OPENOLT_LOG(ERROR, openolt_log_id, "error processing gem cfg request intf_id %d, gem_port_id %d\n",
+                    intf_id, gem_port_id);
+        err = BCM_ERR_INTERNAL;
+    }
+
+    if (err == BCM_ERR_OK) {
+        if (action == GEM_OBJECT_CREATE) {
+            if (result.first.state != GEM_OBJECT_STATE_ACTIVE) {
+                OPENOLT_LOG(ERROR, openolt_log_id, "gem object not in active state intf_id %d, gem_port_id %d gem_obj_state %d\n",
+                            intf_id, gem_port_id, result.first.state);
+               err = BCM_ERR_INTERNAL;
+            } else {
+                OPENOLT_LOG(INFO, openolt_log_id, "Create itupon gem object success, intf_id %d, gem_port_id %d\n",
+                            intf_id, gem_port_id);
+            }
+        } else if (action == GEM_OBJECT_ENCRYPT) {
+            if (result.first.state != GEM_OBJECT_STATE_ACTIVE) {
+                OPENOLT_LOG(ERROR, openolt_log_id, "gem object not in active state intf_id %d, gem_port_id %d gem_obj_state %d\n",
+                            intf_id, gem_port_id, result.first.state);
+               err = BCM_ERR_INTERNAL;
+            } else {
+                OPENOLT_LOG(INFO, openolt_log_id, "Enable itupon gem object encryption success, intf_id %d, gem_port_id %d\n",
+                            intf_id, gem_port_id);
+            }
+        } else { // GEM_OBJECT_DELETE
+              if (result.first.state != GEM_OBJECT_STATE_NOT_CONFIGURED) {
+                  OPENOLT_LOG(ERROR, openolt_log_id, "gem object is not reset intf_id %d, gem_port_id %d gem_obj_state %d\n",
+                              intf_id, gem_port_id, result.first.state);
+                  err = BCM_ERR_INTERNAL;
+              } else {
+                  OPENOLT_LOG(INFO, openolt_log_id, "Remove itupon gem object success, intf_id %d, gem_port_id %d\n",
+                              intf_id, gem_port_id);
+              }
+        }
+    }
+
+    // Remove entry from map
+    bcmos_fastlock_lock(&gem_cfg_wait_lock);
+    gem_cfg_compltd_map.erase(k);
+    bcmos_fastlock_unlock(&gem_cfg_wait_lock, 0);
+    return err;
+}
+
 // This method handles waiting for OnuDeactivate Completed Indication
 bcmos_errno wait_for_onu_deactivate_complete(uint32_t intf_id, uint32_t onu_id) {
     Queue<onu_deactivate_complete_result> deact_result;
@@ -886,7 +953,20 @@
     return err;
 }
 
-Status install_gem_port(int32_t intf_id, int32_t onu_id, int32_t gemport_id) {
+Status install_gem_port(int32_t intf_id, int32_t onu_id, int32_t uni_id, int32_t gemport_id) {
+    gemport_status_map_key_tuple gem_status_key(intf_id, onu_id, uni_id, gemport_id);
+
+    bcmos_fastlock_lock(&data_lock);
+    std::map<gemport_status_map_key_tuple, bool>::const_iterator it = gemport_status_map.find(gem_status_key);
+    if (it != gemport_status_map.end()) {
+        if (it->second) {
+            bcmos_fastlock_unlock(&data_lock, 0);
+            OPENOLT_LOG(INFO, openolt_log_id, "gem port already installed = %d\n", gemport_id);
+            return Status::OK;
+        }
+    }
+    bcmos_fastlock_unlock(&data_lock, 0);
+
     bcmos_errno err;
     bcmolt_itupon_gem_cfg cfg; /* declare main API struct */
     bcmolt_itupon_gem_key key = {}; /* declare key */
@@ -927,18 +1007,43 @@
         return bcm_to_grpc_err(err, "Access_Control set ITU PON Gem port failed");
     }
 
+#ifndef SCALE_AND_PERF
+    err = wait_for_gem_action(intf_id, gemport_id, GEM_OBJECT_CREATE);
+    if (err) {
+        OPENOLT_LOG(ERROR, openolt_log_id, "failed to install gem_port = %d err = %s\n", gemport_id, bcmos_strerror(err));
+        return bcm_to_grpc_err(err, "Access_Control set ITU PON Gem port failed");
+    }
+#endif
+
     OPENOLT_LOG(INFO, openolt_log_id, "gem port installed successfully = %d\n", gemport_id);
 
+    bcmos_fastlock_lock(&data_lock);
+    gemport_status_map[gem_status_key] = true;
+    bcmos_fastlock_unlock(&data_lock, 0);
+
     return Status::OK;
 }
 
-Status remove_gem_port(int32_t intf_id, int32_t gemport_id) {
+Status remove_gem_port(int32_t intf_id, int32_t onu_id, int32_t uni_id, int32_t gemport_id) {
+    gemport_status_map_key_tuple gem_status_key(intf_id, onu_id, uni_id, gemport_id);
+
+    bcmos_fastlock_lock(&data_lock);
+    std::map<gemport_status_map_key_tuple, bool>::const_iterator it = gemport_status_map.find(gem_status_key);
+    if (it == gemport_status_map.end()) {
+        bcmos_fastlock_unlock(&data_lock, 0);
+        OPENOLT_LOG(INFO, openolt_log_id, "gem port already removed = %d\n", gemport_id);
+        return Status::OK;
+    }
+    bcmos_fastlock_unlock(&data_lock, 0);
+
     bcmolt_itupon_gem_cfg gem_cfg;
     bcmolt_itupon_gem_key key = {
         .pon_ni = (bcmolt_interface)intf_id,
         .gem_port_id = (bcmolt_gem_port_id)gemport_id
     };
     bcmos_errno err;
+    bcmolt_interface_state state;
+    bcmolt_status los_status;
 
     BCMOLT_CFG_INIT(&gem_cfg, itupon_gem, key);
     err = bcmolt_cfg_clear(dev_id, &gem_cfg.hdr);
@@ -948,8 +1053,42 @@
         return bcm_to_grpc_err(err, "Access_Control clear ITU PON Gem port failed");
     }
 
+    err = get_pon_interface_status((bcmolt_interface)intf_id, &state, &los_status);
+    if (err == BCM_ERR_OK) {
+        if (state == BCMOLT_INTERFACE_STATE_ACTIVE_WORKING && los_status == BCMOLT_STATUS_OFF) {
+#ifndef SCALE_AND_PERF
+            OPENOLT_LOG(INFO, openolt_log_id, "PON interface: %d is enabled and LoS status is OFF, waiting for gem cfg clear response\n",
+                intf_id);
+            err = wait_for_gem_action(intf_id, gemport_id, GEM_OBJECT_DELETE);
+            if (err) {
+                OPENOLT_LOG(ERROR, openolt_log_id, "failed to remove gem_port = %d err = %s\n", gemport_id, bcmos_strerror(err));
+                return bcm_to_grpc_err(err, "Access_Control clear ITU PON Gem port failed");
+            }
+#endif
+        }
+        else if (state == BCMOLT_INTERFACE_STATE_ACTIVE_WORKING && los_status == BCMOLT_STATUS_ON) {
+            OPENOLT_LOG(INFO, openolt_log_id, "PON interface: %d is enabled but LoS status is ON, not waiting for gem cfg clear response\n",
+                intf_id);
+        }
+        else if (state == BCMOLT_INTERFACE_STATE_INACTIVE) {
+            OPENOLT_LOG(INFO, openolt_log_id, "PON interface: %d is disabled, not waiting for gem cfg clear response\n",
+                intf_id);
+        }
+    } else {
+        OPENOLT_LOG(ERROR, openolt_log_id, "Failed to fetch PON interface status, intf_id = %d, err = %s\n",
+            intf_id, bcmos_strerror(err));
+        return bcm_to_grpc_err(err, "Access_Control clear ITU PON Gem port failed");
+    }
+
     OPENOLT_LOG(INFO, openolt_log_id, "gem port removed successfully = %d\n", gemport_id);
 
+    bcmos_fastlock_lock(&data_lock);
+    it = gemport_status_map.find(gem_status_key);
+    if (it != gemport_status_map.end()) {
+        gemport_status_map.erase(it);
+    }
+    bcmos_fastlock_unlock(&data_lock, 0);
+
     return Status::OK;
 }
 
@@ -974,6 +1113,14 @@
         return bcm_to_grpc_err(err, "Failed to set encryption on GEM port");;
     }
 
+#ifndef SCALE_AND_PERF
+    err = wait_for_gem_action(intf_id, gemport_id, GEM_OBJECT_ENCRYPT);
+    if (err) {
+        OPENOLT_LOG(ERROR, openolt_log_id, "failed to enable gemport encryption, gem_port = %d err = %s\n", gemport_id, bcmos_strerror(err));
+        return bcm_to_grpc_err(err, "Access_Control ITU PON Gem port encryption failed");
+    }
+#endif
+
     OPENOLT_LOG(INFO, openolt_log_id, "encryption set successfully on pon = %d gem_port = %d\n", intf_id, gemport_id);
 
     return Status::OK;
diff --git a/agent/src/core_utils.h b/agent/src/core_utils.h
index c860644..a7a2295 100644
--- a/agent/src/core_utils.h
+++ b/agent/src/core_utils.h
@@ -77,6 +77,7 @@
 void clear_qos_type(uint32_t pon_intf_id, uint32_t onu_id, uint32_t uni_id);
 std::string GetDirection(int direction);
 bcmos_errno wait_for_alloc_action(uint32_t intf_id, uint32_t alloc_id, AllocCfgAction action);
+bcmos_errno wait_for_gem_action(uint32_t intf_id, uint32_t gem_port_id, GemCfgAction action);
 bcmos_errno wait_for_onu_deactivate_complete(uint32_t intf_id, uint32_t onu_id);
 char* openolt_read_sysinfo(const char* field_name, char* field_val);
 Status pushOltOperInd(uint32_t intf_id, const char *type, const char *state);
@@ -92,8 +93,8 @@
 unsigned NumNniIf_();
 unsigned NumPonIf_();
 bcmos_errno get_nni_interface_status(bcmolt_interface id, bcmolt_interface_state *state);
-Status install_gem_port(int32_t intf_id, int32_t onu_id, int32_t gemport_id);
-Status remove_gem_port(int32_t intf_id, int32_t gemport_id);
+Status install_gem_port(int32_t intf_id, int32_t onu_id, int32_t uni_id, int32_t gemport_id);
+Status remove_gem_port(int32_t intf_id, int32_t onu_id, int32_t uni_id, int32_t gemport_id);
 Status enable_encryption_for_gem_port(int32_t intf_id, int32_t gemport_id);
 Status update_acl_interface(int32_t intf_id, bcmolt_interface_type intf_type, uint32_t access_control_id,
                 bcmolt_members_update_command acl_cmd);
diff --git a/agent/src/indications.cc b/agent/src/indications.cc
index 304cb45..0407260 100644
--- a/agent/src/indications.cc
+++ b/agent/src/indications.cc
@@ -35,10 +35,6 @@
 
 using grpc::Status;
 
-extern Queue<openolt::Indication> oltIndQ;
-extern std::map<alloc_cfg_compltd_key,  Queue<alloc_cfg_complete_result> *> alloc_cfg_compltd_map;
-extern bcmos_fastlock alloc_cfg_wait_lock;
-
 bool subscribed = false;
 uint32_t nni_intf_id = 0;
 #define current_device 0
@@ -596,6 +592,84 @@
     bcmolt_msg_free(msg);
 }
 
+static void ItuPonGemConfigCompletedInd(bcmolt_devid olt, bcmolt_msg *msg) {
+
+    switch (msg->obj_type) {
+        case BCMOLT_OBJ_ID_ITUPON_GEM:
+            switch (msg->subgroup) {
+                case BCMOLT_ITUPON_GEM_AUTO_SUBGROUP_CONFIGURATION_COMPLETED:
+                {
+                    bcmolt_itupon_gem_configuration_completed *pkt =
+                        (bcmolt_itupon_gem_configuration_completed*)msg;
+                    bcmolt_itupon_gem_configuration_completed_data *pkt_data =
+                        &((bcmolt_itupon_gem_configuration_completed*)msg)->data;
+
+                    gem_cfg_compltd_key key((uint32_t)pkt->key.pon_ni, (uint32_t) pkt->key.gem_port_id);
+                    gem_cfg_complete_result res;
+                    res.pon_intf_id = pkt->key.pon_ni;
+                    res.gem_port_id = pkt->key.gem_port_id;
+
+                    pkt_data->status == BCMOLT_RESULT_SUCCESS ? res.status = GEM_CFG_STATUS_SUCCESS: res.status = GEM_CFG_STATUS_FAIL;
+                    switch (pkt_data->new_state) {
+                        case BCMOLT_ACTIVATION_STATE_NOT_CONFIGURED:
+                            res.state = GEM_OBJECT_STATE_NOT_CONFIGURED;
+                            break;
+                        case BCMOLT_ACTIVATION_STATE_INACTIVE:
+                            res.state = GEM_OBJECT_STATE_INACTIVE;
+                            break;
+                        case BCMOLT_ACTIVATION_STATE_PROCESSING:
+                            res.state = GEM_OBJECT_STATE_PROCESSING;
+                            break;
+                        case BCMOLT_ACTIVATION_STATE_ACTIVE:
+                            res.state = GEM_OBJECT_STATE_ACTIVE;
+                            break;
+                        default:
+                            OPENOLT_LOG(ERROR, openolt_log_id, "invalid itu pon gem activation new_state, pon_intf %u, gem_port_id %u, new_state %d\n",
+                                    pkt->key.pon_ni, pkt->key.gem_port_id, pkt_data->new_state);
+                            res.state = GEM_OBJECT_STATE_NOT_CONFIGURED;
+                    }
+                    OPENOLT_LOG(INFO, openolt_log_id, "received itu pon gem cfg complete ind, pon intf %u, gem_port_id %u, status %u, new_state %u\n",
+                            pkt->key.pon_ni, pkt->key.gem_port_id, pkt_data->status, pkt_data->new_state);
+
+                    uint32_t gem_cfg_key_check_counter = 1;
+                    std::map<gem_cfg_compltd_key,  Queue<gem_cfg_complete_result> *>::iterator it;
+                    while(true) {
+                        bcmos_fastlock_lock(&gem_cfg_wait_lock);
+                        // Push the result from BAL to queue
+                        it = gem_cfg_compltd_map.find(key);
+
+                        if (it != gem_cfg_compltd_map.end()) {
+                            bcmos_fastlock_unlock(&gem_cfg_wait_lock, 0);
+                            break;
+                        } else if (it == gem_cfg_compltd_map.end() && gem_cfg_key_check_counter < MAX_GEM_CFG_KEY_CHECK) {
+                            /*During removal of gemport, indication from BAL arriving soon even before we start waiting for gemport cfg completion
+                            by pushing empty cfg_result for gem_cfg_compltd_key. To handle this scenario delaying to push gem cfg completion indication
+                            to Queue by 6ms.*/
+                            bcmos_fastlock_unlock(&gem_cfg_wait_lock, 0);
+                            bcmos_usleep(6000);
+                        } else {
+                            // could be case of spurious aysnc response, OR, the application timed-out waiting for response and cleared the key.
+                            bcmolt_msg_free(msg);
+                            OPENOLT_LOG(ERROR, openolt_log_id, "gem config key not found for gem_port_id = %u, pon_intf = %u\n", pkt->key.gem_port_id, pkt->key.pon_ni);
+                            bcmos_fastlock_unlock(&gem_cfg_wait_lock, 0);
+                            return;
+                        }
+                        gem_cfg_key_check_counter++;
+                    }
+
+                    bcmos_fastlock_lock(&gem_cfg_wait_lock);
+                    if (it->second) {
+                        // Push the result
+                        it->second->push(res);
+                    }
+                    bcmos_fastlock_unlock(&gem_cfg_wait_lock, 0);
+                }
+            }
+    }
+
+    bcmolt_msg_free(msg);
+}
+
 static void FlowOperIndication(bcmolt_devid olt, bcmolt_msg *msg) {
     openolt::Indication ind;
     OPENOLT_LOG(DEBUG, openolt_log_id, "flow oper state indication\n");
@@ -1371,6 +1445,14 @@
     rc = bcmolt_ind_subscribe(current_device, &rx_cfg);
     if(rc != BCM_ERR_OK)
         return Status(grpc::StatusCode::INTERNAL, "ITU PON Alloc Configuration Complete Indication subscribe failed");
+
+    rx_cfg.obj_type = BCMOLT_OBJ_ID_ITUPON_GEM;
+    rx_cfg.rx_cb = ItuPonGemConfigCompletedInd;
+    rx_cfg.flags = BCMOLT_AUTO_FLAGS_NONE;
+    rx_cfg.subgroup = bcmolt_itupon_gem_auto_subgroup_configuration_completed;
+    rc = bcmolt_ind_subscribe(current_device, &rx_cfg);
+    if(rc != BCM_ERR_OK)
+        return Status(grpc::StatusCode::INTERNAL, "ITU PON Gem Configuration Complete Indication subscribe failed");
 #endif
 
     rx_cfg.obj_type = BCMOLT_OBJ_ID_GROUP;