VOL-549: Introduce a GRPC timeout to avoid GRPC connection going into a hung
state in case of communication failure.
Change-Id: I7ea158a6bfc697345d2c75f7f9cb4f2e5229aff2
diff --git a/voltha/adapters/asfvolt16_olt/asfvolt16_device_handler.py b/voltha/adapters/asfvolt16_olt/asfvolt16_device_handler.py
index 8ce5c14..82cb37e 100644
--- a/voltha/adapters/asfvolt16_olt/asfvolt16_device_handler.py
+++ b/voltha/adapters/asfvolt16_olt/asfvolt16_device_handler.py
@@ -374,7 +374,7 @@
if self.is_heartbeat_started == 0:
self.log.info('heart-beat-is-not-yet-started-starting-now')
- self.heartbeat(device)
+ self.start_heartbeat()
# Now set the initial PM configuration for this device
self.pm_metrics=Asfvolt16OltPmMetrics(device)
@@ -406,7 +406,9 @@
@inlineCallbacks
- def heartbeat(self, device, state = 'run'):
+ def heartbeat(self, state = 'run'):
+ device = self.adapter_agent.get_device(self.device_id)
+
self.log.debug('olt-heartbeat', device=device, state=state,
count=self.heartbeat_count)
self.is_heartbeat_started = 1
@@ -440,9 +442,7 @@
try:
d = yield self.bal.get_bal_heartbeat(self.device_id.__str__())
except Exception as e:
- d = None
-
- _device = device
+ d = None
if d == None:
# something is not right - OLT is not Reachable
@@ -453,7 +453,7 @@
if self.heartbeat_miss > 0:
self.heartbeat_miss = 0
if d.is_reboot == bal_pb2.BAL_OLT_UP_AFTER_REBOOT:
- self.log.info('Activating-OLT-again-after-reboot')
+ self.log.info('activating-olt-again-after-reboot')
# Since OLT is reachable after reboot, OLT should configurable with
# all the old existing flows. NNI port should be mark it as down for
@@ -470,26 +470,28 @@
self.adapter_agent.publish_inter_adapter_message(child_device.id,
msg)
#Activate Device
- self.activate(device);
+ self.activate(device)
else:
- _device.connect_status = ConnectStatus.REACHABLE
- _device.oper_status = OperStatus.ACTIVE
- _device.reason = ''
- self.adapter_agent.update_device(_device)
- self.log.info('Clearing-the-Hearbeat-Alarm')
- heartbeat_alarm(_device, 0)
+ device.connect_status = ConnectStatus.REACHABLE
+ device.oper_status = OperStatus.ACTIVE
+ device.reason = ''
+ self.adapter_agent.update_device(device)
+ # Update the device control block with the latest update
+ self.log.info("all-fine-no-heartbeat-miss",device=device)
+ self.log.info('clearing-hearbeat-alarm')
+ heartbeat_alarm(device, 0)
if (self.heartbeat_miss >= self.heartbeat_failed_limit) and \
- (_device.connect_status == ConnectStatus.REACHABLE):
+ (device.connect_status == ConnectStatus.REACHABLE):
self.log.info('olt-heartbeat-failed', count=self.heartbeat_miss)
- _device.connect_status = ConnectStatus.UNREACHABLE
- _device.oper_status = OperStatus.FAILED
- _device.reason = 'Lost connectivity to OLT'
- self.adapter_agent.update_device(_device)
+ device.connect_status = ConnectStatus.UNREACHABLE
+ device.oper_status = OperStatus.FAILED
+ device.reason = 'Lost connectivity to OLT'
+ self.adapter_agent.update_device(device)
heartbeat_alarm(device, 1, self.heartbeat_miss)
self.heartbeat_count += 1
- reactor.callLater(self.heartbeat_interval, self.heartbeat, device)
+ reactor.callLater(self.heartbeat_interval, self.heartbeat)
@inlineCallbacks
def reboot(self):
@@ -828,7 +830,7 @@
#heart beat - To health checkup of OLT
if self.is_heartbeat_started == 0:
self.log.info('Heart-beat-is-not-yet-started-starting-now')
- self.heartbeat(device)
+ self.start_heartbeat()
self.pm_metrics=Asfvolt16OltPmMetrics(device)
pm_config = self.pm_metrics.make_proto()
@@ -849,6 +851,9 @@
reactor.callLater(15, self.activate, device)
return
+ def start_heartbeat(self):
+ reactor.callLater(0, self.heartbeat)
+
def handle_not_started_onu(self, child_device, ind_info):
if ind_info['_sub_group_type'] == 'onu_discovery':
self.log.info('Onu-discovered', olt_id=self.olt_id,
@@ -1024,7 +1029,7 @@
try:
if isinstance(data, ChannelgroupConfig):
if data.name in self.channel_groups:
- self.log('Channel-Group-already-present',
+ self.log.info('Channel-Group-already-present',
channel_group=data)
else:
channel_group_config = ChannelgroupConfig()
@@ -1032,7 +1037,7 @@
self.channel_groups[data.name] = channel_group_config
if isinstance(data, ChannelpartitionConfig):
if data.name in self.channel_partitions:
- self.log('Channel-partition-already-present',
+ self.log.info('Channel-partition-already-present',
channel_partition=data)
else:
channel_partition_config = ChannelpartitionConfig()
@@ -1041,7 +1046,7 @@
channel_partition_config
if isinstance(data, ChannelpairConfig):
if data.name in self.channel_pairs:
- self.log('Channel-pair-already-present',
+ self.log.info('Channel-pair-already-present',
channel_pair=data)
else:
channel_pair_config = ChannelpairConfig()
@@ -1062,16 +1067,26 @@
channel_termination_config.CopyFrom(data)
self.channel_terminations[data.name] = \
channel_termination_config
+ self.log.info('channel-termnination-data',
+ data=data,chan_term=self.channel_terminations)
if isinstance(data, VOntaniConfig):
+ self.handle_v_ont_ani_config(data)
if data.name in self.v_ont_anis:
self.log.info('v_ont_ani-already-present',
v_ont_ani=data)
else:
- self.handle_v_ont_ani_config(data)
v_ont_ani_config = VOntAniHandler()
v_ont_ani_config.v_ont_ani.CopyFrom(data)
self.v_ont_anis[data.name] = v_ont_ani_config
if isinstance(data, VEnetConfig):
+ self.adapter_agent.add_port(self.device_id, Port(
+ port_no=self._get_next_uni_port(),
+ label=data.interface.name,
+ type=Port.ETHERNET_UNI,
+ admin_state=AdminState.ENABLED,
+ oper_status=OperStatus.ACTIVE
+ ))
+
if data.name in self.v_enets:
self.log.info('v_enet-already-present',
v_enet=data)
@@ -1079,13 +1094,6 @@
v_enet_config = VEnetHandler()
v_enet_config.v_enet.CopyFrom(data)
self.log.info("creating-port-at-olt")
- self.adapter_agent.add_port(self.device_id, Port(
- port_no=self._get_next_uni_port(),
- label=data.interface.name,
- type=Port.ETHERNET_UNI,
- admin_state=AdminState.ENABLED,
- oper_status=OperStatus.ACTIVE
- ))
self.v_enets[data.name] = v_enet_config
if isinstance(data, OntaniConfig):
if data.name in self.ont_anis:
diff --git a/voltha/adapters/asfvolt16_olt/asfvolt16_ind_handler.py b/voltha/adapters/asfvolt16_olt/asfvolt16_ind_handler.py
index 75325ed..2ffa2d2 100755
--- a/voltha/adapters/asfvolt16_olt/asfvolt16_ind_handler.py
+++ b/voltha/adapters/asfvolt16_olt/asfvolt16_ind_handler.py
@@ -242,7 +242,8 @@
ind_info['_vendor_id'] = onu_data.data.serial_number.vendor_id
ind_info['_vendor_specific'] = \
onu_data.data.serial_number.vendor_specific
- if (bal_model_types_pb2.BAL_STATE_DOWN == onu_data.data.admin_state):
+ if ((bal_model_types_pb2.BAL_STATE_DOWN == onu_data.data.admin_state)\
+ or (bal_model_types_pb2.BAL_STATUS_UP != onu_data.data.oper_status)):
ind_info['activation_successful'] = False
elif (bal_model_types_pb2.BAL_STATE_UP == onu_data.data.admin_state):
ind_info['activation_successful'] = True
diff --git a/voltha/adapters/asfvolt16_olt/asfvolt16_olt.py b/voltha/adapters/asfvolt16_olt/asfvolt16_olt.py
index 1b1f284..47c9554 100644
--- a/voltha/adapters/asfvolt16_olt/asfvolt16_olt.py
+++ b/voltha/adapters/asfvolt16_olt/asfvolt16_olt.py
@@ -33,7 +33,7 @@
device_handler_class=Asfvolt16Handler,
name='asfvolt16_olt',
vendor='Edgecore',
- version='0.94',
+ version='0.95',
device_type='asfvolt16_olt')
# register for adapter messages
self.port = 60001
diff --git a/voltha/adapters/asfvolt16_olt/bal.py b/voltha/adapters/asfvolt16_olt/bal.py
index 43c3467..99b2ffd 100644
--- a/voltha/adapters/asfvolt16_olt/bal.py
+++ b/voltha/adapters/asfvolt16_olt/bal.py
@@ -30,6 +30,8 @@
"""
ADAPTER_PORT = 60001
+GRPC_TIMEOUT = 5
+
class Bal(object):
def __init__(self, olt, log):
@@ -70,7 +72,7 @@
self.log.info('Adapter-port-IP', init.voltha_adapter_ip_port)
self.log.info('connecting-olt', host_and_port=host_and_port,
init_details=init)
- yield self.stub.BalApiInit(init)
+ yield self.stub.BalApiInit(init, timeout=GRPC_TIMEOUT)
def activate_olt(self):
self.log.info('activating-olt')
@@ -86,7 +88,7 @@
self.log.info('Activating-Access-Terminal-Device',
admin_state=admin_state, device_id=self.device_id,
access_terminal_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
@inlineCallbacks
def activate_pon_port(self, olt_no, pon_port):
@@ -104,7 +106,7 @@
self.log.info('activating-pon-port-in-olt',
olt=olt_no, pon_port=pon_port,
pon_port_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('activating-pon-port-in-olt-exception', exc=str(e))
return
@@ -147,7 +149,7 @@
self.log.info('send_omci_request_message',
proxy_address=proxy_address.channel_id,
omci_msg_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('send-proxied_message-exception', exc=str(e))
return
@@ -170,7 +172,7 @@
'202020202020202020202020202020202020202020202020202020202020202020202020'
self.log.info('activating-ONU-in-olt',
onu_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('activating-ONU-exception',
onu_info['onu_id'], exc=str(e))
@@ -213,7 +215,7 @@
obj.packet.data.pkt = pkt
self.log.info('sending-packet-out',
packet_out_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
@inlineCallbacks
def add_flow(self, onu_id, intf_id, flow_id, gem_port,
@@ -331,7 +333,7 @@
return
self.log.info('adding-flow-to-OLT-Device',
flow_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('add_flow-exception',
flow_id, onu_id, exc=str(e))
@@ -360,7 +362,7 @@
obj.flow.data.sub_term_id = onu_id
self.log.info('deleting-flows-from-OLT-Device',
flow_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('delete_flow-exception',
flow_id, onu_id, exc=str(e))
@@ -407,7 +409,7 @@
obj.tm_sched_cfg.data.num_priorities = num_priority
self.log.info('Creating-Scheduler',
scheduler_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('creat-scheduler-exception',
olt=self.olt.olt_id,
@@ -436,7 +438,7 @@
'202020202020202020202020202020202020202020202020202020202020202020202020'
self.log.info('deactivating-ONU-in-olt',
onu_details=obj)
- yield self.stub.BalCfgSet(obj)
+ yield self.stub.BalCfgSet(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('deactivating-ONU-exception',
onu_info['onu_id'], exc=str(e))
@@ -457,7 +459,7 @@
obj.tm_sched_key.id = id
self.log.info('Deleting Scheduler',
scheduler_details=obj)
- yield self.stub.BalCfgClear(obj)
+ yield self.stub.BalCfgClear(obj, timeout=GRPC_TIMEOUT)
except Exception as e:
self.log.info('creat-scheduler-exception',
olt=self.olt.olt_id,
@@ -473,7 +475,7 @@
obj = bal_model_types_pb2.BalInterfaceKey()
obj.intf_id = intf_id
obj.intf_type = interface_type
- stats = yield self.stub.BalCfgStatGet(obj)
+ stats = yield self.stub.BalCfgStatGet(obj, timeout=GRPC_TIMEOUT)
self.log.info('Fetching-statistics-success',
stats_data=stats.data)
returnValue(stats)
@@ -486,7 +488,7 @@
try:
obj = bal_pb2.BalReboot()
obj.device_id = device_id
- err = yield self.stub.BalApiReboot(obj)
+ err = yield self.stub.BalApiReboot(obj, timeout=GRPC_TIMEOUT)
self.log.info('OLT-Reboot-Success', reboot_err=err)
returnValue(err)
except Exception as e:
@@ -498,7 +500,7 @@
try:
obj = bal_pb2.BalHeartbeat()
obj.device_id = device_id
- rebootStatus = yield self.stub.BalApiHeartbeat(obj)
+ rebootStatus = yield self.stub.BalApiHeartbeat(obj, timeout=GRPC_TIMEOUT)
self.log.info('OLT-HeartBeat-Response-Received-from',
device=device_id, rebootStatus=rebootStatus)
returnValue(rebootStatus)
@@ -510,7 +512,7 @@
try:
obj = bal_pb2.BalDefault()
obj.device_id = str(device_id)
- bal_ind = self.ind_stub.BalGetIndFromDevice(obj)
+ bal_ind = self.ind_stub.BalGetIndFromDevice(obj, timeout=GRPC_TIMEOUT)
if bal_ind.ind_present == True:
self.log.info('Indication-received',
device=device_id, bal_ind=bal_ind)
diff --git a/voltha/adapters/broadcom_onu/broadcom_onu.py b/voltha/adapters/broadcom_onu/broadcom_onu.py
index f3a73d7..2ddfe32 100644
--- a/voltha/adapters/broadcom_onu/broadcom_onu.py
+++ b/voltha/adapters/broadcom_onu/broadcom_onu.py
@@ -72,7 +72,7 @@
self.descriptor = Adapter(
id=self.name,
vendor='Voltha project',
- version='0.4',
+ version='0.41',
config=AdapterConfig(log_level=LogLevel.INFO)
)
self.devices_handlers = dict() # device_id -> BroadcomOnuHandler()
@@ -341,6 +341,12 @@
device.oper_status = OperStatus.DISCOVERED
self.adapter_agent.update_device(device)
+ elif (event_msg['event'] == 'deactivate-onu'):
+ device = self.adapter_agent.get_device(self.device_id)
+ device.connect_status = ConnectStatus.UNREACHABLE
+ device.oper_status = OperStatus.DISCOVERED
+ self.adapter_agent.update_device(device)
+
elif event_msg['event'] == 'ranging-completed':
if event_msg['event_data']['ranging_successful'] == True: