| #!/bin/bash |
| |
| #Copyright 2020-present Open Networking Foundation |
| # |
| #Licensed under the Apache License, Version 2.0 (the "License"); |
| #you may not use this file except in compliance with the License. |
| #You may obtain a copy of the License at |
| # |
| #http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| #Unless required by applicable law or agreed to in writing, software |
| #distributed under the License is distributed on an "AS IS" BASIS, |
| #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| #See the License for the specific language governing permissions and |
| #limitations under the License. |
| |
| # This script is a watchdog to monitor openolt and dev_mgmt_daemon processes |
| # It restarts the processes when they crash |
| # When OLT is working in in-band mode, it just monitors the dev_mgmt_daemon |
| # process to start with. Once the openolt processes is also up, it starts |
| # monitoring both the processes. |
| # When OLT is working in out-of-band, it monitors both the processes to start |
| # with. When either of the processes crash, it restarts both the processes. |
| |
| PID_MONITOR_INTERVAL=1 # in seconds |
| MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START=120 |
| |
| WATCHDOG_LOG_FILE="/var/log/openolt_process_watchdog.log" |
| |
| BRCM_DIR="/broadcom" |
| |
| # inband config file |
| INBAND_CONFIG_FILE="${BRCM_DIR}/inband.config" |
| |
| OLT_MODEL=$(cat /sys/devices/virtual/dmi/id/board_name) |
| |
| ASF16_MODEL="ASXvOLT16" |
| |
| # vlan id for asfvolt16 |
| ASFVOLT16_VLAN_ID_ETH2= |
| |
| # vlan id for asgvolt64 |
| ASGVOLT64_VLAN_ID_ETH1= |
| |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: LOG |
| # Description: |
| # Logs to WATCHDOG_LOG_FILE |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # string to be logged |
| # |
| # Returns: |
| # None |
| #------------------------------------------------------------------------------ |
| LOG() { |
| echo `date`" $1" >> ${WATCHDOG_LOG_FILE} |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: get_vlan_ids |
| # Description: |
| # This function facilitates to fetch vlan id from inband configuration file |
| # located at /broadcom/inband.config |
| # |
| # Globals: |
| # INBAND_CONFIG_FILE, ASFVOLT16_VLAN_ID_ETH2, ASFVOLT16_VLAN_ID_ETH3, |
| # ASGVOLT64_VLAN_ID_ETH1, ASGVOLT64_VLAN_ID_ETH2 |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # None |
| #------------------------------------------------------------------------------ |
| get_vlan_ids() { |
| # Read inband.config file to fetch vlan id information |
| if [ -f ${INBAND_CONFIG_FILE} ]; then |
| if [ "${OLT_MODEL}" = ${ASF16_MODEL} ]; then |
| ASFVOLT16_VLAN_ID_ETH2=$(awk '/asfvolt16_vlan_id_eth2/{print $0}' ${INBAND_CONFIG_FILE} | awk -F "=" '{print $2}') |
| else |
| ASGVOLT64_VLAN_ID_ETH1=$(awk '/asgvolt64_vlan_id_eth1/{print $0}' ${INBAND_CONFIG_FILE} | awk -F "=" '{print $2}') |
| fi |
| |
| if [ -z ${ASFVOLT16_VLAN_ID_ETH2} ] && [ -z ${ASGVOLT64_VLAN_ID_ETH1} ]; then |
| LOG "ERROR: vlan ids not valid" |
| exit 1 |
| fi |
| else |
| LOG "ERROR: ${INBAND_CONFIG_FILE} not found, using default value 4093" |
| fi |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: monitor_openolt_and_dev_mgmt_daemon_process |
| # Description: |
| # Monitors openolt and dev_mgmt_daemon processes in an infinite loop. |
| # Restarts both the processes if either of them exits |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # |
| #------------------------------------------------------------------------------ |
| monitor_openolt_and_dev_mgmt_daemon_process() { |
| LOG "start monitor openolt and dev_mgmt_daemon processes" |
| |
| get_vlan_ids |
| if [ "${OLT_MODEL}" = ${ASF16_MODEL} ]; then |
| INBAND_IF="eth2.${ASFVOLT16_VLAN_ID_ETH2}" |
| else |
| INBAND_IF="eth1.${ASGVOLT64_VLAN_ID_ETH1}" |
| fi |
| |
| # This is builtin bash variable that tracks the number of seconds |
| # elapsed since the shell started. We can reset to 0 and the timer |
| # starts from here |
| SECONDS=0 |
| while true; do |
| |
| DEV_MGMT_DAEMON_PID=`pidof /broadcom/dev_mgmt_daemon` |
| OPENOLT_PID=$(pidof /broadcom/openolt) |
| if [ -z ${OPENOLT_PID} ] || [ -z ${DEV_MGMT_DAEMON_PID} ];then |
| |
| if [ -z ${DEV_MGMT_DAEMON_PID} ]; then |
| LOG "dev_mgmt_daemon processes not available" |
| fi |
| |
| if [ -z ${OPENOLT_PID} ]; then |
| LOG "openolt processes not available" |
| fi |
| |
| # Reboot OLT so that everything recovers in the right state |
| reboot -n |
| exit 1 |
| fi |
| |
| sleep ${PID_MONITOR_INTERVAL} |
| |
| if [ $(is_olt_in_inband_mode; echo $?) -eq 1 ]; then |
| INBAND_IF_IP_ADDR=`ip -o -4 addr list ${INBAND_IF} | awk '{print $4}' | cut -d/ -f1` |
| |
| if [ -z ${INBAND_IF_IP_ADDR} ] && [ ${SECONDS} -gt 60 ]; then |
| # If the in-band interface does not have an IP for more than 60s, reboot the OLT |
| LOG "in-band interface ${INBAND_IF} does not have IP for more than 60s, rebooting OLT" |
| reboot -n |
| exit 1 |
| elif [ ${#INBAND_IF_IP_ADDR} -gt 0 ]; then |
| # This is case where we have an IP address on the in-band interface |
| |
| # Reset timer if the in-band interface has an IP address |
| SECONDS=0 |
| fi |
| fi |
| done |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: monitor_dev_mgmt_daemon_process |
| # Description: |
| # Monitors dev_mgmt_daemon in an infinite loop. The loop breaks if openolt |
| # agent process starts |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # |
| #------------------------------------------------------------------------------ |
| monitor_dev_mgmt_daemon_process() { |
| |
| # This is builtin bash variable that tracks the number of seconds |
| # elapsed since the shell started. We can reset to 0 and the timer |
| # starts from here |
| SECONDS=0 |
| while true; do |
| DEV_MGMT_DAEMON_PID=$(pidof /broadcom/dev_mgmt_daemon) |
| if [ -z ${DEV_MGMT_DAEMON_PID} ];then |
| # Reboot OLT so that everything recovers in the right state |
| LOG "dev_mgmt_daemon process crashed or not available in in-band mode of OLT, rebooting OLT" |
| reboot -n |
| fi |
| |
| sleep ${PID_MONITOR_INTERVAL} |
| |
| OPENOLT_PID=$(pidof /broadcom/openolt) |
| if [ ${OPENOLT_PID} ]; then |
| LOG "openolt process has started. start monitoring both dev_mgmt_daemon and openolt processes" |
| break |
| elif [ ${SECONDS} -ge ${MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START} ]; then |
| # Typically the openolt process is supposed to start within MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START |
| # If that does not happen, reboot the OLT. |
| LOG "openolt process did not start for more than ${MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START} seconds, rebooting OLT" |
| reboot -n |
| exit 1 |
| fi |
| done |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: is_olt_in_inband_mode |
| # Description: |
| # Checks if OLT is working in in-band mode |
| # Presence of in-band startup scripts indicates in-band mode. These scripts |
| # are not packaged as part of the debian package or the ONL when OLT is working |
| # in out-of-band mode. |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # |
| #------------------------------------------------------------------------------ |
| is_olt_in_inband_mode() { |
| ! [ -f "/etc/init.d/start_inband_oltservices.sh" ] |
| } |
| |
| |
| # Execution starts here |
| LOG "---- watchdog started ----" |
| |
| # Source watchdog configuration |
| . /opt/openolt/watchdog.config |
| |
| if [ ${enable_watchdog} = "yes" ]; then |
| LOG "watchdog is enabled" |
| else |
| LOG "watchdog is disabled - exiting" |
| exit 0 |
| fi |
| |
| # If OLT is working in out-of-band start with monitoring only dev_mgmt_daemon process |
| if [ $(is_olt_in_inband_mode; echo $?) -eq 1 ]; then |
| LOG "openolt working in in-band mode, starting to monitor dev_mgmt_daemon process first" |
| monitor_dev_mgmt_daemon_process |
| # If we are here, that means the openolt process has just started and we now need to |
| # monitor both dev_mgmt_daemon and openolt process |
| else |
| sleep ${watchdog_startup_timer} # Allow some time for openolt and dev_mgmt_daemon process to start |
| LOG "openolt working in out-of-band mode" |
| fi |
| |
| monitor_openolt_and_dev_mgmt_daemon_process |