agent/scripts/watchdog/openolt_dev_mgmt_daemon_process_watchdog - openolt - Gitiles

 #!/bin/bash

 #Copyright 2020-present Open Networking Foundation
 #
 #Licensed under the Apache License, Version 2.0 (the "License");
 #you may not use this file except in compliance with the License.
 #You may obtain a copy of the License at
 #
 #http://www.apache.org/licenses/LICENSE-2.0
 #
 #Unless required by applicable law or agreed to in writing, software
 #distributed under the License is distributed on an "AS IS" BASIS,
 #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #See the License for the specific language governing permissions and
 #limitations under the License.

 # This script is a watchdog to monitor openolt and dev_mgmt_daemon processes
 # It restarts the processes when they crash
 # When OLT is working in in-band mode, it just monitors the dev_mgmt_daemon
 # process to start with. Once the openolt processes is also up, it starts
 # monitoring both the processes.
 # When OLT is working in out-of-band, it monitors both the processes to start
 # with. When either of the processes crash, it restarts both the processes.

 PID_MONITOR_INTERVAL=1 # in seconds
 MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START=120

 WATCHDOG_LOG_FILE="/var/log/openolt_process_watchdog.log"

 BRCM_DIR="/broadcom"

 # inband config file
 INBAND_CONFIG_FILE="${BRCM_DIR}/inband.config"

 OLT_MODEL=$(cat /sys/devices/virtual/dmi/id/board_name)

 ASF16_MODEL="ASXvOLT16"

 # vlan id for asfvolt16
 ASFVOLT16_VLAN_ID_ETH2=

 # vlan id for asgvolt64
 ASGVOLT64_VLAN_ID_ETH1=


 #------------------------------------------------------------------------------
 # Function Name: LOG
 # Description:
 #    Logs to WATCHDOG_LOG_FILE
 #
 # Globals:
 #    None
 #
 # Arguments:
 #    string to be logged
 #
 # Returns:
 #    None
 #------------------------------------------------------------------------------
 LOG() {
     echo `date`" $1" >> ${WATCHDOG_LOG_FILE}
 }

 #------------------------------------------------------------------------------
 # Function Name: get_vlan_ids
 # Description:
 #    This function facilitates to fetch vlan id from inband configuration file
 #    located at /broadcom/inband.config
 #
 # Globals:
 #    INBAND_CONFIG_FILE, ASFVOLT16_VLAN_ID_ETH2, ASFVOLT16_VLAN_ID_ETH3,
 #    ASGVOLT64_VLAN_ID_ETH1, ASGVOLT64_VLAN_ID_ETH2
 #
 # Arguments:
 #    None
 #
 # Returns:
 #    None
 #------------------------------------------------------------------------------
 get_vlan_ids() {
     # Read inband.config file to fetch vlan id information
     if [ -f ${INBAND_CONFIG_FILE} ]; then
         if [ "${OLT_MODEL}" = ${ASF16_MODEL} ]; then
             ASFVOLT16_VLAN_ID_ETH2=$(awk '/asfvolt16_vlan_id_eth2/{print $0}' ${INBAND_CONFIG_FILE} | awk -F "=" '{print $2}')
         else
             ASGVOLT64_VLAN_ID_ETH1=$(awk '/asgvolt64_vlan_id_eth1/{print $0}' ${INBAND_CONFIG_FILE} | awk -F "=" '{print $2}')
         fi

         if [ -z ${ASFVOLT16_VLAN_ID_ETH2} ] && [ -z ${ASGVOLT64_VLAN_ID_ETH1} ]; then
             LOG "ERROR: vlan ids not valid"
             exit 1
         fi
     else
         LOG "ERROR: ${INBAND_CONFIG_FILE} not found, using default value 4093"
     fi
 }

 #------------------------------------------------------------------------------
 # Function Name: monitor_openolt_and_dev_mgmt_daemon_process
 # Description:
 #    Monitors openolt and dev_mgmt_daemon processes in an infinite loop.
 #    Restarts both the processes if either of them exits
 #
 # Globals:
 #    None
 #
 # Arguments:
 #    None
 #
 # Returns:
 #
 #------------------------------------------------------------------------------
 monitor_openolt_and_dev_mgmt_daemon_process() {
     LOG "start monitor openolt and dev_mgmt_daemon processes"

     get_vlan_ids
     if [ "${OLT_MODEL}" = ${ASF16_MODEL} ]; then
         INBAND_IF="eth2.${ASFVOLT16_VLAN_ID_ETH2}"
     else
         INBAND_IF="eth1.${ASGVOLT64_VLAN_ID_ETH1}"
     fi

     # This is builtin bash variable that tracks the number of seconds
     # elapsed since the shell started. We can reset to 0 and the timer
     # starts from here
     SECONDS=0
     while true; do

         DEV_MGMT_DAEMON_PID=`pidof /broadcom/dev_mgmt_daemon`
         OPENOLT_PID=$(pidof /broadcom/openolt)
         if [ -z ${OPENOLT_PID} ] || [ -z ${DEV_MGMT_DAEMON_PID} ];then

             if [ -z ${DEV_MGMT_DAEMON_PID} ]; then
                LOG "dev_mgmt_daemon processes not available"
             fi

             if [ -z ${OPENOLT_PID} ]; then
                 LOG "openolt processes not available"
             fi

             # Reboot OLT so that everything recovers in the right state
             reboot -n
             exit 1
         fi

         sleep ${PID_MONITOR_INTERVAL}

         if [ $(is_olt_in_inband_mode; echo $?) -eq 1 ]; then
             INBAND_IF_IP_ADDR=`ip -o -4 addr list ${INBAND_IF} | awk '{print $4}' | cut -d/ -f1`

             if [ -z ${INBAND_IF_IP_ADDR} ] && [ ${SECONDS} -gt 60 ]; then
                 # If the in-band interface does not have an IP for more than 60s, reboot the OLT
                 LOG "in-band interface ${INBAND_IF} does not have IP for more than 60s, rebooting OLT"
                 reboot -n
                 exit 1
             elif [ ${#INBAND_IF_IP_ADDR} -gt 0 ]; then
                 # This is case where we have an IP address on the in-band interface

                 # Reset timer if the in-band interface has an IP address
                 SECONDS=0
             fi
         fi
     done
 }

 #------------------------------------------------------------------------------
 # Function Name: monitor_dev_mgmt_daemon_process
 # Description:
 #    Monitors dev_mgmt_daemon in an infinite loop. The loop breaks if openolt
 #    agent process starts
 #
 # Globals:
 #    None
 #
 # Arguments:
 #    None
 #
 # Returns:
 #
 #------------------------------------------------------------------------------
 monitor_dev_mgmt_daemon_process() {

     # This is builtin bash variable that tracks the number of seconds
     # elapsed since the shell started. We can reset to 0 and the timer
     # starts from here
     SECONDS=0
     while true; do
         DEV_MGMT_DAEMON_PID=$(pidof /broadcom/dev_mgmt_daemon)
         if [ -z ${DEV_MGMT_DAEMON_PID} ];then
             # Reboot OLT so that everything recovers in the right state
             LOG "dev_mgmt_daemon process crashed or not available in in-band mode of OLT, rebooting OLT"
             reboot -n
         fi

         sleep ${PID_MONITOR_INTERVAL}

         OPENOLT_PID=$(pidof /broadcom/openolt)
         if [ ${OPENOLT_PID} ]; then
             LOG "openolt process has started. start monitoring both dev_mgmt_daemon and openolt processes"
             break
         elif [ ${SECONDS} -ge ${MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START} ]; then
             # Typically the openolt process is supposed to start within MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START
             # If that does not happen, reboot the OLT.
             LOG "openolt process did not start for more than ${MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START} seconds, rebooting OLT"
             reboot -n
             exit 1
         fi
     done
 }

 #------------------------------------------------------------------------------
 # Function Name: is_olt_in_inband_mode
 # Description:
 #    Checks if OLT is working in in-band mode
 #    Presence of in-band startup scripts indicates in-band mode. These scripts
 #    are not packaged as part of the debian package or the ONL when OLT is working
 #    in out-of-band mode.
 #
 # Globals:
 #    None
 #
 # Arguments:
 #    None
 #
 # Returns:
 #
 #------------------------------------------------------------------------------
 is_olt_in_inband_mode() {
     ! [ -f "/etc/init.d/start_inband_oltservices.sh" ]
 }


 # Execution starts here
 LOG "---- watchdog started ----"

 # Source watchdog configuration
  . /opt/openolt/watchdog.config

 if [ ${enable_watchdog} = "yes" ]; then
     LOG "watchdog is enabled"
 else
     LOG "watchdog is disabled - exiting"
     exit 0
 fi

 # If OLT is working in out-of-band start with monitoring only dev_mgmt_daemon process
 if [ $(is_olt_in_inband_mode; echo $?) -eq 1 ]; then
     LOG "openolt working in in-band mode, starting to monitor dev_mgmt_daemon process first"
     monitor_dev_mgmt_daemon_process
     # If we are here, that means the openolt process has just started and we now need to
     # monitor both dev_mgmt_daemon and openolt process
 else
     sleep ${watchdog_startup_timer} # Allow some time for openolt and dev_mgmt_daemon process to start
     LOG "openolt working in out-of-band mode"
 fi

 monitor_openolt_and_dev_mgmt_daemon_process
	#!/bin/bash

	#Copyright 2020-present Open Networking Foundation
	#
	#Licensed under the Apache License, Version 2.0 (the "License");
	#you may not use this file except in compliance with the License.
	#You may obtain a copy of the License at
	#
	#http://www.apache.org/licenses/LICENSE-2.0
	#
	#Unless required by applicable law or agreed to in writing, software
	#distributed under the License is distributed on an "AS IS" BASIS,
	#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	#See the License for the specific language governing permissions and
	#limitations under the License.

	# This script is a watchdog to monitor openolt and dev_mgmt_daemon processes
	# It restarts the processes when they crash
	# When OLT is working in in-band mode, it just monitors the dev_mgmt_daemon
	# process to start with. Once the openolt processes is also up, it starts
	# monitoring both the processes.
	# When OLT is working in out-of-band, it monitors both the processes to start
	# with. When either of the processes crash, it restarts both the processes.

	PID_MONITOR_INTERVAL=1 # in seconds
	MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START=120

	WATCHDOG_LOG_FILE="/var/log/openolt_process_watchdog.log"

	BRCM_DIR="/broadcom"

	# inband config file
	INBAND_CONFIG_FILE="${BRCM_DIR}/inband.config"

	OLT_MODEL=$(cat /sys/devices/virtual/dmi/id/board_name)

	ASF16_MODEL="ASXvOLT16"

	# vlan id for asfvolt16
	ASFVOLT16_VLAN_ID_ETH2=

	# vlan id for asgvolt64
	ASGVOLT64_VLAN_ID_ETH1=


	#------------------------------------------------------------------------------
	# Function Name: LOG
	# Description:
	# Logs to WATCHDOG_LOG_FILE
	#
	# Globals:
	# None
	#
	# Arguments:
	# string to be logged
	#
	# Returns:
	# None
	#------------------------------------------------------------------------------
	LOG() {
	echo `date`" $1" >> ${WATCHDOG_LOG_FILE}
	}

	#------------------------------------------------------------------------------
	# Function Name: get_vlan_ids
	# Description:
	# This function facilitates to fetch vlan id from inband configuration file
	# located at /broadcom/inband.config
	#
	# Globals:
	# INBAND_CONFIG_FILE, ASFVOLT16_VLAN_ID_ETH2, ASFVOLT16_VLAN_ID_ETH3,
	# ASGVOLT64_VLAN_ID_ETH1, ASGVOLT64_VLAN_ID_ETH2
	#
	# Arguments:
	# None
	#
	# Returns:
	# None
	#------------------------------------------------------------------------------
	get_vlan_ids() {
	# Read inband.config file to fetch vlan id information
	if [ -f ${INBAND_CONFIG_FILE} ]; then
	if [ "${OLT_MODEL}" = ${ASF16_MODEL} ]; then
	ASFVOLT16_VLAN_ID_ETH2=$(awk '/asfvolt16_vlan_id_eth2/{print $0}' ${INBAND_CONFIG_FILE} \| awk -F "=" '{print $2}')
	else
	ASGVOLT64_VLAN_ID_ETH1=$(awk '/asgvolt64_vlan_id_eth1/{print $0}' ${INBAND_CONFIG_FILE} \| awk -F "=" '{print $2}')
	fi

	if [ -z ${ASFVOLT16_VLAN_ID_ETH2} ] && [ -z ${ASGVOLT64_VLAN_ID_ETH1} ]; then
	LOG "ERROR: vlan ids not valid"
	exit 1
	fi
	else
	LOG "ERROR: ${INBAND_CONFIG_FILE} not found, using default value 4093"
	fi
	}

	#------------------------------------------------------------------------------
	# Function Name: monitor_openolt_and_dev_mgmt_daemon_process
	# Description:
	# Monitors openolt and dev_mgmt_daemon processes in an infinite loop.
	# Restarts both the processes if either of them exits
	#
	# Globals:
	# None
	#
	# Arguments:
	# None
	#
	# Returns:
	#
	#------------------------------------------------------------------------------
	monitor_openolt_and_dev_mgmt_daemon_process() {
	LOG "start monitor openolt and dev_mgmt_daemon processes"

	get_vlan_ids
	if [ "${OLT_MODEL}" = ${ASF16_MODEL} ]; then
	INBAND_IF="eth2.${ASFVOLT16_VLAN_ID_ETH2}"
	else
	INBAND_IF="eth1.${ASGVOLT64_VLAN_ID_ETH1}"
	fi

	# This is builtin bash variable that tracks the number of seconds
	# elapsed since the shell started. We can reset to 0 and the timer
	# starts from here
	SECONDS=0
	while true; do

	DEV_MGMT_DAEMON_PID=`pidof /broadcom/dev_mgmt_daemon`
	OPENOLT_PID=$(pidof /broadcom/openolt)
	if [ -z ${OPENOLT_PID} ] \|\| [ -z ${DEV_MGMT_DAEMON_PID} ];then

	if [ -z ${DEV_MGMT_DAEMON_PID} ]; then
	LOG "dev_mgmt_daemon processes not available"
	fi

	if [ -z ${OPENOLT_PID} ]; then
	LOG "openolt processes not available"
	fi

	# Reboot OLT so that everything recovers in the right state
	reboot -n
	exit 1
	fi

	sleep ${PID_MONITOR_INTERVAL}

	if [ $(is_olt_in_inband_mode; echo $?) -eq 1 ]; then
	INBAND_IF_IP_ADDR=`ip -o -4 addr list ${INBAND_IF} \| awk '{print $4}' \| cut -d/ -f1`

	if [ -z ${INBAND_IF_IP_ADDR} ] && [ ${SECONDS} -gt 60 ]; then
	# If the in-band interface does not have an IP for more than 60s, reboot the OLT
	LOG "in-band interface ${INBAND_IF} does not have IP for more than 60s, rebooting OLT"
	reboot -n
	exit 1
	elif [ ${#INBAND_IF_IP_ADDR} -gt 0 ]; then
	# This is case where we have an IP address on the in-band interface

	# Reset timer if the in-band interface has an IP address
	SECONDS=0
	fi
	fi
	done
	}

	#------------------------------------------------------------------------------
	# Function Name: monitor_dev_mgmt_daemon_process
	# Description:
	# Monitors dev_mgmt_daemon in an infinite loop. The loop breaks if openolt
	# agent process starts
	#
	# Globals:
	# None
	#
	# Arguments:
	# None
	#
	# Returns:
	#
	#------------------------------------------------------------------------------
	monitor_dev_mgmt_daemon_process() {

	# This is builtin bash variable that tracks the number of seconds
	# elapsed since the shell started. We can reset to 0 and the timer
	# starts from here
	SECONDS=0
	while true; do
	DEV_MGMT_DAEMON_PID=$(pidof /broadcom/dev_mgmt_daemon)
	if [ -z ${DEV_MGMT_DAEMON_PID} ];then
	# Reboot OLT so that everything recovers in the right state
	LOG "dev_mgmt_daemon process crashed or not available in in-band mode of OLT, rebooting OLT"
	reboot -n
	fi

	sleep ${PID_MONITOR_INTERVAL}

	OPENOLT_PID=$(pidof /broadcom/openolt)
	if [ ${OPENOLT_PID} ]; then
	LOG "openolt process has started. start monitoring both dev_mgmt_daemon and openolt processes"
	break
	elif [ ${SECONDS} -ge ${MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START} ]; then
	# Typically the openolt process is supposed to start within MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START
	# If that does not happen, reboot the OLT.
	LOG "openolt process did not start for more than ${MAX_WAIT_TIMER_FOR_OPENOLT_PROCESS_TO_START} seconds, rebooting OLT"
	reboot -n
	exit 1
	fi
	done
	}

	#------------------------------------------------------------------------------
	# Function Name: is_olt_in_inband_mode
	# Description:
	# Checks if OLT is working in in-band mode
	# Presence of in-band startup scripts indicates in-band mode. These scripts
	# are not packaged as part of the debian package or the ONL when OLT is working
	# in out-of-band mode.
	#
	# Globals:
	# None
	#
	# Arguments:
	# None
	#
	# Returns:
	#
	#------------------------------------------------------------------------------
	is_olt_in_inband_mode() {
	! [ -f "/etc/init.d/start_inband_oltservices.sh" ]
	}


	# Execution starts here
	LOG "---- watchdog started ----"

	# Source watchdog configuration
	. /opt/openolt/watchdog.config

	if [ ${enable_watchdog} = "yes" ]; then
	LOG "watchdog is enabled"
	else
	LOG "watchdog is disabled - exiting"
	exit 0
	fi

	# If OLT is working in out-of-band start with monitoring only dev_mgmt_daemon process
	if [ $(is_olt_in_inband_mode; echo $?) -eq 1 ]; then
	LOG "openolt working in in-band mode, starting to monitor dev_mgmt_daemon process first"
	monitor_dev_mgmt_daemon_process
	# If we are here, that means the openolt process has just started and we now need to
	# monitor both dev_mgmt_daemon and openolt process
	else
	sleep ${watchdog_startup_timer} # Allow some time for openolt and dev_mgmt_daemon process to start
	LOG "openolt working in out-of-band mode"
	fi

	monitor_openolt_and_dev_mgmt_daemon_process