| #!/bin/bash |
| |
| #Copyright 2020-present Open Networking Foundation |
| # |
| #Licensed under the Apache License, Version 2.0 (the "License"); |
| #you may not use this file except in compliance with the License. |
| #You may obtain a copy of the License at |
| # |
| #http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| #Unless required by applicable law or agreed to in writing, software |
| #distributed under the License is distributed on an "AS IS" BASIS, |
| #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| #See the License for the specific language governing permissions and |
| #limitations under the License. |
| # |
| |
| ### BEGIN INIT INFO |
| # Description: |
| # This script swaps the block id labels if expected services |
| # like system services onlp, sshd, bal_core_dist, dhclient, physical |
| # ,vlan interfaces status and olt services dev_mgmt_daemon, openolt services |
| # are UP and RUNNING. If all services are not UP and RUNNING, then roll back |
| # to previous state. |
| ### END INIT INFO |
| |
| # Labels names |
| ONL_ACTIVE_BOOT="ONL-ACTIVE-BOOT" |
| ONL_ACTIVE_DATA="ONL-ACTIVE-DATA" |
| ONL_STANDBY_BOOT="ONL-STANDBY-BOOT" |
| ONL_STANDBY_DATA="ONL-STANDBY-DATA" |
| |
| RUNNING_ROOT_DEV= |
| RUNNING_ROOT_LABEL= |
| RUNNING_BOOT_LABEL= |
| RUNNING_BOOT_DEV= |
| |
| STANDBY_BOOT_DEV= |
| STANDBY_BOOT_LABEL= |
| STANDBY_ROOT_LABEL= |
| STANDBY_ROOT_DEV= |
| |
| # Configuration file about upgrade |
| ONL_ROOT="/mnt/onl" |
| IMAGES="${ONL_ROOT}/images" |
| UPGRADE_PATH="${IMAGES}/upgrade" |
| ACTIVE_GRUB_ENV_FILE="${ONL_ROOT}/active_boot/grub/grubenv" |
| STANDBY_GRUB_ENV_FILE="${ONL_ROOT}/standby_boot/grub/grubenv" |
| GRUB_STAND_PATH="${ONL_ROOT}/standby_boot/grub" |
| GRUB_ACTIVE_PATH="${ONL_ROOT}/active_boot/grub" |
| STANDBY_GRUB_ACTIVE_PATH="${ONL_ROOT}/standby_boot/grub" |
| GRUB_CFG="grub.cfg" |
| GRUB_CFG_BACKUP="grub.cfg.back" |
| GRUB_CONFIG_FILE="${ONL_ROOT}/config/grub.cfg" |
| # Time interval in seconds |
| TIME_INTERVAL=5 |
| |
| #in band interface IP |
| ASFVOLT16_ETH2_VLAN_INF_IP= |
| |
| ASGVOLT64_ETH1_VLAN_INF_IP= |
| |
| ETH3_VLAN_IP= |
| |
| BRCM_DIR='/broadcom' |
| |
| # vlan config file |
| VLAN_CONFIG_FILE="${BRCM_DIR}/inband.config" |
| |
| # vlan id for asfvolt16 |
| ASFVOLT16_VLAN_ID_ETH2= |
| |
| #vlan id for asgvolt64 |
| ASGVOLT64_VLAN_ID_ETH1= |
| |
| OLT_MODEL=$(cat /sys/devices/virtual/dmi/id/board_name) |
| ASXvOLT16="ASXvOLT16" |
| ASGvOLT64="ASGvOLT64" |
| |
| VLAN_ID_1= |
| VLAN_ID_2= |
| |
| # interfaces |
| ETH1=eth1 |
| ETH2=eth2 |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: does_logger_exist |
| # Description: |
| # This function check if logger exist and executable. |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # returns 0 if exist and executable else 1 |
| #------------------------------------------------------------------------------ |
| does_logger_exist() |
| { |
| cmd=/usr/bin/logger |
| if [ -x ${cmd} ]; then |
| return 0 |
| else |
| return 1 |
| fi |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: info_message |
| # Description: |
| # This function print the info message information to the console |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # string message |
| # |
| # Returns: |
| # None |
| #------------------------------------------------------------------------------ |
| info_message() |
| { |
| echo "INFO: $1" |
| logger -p user.info "$1" |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: error_message |
| # Description: |
| # This function print the error message information to the console |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # string message |
| # |
| # Returns: |
| # returns 1 |
| #------------------------------------------------------------------------------ |
| error_message() |
| { |
| echo "ERROR: $1" |
| logger -p user.err "$1" |
| return 1 |
| } |
| |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: error_message |
| # Description: |
| # This function print the error message information to the console |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # string message |
| # |
| # Returns: |
| # returns 1 |
| #------------------------------------------------------------------------------ |
| display_services() |
| { |
| echo "Sanity result: |
| Service Name - Status |
| -------------------------------------- |
| 1. eth2 - UP & RUNNING |
| 2. eth1 - UP & RUNNING |
| 3. bal_service - RUNNING |
| 4. system services - RUNNING" |
| |
| if [ "${OLT_MODEL}" = ${ASXvOLT16} ];then |
| echo " |
| 5. eth2.${ASFVOLT16_VLAN_ID_ETH2} - UP & RUNNING |
| 6. eth2.${ASFVOLT16_VLAN_ID_ETH2} IP - ${ASFVOLT16_ETH2_VLAN_INF_IP}" |
| elif [ "${OLT_MODEL}" = ${ASGvOLT64} ];then |
| echo" |
| 5. eth1.${ASGVOLT64_VLAN_ID_ETH1} - UP & RUNNING |
| 6. eth1.${ASGVOLT64_VLAN_ID_ETH1} IP - ${ASGVOLT64_ETH1_VLAN_INF_IP}" |
| fi |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: is_interface_up_running |
| # Description: |
| # This function validate the interface whether it is UP and RUNNING or DOWN. |
| # |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # interface name |
| # |
| # Returns: |
| # returns 0 if interface is up and running and returns 1 if interface is down |
| #------------------------------------------------------------------------------ |
| is_interface_up_running() |
| { |
| info_message "Validating interface - $1" |
| interface_name=$1 |
| ifconfig ${interface_name} | grep -q "UP BROADCAST RUNNING MULTICAST" |
| if [ $? -eq 0 ]; then |
| info_message "${interface_name} UP & RUNNING" |
| echo "---------------------------------------------------------------------" |
| return 0 |
| else |
| error_message "${interface_name} is DOWN" |
| echo "---------------------------------------------------------------------" |
| return 1 |
| fi |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: validate_ip |
| # Description: |
| # This function get the ip from the interface and validates it. |
| # |
| # Globals: |
| # OLT_MODEL, ASXvOLT16, ASGvOLT64, ASFVOLT16_VLAN_ID_ETH2, |
| # ASFVOLT16_VLAN_ID_ETH2, ASGVOLT64_VLAN_ID_ETH1 |
| # |
| # |
| # Arguments: |
| # interface name |
| # |
| # Returns: |
| # ip address |
| #------------------------------------------------------------------------------ |
| validate_ip() |
| { |
| interface_name=$1 |
| ip_address=$(ifconfig $interface_name | grep "inet addr" | awk 'BEGIN {FS =":"} {print $2}' | awk '{print $1}') |
| |
| if [ "${OLT_MODEL}" = ${ASXvOLT16} ];then |
| if [ ! -z $ip_address ] && [ $interface_name = ${ETH2}.${ASFVOLT16_VLAN_ID_ETH2} ]; then |
| ASFVOLT16_ETH2_VLAN_INF_IP=$ip_address |
| fi |
| elif [ "${OLT_MODEL}" = ${ASGvOLT64} ];then |
| if [ ! -z $ip_address ] && [ $interface_name = ${ETH1}.${ASGVOLT64_VLAN_ID_ETH1} ]; then |
| ASGVOLT64_ETH1_VLAN_INF_IP=$ip_address |
| fi |
| fi |
| |
| info_message "Validating $1 ip address - $ip_address" |
| if expr "$ip_address" : '[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$' >/dev/null; then |
| for i in 1 2 3 4; do |
| if [ $(echo "$ip_address" | cut -d. -f$i) -gt 255 ]; then |
| return 1 |
| fi |
| done |
| return 0 |
| else |
| return 1 |
| fi |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name : validate_interfaces |
| # Description: |
| # This function validate interfaces ${ETH2}.${ASFVOLT16_VLAN_ID_ETH2} and |
| # or ${ETH1}.${ASGVOLT64_VLAN_ID_ETH1} and based on OLT model. |
| # Basically it validates if these interfaces are UP and RUNNING or not |
| # |
| # Globals: |
| # ASFVOLT16_VLAN_ID_ETH2, ASGVOLT64_VLAN_ID_ETH1, |
| # ETH1, ETH2, ASFVOLT16_BAL_PACKAGE |
| # |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # return 1 if interface are down else returns 0 |
| #------------------------------------------------------------------------------ |
| validate_interfaces() |
| { |
| # Validating interfaces whether they are UP and RUNNING or not |
| if [ "${OLT_MODEL}" = ${ASXvOLT16} ];then |
| is_interface_up_running ${ETH2}.${ASFVOLT16_VLAN_ID_ETH2} |
| if [ $? -eq 0 ]; then |
| return 0 |
| else |
| return 1 |
| fi |
| elif [ "${OLT_MODEL}" = ${ASGvOLT64} ];then |
| is_interface_up_running ${ETH1}.${ASGVOLT64_VLAN_ID_ETH1} |
| if [ $? -eq 0 ]; then |
| return 0 |
| else |
| return 1 |
| fi |
| fi |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: validate_vlan_intf_ip |
| # Description: |
| # This function validate the inband vlan interfaces ip, checks if DHCP server has |
| # assigned proper IP addresses. |
| # |
| # |
| # Globals: |
| # ASFVOLT16_VLAN_ID_ETH2, ASGVOLT64_VLAN_ID_ETH1, |
| # ETH1, ETH2, OLT_MODEL, ASXvOLT16 and ASGvOLT64 |
| # |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # returns 0 if ip adresses are valid else return 1 |
| #------------------------------------------------------------------------------ |
| validate_vlan_intf_ip() |
| { |
| # Validating vlan interfaces and their IP |
| if [ "${OLT_MODEL}" = ${ASXvOLT16} ];then |
| validate_ip ${ETH2}.${ASFVOLT16_VLAN_ID_ETH2} |
| if [ $? -eq 0 ]; then |
| return 0 |
| else |
| return 1 |
| fi |
| elif [ "${OLT_MODEL}" = ${ASGvOLT64} ];then |
| validate_ip ${ETH1}.${ASGVOLT64_VLAN_ID_ETH1} |
| if [ $? -eq 0 ]; then |
| return 0 |
| else |
| return 1 |
| fi |
| |
| fi |
| return 0 |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: validate_system_services |
| # Description: |
| # This function checks if the below services/processes are running or not. |
| # 1. dev_mgmt_daemon |
| # 2. openolt |
| # 3. onlp |
| # 4. sshd |
| # 5. dhclient |
| # Globals: |
| # None |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # returns 0 |
| #------------------------------------------------------------------------------ |
| validate_system_services() |
| { |
| echo "---------------------------------------------------------------------" |
| echo "Validating Services" |
| echo "---------------------------------------------------------------------" |
| |
| dhclient_interface dhclient_val1 dhclient_val2 |
| for service_name in dev_mgmt_daemon openolt onlp sshd ${dhclient_val1} ${dhclient_val2} |
| do |
| echo "---------------------------------------------------------------------" |
| ps -ef | grep -v grep | grep ${service_name} |
| if [ $? -eq 0 ]; then |
| info_message "${service_name} service is running" |
| else |
| error_message "${service_name} is not running" |
| return 1 |
| fi |
| echo "---------------------------------------------------------------------" |
| done |
| return 0 |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: dhclient_interface |
| # Description: |
| # This function sets values to which tagged interface dhclient service need |
| # to be tested based on the OLT model package. |
| # |
| # Globals: |
| # OLT_MODEL, ASFVOLT16_VLAN_ID_ETH2, |
| # ASGVOLT64_VLAN_ID_ETH1, ETH1, ETH2, |
| # ASGvOLT64, ASXvOLT16. |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # None |
| #------------------------------------------------------------------------------ |
| dhclient_interface() |
| { |
| local value1=$1 |
| local value2=$2 |
| if [ "${OLT_MODEL}" = ${ASXvOLT16} ];then |
| dhclient1=dhclient.${ETH2}.${ASFVOLT16_VLAN_ID_ETH2} |
| eval $value1="'$dhclient1'" |
| elif [ "${OLT_MODEL}" = ${ASGvOLT64} ];then |
| dhclient3=dhclient.${ETH1}.${ASGVOLT64_VLAN_ID_ETH1} |
| eval $value1="'$dhclient3'" |
| fi |
| |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: check_services |
| # Description: |
| # This function check the expected services, like system services for ex: |
| # onlp, sshd, bal_core_dist, dhclient and physical and vlan interfaces status. |
| # |
| # Globals: |
| # VLAN_CONFIG_FILE, ASFVOLT16_VLAN_ID_ETH2, ASGVOLT64_VLAN_ID_ETH1 |
| # TIME_INTERVAL, OLT_MODEL, ASXvOLT16, ASGvOLT64 |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # None |
| #------------------------------------------------------------------------------ |
| check_services() |
| { |
| # Let bal services are up and running |
| waiting_time=0 |
| while true; do |
| sleep ${TIME_INTERVAL} |
| if [ $waiting_time -eq 70 ]; then |
| break |
| fi |
| waiting_time=$((waiting_time+${TIME_INTERVAL})) |
| done |
| |
| if [ -f ${VLAN_CONFIG_FILE} ]; then |
| ASFVOLT16_VLAN_ID_ETH2=$(awk '/asfvolt16_vlan_id_eth2/{print $0}' \ |
| ${VLAN_CONFIG_FILE} | awk -F "=" '{print $2}') |
| ASGVOLT64_VLAN_ID_ETH1=$(awk '/asgvolt64_vlan_id_eth1/{print $0}' \ |
| ${VLAN_CONFIG_FILE} | awk -F "=" '{print $2}') |
| if [ "${OLT_MODEL}" = ${ASXvOLT16} ];then |
| if [ ${ASFVOLT16_VLAN_ID_ETH2} -gt 4094 ] || [ ${ASFVOLT16_VLAN_ID_ETH2} -lt 1 ]; then |
| error_message "vlan ids not in range" |
| exit 1 |
| fi |
| elif [ "${OLT_MODEL}" = ${ASGvOLT64} ];then |
| if [ ${ASGVOLT64_VLAN_ID_ETH1} -gt 4094 ] || [ ${ASGVOLT64_VLAN_ID_ETH1} -lt 1 ]; then |
| error_message "vlan ids not in range" |
| exit 1 |
| fi |
| fi |
| else |
| error_message "${VLAN_CONFIG_FILE} does not exist" |
| exit 1 |
| fi |
| |
| info_message "Sanity check" |
| for func_name in validate_interfaces validate_vlan_intf_ip validate_system_services |
| do |
| counter=0 |
| while true; do |
| sleep ${TIME_INTERVAL} |
| $func_name |
| if [ $? -eq 0 ]; then |
| break |
| fi |
| if [ $counter -eq 100 ]; then |
| error_message "Time out ,all services are not up" |
| return 1 |
| fi |
| counter=$((counter+${TIME_INTERVAL})) |
| done |
| done |
| display_services |
| return 0 |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: change_labels |
| # Description: |
| # This function does the following functions |
| # 1) After the upgrade procedure installs the image in stand_by_partition, |
| # This function checks for the system services after upgrade, |
| # if all ther services are up and running then following steps takes place |
| # a) It changes the device labels from Active to Standby and |
| # Standby to Active. |
| # b) It sets the grub menu entry from 1 to 0 which is default |
| # grub entry. After reboot in the grub menu entry will |
| # have one entry. |
| # c) It will swap the upgrade image to Image directory and vice |
| # versa. |
| # 2) If system services fails to start after image upgrade, |
| # roll-back to previous state of OLT i.e |
| # boot the OLT back from the active partition. |
| # |
| # Globals: |
| # RUNNING_BOOT_DEV, STANDBY_BOOT_LABEL, RUNNING_ROOT_DEV, STANDBY_ROOT_LABEL |
| # STANDBY_BOOT_DEV, RUNNING_BOOT_LABEL, STANDBY_ROOT_DEV, RUNNING_ROOT_LABEL |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # None |
| #------------------------------------------------------------------------------ |
| change_labels() |
| { |
| grep "data-standby" /proc/cmdline > /dev/null |
| if [ $? -eq 0 ]; then |
| check_services |
| # if the OLT system services fail to start, fall back to previous state |
| if [ $? -eq 1 ]; then |
| reboot -f |
| fi |
| # Change running boot device label to standby boot label |
| e2label $RUNNING_BOOT_DEV $STANDBY_BOOT_LABEL |
| # Change running root device label to standby root label |
| e2label $RUNNING_ROOT_DEV $STANDBY_ROOT_LABEL |
| # Change standby boot device label to active boot label |
| e2label $STANDBY_BOOT_DEV $RUNNING_BOOT_LABEL |
| # Change standby root device label to active root label |
| e2label $STANDBY_ROOT_DEV $RUNNING_ROOT_LABEL |
| partprobe |
| |
| if [ -f ${ACTIVE_GRUB_ENV_FILE} ]; then |
| next_entry=$(grub-editenv ${ACTIVE_GRUB_ENV_FILE} list | sed -n 's/^next_entry=//p') |
| saved_entry=$(grub-editenv ${ACTIVE_GRUB_ENV_FILE} list | sed -n 's/^saved_entry=//p') |
| if [ ${next_entry} -eq 1 ] || [ ${saved_entry} -eq 1 ]; then |
| grub-editenv ${ACTIVE_GRUB_ENV_FILE} set next_entry=0 |
| grub-editenv ${ACTIVE_GRUB_ENV_FILE} set saved_entry=0 |
| fi |
| fi |
| |
| if [ -f ${STANDBY_GRUB_ENV_FILE} ]; then |
| next_entry=$(grub-editenv ${STANDBY_GRUB_ENV_FILE} list | sed -n 's/^next_entry=//p') |
| saved_entry=$(grub-editenv ${STANDBY_GRUB_ENV_FILE} list | sed -n 's/^saved_entry=//p') |
| if [ ${next_entry} -eq 1 ] || [ ${saved_entry} -eq 1 ]; then |
| grub-editenv ${STANDBY_GRUB_ENV_FILE} set next_entry=0 |
| grub-editenv ${STANDBY_GRUB_ENV_FILE} set saved_entry=0 |
| fi |
| fi |
| |
| if [ -f ${GRUB_CONFIG_FILE} ]; then |
| if [ -f "${GRUB_ACTIVE_PATH}/${GRUB_CFG}" ]; then |
| cp ${GRUB_CONFIG_FILE} ${GRUB_ACTIVE_PATH}/${GRUB_CFG} |
| fi |
| if [ -f "${GRUB_STAND_PATH}/${GRUB_CFG}" ]; then |
| cp ${GRUB_CONFIG_FILE} ${GRUB_STAND_PATH}/${GRUB_CFG} |
| fi |
| fi |
| |
| if [ -d ${UPGRADE_PATH} ]; then |
| cd ${UPGRADE_PATH} |
| upgrade_image=$(ls *.swi) |
| mv $upgrade_image "$upgrade_image.bak" |
| cd ${IMAGES} |
| active_image=$(ls *.swi) |
| mv $active_image "$active_image.bak" |
| mv ${UPGRADE_PATH}/"$upgrade_image.bak" ${IMAGES}/$upgrade_image |
| mv ${IMAGES}/"$active_image.bak" ${UPGRADE_PATH}/$active_image |
| fi |
| fi |
| |
| } |
| |
| #------------------------------------------------------------------------------ |
| # Function Name: get_labels |
| # Description: |
| # This function get the labels of the devices and also find out the |
| # Running and Standby devices. |
| # |
| # Globals: |
| # RUNNING_ROOT_DEV, RUNNING_ROOT_LABEL, RUNNING_BOOT_LABEL, STANDBY_BOOT_LABEL |
| # RUNNING_ROOT_LABEL, STANDBY_ROOT_LABEL |
| # |
| # Arguments: |
| # None |
| # |
| # Returns: |
| # None |
| #------------------------------------------------------------------------------ |
| get_labels() |
| { |
| RUNNING_ROOT_DEV=$(cat /proc/mounts | grep " \/ ext4" | awk '{print $1}') |
| RUNNING_ROOT_LABEL=$(blkid | grep ${RUNNING_ROOT_DEV} | awk '{print $2}' | cut -c 7- | tr -d \") |
| if [ ${RUNNING_ROOT_LABEL} = "ONL-ACTIVE-DATA" ]; then |
| RUNNING_BOOT_LABEL=${ONL_ACTIVE_BOOT} |
| STANDBY_BOOT_LABEL=${ONL_STANDBY_BOOT} |
| RUNNING_ROOT_LABEL=${ONL_ACTIVE_DATA} |
| STANDBY_ROOT_LABEL=${ONL_STANDBY_DATA} |
| else |
| RUNNING_BOOT_LABEL=${ONL_STANDBY_BOOT} |
| RUNNING_ROOT_LABEL=${ONL_STANDBY_DATA} |
| STANDBY_BOOT_LABEL=${ONL_ACTIVE_BOOT} |
| STANDBY_ROOT_LABEL=${ONL_ACTIVE_DATA} |
| fi |
| STANDBY_ROOT_DEV=$(blkid | grep " LABEL=\"$STANDBY_ROOT_LABEL\"" | awk '{print $1}' | tr -d :) |
| RUNNING_BOOT_DEV=$(blkid | grep " LABEL=\"$RUNNING_BOOT_LABEL\"" | awk '{print $1}' | tr -d :) |
| STANDBY_BOOT_DEV=$(blkid | grep " LABEL=\"$STANDBY_BOOT_LABEL\"" | awk '{print $1}' | tr -d :) |
| } |
| |
| # Starts from here |
| does_logger_exist |
| if [ $? -eq 0 ]; then |
| get_labels |
| change_labels |
| else |
| error_message "logger does not exist" |
| exit 1 |
| fi |