# March 26, 2020 (Raj Patel)
# Edited September 09, 2021
# Edited April 13, 2024
# If CMDB was working, and PowerVC or HMC events are no longer in sync with CMDB, viosvc.log or viossvc.err additinal steps from HMC may be required. 
# In such cases, you may need to run below two command. If there are still problems work with approriate teams.

# commands ro be run from HMC if the CMDB and HMC events are not in sync 
  # https://[hmc_ip]:12443/rest/api/uom/Debug/ClearVIOSCache
  # https://[hmc_ip]:12443/rest/api/uom/Debug/DumpVIOSCache

# THIS SCRIPT IS TO BE USEDS FOR RESTARTING CMDB WHEN ANY CHAGES ARE MADE TO HOSTNAME RESOLUTION

####################################################################
# This script is intended to be used on rare situation especially
# where the CMDB needs to be removed and new tables, triggers and
# procedures need to be rebuilt for CMDB.
###################################################################

####################################################################
# Basic Checks before using this script:
####################################################################
# a) CPU Requirements & Memory requirements for SSP, PowerVC, NovaLink.
# b) Load on configuration based on network and storage configured.
# c) Correctly configured network IPs, hostname and  DNS so that
#    Name Server lookup (nslookup), Reverse Lookup and Host DNS Lookup
#    is correctly resolved on PowerVC, NovaLink, HMC and VIOs
#    using the Fully Qulified Name (FQN).
# d) Basic logs on PowerVC, HMC and VIOs based on problem and error.
####################################################################

##########################################################################
# DEBUGGING: Collecting Basic data and analizing before running the script
##########################################################################
# PowerVC: http://www-01.ibm.com/support/docview.wss?uid=nas8N1020600
# NovaLink: http://www-01.ibm.com/support/docview.wss?uid=nas8N1020600
# HMC: http://www-01.ibm.com/support/docview.wss?uid=isg3T1012079
# VIOs: http://www-01.ibm.com/support/docview.wss?uid=isg3T1013064
# Screen shot errors from GUI
# NOTE: Make sure date and time and zone are synced on all above.

####################################################################
# Run this from oem_setup_env on the problem node.
# NOTE: For SSP, a new  DBN node will be elected when run on DBN node with no impact. 
####################################################################

#----------------------------------------------------------------------

handle_CMDB()
{

set done=0
set retry=0
set myPID=0

if [ -d /tmp/CMreset  ]
then
    rm -R /tmp/CMreset
fi

mkdir /tmp/CMreset

# Write to log directory
  echo "Saving logs in: /tmp/CMreset/CMcleanup.history.log\n" >> /tmp/CMreset/CMcleanup.history.log

  # get host lookup details
    echo "====== locate hostname ======" >> /tmp/CMreset/CMcleanup.history.log
    host `hostname` >> /tmp/CMreset/CMcleanup.history.log

  # get ps for CM  before cleanup
  echo "======" >> /tmp/CMreset/CMcleanup.history.log
  date >> /tmp/CMreset/CMcleanup.history.log
  echo "======  ps -ef of CM before cleanup starting ======" >> /tmp/CMreset/CMcleanup.history.log
  ps_cm_before=`ps -ef |grep CM | tee >> /tmp/CMreset/CMcleanup.history.log`

   # Stop inetd
     stopsrc -s inetd ;

   # Restart inetd
     startsrc -s inetd
     sleep 10;

   # stop the vio_daemon and restart the vio_daemon

   stopsrc -s vio_daemon
   while [[ $done -ne 1 ]]
   do
       # check if the daemon is stopped or not
       LC_ALL=C lssrc -s vio_daemon | grep "inoperative"
       if [ $? -ne 0 ]
       then
          sleep 3
          retry=$retry+1
          if [ $retry -ge 7 ]
          then
              myPID=`lssrc -s vio_daemon | grep vio_daemon |tr -s " " | cut -d " " -f 3`
              kill -9 $myPID
              sleep 1
              done=1
          fi
       else
          done=1
       fi
   done


if [ -d /var/vio/CM ]
then
    rm -R /var/vio/CM
fi

if [ -d /home/ios/CM ]
then
    rm -R /home/ios/CM
fi

slibclean

startsrc -s vio_daemon

sleep 60

 # get ps for CM  after cleanup
 echo "======" >> /tmp/CMreset/CMcleanup.history.log
 date >> /tmp/CMreset/CMcleanup.history.log
 echo "======" >> /tmp/CMreset/CMcleanup.history.log
 echo "======   ps -ef of CM after cleanup completed  ======" >> /tmp/CMreset/CMcleanup.history.log
 echo "======" >> /tmp/CMreset/CMcleanup.history.log
 ps_cm_after=`ps -ef |grep CM | tee >> /tmp/CMreset/CMcleanup.history.log`
 echo "======" >> /tmp/CMreset/CMcleanup.history.log
 echo "====== listing files in /home/ios/CM/DB after cleanup ======" >> /tmp/CMreset/CMcleanup.history.log
 ls_cm_db=`ls -ltrb /home/ios/CM/DB | tee >> /tmp/CMreset/CMcleanup.history.log`
 sleep 180;
 echo "====== listing files in /home/ios/CM/DB after 180 seconds ======" >> /tmp/CMreset/CMcleanup.history.log
 ls_cm_db_180sec=`ls -ltrb /home/ios/CM/DB | tee >> /tmp/CMreset/CMcleanup.history.log`
 echo "======" >> /tmp/CMreset/CMcleanup.history.log
 echo "====== Last few lines from viod_CM.log after cleanup ======" >> /tmp/CMreset/CMcleanup.history.log
 tail_cm_db=`tail -n 60 /home/ios/logs/viod_CM.log | tee >>  /tmp/CMreset/CMcleanup.history.log &`
 echo "======" >> /tmp/CMreset/CMcleanup.history.log

 pid=`ps -ef |grep -v grep|grep vio_daemon|awk '{print $2}'`;echo $pid;kill -1 $pid

}

#----------------------------------------------------------------------
# *  MAIN SCRIPT  *
#----------------------------------------------------------------------

# handle CMDB
handle_CMDB

echo "upload file /tmp/CMreset/CMcleanup.history.log if requested if problem is not resolved with snap data"

exit 0