#!/bin/ksh
#
#
# Copyright IBM Corp. 1998, 2011
#
# $Workfile:   ui $
# $
#
#*******************************************************************************
#*******************************************************************************
#
#*******************************************************************************
#
# For debug
# set -xv
#
#*******************************************************************************
#*******************************************************************************
#
#   Author:   WTX L3 support
#
#   Description:
#   This script is being written to capture local information when a product 
#   abort or hang has occurred.
#    This program is designed to capture - 
#      1) system information
#      2) local log information
#      3) stackt trace information
#   The stack trace information is collected via the use of a local debugger
#      Platform       Debugger
#      AIX            dbx
#      HPUX           gdb
#      HPUX (Itanium) gdb
#      Solaris        dbx/gdb
#      Tru64          dbx/gdb
#			 Linux (RedHat) 
#     

#*******************************************************************************
#*******************************************************************************
#
# History:
# Version: 1.01 - 01apr2002
#    Ron B. - 1st draft at muti-platform information collection script
#    Currently untried for Tru64
# Version: 1.01 - 02apr2002
#    Added hostname logging       
# Version: 1.02 - 05apr2002
#    Added installation notes in commentary; updated comments
# Version: 1.03 - 08apr2002
#    Added more directory logging
# Version: 1.04 - 08apr2002
#    Added running pstack for Solaris even if dbx/gdb debuggers not found.
# Version: 1.05 - 02may02
#    Added setting output directory for log files
# Version: 1.06 - 08may02
#    Added more for HPUX - Memory and Kernel Params
# Version: 1.07 - 24may02
#    Added Date Stamp check for core files
# Version: 1.08 - 28may02
#    If second argument is NULL and local core file does not exist then
#    get system and environmental info only - no debugger session
# Version: 1.09 - 03jun02
#    Output version of ui to logs
# Version: 1.10 - 03jun02
#    MERC_HOME_DIR/bin - needed delineating title
#    - update kmtune location if not initially found
# Version: 1.11 - 06jun02
#    Check for AIX processor type - 32 vs 64 bit
# Version: 1.12 - 26jun02
#    Add merchttpd to list of executables
#    For Solaris; check for existence of prtmem before execution
#    Include size of script at top for reference
# Version: 1.13 - 23jul02
#    Added more ps info to ps.log file for AIX & SUN
#    to make it 'top' like. Still need to do HP
# Version: 1.14 - 24jul02
#    Added more ps info to ps.log file for HP
#    to make it 'top' like.
# Version: 1.15 - 31jul02
#    Added executable and library check for AIX
# Version: 1.16 - 21aug02
#    Added swap info for AIX and Solaris
#    For "df" command add -t for totoal usage too
# Version: 1.17 - 21jan03
#    Add patch info for AIX/HP/Solaris
# Version: 1.17a - 10mar03
#    Added back in capture for environmental info(i.e lost from 1.16)
# Version 1.18 - 10mar03
#    Added tar'ring up files for ease of delivery to support.
#    Also, save mercator.ini file if available.
# Version 1.19 - 12mar03
#    Added count for # of CPUs & Processor Info
# Version 1.20 - 03apr
#    Added more Header Info and Time Stamps between sections 
#    being processed.
# Version 1.21 - 14apr
#    Updated cross check for Process Id 
#    Added saving of crash.log file if in same dir as core file
#    Added saving of ABORT-<pid>.txt file if it exist
#    Added savign of mercAbort.log file
# Version 1.22 - 14apr
#    Added getting logs dir files for last day
# Version 1.23 - 06may
#    Added getting OS level for AIX
#    Change launcher check from doing ps to using pid from .launcherPID
# Version 1.24 - 03Jan
#    Change thread list processing - needed for the latest gdb on HPUX
# Version 1.25 - 14Apr 2005 - Vlad
#    Make it to work Linux and Itanium on all versions
# Version 1.26 - 19Apr 2005 - Vlad
#    Correct version info problem
# Version 1.27 - 06Sep 2006 - Vlad
#    add java to the list of processes
# Version 1.29 - May2007 - Vlad
# remove test for the process name
# Version 1.30 - 21Oct 2008 - Chris
#    add evntagnt to the list of processes, and increased version number just
#    to be safe.
# Version 1.31 - Nov20 2008 - Vlad
#    fix Itanium issue - shared lib extension is .so
# Version 1.32 - March20 2009 - Vlad
#    add AIX genkld for list of loaded libs
# Version 1.33 - April 30 2009 - Vlad
#    added dtxcmdsv and dstx
# Version 1.34 - Jan 30 2010 - Vlad
#    64 bits aix dump
# Version 1.35 - Feb 16 2010 - Vlad
#    remove test for launcher pid, fix naming
# Version 1.36 - March 23 2010 - Vlad
#    AIX -d option for deep stack trace and dbx
# Version 1.37 - March 23 2010 - Vlad
#    remove misleading launcher pid cross check message
# Version 1.38 - Nov 30 2011 - Vlad
#    copyright statement
# Version 1.39 - Feb 02 2012 - Vlad
#    Linux patches rpm report
#
#*******************************************************************************
#*******************************************************************************
#
# BEGINNING of commentary
#
# This script is intended to assist in collecting product and environmental 
# information from an aborted or hung process.
# 

# INSTALLATION:
# The script needs to be installed in the directory:
#   $MERC_HOME_DIR/bin
# and it needs the execute permissions set.
# 

# Pre-requisite:
# It is expected that the user has already run the setup script with the 
# dot command as in:
#    . ./setup
# from the $MERC_HOME_DIR directory.
# 

# The syntax for usage is:
#       ui <product> <core>/<pid> [OutputDirectory]
# where <product> is one of 
#       launcher, ldirmon, lci, mercator, merchttpd, java, evntagnt
# and, where <core> is the name of the core file; 
#   if the core file is in another directory you need to add the leading path;
# or, where <pid> is the process id of the running process; 
#   typically obtained from the "ps" command.
# and, where you can specify an optional output directory, the default is the
#   current directory.
# 

# Without being verbose - a simple example serves as an OK explanation.
# 

# An ABORT scenario:
# To run against a core file from the Event Server - 
# assuming you are in directory $MERC_TMP_DIR/launcher_4567
#       ui launcher core
# The output logs would include:
#       <date/time>_core_ThreadList.txt
#       <date/time>_core__launcher.log
#       <date/time>_core_ps.log
#       <date/time>_core_patches.log
# where <date/time> is a date and time stamp from the time 
#   which the "ui" script is run.
# 

# Alternately, the second argument is optional in the case of a core file.
# If none is specified then a default of "core" is assumed.  However, if
# a local core file is not found then we get system and environmental 
# information only.
# In the case of getting the system and environmental info only, the output
# file format changes as follows - 
# Example:
#   ui launcher
# The output logs would include:
#       <date/time>_misc__launcher.log
#       <date/time>_misc_ps.log
#       <date/time>_misc_patches.log
# where <date/time> is a date and time stamp from the time 
#   which the "ui" script is run.
# 

# A HANG scenario:
# To run against a running Event Server with process id of 4567
#       ui launcher 4567
# The output logs would include:
#       <date/time>_<pid>_ThreadList.txt
#       <date/time>_<pid>_launcher.log
#       <date/time>_<pid>_ps.log
#       <date/time>_<pid>_patches.log
# where <date/time> is a date and time stamp from the time 
#   which the "ui" script is run.
# and <pid> is the process which you ran the "ui" script against
# 
 
# The OUTPUT file:
#    <date/time>_<core/pid>_<product>.log 
# (i.e. *<product>.log) file is broken up into different sections.
# To go from section to section search for the keyword "Information"
#

# For a non-typical case -  an example:
# Sometimes a customer needs to be in a certain directory - i.e. to access 
#   private shared libraries.
# 
# Lets say the Event Server is installed in /mercator and they are in 
# directory /mercator/maps which contains the private shared libraries.  
# The core file is located in /mercator/tmp/launcher_4567
# 
# To examine a core file for the above situation do:
#       ui launcher ../tmp/launcher_4567/core
# or
#       ui launcher /mercator/tmp/launcher_4567/core
# 

# HANG vs ABORT -  what to do:
# If the situation is a hang scenario then I typically advise that they:
# 1)   run the "ui" script
# 2)   wait a couple of minutes and run the "ui" script again
# 3)   run the lci snapshots sub-menus
# 4)   kill the process with a "-6" to produce a core file; 
#         sometimes a second kill -6 is needed
# 5)   if the process does not abort then kill it with a "-9"
# 6)   if a core file was produced then run the ui script on it
# 7)   use the "ps" command to ensure the ldirmon process is not running 
#         before restarting the Event Server.
# 8)   Restart the Event Server when ready.
# 9)   Collect all logs generated by the "ui" script and send them to support.
# 
# The second run of the "ui" script is needed so we can easily see 
#   which threads are hung and which are not.
#
# If the situation is an abort then skip to step 6), above.
#

# Information for JAVA debug:
# ui /fullpathtowherejava_executable_is_located/java PID
# Full path to where the java is located and PID can be found using:
# ps -afe | grep java
# for core dumps:
# ui /fullpathwherejava_executable_is_located/java /fullpathto/the/core
#
# END of commentary
#*******************************************************************************
#*******************************************************************************

UI_VERSION=1.38

mercver 1 > /dev/null 2>&1 ; if [ $? != "0" ] ; then 
	 echo "set mercver to dstxver"
	 alias mercver=dstxver
fi
header_info()
{
echo "#***********************************************************************"
echo "Date: `date`"
echo "Info: $1"
echo "#***********************************************************************"
echo " "
}

usage="\n 
   Usage: `basename $0` <executable> <pid/core> [OutputDirectory]\n
   \n
   Where <executable> is the process name\n
   and where <pid/core> is either the process id of the running executable\n
   or, a core file produced by the indicated executable.\n
   You can also specify an optional Output Directory.\n
   \n
   Exiting\n
   "

if [ "$1" = "-help" -o "$1" = "-?" ]; then
   echo " "
   echo "This program requires the user to:"
   echo "  1) Have the debugger in their PATH variable."
   echo "  2) Know the Process ID, or <PID>, for the executable"
   echo "     or, know the location of the core file."
   echo "  3) in the case of a running process, you need to be the user"
	echo "     who started it."
   echo "  4) Have previously used the dot command on the setup script"
   echo "     from the current terminal session."
   echo " "
   echo $usage
   exit
fi

# Section Info
header_info "Begin UI Script Processing, Version:  $UI_VERSION"

NumberOfArgs="$#"
OutputDir=""
if [ $NumberOfArgs -lt 1 ]; then
   echo " "
   echo " Info: Missing Argument"
   echo $usage
   exit 0
fi

if [ $NumberOfArgs -eq 1 ]; then
	arg2=core
else
   arg2=$2
fi

cwd=`pwd`
if [ $NumberOfArgs -eq 3 ]; then
  OutputDir="$3"
  if [ ! -d "$OutputDir" ]; then
     echo " "
     echo " Warning: Invalid Output Directory of:"
     echo " $OutputDir"
     exit 0
  fi
else
  OutputDir=$cwd
fi

# check for MERC_HOME_DIR and MERC_TMP_DIR
if [ -z "$MERC_HOME_DIR" ]; then
   echo "\n
   This script requires knowledge of the MERC_HOME_DIR directory
   which is not currently present.  You need to run the setup
   script with the dot command before running this script.
   Exiting.
   \n"
   exit 0
else
   mhd=$MERC_HOME_DIR
fi
if [ ! -d "$mhd" ]; then
   echo " "
   echo Error, MERC_HOME_DIR not found: $mhd
   echo Exiting
   echo " "
fi

if [ -z "$MERC_TMP_DIR" ]; then
   echo " "
   echo This script requires knowledge of the MERC_TMP_DIR directory
   echo which is not currently set.  Using default of /tmp.
   echo Continuing.
   echo " "
   MERC_TMP_DIR=/tmp
   export MERC_TMP_DIR
fi
mtd=$MERC_TMP_DIR
if [ ! -d "$mhd" ]; then
   echo " "
   echo Error, MERC_HOME_DIR not found: $mhd
   echo Exiting
   echo " "
fi
if [ ! -d "$mtd" ]; then
   echo " "
   echo Error, MERC_TMP_DIR not found: $mtd
   echo Exiting
   echo " "
fi
 

script=$0
product=$1
prod=`basename $product`
a2=`basename $arg2`
pid=""

#if [ "$prod" != "launcher" -a "$prod" != "mercator" -a "$prod" != "dstx" -a "$prod" != "lci" -a "$prod" != "ldirmon" -a "$prod" != "merchttpd" -a "$prod" != "java" ]; then
#  echo " "
#  echo " Invalid executable name"
#  echo $usage
#  exit 0
#fi

# check for word core in core file
echo "$a2" | grep core 1>/dev/null 2>&1
corestat=$?

if [ ! -f $arg2 -a "$corestat" = 1 ]; then # must be pid
# the following was intended as a sanity type test to ensure customer didn't 
# accidentally enter incorrect process id
  if [ "$product" != "launcher" ]; then
    pid=`ps -ef | grep $arg2 | grep -v grep | grep -v $script | awk '{print $2; exit}' 2>/dev/null`
     if [ -z "$pid" ]; then
       echo " Argument $2 does not appear to be a valid pid."
       echo " ReCheck the value and try again."
       echo $usage
       exit 0
     fi
  fi

  if [ "$product" = "launcher" ]; then
	  if [ ! -s "$MERC_HOME_DIR/.launcherPID" ]; then
		 echo "Warning - Launcher Process is not currently seen as running."
		 echo "Continuing anyway . . ."
		 echo " "
		 pid=$arg2
	  else
		  launcherPid=`cat $MERC_HOME_DIR/.launcherPID`

		  pid=$arg2
	  fi
  fi
  tc=pid
else
  miscInfoOnly=no
  if [ ! -f $arg2 ]; then
    echo " " 
    echo " Core file not found, getting system & environmental info ..."
    echo " " 
  	 miscInfoOnly=yes
  fi
  tc=core
  coreFile=$arg2
fi

# if the executable is launcher or mercator then use the one in $MERC_HOME_DIR
# otherwise, try to use the one specified
if [ "$prod" = "launcher" -o "$prod" = "mercator" -o "$prod" = "lci" -o "$prod" = "ldirmon" -o "$prod" = "merchttpd" -o "$prod" = "evntagnt" -o "$prod" = "dtxcmdsv" -o "$prod" = "dstx" ]; then
	if [ -f $MERC_HOME_DIR/bin/$prod ]; then
		product=$MERC_HOME_DIR/bin/$prod
	else
		product=$prod
	fi
 
fi

if [ ! -f "$product" ]; then
	echo " "
	echo " Executable not found: $product"
   echo $usage
   echo " "
   exit 0
fi

# Variable setup
mypid=$$
today=`date +"%d%b%y@%H%M.%S"`
if [ $tc = "pid" ]; then
  psLog=$OutputDir/${today}_${pid}_ps.log
  patchLog=$OutputDir/${today}_${pid}_patches.log
  threadList=$OutputDir/${today}_${pid}_ThreadList.txt
  tmpThreadList=$OutputDir/${today}_${pid}_tmpThreadList.txt
  dbxCmdList=$OutputDir/${today}_${pid}_dbxCmdList.txt
  gdbCmdList=$OutputDir/${today}_${pid}_gdbCmdList.txt
  outLog=$OutputDir/${today}_${pid}_${prod}.log
  envLog=$OutputDir/${today}_${pid}_env.log
  kernelLog=$OutputDir/${today}_${pid}_kernel.log
else
  if [ "$miscInfoOnly" = "yes" ]; then
	  psLog=$OutputDir/${today}_misc_ps.log
	  patchLog=$OutputDir/${today}_misc_patches.log
	  threadList=$OutputDir/${today}_misc_ThreadList.txt
	  tmpThreadList=$OutputDir/${today}_misc_tmpThreadList.txt
	  dbxCmdList=$OutputDir/${today}_misc_dbxCmdList.txt
	  gdbCmdList=$OutputDir/${today}_misc_gdbCmdList.txt
	  outLog=$OutputDir/${today}_misc_${prod}.log
	  envLog=$OutputDir/${today}_misc_env.log
	  kernelLog=$OutputDir/${today}_misc_kernel.log
  else
	  psLog=$OutputDir/${today}_core_ps.log
	  patchLog=$OutputDir/${today}_core_patches.log
	  threadList=$OutputDir/${today}_core_ThreadList.txt
	  tmpThreadList=$OutputDir/${today}_core_tmpThreadList.txt
	  dbxCmdList=$OutputDir/${today}_core_dbxCmdList.txt
	  gdbCmdList=$OutputDir/${today}_core_gdbCmdList.txt
	  outLog=$OutputDir/${today}_core_${prod}.log
	  envLog=$OutputDir/${today}_core_env.log
	  kernelLog=$OutputDir/${today}_core_kernel.log
  fi
fi
#errlog=$OutputDir/${today}_errpt.log

#
# tar up log files into tar file
exit_time()
{
cdir=`pwd`

#if we have a logs directory - save log files for last day
if [ -d $MERC_HOME_DIR/logs ] ;then
  cd $MERC_HOME_DIR/logs
  find . -mtime -1 -print  >  $OutputDir/${today}_logsdir 2>/dev/null
  tar -cvf $OutputDir/${today}_logsdir.tar  `cat $OutputDir/${today}_logsdir` 1>/dev/null 2>&1
fi

cd $OutputDir

ls ./${today}_* > .$$_files
if [ -f $MERC_HOME_DIR/config/mercator.ini ]; then
   cp $MERC_HOME_DIR/config/mercator.ini ${today}_mercator.ini
   ls ./${today}_mercator.ini >> .$$_files
elif [ -f $MERC_HOME_DIR/config/LCONFIG ]; then
   cp $MERC_HOME_DIR/config/LCONFIG ${today}_LCONFIG
   ls ./${today}_LCONFIG >> .$$_files
fi

# if we have a core file
if [ $tc = "core" ]; then
	coreDir="`dirname $coreFile`"
	if [ "$coreDir" = "." ]; then
	  coreDir=`pwd`
	fi
   if [ -f $coreDir/crash.log ]; then
      cp $coreDir/crash.log ${today}_crash.log
      ls ./${today}_crash.log  >> .$$_files
   fi

# get ABORT-<pid>.txt file if it exist
   myPid=`grep "^PID" ./${today}_crash.log | awk '{print $3; exit}'`
   if [ -f ${MERC_HOME_DIR}/ABORT-${myPid}.txt ]; then
      cp ${MERC_HOME_DIR}/ABORT-${myPid}.txt  ${today}_ABORT-${myPid}.txt
      ls ./${today}_ABORT-${myPid}.txt  >> .$$_files
   fi
fi

# save mercAbort.log file if it exist
if [ -f "$MERC_TMP_DIR/mercAbort.log" ]; then
  cp $MERC_TMP_DIR/mercAbort.log ./${today}_mercAbort.log 
  ls ./${today}_mercAbort.log >> .$$_files
fi


tar -cvf ${today}_uilogs.tar .$$_files `cat .$$_files` 1>/dev/null 2>&1
rm -f ${today}_LCONFIG ${today}_mercator.ini ${today}_crash.log ./${today}_ABORT-${myPid}.txt ./${today}_mercAbort.log

compress ${today}_uilogs.tar
for file in `cat .$$_files`
do
   rm -f $file
done
rm .$$_files
echo " "
echo "The file "
echo "    $OutputDir/${today}_uilogs.tar.Z"
echo "needs to be given to IBM Websphere Transformation Extender support. Thank You."
echo " "
}

# Get Processor Info
get_processor_info()
{
echo "Processor Information -" >> $outLog
echo "Number of processors on this machine:" >> $outLog
if [ "`uname`" = "HP-UX" ]; then
	if [ -x /usr/sbin/ioscan ]; then
		/usr/sbin/ioscan -fnkC processor|grep -c '^processor' >> $outLog
	else
      echo "Unavailable" >> $outLog
	fi
elif [ "`uname`" = "AIX" ]; then
	if [ -x /usr/sbin/lsdev ]; then
		/usr/sbin/lsdev -C -c processor -S Available | grep -c . >> $outLog
	else
      echo "Unavailable" >> $outLog
	fi
elif [ "`uname`" = "SunOS" ]; then
	uname -X | grep NumCPU >> $outLog
elif [ "`uname`" = "Linux" ]; then
	cat /proc/cpuinfo | grep processor >> $outLog 
else
	if [ -x /usr/sbin/psrinfo ]; then
		/usr/sbin/psrinfo -v | grep "on-line" | wc -l >> $outLog 2>/dev/null
	else
      echo "Unavailable" >> $outLog
	fi
fi

echo " " >> $outLog
echo " " >> $outLog
echo "General Processor Information - " >> $outLog
if [ "`uname`" = "HP-UX" ]; then
	if [ -x /usr/sbin/ioscan ]; then
		/usr/sbin/ioscan -fnkC processor >> $outLog
	else
      echo "Unavailable" >> $outLog
	fi
elif [ "`uname`" = "Linux" ]; then
	cat /proc/cpuinfo >> $outLog 
elif [ "`uname`" = "AIX" ]; then
	if [ -x /usr/sbin/lsdev ]; then
		/usr/sbin/lsdev -C -c processor -S Available  >> $outLog
	else
      echo "Unavailable" >> $outLog
	fi
else
	if [ -x /usr/sbin/psrinfo ]; then
		/usr/sbin/psrinfo -v >> $outLog 2>/dev/null
	else
      echo "Unavailable" >> $outLog
	fi
fi
echo " " >> $outLog
}

# Section Info
header_info "Getting Environmental Info ..." > $outLog

# get local env log info
echo "UI Version: $UI_VERSION" > $envLog
echo " " >> $envLog
echo "Date: `date`" >> $envLog
echo " " >> $envLog
echo "User: `id`" >> $envLog
echo " " >> $envLog
echo "Host: `hostname`" >> $envLog
echo " " >> $envLog
echo "Current Working Directory: `pwd`" >> $envLog
echo " " >> $envLog
echo "Environmental info:" >> $envLog
echo " " >> $envLog
echo "`env`" >> $envLog
echo " " >> $envLog

# get kernel info
echo "UI Version: $UI_VERSION" > $kernelLog
echo " " >> $kernelLog
echo "Date: `date`" >> $kernelLog
echo " " >> $kernelLog
echo "User: `id`" >> $kernelLog
echo " " >> $kernelLog
echo "Host: `hostname`" >> $kernelLog
echo " " >> $kernelLog
echo "Current Working Directory: `pwd`" >> $kernelLog
echo " " >> $kernelLog


# Section Info
header_info "Getting Kernel Info ..." >> $outLog

echo "==== Kernel Information =================================================" >> $kernelLog
echo " " >> $kernelLog
# kernel information
if [ "`uname`" = AIX ]; then
	lsattr -E -l sys0 >> $kernelLog
elif [ "`uname`" = "HP-UX" ]; then
   type kmtune 1>/dev/null 2>&1
   if [ "$?" -eq 0 ]; then
		kmtune >> $kernelLog
   else
      if [ -x /usr/sbin/kmtune ]; then
			/usr/sbin/kmtune >> $kernelLog 2>&1
      fi
   fi
elif [ "`uname`" = "OSF1" ]; then
	/sbin/sysconfigdb -l >> $kernelLog
elif [ "`uname`" = "SunOS" ]; then
	/usr/sbin/sysdef -i >> $kernelLog
elif [ "`uname`" = "Linux" ]; then
	 cat /proc/version >> $kernelLog
	 cat /proc/uptime >> $kernelLog
	 cat /proc/loadavg >> $kernelLog
	 cat /proc/stat >> $kernelLog
fi
echo " " >> $kernelLog

# Section Info
header_info "Getting Process Status Info ..." >> $outLog

# get local log info
echo "UI Version: $UI_VERSION" > $psLog
echo " " >> $psLog
echo "Date: `date`" >> $psLog
echo " " >> $psLog
echo "User: `id`" >> $psLog
echo " " >> $psLog
echo "Host: `hostname`" >> $psLog
echo " " >> $psLog
echo "Current Working Directory: `pwd`" >> $psLog
echo " " >> $psLog
echo "Executable used: $product" >> $psLog
echo " " >> $psLog
pscmd="ps -efl" 
if [ "`uname`" = "HP-UX" ]; then
  UNIX95=1
  export UNIX95
  pmem=""
elif [ "`uname`" = "AIX" ]; then
  pscmd="ps -emo THREAD"
  pmem="-opmem"
else
  pmem="-opmem"
fi
if [ $tc = "pid" ]; then
   echo "Process Status for process: ${pid}\n" >> $psLog
	ps -opcpu -ovsz $pmem -p$pid >> $psLog
	echo " " >> $psLog
fi
echo "General Process Status Listing." >> $psLog
# used to see if extraneous ldirmon processes are running
$pscmd >> $psLog
echo " " >> $psLog
# For AIX only
#   ps_header=`ps -e -o "%U %p %G %C %n %z %x %c" | head -1` 
ps_header=`ps -ouser -opid -ogroup -opcpu -onice -ovsz -otime -ocomm | head -1`
#
echo "Top for `uname`" >> $psLog
echo "Sorted by %CPU" >> $psLog
echo $ps_header >> $psLog
ps -e -ouser -opid -ogroup -opcpu -onice -ovsz -otime -ocomm | grep -v "USER   PID    GROUP" | grep -v "USER       PID GROUP" | sort -nr -k 4 >> $psLog
echo " " >> $psLog
echo "Sorted by VSZ" >> $psLog
echo $ps_header >> $psLog
ps -e -ouser -opid -ogroup -opcpu -onice -ovsz -otime -ocomm | grep -v "USER   PID    GROUP" | grep -v "USER       PID GROUP" | sort -nr -k 6 >> $psLog
echo " " >> $psLog

# Section Info
header_info "Getting OS Patch Info ..." >> $outLog

# get local patch info
echo "UI Version: $UI_VERSION" > $patchLog
echo " " >> $patchLog
echo "Date: `date`" >> $patchLog
echo " " >> $patchLog
echo "User: `id`" >> $patchLog
echo " " >> $patchLog
echo "Host: `hostname`" >> $patchLog
echo " " >> $patchLog
echo "Current Working Directory: `pwd`" >> $patchLog
echo " " >> $patchLog
echo "Executable used: $product" >> $patchLog
echo " " >> $patchLog
if [ "`uname`" = "HP-UX" ]; then
	swlist > $patchLog 2>&1
elif [ "`uname`" = "AIX" ]; then
	lslpp -L all > $patchLog 2>&1
elif [ "`uname`" = "Linux" ]; then
	rpm -qav > $patchLog 2>&1
else
	showrev -a > $patchLog 2>&1
fi

# Section Info
header_info "Logging System Info ..." >> $outLog

# Date
echo "UI Version: $UI_VERSION" >> $outLog
echo " " >> $outLog
echo "Date: `date`" >> $outLog
echo " " >> $outLog
echo "User: `id`" >> $outLog
echo " " >> $outLog
echo "Host: `hostname`" >> $outLog
echo " " >> $outLog
echo "Current Working Directory: `pwd`" >> $outLog
echo " " >> $outLog
echo "Executable used: $product" >> $outLog
echo " " >> $outLog
if [ $tc = "pid" ]; then
   echo "Process ID used: $pid" >> $outLog
else
   echo "Core File used: $coreFile" >> $outLog
fi
echo " " >> $outLog

echo "==== System Information =================================================" >> $outLog
uname -a >> $outLog

if [ "`uname`" = AIX ]; then
	echo " " >> $outLog
	echo "Aix OS Level - " >> $outLog
	oslevel >> $outLog
fi

if [ "`uname`" = "HP-UX" ]; then
  theShell="sh"
else
  theShell=""
fi
echo "\nSystem Soft Limits - " >> $outLog
$theShell ulimit -Sa >> $outLog
echo "\nSystem Hard Limits - " >> $outLog
$theShell ulimit -Ha >> $outLog

echo " " >> $outLog
echo "\nAmount of Memory on this machine - " >> $outLog
if [ "`uname`" = AIX ]; then
	/usr/sbin/lsattr -E -l sys0 | grep realmem >> $outLog
   echo " " >> $outLog
   echo "Swap/Paging Info: " >> $outLog
	lsps -a >> $outLog
	lsps -s >> $outLog

elif [ "`uname`" = "HP-UX" ]; then
   echo "Dmesg: " >> $outLog
	/usr/sbin/dmesg | grep Physical >> $outLog
   echo " " >> $outLog
	echo "Syslog: " >> $outLog
	grep Physical /usr/adm/syslog/syslog.log | tail -1 | awk ' { print $7 / 1024 " MB System Ram" } ' >> $outLog
   echo " " >> $outLog
	echo "Swapinfo: " >> $outLog
	/usr/sbin/swapinfo -t >> $outLog
elif [ "`uname`" = SunOS ] ; then
	/usr/sbin/prtconf | grep Memory >> $outLog
   if [ -x /usr/bin/prtmem ]; then
	   /usr/bin/prtmem >> $outLog
	fi
	echo "Swap Info: " >> $outLog
   /usr/sbin/swap -s >> $outLog
elif [ "`uname`" = Linux ] ; then
	 cat /proc/meminfo >> $outLog
fi
echo " " >> $outLog

if [ "`uname`" = AIX ]; then
	 echo "AIX loaded libs" >> $outLog
	 genkld >> $outLog
fi
# Section Info
header_info "Getting Processor Info ..." >> $outLog

# Get Processor Info
get_processor_info


# Section Info
header_info "Getting Disk Info ..." >> $outLog

# Disk stats
echo "==== Disk Capacity/Usage ================================================" >> $outLog
# 14apr03 - route errors to /dev/null
if [ "`uname`" = Linux ]; then
df >> $outLog 2>/dev/null
cat /proc/mounts >> $outLog 2>/dev/null
else
df -kt >> $outLog 2>/dev/null
fi
echo " " >> $outLog

# Section Info
header_info "Getting IPCS Info ..." >> $outLog

# ipcs information
echo "==== IPCS Information ===================================================" >> $outLog
ipcs >> $outLog 2>/dev/null
echo " " >> $outLog

# Section Info
header_info "Directory Listing Info ..." >> $outLog

# MERC_TMP_DIR info
# directory listing
echo "==== MERC_TMP_DIR Directory  Information ================================" >> $outLog
echo dir: $MERC_TMP_DIR >> $outLog
ls -altR $MERC_TMP_DIR >> $outLog
echo " " >> $outLog

# MERC_HOME_DIR info
# directory listing
# checksum listing
echo "==== MERC_HOME_DIR Directory  Information ===============================" >> $outLog
echo dir: $MERC_HOME_DIR >> $outLog
ls -alt $MERC_HOME_DIR >> $outLog
echo dir: $MERC_HOME_DIR/bin >> $outLog
ls -alt $MERC_HOME_DIR/bin >> $outLog
echo " " >> $outLog

# Section Info
header_info "Extracting BIN Mercver Info ..." >> $outLog

echo "==== MERC_HOME_DIR/bin Mercver  Information =============================" >> $outLog
echo dir: ${MERC_HOME_DIR}/bin >> $outLog
cd $MERC_HOME_DIR/bin
mercver >> $outLog 2>&1
echo " " >> $outLog

# Section Info
header_info "Extracting mercAbort.log Info ..." >> $outLog

echo "==== mercAbort.log Information ==========================================" >> $outLog
echo " " >> $outLog
if [ -s "$MERC_TMP_DIR/mercAbort.log" ]; then
	echo "File: mercAbort.log contains entries !!" >> $outLog
	echo "We are experiencing 'Out of memory' errors " >> $outLog
	echo " " >> $outLog
	tail -5 $MERC_TMP_DIR/mercAbort.log >> $outLog
	echo " " >> $outLog
else
   echo "File: mercAbort.log is empty." >> $outLog
	echo "We are NOT experiencing 'Out of memory' errors " >> $outLog
	echo " " >> $outLog
fi


# Section Info
header_info "BIN Dir Checksum Listing Info ..." >> $outLog

echo "==== Checksum Listing for MERC_HOME_DIR/bin Directory ===================" >> $outLog
cksum $MERC_HOME_DIR/bin/l* >> $outLog
cksum $MERC_HOME_DIR/bin/mercator >> $outLog
cksum $MERC_HOME_DIR/bin/*_abort.sh >> $outLog
if [ -s "$MERC_HOME_DIR/bin/evntagnt" ]; then
    cksum $MERC_HOME_DIR/bin/evntagnt >> $outLog
fi
if [ -s "$MERC_HOME_DIR/bin/merchttpd" ]; then
    cksum $MERC_HOME_DIR/bin/merchttpd >> $outLog
fi
echo " " >> $outLog

if [ "`uname`" = "HP-UX" ]; then
 if [ "`uname -m`" = "ia64" ]; then
  sosl=so
 else
  sosl=sl
 fi
else
  sosl=so
fi

# Section Info
header_info "Extracting LIBS Mercver Info ..." >> $outLog

echo "==== MERC_HOME_DIR/libs Mercver Information =============================" >> $outLog
echo dir: ${MERC_HOME_DIR}/libs >> $outLog
cd $MERC_HOME_DIR/libs
for lib in `ls *.$sosl`
do
   echo "lib: $lib" >> $outLog
	mercver $lib >> $outLog 2>&1
done
echo " " >> $outLog

# Section Info
header_info "LIBS Dir Checksum Listing Info ..." >> $outLog

echo "==== Checksum Listing for MERC_HOME_DIR/libs Directory ==================" >> $outLog
echo dir: ${MERC_HOME_DIR}/libs >> $outLog
cksum $MERC_HOME_DIR/libs/*.$sosl >> $outLog
echo " " >> $outLog

# return to working directory
cd $cwd

# for AIX check on data size limitation of executable
if [ "`uname`" = "AIX" ]; then

# Section Info
header_info "Processor Type Info ..." >> $outLog

	echo "==== Checking for Processor Type =====================================" >> $outLog
	echo "
The following are known 64 bit processors - 
        PowerPC_RS64
        PowerPC_RS64 II
        PowerPC_RS64 III
        PowerPC_Power3
        PowerPC_Power3II
" >> $outLog
	echo "Current Processor type: " >> $outLog
	lsattr -El proc0 >> $outLog 2>&1
	echo " " >> $outLog
	echo "==== Data Size Limit for Application =================================" >> $outLog
	dump -Xany -ov $product >> $outLog 2>&1
	echo " " >> $outLog

# Section Info
header_info "Dumping Header Info ..." >> $outLog

# Check on header info for $product and mercpi
	if [ -f $product ]; then
		echo "==== Dumping Header info for executable  ============================" >> $outLog
		echo "Executable: $product" >> $outLog
		dump -Xany -H $product >> $outLog
		echo " " >> $outLog
	fi
#
	if [ "`uname`" = "HP-UX" ]; then
	  sosl=sl
	else
	  sosl=so
	fi

	for lib in libmercpi.$sosl libmercadpt.$sosl libmresname.$sosl libm4file.$sosl
		do
		if [ -f $MERC_HOME_DIR/libs/$lib ]; then
			echo "==== Dumping Header info for library ================================" >> $outLog
			echo "Library: $lib" >> $outLog
			dump -Xany -H $MERC_HOME_DIR/libs/$lib >> $outLog
			echo " " >> $outLog
		fi
		done
fi

if [ "$miscInfoOnly" = "yes" ]; then
  echo " " >> $outLog
  echo " Obtained miscellaneous System & Environmental Info Only" >> $outLog
  echo " Exiting." >> $outLog
  exit_time
  exit
fi

# who produced the core file
if [ "$tc" = "core" ]; then
# Section Info
header_info "General Core File Info ..." >> $outLog

echo "==== General Core File Information ======================================" >> $outLog
if [ "`dirname $coreFile`" = "." ]; then
	echo core: `pwd`/$coreFile >> $outLog
else
	echo "Current Directory: `pwd`" >> $outLog
	echo core: $coreFile >> $outLog
fi
echo " " >> $outLog
echo "Checksum: `cksum $coreFile`" >> $outLog
echo " " >> $outLog
echo "Date Stamp Check: `ls -al $coreFile`" >> $outLog
echo " " >> $outLog
	echo " Check for App which created core dump: " >> $outLog
	if [ "`uname`" = "AIX" ]; then
		lquerypv -h $coreFile 6b0 64 >> $outLog 2>&1
	else
		file $coreFile 2>&1 >> $outLog
	fi
	echo " " >> $outLog
fi

# Section Info
header_info "Getting Stack Trace Info ..." >> $outLog

myPlat=`uname`
# debugger setup
type dbx 1>/dev/null 2>&1
if [ $? -eq 0 ]; then
	myDebugger=dbx
	echo " " >> $outLog
	echo "Using the dbx debugger." >> $outLog
	echo " " >> $outLog
else
	type gdb 2>&1 1>/dev/null
	if [ $? -eq 0 ]; then
		myDebugger=gdb
		echo " " >> $outLog
		echo "Using the gdb debugger." >> $outLog
		echo " " >> $outLog
	else
		echo " " >> $outLog
		if [ $myPlat = "HP-UX" -o $myPlat = "Linux" ]; then
			echo "The gdb debugger was Not found, exiting." >> $outLog
		else
			if [ "$myPlat" = SunOS ]; then

# Section Info
header_info "Getting PMAP Info ..." >> $outLog
				echo " " >> $outLog
				echo "==== P-Map Listing Information =======================================" >> $outLog
				/usr/proc/bin/pmap $pid >> $outLog
				echo " " >> $outLog
				echo " " >> $outLog

# Section Info
header_info "Getting PSTACK Info ..." >> $outLog
				echo "==== P-Stack Listing Information ====================================+" >> $outLog
				/usr/proc/bin/pstack $pid >> $outLog
				echo " " >> $outLog
				echo " " >> $outLog
			fi
			echo "Neither the dbx or gdb debugger were found, exiting." >> $outLog
		fi
		echo " " >> $outLog
		exit_time
		exit
	fi
fi


if [ $myPlat = "HP-UX" -o $myPlat = "Linux" ]; then
	myPrint="she echo" 
elif [ "$myPlat" = "SunOS" ]; then
  if [ "$myDebugger" = "gdb" ]; then
	myPrint=echo
	NL="\n"
  else
	myPrint=print
	NL=""
  fi

else
	myPrint=print
fi

# get stack information
# first - are we dealing with a running process or a core file
if [ "$tc" = "core" ]; then
	echo "==== Thread Listing Information ======================================" >> $outLog
	echo " " >> $outLog

	if [ $myDebugger = dbx ]; then
		if [ "$myPlat" = "AIX" ]; then
			dbx -d 2000 $product $coreFile >> $outLog 2>&1 <<EOF
$myPrint " "
$myPrint "==== Threads Listing ================================================="
thread
thread > $threadList
$myPrint " "
$myPrint "==== Stack Listing ==================================================="
where
$myPrint " "
$myPrint "==== Register Listing ================================================"
registers
$myPrint " "
$myPrint " "
$myPrint "==== Memory Map Information =========================================="
map
$myPrint " "
$myPrint "==== File Descriptor Information =========================================="
fd
$myPrint " "
quit
EOF
		else
			$myDebugger $product $coreFile >> $outLog 2>&1 <<EOF
$myPrint " "
$myPrint "==== Threads Listing ================================================="
threads
threads > $threadList
$myPrint " "
$myPrint "==== Stack Listing ==================================================="
where
$myPrint " "
$myPrint "==== Register Listing ================================================"
regs
$myPrint " "
$myPrint "==== Memory Map Information =========================================="
#map
$myPrint "Todo - Currently not available for core files?!"
$myPrint " " 
$myPrint " " 
quit
EOF
		fi
	else
# for GDB
	$myDebugger -silent $product $coreFile >> $outLog 2>&1 <<EOF
$myPrint " $NL"
$myPrint "\n==== Threads Listing =================================================$NL"
info thread
$myPrint " $NL"
$myPrint "\n==== Stack Listing ===================================================$NL"
where
$myPrint " $NL"
$myPrint "\n==== Register Listing ================================================$NL"
info reg
$myPrint " $NL"
$myPrint "\n==== Memory Map Information ==========================================$NL"
info share
$myPrint " $NL"
$myPrint " $NL"
quit
EOF
	$myDebugger -silent $product $coreFile >> $threadList 2>&1 <<EOF
info thread
quit
EOF
	fi
	if [ $myDebugger = dbx ]; then
		if [ ! -s ${threadList} ]; then
		  echo "Only Faulting Thread Found, exiting." | tee -a $outLog
		  echo " "
		  rm -f $threadList $dbxCmdList
		  echo "All done."
		  echo " "
		  exit_time
		  exit
		else
		  echo " " | tee -a $outLog
		  echo "Threads Found, Continuing Analysis:" | tee -a $outLog
		  echo " " | tee -a $outLog
		fi
	else
		cat ${threadList} | grep "system thread" > $$_tmpThreadList
		mv $$_tmpThreadList $threadList
		numThreads=`cat ${threadList} | wc -l`
		if [ "$numThreads" = "1" ]; then
		  echo "Only Faulting Thread Found, exiting." | tee -a $outLog
		  echo " "
		  rm -f $threadList $dbxCmdList
		  echo "All done."
		  echo " "
		  exit_time
		  exit
		else
		  echo " " | tee -a $outLog
		  echo "Threads Found, Continuing Analysis:" | tee -a $outLog
		  echo " " | tee -a $outLog
		fi

	fi

	if [ $myDebugger = dbx ]; then
		echo "==== Listing of 'Stack Traces per Thread' Information  ===============" >> $outLog
		rm -f $dbxCmdList
		if [ "$myPlat" = AIX ]; then
			cat $threadList | grep '$t' | cut -f2 -d"t" | cut -f1 -d" " > $tmpThreadList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			for thread in `cat $tmpThreadList`
			do
				echo "sh echo ==== Thread: $thread ================================================= >> $outLog" >> $dbxCmdList
				echo "thread current $thread"      >> $dbxCmdList
				echo "thread current >> $outLog"   >> $dbxCmdList
				echo "sh echo ==== Stack Trace Listing ============================================= >> $outLog" >> $dbxCmdList
				echo "where >> $outLog"            >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
				echo "sh echo ==== Registers ======================================================= >> $outLog" >> $dbxCmdList
				echo "registers >> $outLog"        >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
			done
		else
			cat $threadList | grep 't@' | cut -f2 -d"t" | cut -f2 -d"@" | cut -f1 -d" " > $tmpThreadList
			for thread in `cat $tmpThreadList`
			do
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
		  		echo "sh echo \"==== Thread: $thread ============================================\" >> $outLog" >> $dbxCmdList
				echo "thread t@${thread} >> $outLog"      >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
		  		echo "sh echo \"==== Stack Trace Listing ============================================\" >> $outLog" >> $dbxCmdList
				echo "where >> $outLog"            >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
		  		echo "sh echo \"==== Registers =======================================================\" >> $outLog" >> $dbxCmdList
				echo "regs >> $outLog"        >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
			done
			echo "quit >> $outLog" >> $dbxCmdList
		fi
		dbx $product $coreFile  1>>$outLog 2>&1 < $dbxCmdList
		rm -f $tmpThreadList $dbxCmdList
	else
# else gdb . . .
		cdir=`pwd`
		tmpGdbFile=$cdir/$$_1.gdb
		tmpGdbCmdList=$cdir/$$_2.gdb

		echo "info thread" > $tmpGdbFile
#		echo "detach" >> $tmpGdbFile
		echo "quit" >> $tmpGdbFile

		if [ "$myPlat" = HP-UX ]; then
		  THREADIDENT="system thread"
		  COL_TO_CUT="-f3"
		elif [ "$myPlat" = Linux ]; then
		  THREADIDENT="process"
		  COL_TO_CUT="-f5"			 
		else
		  THREADIDENT="Thread"
		  COL_TO_CUT="-f5"
		fi

		$myDebugger -batch -silent -x $tmpGdbFile $product $coreFile 2>/dev/null | grep "$THREADIDENT" | sed -e "s/(gdb) //g" |sed -e "s/* /  /g" > $tmpGdbCmdList
#VPERIC - make this more robust
		for thread in `cat $tmpGdbCmdList | awk '{print $1}'`
#		for thread in `cat $tmpGdbCmdList | cut $COL_TO_CUT -d" "`
		do
		  echo "she echo \" \"$NL" >> $gdbCmdList
		  echo "she echo \"==== Thread: $thread ============================================\"$NL" >> $gdbCmdList
		  echo "thread $thread" >> $gdbCmdList
		  echo "she echo \" \"$NL" >> $gdbCmdList
		  echo "she echo \"==== Stack Trace Listing ============================================\"$NL" >> $gdbCmdList
		  echo where >> $gdbCmdList
		  echo "she echo \" \"$NL" >> $gdbCmdList
		  echo "she echo \"==== Registers =======================================================\"$NL" >> $gdbCmdList
		  echo "info reg" >> $gdbCmdList
		  echo "she echo \" \"$NL" >> $gdbCmdList
		done
		echo quit >> $gdbCmdList

		$myDebugger -silent -batch -x $gdbCmdList $product $coreFile 2>/dev/null | grep -v "no debugging" >> $outLog
		rm $tmpGdbFile $tmpGdbCmdList $gdbCmdList

		cat $outLog | sed -e "s/(gdb) //g" > ${outLog}_$$
		mv ${outLog}_$$ $outLog
	fi
else
# - are we dealing with a running process 
	echo "==== Thread Listing Information =========================================" >> $outLog
	if [ $myDebugger = dbx ]; then
		if [ "$myPlat" = AIX ]; then
			dbx -a $pid >$threadList 2>&1 <<EOF
th
detach
EOF
			echo " " >> $outLog
			cat $threadList | grep '$t' | grep -v "stopped" | cut -f2 -d"t" | cut -f1 -d" " > $tmpThreadList

			echo "sh echo ==== Listing of Threads ============================================== >> $outLog" > $dbxCmdList
			echo "thread >> $outLog"              >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo ==== Listing of Locked Mutexes ======================================= >> $outLog" >> $dbxCmdList
			echo "mutex lock >> $outLog"          >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo ==== Listing of Condition Variables in Wait State ==================== >> $outLog" >> $dbxCmdList
			echo "condition wait >> $outLog"          >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo ==== Listing of Stack Traces per Thread ============================== >> $outLog" >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			for thread in `cat $tmpThreadList`
			do
				echo "sh echo ==== Thread: $thread ================================================= >> $outLog" >> $dbxCmdList
				echo "thread current $thread"        >> $dbxCmdList
				echo "thread current >> $outLog"   >> $dbxCmdList
				echo "sh echo ==== Stack Trace Listing ============================================= >> $outLog" >> $dbxCmdList
				echo "where >> $outLog"            >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
				echo "sh echo ==== Registers ======================================================= >> $outLog" >> $dbxCmdList
				echo "registers >> $outLog"        >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
			done
			echo "sh echo \"\" >> 		$outLog"     	>> $dbxCmdList
			echo "sh echo ==== \"Memory Map\" Information ======================================= >> $outLog"  >> $dbxCmdList
			echo "map >> 					$outLog"       >> $dbxCmdList
			echo "detach" 										>> $dbxCmdList

			$myDebugger -a $pid -c $dbxCmdList 1>/dev/null 2>&1
		else
# dbx for Solaris and ...
			$myDebugger $product $pid >$threadList 2>&1 <<EOF
threads
detach
quit
EOF
			echo " " >> $outLog
			cat $threadList | grep 't@' | grep -v "^t@" | cut -f2 -d"t" | cut -f2 -d"@" | cut -f1 -d" " > $tmpThreadList

			echo "sh echo Listing of Threads. >> $outLog" > $dbxCmdList
			echo "threads >> $outLog"              >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo \"\" >> $outLog"        >> $dbxCmdList
			echo "sh echo Listing of Stack Traces per Thread. >> $outLog" >> $dbxCmdList
			for thread in `cat $tmpThreadList`
			do
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
		  		echo "sh echo \"==== Thread: $thread ============================================\" >> $outLog" >> $dbxCmdList
				echo "thread t@${thread} >> $outLog"      >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
		  		echo "sh echo \"==== Stack Trace Listing ============================================\" >> $outLog" >> $dbxCmdList
				echo "where >> $outLog"            >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
		  		echo "sh echo \"==== Registers =======================================================\" >> $outLog" >> $dbxCmdList
				echo "regs >> $outLog"        >> $dbxCmdList
				echo "sh echo \"\" >> $outLog"     >> $dbxCmdList
			done
			echo "sh echo \"\" >> 		$outLog"     	>> $dbxCmdList
			echo "detach" 										>> $dbxCmdList
			echo "quit" 										>> $dbxCmdList

			$myDebugger $product $pid 1>/dev/null 2>&1 < $dbxCmdList
		fi
		rm -f $tmpThreadList $dbxCmdList
		if [ "$myPlat" = SunOS ]; then
			echo " " >> $outLog
			echo "==== P-Map Listing Information =======================================" >> $outLog
			/usr/proc/bin/pmap $pid >> $outLog
			echo " " >> $outLog
			echo " " >> $outLog
			echo "==== P-Stack Listing Information ====================================+" >> $outLog
			/usr/proc/bin/pstack $pid >> $outLog
			echo " " >> $outLog
			echo " " >> $outLog
		fi
	else
		cdir=`pwd`
		tmpGdbFile=$cdir/$$_1.gdb
		tmpGdbCmdList=$cdir/$$_2.gdb

		echo "she echo \"\" "        >> $tmpGdbCmdList
		echo "she echo ==== Listing of Threads ==============================================" >> $tmpGdbCmdList
		echo "info thread"              >> $tmpGdbCmdList
		echo "she echo \"\" "        >> $tmpGdbCmdList
		echo "she echo \"\" "        >> $tmpGdbCmdList
		echo "she echo ==== Listing of Stack Traces per Thread ==============================" >> $tmpGdbCmdList

		echo "info thread" > $tmpGdbFile
		echo "detach" >> $tmpGdbFile
		echo "quit" >> $tmpGdbFile

		if [ "$myPlat" = HP-UX ]; then
		  THREADIDENT="system thread"
		  COL_TO_CUT="-f3"
		elif [ "$myPlat" = Linux ]; then
		  THREADIDENT=" Thread "
		  COL_TO_CUT="-f5"			 
		else
		  THREADIDENT="Thread"
		  COL_TO_CUT="-f5"
		fi


#		for thread in `$myDebugger -batch -silent -x $tmpGdbFile $product $pid 2>/dev/null | grep "system thread" | sed -e "s/* /  /g" | cut -f3 -d" "`

		if [ "$myPlat" = Linux ]; then
   		$myDebugger -batch -silent -x $tmpGdbFile $product $pid 2>/dev/null | grep "$THREADIDENT" | grep -v "New Thread" | sed -e "s/* /  /g" > $threadList
		else
   		$myDebugger -batch -silent -x $tmpGdbFile $product $pid 2>/dev/null | grep "$THREADIDENT" | sed -e "s/* /  /g" > $threadList
      fi

#VPERIC - make this more robust - new hp gdb creates 5 spaces
		for thread in `cat $threadList | awk '{print $1}'`
#		for thread in `cat $threadList | cut $COL_TO_CUT -d" "`
		do
			echo "she echo \"\"$NL"     >> $tmpGdbCmdList
		   echo "she echo \"==== Thread: $thread ============================================\"$NL" >> $tmpGdbCmdList
			echo "thread $thread"        >> $tmpGdbCmdList
			echo "she echo \"\"$NL"     >> $tmpGdbCmdList
		   echo "she echo \"==== Stack Trace Listing ============================================\"$NL" >> $tmpGdbCmdList
			echo "where"            >> $tmpGdbCmdList
			echo "she echo \"\"$NL"     >> $tmpGdbCmdList
		   echo "she echo \"==== Registers =======================================================\"$NL" >> $tmpGdbCmdList
			echo "info reg"  >> $tmpGdbCmdList
			echo "she echo \"\"$NL"     >> $tmpGdbCmdList
		done
		echo "she echo \"==== Memory Map Information==========================================\"$NL"     >> $tmpGdbCmdList
		echo "info share"            >> $tmpGdbCmdList
		echo "detach" >> $tmpGdbCmdList
		echo "quit" >> $tmpGdbCmdList
		$myDebugger -silent -batch -x $tmpGdbCmdList $product $pid 2>/dev/null  1>>$outLog
		rm $tmpGdbFile
		rm -f $tmpThreadList $tmpGdbCmdList

	fi
fi


# Debugger to use - see above - start with check for dbx and then try gdb 
#  

if [ -n "$pid" ] ; then
 if [ $myPlat = "SunOS" -o $myPlat = "Linux" ]; then

# Section Info - open files
  echo " " >> $outLog
  header_info "Open Files ..." >> $outLog
  echo " " >> $outLog
  ls -l /proc/$pid/fd >> $outLog
 fi
 if [ $myPlat = "Linux" ]; then
  echo " " >> $outLog
  header_info "PROC status ..." >> $outLog
  echo " " >> $outLog
  cat /proc/$pid/status >> $outLog
 fi
fi

# Section Info
echo " " >> $outLog
header_info "Getting Reay To Exit ..." >> $outLog
echo " " >> $outLog

exit_time

header_info "All done."